@financial-times/cp-content-pipeline-schema 2.2.2 → 2.3.0

package/src/model/RichText.test.ts

@@ -1,6 +1,16 @@
 import { baseCapiObject } from '../fixtures/capiObject'
 import { RichText } from '../model/RichText'
 
+jest.mock('@dotcom-reliability-kit/logger', () => ({
+  Logger: jest.fn(() => ({
+    debug: jest.fn(),
+    error: jest.fn(),
+    fatal: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+  })),
+}))
+
 describe('RichText resolver', () => {
   it('should transform bodyXML to an AST', async () => {
     const model = new RichText('bodyXML', baseCapiObject.bodyXML)

package/src/model/RichText.ts

@@ -1,16 +1,26 @@
-import bodyXMLToTree from '../resolvers/content-tree/bodyXMLToTree'
+import Piscina from 'piscina'
+
+import type { QueryContext } from '..'
+import bodyXMLToTreeWorker from '../resolvers/content-tree/bodyXMLToTreeWorker'
 import extractText from '../resolvers/content-tree/extractText'
+import type { PredicateError } from '../resolvers/content-tree/nodePredicates'
 import updateTreeWithReferenceIds from '../resolvers/content-tree/updateTreeWithReferenceIds'
 import { LiteralUnionScalarValues } from '../resolvers/literal-union'
 import { RichTextSource } from '../resolvers/scalars'
 import { CapiResponse } from './CapiResponse'
-import {
-  commonTagMappings,
-  articleTagMappings,
-  liveBlogPostTagMappings,
-} from '../resolvers/content-tree/tagMappings'
+import { OperationalError } from '@dotcom-reliability-kit/errors'
 import { ContentTree } from '@financial-times/content-tree'
-import { QueryContext } from '..'
+
+let piscina: Piscina | undefined
+// Don't run the thread pool during tests as it creates loads of threads that
+// aren't properly closed and leaves the process hanging. We do our best to
+// keep the same behaviour when running on the main thread vs a worker thread,
+// with the main difference being the lack of logs on the main thread.
+if (process.env.NODE_ENV !== 'test') {
+  piscina = new Piscina({
+    filename: require.resolve('../resolvers/content-tree/bodyXMLToTreeWorker'),
+  })
+}
 
 export class RichText {
   constructor(
@@ -24,27 +34,29 @@ export class RichText {
   }
 
   async structured(context?: QueryContext) {
-    const tree = await new Promise<ContentTree.Body>((resolve, reject) => {
-      // bodyXMLToTree is synchronous and slow. scheduling it in a setImmediate
-      // prevents it from blocking the event loop, so the app can still handle
-      // requests, and it won't skew resolver timing metrics by blocking the
-      // timers for quicker, asynchronous resolvers from finishing.
-
-      const tagMappings =
-        this.contentApiData?.type() === 'LiveBlogPost'
-          ? { ...commonTagMappings, ...liveBlogPostTagMappings }
-          : {
-              ...commonTagMappings,
-              ...articleTagMappings(this.contentApiData),
-            }
-      setImmediate(() => {
-        try {
-          resolve(bodyXMLToTree(this.value ?? '', tagMappings, context))
-        } catch (error) {
-          reject(error)
-        }
-      })
+    const args = {
+      xml: this.value ?? '',
+      responseMetadata: this.contentApiData
+        ? {
+            isLiveBlogPost: this.contentApiData.type() === 'LiveBlogPost',
+            topperHasImage: this.contentApiData.topperHasImage(),
+          }
+        : undefined,
+    }
+    // forward errors from the worker threads to the logger
+    piscina?.on('message', ({ event, error, ...predError }: PredicateError) => {
+      const { stack } = error
+      const opError = new OperationalError(predError)
+      opError.stack = stack
+      context?.logger.error({ event, error: opError })
     })
+    // bodyXMLToTree is synchronous and slow. Offload its processing to a
+    // worker thread so that it does not block the event loop. This allows the
+    // app to continue to handle other requests whilst processing an expensive
+    // one, as well as not skewing timing metrics.
+    const tree: ContentTree.Body = piscina
+      ? await piscina.run(args)
+      : bodyXMLToTreeWorker(args)
 
     const { tree: treeWithReferences, references } = updateTreeWithReferenceIds(
       tree,

package/src/resolvers/content-tree/bodyXMLToTree.test.ts

@@ -1,20 +1,16 @@
+import { parentPort } from 'worker_threads'
 import { ContentTree } from '@financial-times/content-tree'
 import bodyXMLToTree, { TagMappings } from './bodyXMLToTree'
 import tags from './tagMappings'
-import { Logger } from '@dotcom-reliability-kit/logger'
-import { QueryContext } from '../..'
 
-const mockLogger = new Logger()
-const mockLogError = jest.spyOn(mockLogger, 'error')
-
-const mockContext = {
-  logger: mockLogger,
-} as QueryContext
+jest.mock('worker_threads')
+// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+const mockPostMessage = jest.mocked(parentPort)!.postMessage
 
 describe('bodyXMLToTree', () => {
   it('converts XML to tree', () => {
     const xml = `<body><p>Hello world</p></body>`
-    expect(bodyXMLToTree(xml, tags, mockContext)).toMatchInlineSnapshot(`
+    expect(bodyXMLToTree(xml, tags)).toMatchInlineSnapshot(`
       Object {
         "children": Array [
           Object {
@@ -220,14 +216,14 @@ describe('bodyXMLToTree', () => {
           "version": 1,
         }
       `)
-      expect(mockLogError).not.toBeCalled()
+      expect(mockPostMessage).not.toBeCalled()
     })
 
     it('should handle heading and slots', () => {
       const xml =
         '<body><div class="n-content-layout"><h3></h3><div class="n-content-layout__slot"></div><div class="n-content-layout__slot"></div></body>'
 
-      expect(bodyXMLToTree(xml, tags, mockContext)).toMatchInlineSnapshot(`
+      expect(bodyXMLToTree(xml, tags)).toMatchInlineSnapshot(`
         Object {
           "children": Array [
             Object {
@@ -255,14 +251,14 @@ describe('bodyXMLToTree', () => {
           "version": 1,
         }
       `)
-      expect(mockLogError).not.toBeCalled()
+      expect(mockPostMessage).not.toBeCalled()
     })
 
     it('should log an error on unexpected child after heading', () => {
       const xml =
         '<body><div class="n-content-layout"><h3></h3><div class="n-content-layout__slot"></div><p></p></body>'
 
-      expect(bodyXMLToTree(xml, tags, mockContext)).toMatchInlineSnapshot(`
+      expect(bodyXMLToTree(xml, tags)).toMatchInlineSnapshot(`
         Object {
           "children": Array [
             Object {
@@ -290,12 +286,19 @@ describe('bodyXMLToTree', () => {
           "version": 1,
         }
       `)
-      expect(mockLogError).toBeCalled()
-      expect(mockLogError.mock.lastCall).toMatchInlineSnapshot(`
+      expect(mockPostMessage).toBeCalled()
+      expect(mockPostMessage.mock.lastCall).toMatchInlineSnapshot(`
         Array [
           Object {
-            "error": [OperationalError: Unexpected children types for layout],
+            "actual": Array [
+              "layout-slot",
+              "paragraph",
+            ],
+            "code": "BODY_XML_UNEXPECTED_STRUCTURE",
+            "error": [Error],
             "event": "RECOVERABLE_ERROR",
+            "expected": "layout-slot",
+            "message": "Unexpected children types for layout",
           },
         ]
       `)
@@ -305,7 +308,7 @@ describe('bodyXMLToTree', () => {
       const xml =
         '<body><div class="n-content-layout"><div class="n-content-layout__slot"></div><p></p></body>'
 
-      expect(bodyXMLToTree(xml, tags, mockContext)).toMatchInlineSnapshot(`
+      expect(bodyXMLToTree(xml, tags)).toMatchInlineSnapshot(`
         Object {
           "children": Array [
             Object {
@@ -328,12 +331,19 @@ describe('bodyXMLToTree', () => {
           "version": 1,
         }
       `)
-      expect(mockLogError).toBeCalled()
-      expect(mockLogError.mock.lastCall).toMatchInlineSnapshot(`
+      expect(mockPostMessage).toBeCalled()
+      expect(mockPostMessage.mock.lastCall).toMatchInlineSnapshot(`
         Array [
           Object {
-            "error": [OperationalError: Unexpected children types for layout],
+            "actual": Array [
+              "layout-slot",
+              "paragraph",
+            ],
+            "code": "BODY_XML_UNEXPECTED_STRUCTURE",
+            "error": [Error],
             "event": "RECOVERABLE_ERROR",
+            "expected": "layout-slot",
+            "message": "Unexpected children types for layout",
           },
         ]
       `)
@@ -345,7 +355,7 @@ describe('bodyXMLToTree', () => {
       const xml =
         '<table class="data-table" data-table-collapse-rownum="" data-table-layout-largescreen="auto" data-table-layout-smallscreen="auto" data-table-theme="auto"><caption>Nulla iaculis tempus augue</caption><thead><tr><th data-column-hidden="none" data-column-sortable="false" data-column-type="string">libero mollis</th><th data-column-hidden="none" data-column-sortable="false" data-column-type="string">pretium nunc</th><th data-column-hidden="none" data-column-sortable="false" data-column-type="string">euismod nunc</th></tr></thead><tbody><tr><td>Aenean </td><td>14134</td><td>dfdsfd</td></tr><tr><td>lobortis </td><td>3434</td><td>fdsf dsf </td></tr><tr><td>volutpat </td><td>234234</td><td>sd fsd</td></tr><tr><td>vitae </td><td>2423</td><td>s fsdf</td></tr><tr><td>elementumus</td><td>23423</td><td>f sdf</td></tr></tbody><tfoot><tr><td colspan="1000">Aenean sodales sapien</td></tr></tfoot></table>'
 
-      expect(bodyXMLToTree(xml, tags, mockContext)).toMatchInlineSnapshot(`
+      expect(bodyXMLToTree(xml, tags)).toMatchInlineSnapshot(`
         Object {
           "children": Array [
             Object {
@@ -604,7 +614,7 @@ describe('bodyXMLToTree', () => {
       const xml =
         '<table class="data-table" id="U1140244733565W0C"><caption>Emerging markets outlook for 2017</caption><tbody><tr><td colspan="2"><p>Brazil</p><p>Brazilian shares were the best-performing asset globally over the 12 months to the end of January, returning 121 per cent, according to data from BofA Merrill Lynch. Brazilian stocks did very well through January, but investors including Lazard and Eastspring have scaled back their exposure after strong growth last year.</p></td></tr></tbody></table>'
 
-      expect(bodyXMLToTree(xml, tags, mockContext)).toMatchInlineSnapshot(`
+      expect(bodyXMLToTree(xml, tags)).toMatchInlineSnapshot(`
         Object {
           "children": Array [
             Object {

package/src/resolvers/content-tree/bodyXMLToTree.ts

@@ -1,9 +1,8 @@
+import type { ContentTree } from '@financial-times/content-tree'
 import * as cheerio from 'cheerio'
+import { isTag, isText } from 'domhandler'
 
-import type { ContentTree } from '@financial-times/content-tree'
 import { AnyNode } from './Workarounds'
-import { QueryContext } from '../..'
-import { isTag, isText } from 'domhandler'
 
 function isNode(node: ContentTree.Node | undefined): node is ContentTree.Node {
   return Boolean(node && 'type' in node)
@@ -11,8 +10,7 @@ function isNode(node: ContentTree.Node | undefined): node is ContentTree.Node {
 
 type ContentTreeTransform = (
   $el: cheerio.Cheerio<any>,
-  traverse: () => AnyNode[],
-  context?: QueryContext
+  traverse: () => AnyNode[]
 ) => AnyNode | AnyNode[]
 
 export type TagMappings = Record<string, ContentTreeTransform>
@@ -23,8 +21,7 @@ export type TagMappings = Record<string, ContentTreeTransform>
 */
 export default function bodyXMLToTree(
   xml: string,
-  tagMappings: TagMappings,
-  context?: QueryContext
+  tagMappings: TagMappings
 ): ContentTree.Body {
   const $ = cheerio.load(xml)
 
@@ -44,10 +41,8 @@ export default function bodyXMLToTree(
 
       if (matchedSelector) {
         const contentTreeTransform = tagMappings[matchedSelector]
-        return contentTreeTransform(
-          $(node),
-          () => flattenAndTraverseChildren(node.children),
-          context
+        return contentTreeTransform($(node), () =>
+          flattenAndTraverseChildren(node.children)
         )
       }
 

package/src/resolvers/content-tree/bodyXMLToTreeWorker.ts

@@ -0,0 +1,31 @@
+import type { ContentTree } from '@financial-times/content-tree'
+
+import bodyXMLToTree from './bodyXMLToTree'
+
+import {
+  commonTagMappings,
+  articleTagMappings,
+  liveBlogPostTagMappings,
+} from './tagMappings'
+
+export interface ResponseMetadata {
+  isLiveBlogPost: boolean
+  topperHasImage: boolean
+}
+
+export default function bodyXMLToTreeWorker({
+  xml,
+  responseMetadata,
+}: {
+  xml: string
+  responseMetadata?: ResponseMetadata
+}): ContentTree.Body {
+  const tagMappings = responseMetadata?.isLiveBlogPost
+    ? { ...commonTagMappings, ...liveBlogPostTagMappings }
+    : {
+        ...commonTagMappings,
+        ...articleTagMappings(responseMetadata?.topperHasImage ?? false),
+      }
+
+  return bodyXMLToTree(xml, tagMappings)
+}

package/src/resolvers/content-tree/nodePredicates.ts

@@ -1,6 +1,6 @@
-import { OperationalError } from '@dotcom-reliability-kit/errors'
-import { AnyNode } from './Workarounds'
-import { QueryContext } from '../..'
+import { parentPort } from 'worker_threads'
+
+import type { AnyNode } from './Workarounds'
 
 type ValuesOfTuple<Tuple extends readonly string[]> = Tuple[number]
 
@@ -19,24 +19,39 @@ export const phrasingTypes = [
   'link',
 ] as const
 
+export interface PredicateError {
+  event: string
+  code: string
+  message: string
+  expected: AnyNode['type'] | readonly AnyNode['type'][]
+  actual: AnyNode['type'][]
+  error: Error
+}
+
+// reliability-kit's logger creates a worker thread under the hood but we can't
+// access this thread directly from these worker threads. Instead let's send
+// all our recoverable errors to the piscina pool managing us.
+const postLoggerError = (error: Omit<PredicateError, 'error'>) => {
+  // stacks will only copy over if they're in an Error object
+  const errorWithStack = { ...error, error: new Error() }
+  parentPort?.postMessage(errorWithStack)
+}
+
 export const findChildOftype = <NodeType extends AnyNode>(
   type: NodeType['type'],
   nodes: AnyNode[],
-  parentType: AnyNode['type'],
-  context?: QueryContext
+  parentType: AnyNode['type']
 ): NodeType | undefined => {
   const predicate = (node: AnyNode): node is NodeType => node.type === type
   const child = nodes.find(predicate)
 
   if (!child) {
-    context?.logger.error({
+    postLoggerError({
       event: 'RECOVERABLE_ERROR',
-      error: new OperationalError({
-        code: 'BODY_XML_UNEXPECTED_STRUCTURE',
-        message: `Didn't find expected child type in ${parentType}`,
-        expected: type,
-        actual: nodes.map((node) => node.type),
-      }),
+      code: 'BODY_XML_UNEXPECTED_STRUCTURE',
+      message: `Didn't find expected child type in ${parentType}`,
+      expected: type,
+      actual: nodes.map((node) => node.type),
     })
   }
 
@@ -46,21 +61,18 @@ export const findChildOftype = <NodeType extends AnyNode>(
 export const everyChildIsType = <NodeType extends AnyNode>(
   type: NodeType['type'],
   nodes: AnyNode[],
-  parentType: AnyNode['type'],
-  context?: QueryContext
+  parentType: AnyNode['type']
 ): NodeType[] => {
   const predicate = (node: AnyNode): node is NodeType => node.type === type
   const allChildrenAreType = nodes.every(predicate)
 
   if (!allChildrenAreType) {
-    context?.logger.error({
+    postLoggerError({
       event: 'RECOVERABLE_ERROR',
-      error: new OperationalError({
-        code: 'BODY_XML_UNEXPECTED_STRUCTURE',
-        message: `Unexpected children types for ${parentType}`,
-        expected: type,
-        actual: nodes.map((node) => node.type),
-      }),
+      code: 'BODY_XML_UNEXPECTED_STRUCTURE',
+      message: `Unexpected children types for ${parentType}`,
+      expected: type,
+      actual: nodes.map((node) => node.type),
     })
   }
 
@@ -70,22 +82,19 @@ export const everyChildIsType = <NodeType extends AnyNode>(
 export const childrenOfTypes = <Types extends readonly AnyNode['type'][]>(
   types: Types,
   nodes: AnyNode[],
-  parentType: AnyNode['type'],
-  context?: QueryContext
+  parentType: AnyNode['type']
 ): NodeOfType<ValuesOfTuple<Types>>[] => {
   const predicate = (node: AnyNode): node is NodeOfType<ValuesOfTuple<Types>> =>
     types.includes(node.type)
   const allChildrenAreType = nodes.every(predicate)
 
   if (!allChildrenAreType) {
-    context?.logger.error({
+    postLoggerError({
       event: 'RECOVERABLE_ERROR',
-      error: new OperationalError({
-        code: 'BODY_XML_UNEXPECTED_STRUCTURE',
-        message: `Unexpected ordered children types for ${parentType}`,
-        expected: types,
-        actual: nodes.map((node) => node.type),
-      }),
+      code: 'BODY_XML_UNEXPECTED_STRUCTURE',
+      message: `Unexpected ordered children types for ${parentType}`,
+      expected: types,
+      actual: nodes.map((node) => node.type),
     })
   }
 
@@ -110,21 +119,18 @@ export const childrenOfOrderedTypes = <
 >(
   types: Types,
   nodes: AnyNode[],
-  parentType: AnyNode['type'],
-  context?: QueryContext
+  parentType: AnyNode['type']
 ): NodesOfTypes<Types> => {
   if (nodesAreOrderedTypes(types, nodes)) {
     return nodes
   }
 
-  context?.logger.error({
+  postLoggerError({
     event: 'RECOVERABLE_ERROR',
-    error: new OperationalError({
-      code: 'BODY_XML_UNEXPECTED_STRUCTURE',
-      message: `Unexpected children types for ${parentType}`,
-      expected: types,
-      actual: nodes.map((node) => node.type),
-    }),
+    code: 'BODY_XML_UNEXPECTED_STRUCTURE',
+    message: `Unexpected children types for ${parentType}`,
+    expected: types,
+    actual: nodes.map((node) => node.type),
   })
 
   return nodes as unknown as NodesOfTypes<Types>