@jsreport/jsreport-core 4.3.0 → 4.4.0

package/README.md

@@ -282,6 +282,16 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 4.4.0
+
+- trigger async report using header to avoid allocating worker
+- fix require('..') and require('.') in sandbox
+- fix component rendering in loop with async work on helpers
+
+### 4.3.1
+
+- fix `waitForAsyncHelper`, `waitForAsyncHelpers` not working with trustUserCode: true
+
 ### 4.3.0
 
 - expose safe properties of `req.context.user` in sandbox

package/lib/main/profiler.js

@@ -159,6 +159,10 @@ module.exports = (reporter) => {
   reporter.beforeRenderWorkerAllocatedListeners.add('profiler', async (req) => {
     req.context.profiling = req.context.profiling || {}
 
+    if (req.context.profiling.enabled === false) {
+      return
+    }
+
     if (req.context.profiling.mode == null) {
       const profilerSettings = await reporter.settings.findValue('profiler', req)
       const defaultMode = reporter.options.profiler.defaultMode || 'standard'

package/lib/main/reporter.js

@@ -382,7 +382,12 @@ class MainReporter extends Reporter {
     const res = Response(this, req.context.id)
 
     try {
-      await this.beforeRenderWorkerAllocatedListeners.fire(req)
+      await this.beforeRenderWorkerAllocatedListeners.fire(req, res)
+      if (req.context.clientNotification) {
+        const r = req.context.clientNotification
+        delete req.context.clientNotification
+        return r
+      }
 
       worker = await this._workersManager.allocate(req, {
         timeout: this.getReportTimeout(req)

package/lib/worker/render/executeEngine.js

@@ -14,7 +14,7 @@ module.exports = (reporter) => {
 
   reporter.templatingEngines = { cache: templatesCache }
 
-  const contextExecutionChainMap = new WeakMap()
+  const contextExecutionChainMap = new Map()
   const executionFnParsedParamsMap = new Map()
   const executionAsyncResultsMap = new Map()
   const executionAsyncCallChainMap = new Map()
@@ -69,7 +69,7 @@ module.exports = (reporter) => {
         return templatingEnginesEvaluate(false, executionInfo, entityInfo, req)
       },
       waitForAsyncHelper: async (maybeAsyncContent) => {
-        const executionChain = contextExecutionChainMap.get(context) || []
+        const executionChain = contextExecutionChainMap.get(context.__sandboxId) || []
         const executionId = executionChain[executionChain.length - 1]
 
         if (
@@ -99,7 +99,7 @@ module.exports = (reporter) => {
         return content
       },
       waitForAsyncHelpers: async () => {
-        const executionChain = contextExecutionChainMap.get(context) || []
+        const executionChain = contextExecutionChainMap.get(context.__sandboxId) || []
         const executionId = executionChain[executionChain.length - 1]
 
         if (executionId != null && executionAsyncResultsMap.has(executionId)) {
@@ -115,7 +115,7 @@ module.exports = (reporter) => {
         }
       },
       addFinishListener: (fn) => {
-        const executionChain = contextExecutionChainMap.get(context) || []
+        const executionChain = contextExecutionChainMap.get(context.__sandboxId) || []
         const executionId = executionChain[executionChain.length - 1]
 
         if (executionId && executionFinishListenersMap.has(executionId)) {
@@ -123,7 +123,7 @@ module.exports = (reporter) => {
         }
       },
       createAsyncHelperResult: (v) => {
-        const executionChain = contextExecutionChainMap.get(context) || []
+        const executionChain = contextExecutionChainMap.get(context.__sandboxId) || []
         const executionId = executionChain[executionChain.length - 1]
 
         const asyncResultMap = executionAsyncResultsMap.get(executionId)
@@ -170,6 +170,7 @@ module.exports = (reporter) => {
 
     const normalizedHelpers = `${helpers || ''}`
     const executionFnParsedParamsKey = `entity:${entity.shortid || 'anonymous'}:helpers:${normalizedHelpers}`
+    let sandboxId
 
     const initFn = async (getTopLevelFunctions, compileScript) => {
       if (systemHelpersCache != null) {
@@ -213,112 +214,118 @@ module.exports = (reporter) => {
     }
 
     const executionFn = async ({ require, console, topLevelFunctions, context }) => {
+      sandboxId = context.__sandboxId
       const asyncResultMap = new Map()
       const asyncCallChainSet = new Set()
 
-      if (!contextExecutionChainMap.has(context)) {
-        contextExecutionChainMap.set(context, [])
+      if (!contextExecutionChainMap.has(sandboxId)) {
+        contextExecutionChainMap.set(sandboxId, [])
       }
 
-      contextExecutionChainMap.get(context).push(executionId)
+      contextExecutionChainMap.get(sandboxId).push(executionId)
 
       executionAsyncResultsMap.set(executionId, asyncResultMap)
       executionAsyncCallChainMap.set(executionId, asyncCallChainSet)
       executionFinishListenersMap.set(executionId, reporter.createListenerCollection())
       executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions, context })
 
-      const key = engine.buildTemplateCacheKey
-        ? engine.buildTemplateCacheKey({ content }, req)
-        : `template:${content}:${engine.name}`
-
-      if (!templatesCache.has(key)) {
-        try {
-          templatesCache.set(key, engine.compile(content, { require }))
-        } catch (e) {
-          e.property = 'content'
-          throw e
+      try {
+        const key = engine.buildTemplateCacheKey
+          ? engine.buildTemplateCacheKey({ content }, req)
+          : `template:${content}:${engine.name}`
+
+        if (!templatesCache.has(key)) {
+          try {
+            templatesCache.set(key, engine.compile(content, { require }))
+          } catch (e) {
+            e.property = 'content'
+            throw e
+          }
         }
-      }
 
-      const compiledTemplate = templatesCache.get(key)
-      const wrappedTopLevelFunctions = {}
+        const compiledTemplate = templatesCache.get(key)
+        const wrappedTopLevelFunctions = {}
 
-      for (const h of Object.keys(topLevelFunctions)) {
-        // extra wrapping for enhance the error with the helper name
-        wrappedTopLevelFunctions[h] = wrapHelperForHelperNameWhenError(topLevelFunctions[h], h, () => executionFnParsedParamsMap.has(req.context.id))
+        for (const h of Object.keys(topLevelFunctions)) {
+          // extra wrapping for enhance the error with the helper name
+          wrappedTopLevelFunctions[h] = wrapHelperForHelperNameWhenError(topLevelFunctions[h], h, () => executionFnParsedParamsMap.has(req.context.id))
 
-        if (engine.getWrappingHelpersEnabled && engine.getWrappingHelpersEnabled(req) === false) {
-          wrappedTopLevelFunctions[h] = engine.wrapHelper(wrappedTopLevelFunctions[h], { context })
-        } else {
-          wrappedTopLevelFunctions[h] = wrapHelperForAsyncSupport(wrappedTopLevelFunctions[h], asyncResultMap, asyncCallChainSet)
+          if (engine.getWrappingHelpersEnabled && engine.getWrappingHelpersEnabled(req) === false) {
+            wrappedTopLevelFunctions[h] = engine.wrapHelper(wrappedTopLevelFunctions[h], { context })
+          } else {
+            wrappedTopLevelFunctions[h] = wrapHelperForAsyncSupport(wrappedTopLevelFunctions[h], asyncResultMap, asyncCallChainSet)
+          }
         }
-      }
 
-      let contentResult = await engine.execute(compiledTemplate, wrappedTopLevelFunctions, data, { require })
+        let contentResult = await engine.execute(compiledTemplate, wrappedTopLevelFunctions, data, { require })
 
-      const resolvedResultsMap = new Map()
+        const resolvedResultsMap = new Map()
 
-      // we need to use the cloned map, because there can be a waitForAsyncHelper pending that needs the asyncResultMap values
-      let clonedMap = new Map(asyncResultMap)
+        // we need to use the cloned map, because there can be a waitForAsyncHelper pending that needs the asyncResultMap values
+        let clonedMap = new Map(asyncResultMap)
 
-      while (clonedMap.size > 0) {
-        const keysEvaluated = [...clonedMap.keys()]
+        while (clonedMap.size > 0) {
+          const keysEvaluated = [...clonedMap.keys()]
 
-        await Promise.all(keysEvaluated.map(async (k) => {
-          const result = await clonedMap.get(k)
-          asyncCallChainSet.delete(k)
-          resolvedResultsMap.set(k, `${result}`)
-          clonedMap.delete(k)
-        }))
+          await Promise.all(keysEvaluated.map(async (k) => {
+            const result = await clonedMap.get(k)
+            asyncCallChainSet.delete(k)
+            resolvedResultsMap.set(k, `${result}`)
+            clonedMap.delete(k)
+          }))
 
-        // we need to remove the keys processed from the original map at this point
-        // (after the await) because during the async work the asyncResultMap will be read
-        for (const k of keysEvaluated) {
-          asyncResultMap.delete(k)
-        }
-
-        // we want to process the new generated pending async results
-        if (asyncResultMap.size > 0) {
-          clonedMap = new Map(asyncResultMap)
-        }
-      }
+          // we need to remove the keys processed from the original map at this point
+          // (after the await) because during the async work the asyncResultMap will be read
+          for (const k of keysEvaluated) {
+            asyncResultMap.delete(k)
+          }
 
-      while (contentResult.includes('{#asyncHelperResult')) {
-        contentResult = contentResult.replace(/{#asyncHelperResult ([^{}]+)}/g, (str, p1) => {
-          const asyncResultId = p1
-          // this can happen if a child jsreport.templatingEngines.evaluate receives an async value from outer scope
-          // because every evaluate uses a unique map of async results
-          // example is the case when component receives as a value async thing
-          // instead of returning "undefined" we let the outer eval to do the replace
-          if (!resolvedResultsMap.has(asyncResultId)) {
-            // returning asyncUnresolvedHelperResult just to avoid endless loop, after replace we put it back to asyncHelperResult
-            return `{#asyncUnresolvedHelperResult ${asyncResultId}}`
+          // we want to process the new generated pending async results
+          if (asyncResultMap.size > 0) {
+            clonedMap = new Map(asyncResultMap)
           }
-          return `${resolvedResultsMap.get(asyncResultId)}`
-        })
-      }
+        }
 
-      contentResult = contentResult.replace(/asyncUnresolvedHelperResult/g, 'asyncHelperResult')
+        while (contentResult.includes('{#asyncHelperResult')) {
+          contentResult = contentResult.replace(/{#asyncHelperResult ([^{}]+)}/g, (str, p1) => {
+            const asyncResultId = p1
+            // this can happen if a child jsreport.templatingEngines.evaluate receives an async value from outer scope
+            // because every evaluate uses a unique map of async results
+            // example is the case when component receives as a value async thing
+            // instead of returning "undefined" we let the outer eval to do the replace
+            if (!resolvedResultsMap.has(asyncResultId)) {
+              // returning asyncUnresolvedHelperResult just to avoid endless loop, after replace we put it back to asyncHelperResult
+              return `{#asyncUnresolvedHelperResult ${asyncResultId}}`
+            }
+            return `${resolvedResultsMap.get(asyncResultId)}`
+          })
+        }
 
-      await executionFinishListenersMap.get(executionId).fire()
+        contentResult = contentResult.replace(/asyncUnresolvedHelperResult/g, 'asyncHelperResult')
 
-      contextExecutionChainMap.set(context, contextExecutionChainMap.get(context).filter((id) => id !== executionId))
+        await executionFinishListenersMap.get(executionId).fire()
 
-      return {
-        // handlebars escapes single brackets before execution to prevent errors on {#asset}
-        // we need to unescape them later here, because at the moment the engine.execute finishes
-        // the async helpers aren't executed yet
-        content: engine.unescape ? engine.unescape(contentResult) : contentResult
+        return {
+          // handlebars escapes single brackets before execution to prevent errors on {#asset}
+          // we need to unescape them later here, because at the moment the engine.execute finishes
+          // the async helpers aren't executed yet
+          content: engine.unescape ? engine.unescape(contentResult) : contentResult
+        }
+      } finally {
+        // ensure we clean the execution from the chain always, even on errors
+        contextExecutionChainMap.set(sandboxId, contextExecutionChainMap.get(sandboxId).filter((id) => id !== executionId))
       }
     }
 
-    // executionFnParsedParamsMap is there to cache parsed components helpers to speed up longer loops
-    // we store there for the particular request and component a promise and only the first component gets compiled
-    if (executionFnParsedParamsMap.get(req.context.id).has(executionFnParsedParamsKey)) {
-      const { require, console, topLevelFunctions, context } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
+    try {
+      // executionFnParsedParamsMap is there to cache parsed components helpers to speed up longer loops
+      // we store there for the particular request and component a promise and only the first component gets compiled
+      if (executionFnParsedParamsMap.get(req.context.id).has(executionFnParsedParamsKey)) {
+        const { require, console, topLevelFunctions, context } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
+
+        return await executionFn({ require, console, topLevelFunctions, context })
+      }
 
-      return executionFn({ require, console, topLevelFunctions, context })
-    } else {
       const awaiter = {}
 
       awaiter.promise = new Promise((resolve) => {
@@ -326,67 +333,71 @@ module.exports = (reporter) => {
       })
 
       executionFnParsedParamsMap.get(req.context.id).set(executionFnParsedParamsKey, awaiter)
-    }
 
-    if (reporter.options.sandbox.cache && reporter.options.sandbox.cache.enabled === false) {
-      templatesCache.reset()
-    }
+      if (reporter.options.sandbox.cache && reporter.options.sandbox.cache.enabled === false) {
+        templatesCache.reset()
+      }
 
-    try {
-      return await reporter.runInSandbox({
-        context: {
-          ...(engine.createContext ? engine.createContext(req) : {})
-        },
-        userCode: normalizedHelpers,
-        initFn,
-        executionFn,
-        currentPath: entityPath,
-        onRequire: (moduleName, { context }) => {
-          if (engine.onRequire) {
-            return engine.onRequire(moduleName, { context })
+      try {
+        return await reporter.runInSandbox({
+          context: {
+            ...(engine.createContext ? engine.createContext(req) : {})
+          },
+          userCode: normalizedHelpers,
+          initFn,
+          executionFn,
+          currentPath: entityPath,
+          onRequire: (moduleName, { context }) => {
+            if (engine.onRequire) {
+              return engine.onRequire(moduleName, { context })
+            }
           }
+        }, req)
+      } catch (e) {
+        if (!handleErrors) {
+          throw e
         }
-      }, req)
-    } catch (e) {
-      if (!handleErrors) {
-        throw e
-      }
 
-      const nestedErrorWithEntity = e.entity != null
+        const nestedErrorWithEntity = e.entity != null
 
-      const templatePath = req.template._id ? await reporter.folders.resolveEntityPath(req.template, 'templates', req) : 'anonymous'
+        const templatePath = req.template._id ? await reporter.folders.resolveEntityPath(req.template, 'templates', req) : 'anonymous'
 
-      const newError = reporter.createError(`Error when evaluating engine ${engine.name} for template ${templatePath}`, { original: e })
+        const newError = reporter.createError(`Error when evaluating engine ${engine.name} for template ${templatePath}`, { original: e })
 
-      if (templatePath !== 'anonymous' && !nestedErrorWithEntity) {
-        const templateFound = await reporter.folders.resolveEntityFromPath(templatePath, 'templates', req)
+        if (templatePath !== 'anonymous' && !nestedErrorWithEntity) {
+          const templateFound = await reporter.folders.resolveEntityFromPath(templatePath, 'templates', req)
 
-        if (templateFound != null) {
-          newError.entity = {
-            shortid: templateFound.entity.shortid,
-            name: templateFound.entity.name,
-            content
+          if (templateFound != null) {
+            newError.entity = {
+              shortid: templateFound.entity.shortid,
+              name: templateFound.entity.name,
+              content
+            }
           }
         }
-      }
 
-      if (!nestedErrorWithEntity && e.property !== 'content') {
-        newError.property = 'helpers'
-      }
+        if (!nestedErrorWithEntity && e.property !== 'content') {
+          newError.property = 'helpers'
+        }
 
-      if (nestedErrorWithEntity) {
-        // errors from nested assets evals needs an unwrap for some reason
-        newError.entity = { ...e.entity }
-      }
+        if (nestedErrorWithEntity) {
+          // errors from nested assets evals needs an unwrap for some reason
+          newError.entity = { ...e.entity }
+        }
 
-      // we remove the decoratedSuffix (created from sandbox) from the stack trace (if it is there) because it
-      // just creates noise and duplication when printing the error,
-      // we just want the decoration on the message not in the stack trace
-      if (e.decoratedSuffix != null && newError.stack.includes(e.decoratedSuffix)) {
-        newError.stack = newError.stack.replace(e.decoratedSuffix, '')
-      }
+        // we remove the decoratedSuffix (created from sandbox) from the stack trace (if it is there) because it
+        // just creates noise and duplication when printing the error,
+        // we just want the decoration on the message not in the stack trace
+        if (e.decoratedSuffix != null && newError.stack.includes(e.decoratedSuffix)) {
+          newError.stack = newError.stack.replace(e.decoratedSuffix, '')
+        }
 
-      throw newError
+        throw newError
+      }
+    } finally {
+      if (sandboxId != null && contextExecutionChainMap.get(sandboxId)?.length === 0) {
+        contextExecutionChainMap.delete(sandboxId)
+      }
     }
   }
 

package/lib/worker/reporter.js

@@ -88,7 +88,7 @@ class WorkerReporter extends Reporter {
       require('@jsreport/ses')
 
       // eslint-disable-next-line
-      lockdown({
+      repairIntrinsics({
         // don't change locale based methods which users may be using in their templates
         localeTaming: 'unsafe',
         errorTaming: 'unsafe',
@@ -108,6 +108,8 @@ class WorkerReporter extends Reporter {
         overrideTaming: 'severe'
       })
 
+      // NOTE: we need to add these overrides between repairIntrinsics and hardenIntrinsics
+      // for them to be valid.
       // in this mode we alias the unsafe methods to safe ones
       Buffer.allocUnsafe = function allocUnsafe (size) {
         return Buffer.alloc(size)
@@ -117,6 +119,9 @@ class WorkerReporter extends Reporter {
         return Buffer.alloc(size)
       }
 
+      // eslint-disable-next-line
+      hardenIntrinsics()
+
       // we also harden Buffer because we expose it to sandbox
       // eslint-disable-next-line
       harden(Buffer)

package/lib/worker/sandbox/isolatedRequire.js

@@ -12,7 +12,7 @@ const ISOLATED_PACKAGE_JSON_CACHE = new Map()
 // to bring isolated modules across renders and without memory leaks.
 // most of the code is copied from node.js source code and adapted a bit
 // (you will see in some parts specific links to node.js source code counterpart for reference)
-function isolatedRequire (_moduleId, modulesMeta, requireFromRootDirectory) {
+function isolatedRequire (_moduleId, modulesMeta, resolveModule) {
   const parentModule = typeof _moduleId !== 'string' ? _moduleId.parent : null
   const moduleId = parentModule ? _moduleId.moduleId : _moduleId
 
@@ -32,10 +32,14 @@ function isolatedRequire (_moduleId, modulesMeta, requireFromRootDirectory) {
   }
 
   const { rootModule, modulesCache, requireExtensions } = modulesMeta
-
-  const fullModulePath = resolveFilename(ISOLATED_REQUIRE_RESOLVE_CACHE, requireFromRootDirectory.resolve, moduleId, { parentModulePath: parentModule?.path })
+  const fullModulePath = resolveFilename(ISOLATED_REQUIRE_RESOLVE_CACHE, resolveModule, moduleId, { parentModulePath: parentModule?.path })
 
   if (modulesCache[fullModulePath]) {
+    // if module was already tried to be loaded and ended with error we rethrow the error
+    if (modulesCache[fullModulePath].loadingError != null) {
+      throw modulesCache[fullModulePath].loadingError
+    }
+
     return modulesCache[fullModulePath].exports
   }
 
@@ -50,29 +54,33 @@ function isolatedRequire (_moduleId, modulesMeta, requireFromRootDirectory) {
   // https://github.com/nodejs/node/blob/v18.14.2/lib/internal/modules/cjs/loader.js#L1133
   // we can not add this to the IsolatedModule.prototype because we need access to other variables
   mod.require = function (id) {
-    return isolatedRequire({ parent: this, moduleId: id }, modulesMeta, requireFromRootDirectory)
+    return isolatedRequire({ parent: this, moduleId: id }, modulesMeta, resolveModule)
   }
 
   modulesCache[fullModulePath] = mod
 
-  mod.filename = fullModulePath
-  mod.paths = Module._nodeModulePaths(path.dirname(fullModulePath))
+  try {
+    mod.filename = fullModulePath
+    mod.paths = Module._nodeModulePaths(path.dirname(fullModulePath))
 
-  const extension = findLongestRegisteredExtension(fullModulePath, requireExtensions)
+    const extension = findLongestRegisteredExtension(fullModulePath, requireExtensions)
 
-  // https://github.com/nodejs/node/blob/v18.14.2/lib/internal/modules/cjs/loader.js#L1113
-  // allow .mjs to be overridden
-  if (fullModulePath.endsWith('.mjs') && !requireExtensions['.mjs']) {
-    throw createRequireESMError(fullModulePath)
-  }
-
-  const moduleResolver = requireExtensions[extension]
+    // https://github.com/nodejs/node/blob/v18.14.2/lib/internal/modules/cjs/loader.js#L1113
+    // allow .mjs to be overridden
+    if (fullModulePath.endsWith('.mjs') && !requireExtensions['.mjs']) {
+      throw createRequireESMError(fullModulePath)
+    }
 
-  moduleResolver(mod, fullModulePath)
+    const moduleResolver = requireExtensions[extension]
 
-  mod.loaded = true
+    moduleResolver(mod, fullModulePath)
 
-  return mod.exports
+    mod.loaded = true
+    return mod.exports
+  } catch (error) {
+    mod.loadingError = error
+    throw error
+  }
 }
 
 function setDefaultRequireExtensions (currentExtensions, requireFromRootDirectory, compileScript) {

package/lib/worker/sandbox/requireSandbox.js

@@ -80,11 +80,24 @@ function doRequire (moduleId, requireFromRootDirectory, _requirePaths, modulesMe
   const _require = isolateModules ? isolatedRequire : requireFromRootDirectory
   const extraRequireParams = []
 
-  if (isolateModules) {
-    extraRequireParams.push(modulesMeta, requireFromRootDirectory)
+  const resolveModule = (mId, ...args) => {
+    let normalizedModuleId = mId
+
+    if (normalizedModuleId === '..' || normalizedModuleId === '.') {
+      // NOTE: we need to manually normalize because node has a bug
+      // https://github.com/nodejs/node/issues/47000
+      // when using require.resolve and using options.paths, it does not recognize for "..", "."
+      // to be relative, just other cases work like "../", "..\\",
+      // so when we detect this case we normalize it in order for node to resolve correctly
+      normalizedModuleId = normalizedModuleId + path.sep
+    }
+
+    return requireFromRootDirectory.resolve(normalizedModuleId, ...args)
   }
 
-  const resolveModule = requireFromRootDirectory.resolve
+  if (isolateModules) {
+    extraRequireParams.push(modulesMeta, resolveModule)
+  }
 
   let result = executeRequire(_require, resolveModule, moduleId, searchedPaths, ...extraRequireParams)
 

package/lib/worker/sandbox/runInSandbox.js

@@ -25,6 +25,9 @@ module.exports = function createRunInSandbox (reporter) {
     // it may turn out it is a bad approach in assets so we gonna delete it here
     const executionFnName = `${nanoid()}_executionFn`
 
+    // creating new id different than execution to ensure user code can not get access to
+    // internal functions by using the __sandboxId
+    context.__sandboxId = nanoid()
     context[executionFnName] = executionFn
     context.__appDirectory = reporter.options.appDirectory
     context.__rootDirectory = reporter.options.rootDirectory

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "4.3.0",
+  "version": "4.4.0",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -44,7 +44,7 @@
     "@jsreport/mingo": "2.4.1",
     "@jsreport/reap": "0.1.0",
     "@jsreport/serializator": "1.0.0",
-    "@jsreport/ses": "1.0.1",
+    "@jsreport/ses": "1.1.0",
     "ajv": "6.12.6",
     "app-root-path": "3.0.0",
     "bytes": "3.1.2",