@jsreport/jsreport-core 3.4.1 → 3.6.0

package/lib/worker/render/executeEngine.js

@@ -9,11 +9,13 @@ const LRU = require('lru-cache')
 const { nanoid } = require('nanoid')
 
 module.exports = (reporter) => {
-  const cache = LRU(reporter.options.sandbox.cache || { max: 100 })
+  const templatesCache = LRU(reporter.options.sandbox.cache)
+  let systemHelpersCache
 
-  reporter.templatingEngines = { cache }
+  reporter.templatingEngines = { cache: templatesCache }
 
   const executionFnParsedParamsMap = new Map()
+  const executionAsyncResultsMap = new Map()
 
   const templatingEnginesEvaluate = async (mainCall, { engine, content, helpers, data }, { entity, entitySet }, req) => {
     const engineImpl = reporter.extensionsManager.engines.find((e) => e.name === engine)
@@ -28,19 +30,23 @@ module.exports = (reporter) => {
       executionFnParsedParamsMap.set(req.context.id, new Map())
     }
 
+    const executionId = nanoid(7)
+
     try {
       const res = await executeEngine({
         engine: engineImpl,
         content,
         helpers,
         data
-      }, { handleErrors: false, entity, entitySet }, req)
+      }, { executionId, handleErrors: false, entity, entitySet }, req)
 
       return res.content
     } finally {
       if (mainCall) {
         executionFnParsedParamsMap.delete(req.context.id)
       }
+
+      executionAsyncResultsMap.delete(executionId)
     }
   }
 
@@ -54,6 +60,39 @@ module.exports = (reporter) => {
     proxy.templatingEngines = {
       evaluate: async (executionInfo, entityInfo) => {
         return templatingEnginesEvaluate(false, executionInfo, entityInfo, req)
+      },
+      waitForAsyncHelper: async (maybeAsyncContent) => {
+        if (
+          context.__executionId == null ||
+          !executionAsyncResultsMap.has(context.__executionId) ||
+          typeof maybeAsyncContent !== 'string'
+        ) {
+          return maybeAsyncContent
+        }
+
+        const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
+        const asyncHelperResultRegExp = /{#asyncHelperResult ([^{}]+)}/
+        let content = maybeAsyncContent
+        let matchResult
+
+        do {
+          if (matchResult != null) {
+            const matchedPart = matchResult[0]
+            const asyncResultId = matchResult[1]
+            const result = await asyncResultMap.get(asyncResultId)
+            content = `${content.slice(0, matchResult.index)}${result}${content.slice(matchResult.index + matchedPart.length)}`
+          }
+
+          matchResult = content.match(asyncHelperResultRegExp)
+        } while (matchResult != null)
+
+        return content
+      },
+      waitForAsyncHelpers: async () => {
+        if (context.__executionId != null && executionAsyncResultsMap.has(context.__executionId)) {
+          const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
+          return Promise.all([...asyncResultMap.keys()].map((k) => asyncResultMap.get(k)))
+        }
       }
     }
   })
@@ -64,6 +103,8 @@ module.exports = (reporter) => {
     req.data.__rootDirectory = reporter.options.rootDirectory
     req.data.__parentModuleDirectory = reporter.options.parentModuleDirectory
 
+    const executionId = nanoid(7)
+
     try {
       return await executeEngine({
         engine,
@@ -71,55 +112,88 @@ module.exports = (reporter) => {
         helpers: req.template.helpers,
         data: req.data
       }, {
+        executionId,
         handleErrors: true,
         entity: req.template,
         entitySet: 'templates'
       }, req)
     } finally {
       executionFnParsedParamsMap.delete(req.context.id)
+      executionAsyncResultsMap.delete(executionId)
     }
   }
 
-  async function executeEngine ({ engine, content, helpers, data }, { handleErrors, entity, entitySet }, req) {
+  async function executeEngine ({ engine, content, helpers, data }, { executionId, handleErrors, entity, entitySet }, req) {
     let entityPath
 
     if (entity._id) {
       entityPath = await reporter.folders.resolveEntityPath(entity, entitySet, req)
     }
 
-    const registerResults = await reporter.registerHelpersListeners.fire(req)
-    const systemHelpers = []
+    const normalizedHelpers = `${helpers || ''}`
+    const executionFnParsedParamsKey = `entity:${entity.shortid || 'anonymous'}:helpers:${normalizedHelpers}`
 
-    for (const result of registerResults) {
-      if (result == null) {
-        continue
+    const initFn = async (getTopLevelFunctions, compileScript) => {
+      if (systemHelpersCache != null) {
+        return systemHelpersCache
       }
 
-      if (typeof result === 'string') {
-        systemHelpers.push(result)
+      const registerResults = await reporter.registerHelpersListeners.fire()
+      const systemHelpers = []
+
+      for (const result of registerResults) {
+        if (result == null) {
+          continue
+        }
+
+        if (typeof result === 'string') {
+          systemHelpers.push(result)
+        }
       }
-    }
 
-    const systemHelpersStr = systemHelpers.join('\n')
-    const joinedHelpers = systemHelpersStr + '\n' + (helpers || '')
-    const executionFnParsedParamsKey = `entity:${entity.shortid || 'anonymous'}:helpers:${joinedHelpers}`
+      const systemHelpersStr = systemHelpers.join('\n')
+
+      const functionNames = getTopLevelFunctions(systemHelpersStr)
 
-    const executionFn = async ({ require, console, topLevelFunctions }) => {
+      const exposeSystemHelpersCode = `for (const fName of ${JSON.stringify(functionNames)}) { this[fName] = __topLevelFunctions[fName] }`
+
+      // we sync the __topLevelFunctions with system helpers and expose it immediately to the global context
+      const userCode = `(async () => { ${systemHelpersStr};
+      __topLevelFunctions = {...__topLevelFunctions, ${functionNames.map(h => `"${h}": ${h}`).join(',')}}; ${exposeSystemHelpersCode}
+      })()`
+
+      const filename = 'system-helpers.js'
+      const script = compileScript(userCode, filename)
+
+      systemHelpersCache = {
+        filename,
+        source: systemHelpersStr,
+        script
+      }
+
+      return systemHelpersCache
+    }
+
+    const executionFn = async ({ require, console, topLevelFunctions, context }) => {
       const asyncResultMap = new Map()
-      executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions })
+
+      context.__executionId = executionId
+
+      executionAsyncResultsMap.set(executionId, asyncResultMap)
+      executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions, context })
+
       const key = `template:${content}:${engine.name}`
 
-      if (!cache.has(key)) {
+      if (!templatesCache.has(key)) {
         try {
-          cache.set(key, engine.compile(content, { require }))
+          templatesCache.set(key, engine.compile(content, { require }))
         } catch (e) {
           e.property = 'content'
           throw e
         }
       }
 
-      const compiledTemplate = cache.get(key)
-
+      const compiledTemplate = templatesCache.get(key)
       const wrappedTopLevelFunctions = {}
 
       for (const h of Object.keys(topLevelFunctions)) {
@@ -127,7 +201,9 @@ module.exports = (reporter) => {
       }
 
       let contentResult = await engine.execute(compiledTemplate, wrappedTopLevelFunctions, data, { require })
+
       const resolvedResultsMap = new Map()
+
       while (asyncResultMap.size > 0) {
         await Promise.all([...asyncResultMap.keys()].map(async (k) => {
           resolvedResultsMap.set(k, `${await asyncResultMap.get(k)}`)
@@ -143,26 +219,31 @@ module.exports = (reporter) => {
       }
 
       return {
-        content: contentResult
+        // handlebars escapes single brackets before execution to prevent errors on {#asset}
+        // we need to unescape them later here, because at the moment the engine.execute finishes
+        // the async helpers aren't executed yet
+        content: engine.unescape ? engine.unescape(contentResult) : contentResult
       }
     }
 
     // executionFnParsedParamsMap is there to cache parsed components helpers to speed up longer loops
     // we store there for the particular request and component a promise and only the first component gets compiled
     if (executionFnParsedParamsMap.get(req.context.id).has(executionFnParsedParamsKey)) {
-      const { require, console, topLevelFunctions } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
+      const { require, console, topLevelFunctions, context } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
 
-      return executionFn({ require, console, topLevelFunctions })
+      return executionFn({ require, console, topLevelFunctions, context })
     } else {
       const awaiter = {}
+
       awaiter.promise = new Promise((resolve) => {
         awaiter.resolve = resolve
       })
+
       executionFnParsedParamsMap.get(req.context.id).set(executionFnParsedParamsKey, awaiter)
     }
 
     if (reporter.options.sandbox.cache && reporter.options.sandbox.cache.enabled === false) {
-      cache.reset()
+      templatesCache.reset()
     }
 
     try {
@@ -170,10 +251,10 @@ module.exports = (reporter) => {
         context: {
           ...(engine.createContext ? engine.createContext() : {})
         },
-        userCode: joinedHelpers,
+        userCode: normalizedHelpers,
+        initFn,
         executionFn,
         currentPath: entityPath,
-        errorLineNumberOffset: systemHelpersStr.split('\n').length,
         onRequire: (moduleName, { context }) => {
           if (engine.onRequire) {
             return engine.onRequire(moduleName, { context })

package/lib/worker/render/moduleHelper.js

@@ -4,7 +4,7 @@ const path = require('path')
 module.exports = (reporter) => {
   let helpersScript
 
-  reporter.registerHelpersListeners.add('core-helpers', (req) => {
+  reporter.registerHelpersListeners.add('core-helpers', () => {
     return helpersScript
   })
 
@@ -12,11 +12,11 @@ module.exports = (reporter) => {
     helpersScript = await fs.readFile(path.join(__dirname, '../../static/helpers.js'), 'utf8')
   })
 
-  reporter.extendProxy((proxy, req, { safeRequire }) => {
+  reporter.extendProxy((proxy, req, { sandboxRequire }) => {
     proxy.module = async (module) => {
-      if (!reporter.options.allowLocalFilesAccess && reporter.options.sandbox.allowedModules !== '*') {
+      if (!reporter.options.trustUserCode && reporter.options.sandbox.allowedModules !== '*') {
         if (reporter.options.sandbox.allowedModules.indexOf(module) === -1) {
-          throw reporter.createError(`require of module ${module} was rejected. Either set allowLocalFilesAccess=true or sandbox.allowLocalModules='*' or sandbox.allowLocalModules=['${module}'] `, { status: 400 })
+          throw reporter.createError(`require of module ${module} was rejected. Either set trustUserCode=true or sandbox.allowLocalModules='*' or sandbox.allowLocalModules=['${module}'] `, { status: 400 })
         }
       }
 

package/lib/worker/render/profiler.js

@@ -57,9 +57,9 @@ class Profiler {
 
     if (m.type !== 'log') {
       req.context.profiling.lastEventId = m.id
+      m.operationId = m.operationId || generateRequestId()
     }
 
-    m.operationId = m.operationId || generateRequestId()
     if (m.previousOperationId == null && req.context.profiling.lastOperationId) {
       m.previousOperationId = req.context.profiling.lastOperationId
     }
@@ -72,15 +72,21 @@ class Profiler {
       let content = res.content
 
       if (content != null) {
-        if (isbinaryfile(content)) {
+        if (content.length > this.reporter.options.profiler.maxDiffSize) {
           content = {
-            content: res.content.toString('base64'),
-            encoding: 'base64'
+            tooLarge: true
           }
         } else {
-          content = {
-            content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
-            encoding: 'diff'
+          if (isbinaryfile(content)) {
+            content = {
+              content: res.content.toString('base64'),
+              encoding: 'base64'
+            }
+          } else {
+            content = {
+              content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
+              encoding: 'diff'
+            }
           }
         }
       }
@@ -91,9 +97,14 @@ class Profiler {
 
       const stringifiedReq = JSON.stringify({ template: req.template, data: req.data }, null, 2)
 
-      m.req = { diff: createPatch('req', req.context.profiling.reqLastVal || '', stringifiedReq, 0) }
+      m.req = { }
+      if (stringifiedReq.length * 4 > this.reporter.options.profiler.maxDiffSize) {
+        m.req.tooLarge = true
+      } else {
+        m.req.diff = createPatch('req', req.context.profiling.reqLastVal || '', stringifiedReq, 0)
+      }
 
-      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content)) ? null : res.content.toString()
+      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content) || content.tooLarge) ? null : res.content.toString()
       req.context.profiling.resMetaLastVal = stringifiedResMeta
       req.context.profiling.reqLastVal = stringifiedReq
     }
@@ -129,10 +140,6 @@ class Profiler {
       previousOperationId: parentReq ? parentReq.context.profiling.lastOperationId : null
     }
 
-    if (!req.context.isChildRequest) {
-      profilerEvent.profileId = req.context.profiling.entity._id
-    }
-
     return this.emit(profilerEvent, req, res)
   }
 

package/lib/worker/render/render.js

@@ -10,7 +10,6 @@ const Request = require('../../shared/request')
 const generateRequestId = require('../../shared/generateRequestId')
 const resolveReferences = require('./resolveReferences.js')
 const moduleHelper = require('./moduleHelper')
-let reportCounter = 0
 
 module.exports = (reporter) => {
   moduleHelper(reporter)
@@ -127,9 +126,6 @@ module.exports = (reporter) => {
         response.meta.reportName = 'report'
       }
 
-      request.context.reportCounter = ++reportCounter
-      request.context.startTimestamp = new Date().getTime()
-
       if (parentReq == null) {
         reporter.requestModulesCache.set(request.context.rootId, Object.create(null))
       }

package/lib/worker/reporter.js

@@ -111,14 +111,14 @@ class WorkerReporter extends Reporter {
     this._proxyRegistrationFns.push(registrationFn)
   }
 
-  createProxy ({ req, runInSandbox, context, getTopLevelFunctions, safeRequire }) {
+  createProxy ({ req, runInSandbox, context, getTopLevelFunctions, sandboxRequire }) {
     const proxyInstance = {}
     for (const fn of this._proxyRegistrationFns) {
       fn(proxyInstance, req, {
         runInSandbox,
         context,
         getTopLevelFunctions,
-        safeRequire
+        sandboxRequire
       })
     }
     return proxyInstance
@@ -137,10 +137,11 @@ class WorkerReporter extends Reporter {
     manager,
     context,
     userCode,
+    initFn,
     executionFn,
+    currentPath,
     onRequire,
     propertiesConfig,
-    currentPath,
     errorLineNumberOffset
   }, req) {
     // we flush before running code in sandbox because it can potentially
@@ -152,10 +153,11 @@ class WorkerReporter extends Reporter {
       manager,
       context,
       userCode,
+      initFn,
       executionFn,
+      currentPath,
       onRequire,
       propertiesConfig,
-      currentPath,
       errorLineNumberOffset
     }, req)
   }

package/lib/worker/sandbox/{safeSandbox.js → createSandbox.js}

@@ -19,6 +19,7 @@ module.exports = (_sandbox, options = {}) => {
     propertiesConfig = {},
     globalModules = [],
     allowedModules = [],
+    safeExecution,
     requireMap
   } = options
 
@@ -65,7 +66,7 @@ module.exports = (_sandbox, options = {}) => {
       }
     }
 
-    if (allowAllModules || allowedModules === '*') {
+    if (!safeExecution || allowAllModules || allowedModules === '*') {
       return doRequire(moduleName, requirePaths, modulesCache)
     }
 
@@ -111,28 +112,28 @@ module.exports = (_sandbox, options = {}) => {
     require: (m) => _require(m, { context: _sandbox })
   })
 
-  const vm = new VM()
+  let safeVM
+  let vmSandbox
 
-  // NOTE: we wrap the Contextify.object, Decontextify.object methods because those are the
-  // methods that returns the proxies created by vm2 in the sandbox, we want to have a list of those
-  // to later use them
-  const wrapAndSaveProxyResult = (originalFn, thisArg) => {
-    return (value, ...args) => {
-      const result = originalFn.call(thisArg, value, ...args)
+  if (safeExecution) {
+    safeVM = new VM()
 
-      if (result != null && result.isVMProxy === true) {
-        proxiesInVM.set(result, value)
-      }
+    // delete the vm.sandbox.global because it introduces json stringify issues
+    // and we don't need such global in context
+    delete safeVM.sandbox.global
 
-      return result
+    for (const name in sandbox) {
+      safeVM.setGlobal(name, sandbox[name])
     }
-  }
 
-  vm._internal.Contextify.object = wrapAndSaveProxyResult(vm._internal.Contextify.object, vm._internal.Contextify)
-  vm._internal.Decontextify.object = wrapAndSaveProxyResult(vm._internal.Decontextify.object, vm._internal.Decontextify)
+    vmSandbox = safeVM.sandbox
+  } else {
+    vmSandbox = originalVM.createContext(undefined)
+    vmSandbox.Buffer = Buffer
 
-  for (const name in sandbox) {
-    vm._internal.Contextify.setGlobal(name, sandbox[name])
+    for (const name in sandbox) {
+      vmSandbox[name] = sandbox[name]
+    }
   }
 
   // processing top level props because getter/setter descriptors
@@ -141,57 +142,46 @@ module.exports = (_sandbox, options = {}) => {
     const currentConfig = propsConfig[key]
 
     if (currentConfig.root && currentConfig.root.sandboxReadOnly) {
-      readOnlyProp(vm._context, key, [], customProxies, { onlyTopLevel: true })
+      readOnlyProp(vmSandbox, key, [], customProxies, { onlyTopLevel: true })
     }
   })
 
   const sourceFilesInfo = new Map()
 
   return {
-    sandbox: vm._context,
+    sandbox: vmSandbox,
     console: _console,
     sourceFilesInfo,
-    contextifyValue: (value) => {
-      return vm._internal.Contextify.value(value)
-    },
-    decontextifyValue: (value) => {
-      return vm._internal.Decontextify.value(value)
+    compileScript: (code, filename) => {
+      return doCompileScript(code, filename, safeExecution)
     },
     restore: () => {
-      return restoreProperties(vm._context, originalValues, proxiesInVM, customProxies)
-    },
-    unproxyValue: (value) => {
-      return getOriginalFromProxy(proxiesInVM, customProxies, value)
+      return restoreProperties(vmSandbox, originalValues, proxiesInVM, customProxies)
     },
-    safeRequire: (modulePath) => _require(modulePath, { context: _sandbox, allowAllModules: true }),
-    run: async (code, { filename, errorLineNumberOffset = 0, source, entity, entitySet } = {}) => {
-      const script = new VMScript(code, filename)
+    sandboxRequire: (modulePath) => _require(modulePath, { context: _sandbox, allowAllModules: true }),
+    run: async (codeOrScript, { filename, errorLineNumberOffset = 0, source, entity, entitySet } = {}) => {
+      let run
 
       if (filename != null && source != null) {
         sourceFilesInfo.set(filename, { filename, source, entity, entitySet, errorLineNumberOffset })
       }
 
-      // NOTE: if we need to upgrade vm2 we will need to check the source of this function
-      // in vm2 repo and see if we need to change this,
-      // we needed to override this method because we want "displayErrors" to be true in order
-      // to show nice error when the compile of a script fails
-      script._compile = function (prefix, suffix) {
-        return new originalVM.Script(prefix + this.getCompiledCode() + suffix, {
-          filename: this.filename,
-          displayErrors: true,
-          lineOffset: this.lineOffset,
-          columnOffset: this.columnOffset,
-          // THIS FN WAS TAKEN FROM vm2 source, nothing special here
-          importModuleDynamically: () => {
-            // We can't throw an error object here because since vm.Script doesn't store a context, we can't properly contextify that error object.
-            // eslint-disable-next-line no-throw-literal
-            throw 'Dynamic imports are not allowed.'
-          }
-        })
+      const script = typeof codeOrScript !== 'string' ? codeOrScript : doCompileScript(codeOrScript, filename, safeExecution)
+
+      if (safeExecution) {
+        run = async () => {
+          return safeVM.run(script)
+        }
+      } else {
+        run = async () => {
+          return script.runInContext(vmSandbox, {
+            displayErrors: true
+          })
+        }
       }
 
       try {
-        const result = await vm.run(script)
+        const result = await run()
         return result
       } catch (e) {
         decorateErrorMessage(e, sourceFilesInfo)
@@ -202,10 +192,53 @@ module.exports = (_sandbox, options = {}) => {
   }
 }
 
+function doCompileScript (code, filename, safeExecution) {
+  let script
+
+  if (safeExecution) {
+    script = new VMScript(code, filename)
+
+    // NOTE: if we need to upgrade vm2 we will need to check the source of this function
+    // in vm2 repo and see if we need to change this,
+    // we needed to override this method because we want "displayErrors" to be true in order
+    // to show nice error when the compile of a script fails
+    script._compile = function (prefix, suffix) {
+      return new originalVM.Script(prefix + this.getCompiledCode() + suffix, {
+        __proto__: null,
+        filename: this.filename,
+        displayErrors: true,
+        lineOffset: this.lineOffset,
+        columnOffset: this.columnOffset,
+        // THIS FN WAS TAKEN FROM vm2 source, nothing special here
+        importModuleDynamically: () => {
+          // We can't throw an error object here because since vm.Script doesn't store a context, we can't properly contextify that error object.
+          // eslint-disable-next-line no-throw-literal
+          throw 'Dynamic imports are not allowed.'
+        }
+      })
+    }
+
+    // do the compilation
+    script._compileVM()
+  } else {
+    script = new originalVM.Script(code, {
+      filename,
+      displayErrors: true,
+      importModuleDynamically: () => {
+        // We can't throw an error object here because since vm.Script doesn't store a context, we can't properly contextify that error object.
+        // eslint-disable-next-line no-throw-literal
+        throw 'Dynamic imports are not allowed.'
+      }
+    })
+  }
+
+  return script
+}
+
 function doRequire (moduleName, requirePaths = [], modulesCache) {
   const searchedPaths = []
 
-  function safeRequire (require, modulePath) {
+  function optimizedRequire (require, modulePath) {
     // save the current module cache, we will use this to restore the cache to the
     // original values after the require finish
     const originalModuleCache = Object.assign(Object.create(null), require.cache)
@@ -260,13 +293,13 @@ function doRequire (moduleName, requirePaths = [], modulesCache) {
     }
   }
 
-  let result = safeRequire(require, moduleName)
+  let result = optimizedRequire(require, moduleName)
 
   if (!result) {
     let pathsSearched = 0
 
     while (!result && pathsSearched < requirePaths.length) {
-      result = safeRequire(require, path.join(requirePaths[pathsSearched], moduleName))
+      result = optimizedRequire(require, path.join(requirePaths[pathsSearched], moduleName))
       pathsSearched++
     }
   }