@jsreport/jsreport-core 3.2.0 → 3.4.2

package/README.md

@@ -36,7 +36,7 @@ const result = await jsreport.render({
 		foo: "world"
 	}
 })
-await fs.writeFile('out.pdf', resp.content)
+await fs.writeFile('out.pdf', result.content)
 ```
 
 ## Render
@@ -282,6 +282,22 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 3.4.2
+
+- update dep `vm2` to fix security vulnerability in sandbox
+
+### 3.4.1
+
+- fix passing data to async report
+- fix blob appends
+
+### 3.4.0
+
+- fix for reports execution
+- fix for render profiling
+- fix for blob storage remove
+- update deps to fix npm audit
+
 ### 3.1.0
 
 - fix blob storage append to not existing blob (mongo)

package/lib/main/blobStorage/blobStorage.js

@@ -25,6 +25,7 @@ module.exports = (reporter, options) => {
         let existingBuf = Buffer.from([])
         try {
           existingBuf = await provider.read(blobName, req)
+          await provider.remove(blobName, req)
         } catch (e) {
           // so far blob storage throws when blob doesnt exit
         }

package/lib/main/logger.js

@@ -37,14 +37,25 @@ function createLogger () {
 }
 
 function configureLogger (logger, _transports) {
+  const transports = _transports || {}
+  const transportFormatMap = new WeakMap()
+
+  // we ensure we do .format cleanup on options first before checking if the logger
+  // is configured or not, this ensure that options are properly cleaned up when
+  // configureLogger is called more than once (like when execution cli commands from extensions)
+  for (const [, transpOptions] of Object.entries(transports)) {
+    if (transpOptions.format != null) {
+      transportFormatMap.set(transpOptions, transpOptions.format)
+      delete transpOptions.format
+    }
+  }
+
   const configuredPreviously = logger.__configured__ === true
 
   if (configuredPreviously) {
     return
   }
 
-  const transports = _transports || {}
-
   const knownTransports = {
     debug: DebugTransport,
     console: winston.transports.Console,
@@ -78,22 +89,21 @@ function configureLogger (logger, _transports) {
       continue
     }
 
+    let originalFormat
+
+    if (transportFormatMap.has(transpOptions)) {
+      originalFormat = transportFormatMap.get(transpOptions)
+    }
+
     if (
-      transpOptions.format != null &&
-      typeof transpOptions.format.constructor !== 'function'
+      originalFormat != null &&
+      typeof originalFormat.constructor !== 'function'
     ) {
       throw new Error(`Invalid option for transport object "${
         transpName
       }", option "format" has an incorrect value, must be an instance of loggerFormat. check your "logger" config`)
     }
 
-    let originalFormat
-
-    if (transpOptions.format != null) {
-      originalFormat = transpOptions.format
-      delete transpOptions.format
-    }
-
     const options = Object.assign(omit(transpOptions, knownOptions), {
       name: transpName
     })

package/lib/main/monitoring.js

@@ -84,7 +84,8 @@ module.exports = (reporter) => {
 
   reporter.documentStore.registerEntitySet('monitoring', {
     entityType: 'jsreport.MonitoringType',
-    exportable: false
+    exportable: false,
+    shared: true
   })
 
   reporter.monitoring = new Monitoring(reporter)

package/lib/main/optionsLoad.js

@@ -133,15 +133,9 @@ async function loadConfig (defaults, options, loadExternal = true) {
         const currentValue = obj.value[extensionKey]
         delete obj.value[extensionKey]
 
-        if (realExtensionName !== extensionKey && obj.value[realExtensionName]) {
-          obj.value[realExtensionName] = extend(
-            true,
-            obj.value[realExtensionName],
-            currentValue
-          )
-        } else {
-          obj.value[realExtensionName] = currentValue
-        }
+        // the camelCase key version should already contain all merged values
+        // (from both the real extension name with "-" and camel case)
+        obj.value[realExtensionName] = currentValue
       })
     } else if (!normalize && obj.key.startsWith('extensions')) {
       // the transform ensures that camelCase alias keys of extensions

package/lib/main/profiler.js

@@ -26,9 +26,13 @@ module.exports = (reporter) => {
       return
     }
 
+    let lastOperation
+
     for (const m of events) {
       if (m.type === 'log') {
         reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp })
+      } else {
+        lastOperation = m
       }
 
       if (profilersMap.has(req.context.rootId)) {
@@ -36,8 +40,15 @@ module.exports = (reporter) => {
       }
     }
 
+    if (lastOperation != null) {
+      req.context.profiling.lastOperation = lastOperation
+    }
+
     profilerAppendChain.set(req.context.rootId, profilerAppendChain.get(req.context.rootId).then(() => {
-      return reporter.blobStorage.append(req.context.profiling.entity.blobName, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req).catch(e => {
+      return reporter.blobStorage.append(
+        req.context.profiling.entity.blobName,
+        Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req
+      ).catch(e => {
         reporter.logger.error('Failed to append to profile blob', e)
       })
     }))
@@ -64,6 +75,7 @@ module.exports = (reporter) => {
     profilerAppendChain.set(req.context.rootId, Promise.resolve())
 
     req.context.profiling = req.context.profiling || {}
+    req.context.profiling.lastOperation = null
 
     let blobName = `profiles/${req.context.rootId}.log`
 

package/lib/main/reporter.js

@@ -25,7 +25,7 @@ const setupValidateShortid = require('./store/setupValidateShortid')
 const documentStoreActions = require('./store/mainActions')
 const blobStorageActions = require('./blobStorage/mainActions')
 const Reporter = require('../shared/reporter')
-const Request = require('../shared/request')
+const Request = require('./request')
 const generateRequestId = require('../shared/generateRequestId')
 const Profiler = require('./profiler')
 const Monitoring = require('./monitoring')
@@ -315,7 +315,7 @@ class MainReporter extends Reporter {
    *
    * @public
    */
-  async render (req, parentReq) {
+  async render (req, options = {}) {
     if (!this._initialized) {
       throw new Error('Not initialized, you need to call jsreport.init().then before rendering')
     }
@@ -325,21 +325,18 @@ class MainReporter extends Reporter {
     req.context.rootId = req.context.rootId || generateRequestId()
     req.context.id = req.context.rootId
 
-    const worker = await this._workersManager.allocate(req, {
+    const worker = options.worker || await this._workersManager.allocate(req, {
       timeout: this.options.reportTimeout
     })
 
+    let keepWorker
     let workerAborted
-    if (parentReq && !parentReq.__isJsreportRequest__) {
-      const options = parentReq
-      parentReq = null
-
-      if (options.abortEmitter) {
-        options.abortEmitter.once('abort', () => {
-          workerAborted = true
-          worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
-        })
-      }
+
+    if (options.abortEmitter) {
+      options.abortEmitter.once('abort', () => {
+        workerAborted = true
+        worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+      })
     }
 
     const res = { meta: {} }
@@ -348,10 +345,7 @@ class MainReporter extends Reporter {
         throw this.createError('Request aborted by client')
       }
 
-      let isDataStoredInWorker = false
-
       if (req.rawContent) {
-        isDataStoredInWorker = true
         const result = await worker.execute({
           actionName: 'parse',
           req,
@@ -362,13 +356,7 @@ class MainReporter extends Reporter {
         req = result
       }
 
-      req = Request(req, parentReq)
-
-      if (isDataStoredInWorker) {
-        // we unset this because we want the Request() call in worker to evaluate the data
-        // and determine if the original was empty or not
-        delete req.context.originalInputDataIsEmpty
-      }
+      req = Request(req)
 
       // TODO: we will probably validate in the thread
       if (this.entityTypeValidator.getSchema('TemplateType') != null) {
@@ -388,13 +376,31 @@ class MainReporter extends Reporter {
         req.options &&
         req.options.timeout != null
       ) {
-        reportTimeout = req
+        reportTimeout = req.options.timeout
       }
 
-      await this.beforeRenderListeners.fire(req, res)
+      await this.beforeRenderListeners.fire(req, res, { worker })
 
-      if (req.context.isFinished) {
+      // this is used so far just in the reports extension
+      // it wants to send to the client immediate response with link to the report status
+      // but the previous steps already allocated worker which has the parsed input request
+      // so we need to keep the worker active and let the subsequent real render call use it
+      // we cant move the main beforeRenderListener before the worker allocation, because at that point
+      // the request isn't parsed and we don't know the template and options
+      if (req.context.returnResponseAndKeepWorker) {
+        keepWorker = true
         res.stream = Readable.from(res.content)
+
+        // just temporary workaround until we change how report render works
+        await this.documentStore.collection('profiles').update({
+          _id: req.context.profiling.entity._id
+        }, {
+          $set: {
+            state: 'success',
+            finishedOn: new Date(),
+            blobPersisted: true
+          }
+        }, req)
         return res
       }
 
@@ -414,6 +420,15 @@ class MainReporter extends Reporter {
     } catch (err) {
       if (err.code === 'WORKER_TIMEOUT') {
         err.message = 'Report timeout'
+        if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
+          err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
+        }
+
+        if (req.context.http != null) {
+          const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
+          err.message += `. You can inspect and find more details here: ${profileUrl}`
+        }
+
         err.weak = true
       }
 
@@ -429,7 +444,7 @@ class MainReporter extends Reporter {
       await this.renderErrorListeners.fire(req, res, err)
       throw err
     } finally {
-      if (!workerAborted) {
+      if (!workerAborted && !keepWorker) {
         await worker.release(req)
       }
     }

package/lib/main/request.js

@@ -0,0 +1,21 @@
+const extend = require('node.extend.without.arrays')
+
+module.exports = (obj) => {
+  const request = Object.create({}, {
+    __isJsreportRequest__: {
+      value: true,
+      writable: false,
+      configurable: false,
+      enumerable: false
+    }
+  })
+
+  request.template = extend(true, {}, obj.template)
+
+  request.options = extend(true, {}, request.options, obj.options)
+  request.context = extend(true, {}, request.context, obj.context)
+  request.context.shared = extend(true, {}, request.context.shared)
+  request.data = obj.data
+
+  return request
+}

package/lib/worker/render/executeEngine.js

@@ -85,7 +85,6 @@ module.exports = (reporter) => {
 
     if (entity._id) {
       entityPath = await reporter.folders.resolveEntityPath(entity, entitySet, req)
-      entityPath = entityPath.substring(0, entityPath.lastIndexOf('/'))
     }
 
     const registerResults = await reporter.registerHelpersListeners.fire(req)
@@ -106,12 +105,6 @@ module.exports = (reporter) => {
     const executionFnParsedParamsKey = `entity:${entity.shortid || 'anonymous'}:helpers:${joinedHelpers}`
 
     const executionFn = async ({ require, console, topLevelFunctions }) => {
-      // cached components dont call runInSandbox but share the proxy, we get to it here
-      if (entitySet !== 'templates') {
-        const jsreport = require('jsreport-proxy')
-        jsreport.currentPath = entityPath
-      }
-
       const asyncResultMap = new Map()
       executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions })
       const key = `template:${content}:${engine.name}`

package/lib/worker/render/profiler.js

@@ -8,7 +8,7 @@ class Profiler {
   constructor (reporter) {
     this.reporter = reporter
 
-    this.reporter.addRequestContextMetaConfig('profiling', { sandboxReadOnly: true })
+    this.reporter.addRequestContextMetaConfig('profiling', { sandboxHidden: true })
     this.reporter.addRequestContextMetaConfig('resolvedTemplate', { sandboxHidden: true })
 
     this.reporter.beforeMainActionListeners.add('profiler', (actionName, data, req) => {
@@ -18,16 +18,7 @@ class Profiler {
     })
 
     this.profiledRequestsMap = new Map()
-    const profileEventsFlushInterval = setInterval(async () => {
-      for (const id of [...this.profiledRequestsMap.keys()]) {
-        const profilingInfo = this.profiledRequestsMap.get(id)
-        if (profilingInfo) {
-          const batch = profilingInfo.batch
-          profilingInfo.batch = []
-          await this.reporter.executeMainAction('profile', batch, profilingInfo.req).catch((e) => this.reporter.logger.error(e, profilingInfo.req))
-        }
-      }
-    }, 100)
+    const profileEventsFlushInterval = setInterval(() => this.flush(), 100)
     profileEventsFlushInterval.unref()
 
     this.reporter.closeListeners.add('profiler', this, () => {
@@ -37,6 +28,19 @@ class Profiler {
     })
   }
 
+  async flush (id) {
+    const toProcess = id == null ? [...this.profiledRequestsMap.keys()] : [id]
+
+    for (const id of toProcess) {
+      const profilingInfo = this.profiledRequestsMap.get(id)
+      if (profilingInfo) {
+        const batch = profilingInfo.batch
+        profilingInfo.batch = []
+        await this.reporter.executeMainAction('profile', batch, profilingInfo.req).catch((e) => this.reporter.logger.error(e, profilingInfo.req))
+      }
+    }
+  }
+
   emit (m, req, res) {
     m.timestamp = m.timestamp || new Date().getTime()
 
@@ -89,7 +93,7 @@ class Profiler {
 
       m.req = { diff: createPatch('req', req.context.profiling.reqLastVal || '', stringifiedReq, 0) }
 
-      req.context.profiling.resLastVal = res.content
+      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content)) ? null : res.content.toString()
       req.context.profiling.resMetaLastVal = stringifiedResMeta
       req.context.profiling.reqLastVal = stringifiedReq
     }

package/lib/worker/reporter.js

@@ -143,6 +143,11 @@ class WorkerReporter extends Reporter {
     currentPath,
     errorLineNumberOffset
   }, req) {
+    // we flush before running code in sandbox because it can potentially
+    // include code that blocks the whole process (like `while (true) {}`) and we
+    // want to ensure that the batched messages are flushed before trying to execute the code
+    await this.profiler.flush(req.context.rootId)
+
     return this._runInSandbox({
       manager,
       context,

package/lib/worker/sandbox/runInSandbox.js

@@ -1,6 +1,7 @@
 const LRU = require('lru-cache')
-const safeSandbox = require('./safeSandbox')
+const stackTrace = require('stack-trace')
 const { customAlphabet } = require('nanoid')
+const safeSandbox = require('./safeSandbox')
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
 
 module.exports = (reporter) => {
@@ -27,7 +28,7 @@ module.exports = (reporter) => {
     context.__topLevelFunctions = {}
     context.__handleError = (err) => handleError(reporter, err)
 
-    const { run, restore, contextifyValue, decontextifyValue, unproxyValue, sandbox, safeRequire } = safeSandbox(context, {
+    const { sourceFilesInfo, run, restore, sandbox, safeRequire } = safeSandbox(context, {
       onLog: (log) => {
         reporter.logger[log.level](log.message, { ...req, timestamp: log.timestamp })
       },
@@ -61,16 +62,57 @@ module.exports = (reporter) => {
     })
 
     jsreportProxy = reporter.createProxy({ req, runInSandbox: run, context: sandbox, getTopLevelFunctions, safeRequire })
-    jsreportProxy.currentPath = currentPath
+
+    jsreportProxy.currentPath = async () => {
+      // we get the current path by throwing an error, which give us a stack trace
+      // which we analyze and see if some source file is associated to an entity
+      // if it is then we can properly get the path associated to it, if not we
+      // fallback to the current path passed as options
+      const filesCount = sourceFilesInfo.size
+      let resolvedPath = currentPath
+
+      if (filesCount > 0) {
+        const err = new Error('get me stack trace please')
+        const trace = stackTrace.parse(err)
+
+        for (let i = 0; i < trace.length; i++) {
+          const current = trace[i]
+
+          if (sourceFilesInfo.has(current.getFileName())) {
+            const { entity, entitySet } = sourceFilesInfo.get(current.getFileName())
+
+            if (entity != null && entitySet != null) {
+              resolvedPath = await reporter.folders.resolveEntityPath(entity, entitySet, req)
+              break
+            }
+          }
+        }
+      }
+
+      return resolvedPath
+    }
+
+    jsreportProxy.currentDirectoryPath = async () => {
+      const currentPath = await jsreportProxy.currentPath()
+
+      if (currentPath != null) {
+        const localPath = currentPath.substring(0, currentPath.lastIndexOf('/'))
+
+        if (localPath === '') {
+          return '/'
+        }
+
+        return localPath
+      }
+
+      return currentPath
+    }
 
     // NOTE: it is important that cleanup, restore methods are not called from a function attached to the
     // sandbox, because the arguments and return value of such function call will be sandboxed again, to solve this
     // we don't attach these methods to the sandbox, and instead share them through a "manager" object that should
     // be passed in options
     manager.restore = restore
-    manager.contextifyValue = contextifyValue
-    manager.decontextifyValue = decontextifyValue
-    manager.unproxyValue = unproxyValue
 
     const functionNames = getTopLevelFunctions(userCode)
     const functionsCode = `return {${functionNames.map(h => `"${h}": ${h}`).join(',')}}`

package/lib/worker/sandbox/safeSandbox.js

@@ -113,26 +113,12 @@ module.exports = (_sandbox, options = {}) => {
 
   const vm = new VM()
 
-  // NOTE: we wrap the Contextify.object, Decontextify.object methods because those are the
-  // methods that returns the proxies created by vm2 in the sandbox, we want to have a list of those
-  // to later use them
-  const wrapAndSaveProxyResult = (originalFn, thisArg) => {
-    return (value, ...args) => {
-      const result = originalFn.call(thisArg, value, ...args)
-
-      if (result != null && result.isVMProxy === true) {
-        proxiesInVM.set(result, value)
-      }
-
-      return result
-    }
-  }
-
-  vm._internal.Contextify.object = wrapAndSaveProxyResult(vm._internal.Contextify.object, vm._internal.Contextify)
-  vm._internal.Decontextify.object = wrapAndSaveProxyResult(vm._internal.Decontextify.object, vm._internal.Decontextify)
+  // delete the vm.sandbox.global because it introduces json stringify issues
+  // and we don't need such global in context
+  delete vm.sandbox.global
 
   for (const name in sandbox) {
-    vm._internal.Contextify.setGlobal(name, sandbox[name])
+    vm.setGlobal(name, sandbox[name])
   }
 
   // processing top level props because getter/setter descriptors
@@ -141,33 +127,25 @@ module.exports = (_sandbox, options = {}) => {
     const currentConfig = propsConfig[key]
 
     if (currentConfig.root && currentConfig.root.sandboxReadOnly) {
-      readOnlyProp(vm._context, key, [], customProxies, { onlyTopLevel: true })
+      readOnlyProp(vm.sandbox, key, [], customProxies, { onlyTopLevel: true })
     }
   })
 
   const sourceFilesInfo = new Map()
 
   return {
-    sandbox: vm._context,
+    sandbox: vm.sandbox,
     console: _console,
-    contextifyValue: (value) => {
-      return vm._internal.Contextify.value(value)
-    },
-    decontextifyValue: (value) => {
-      return vm._internal.Decontextify.value(value)
-    },
+    sourceFilesInfo,
     restore: () => {
-      return restoreProperties(vm._context, originalValues, proxiesInVM, customProxies)
-    },
-    unproxyValue: (value) => {
-      return getOriginalFromProxy(proxiesInVM, customProxies, value)
+      return restoreProperties(vm.sandbox, originalValues, proxiesInVM, customProxies)
     },
     safeRequire: (modulePath) => _require(modulePath, { context: _sandbox, allowAllModules: true }),
-    run: async (code, { filename, errorLineNumberOffset = 0, source, entity } = {}) => {
+    run: async (code, { filename, errorLineNumberOffset = 0, source, entity, entitySet } = {}) => {
       const script = new VMScript(code, filename)
 
       if (filename != null && source != null) {
-        sourceFilesInfo.set(filename, { filename, source, entity, errorLineNumberOffset })
+        sourceFilesInfo.set(filename, { filename, source, entity, entitySet, errorLineNumberOffset })
       }
 
       // NOTE: if we need to upgrade vm2 we will need to check the source of this function
@@ -176,6 +154,7 @@ module.exports = (_sandbox, options = {}) => {
       // to show nice error when the compile of a script fails
       script._compile = function (prefix, suffix) {
         return new originalVM.Script(prefix + this.getCompiledCode() + suffix, {
+          __proto__: null,
           filename: this.filename,
           displayErrors: true,
           lineOffset: this.lineOffset,

package/lib/worker/workerHandler.js

@@ -31,6 +31,7 @@ module.exports = (userInitData, { executeMain, convertUint8ArrayToBuffer }) => {
             ...JSON.parse(req.rawContent),
             context: req.context
           }
+          parsedReq.context.parsedInWorker = true
 
           return omit(parsedReq, 'data')
         } catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "3.2.0",
+  "version": "3.4.2",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -53,8 +53,8 @@
     "lodash.set": "4.3.2",
     "lru-cache": "4.1.1",
     "ms": "2.1.3",
-    "nanoid": "3.1.16",
-    "nconf": "0.10.0",
+    "nanoid": "3.2.0",
+    "nconf": "0.11.3",
     "node.extend.without.arrays": "1.1.6",
     "reap2": "1.0.1",
     "semver": "7.3.5",
@@ -63,8 +63,7 @@
     "triple-beam": "1.3.0",
     "unset-value": "1.0.0",
     "uuid": "8.3.2",
-    "v8-compile-cache": "*",
-    "vm2": "3.9.5",
+    "vm2": "3.9.7",
     "winston": "3.3.3",
     "winston-transport": "4.4.0"
   },

package/test/blobStorage/common.js

@@ -18,4 +18,8 @@ module.exports = (storage) => {
     const buf = await storage().read('foldera/folderb/myblob.txt')
     buf.toString().should.be.eql('hula')
   })
+
+  it('remove shouldnt fail for missing blob', async () => {
+    await storage().remove('foo')
+  })
 }