@jsreport/jsreport-core 3.4.2 → 3.6.1

package/README.md

@@ -282,6 +282,24 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 3.6.0
+
+- improve handling of worker exit
+- add a way to disable the safe execution and prevent the performance penalty
+- cache system helpers function
+- `profiler.maxDiffSize` applies to both req and res
+- improved the support for running jsreport as serverless function
+- added new apis in `jsreport-proxy` for better working with async helpers
+
+### 3.5.0
+
+- fix parsing issue of code with comment in the sandbox (helpers, scripts)
+- improve profiling when there is big data
+- make transactions support in store configurable
+- improve timeout for the whole request
+- fix applying req.options.timeout when enableRequestReportTimeout is true
+- optimization regarding profile persistence
+
 ### 3.4.2
 
 - update dep `vm2` to fix security vulnerability in sandbox

package/lib/main/blobStorage/blobStorage.js

@@ -46,6 +46,14 @@ module.exports = (reporter, options) => {
       }
     },
 
+    get supportsAppend () {
+      return provider.append instanceof Function
+    },
+
+    get _provider () {
+      return provider
+    },
+
     registerProvider (p) {
       provider = p
     }

package/lib/main/logger.js

@@ -166,6 +166,9 @@ function configureLogger (logger, _transports) {
   }
 
   for (const { TransportClass, options } of transportsToAdd) {
+    if (options.silent) {
+      continue
+    }
     const transportInstance = new TransportClass(options)
 
     const existingTransport = logger.transports.find((t) => t.name === transportInstance.name)

package/lib/main/optionsLoad.js

@@ -39,6 +39,7 @@ async function optionsLoad ({
     loadConfigResult = await loadConfig(defaults, options, false)
   }
 
+  const explicitOptions = loadConfigResult[0]
   const appliedConfigFile = loadConfigResult[1]
 
   options.loadConfig = shouldLoadExternalConfig
@@ -79,9 +80,16 @@ async function optionsLoad ({
   options.store = options.store || { provider: 'memory' }
 
   options.sandbox = options.sandbox || {}
-  if (options.allowLocalFilesAccess === true) {
+
+  // NOTE: handling back-compatible introduction of "trustUserCode" option, "allowLocalFilesAccess" is deprecated
+  if (explicitOptions.allowLocalFilesAccess === true && explicitOptions.trustUserCode == null) {
+    options.trustUserCode = true
+  }
+
+  if (options.trustUserCode === true) {
     options.sandbox.allowedModules = '*'
   }
+
   options.sandbox.nativeModules = options.sandbox.nativeModules || []
   options.sandbox.modules = options.sandbox.modules || []
   options.sandbox.allowedModules = options.sandbox.allowedModules || []
@@ -98,7 +106,7 @@ async function optionsLoad ({
     fs.mkdirSync(options.tempCoreDirectory, { recursive: true })
   }
 
-  return appliedConfigFile
+  return [explicitOptions, appliedConfigFile]
 }
 
 /**

package/lib/main/optionsSchema.js

@@ -59,6 +59,7 @@ module.exports.getRootSchemaOptions = () => ({
       default: '2s'
     },
     enableRequestReportTimeout: { type: 'boolean', default: false, description: 'option that enables passing a custom report timeout per request using req.options.timeout. this enables that the caller of the report generation control the report timeout so enable it only when you trust the caller' },
+    trustUserCode: { type: 'boolean', default: false, description: 'option that control whether code sandboxing is enabled or not, code sandboxing has an impact on performance when rendering large reports. when true code sandboxing will be disabled meaning that users can potentially penetrate the local system if you allow code from external users to be part of your reports' },
     allowLocalFilesAccess: { type: 'boolean', default: false },
     encryption: {
       type: 'object',
@@ -77,6 +78,7 @@ module.exports.getRootSchemaOptions = () => ({
     },
     sandbox: {
       type: 'object',
+      default: {},
       properties: {
         allowedModules: {
           anyOf: [{
@@ -89,9 +91,10 @@ module.exports.getRootSchemaOptions = () => ({
         },
         cache: {
           type: 'object',
+          default: {},
           properties: {
-            max: { type: 'number' },
-            enabled: { type: 'boolean' }
+            max: { type: 'number', default: 100 },
+            enabled: { type: 'boolean', default: true }
           }
         }
       }
@@ -126,7 +129,13 @@ module.exports.getRootSchemaOptions = () => ({
     store: {
       type: 'object',
       properties: {
-        provider: { type: 'string', enum: ['memory'] }
+        provider: { type: 'string', enum: ['memory'] },
+        transactions: {
+          type: 'object',
+          properties: {
+            enabled: { type: 'boolean', default: true }
+          }
+        }
       }
     },
     blobStorage: {
@@ -163,6 +172,10 @@ module.exports.getRootSchemaOptions = () => ({
       type: 'object',
       default: {},
       properties: {
+        defaultMode: {
+          type: 'string',
+          default: 'standard'
+        },
         maxProfilesHistory: {
           type: 'number',
           default: 1000
@@ -171,6 +184,11 @@ module.exports.getRootSchemaOptions = () => ({
           type: ['string', 'number'],
           '$jsreport-acceptsDuration': true,
           default: '1m'
+        },
+        maxDiffSize: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsSize': true,
+          default: '50mb'
         }
       }
     }

package/lib/main/profiler.js

@@ -1,6 +1,7 @@
 const EventEmitter = require('events')
 const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
+const fs = require('fs/promises')
 
 module.exports = (reporter) => {
   reporter.documentStore.registerEntityType('ProfileType', {
@@ -8,9 +9,9 @@ module.exports = (reporter) => {
     timestamp: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     finishedOn: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     state: { type: 'Edm.String' },
-    blobPersisted: { type: 'Edm.Boolean' },
-    blobName: { type: 'Edm.String' },
-    error: { type: 'Edm.String' }
+    error: { type: 'Edm.String' },
+    mode: { type: 'Edm.String', schema: { enum: ['full', 'standard', 'disabled'] } },
+    blobName: { type: 'Edm.String' }
   })
 
   reporter.documentStore.registerEntitySet('profiles', {
@@ -19,9 +20,41 @@ module.exports = (reporter) => {
   })
 
   const profilersMap = new Map()
-  const profilerAppendChain = new Map()
 
-  async function emitProfiles (events, req) {
+  const profilerOperationsChainsMap = new Map()
+  function runInProfilerChain (fn, req) {
+    if (req.context.profiling.mode === 'disabled') {
+      return
+    }
+
+    profilerOperationsChainsMap.set(req.context.rootId, profilerOperationsChainsMap.get(req.context.rootId).then(async () => {
+      if (req.context.profiling.chainFailed) {
+        return
+      }
+
+      try {
+        await fn()
+      } catch (e) {
+        reporter.logger.warn('Failed persist profile', e)
+        req.context.profiling.chainFailed = true
+      }
+    }))
+  }
+
+  function createProfileMessage (m, req) {
+    m.timestamp = new Date().getTime()
+    m.id = generateRequestId()
+    m.previousOperationId = m.previousOperationId || null
+    if (m.type !== 'log') {
+      m.operationId = m.operationId || generateRequestId()
+      req.context.profiling.lastOperationId = m.operationId
+      req.context.profiling.lastEventId = m.id
+    }
+
+    return m
+  }
+
+  function emitProfiles (events, req) {
     if (events.length === 0) {
       return
     }
@@ -44,14 +77,16 @@ module.exports = (reporter) => {
       req.context.profiling.lastOperation = lastOperation
     }
 
-    profilerAppendChain.set(req.context.rootId, profilerAppendChain.get(req.context.rootId).then(() => {
+    runInProfilerChain(() => {
+      if (req.context.profiling.logFilePath) {
+        return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
+      }
+
       return reporter.blobStorage.append(
         req.context.profiling.entity.blobName,
         Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req
-      ).catch(e => {
-        reporter.logger.error('Failed to append to profile blob', e)
-      })
-    }))
+      )
+    }, req)
   }
 
   reporter.registerMainAction('profile', async (events, req) => {
@@ -62,8 +97,7 @@ module.exports = (reporter) => {
     req.context = req.context || {}
     req.context.rootId = reporter.generateRequestId()
     req.context.profiling = {
-      mode: profileMode == null ? 'full' : profileMode,
-      isAttached: true
+      mode: profileMode == null ? 'full' : profileMode
     }
     const profiler = new EventEmitter()
     profilersMap.set(req.context.rootId, profiler)
@@ -71,88 +105,136 @@ module.exports = (reporter) => {
     return profiler
   }
 
-  reporter.beforeRenderListeners.add('profiler', async (req, res) => {
-    profilerAppendChain.set(req.context.rootId, Promise.resolve())
-
+  reporter.beforeRenderWorkerAllocatedListeners.add('profiler', async (req) => {
     req.context.profiling = req.context.profiling || {}
+
+    if (req.context.profiling.mode == null) {
+      const profilerSettings = await reporter.settings.findValue('profiler', req)
+      const defaultMode = reporter.options.profiler.defaultMode || 'standard'
+      req.context.profiling.mode = (profilerSettings != null && profilerSettings.mode != null) ? profilerSettings.mode : defaultMode
+    }
+
+    profilerOperationsChainsMap.set(req.context.rootId, Promise.resolve())
+
     req.context.profiling.lastOperation = null
 
-    let blobName = `profiles/${req.context.rootId}.log`
+    const blobName = `profiles/${req.context.rootId}.log`
 
-    const template = await reporter.templates.resolveTemplate(req)
+    const profile = {
+      _id: reporter.documentStore.generateId(),
+      timestamp: new Date(),
+      state: 'queued',
+      mode: req.context.profiling.mode,
+      blobName
+    }
 
-    if (template && template._id) {
-      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
-      blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
-      // store a copy to prevent side-effects
-      req.context.resolvedTemplate = extend(true, {}, template)
+    if (!reporter.blobStorage.supportsAppend) {
+      const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
+      req.context.profiling.logFilePath = pathToFile
     }
 
-    if (!req.context.profiling.isAttached) {
-      const setting = await reporter.documentStore.collection('settings').findOne({ key: 'fullProfilerRunning' }, req)
-      if (setting && JSON.parse(setting.value)) {
-        req.context.profiling.isAttached = true
-        req.context.profiling.mode = 'full'
-      }
+    runInProfilerChain(async () => {
+      req.context.skipValidationFor = profile
+      await reporter.documentStore.collection('profiles').insert(profile, req)
+    }, req)
+
+    req.context.profiling.entity = profile
+
+    const profileStartOperation = createProfileMessage({
+      type: 'operationStart',
+      subtype: 'profile',
+      data: profile,
+      doDiffs: false
+    }, req)
+
+    req.context.profiling.profileStartOperationId = profileStartOperation.operationId
+
+    emitProfiles([profileStartOperation], req)
+
+    emitProfiles([createProfileMessage({
+      type: 'log',
+      level: 'info',
+      message: `Render request ${req.context.reportCounter} queued for execution and waiting for availible worker`,
+      previousOperationId: profileStartOperation.operationId
+    }, req)], req)
+  })
+
+  reporter.beforeRenderListeners.add('profiler', async (req, res) => {
+    const update = {
+      state: 'running'
     }
 
-    if (req.context.profiling.mode == null) {
-      req.context.profiling.mode = 'standard'
+    const template = await reporter.templates.resolveTemplate(req)
+    if (template && template._id) {
+      req.context.resolvedTemplate = extend(true, {}, template)
+      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
+      const blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
+      update.templateShortid = template.shortid
+
+      const originalBlobName = req.context.profiling.entity.blobName
+      // we want to store the profile into blobName path reflecting the template path so we need to copy the blob to new path now
+      runInProfilerChain(async () => {
+        if (req.context.profiling.logFilePath == null) {
+          const content = await reporter.blobStorage.read(originalBlobName, req)
+          await reporter.blobStorage.write(blobName, content, req)
+          return reporter.blobStorage.remove(originalBlobName, req)
+        }
+      }, req)
+
+      update.blobName = blobName
     }
 
-    const profile = await reporter.documentStore.collection('profiles').insert({
-      templateShortid: template != null ? template.shortid : null,
-      timestamp: new Date(),
-      state: 'running',
-      blobName,
-      fullRequestProfiling: req.context.profiling.mode === 'full'
+    runInProfilerChain(() => {
+      req.context.skipValidationFor = update
+      return reporter.documentStore.collection('profiles').update({
+        _id: req.context.profiling.entity._id
+      }, {
+        $set: update
+      }, req)
     }, req)
 
-    req.context.profiling.entity = profile
+    Object.assign(req.context.profiling.entity, update)
   })
 
   reporter.afterRenderListeners.add('profiler', async (req, res) => {
+    emitProfiles([createProfileMessage({
+      type: 'operationEnd',
+      doDiffs: false,
+      previousEventId: req.context.profiling.lastEventId,
+      previousOperationId: req.context.profiling.lastOperationId,
+      operationId: req.context.profiling.profileStartOperationId
+    }, req)], req)
+
     res.meta.profileId = req.context.profiling?.entity?._id
-    profilersMap.delete(req.context.rootId)
-    const profilerBlobPersistPromise = profilerAppendChain.get(req.context.rootId)
-    profilerAppendChain.delete(req.context.rootId)
-
-    await reporter.documentStore.collection('profiles').update({
-      _id: req.context.profiling.entity._id
-    }, {
-      $set: {
+
+    runInProfilerChain(async () => {
+      if (req.context.profiling.logFilePath != null) {
+        const content = await fs.readFile(req.context.profiling.logFilePath)
+        await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
+        await fs.unlink(req.context.profiling.logFilePath)
+      }
+
+      const update = {
         state: 'success',
         finishedOn: new Date()
       }
-    }, req)
-    profilerBlobPersistPromise.finally(() => {
-      reporter.documentStore.collection('profiles').update({
+      req.context.skipValidationFor = update
+      await reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
-        $set: {
-          blobPersisted: true
-        }
-      }, req).catch((e) => reporter.logger.error('Failed to update profile blobPersisted', e))
-    })
+        $set: update
+      }, req)
+    }, req)
+
+    // we don't remove from profiler requests map, because the renderErrorListeners are invoked if the afterRenderListener fails
   })
 
   reporter.renderErrorListeners.add('profiler', async (req, res, e) => {
     try {
       res.meta.profileId = req.context.profiling?.entity?._id
-      const profilerBlobPersistPromise = profilerAppendChain.get(req.context.rootId)
 
       if (req.context.profiling?.entity != null) {
-        await reporter.documentStore.collection('profiles').update({
-          _id: req.context.profiling.entity._id
-        }, {
-          $set: {
-            state: 'error',
-            finishedOn: new Date(),
-            error: e.toString()
-          }
-        }, req)
-
-        await emitProfiles([{
+        emitProfiles([{
           type: 'error',
           timestamp: new Date().getTime(),
           ...e,
@@ -160,20 +242,29 @@ module.exports = (reporter) => {
           stack: e.stack,
           message: e.message
         }], req)
+        runInProfilerChain(async () => {
+          if (req.context.profiling.logFilePath != null) {
+            const content = await fs.readFile(req.context.profiling.logFilePath, 'utf8')
+            await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
+            await fs.unlink(req.context.profiling.logFilePath)
+          }
 
-        profilerBlobPersistPromise.finally(() => {
-          reporter.documentStore.collection('profiles').update({
+          const update = {
+            state: 'error',
+            finishedOn: new Date(),
+            error: e.toString()
+          }
+          req.context.skipValidationFor = update
+          await reporter.documentStore.collection('profiles').update({
             _id: req.context.profiling.entity._id
           }, {
-            $set: {
-              blobPersisted: true
-            }
-          }, req).catch((e) => reporter.logger.error('Failed to update profile blobPersisted', e))
-        })
+            $set: update
+          }, req)
+        }, req)
       }
     } finally {
       profilersMap.delete(req.context.rootId)
-      profilerAppendChain.delete(req.context.rootId)
+      profilerOperationsChainsMap.delete(req.context.rootId)
     }
   })
 
@@ -197,8 +288,8 @@ module.exports = (reporter) => {
       clearInterval(profilesCleanupInterval)
     }
 
-    for (const key of profilerAppendChain.keys()) {
-      const profileAppendPromise = profilerAppendChain.get(key)
+    for (const key of profilerOperationsChainsMap.keys()) {
+      const profileAppendPromise = profilerOperationsChainsMap.get(key)
       if (profileAppendPromise) {
         await profileAppendPromise
       }