@jsreport/jsreport-core 3.4.2 → 3.5.0

package/README.md

@@ -282,6 +282,15 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 3.5.0
+
+- fix parsing issue of code with comment in the sandbox (helpers, scripts)
+- improve profiling when there is big data
+- make transactions support in store configurable
+- improve timeout for the whole request
+- fix applying req.options.timeout when enableRequestReportTimeout is true
+- optimization regarding profile persistence
+
 ### 3.4.2
 
 - update dep `vm2` to fix security vulnerability in sandbox

package/lib/main/blobStorage/blobStorage.js

@@ -46,6 +46,14 @@ module.exports = (reporter, options) => {
       }
     },
 
+    get supportsAppend () {
+      return provider.append instanceof Function
+    },
+
+    get _provider () {
+      return provider
+    },
+
     registerProvider (p) {
       provider = p
     }

package/lib/main/optionsSchema.js

@@ -126,7 +126,13 @@ module.exports.getRootSchemaOptions = () => ({
     store: {
       type: 'object',
       properties: {
-        provider: { type: 'string', enum: ['memory'] }
+        provider: { type: 'string', enum: ['memory'] },
+        transactions: {
+          type: 'object',
+          properties: {
+            enabled: { type: 'boolean', default: true }
+          }
+        }
       }
     },
     blobStorage: {
@@ -163,6 +169,10 @@ module.exports.getRootSchemaOptions = () => ({
       type: 'object',
       default: {},
       properties: {
+        defaultMode: {
+          type: 'string',
+          default: 'standard'
+        },
         maxProfilesHistory: {
           type: 'number',
           default: 1000
@@ -171,6 +181,11 @@ module.exports.getRootSchemaOptions = () => ({
           type: ['string', 'number'],
           '$jsreport-acceptsDuration': true,
           default: '1m'
+        },
+        maxResponseSize: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsSize': true,
+          default: '50mb'
         }
       }
     }

package/lib/main/profiler.js

@@ -1,6 +1,7 @@
 const EventEmitter = require('events')
 const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
+const fs = require('fs/promises')
 
 module.exports = (reporter) => {
   reporter.documentStore.registerEntityType('ProfileType', {
@@ -8,9 +9,9 @@ module.exports = (reporter) => {
     timestamp: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     finishedOn: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     state: { type: 'Edm.String' },
-    blobPersisted: { type: 'Edm.Boolean' },
-    blobName: { type: 'Edm.String' },
-    error: { type: 'Edm.String' }
+    error: { type: 'Edm.String' },
+    mode: { type: 'Edm.String', schema: { enum: ['full', 'standard', 'disabled'] } },
+    blobName: { type: 'Edm.String' }
   })
 
   reporter.documentStore.registerEntitySet('profiles', {
@@ -19,9 +20,41 @@ module.exports = (reporter) => {
   })
 
   const profilersMap = new Map()
-  const profilerAppendChain = new Map()
 
-  async function emitProfiles (events, req) {
+  const profilerOperationsChainsMap = new Map()
+  function runInProfilerChain (fn, req) {
+    if (req.context.profiling.mode === 'disabled') {
+      return
+    }
+
+    profilerOperationsChainsMap.set(req.context.rootId, profilerOperationsChainsMap.get(req.context.rootId).then(async () => {
+      if (req.context.profiling.chainFailed) {
+        return
+      }
+
+      try {
+        await fn()
+      } catch (e) {
+        reporter.logger.warn('Failed persist profile', e)
+        req.context.profiling.chainFailed = true
+      }
+    }))
+  }
+
+  function createProfileMessage (m, req) {
+    m.timestamp = new Date().getTime()
+    m.id = generateRequestId()
+    m.previousOperationId = m.previousOperationId || null
+    if (m.type !== 'log') {
+      m.operationId = m.operationId || generateRequestId()
+      req.context.profiling.lastOperationId = m.operationId
+      req.context.profiling.lastEventId = m.id
+    }
+
+    return m
+  }
+
+  function emitProfiles (events, req) {
     if (events.length === 0) {
       return
     }
@@ -44,14 +77,16 @@ module.exports = (reporter) => {
       req.context.profiling.lastOperation = lastOperation
     }
 
-    profilerAppendChain.set(req.context.rootId, profilerAppendChain.get(req.context.rootId).then(() => {
+    runInProfilerChain(() => {
+      if (req.context.profiling.logFilePath) {
+        return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
+      }
+
       return reporter.blobStorage.append(
         req.context.profiling.entity.blobName,
         Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req
-      ).catch(e => {
-        reporter.logger.error('Failed to append to profile blob', e)
-      })
-    }))
+      )
+    }, req)
   }
 
   reporter.registerMainAction('profile', async (events, req) => {
@@ -62,8 +97,7 @@ module.exports = (reporter) => {
     req.context = req.context || {}
     req.context.rootId = reporter.generateRequestId()
     req.context.profiling = {
-      mode: profileMode == null ? 'full' : profileMode,
-      isAttached: true
+      mode: profileMode == null ? 'full' : profileMode
     }
     const profiler = new EventEmitter()
     profilersMap.set(req.context.rootId, profiler)
@@ -71,88 +105,136 @@ module.exports = (reporter) => {
     return profiler
   }
 
-  reporter.beforeRenderListeners.add('profiler', async (req, res) => {
-    profilerAppendChain.set(req.context.rootId, Promise.resolve())
-
+  reporter.beforeRenderWorkerAllocatedListeners.add('profiler', async (req) => {
     req.context.profiling = req.context.profiling || {}
+
+    if (req.context.profiling.mode == null) {
+      const profilerSettings = await reporter.settings.findValue('profiler', req)
+      const defaultMode = reporter.options.profiler.defaultMode || 'standard'
+      req.context.profiling.mode = (profilerSettings != null && profilerSettings.mode != null) ? profilerSettings.mode : defaultMode
+    }
+
+    profilerOperationsChainsMap.set(req.context.rootId, Promise.resolve())
+
     req.context.profiling.lastOperation = null
 
-    let blobName = `profiles/${req.context.rootId}.log`
+    const blobName = `profiles/${req.context.rootId}.log`
 
-    const template = await reporter.templates.resolveTemplate(req)
+    const profile = {
+      _id: reporter.documentStore.generateId(),
+      timestamp: new Date(),
+      state: 'queued',
+      mode: req.context.profiling.mode,
+      blobName
+    }
 
-    if (template && template._id) {
-      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
-      blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
-      // store a copy to prevent side-effects
-      req.context.resolvedTemplate = extend(true, {}, template)
+    if (!reporter.blobStorage.supportsAppend) {
+      const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
+      req.context.profiling.logFilePath = pathToFile
     }
 
-    if (!req.context.profiling.isAttached) {
-      const setting = await reporter.documentStore.collection('settings').findOne({ key: 'fullProfilerRunning' }, req)
-      if (setting && JSON.parse(setting.value)) {
-        req.context.profiling.isAttached = true
-        req.context.profiling.mode = 'full'
-      }
+    runInProfilerChain(async () => {
+      req.context.skipValidationFor = profile
+      await reporter.documentStore.collection('profiles').insert(profile, req)
+    }, req)
+
+    req.context.profiling.entity = profile
+
+    const profileStartOperation = createProfileMessage({
+      type: 'operationStart',
+      subtype: 'profile',
+      data: profile,
+      doDiffs: false
+    }, req)
+
+    req.context.profiling.profileStartOperationId = profileStartOperation.operationId
+
+    emitProfiles([profileStartOperation], req)
+
+    emitProfiles([createProfileMessage({
+      type: 'log',
+      level: 'info',
+      message: `Render request ${req.context.reportCounter} queued for execution and waiting for availible worker`,
+      previousOperationId: profileStartOperation.operationId
+    }, req)], req)
+  })
+
+  reporter.beforeRenderListeners.add('profiler', async (req, res) => {
+    const update = {
+      state: 'running'
     }
 
-    if (req.context.profiling.mode == null) {
-      req.context.profiling.mode = 'standard'
+    const template = await reporter.templates.resolveTemplate(req)
+    if (template && template._id) {
+      req.context.resolvedTemplate = extend(true, {}, template)
+      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
+      const blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
+      update.templateShortid = template.shortid
+
+      const originalBlobName = req.context.profiling.entity.blobName
+      // we want to store the profile into blobName path reflecting the template path so we need to copy the blob to new path now
+      runInProfilerChain(async () => {
+        if (req.context.profiling.logFilePath == null) {
+          const content = await reporter.blobStorage.read(originalBlobName, req)
+          await reporter.blobStorage.write(blobName, content, req)
+          return reporter.blobStorage.remove(originalBlobName, req)
+        }
+      }, req)
+
+      update.blobName = blobName
     }
 
-    const profile = await reporter.documentStore.collection('profiles').insert({
-      templateShortid: template != null ? template.shortid : null,
-      timestamp: new Date(),
-      state: 'running',
-      blobName,
-      fullRequestProfiling: req.context.profiling.mode === 'full'
+    runInProfilerChain(() => {
+      req.context.skipValidationFor = update
+      return reporter.documentStore.collection('profiles').update({
+        _id: req.context.profiling.entity._id
+      }, {
+        $set: update
+      }, req)
     }, req)
 
-    req.context.profiling.entity = profile
+    Object.assign(req.context.profiling.entity, update)
   })
 
   reporter.afterRenderListeners.add('profiler', async (req, res) => {
+    emitProfiles([createProfileMessage({
+      type: 'operationEnd',
+      doDiffs: false,
+      previousEventId: req.context.profiling.lastEventId,
+      previousOperationId: req.context.profiling.lastOperationId,
+      operationId: req.context.profiling.profileStartOperationId
+    }, req)], req)
+
     res.meta.profileId = req.context.profiling?.entity?._id
-    profilersMap.delete(req.context.rootId)
-    const profilerBlobPersistPromise = profilerAppendChain.get(req.context.rootId)
-    profilerAppendChain.delete(req.context.rootId)
-
-    await reporter.documentStore.collection('profiles').update({
-      _id: req.context.profiling.entity._id
-    }, {
-      $set: {
+
+    runInProfilerChain(async () => {
+      if (req.context.profiling.logFilePath != null) {
+        const content = await fs.readFile(req.context.profiling.logFilePath)
+        await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
+        await fs.unlink(req.context.profiling.logFilePath)
+      }
+
+      const update = {
         state: 'success',
         finishedOn: new Date()
       }
-    }, req)
-    profilerBlobPersistPromise.finally(() => {
-      reporter.documentStore.collection('profiles').update({
+      req.context.skipValidationFor = update
+      await reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
-        $set: {
-          blobPersisted: true
-        }
-      }, req).catch((e) => reporter.logger.error('Failed to update profile blobPersisted', e))
-    })
+        $set: update
+      }, req)
+    }, req)
+
+    // we don't remove from profiler requests map, because the renderErrorListeners are invoked if the afterRenderListener fails
   })
 
   reporter.renderErrorListeners.add('profiler', async (req, res, e) => {
     try {
       res.meta.profileId = req.context.profiling?.entity?._id
-      const profilerBlobPersistPromise = profilerAppendChain.get(req.context.rootId)
 
       if (req.context.profiling?.entity != null) {
-        await reporter.documentStore.collection('profiles').update({
-          _id: req.context.profiling.entity._id
-        }, {
-          $set: {
-            state: 'error',
-            finishedOn: new Date(),
-            error: e.toString()
-          }
-        }, req)
-
-        await emitProfiles([{
+        emitProfiles([{
           type: 'error',
           timestamp: new Date().getTime(),
           ...e,
@@ -160,20 +242,29 @@ module.exports = (reporter) => {
           stack: e.stack,
           message: e.message
         }], req)
+        runInProfilerChain(async () => {
+          if (req.context.profiling.logFilePath != null) {
+            const content = await fs.readFile(req.context.profiling.logFilePath, 'utf8')
+            await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
+            await fs.unlink(req.context.profiling.logFilePath)
+          }
 
-        profilerBlobPersistPromise.finally(() => {
-          reporter.documentStore.collection('profiles').update({
+          const update = {
+            state: 'error',
+            finishedOn: new Date(),
+            error: e.toString()
+          }
+          req.context.skipValidationFor = update
+          await reporter.documentStore.collection('profiles').update({
             _id: req.context.profiling.entity._id
           }, {
-            $set: {
-              blobPersisted: true
-            }
-          }, req).catch((e) => reporter.logger.error('Failed to update profile blobPersisted', e))
-        })
+            $set: update
+          }, req)
+        }, req)
       }
     } finally {
       profilersMap.delete(req.context.rootId)
-      profilerAppendChain.delete(req.context.rootId)
+      profilerOperationsChainsMap.delete(req.context.rootId)
     }
   })
 
@@ -197,8 +288,8 @@ module.exports = (reporter) => {
       clearInterval(profilesCleanupInterval)
     }
 
-    for (const key of profilerAppendChain.keys()) {
-      const profileAppendPromise = profilerAppendChain.get(key)
+    for (const key of profilerOperationsChainsMap.keys()) {
+      const profileAppendPromise = profilerOperationsChainsMap.get(key)
       if (profileAppendPromise) {
         await profileAppendPromise
       }

package/lib/main/reporter.js

@@ -28,10 +28,10 @@ const Reporter = require('../shared/reporter')
 const Request = require('./request')
 const generateRequestId = require('../shared/generateRequestId')
 const Profiler = require('./profiler')
-const Monitoring = require('./monitoring')
 const migrateXlsxTemplatesToAssets = require('./migration/xlsxTemplatesToAssets')
 const migrateResourcesToAssets = require('./migration/resourcesToAssets')
 const semver = require('semver')
+let reportCounter = 0
 
 class MainReporter extends Reporter {
   constructor (options, defaults) {
@@ -69,6 +69,7 @@ class MainReporter extends Reporter {
 
     this.logger = createLogger()
     this.beforeMainActionListeners = this.createListenerCollection('beforeMainAction')
+    this.beforeRenderWorkerAllocatedListeners = this.createListenerCollection('beforeRenderWorkerAllocated')
   }
 
   discover () {
@@ -165,9 +166,11 @@ class MainReporter extends Reporter {
     this._initializing = true
 
     if (this.compilation) {
-      this.compilation.resource('sandbox.js', require.resolve('vm2/lib/sandbox.js'))
-      this.compilation.resource('contextify.js', require.resolve('vm2/lib/contextify.js'))
-      this.compilation.resource('fixasync.js', require.resolve('vm2/lib/fixasync.js'))
+      this.compilation.resource('vm2-events.js', require.resolve('vm2/lib/events.js'))
+      this.compilation.resource('vm2-resolver-compat.js', require.resolve('vm2/lib/resolver-compat.js'))
+      this.compilation.resource('vm2-resolver.js', require.resolve('vm2/lib/resolver.js'))
+      this.compilation.resource('vm2-setup-node-sandbox.js', require.resolve('vm2/lib/setup-node-sandbox.js'))
+      this.compilation.resource('vm2-setup-sandbox.js', require.resolve('vm2/lib/setup-sandbox.js'))
     }
 
     try {
@@ -180,7 +183,6 @@ class MainReporter extends Reporter {
       blobStorageActions(this)
       Templates(this)
       Profiler(this)
-      Monitoring(this)
 
       this.folders = Object.assign(this.folders, Folders(this))
 
@@ -267,8 +269,6 @@ class MainReporter extends Reporter {
         name: 'none'
       })
 
-      this.monitoring.init()
-
       this.logger.info('reporter initialized')
       this._initialized = true
       this._initExecution.resolve()
@@ -306,6 +306,34 @@ class MainReporter extends Reporter {
     await validateReservedName(this, c, doc)
   }
 
+  async _handleRenderError (req, res, err) {
+    if (err.code === 'WORKER_TIMEOUT') {
+      err.message = 'Report timeout'
+      if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
+        err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
+      }
+
+      if (req.context.http != null) {
+        const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
+        err.message += `. You can inspect and find more details here: ${profileUrl}`
+      }
+
+      err.weak = true
+    }
+
+    if (err.code === 'WORKER_ABORTED') {
+      err.message = 'Report cancelled'
+      err.weak = true
+    }
+
+    if (!err.logged) {
+      const logFn = err.weak ? this.logger.warn : this.logger.error
+      logFn(`Report render failed: ${err.message}${err.stack != null ? ' ' + err.stack : ''}`, req)
+    }
+    await this.renderErrorListeners.fire(req, res, err)
+    throw err
+  }
+
   /**
    * Main method for invoking rendering
    * render({ template: { content: 'foo', engine: 'none', recipe: 'html' }, data: { foo: 'hello' } })
@@ -324,23 +352,30 @@ class MainReporter extends Reporter {
     req.context = Object.assign({}, req.context)
     req.context.rootId = req.context.rootId || generateRequestId()
     req.context.id = req.context.rootId
+    req.context.reportCounter = ++reportCounter
+    req.context.startTimestamp = new Date().getTime()
 
-    const worker = options.worker || await this._workersManager.allocate(req, {
-      timeout: this.options.reportTimeout
-    })
-
-    let keepWorker
+    let worker
     let workerAborted
+    let dontCloseProcessing
+    const res = { meta: {} }
+    try {
+      await this.beforeRenderWorkerAllocatedListeners.fire(req)
 
-    if (options.abortEmitter) {
-      options.abortEmitter.once('abort', () => {
-        workerAborted = true
-        worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+      worker = await this._workersManager.allocate(req, {
+        timeout: this.getReportTimeout(req)
       })
-    }
 
-    const res = { meta: {} }
-    try {
+      if (options.abortEmitter) {
+        options.abortEmitter.once('abort', () => {
+          if (workerAborted) {
+            return
+          }
+          workerAborted = true
+          worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+        })
+      }
+
       if (workerAborted) {
         throw this.createError('Request aborted by client')
       }
@@ -351,7 +386,7 @@ class MainReporter extends Reporter {
           req,
           data: {}
         }, {
-          timeout: this.options.reportTimeout
+          timeout: this.getReportTimeout(req)
         })
         req = result
       }
@@ -369,39 +404,40 @@ class MainReporter extends Reporter {
         }
       }
 
-      let reportTimeout = this.options.reportTimeout
-
-      if (
-        this.options.enableRequestReportTimeout &&
-        req.options &&
-        req.options.timeout != null
-      ) {
-        reportTimeout = req.options.timeout
-      }
+      const reportTimeout = this.getReportTimeout(req)
 
       await this.beforeRenderListeners.fire(req, res, { worker })
 
-      // this is used so far just in the reports extension
-      // it wants to send to the client immediate response with link to the report status
-      // but the previous steps already allocated worker which has the parsed input request
-      // so we need to keep the worker active and let the subsequent real render call use it
-      // we cant move the main beforeRenderListener before the worker allocation, because at that point
-      // the request isn't parsed and we don't know the template and options
-      if (req.context.returnResponseAndKeepWorker) {
-        keepWorker = true
-        res.stream = Readable.from(res.content)
-
-        // just temporary workaround until we change how report render works
-        await this.documentStore.collection('profiles').update({
-          _id: req.context.profiling.entity._id
-        }, {
-          $set: {
-            state: 'success',
-            finishedOn: new Date(),
-            blobPersisted: true
+      if (workerAborted) {
+        throw this.createError('Request aborted by client')
+      }
+
+      if (req.context.clientNotification) {
+        process.nextTick(async () => {
+          try {
+            const responseResult = await this.executeWorkerAction('render', {}, {
+              timeout: reportTimeout + this.options.reportTimeoutMargin,
+              worker
+            }, req)
+
+            Object.assign(res, responseResult)
+            await this.afterRenderListeners.fire(req, res)
+          } catch (err) {
+            await this._handleRenderError(req, res, err).catch((e) => {})
+          } finally {
+            if (!workerAborted) {
+              await worker.release(req)
+            }
           }
-        }, req)
-        return res
+        })
+
+        dontCloseProcessing = true
+        const r = {
+          ...req.context.clientNotification,
+          stream: Readable.from(req.context.clientNotification.content)
+        }
+        delete req.context.clientNotification
+        return r
       }
 
       if (workerAborted) {
@@ -418,33 +454,10 @@ class MainReporter extends Reporter {
       res.stream = Readable.from(res.content)
       return res
     } catch (err) {
-      if (err.code === 'WORKER_TIMEOUT') {
-        err.message = 'Report timeout'
-        if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
-          err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
-        }
-
-        if (req.context.http != null) {
-          const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
-          err.message += `. You can inspect and find more details here: ${profileUrl}`
-        }
-
-        err.weak = true
-      }
-
-      if (err.code === 'WORKER_ABORTED') {
-        err.message = 'Report cancelled'
-        err.weak = true
-      }
-
-      if (!err.logged) {
-        const logFn = err.weak ? this.logger.warn : this.logger.error
-        logFn(`Report render failed: ${err.message}${err.stack != null ? ' ' + err.stack : ''}`, req)
-      }
-      await this.renderErrorListeners.fire(req, res, err)
+      await this._handleRenderError(req, res, err)
       throw err
     } finally {
-      if (!workerAborted && !keepWorker) {
+      if (worker && !workerAborted && !dontCloseProcessing) {
         await worker.release(req)
       }
     }

package/lib/main/schemaValidator.js

@@ -3,6 +3,7 @@ const set = require('lodash.set')
 const hasOwn = require('has-own-deep')
 const unsetValue = require('unset-value')
 const ms = require('ms')
+const bytes = require('bytes')
 const Ajv = require('ajv')
 
 const validatorCollection = new WeakMap()
@@ -126,6 +127,35 @@ class SchemaValidator {
       }
     })
 
+    validator.addKeyword('$jsreport-acceptsSize', {
+      modifying: true,
+      compile: (sch) => {
+        if (sch !== true) {
+          return () => true
+        }
+
+        return (data, dataPath, parentData, parentDataProperty) => {
+          if (typeof data !== 'string' && typeof data !== 'number') {
+            return false
+          }
+
+          if (typeof data === 'number') {
+            return true
+          }
+
+          const newData = bytes(data)
+
+          if (newData == null) {
+            return false
+          }
+
+          parentData[parentDataProperty] = newData
+
+          return true
+        }
+      }
+    })
+
     let rootValidate
 
     if (options.rootSchema != null) {

package/lib/main/settings.js

@@ -25,7 +25,7 @@ Settings.prototype.get = function (key) {
 }
 
 Settings.prototype.findValue = async function (key, req) {
-  const res = await this.documentStore.collection('settings').find({ key: key }, req)
+  const res = await this.documentStore.collection('settings').find({ key: key }, localReqWithoutAuthorization(req))
   if (res.length !== 1) {
     return null
   }
@@ -49,7 +49,6 @@ Settings.prototype.addOrSet = async function (key, avalue, req) {
   const value = typeof avalue !== 'string' ? JSON.stringify(avalue) : avalue
 
   const updateCount = await this.documentStore.collection('settings').update({ key }, { $set: { key: key, value: value } }, localReqWithoutAuthorization(req))
-
   if (updateCount === 0) {
     await this.documentStore.collection('settings').insert({ key: key, value: value }, localReqWithoutAuthorization(req))
     return 1

package/lib/main/store/collection.js

@@ -86,16 +86,18 @@ module.exports = (entitySet, provider, model, validator, encryption, transaction
       validateEntityName(data.name)
     }
 
-    const entityType = model.entitySets[entitySet] ? model.entitySets[entitySet].normalizedEntityTypeName : null
+    if (req == null || req.context.skipValidationFor !== data) {
+      const entityType = model.entitySets[entitySet] ? model.entitySets[entitySet].normalizedEntityTypeName : null
 
-    if (entityType != null && validator.getSchema(entityType) != null) {
-      const validationResult = validator.validate(entityType, data)
+      if (entityType != null && validator.getSchema(entityType) != null) {
+        const validationResult = validator.validate(entityType, data)
 
-      if (!validationResult.valid) {
-        throw createError(`Error when trying to insert into "${entitySet}" collection. input contain values that does not match the schema. ${validationResult.fullErrorMessage}`, {
-          weak: true,
-          statusCode: 400
-        })
+        if (!validationResult.valid) {
+          throw createError(`Error when trying to insert into "${entitySet}" collection. input contain values that does not match the schema. ${validationResult.fullErrorMessage}`, {
+            weak: true,
+            statusCode: 400
+          })
+        }
       }
     }
 

package/lib/main/store/documentStore.js

@@ -372,6 +372,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async beginTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       if (req.context.storeTransaction && transactions.has(req.context.storeTransaction)) {
         throw new Error('Can not call store.beginTransaction when an active transaction already exists, make sure you are not calling store.beginTransaction more than once')
       }
@@ -386,6 +390,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async commitTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       const tranId = req.context.storeTransaction
       const tran = transactions.get(tranId)
 
@@ -400,6 +408,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async rollbackTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       const tranId = req.context.storeTransaction
       const tran = transactions.get(tranId)
 
@@ -411,6 +423,13 @@ const DocumentStore = (options, validator, encryption) => {
 
       transactions.delete(tranId)
       delete req.context.storeTransaction
+    },
+
+    generateId () {
+      if (this.provider.generateId) {
+        return this.provider.generateId()
+      }
+      return uuidv4()
     }
   }
 

package/lib/main/store/setupValidateId.js

@@ -3,10 +3,16 @@ module.exports = (reporter) => {
   reporter.initializeListeners.add('core-validate-id', () => {
     for (const c of Object.keys(reporter.documentStore.collections)) {
       reporter.documentStore.collection(c).beforeInsertListeners.add('validate-id', (doc, req) => {
-        return validateIdForStoreChange(reporter, c, doc._id, undefined, req)
+        if (req == null || req.context.skipValidationFor !== doc) {
+          return validateIdForStoreChange(reporter, c, doc._id, undefined, req)
+        }
       })
 
       reporter.documentStore.collection(c).beforeUpdateListeners.add('validate-id', async (q, update, opts, req) => {
+        if (req != null && req.context.skipValidationFor === update) {
+          return
+        }
+
         if (update.$set && opts && opts.upsert === true) {
           await validateIdForStoreChange(reporter, c, update.$set._id, undefined, req)
         }

package/lib/main/store/setupValidateShortid.js

@@ -31,6 +31,10 @@ module.exports = (reporter) => {
 }
 
 async function validateShortid (reporter, collectionName, doc, originalIdValue, req) {
+  if (req != null && req.context.skipValidationFor === doc) {
+    return
+  }
+
   const shortid = doc.shortid
 
   if (!shortid) {

package/lib/shared/normalizeMetaFromLogs.js

@@ -13,7 +13,7 @@ module.exports = (level, msg, meta) => {
 
     // TODO adding cancel looks bad, its before script is adding req.cancel()
     // excluding non relevant properties for the log
-    const newMeta = Object.assign({}, omit(meta, ['template', 'options', 'data', 'context', 'timestamp', 'cancel']))
+    const newMeta = Object.assign({}, omit(meta, ['rawContent', 'template', 'options', 'data', 'context', 'timestamp', 'cancel']))
 
     if (newMeta.rootId == null && meta.context.rootId != null) {
       newMeta.rootId = meta.context.rootId

package/lib/shared/reporter.js

@@ -51,6 +51,22 @@ class Reporter extends EventEmitter {
     return generateRequestId()
   }
 
+  /**
+   * @public Ensures that we get the proper report timeout in case when custom timeout per request was enabled
+   */
+  getReportTimeout (req) {
+    const elapsedTime = req.context.startTimestamp ? (new Date().getTime() - req.context.startTimestamp) : 0
+    if (
+      this.options.enableRequestReportTimeout &&
+      req.options != null &&
+      req.options.timeout != null
+    ) {
+      return Math.max(0, req.options.timeout - elapsedTime)
+    }
+
+    return Math.max(0, this.options.reportTimeout - elapsedTime)
+  }
+
   /**
    * Ensures that the jsreport auto-cleanup temp directory (options.tempAutoCleanupDirectory) exists by doing a mkdir call
    *

package/lib/worker/render/executeEngine.js

@@ -14,6 +14,7 @@ module.exports = (reporter) => {
   reporter.templatingEngines = { cache }
 
   const executionFnParsedParamsMap = new Map()
+  const executionAsyncResultsMap = new Map()
 
   const templatingEnginesEvaluate = async (mainCall, { engine, content, helpers, data }, { entity, entitySet }, req) => {
     const engineImpl = reporter.extensionsManager.engines.find((e) => e.name === engine)
@@ -28,19 +29,23 @@ module.exports = (reporter) => {
       executionFnParsedParamsMap.set(req.context.id, new Map())
     }
 
+    const executionId = nanoid(7)
+
     try {
       const res = await executeEngine({
         engine: engineImpl,
         content,
         helpers,
         data
-      }, { handleErrors: false, entity, entitySet }, req)
+      }, { executionId, handleErrors: false, entity, entitySet }, req)
 
       return res.content
     } finally {
       if (mainCall) {
         executionFnParsedParamsMap.delete(req.context.id)
       }
+
+      executionAsyncResultsMap.delete(executionId)
     }
   }
 
@@ -54,6 +59,12 @@ module.exports = (reporter) => {
     proxy.templatingEngines = {
       evaluate: async (executionInfo, entityInfo) => {
         return templatingEnginesEvaluate(false, executionInfo, entityInfo, req)
+      },
+      waitForAsyncHelpers: async () => {
+        if (context.__executionId != null && executionAsyncResultsMap.has(context.__executionId)) {
+          const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
+          return Promise.all([...asyncResultMap.keys()].map((k) => asyncResultMap.get(k)))
+        }
       }
     }
   })
@@ -64,6 +75,8 @@ module.exports = (reporter) => {
     req.data.__rootDirectory = reporter.options.rootDirectory
     req.data.__parentModuleDirectory = reporter.options.parentModuleDirectory
 
+    const executionId = nanoid(7)
+
     try {
       return await executeEngine({
         engine,
@@ -71,16 +84,18 @@ module.exports = (reporter) => {
         helpers: req.template.helpers,
         data: req.data
       }, {
+        executionId,
         handleErrors: true,
         entity: req.template,
         entitySet: 'templates'
       }, req)
     } finally {
       executionFnParsedParamsMap.delete(req.context.id)
+      executionAsyncResultsMap.delete(executionId)
     }
   }
 
-  async function executeEngine ({ engine, content, helpers, data }, { handleErrors, entity, entitySet }, req) {
+  async function executeEngine ({ engine, content, helpers, data }, { executionId, handleErrors, entity, entitySet }, req) {
     let entityPath
 
     if (entity._id) {
@@ -104,9 +119,14 @@ module.exports = (reporter) => {
     const joinedHelpers = systemHelpersStr + '\n' + (helpers || '')
     const executionFnParsedParamsKey = `entity:${entity.shortid || 'anonymous'}:helpers:${joinedHelpers}`
 
-    const executionFn = async ({ require, console, topLevelFunctions }) => {
+    const executionFn = async ({ require, console, topLevelFunctions, context }) => {
       const asyncResultMap = new Map()
-      executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions })
+
+      context.__executionId = executionId
+
+      executionAsyncResultsMap.set(executionId, asyncResultMap)
+      executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions, context })
+
       const key = `template:${content}:${engine.name}`
 
       if (!cache.has(key)) {
@@ -143,16 +163,19 @@ module.exports = (reporter) => {
       }
 
       return {
-        content: contentResult
+        // handlebars escapes single brackets before execution to prevent errors on {#asset}
+        // we need to unescape them later here, because at the moment the engine.execute finishes
+        // the async helpers aren't executed yet
+        content: engine.unescape ? engine.unescape(contentResult) : contentResult
       }
     }
 
     // executionFnParsedParamsMap is there to cache parsed components helpers to speed up longer loops
     // we store there for the particular request and component a promise and only the first component gets compiled
     if (executionFnParsedParamsMap.get(req.context.id).has(executionFnParsedParamsKey)) {
-      const { require, console, topLevelFunctions } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
+      const { require, console, topLevelFunctions, context } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
 
-      return executionFn({ require, console, topLevelFunctions })
+      return executionFn({ require, console, topLevelFunctions, context })
     } else {
       const awaiter = {}
       awaiter.promise = new Promise((resolve) => {

package/lib/worker/render/profiler.js

@@ -57,9 +57,9 @@ class Profiler {
 
     if (m.type !== 'log') {
       req.context.profiling.lastEventId = m.id
+      m.operationId = m.operationId || generateRequestId()
     }
 
-    m.operationId = m.operationId || generateRequestId()
     if (m.previousOperationId == null && req.context.profiling.lastOperationId) {
       m.previousOperationId = req.context.profiling.lastOperationId
     }
@@ -72,15 +72,21 @@ class Profiler {
       let content = res.content
 
       if (content != null) {
-        if (isbinaryfile(content)) {
+        if (content.length > this.reporter.options.profiler.maxResponseSize) {
           content = {
-            content: res.content.toString('base64'),
-            encoding: 'base64'
+            tooLarge: true
           }
         } else {
-          content = {
-            content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
-            encoding: 'diff'
+          if (isbinaryfile(content)) {
+            content = {
+              content: res.content.toString('base64'),
+              encoding: 'base64'
+            }
+          } else {
+            content = {
+              content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
+              encoding: 'diff'
+            }
           }
         }
       }
@@ -93,7 +99,7 @@ class Profiler {
 
       m.req = { diff: createPatch('req', req.context.profiling.reqLastVal || '', stringifiedReq, 0) }
 
-      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content)) ? null : res.content.toString()
+      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content) || content.tooLarge) ? null : res.content.toString()
       req.context.profiling.resMetaLastVal = stringifiedResMeta
       req.context.profiling.reqLastVal = stringifiedReq
     }
@@ -129,10 +135,6 @@ class Profiler {
       previousOperationId: parentReq ? parentReq.context.profiling.lastOperationId : null
     }
 
-    if (!req.context.isChildRequest) {
-      profilerEvent.profileId = req.context.profiling.entity._id
-    }
-
     return this.emit(profilerEvent, req, res)
   }
 

package/lib/worker/render/render.js

@@ -10,7 +10,6 @@ const Request = require('../../shared/request')
 const generateRequestId = require('../../shared/generateRequestId')
 const resolveReferences = require('./resolveReferences.js')
 const moduleHelper = require('./moduleHelper')
-let reportCounter = 0
 
 module.exports = (reporter) => {
   moduleHelper(reporter)
@@ -127,9 +126,6 @@ module.exports = (reporter) => {
         response.meta.reportName = 'report'
       }
 
-      request.context.reportCounter = ++reportCounter
-      request.context.startTimestamp = new Date().getTime()
-
       if (parentReq == null) {
         reporter.requestModulesCache.set(request.context.rootId, Object.create(null))
       }

package/lib/worker/sandbox/runInSandbox.js

@@ -116,7 +116,7 @@ module.exports = (reporter) => {
 
     const functionNames = getTopLevelFunctions(userCode)
     const functionsCode = `return {${functionNames.map(h => `"${h}": ${h}`).join(',')}}`
-    const executionCode = `;(async () => { ${userCode}; ${functionsCode} })()
+    const executionCode = `;(async () => { ${userCode} \n\n;${functionsCode} })()
         .then((topLevelFunctions) => {
           const mergedTopLevelFunctions = { ...topLevelFunctions, ...__topLevelFunctions }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "3.4.2",
+  "version": "3.5.0",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -32,11 +32,11 @@
     "@babel/code-frame": "7.12.13",
     "@babel/parser": "7.14.4",
     "@babel/traverse": "7.12.9",
-    "@jsreport/advanced-workers": "1.2.0",
+    "@jsreport/advanced-workers": "1.2.1",
     "@jsreport/mingo": "2.4.1",
     "ajv": "6.12.6",
     "app-root-path": "2.0.1",
-    "async-replace": "1.0.1",
+    "bytes": "3.1.2",
     "camelcase": "5.0.0",
     "debug": "4.3.2",
     "decamelize": "2.0.0",
@@ -46,7 +46,7 @@
     "enhanced-resolve": "5.8.3",
     "has-own-deep": "1.1.0",
     "isbinaryfile": "4.0.0",
-    "listener-collection": "1.2.0",
+    "listener-collection": "2.0.0",
     "lodash.get": "4.4.2",
     "lodash.groupby": "4.6.0",
     "lodash.omit": "4.5.0",
@@ -54,7 +54,7 @@
     "lru-cache": "4.1.1",
     "ms": "2.1.3",
     "nanoid": "3.2.0",
-    "nconf": "0.11.3",
+    "nconf": "0.12.0",
     "node.extend.without.arrays": "1.1.6",
     "reap2": "1.0.1",
     "semver": "7.3.5",
@@ -68,7 +68,7 @@
     "winston-transport": "4.4.0"
   },
   "devDependencies": {
-    "mocha": "8.2.1",
+    "mocha": "9.2.2",
     "should": "13.2.3",
     "standard": "16.0.4",
     "std-mocks": "1.0.1",

package/lib/main/monitoring.js

@@ -1,92 +0,0 @@
-const os = require('os')
-
-function cpu () {
-  // Create function to get CPU information
-  function cpuAverage () {
-  // Initialise sum of idle and time of cores and fetch CPU info
-    let totalIdle = 0; let totalTick = 0
-    const cpus = os.cpus()
-
-    // Loop through CPU cores
-    for (let i = 0, len = cpus.length; i < len; i++) {
-    // Select CPU core
-      const cpu = cpus[i]
-
-      // Total up the time in the cores tick
-      for (const type in cpu.times) {
-        totalTick += cpu.times[type]
-      }
-
-      // Total up the idle time of the core
-      totalIdle += cpu.times.idle
-    }
-
-    // Return the average Idle and Tick times
-    return { idle: totalIdle / cpus.length, total: totalTick / cpus.length }
-  }
-
-  // Grab first CPU Measure
-  const startMeasure = cpuAverage()
-
-  return new Promise((resolve) => {
-    // Set delay for second Measure
-    setTimeout(function () {
-    // Grab second Measure
-      const endMeasure = cpuAverage()
-
-      // Calculate the difference in idle and total time between the measures
-      const idleDifference = endMeasure.idle - startMeasure.idle
-      const totalDifference = endMeasure.total - startMeasure.total
-
-      // Calculate the average percentage CPU usage
-      const percentageCPU = 100 - ~~(100 * idleDifference / totalDifference)
-
-      // Output result to console
-      resolve(percentageCPU)
-    }, 1000)
-  })
-}
-
-class Monitoring {
-  constructor (reporter) {
-    this.reporter = reporter
-  }
-
-  async execute () {
-    const monitoring = {
-      cpu: await cpu(),
-      freemem: Math.round(os.freemem() / 1024 / 1024),
-      timestamp: new Date(),
-      hostname: os.hostname()
-    }
-    return this.reporter.documentStore.collection('monitoring').insert(monitoring)
-  }
-
-  init () {
-    this._interval = setInterval(() => {
-      this.execute().catch((e) => this.reporter.logger.warn('unable to persist monitoring info, but no need to worry, we will retry, details:' + e.stack))
-    }, 60000)
-    this._interval.unref()
-  }
-
-  close () {
-    clearInterval(this._interval)
-  }
-}
-
-module.exports = (reporter) => {
-  reporter.documentStore.registerEntityType('MonitoringType', {
-    cpu: { type: 'Edm.Int32' },
-    freemem: { type: 'Edm.Int32' },
-    timestamp: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
-    hostname: { type: 'Edm.String' }
-  })
-
-  reporter.documentStore.registerEntitySet('monitoring', {
-    entityType: 'jsreport.MonitoringType',
-    exportable: false,
-    shared: true
-  })
-
-  reporter.monitoring = new Monitoring(reporter)
-}