@jsreport/jsreport-core 3.4.0 → 3.5.0

package/lib/main/reporter.js

@@ -25,13 +25,13 @@ const setupValidateShortid = require('./store/setupValidateShortid')
 const documentStoreActions = require('./store/mainActions')
 const blobStorageActions = require('./blobStorage/mainActions')
 const Reporter = require('../shared/reporter')
-const Request = require('../shared/request')
+const Request = require('./request')
 const generateRequestId = require('../shared/generateRequestId')
 const Profiler = require('./profiler')
-const Monitoring = require('./monitoring')
 const migrateXlsxTemplatesToAssets = require('./migration/xlsxTemplatesToAssets')
 const migrateResourcesToAssets = require('./migration/resourcesToAssets')
 const semver = require('semver')
+let reportCounter = 0
 
 class MainReporter extends Reporter {
   constructor (options, defaults) {
@@ -69,6 +69,7 @@ class MainReporter extends Reporter {
 
     this.logger = createLogger()
     this.beforeMainActionListeners = this.createListenerCollection('beforeMainAction')
+    this.beforeRenderWorkerAllocatedListeners = this.createListenerCollection('beforeRenderWorkerAllocated')
   }
 
   discover () {
@@ -165,9 +166,11 @@ class MainReporter extends Reporter {
     this._initializing = true
 
     if (this.compilation) {
-      this.compilation.resource('sandbox.js', require.resolve('vm2/lib/sandbox.js'))
-      this.compilation.resource('contextify.js', require.resolve('vm2/lib/contextify.js'))
-      this.compilation.resource('fixasync.js', require.resolve('vm2/lib/fixasync.js'))
+      this.compilation.resource('vm2-events.js', require.resolve('vm2/lib/events.js'))
+      this.compilation.resource('vm2-resolver-compat.js', require.resolve('vm2/lib/resolver-compat.js'))
+      this.compilation.resource('vm2-resolver.js', require.resolve('vm2/lib/resolver.js'))
+      this.compilation.resource('vm2-setup-node-sandbox.js', require.resolve('vm2/lib/setup-node-sandbox.js'))
+      this.compilation.resource('vm2-setup-sandbox.js', require.resolve('vm2/lib/setup-sandbox.js'))
     }
 
     try {
@@ -180,7 +183,6 @@ class MainReporter extends Reporter {
       blobStorageActions(this)
       Templates(this)
       Profiler(this)
-      Monitoring(this)
 
       this.folders = Object.assign(this.folders, Folders(this))
 
@@ -267,8 +269,6 @@ class MainReporter extends Reporter {
         name: 'none'
       })
 
-      this.monitoring.init()
-
       this.logger.info('reporter initialized')
       this._initialized = true
       this._initExecution.resolve()
@@ -306,6 +306,34 @@ class MainReporter extends Reporter {
     await validateReservedName(this, c, doc)
   }
 
+  async _handleRenderError (req, res, err) {
+    if (err.code === 'WORKER_TIMEOUT') {
+      err.message = 'Report timeout'
+      if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
+        err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
+      }
+
+      if (req.context.http != null) {
+        const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
+        err.message += `. You can inspect and find more details here: ${profileUrl}`
+      }
+
+      err.weak = true
+    }
+
+    if (err.code === 'WORKER_ABORTED') {
+      err.message = 'Report cancelled'
+      err.weak = true
+    }
+
+    if (!err.logged) {
+      const logFn = err.weak ? this.logger.warn : this.logger.error
+      logFn(`Report render failed: ${err.message}${err.stack != null ? ' ' + err.stack : ''}`, req)
+    }
+    await this.renderErrorListeners.fire(req, res, err)
+    throw err
+  }
+
   /**
    * Main method for invoking rendering
    * render({ template: { content: 'foo', engine: 'none', recipe: 'html' }, data: { foo: 'hello' } })
@@ -315,63 +343,55 @@ class MainReporter extends Reporter {
    *
    * @public
    */
-  async render (req, parentReq) {
+  async render (req, options = {}) {
     if (!this._initialized) {
       throw new Error('Not initialized, you need to call jsreport.init().then before rendering')
     }
 
-    let options = {}
-    if (parentReq && !parentReq.__isJsreportRequest__) {
-      options = parentReq
-      parentReq = null
-    }
-
     req = Object.assign({}, req)
     req.context = Object.assign({}, req.context)
     req.context.rootId = req.context.rootId || generateRequestId()
     req.context.id = req.context.rootId
+    req.context.reportCounter = ++reportCounter
+    req.context.startTimestamp = new Date().getTime()
 
-    const worker = options.worker || await this._workersManager.allocate(req, {
-      timeout: this.options.reportTimeout
-    })
-
-    let keepWorker
+    let worker
     let workerAborted
+    let dontCloseProcessing
+    const res = { meta: {} }
+    try {
+      await this.beforeRenderWorkerAllocatedListeners.fire(req)
 
-    if (options.abortEmitter) {
-      options.abortEmitter.once('abort', () => {
-        workerAborted = true
-        worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+      worker = await this._workersManager.allocate(req, {
+        timeout: this.getReportTimeout(req)
       })
-    }
 
-    const res = { meta: {} }
-    try {
+      if (options.abortEmitter) {
+        options.abortEmitter.once('abort', () => {
+          if (workerAborted) {
+            return
+          }
+          workerAborted = true
+          worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+        })
+      }
+
       if (workerAborted) {
         throw this.createError('Request aborted by client')
       }
 
-      let isDataStoredInWorker = false
-
       if (req.rawContent) {
-        isDataStoredInWorker = true
         const result = await worker.execute({
           actionName: 'parse',
           req,
           data: {}
         }, {
-          timeout: this.options.reportTimeout
+          timeout: this.getReportTimeout(req)
         })
         req = result
       }
 
-      req = Request(req, parentReq)
-
-      if (isDataStoredInWorker) {
-        // we unset this because we want the Request() call in worker to evaluate the data
-        // and determine if the original was empty or not
-        delete req.context.originalInputDataIsEmpty
-      }
+      req = Request(req)
 
       // TODO: we will probably validate in the thread
       if (this.entityTypeValidator.getSchema('TemplateType') != null) {
@@ -384,29 +404,40 @@ class MainReporter extends Reporter {
         }
       }
 
-      let reportTimeout = this.options.reportTimeout
+      const reportTimeout = this.getReportTimeout(req)
 
-      if (
-        this.options.enableRequestReportTimeout &&
-        req.options &&
-        req.options.timeout != null
-      ) {
-        reportTimeout = req.options.timeout
+      await this.beforeRenderListeners.fire(req, res, { worker })
+
+      if (workerAborted) {
+        throw this.createError('Request aborted by client')
       }
 
-      await this.beforeRenderListeners.fire(req, res, { worker })
+      if (req.context.clientNotification) {
+        process.nextTick(async () => {
+          try {
+            const responseResult = await this.executeWorkerAction('render', {}, {
+              timeout: reportTimeout + this.options.reportTimeoutMargin,
+              worker
+            }, req)
+
+            Object.assign(res, responseResult)
+            await this.afterRenderListeners.fire(req, res)
+          } catch (err) {
+            await this._handleRenderError(req, res, err).catch((e) => {})
+          } finally {
+            if (!workerAborted) {
+              await worker.release(req)
+            }
+          }
+        })
 
-      // this is used so far just in the reports extension
-      // it wants to send to the client immediate response with link to the report status
-      // but the previous steps already allocated worker which has the parsed input request
-      // so we need to keep the worker active and let the subsequent real render call use it
-      // we cant move the main beforeRenderListener before the worker allocation, because at that point
-      // the request isn't parsed and we don't know the template and options
-      if (req.context.returnResponseAndKeepWorker) {
-        keepWorker = true
-        res.stream = Readable.from(res.content)
-        await this.afterRenderListeners.fire(req, res)
-        return res
+        dontCloseProcessing = true
+        const r = {
+          ...req.context.clientNotification,
+          stream: Readable.from(req.context.clientNotification.content)
+        }
+        delete req.context.clientNotification
+        return r
       }
 
       if (workerAborted) {
@@ -423,33 +454,10 @@ class MainReporter extends Reporter {
       res.stream = Readable.from(res.content)
       return res
     } catch (err) {
-      if (err.code === 'WORKER_TIMEOUT') {
-        err.message = 'Report timeout'
-        if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
-          err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
-        }
-
-        if (req.context.http != null) {
-          const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
-          err.message += `. You can inspect and find more details here: ${profileUrl}`
-        }
-
-        err.weak = true
-      }
-
-      if (err.code === 'WORKER_ABORTED') {
-        err.message = 'Report cancelled'
-        err.weak = true
-      }
-
-      if (!err.logged) {
-        const logFn = err.weak ? this.logger.warn : this.logger.error
-        logFn(`Report render failed: ${err.message}${err.stack != null ? ' ' + err.stack : ''}`, req)
-      }
-      await this.renderErrorListeners.fire(req, res, err)
+      await this._handleRenderError(req, res, err)
       throw err
     } finally {
-      if (!workerAborted && !keepWorker) {
+      if (worker && !workerAborted && !dontCloseProcessing) {
         await worker.release(req)
       }
     }

package/lib/main/request.js

@@ -0,0 +1,21 @@
+const extend = require('node.extend.without.arrays')
+
+module.exports = (obj) => {
+  const request = Object.create({}, {
+    __isJsreportRequest__: {
+      value: true,
+      writable: false,
+      configurable: false,
+      enumerable: false
+    }
+  })
+
+  request.template = extend(true, {}, obj.template)
+
+  request.options = extend(true, {}, request.options, obj.options)
+  request.context = extend(true, {}, request.context, obj.context)
+  request.context.shared = extend(true, {}, request.context.shared)
+  request.data = obj.data
+
+  return request
+}

package/lib/main/schemaValidator.js

@@ -3,6 +3,7 @@ const set = require('lodash.set')
 const hasOwn = require('has-own-deep')
 const unsetValue = require('unset-value')
 const ms = require('ms')
+const bytes = require('bytes')
 const Ajv = require('ajv')
 
 const validatorCollection = new WeakMap()
@@ -126,6 +127,35 @@ class SchemaValidator {
       }
     })
 
+    validator.addKeyword('$jsreport-acceptsSize', {
+      modifying: true,
+      compile: (sch) => {
+        if (sch !== true) {
+          return () => true
+        }
+
+        return (data, dataPath, parentData, parentDataProperty) => {
+          if (typeof data !== 'string' && typeof data !== 'number') {
+            return false
+          }
+
+          if (typeof data === 'number') {
+            return true
+          }
+
+          const newData = bytes(data)
+
+          if (newData == null) {
+            return false
+          }
+
+          parentData[parentDataProperty] = newData
+
+          return true
+        }
+      }
+    })
+
     let rootValidate
 
     if (options.rootSchema != null) {

package/lib/main/settings.js

@@ -25,7 +25,7 @@ Settings.prototype.get = function (key) {
 }
 
 Settings.prototype.findValue = async function (key, req) {
-  const res = await this.documentStore.collection('settings').find({ key: key }, req)
+  const res = await this.documentStore.collection('settings').find({ key: key }, localReqWithoutAuthorization(req))
   if (res.length !== 1) {
     return null
   }
@@ -49,7 +49,6 @@ Settings.prototype.addOrSet = async function (key, avalue, req) {
   const value = typeof avalue !== 'string' ? JSON.stringify(avalue) : avalue
 
   const updateCount = await this.documentStore.collection('settings').update({ key }, { $set: { key: key, value: value } }, localReqWithoutAuthorization(req))
-
   if (updateCount === 0) {
     await this.documentStore.collection('settings').insert({ key: key, value: value }, localReqWithoutAuthorization(req))
     return 1

package/lib/main/store/collection.js

@@ -86,16 +86,18 @@ module.exports = (entitySet, provider, model, validator, encryption, transaction
       validateEntityName(data.name)
     }
 
-    const entityType = model.entitySets[entitySet] ? model.entitySets[entitySet].normalizedEntityTypeName : null
+    if (req == null || req.context.skipValidationFor !== data) {
+      const entityType = model.entitySets[entitySet] ? model.entitySets[entitySet].normalizedEntityTypeName : null
 
-    if (entityType != null && validator.getSchema(entityType) != null) {
-      const validationResult = validator.validate(entityType, data)
+      if (entityType != null && validator.getSchema(entityType) != null) {
+        const validationResult = validator.validate(entityType, data)
 
-      if (!validationResult.valid) {
-        throw createError(`Error when trying to insert into "${entitySet}" collection. input contain values that does not match the schema. ${validationResult.fullErrorMessage}`, {
-          weak: true,
-          statusCode: 400
-        })
+        if (!validationResult.valid) {
+          throw createError(`Error when trying to insert into "${entitySet}" collection. input contain values that does not match the schema. ${validationResult.fullErrorMessage}`, {
+            weak: true,
+            statusCode: 400
+          })
+        }
       }
     }
 

package/lib/main/store/documentStore.js

@@ -372,6 +372,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async beginTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       if (req.context.storeTransaction && transactions.has(req.context.storeTransaction)) {
         throw new Error('Can not call store.beginTransaction when an active transaction already exists, make sure you are not calling store.beginTransaction more than once')
       }
@@ -386,6 +390,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async commitTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       const tranId = req.context.storeTransaction
       const tran = transactions.get(tranId)
 
@@ -400,6 +408,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async rollbackTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       const tranId = req.context.storeTransaction
       const tran = transactions.get(tranId)
 
@@ -411,6 +423,13 @@ const DocumentStore = (options, validator, encryption) => {
 
       transactions.delete(tranId)
       delete req.context.storeTransaction
+    },
+
+    generateId () {
+      if (this.provider.generateId) {
+        return this.provider.generateId()
+      }
+      return uuidv4()
     }
   }
 

package/lib/main/store/setupValidateId.js

@@ -3,10 +3,16 @@ module.exports = (reporter) => {
   reporter.initializeListeners.add('core-validate-id', () => {
     for (const c of Object.keys(reporter.documentStore.collections)) {
       reporter.documentStore.collection(c).beforeInsertListeners.add('validate-id', (doc, req) => {
-        return validateIdForStoreChange(reporter, c, doc._id, undefined, req)
+        if (req == null || req.context.skipValidationFor !== doc) {
+          return validateIdForStoreChange(reporter, c, doc._id, undefined, req)
+        }
       })
 
       reporter.documentStore.collection(c).beforeUpdateListeners.add('validate-id', async (q, update, opts, req) => {
+        if (req != null && req.context.skipValidationFor === update) {
+          return
+        }
+
         if (update.$set && opts && opts.upsert === true) {
           await validateIdForStoreChange(reporter, c, update.$set._id, undefined, req)
         }

package/lib/main/store/setupValidateShortid.js

@@ -31,6 +31,10 @@ module.exports = (reporter) => {
 }
 
 async function validateShortid (reporter, collectionName, doc, originalIdValue, req) {
+  if (req != null && req.context.skipValidationFor === doc) {
+    return
+  }
+
   const shortid = doc.shortid
 
   if (!shortid) {

package/lib/shared/normalizeMetaFromLogs.js

@@ -13,7 +13,7 @@ module.exports = (level, msg, meta) => {
 
     // TODO adding cancel looks bad, its before script is adding req.cancel()
     // excluding non relevant properties for the log
-    const newMeta = Object.assign({}, omit(meta, ['template', 'options', 'data', 'context', 'timestamp', 'cancel']))
+    const newMeta = Object.assign({}, omit(meta, ['rawContent', 'template', 'options', 'data', 'context', 'timestamp', 'cancel']))
 
     if (newMeta.rootId == null && meta.context.rootId != null) {
       newMeta.rootId = meta.context.rootId

package/lib/shared/reporter.js

@@ -51,6 +51,22 @@ class Reporter extends EventEmitter {
     return generateRequestId()
   }
 
+  /**
+   * @public Ensures that we get the proper report timeout in case when custom timeout per request was enabled
+   */
+  getReportTimeout (req) {
+    const elapsedTime = req.context.startTimestamp ? (new Date().getTime() - req.context.startTimestamp) : 0
+    if (
+      this.options.enableRequestReportTimeout &&
+      req.options != null &&
+      req.options.timeout != null
+    ) {
+      return Math.max(0, req.options.timeout - elapsedTime)
+    }
+
+    return Math.max(0, this.options.reportTimeout - elapsedTime)
+  }
+
   /**
    * Ensures that the jsreport auto-cleanup temp directory (options.tempAutoCleanupDirectory) exists by doing a mkdir call
    *

package/lib/worker/render/executeEngine.js

@@ -14,6 +14,7 @@ module.exports = (reporter) => {
   reporter.templatingEngines = { cache }
 
   const executionFnParsedParamsMap = new Map()
+  const executionAsyncResultsMap = new Map()
 
   const templatingEnginesEvaluate = async (mainCall, { engine, content, helpers, data }, { entity, entitySet }, req) => {
     const engineImpl = reporter.extensionsManager.engines.find((e) => e.name === engine)
@@ -28,19 +29,23 @@ module.exports = (reporter) => {
       executionFnParsedParamsMap.set(req.context.id, new Map())
     }
 
+    const executionId = nanoid(7)
+
     try {
       const res = await executeEngine({
         engine: engineImpl,
         content,
         helpers,
         data
-      }, { handleErrors: false, entity, entitySet }, req)
+      }, { executionId, handleErrors: false, entity, entitySet }, req)
 
       return res.content
     } finally {
       if (mainCall) {
         executionFnParsedParamsMap.delete(req.context.id)
       }
+
+      executionAsyncResultsMap.delete(executionId)
     }
   }
 
@@ -54,6 +59,12 @@ module.exports = (reporter) => {
     proxy.templatingEngines = {
       evaluate: async (executionInfo, entityInfo) => {
         return templatingEnginesEvaluate(false, executionInfo, entityInfo, req)
+      },
+      waitForAsyncHelpers: async () => {
+        if (context.__executionId != null && executionAsyncResultsMap.has(context.__executionId)) {
+          const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
+          return Promise.all([...asyncResultMap.keys()].map((k) => asyncResultMap.get(k)))
+        }
       }
     }
   })
@@ -64,6 +75,8 @@ module.exports = (reporter) => {
     req.data.__rootDirectory = reporter.options.rootDirectory
     req.data.__parentModuleDirectory = reporter.options.parentModuleDirectory
 
+    const executionId = nanoid(7)
+
     try {
       return await executeEngine({
         engine,
@@ -71,16 +84,18 @@ module.exports = (reporter) => {
         helpers: req.template.helpers,
         data: req.data
       }, {
+        executionId,
         handleErrors: true,
         entity: req.template,
         entitySet: 'templates'
       }, req)
     } finally {
       executionFnParsedParamsMap.delete(req.context.id)
+      executionAsyncResultsMap.delete(executionId)
     }
   }
 
-  async function executeEngine ({ engine, content, helpers, data }, { handleErrors, entity, entitySet }, req) {
+  async function executeEngine ({ engine, content, helpers, data }, { executionId, handleErrors, entity, entitySet }, req) {
     let entityPath
 
     if (entity._id) {
@@ -104,9 +119,14 @@ module.exports = (reporter) => {
     const joinedHelpers = systemHelpersStr + '\n' + (helpers || '')
     const executionFnParsedParamsKey = `entity:${entity.shortid || 'anonymous'}:helpers:${joinedHelpers}`
 
-    const executionFn = async ({ require, console, topLevelFunctions }) => {
+    const executionFn = async ({ require, console, topLevelFunctions, context }) => {
       const asyncResultMap = new Map()
-      executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions })
+
+      context.__executionId = executionId
+
+      executionAsyncResultsMap.set(executionId, asyncResultMap)
+      executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions, context })
+
       const key = `template:${content}:${engine.name}`
 
       if (!cache.has(key)) {
@@ -143,16 +163,19 @@ module.exports = (reporter) => {
       }
 
       return {
-        content: contentResult
+        // handlebars escapes single brackets before execution to prevent errors on {#asset}
+        // we need to unescape them later here, because at the moment the engine.execute finishes
+        // the async helpers aren't executed yet
+        content: engine.unescape ? engine.unescape(contentResult) : contentResult
       }
     }
 
     // executionFnParsedParamsMap is there to cache parsed components helpers to speed up longer loops
     // we store there for the particular request and component a promise and only the first component gets compiled
     if (executionFnParsedParamsMap.get(req.context.id).has(executionFnParsedParamsKey)) {
-      const { require, console, topLevelFunctions } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
+      const { require, console, topLevelFunctions, context } = await (executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).promise)
 
-      return executionFn({ require, console, topLevelFunctions })
+      return executionFn({ require, console, topLevelFunctions, context })
     } else {
       const awaiter = {}
       awaiter.promise = new Promise((resolve) => {

package/lib/worker/render/profiler.js

@@ -57,9 +57,9 @@ class Profiler {
 
     if (m.type !== 'log') {
       req.context.profiling.lastEventId = m.id
+      m.operationId = m.operationId || generateRequestId()
     }
 
-    m.operationId = m.operationId || generateRequestId()
     if (m.previousOperationId == null && req.context.profiling.lastOperationId) {
       m.previousOperationId = req.context.profiling.lastOperationId
     }
@@ -72,15 +72,21 @@ class Profiler {
       let content = res.content
 
       if (content != null) {
-        if (isbinaryfile(content)) {
+        if (content.length > this.reporter.options.profiler.maxResponseSize) {
           content = {
-            content: res.content.toString('base64'),
-            encoding: 'base64'
+            tooLarge: true
           }
         } else {
-          content = {
-            content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
-            encoding: 'diff'
+          if (isbinaryfile(content)) {
+            content = {
+              content: res.content.toString('base64'),
+              encoding: 'base64'
+            }
+          } else {
+            content = {
+              content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
+              encoding: 'diff'
+            }
           }
         }
       }
@@ -93,7 +99,7 @@ class Profiler {
 
       m.req = { diff: createPatch('req', req.context.profiling.reqLastVal || '', stringifiedReq, 0) }
 
-      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content)) ? null : res.content.toString()
+      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content) || content.tooLarge) ? null : res.content.toString()
       req.context.profiling.resMetaLastVal = stringifiedResMeta
       req.context.profiling.reqLastVal = stringifiedReq
     }
@@ -129,10 +135,6 @@ class Profiler {
       previousOperationId: parentReq ? parentReq.context.profiling.lastOperationId : null
     }
 
-    if (!req.context.isChildRequest) {
-      profilerEvent.profileId = req.context.profiling.entity._id
-    }
-
     return this.emit(profilerEvent, req, res)
   }