@jsreport/jsreport-core 3.4.1 → 3.6.0

package/lib/main/reporter.js

@@ -5,7 +5,7 @@
  */
 const path = require('path')
 const { Readable } = require('stream')
-const Reaper = require('reap2')
+const Reaper = require('@jsreport/reap')
 const optionsLoad = require('./optionsLoad')
 const { createLogger, configureLogger, silentLogs } = require('./logger')
 const checkEntityName = require('./validateEntityName')
@@ -28,10 +28,10 @@ const Reporter = require('../shared/reporter')
 const Request = require('./request')
 const generateRequestId = require('../shared/generateRequestId')
 const Profiler = require('./profiler')
-const Monitoring = require('./monitoring')
 const migrateXlsxTemplatesToAssets = require('./migration/xlsxTemplatesToAssets')
 const migrateResourcesToAssets = require('./migration/resourcesToAssets')
 const semver = require('semver')
+let reportCounter = 0
 
 class MainReporter extends Reporter {
   constructor (options, defaults) {
@@ -69,6 +69,7 @@ class MainReporter extends Reporter {
 
     this.logger = createLogger()
     this.beforeMainActionListeners = this.createListenerCollection('beforeMainAction')
+    this.beforeRenderWorkerAllocatedListeners = this.createListenerCollection('beforeRenderWorkerAllocated')
   }
 
   discover () {
@@ -89,8 +90,8 @@ class MainReporter extends Reporter {
     return this
   }
 
-  async extensionsLoad (opts) {
-    const appliedConfigFile = await optionsLoad({
+  async extensionsLoad (_opts = {}) {
+    const [explicitOptions, appliedConfigFile] = await optionsLoad({
       defaults: this.defaults,
       options: this.options,
       validator: this.optionsValidator,
@@ -105,6 +106,8 @@ class MainReporter extends Reporter {
       silentLogs(this.logger)
     }
 
+    const { onConfigDetails, ...opts } = _opts
+
     this.logger.info(`Initializing jsreport (version: ${this.version}, configuration file: ${appliedConfigFile || 'none'}, nodejs: ${process.versions.node})`)
 
     await this.extensionsManager.load(opts)
@@ -129,6 +132,10 @@ class MainReporter extends Reporter {
       throw new Error(`options contain values that does not match the defined full root schema. ${rootOptionsValidation.fullErrorMessage}`)
     }
 
+    if (typeof onConfigDetails === 'function') {
+      onConfigDetails(explicitOptions)
+    }
+
     return this
   }
 
@@ -156,6 +163,7 @@ class MainReporter extends Reporter {
    */
   async init () {
     this.closing = this.closed = false
+
     if (this._initialized || this._initializing) {
       throw new Error('jsreport already initialized or just initializing. Make sure init is called only once')
     }
@@ -165,14 +173,23 @@ class MainReporter extends Reporter {
     this._initializing = true
 
     if (this.compilation) {
-      this.compilation.resource('sandbox.js', require.resolve('vm2/lib/sandbox.js'))
-      this.compilation.resource('contextify.js', require.resolve('vm2/lib/contextify.js'))
-      this.compilation.resource('fixasync.js', require.resolve('vm2/lib/fixasync.js'))
+      this.compilation.resource('vm2-events.js', require.resolve('vm2/lib/events.js'))
+      this.compilation.resource('vm2-resolver-compat.js', require.resolve('vm2/lib/resolver-compat.js'))
+      this.compilation.resource('vm2-resolver.js', require.resolve('vm2/lib/resolver.js'))
+      this.compilation.resource('vm2-setup-node-sandbox.js', require.resolve('vm2/lib/setup-node-sandbox.js'))
+      this.compilation.resource('vm2-setup-sandbox.js', require.resolve('vm2/lib/setup-sandbox.js'))
     }
 
     try {
       this._registerLogMainAction()
-      await this.extensionsLoad()
+
+      let explicitOptions
+
+      await this.extensionsLoad({
+        onConfigDetails: (_explicitOptions) => {
+          explicitOptions = _explicitOptions
+        }
+      })
 
       this.documentStore = DocumentStore(Object.assign({}, this.options, { logger: this.logger }), this.entityTypeValidator, this.encryption)
       documentStoreActions(this)
@@ -180,7 +197,6 @@ class MainReporter extends Reporter {
       blobStorageActions(this)
       Templates(this)
       Profiler(this)
-      Monitoring(this)
 
       this.folders = Object.assign(this.folders, Folders(this))
 
@@ -198,6 +214,14 @@ class MainReporter extends Reporter {
 
       await this.extensionsManager.init()
 
+      if (this.options.trustUserCode) {
+        this.logger.info('Code sandboxing is disabled, users can potentially penetrate the local system if you allow code from external users to be part of your reports')
+      }
+
+      if (explicitOptions.trustUserCode == null && explicitOptions.allowLocalFilesAccess != null) {
+        this.logger.warn('options.allowLocalFilesAccess is deprecated, use options.trustUserCode instead')
+      }
+
       this.logger.info(`Using general timeout for rendering (reportTimeout: ${this.options.reportTimeout})`)
 
       if (this.options.store.provider === 'memory') {
@@ -267,8 +291,6 @@ class MainReporter extends Reporter {
         name: 'none'
       })
 
-      this.monitoring.init()
-
       this.logger.info('reporter initialized')
       this._initialized = true
       this._initExecution.resolve()
@@ -306,6 +328,34 @@ class MainReporter extends Reporter {
     await validateReservedName(this, c, doc)
   }
 
+  async _handleRenderError (req, res, err) {
+    if (err.code === 'WORKER_TIMEOUT') {
+      err.message = 'Report timeout'
+      if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
+        err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
+      }
+
+      if (req.context.http != null) {
+        const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
+        err.message += `. You can inspect and find more details here: ${profileUrl}`
+      }
+
+      err.weak = true
+    }
+
+    if (err.code === 'WORKER_ABORTED') {
+      err.message = 'Report cancelled'
+      err.weak = true
+    }
+
+    if (!err.logged) {
+      const logFn = err.weak ? this.logger.warn : this.logger.error
+      logFn(`Report render failed: ${err.message}${err.stack != null ? ' ' + err.stack : ''}`, req)
+    }
+    await this.renderErrorListeners.fire(req, res, err)
+    throw err
+  }
+
   /**
    * Main method for invoking rendering
    * render({ template: { content: 'foo', engine: 'none', recipe: 'html' }, data: { foo: 'hello' } })
@@ -324,23 +374,30 @@ class MainReporter extends Reporter {
     req.context = Object.assign({}, req.context)
     req.context.rootId = req.context.rootId || generateRequestId()
     req.context.id = req.context.rootId
+    req.context.reportCounter = ++reportCounter
+    req.context.startTimestamp = new Date().getTime()
 
-    const worker = options.worker || await this._workersManager.allocate(req, {
-      timeout: this.options.reportTimeout
-    })
-
-    let keepWorker
+    let worker
     let workerAborted
+    let dontCloseProcessing
+    const res = { meta: {} }
+    try {
+      await this.beforeRenderWorkerAllocatedListeners.fire(req)
 
-    if (options.abortEmitter) {
-      options.abortEmitter.once('abort', () => {
-        workerAborted = true
-        worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+      worker = await this._workersManager.allocate(req, {
+        timeout: this.getReportTimeout(req)
       })
-    }
 
-    const res = { meta: {} }
-    try {
+      if (options.abortEmitter) {
+        options.abortEmitter.once('abort', () => {
+          if (workerAborted) {
+            return
+          }
+          workerAborted = true
+          worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+        })
+      }
+
       if (workerAborted) {
         throw this.createError('Request aborted by client')
       }
@@ -351,7 +408,7 @@ class MainReporter extends Reporter {
           req,
           data: {}
         }, {
-          timeout: this.options.reportTimeout
+          timeout: this.getReportTimeout(req)
         })
         req = result
       }
@@ -369,39 +426,40 @@ class MainReporter extends Reporter {
         }
       }
 
-      let reportTimeout = this.options.reportTimeout
-
-      if (
-        this.options.enableRequestReportTimeout &&
-        req.options &&
-        req.options.timeout != null
-      ) {
-        reportTimeout = req.options.timeout
-      }
+      const reportTimeout = this.getReportTimeout(req)
 
       await this.beforeRenderListeners.fire(req, res, { worker })
 
-      // this is used so far just in the reports extension
-      // it wants to send to the client immediate response with link to the report status
-      // but the previous steps already allocated worker which has the parsed input request
-      // so we need to keep the worker active and let the subsequent real render call use it
-      // we cant move the main beforeRenderListener before the worker allocation, because at that point
-      // the request isn't parsed and we don't know the template and options
-      if (req.context.returnResponseAndKeepWorker) {
-        keepWorker = true
-        res.stream = Readable.from(res.content)
-
-        // just temporary workaround until we change how report render works
-        await this.documentStore.collection('profiles').update({
-          _id: req.context.profiling.entity._id
-        }, {
-          $set: {
-            state: 'success',
-            finishedOn: new Date(),
-            blobPersisted: true
+      if (workerAborted) {
+        throw this.createError('Request aborted by client')
+      }
+
+      if (req.context.clientNotification) {
+        process.nextTick(async () => {
+          try {
+            const responseResult = await this.executeWorkerAction('render', {}, {
+              timeout: reportTimeout + this.options.reportTimeoutMargin,
+              worker
+            }, req)
+
+            Object.assign(res, responseResult)
+            await this.afterRenderListeners.fire(req, res)
+          } catch (err) {
+            await this._handleRenderError(req, res, err).catch((e) => {})
+          } finally {
+            if (!workerAborted) {
+              await worker.release(req)
+            }
           }
-        }, req)
-        return res
+        })
+
+        dontCloseProcessing = true
+        const r = {
+          ...req.context.clientNotification,
+          stream: Readable.from(req.context.clientNotification.content)
+        }
+        delete req.context.clientNotification
+        return r
       }
 
       if (workerAborted) {
@@ -418,33 +476,10 @@ class MainReporter extends Reporter {
       res.stream = Readable.from(res.content)
       return res
     } catch (err) {
-      if (err.code === 'WORKER_TIMEOUT') {
-        err.message = 'Report timeout'
-        if (req.context.profiling?.lastOperation != null && req.context.profiling?.entity != null) {
-          err.message += `. Last profiler operation: (${req.context.profiling.lastOperation.subtype}) ${req.context.profiling.lastOperation.name}`
-        }
-
-        if (req.context.http != null) {
-          const profileUrl = `${req.context.http.baseUrl}/studio/profiles/${req.context.profiling.entity._id}`
-          err.message += `. You can inspect and find more details here: ${profileUrl}`
-        }
-
-        err.weak = true
-      }
-
-      if (err.code === 'WORKER_ABORTED') {
-        err.message = 'Report cancelled'
-        err.weak = true
-      }
-
-      if (!err.logged) {
-        const logFn = err.weak ? this.logger.warn : this.logger.error
-        logFn(`Report render failed: ${err.message}${err.stack != null ? ' ' + err.stack : ''}`, req)
-      }
-      await this.renderErrorListeners.fire(req, res, err)
+      await this._handleRenderError(req, res, err)
       throw err
     } finally {
-      if (!workerAborted && !keepWorker) {
+      if (worker && !workerAborted && !dontCloseProcessing) {
         await worker.release(req)
       }
     }

package/lib/main/schemaValidator.js

@@ -3,6 +3,7 @@ const set = require('lodash.set')
 const hasOwn = require('has-own-deep')
 const unsetValue = require('unset-value')
 const ms = require('ms')
+const bytes = require('bytes')
 const Ajv = require('ajv')
 
 const validatorCollection = new WeakMap()
@@ -126,6 +127,35 @@ class SchemaValidator {
       }
     })
 
+    validator.addKeyword('$jsreport-acceptsSize', {
+      modifying: true,
+      compile: (sch) => {
+        if (sch !== true) {
+          return () => true
+        }
+
+        return (data, dataPath, parentData, parentDataProperty) => {
+          if (typeof data !== 'string' && typeof data !== 'number') {
+            return false
+          }
+
+          if (typeof data === 'number') {
+            return true
+          }
+
+          const newData = bytes(data)
+
+          if (newData == null) {
+            return false
+          }
+
+          parentData[parentDataProperty] = newData
+
+          return true
+        }
+      }
+    })
+
     let rootValidate
 
     if (options.rootSchema != null) {

package/lib/main/settings.js

@@ -25,7 +25,7 @@ Settings.prototype.get = function (key) {
 }
 
 Settings.prototype.findValue = async function (key, req) {
-  const res = await this.documentStore.collection('settings').find({ key: key }, req)
+  const res = await this.documentStore.collection('settings').find({ key: key }, localReqWithoutAuthorization(req))
   if (res.length !== 1) {
     return null
   }
@@ -49,7 +49,6 @@ Settings.prototype.addOrSet = async function (key, avalue, req) {
   const value = typeof avalue !== 'string' ? JSON.stringify(avalue) : avalue
 
   const updateCount = await this.documentStore.collection('settings').update({ key }, { $set: { key: key, value: value } }, localReqWithoutAuthorization(req))
-
   if (updateCount === 0) {
     await this.documentStore.collection('settings').insert({ key: key, value: value }, localReqWithoutAuthorization(req))
     return 1

package/lib/main/store/collection.js

@@ -86,16 +86,18 @@ module.exports = (entitySet, provider, model, validator, encryption, transaction
       validateEntityName(data.name)
     }
 
-    const entityType = model.entitySets[entitySet] ? model.entitySets[entitySet].normalizedEntityTypeName : null
+    if (req == null || req.context.skipValidationFor !== data) {
+      const entityType = model.entitySets[entitySet] ? model.entitySets[entitySet].normalizedEntityTypeName : null
 
-    if (entityType != null && validator.getSchema(entityType) != null) {
-      const validationResult = validator.validate(entityType, data)
+      if (entityType != null && validator.getSchema(entityType) != null) {
+        const validationResult = validator.validate(entityType, data)
 
-      if (!validationResult.valid) {
-        throw createError(`Error when trying to insert into "${entitySet}" collection. input contain values that does not match the schema. ${validationResult.fullErrorMessage}`, {
-          weak: true,
-          statusCode: 400
-        })
+        if (!validationResult.valid) {
+          throw createError(`Error when trying to insert into "${entitySet}" collection. input contain values that does not match the schema. ${validationResult.fullErrorMessage}`, {
+            weak: true,
+            statusCode: 400
+          })
+        }
       }
     }
 

package/lib/main/store/documentStore.js

@@ -372,6 +372,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async beginTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       if (req.context.storeTransaction && transactions.has(req.context.storeTransaction)) {
         throw new Error('Can not call store.beginTransaction when an active transaction already exists, make sure you are not calling store.beginTransaction more than once')
       }
@@ -386,6 +390,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async commitTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       const tranId = req.context.storeTransaction
       const tran = transactions.get(tranId)
 
@@ -400,6 +408,10 @@ const DocumentStore = (options, validator, encryption) => {
     },
 
     async rollbackTransaction (req) {
+      if (this.options.store?.transactions?.enabled === false) {
+        return
+      }
+
       const tranId = req.context.storeTransaction
       const tran = transactions.get(tranId)
 
@@ -411,6 +423,13 @@ const DocumentStore = (options, validator, encryption) => {
 
       transactions.delete(tranId)
       delete req.context.storeTransaction
+    },
+
+    generateId () {
+      if (this.provider.generateId) {
+        return this.provider.generateId()
+      }
+      return uuidv4()
     }
   }
 

package/lib/main/store/setupValidateId.js

@@ -3,10 +3,16 @@ module.exports = (reporter) => {
   reporter.initializeListeners.add('core-validate-id', () => {
     for (const c of Object.keys(reporter.documentStore.collections)) {
       reporter.documentStore.collection(c).beforeInsertListeners.add('validate-id', (doc, req) => {
-        return validateIdForStoreChange(reporter, c, doc._id, undefined, req)
+        if (req == null || req.context.skipValidationFor !== doc) {
+          return validateIdForStoreChange(reporter, c, doc._id, undefined, req)
+        }
       })
 
       reporter.documentStore.collection(c).beforeUpdateListeners.add('validate-id', async (q, update, opts, req) => {
+        if (req != null && req.context.skipValidationFor === update) {
+          return
+        }
+
         if (update.$set && opts && opts.upsert === true) {
           await validateIdForStoreChange(reporter, c, update.$set._id, undefined, req)
         }

package/lib/main/store/setupValidateShortid.js

@@ -31,6 +31,10 @@ module.exports = (reporter) => {
 }
 
 async function validateShortid (reporter, collectionName, doc, originalIdValue, req) {
+  if (req != null && req.context.skipValidationFor === doc) {
+    return
+  }
+
   const shortid = doc.shortid
 
   if (!shortid) {

package/lib/shared/normalizeMetaFromLogs.js

@@ -13,7 +13,7 @@ module.exports = (level, msg, meta) => {
 
     // TODO adding cancel looks bad, its before script is adding req.cancel()
     // excluding non relevant properties for the log
-    const newMeta = Object.assign({}, omit(meta, ['template', 'options', 'data', 'context', 'timestamp', 'cancel']))
+    const newMeta = Object.assign({}, omit(meta, ['rawContent', 'template', 'options', 'data', 'context', 'timestamp', 'cancel']))
 
     if (newMeta.rootId == null && meta.context.rootId != null) {
       newMeta.rootId = meta.context.rootId

package/lib/shared/reporter.js

@@ -51,6 +51,22 @@ class Reporter extends EventEmitter {
     return generateRequestId()
   }
 
+  /**
+   * @public Ensures that we get the proper report timeout in case when custom timeout per request was enabled
+   */
+  getReportTimeout (req) {
+    const elapsedTime = req.context.startTimestamp ? (new Date().getTime() - req.context.startTimestamp) : 0
+    if (
+      this.options.enableRequestReportTimeout &&
+      req.options != null &&
+      req.options.timeout != null
+    ) {
+      return Math.max(0, req.options.timeout - elapsedTime)
+    }
+
+    return Math.max(0, this.options.reportTimeout - elapsedTime)
+  }
+
   /**
    * Ensures that the jsreport auto-cleanup temp directory (options.tempAutoCleanupDirectory) exists by doing a mkdir call
    *