@jsreport/jsreport-core 3.11.1 → 3.11.3

package/README.md

@@ -282,6 +282,21 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 3.11.3
+
+- update vm2 to fix security issue
+- automatically disable full profiling after some time to avoid performance degradation
+- improvements to full profile serialization (prevent blocking)
+- fix profiles cleaning and calculate timeout in beforeRender
+
+### 3.11.2
+
+- add `options.onReqReady` to be able to receive the parsed req values
+
+### 3.11.1
+
+- fix error when trying to read `req.options` on reporter main code when `enableRequestReportTimeout` is enabled
+
 ### 3.11.0
 
 - log when worker returns bad res.content

package/lib/main/optionsSchema.js

@@ -176,6 +176,16 @@ module.exports.getRootSchemaOptions = () => ({
           type: 'string',
           default: 'standard'
         },
+        fullModeDurationCheckInterval: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsDuration': true,
+          default: '10m'
+        },
+        fullModeDuration: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsDuration': true,
+          default: '4h'
+        },
         maxProfilesHistory: {
           type: 'number',
           default: 1000
@@ -185,6 +195,11 @@ module.exports.getRootSchemaOptions = () => ({
           '$jsreport-acceptsDuration': true,
           default: '1m'
         },
+        maxUnallocatedProfileAge: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsDuration': true,
+          default: '24h'
+        },
         maxDiffSize: {
           type: ['string', 'number'],
           '$jsreport-acceptsSize': true,

package/lib/main/profiler.js

@@ -4,6 +4,9 @@ const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
 const fs = require('fs/promises')
 const { SPLAT } = require('triple-beam')
+const promisify = require('util').promisify
+const stringifyAsync = promisify(require('yieldable-json').stringifyAsync)
+
 module.exports = (reporter) => {
   reporter.documentStore.registerEntityType('ProfileType', {
     templateShortid: { type: 'Edm.String', referenceTo: 'templates' },
@@ -104,8 +107,11 @@ module.exports = (reporter) => {
       req.context.profiling.lastOperation = lastOperation
     }
 
-    runInProfilerChain(() => {
-      return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
+    runInProfilerChain(async () => {
+      const stringifiedMessages = req.context.mode === 'full'
+        ? await Promise.all(events.map(m => stringifyAsync(m)))
+        : events.map(m => JSON.stringify(m))
+      await fs.appendFile(req.context.profiling.logFilePath, Buffer.from(stringifiedMessages.join('\n') + '\n'))
     }, req)
   }
 
@@ -167,8 +173,7 @@ module.exports = (reporter) => {
       _id: reporter.documentStore.generateId(),
       timestamp: new Date(),
       state: 'queued',
-      mode: req.context.profiling.mode,
-      timeout: reporter.options.enableRequestReportTimeout && req.options.timeout ? req.options.timeout : reporter.options.reportTimeout
+      mode: req.context.profiling.mode
     }
 
     const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
@@ -204,7 +209,9 @@ module.exports = (reporter) => {
 
   reporter.beforeRenderListeners.add('profiler', async (req, res) => {
     const update = {
-      state: 'running'
+      state: 'running',
+      // the timeout needs to be calculated later here, because the req.options.timeout isnt yet parsed in beforeRenderWorkerAllocatedListeners
+      timeout: reporter.options.enableRequestReportTimeout && req.options.timeout ? req.options.timeout : reporter.options.reportTimeout
     }
 
     // we set the request here because this listener will container the req which
@@ -365,6 +372,7 @@ module.exports = (reporter) => {
   }
 
   let profilesCleanupInterval
+  let fullModeDurationCheckInterval
 
   reporter.initializeListeners.add('profiler', async () => {
     reporter.documentStore.collection('profiles').beforeRemoveListeners.add('profiles', async (query, req) => {
@@ -377,13 +385,21 @@ module.exports = (reporter) => {
       }
     })
 
+    // exposing it to jo for override
     function profilesCleanupExec () {
       return reporter._profilesCleanup()
     }
 
+    function fullModeDurationCheckExec () {
+      return reporter._profilesFullModeDurationCheck()
+    }
+
     profilesCleanupInterval = setInterval(profilesCleanupExec, reporter.options.profiler.cleanupInterval)
     profilesCleanupInterval.unref()
 
+    fullModeDurationCheckInterval = setInterval(fullModeDurationCheckExec, reporter.options.profiler.fullModeDurationCheckInterval)
+    fullModeDurationCheckInterval.unref()
+
     await reporter._profilesCleanup()
   })
 
@@ -392,6 +408,10 @@ module.exports = (reporter) => {
       clearInterval(profilesCleanupInterval)
     }
 
+    if (fullModeDurationCheckInterval) {
+      clearInterval(fullModeDurationCheckInterval)
+    }
+
     for (const key of profilerOperationsChainsMap.keys()) {
       const profileAppendPromise = profilerOperationsChainsMap.get(key)
       if (profileAppendPromise) {
@@ -445,11 +465,28 @@ module.exports = (reporter) => {
         }
 
         if (!profile.timeout) {
+          // we can calculate profile timeout only after worker parses request and req.options.timeout is calculated
+          // if the timeout isnt calculated we error orphans that hangs for very long time before worker gets allocated and parses req
+          if ((profile.timestamp.getTime() + reporter.options.profiler.maxUnallocatedProfileAge) < new Date().getTime()) {
+            try {
+              await reporter.documentStore.collection('profiles').update({
+                _id: profile._id
+              }, {
+                $set: {
+                  state: 'error',
+                  finishedOn: new Date(),
+                  error: `The request wasn't parsed before ${reporter.options.profiler.maxUnallocatedProfileAge}ms. This can happen when the server is unexpectedly stopped.`
+                }
+              })
+            } catch (e) {
+              lastError = e
+            }
+          }
           continue
         }
 
-        const whenShouldBeFinished = profile.timestamp + profile.timeout + reporter.options.reportTimeoutMargin * 2
-        if (whenShouldBeFinished < new Date().getTime()) {
+        const whenShouldBeFinished = profile.timestamp.getTime() + profile.timeout + reporter.options.reportTimeoutMargin * 2
+        if (whenShouldBeFinished > new Date().getTime()) {
           continue
         }
 
@@ -478,6 +515,29 @@ module.exports = (reporter) => {
     }
   }
 
+  reporter._profilesFullModeDurationCheck = async function () {
+    try {
+      if (reporter.options.profiler.defaultMode === 'full') {
+        return
+      }
+      const profiler = await reporter.documentStore.collection('settings').findOne({ key: 'profiler' })
+      if (profiler == null || (profiler.modificationDate.getTime() + reporter.options.profiler.fullModeDuration) > new Date().getTime()) {
+        return
+      }
+
+      const profilerValue = JSON.parse(profiler.value)
+
+      if (profilerValue.mode !== 'full') {
+        return
+      }
+
+      reporter.logger.info('Switching full mode profiling back to standard to avoid performance degradation.')
+      await reporter.settings.addOrSet('profiler', { mode: 'standard' })
+    } catch (e) {
+      reporter.logger.warn('Failed to change profiling mode', e)
+    }
+  }
+
   return function cleanProfileInRequest (req) {
     // - req.context.profiling is empty only on an early error
     // that happens before setting the profiler.

package/lib/main/reporter.js

@@ -417,6 +417,7 @@ class MainReporter extends Reporter {
       }
 
       req = Request(req)
+      options.onReqReady?.(req)
 
       // TODO: we will probably validate in the thread
       if (this.entityTypeValidator.getSchema('TemplateType') != null) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "3.11.1",
+  "version": "3.11.3",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -71,9 +71,10 @@
     "triple-beam": "1.3.0",
     "unset-value": "1.0.0",
     "uuid": "8.3.2",
-    "vm2": "3.9.11",
+    "vm2": "3.9.17",
     "winston": "3.8.1",
-    "winston-transport": "4.5.0"
+    "winston-transport": "4.5.0",
+    "yieldable-json": "2.0.1"
   },
   "devDependencies": {
     "mocha": "9.2.2",