@jsreport/jsreport-core 3.4.0 → 3.5.0

package/README.md

@@ -36,7 +36,7 @@ const result = await jsreport.render({
 		foo: "world"
 	}
 })
-await fs.writeFile('out.pdf', resp.content)
+await fs.writeFile('out.pdf', result.content)
 ```
 
 ## Render
@@ -282,6 +282,24 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 3.5.0
+
+- fix parsing issue of code with comment in the sandbox (helpers, scripts)
+- improve profiling when there is big data
+- make transactions support in store configurable
+- improve timeout for the whole request
+- fix applying req.options.timeout when enableRequestReportTimeout is true
+- optimization regarding profile persistence
+
+### 3.4.2
+
+- update dep `vm2` to fix security vulnerability in sandbox
+
+### 3.4.1
+
+- fix passing data to async report
+- fix blob appends
+
 ### 3.4.0
 
 - fix for reports execution

package/lib/main/blobStorage/blobStorage.js

@@ -25,6 +25,7 @@ module.exports = (reporter, options) => {
         let existingBuf = Buffer.from([])
         try {
           existingBuf = await provider.read(blobName, req)
+          await provider.remove(blobName, req)
         } catch (e) {
           // so far blob storage throws when blob doesnt exit
         }
@@ -45,6 +46,14 @@ module.exports = (reporter, options) => {
       }
     },
 
+    get supportsAppend () {
+      return provider.append instanceof Function
+    },
+
+    get _provider () {
+      return provider
+    },
+
     registerProvider (p) {
       provider = p
     }

package/lib/main/optionsSchema.js

@@ -126,7 +126,13 @@ module.exports.getRootSchemaOptions = () => ({
     store: {
       type: 'object',
       properties: {
-        provider: { type: 'string', enum: ['memory'] }
+        provider: { type: 'string', enum: ['memory'] },
+        transactions: {
+          type: 'object',
+          properties: {
+            enabled: { type: 'boolean', default: true }
+          }
+        }
       }
     },
     blobStorage: {
@@ -163,6 +169,10 @@ module.exports.getRootSchemaOptions = () => ({
       type: 'object',
       default: {},
       properties: {
+        defaultMode: {
+          type: 'string',
+          default: 'standard'
+        },
         maxProfilesHistory: {
           type: 'number',
           default: 1000
@@ -171,6 +181,11 @@ module.exports.getRootSchemaOptions = () => ({
           type: ['string', 'number'],
           '$jsreport-acceptsDuration': true,
           default: '1m'
+        },
+        maxResponseSize: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsSize': true,
+          default: '50mb'
         }
       }
     }

package/lib/main/profiler.js

@@ -1,6 +1,7 @@
 const EventEmitter = require('events')
 const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
+const fs = require('fs/promises')
 
 module.exports = (reporter) => {
   reporter.documentStore.registerEntityType('ProfileType', {
@@ -8,9 +9,9 @@ module.exports = (reporter) => {
     timestamp: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     finishedOn: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     state: { type: 'Edm.String' },
-    blobPersisted: { type: 'Edm.Boolean' },
-    blobName: { type: 'Edm.String' },
-    error: { type: 'Edm.String' }
+    error: { type: 'Edm.String' },
+    mode: { type: 'Edm.String', schema: { enum: ['full', 'standard', 'disabled'] } },
+    blobName: { type: 'Edm.String' }
   })
 
   reporter.documentStore.registerEntitySet('profiles', {
@@ -19,9 +20,41 @@ module.exports = (reporter) => {
   })
 
   const profilersMap = new Map()
-  const profilerAppendChain = new Map()
 
-  async function emitProfiles (events, req) {
+  const profilerOperationsChainsMap = new Map()
+  function runInProfilerChain (fn, req) {
+    if (req.context.profiling.mode === 'disabled') {
+      return
+    }
+
+    profilerOperationsChainsMap.set(req.context.rootId, profilerOperationsChainsMap.get(req.context.rootId).then(async () => {
+      if (req.context.profiling.chainFailed) {
+        return
+      }
+
+      try {
+        await fn()
+      } catch (e) {
+        reporter.logger.warn('Failed persist profile', e)
+        req.context.profiling.chainFailed = true
+      }
+    }))
+  }
+
+  function createProfileMessage (m, req) {
+    m.timestamp = new Date().getTime()
+    m.id = generateRequestId()
+    m.previousOperationId = m.previousOperationId || null
+    if (m.type !== 'log') {
+      m.operationId = m.operationId || generateRequestId()
+      req.context.profiling.lastOperationId = m.operationId
+      req.context.profiling.lastEventId = m.id
+    }
+
+    return m
+  }
+
+  function emitProfiles (events, req) {
     if (events.length === 0) {
       return
     }
@@ -44,14 +77,16 @@ module.exports = (reporter) => {
       req.context.profiling.lastOperation = lastOperation
     }
 
-    profilerAppendChain.set(req.context.rootId, profilerAppendChain.get(req.context.rootId).then(() => {
+    runInProfilerChain(() => {
+      if (req.context.profiling.logFilePath) {
+        return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
+      }
+
       return reporter.blobStorage.append(
         req.context.profiling.entity.blobName,
         Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req
-      ).catch(e => {
-        reporter.logger.error('Failed to append to profile blob', e)
-      })
-    }))
+      )
+    }, req)
   }
 
   reporter.registerMainAction('profile', async (events, req) => {
@@ -62,8 +97,7 @@ module.exports = (reporter) => {
     req.context = req.context || {}
     req.context.rootId = reporter.generateRequestId()
     req.context.profiling = {
-      mode: profileMode == null ? 'full' : profileMode,
-      isAttached: true
+      mode: profileMode == null ? 'full' : profileMode
     }
     const profiler = new EventEmitter()
     profilersMap.set(req.context.rootId, profiler)
@@ -71,88 +105,136 @@ module.exports = (reporter) => {
     return profiler
   }
 
-  reporter.beforeRenderListeners.add('profiler', async (req, res) => {
-    profilerAppendChain.set(req.context.rootId, Promise.resolve())
-
+  reporter.beforeRenderWorkerAllocatedListeners.add('profiler', async (req) => {
     req.context.profiling = req.context.profiling || {}
+
+    if (req.context.profiling.mode == null) {
+      const profilerSettings = await reporter.settings.findValue('profiler', req)
+      const defaultMode = reporter.options.profiler.defaultMode || 'standard'
+      req.context.profiling.mode = (profilerSettings != null && profilerSettings.mode != null) ? profilerSettings.mode : defaultMode
+    }
+
+    profilerOperationsChainsMap.set(req.context.rootId, Promise.resolve())
+
     req.context.profiling.lastOperation = null
 
-    let blobName = `profiles/${req.context.rootId}.log`
+    const blobName = `profiles/${req.context.rootId}.log`
 
-    const template = await reporter.templates.resolveTemplate(req)
+    const profile = {
+      _id: reporter.documentStore.generateId(),
+      timestamp: new Date(),
+      state: 'queued',
+      mode: req.context.profiling.mode,
+      blobName
+    }
 
-    if (template && template._id) {
-      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
-      blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
-      // store a copy to prevent side-effects
-      req.context.resolvedTemplate = extend(true, {}, template)
+    if (!reporter.blobStorage.supportsAppend) {
+      const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
+      req.context.profiling.logFilePath = pathToFile
     }
 
-    if (!req.context.profiling.isAttached) {
-      const setting = await reporter.documentStore.collection('settings').findOne({ key: 'fullProfilerRunning' }, req)
-      if (setting && JSON.parse(setting.value)) {
-        req.context.profiling.isAttached = true
-        req.context.profiling.mode = 'full'
-      }
+    runInProfilerChain(async () => {
+      req.context.skipValidationFor = profile
+      await reporter.documentStore.collection('profiles').insert(profile, req)
+    }, req)
+
+    req.context.profiling.entity = profile
+
+    const profileStartOperation = createProfileMessage({
+      type: 'operationStart',
+      subtype: 'profile',
+      data: profile,
+      doDiffs: false
+    }, req)
+
+    req.context.profiling.profileStartOperationId = profileStartOperation.operationId
+
+    emitProfiles([profileStartOperation], req)
+
+    emitProfiles([createProfileMessage({
+      type: 'log',
+      level: 'info',
+      message: `Render request ${req.context.reportCounter} queued for execution and waiting for availible worker`,
+      previousOperationId: profileStartOperation.operationId
+    }, req)], req)
+  })
+
+  reporter.beforeRenderListeners.add('profiler', async (req, res) => {
+    const update = {
+      state: 'running'
     }
 
-    if (req.context.profiling.mode == null) {
-      req.context.profiling.mode = 'standard'
+    const template = await reporter.templates.resolveTemplate(req)
+    if (template && template._id) {
+      req.context.resolvedTemplate = extend(true, {}, template)
+      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
+      const blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
+      update.templateShortid = template.shortid
+
+      const originalBlobName = req.context.profiling.entity.blobName
+      // we want to store the profile into blobName path reflecting the template path so we need to copy the blob to new path now
+      runInProfilerChain(async () => {
+        if (req.context.profiling.logFilePath == null) {
+          const content = await reporter.blobStorage.read(originalBlobName, req)
+          await reporter.blobStorage.write(blobName, content, req)
+          return reporter.blobStorage.remove(originalBlobName, req)
+        }
+      }, req)
+
+      update.blobName = blobName
     }
 
-    const profile = await reporter.documentStore.collection('profiles').insert({
-      templateShortid: template != null ? template.shortid : null,
-      timestamp: new Date(),
-      state: 'running',
-      blobName,
-      fullRequestProfiling: req.context.profiling.mode === 'full'
+    runInProfilerChain(() => {
+      req.context.skipValidationFor = update
+      return reporter.documentStore.collection('profiles').update({
+        _id: req.context.profiling.entity._id
+      }, {
+        $set: update
+      }, req)
     }, req)
 
-    req.context.profiling.entity = profile
+    Object.assign(req.context.profiling.entity, update)
   })
 
   reporter.afterRenderListeners.add('profiler', async (req, res) => {
+    emitProfiles([createProfileMessage({
+      type: 'operationEnd',
+      doDiffs: false,
+      previousEventId: req.context.profiling.lastEventId,
+      previousOperationId: req.context.profiling.lastOperationId,
+      operationId: req.context.profiling.profileStartOperationId
+    }, req)], req)
+
     res.meta.profileId = req.context.profiling?.entity?._id
-    profilersMap.delete(req.context.rootId)
-    const profilerBlobPersistPromise = profilerAppendChain.get(req.context.rootId)
-    profilerAppendChain.delete(req.context.rootId)
-
-    await reporter.documentStore.collection('profiles').update({
-      _id: req.context.profiling.entity._id
-    }, {
-      $set: {
+
+    runInProfilerChain(async () => {
+      if (req.context.profiling.logFilePath != null) {
+        const content = await fs.readFile(req.context.profiling.logFilePath)
+        await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
+        await fs.unlink(req.context.profiling.logFilePath)
+      }
+
+      const update = {
         state: 'success',
         finishedOn: new Date()
       }
-    }, req)
-    profilerBlobPersistPromise.finally(() => {
-      reporter.documentStore.collection('profiles').update({
+      req.context.skipValidationFor = update
+      await reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
-        $set: {
-          blobPersisted: true
-        }
-      }, req).catch((e) => reporter.logger.error('Failed to update profile blobPersisted', e))
-    })
+        $set: update
+      }, req)
+    }, req)
+
+    // we don't remove from profiler requests map, because the renderErrorListeners are invoked if the afterRenderListener fails
   })
 
   reporter.renderErrorListeners.add('profiler', async (req, res, e) => {
     try {
       res.meta.profileId = req.context.profiling?.entity?._id
-      const profilerBlobPersistPromise = profilerAppendChain.get(req.context.rootId)
 
       if (req.context.profiling?.entity != null) {
-        await reporter.documentStore.collection('profiles').update({
-          _id: req.context.profiling.entity._id
-        }, {
-          $set: {
-            state: 'error',
-            finishedOn: new Date(),
-            error: e.toString()
-          }
-        }, req)
-
-        await emitProfiles([{
+        emitProfiles([{
           type: 'error',
           timestamp: new Date().getTime(),
           ...e,
@@ -160,20 +242,29 @@ module.exports = (reporter) => {
           stack: e.stack,
           message: e.message
         }], req)
+        runInProfilerChain(async () => {
+          if (req.context.profiling.logFilePath != null) {
+            const content = await fs.readFile(req.context.profiling.logFilePath, 'utf8')
+            await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
+            await fs.unlink(req.context.profiling.logFilePath)
+          }
 
-        profilerBlobPersistPromise.finally(() => {
-          reporter.documentStore.collection('profiles').update({
+          const update = {
+            state: 'error',
+            finishedOn: new Date(),
+            error: e.toString()
+          }
+          req.context.skipValidationFor = update
+          await reporter.documentStore.collection('profiles').update({
             _id: req.context.profiling.entity._id
           }, {
-            $set: {
-              blobPersisted: true
-            }
-          }, req).catch((e) => reporter.logger.error('Failed to update profile blobPersisted', e))
-        })
+            $set: update
+          }, req)
+        }, req)
       }
     } finally {
       profilersMap.delete(req.context.rootId)
-      profilerAppendChain.delete(req.context.rootId)
+      profilerOperationsChainsMap.delete(req.context.rootId)
     }
   })
 
@@ -197,8 +288,8 @@ module.exports = (reporter) => {
       clearInterval(profilesCleanupInterval)
     }
 
-    for (const key of profilerAppendChain.keys()) {
-      const profileAppendPromise = profilerAppendChain.get(key)
+    for (const key of profilerOperationsChainsMap.keys()) {
+      const profileAppendPromise = profilerOperationsChainsMap.get(key)
       if (profileAppendPromise) {
         await profileAppendPromise
       }