@jsreport/jsreport-core 3.9.0 → 3.11.0

package/README.md

@@ -282,6 +282,29 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 3.11.0
+
+- log when worker returns bad res.content
+- fix profiler leaks
+- remove settings sync API and avoid loading all items to memory
+- throw weak error when validating duplicated entity
+- ensure we end with profiles with error state when there is server or req timeout
+
+### 3.10.0
+
+- `mainReporter.executeWorkerAction` now supports cancellation with `AbortController.signal`
+- add support for specifying what are the main document properties of templates entitySet
+
+### 3.9.0
+
+- add more store methods `collection.findAdmin`, `collection.findOneAdmin`, `reporter.adminRequest` to easily allow execure store queries without taking into account permissions
+- improve logging for child requests and user level logs
+- differentiate between template not found errors and permissions related errors (it is now more clean what is the cause of specific error)
+- normalize to error when non-errors are throw (like throw "string")
+- improve errors in helpers (it now includes the helper name)
+- improve error message when template was not found in child request
+- improve error handling in sandbox
+
 ### 3.8.1
 
 - update vm2 for fix security issue

package/lib/main/folders/validateDuplicatedName.js

@@ -74,6 +74,7 @@ async function validateDuplicatedName (reporter, c, doc, originalIdValue, req) {
 
     throw reporter.createError(msg, {
       statusCode: 400,
+      weak: true,
       code: 'DUPLICATED_ENTITY',
       existingEntity: existingEntity.entity,
       existingEntityEntitySet: existingEntity.entitySet

package/lib/main/profiler.js

@@ -1,9 +1,9 @@
 const EventEmitter = require('events')
-const winston = require('winston')
+const Transport = require('winston-transport')
 const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
 const fs = require('fs/promises')
-
+const { SPLAT } = require('triple-beam')
 module.exports = (reporter) => {
   reporter.documentStore.registerEntityType('ProfileType', {
     templateShortid: { type: 'Edm.String', referenceTo: 'templates' },
@@ -12,7 +12,8 @@ module.exports = (reporter) => {
     state: { type: 'Edm.String' },
     error: { type: 'Edm.String' },
     mode: { type: 'Edm.String', schema: { enum: ['full', 'standard', 'disabled'] } },
-    blobName: { type: 'Edm.String' }
+    blobName: { type: 'Edm.String' },
+    timeout: { type: 'Edm.Int32' }
   })
 
   reporter.documentStore.registerEntitySet('profiles', {
@@ -23,7 +24,6 @@ module.exports = (reporter) => {
   const profilersMap = new Map()
   const profilerOperationsChainsMap = new Map()
   const profilerRequestMap = new Map()
-  const profilerLogRequestMap = new Map()
 
   function runInProfilerChain (fnOrOptions, req) {
     if (req.context.profiling.mode === 'disabled') {
@@ -89,7 +89,7 @@ module.exports = (reporter) => {
     for (const m of events) {
       if (m.type === 'log') {
         if (log) {
-          reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp, fromEmitProfile: true })
+          reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp, logged: true })
         }
       } else {
         lastOperation = m
@@ -167,7 +167,8 @@ module.exports = (reporter) => {
       _id: reporter.documentStore.generateId(),
       timestamp: new Date(),
       state: 'queued',
-      mode: req.context.profiling.mode
+      mode: req.context.profiling.mode,
+      timeout: reporter.options.enableRequestReportTimeout && req.options.timeout ? req.options.timeout : reporter.options.reportTimeout
     }
 
     const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
@@ -325,49 +326,40 @@ module.exports = (reporter) => {
     }
   })
 
+  // we want to add to profiles also log messages from the main
   const configuredPreviously = reporter.logger.__profilerConfigured__ === true
-
   if (!configuredPreviously) {
-    const originalLog = reporter.logger.log
-
-    // we want to catch the original request
-    reporter.logger.log = function (level, msg, ...splat) {
-      const [meta] = splat
+    // we emit from winston transport, so winston formatters can still format message
+    class EmittingProfilesTransport extends Transport {
+      log (info, callback) {
+        setImmediate(() => {
+          this.emit('logged', info)
+        })
+
+        if (info[SPLAT]) {
+          const [req] = info[SPLAT]
+
+          if (req && req.context && req.logged !== true) {
+            emitProfiles({
+              events: [createProfileMessage({
+                type: 'log',
+                level: info.level,
+                message: info.message,
+                previousOperationId: req.context.profiling.lastOperationId
+              }, req)],
+              log: false
+            }, req)
+          }
+        }
 
-      if (typeof meta === 'object' && meta !== null && meta.context?.rootId != null) {
-        profilerLogRequestMap.set(meta.context.rootId, meta)
+        callback()
       }
-
-      return originalLog.call(this, level, msg, ...splat)
     }
 
-    const mainLogsToProfile = winston.format((info) => {
-      // propagate the request logs occurring on main to the profile
-      if (info.rootId != null && info.fromEmitProfile == null && profilerLogRequestMap.has(info.rootId)) {
-        const req = profilerLogRequestMap.get(info.rootId)
-
-        emitProfiles({
-          events: [createProfileMessage({
-            type: 'log',
-            level: info.level,
-            message: info.message,
-            previousOperationId: req.context.profiling.lastOperationId
-          }, req)],
-          log: false
-        }, req)
-      }
-
-      if (info.fromEmitProfile != null) {
-        delete info.fromEmitProfile
-      }
-
-      return info
-    })
-
-    reporter.logger.format = winston.format.combine(
-      reporter.logger.format,
-      mainLogsToProfile()
-    )
+    reporter.logger.add(new EmittingProfilesTransport({
+      format: reporter.logger.format,
+      level: 'debug'
+    }))
 
     reporter.logger.__profilerConfigured__ = true
   }
@@ -410,7 +402,6 @@ module.exports = (reporter) => {
     profilersMap.clear()
     profilerOperationsChainsMap.clear()
     profilerRequestMap.clear()
-    profilerLogRequestMap.clear()
   })
 
   let profilesCleanupRunning = false
@@ -422,11 +413,13 @@ module.exports = (reporter) => {
 
     profilesCleanupRunning = true
 
-    let lastRemoveError
+    let lastError
 
     try {
-      const profiles = await reporter.documentStore.collection('profiles').find({}).sort({ timestamp: -1 })
-      const profilesToRemove = profiles.slice(reporter.options.profiler.maxProfilesHistory)
+      const profilesToRemove = await reporter.documentStore.collection('profiles')
+        .find({}, { _id: 1 }).sort({ timestamp: -1 })
+        .skip(reporter.options.profiler.maxProfilesHistory)
+        .toArray()
 
       for (const profile of profilesToRemove) {
         if (reporter.closed || reporter.closing) {
@@ -438,7 +431,40 @@ module.exports = (reporter) => {
             _id: profile._id
           })
         } catch (e) {
-          lastRemoveError = e
+          lastError = e
+        }
+      }
+
+      const notFinishedProfiles = await reporter.documentStore.collection('profiles')
+        .find({ $or: [{ state: 'running' }, { state: 'queued' }] }, { _id: 1, timeout: 1, timestamp: 1 })
+        .toArray()
+
+      for (const profile of notFinishedProfiles) {
+        if (reporter.closed || reporter.closing) {
+          return
+        }
+
+        if (!profile.timeout) {
+          continue
+        }
+
+        const whenShouldBeFinished = profile.timestamp + profile.timeout + reporter.options.reportTimeoutMargin * 2
+        if (whenShouldBeFinished < new Date().getTime()) {
+          continue
+        }
+
+        try {
+          await reporter.documentStore.collection('profiles').update({
+            _id: profile._id
+          }, {
+            $set: {
+              state: 'error',
+              finishedOn: new Date(),
+              error: 'The server did not update the report profile before its timeout. This can happen when the server is unexpectedly stopped.'
+            }
+          })
+        } catch (e) {
+          lastError = e
         }
       }
     } catch (e) {
@@ -447,8 +473,8 @@ module.exports = (reporter) => {
       profilesCleanupRunning = false
     }
 
-    if (lastRemoveError) {
-      reporter.logger.warn('Profile cleanup failed for some entities, last error:', lastRemoveError)
+    if (lastError) {
+      reporter.logger.warn('Profile cleanup failed for some entities, last error:', lastError)
     }
   }
 
@@ -461,7 +487,6 @@ module.exports = (reporter) => {
       profilersMap.delete(req.context.rootId)
       profilerOperationsChainsMap.delete(req.context.rootId)
       profilerRequestMap.delete(req.context.rootId)
-      profilerLogRequestMap.delete(req.context.rootId)
       return
     }
 
@@ -471,7 +496,6 @@ module.exports = (reporter) => {
         profilersMap.delete(req.context.rootId)
         profilerOperationsChainsMap.delete(req.context.rootId)
         profilerRequestMap.delete(req.context.rootId)
-        profilerLogRequestMap.delete(req.context.rootId)
       }
     }, req)
   }

package/lib/main/reporter.js

@@ -449,6 +449,7 @@ class MainReporter extends Reporter {
           } catch (err) {
             await this._handleRenderError(req, res, err).catch((e) => {})
           } finally {
+            this._cleanProfileInRequest(req)
             if (!workerAborted) {
               await worker.release(req)
             }
@@ -477,6 +478,10 @@ class MainReporter extends Reporter {
 
       await this.afterRenderListeners.fire(req, res)
 
+      if (!res.content) {
+        this.logger.error('Worker didnt return render res.content, returned:' + JSON.stringify(responseResult), req)
+      }
+
       res.stream = Readable.from(res.content)
 
       this._cleanProfileInRequest(req)
@@ -557,6 +562,16 @@ class MainReporter extends Reporter {
         timeout
       })
 
+    const handleAbortSignal = () => {
+      if (worker) {
+        worker.release(req).catch((e) => this.logger.error('Failed to release worker ' + e))
+      }
+    }
+
+    if (options.signal && !options.worker) {
+      options.signal.addEventListener('abort', handleAbortSignal, { once: true })
+    }
+
     try {
       const result = await worker.execute({
         actionName,
@@ -579,7 +594,16 @@ class MainReporter extends Reporter {
       return result
     } finally {
       if (!options.worker) {
-        await worker.release(req)
+        let shouldRelease = true
+
+        if (options.signal) {
+          options.signal.removeEventListener('abort', handleAbortSignal)
+          shouldRelease = options.signal.aborted !== true
+        }
+
+        if (shouldRelease) {
+          await worker.release(req)
+        }
       }
     }
   }

package/lib/main/settings.js

@@ -6,7 +6,7 @@
 const Request = require('../shared/request')
 
 const Settings = module.exports = function () {
-  this._collection = []
+
 }
 
 Settings.prototype.add = function (key, value, req) {
@@ -15,15 +15,9 @@ Settings.prototype.add = function (key, value, req) {
     value: typeof value !== 'string' ? JSON.stringify(value) : value
   }
 
-  this._collection.push(settingItem)
-
   return this.documentStore.collection('settings').insert(settingItem, localReqWithoutAuthorization(req))
 }
 
-Settings.prototype.get = function (key) {
-  return this._collection.find((s) => s.key === key)
-}
-
 Settings.prototype.findValue = async function (key, req) {
   const res = await this.documentStore.collection('settings').find({ key: key }, localReqWithoutAuthorization(req))
   if (res.length !== 1) {
@@ -36,8 +30,6 @@ Settings.prototype.findValue = async function (key, req) {
 Settings.prototype.set = function (key, avalue, req) {
   const value = typeof avalue !== 'string' ? JSON.stringify(avalue) : avalue
 
-  this.get(key).value = value
-
   return this.documentStore.collection('settings').update({
     key: key
   }, {
@@ -58,8 +50,6 @@ Settings.prototype.addOrSet = async function (key, avalue, req) {
 Settings.prototype.init = async function (documentStore, authorization) {
   this.documentStore = documentStore
 
-  const incompatibleSettingsToRemove = []
-
   if (authorization != null) {
     const col = documentStore.collection('settings')
 
@@ -95,30 +85,6 @@ Settings.prototype.init = async function (documentStore, authorization) {
       }
     })
   }
-
-  const res = await documentStore.collection('settings').find({})
-
-  res.forEach((v) => {
-    if (typeof v.value !== 'string') {
-      return this._collection.push({
-        key: v.key,
-        value: v.value
-      })
-    }
-
-    try {
-      return this._collection.push({
-        key: v.key,
-        value: JSON.parse(v.value)
-      })
-    } catch (e) {
-      incompatibleSettingsToRemove.push(v._id)
-    }
-  })
-
-  if (incompatibleSettingsToRemove.length) {
-    return documentStore.collection('settings').remove({ _id: { $in: incompatibleSettingsToRemove } })
-  }
 }
 
 Settings.prototype.registerEntity = function (documentStore) {

package/lib/main/templates.js

@@ -2,9 +2,9 @@
 module.exports = (reporter) => {
   reporter.documentStore.registerEntityType('TemplateType', {
     name: { type: 'Edm.String' },
-    content: { type: 'Edm.String', document: { extension: 'html', engine: true } },
+    content: { type: 'Edm.String', document: { main: true, extension: 'html', engine: true } },
     recipe: { type: 'Edm.String' },
-    helpers: { type: 'Edm.String', document: { extension: 'js' }, schema: { type: 'object' } },
+    helpers: { type: 'Edm.String', document: { main: true, extension: 'js' }, schema: { type: 'object' } },
     engine: { type: 'Edm.String' }
   }, true)
 

package/lib/shared/normalizeMetaFromLogs.js

@@ -13,7 +13,7 @@ module.exports = (level, msg, meta) => {
 
     // TODO adding cancel looks bad, its before script is adding req.cancel()
     // excluding non relevant properties for the log
-    const newMeta = Object.assign({}, omit(meta, ['rawContent', 'template', 'options', 'data', 'context', 'timestamp', 'cancel']))
+    const newMeta = Object.assign({}, omit(meta, ['logged', 'rawContent', 'template', 'options', 'data', 'context', 'timestamp', 'cancel']))
 
     if (newMeta.rootId == null && meta.context.rootId != null) {
       newMeta.rootId = meta.context.rootId

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -40,7 +40,7 @@
     "@babel/parser": "7.14.4",
     "@babel/traverse": "7.12.9",
     "@colors/colors": "1.5.0",
-    "@jsreport/advanced-workers": "1.2.3",
+    "@jsreport/advanced-workers": "1.2.4",
     "@jsreport/mingo": "2.4.1",
     "@jsreport/reap": "0.1.0",
     "ajv": "6.12.6",
@@ -54,7 +54,7 @@
     "diff-match-patch": "1.0.5",
     "enhanced-resolve": "5.8.3",
     "has-own-deep": "1.1.0",
-    "isbinaryfile": "4.0.0",
+    "isbinaryfile": "5.0.0",
     "listener-collection": "2.0.0",
     "lodash.get": "4.4.2",
     "lodash.groupby": "4.6.0",