@jsreport/jsreport-core 4.5.0 → 4.6.1

package/README.md

@@ -282,6 +282,19 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 4.6.1
+
+- clear profiles canceling check interval during reporter close
+
+### 4.6.0
+
+- update nanoid to fix security issue
+- optimize fs store operations for big workspaces
+- reimplement and optimize fs transactions
+- fix async reports with mongo store
+- create store indexes during schema creation fix
+- implement canceling requests from profiler
+
 ### 4.5.0
 
 - fix blobStorage failing to save reports bigger than 1gb

package/index.js

@@ -21,6 +21,7 @@ module.exports.Request = Request
 module.exports.createListenerCollection = createListenerCollection
 module.exports.loggerFormat = winston.format
 module.exports.createDefaultLoggerFormat = createDefaultLoggerFormat
+module.exports.createError = require('./lib/shared/createError')
 
 module.exports.tests = {
   documentStore: () => require('./test/store/common.js'),

package/lib/main/folders/cascadeFolderRemove.js

@@ -1,25 +1,35 @@
 module.exports = (reporter) => {
   reporter.documentStore.collection('folders').beforeRemoveListeners.add('folders', async (q, req) => {
+    async function removeInCol (c, folder) {
+      const entities = await reporter.documentStore.collection(c).find({
+        folder: {
+          shortid: folder.shortid
+        }
+      }, req)
+
+      if (entities.length === 0) {
+        return
+      }
+
+      return reporter.documentStore.collection(c).remove({
+        _id: {
+          $in: entities.map(e => e._id)
+        }
+      }, req)
+    }
+
     const foldersToRemove = await reporter.documentStore.collection('folders').find(q, req)
+    const promises = []
 
     for (const folder of foldersToRemove) {
       for (const c of Object.keys(reporter.documentStore.collections)) {
-        const entities = await reporter.documentStore.collection(c).find({
-          folder: {
-            shortid: folder.shortid
-          }
-        }, req)
-
-        if (entities.length === 0) {
+        if (!reporter.documentStore.model.entitySets[c].entityTypeDef.folder) {
           continue
         }
 
-        for (const e of entities) {
-          await reporter.documentStore.collection(c).remove({
-            _id: e._id
-          }, req)
-        }
+        promises.push(removeInCol(c, folder))
       }
     }
+    return Promise.all(promises)
   })
 }

package/lib/main/folders/getEntitiesInFolder.js

@@ -8,6 +8,10 @@ module.exports = (reporter) => async function getEntitiesInFolder (folderShortId
   const lookup = []
 
   for (const [entitySetName] of Object.entries(reporter.documentStore.model.entitySets)) {
+    if (!reporter.documentStore.model.entitySets[entitySetName].entityTypeDef.folder) {
+      continue
+    }
+
     lookup.push(reporter.documentStore.collection(entitySetName).find({
       folder: {
         shortid: folderShortId

package/lib/main/folders/index.js

@@ -15,18 +15,19 @@ module.exports = (reporter) => {
   })
 
   reporter.documentStore.registerComplexType('FolderRefType', {
-    shortid: { type: 'Edm.String', referenceTo: 'folders' }
+    shortid: { type: 'Edm.String', referenceTo: 'folders', index: true, length: 255 }
   })
 
   // before document store initialization, extend all entity types with folder information
   reporter.documentStore.on('before-init', (documentStore) => {
     Object.entries(documentStore.model.entitySets).forEach(([k, entitySet]) => {
       const entityTypeName = entitySet.entityType.replace(documentStore.model.namespace + '.', '')
-
-      documentStore.model.entityTypes[entityTypeName].folder = {
-        type: 'jsreport.FolderRefType',
-        // folder reference can be null when entity is at the root level
-        schema: { type: 'null' }
+      if (entitySet.exportable !== false) {
+        documentStore.model.entityTypes[entityTypeName].folder = {
+          type: 'jsreport.FolderRefType',
+          // folder reference can be null when entity is at the root level
+          schema: { type: 'null' }
+        }
       }
     })
   })

package/lib/main/folders/validateDuplicatedName.js

@@ -1,18 +1,36 @@
 const resolveEntityPath = require('../../shared/folders/resolveEntityPath')
 
 async function findEntity (reporter, name, folder, req) {
+  async function findEntityInColAndFolder (c, folder) {
+    const entities = await reporter.documentStore.collection(c).findAdmin({
+      folder
+    }, {
+      name: 1
+    }, req)
+
+    return {
+      entities,
+      entitySet: c
+    }
+  }
+
+  const promises = []
   for (const c of Object.keys(reporter.documentStore.collections)) {
     if (!reporter.documentStore.model.entitySets[c].entityTypeDef.name) {
       continue
     }
 
-    const allEntities = await reporter.documentStore.collection(c).findAdmin({
-      folder
-    }, {
-      name: 1
-    }, req)
+    if (folder != null && !reporter.documentStore.model.entitySets[c].entityTypeDef.folder) {
+      continue
+    }
+
+    promises.push(findEntityInColAndFolder(c, folder))
+  }
+
+  const results = await Promise.all(promises)
 
-    const existingEntity = allEntities.find((entity) => {
+  for (const { entities, entitySet } of results) {
+    const existingEntity = entities.find((entity) => {
       if (entity.name) {
         // doing the check for case insensitive string (foo === FOO)
         return entity.name.toLowerCase() === name.toLowerCase()
@@ -22,7 +40,7 @@ async function findEntity (reporter, name, folder, req) {
     })
 
     if (existingEntity) {
-      return { entity: existingEntity, entitySet: c }
+      return { entity: existingEntity, entitySet }
     }
   }
 }

package/lib/main/optionsSchema.js

@@ -197,6 +197,11 @@ module.exports.getRootSchemaOptions = () => ({
           type: ['string', 'number'],
           '$jsreport-acceptsSize': true,
           default: '10mb'
+        },
+        cancelingCheckInterval: {
+          type: ['string', 'number'],
+          '$jsreport-acceptsDuration': true,
+          default: '5s'
         }
       }
     }

package/lib/main/profiler.js

@@ -12,7 +12,7 @@ module.exports = (reporter) => {
     templateShortid: { type: 'Edm.String', referenceTo: 'templates' },
     timestamp: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
     finishedOn: { type: 'Edm.DateTimeOffset', schema: { type: 'null' } },
-    state: { type: 'Edm.String' },
+    state: { type: 'Edm.String', schema: { enum: ['running', 'success', 'queued', 'error', 'canceling'] }, index: true, length: 255 },
     error: { type: 'Edm.String' },
     mode: { type: 'Edm.String', schema: { enum: ['full', 'standard', 'disabled'] } },
     blobName: { type: 'Edm.String' },
@@ -377,6 +377,7 @@ module.exports = (reporter) => {
 
   let profilesCleanupInterval
   let fullModeDurationCheckInterval
+  let profilesCancelingCheckInterval
 
   reporter.initializeListeners.add('profiler', async () => {
     reporter.documentStore.collection('profiles').beforeRemoveListeners.add('profiles', async (query, req) => {
@@ -398,12 +399,40 @@ module.exports = (reporter) => {
       return reporter._profilesFullModeDurationCheck()
     }
 
+    let _profilesCancelingCheckExecRunning = false
+    async function profilesCancelingCheckExec () {
+      if (_profilesCancelingCheckExecRunning) {
+        return
+      }
+      _profilesCancelingCheckExecRunning = true
+
+      try {
+        const cancelingProfiles = await reporter.documentStore.collection('profiles').find({
+          state: 'canceling'
+        })
+
+        for (const profile of cancelingProfiles) {
+          const runningReq = [...reporter.runningRequests.map.values()].find(v => v.req.context.profiling?.entity?._id === profile._id)
+          if (runningReq) {
+            runningReq.options.abortEmitter.emit('abort')
+          }
+        }
+      } catch (e) {
+        reporter.logger.warn('Failed to process cancelling profiles. No worry, it will retry next time.', e)
+      } finally {
+        _profilesCancelingCheckExecRunning = false
+      }
+    }
+
     profilesCleanupInterval = setInterval(profilesCleanupExec, reporter.options.profiler.cleanupInterval)
     profilesCleanupInterval.unref()
 
     fullModeDurationCheckInterval = setInterval(fullModeDurationCheckExec, reporter.options.profiler.fullModeDurationCheckInterval)
     fullModeDurationCheckInterval.unref()
 
+    profilesCancelingCheckInterval = setInterval(profilesCancelingCheckExec, reporter.options.profiler.cancelingCheckInterval)
+    profilesCancelingCheckInterval.unref()
+
     await reporter._profilesCleanup()
   })
 
@@ -416,6 +445,27 @@ module.exports = (reporter) => {
       clearInterval(fullModeDurationCheckInterval)
     }
 
+    if (profilesCancelingCheckInterval) {
+      clearInterval(profilesCancelingCheckInterval)
+    }
+
+    try {
+      const runningRequests = [...reporter.runningRequests.map.values()]
+      await reporter.documentStore.collection('profiles').update({
+        _id: {
+          $in: runningRequests.map(r => r.req.context.profiling?.entity?._id)
+        }
+      }, {
+        $set: {
+          state: 'error',
+          finishedOn: new Date(),
+          error: 'The server unexpectedly stopped during the report rendering.'
+        }
+      })
+    } catch (e) {
+      reporter.logger.warn('Failed to set error state to the running requests when closing.', e)
+    }
+
     for (const key of profilerOperationsChainsMap.keys()) {
       const profileAppendPromise = profilerOperationsChainsMap.get(key)
       if (profileAppendPromise) {
@@ -460,7 +510,7 @@ module.exports = (reporter) => {
       }
 
       const notFinishedProfiles = await reporter.documentStore.collection('profiles')
-        .find({ $or: [{ state: 'running' }, { state: 'queued' }] }, { _id: 1, timeout: 1, timestamp: 1 })
+        .find({ $or: [{ state: 'running' }, { state: 'queued' }, { state: 'canceling' }] }, { _id: 1, timeout: 1, timestamp: 1 })
         .toArray()
 
       for (const profile of notFinishedProfiles) {

package/lib/main/reporter.js

@@ -29,6 +29,7 @@ const Request = require('./request')
 const Response = require('../shared/response')
 const Profiler = require('./profiler')
 const semver = require('semver')
+const EventEmitter = require('events')
 let reportCounter = 0
 
 class MainReporter extends Reporter {
@@ -367,7 +368,10 @@ class MainReporter extends Reporter {
       throw new Error('Not initialized, you need to call jsreport.init().then before rendering')
     }
 
-    req = Object.assign({}, req)
+    options.abortEmitter = options.abortEmitter || new EventEmitter()
+
+    req = Request(req)
+
     req.context = Object.assign({}, req.context)
     req.context.rootId = req.context.rootId || this.generateRequestId()
     req.context.id = req.context.rootId
@@ -375,6 +379,8 @@ class MainReporter extends Reporter {
     req.context.startTimestamp = new Date().getTime()
     req.options = Object.assign({}, req.options)
 
+    this.runningRequests.register(req, options)
+
     let worker
     let workerAborted
     let dontCloseProcessing
@@ -415,10 +421,9 @@ class MainReporter extends Reporter {
         }, {
           timeout: this.getReportTimeout(req)
         })
-        req = result
+        req = Request(result)
       }
 
-      req = Request(req)
       options.onReqReady?.(req)
 
       // TODO: we will probably validate in the thread
@@ -491,6 +496,9 @@ class MainReporter extends Reporter {
       this._cleanProfileInRequest(req)
       throw err
     } finally {
+      options.abortEmitter.removeAllListeners('abort')
+
+      this.runningRequests.unregister(req, options)
       if (worker && !workerAborted && !dontCloseProcessing) {
         await worker.release(req)
       }

package/lib/main/store/documentStore.js

@@ -75,6 +75,11 @@ const DocumentStore = (options, validator, encryption) => {
         es.entityTypeDef = this.model.entityTypes[es.normalizedEntityTypeName]
         const entityType = es.entityTypeDef
 
+        if (entityType.name) {
+          entityType.name.index = true
+          entityType.name.length = 1024
+        }
+
         if (!entityType._id) {
           entityType._id = { type: 'Edm.String' }
 
@@ -94,7 +99,7 @@ const DocumentStore = (options, validator, encryption) => {
         }
 
         if (!entityType.shortid) {
-          entityType.shortid = { type: 'Edm.String' }
+          entityType.shortid = { type: 'Edm.String', index: true, length: 255 }
         }
 
         const referenceProperties = findReferencePropertiesInType(this.model, entityType)

package/lib/shared/reporter.js

@@ -9,7 +9,7 @@ const tempFilesHandler = require('./tempFilesHandler')
 const encryption = require('./encryption')
 const generateRequestId = require('./generateRequestId')
 const adminRequest = require('./adminRequest')
-const ReqStorage = require('./reqStorage')
+const RunningRequests = require('./runningRequests')
 
 class Reporter extends EventEmitter {
   constructor (options) {
@@ -19,7 +19,7 @@ class Reporter extends EventEmitter {
     this.Request = Request
     this.Response = (...args) => Response(this, ...args)
     this.adminRequest = adminRequest
-    this.reqStorage = ReqStorage(this)
+    this.runningRequests = RunningRequests(this)
 
     // since `reporter` instance will be used for other extensions,
     // it will quickly reach the limit of `10` listeners,

package/lib/shared/request.js

@@ -16,6 +16,10 @@ module.exports = (obj, parent) => {
     }
   })
 
+  if (obj.rawContent) {
+    request.rawContent = obj.rawContent
+  }
+
   request.template = extend(true, {}, obj.template)
 
   if (parent) {

package/lib/shared/runningRequests.js

@@ -0,0 +1,30 @@
+// holds running requests and provides additionally keyValueStore
+module.exports = (reporter) => {
+  const runningReqMap = new Map()
+
+  return {
+    keyValueStore: {
+      get: (key, req) => {
+        const keyValueMap = runningReqMap.get(req.context.rootId).keyValueMap
+        return keyValueMap.get(key)
+      },
+      set: (key, val, req) => {
+        const keyValueMap = runningReqMap.get(req.context.rootId).keyValueMap
+        keyValueMap.set(key, val)
+      }
+    },
+
+    map: runningReqMap,
+
+    register: (req, options) => {
+      runningReqMap.set(req.context.rootId, {
+        keyValueMap: new Map(),
+        req,
+        options
+      })
+    },
+    unregister: (req) => {
+      runningReqMap.delete(req.context.rootId)
+    }
+  }
+}

package/lib/worker/render/engineStream.js

@@ -58,14 +58,14 @@ module.exports = (reporter) => {
     if (proxy.templatingEngines) {
       proxy.templatingEngines.createStream = async (opts = {}) => {
         // limiting the number of temp files to avoid breaking server, otherwise I see no reason why having more than 1000 calls per req should be valid usecase
-        const counter = reporter.reqStorage.get('engine-stream-counter', req) || 0
+        const counter = reporter.runningRequests.keyValueStore.get('engine-stream-counter', req) || 0
         if (counter > 1000) {
           throw reporter.createError('Reached maximum limit of templatingEngine.createStream calls', {
             weak: true,
             statusCode: 400
           })
         }
-        reporter.reqStorage.set('engine-stream-counter', counter + 1, req)
+        reporter.runningRequests.keyValueStore.set('engine-stream-counter', counter + 1, req)
 
         req.context.engineStreamEnabled = true
 

package/lib/worker/render/render.js

@@ -110,7 +110,7 @@ module.exports = (reporter) => {
       request.context.id = reporter.generateRequestId()
     }
     if (parentReq == null) {
-      reporter.reqStorage.registerReq(request)
+      reporter.runningRequests.register(request)
     }
 
     const response = Response(reporter, request.context.id)
@@ -195,7 +195,7 @@ module.exports = (reporter) => {
     } finally {
       if (parentReq == null) {
         reporter.requestModulesCache.delete(request.context.rootId)
-        reporter.reqStorage.unregisterReq(request)
+        reporter.runningRequests.unregister(request)
       }
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "4.5.0",
+  "version": "4.6.1",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -63,7 +63,7 @@
     "lodash.omit": "4.5.0",
     "lru-cache": "4.1.1",
     "ms": "2.1.3",
-    "nanoid": "3.2.0",
+    "nanoid": "3.3.8",
     "nconf": "0.12.0",
     "node.extend.without.arrays": "1.1.6",
     "semver": "7.5.4",

package/lib/shared/reqStorage.js

@@ -1,20 +0,0 @@
-module.exports = (reporter) => {
-  const runningReqMap = new Map()
-
-  return {
-    set: (key, val, req) => {
-      const keyValueMap = runningReqMap.get(req.context.rootId)
-      keyValueMap.set(key, val)
-    },
-    get: (key, req) => {
-      const keyValueMap = runningReqMap.get(req.context.rootId)
-      return keyValueMap.get(key)
-    },
-    registerReq: (req) => {
-      runningReqMap.set(req.context.rootId, new Map())
-    },
-    unregisterReq: (req) => {
-      runningReqMap.delete(req.context.rootId)
-    }
-  }
-}