@jsreport/jsreport-core 4.4.0 → 4.5.0

package/README.md

@@ -282,11 +282,24 @@ jsreport.documentStore.collection('templates')
 
 ## Changelog
 
+### 4.5.0
+
+- fix blobStorage failing to save reports bigger than 1gb
+- decrease default value of `options.profile.maxDiffSize` to `10mb`
+- fix logs metadata for main logs
+- fix support for using of async helper that returns a value other than string in template engines
+- improve support for `jsreport.templateEngines.waitForAsyncHelpers` when used in async helper
+
+### 4.4.1
+
+- fix timestamp shown in logging
+
 ### 4.4.0
 
 - trigger async report using header to avoid allocating worker
 - fix require('..') and require('.') in sandbox
 - fix component rendering in loop with async work on helpers
+- update @jsreport/ses to 1.1.0
 
 ### 4.3.1
 

package/lib/main/blobStorage/mainActions.js

@@ -3,13 +3,22 @@ module.exports = (reporter) => {
     const localReq = reporter.Request(originalReq)
     const res = await reporter.blobStorage.read(spec.blobName, localReq)
 
-    return res.toString('base64')
+    if (res.length < 1000 * 1000 * 10) {
+      return { content: res.toString('base64') }
+    }
+
+    const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.blob`, res)
+    return { pathToFile }
   })
 
   reporter.registerMainAction('blobStorage.write', async (spec, originalReq) => {
     const localReq = reporter.Request(originalReq)
+    if (spec.content) {
+      return await reporter.blobStorage.write(spec.blobName, Buffer.from(spec.content, 'base64'), localReq)
+    }
 
-    return await reporter.blobStorage.write(spec.blobName, Buffer.from(spec.content, 'base64'), localReq)
+    const { content } = await reporter.readTempFile(spec.pathToFile)
+    return await reporter.blobStorage.write(spec.blobName, content, localReq)
   })
 
   reporter.registerMainAction('blobStorage.remove', async (spec, originalReq) => {

package/lib/main/createDefaultLoggerFormat.js

@@ -4,8 +4,17 @@ const winston = require('winston')
 
 module.exports = (options = {}) => {
   return winston.format((info) => {
-    const { level, message, ...meta } = info
-    info[MESSAGE] = `${options.timestamp === true ? `${new Date().toISOString()} - ` : ''}${level}: ${info.userLevel === true ? colors.cyan(message) : message}`
+    const { level, message, timestamp, ...meta } = info
+    let logDate
+
+    if (timestamp == null) {
+      logDate = new Date()
+      info.timestamp = logDate.getTime()
+    } else {
+      logDate = new Date(timestamp)
+    }
+
+    info[MESSAGE] = `${options.timestamp === true ? `${logDate.toISOString()} - ` : ''}${level}: ${info.userLevel === true ? colors.cyan(message) : message}`
 
     const metaKeys = Object.keys(meta)
 

package/lib/main/createNormalizeMetaLoggerFormat.js

@@ -1,19 +1,32 @@
+const omit = require('lodash.omit')
 const winston = require('winston')
+const { LEVEL, MESSAGE, SPLAT } = require('triple-beam')
 const normalizeMetaFromLogs = require('../shared/normalizeMetaFromLogs')
 
 module.exports = () => {
   return winston.format((info) => {
-    const { level, message, ...meta } = info
-    const newMeta = normalizeMetaFromLogs(level, message, meta)
+    const { level, message, timestamp: _timestamp, ...meta } = info
 
-    if (newMeta != null) {
-      return {
-        level,
-        message,
-        ...newMeta
+    const symbolProps = [LEVEL, MESSAGE, SPLAT]
+    const originalSymbolProps = {}
+
+    for (const symbolProp of symbolProps) {
+      if (info[symbolProp] != null) {
+        originalSymbolProps[symbolProp] = info[symbolProp]
       }
     }
 
-    return info
+    const timestamp = _timestamp ?? new Date().getTime()
+
+    const targetMeta = omit(meta, symbolProps)
+    const newMeta = normalizeMetaFromLogs(level, message, timestamp, targetMeta)
+
+    return {
+      level,
+      message,
+      timestamp,
+      ...originalSymbolProps,
+      ...newMeta
+    }
   })
 }

package/lib/main/logger.js

@@ -171,6 +171,7 @@ function configureLogger (logger, _transports) {
     if (options.silent) {
       continue
     }
+
     const transportInstance = new TransportClass(options)
 
     const existingTransport = logger.transports.find((t) => t.name === transportInstance.name)
@@ -238,7 +239,6 @@ class DebugTransport extends Transport {
 
     this.format = options.format || winston.format.combine(
       winston.format.colorize(),
-      normalizeMetaLoggerFormat(),
       defaultLoggerFormat()
     )
 

package/lib/main/optionsSchema.js

@@ -196,7 +196,7 @@ module.exports.getRootSchemaOptions = () => ({
         maxDiffSize: {
           type: ['string', 'number'],
           '$jsreport-acceptsSize': true,
-          default: '50mb'
+          default: '10mb'
         }
       }
     }

package/lib/shared/normalizeMetaFromLogs.js

@@ -1,6 +1,8 @@
 const omit = require('lodash.omit')
 
-module.exports = (level, msg, meta) => {
+module.exports = (level, msg, timestamp, meta) => {
+  let result = meta
+
   // detecting if meta is jsreport request object
   if (meta != null && meta.context) {
     meta.context.logs = meta.context.logs || []
@@ -8,7 +10,7 @@ module.exports = (level, msg, meta) => {
     meta.context.logs.push({
       level: level,
       message: msg,
-      timestamp: meta.timestamp || new Date().getTime()
+      timestamp
     })
 
     // TODO adding cancel looks bad, its before script is adding req.cancel()
@@ -23,8 +25,8 @@ module.exports = (level, msg, meta) => {
       newMeta.id = meta.context.id
     }
 
-    return newMeta
+    result = newMeta
   }
 
-  return meta
+  return result != null && Object.keys(result).length > 0 ? result : null
 }

package/lib/shared/response.js

@@ -3,7 +3,6 @@ const fs = require('fs/promises')
 const { Readable } = require('stream')
 const { pipeline } = require('stream/promises')
 const path = require('path')
-const isArrayBufferView = require('util').types.isArrayBufferView
 
 module.exports = (reporter, requestId, obj) => {
   let outputImpl = new BufferOutput(reporter)
@@ -44,7 +43,7 @@ module.exports = (reporter, requestId, obj) => {
       async getSize () { return outputImpl.getSize() },
       async writeToTempFile (...args) { return outputImpl.writeToTempFile(...args) },
       async update (bufOrStreamOrPath) {
-        if (Buffer.isBuffer(bufOrStreamOrPath) || isArrayBufferView(bufOrStreamOrPath)) {
+        if (Buffer.isBuffer(bufOrStreamOrPath) || ArrayBuffer.isView(bufOrStreamOrPath)) {
           return outputImpl.setBuffer(bufOrStreamOrPath)
         }
 

package/lib/worker/blobStorage.js

@@ -1,17 +1,31 @@
-module.exports = (executeMainAction) => {
+module.exports = (executeMainAction, { writeTempFile, readTempFile }) => {
   return {
     async read (blobName, req) {
       const r = await executeMainAction('blobStorage.read', {
         blobName
       }, req)
-      return Buffer.from(r, 'base64')
+
+      if (r.content) {
+        return Buffer.from(r.content, 'base64')
+      }
+
+      const { content } = await readTempFile(r.pathToFile)
+      return content
     },
 
-    write (blobName, content, req) {
-      return executeMainAction('blobStorage.write', {
-        blobName,
-        content: Buffer.from(content).toString('base64')
-      }, req)
+    async write (blobName, content, req) {
+      const message = {
+        blobName
+      }
+
+      if (content.length < 1000 * 1000 * 10) {
+        message.content = Buffer.from(content).toString('base64')
+      } else {
+        const { pathToFile } = await writeTempFile((uuid) => `${uuid}.blob`, content)
+        message.pathToFile = pathToFile
+      }
+
+      return executeMainAction('blobStorage.write', message, req)
     },
 
     remove (blobName, req) {

package/lib/worker/logger.js

@@ -35,7 +35,7 @@ function logFn (level, profiler, ...args) {
     message: util.format.apply(util, msgArgs)
   }
 
-  const meta = normalizeMetaFromLogs(level, log.message, lastArg)
+  const meta = normalizeMetaFromLogs(level, log.message, log.timestamp, lastArg)
 
   if (meta != null) {
     log.meta = meta

package/lib/worker/render/executeEngine.js

@@ -7,8 +7,10 @@
  */
 const LRU = require('lru-cache')
 const { nanoid } = require('nanoid')
+const { AsyncLocalStorage } = require('node:async_hooks')
 
 module.exports = (reporter) => {
+  const helperCallerAsyncLocalStorage = new AsyncLocalStorage()
   const templatesCache = LRU(reporter.options.sandbox.cache)
   let systemHelpersCache
 
@@ -17,7 +19,6 @@ module.exports = (reporter) => {
   const contextExecutionChainMap = new Map()
   const executionFnParsedParamsMap = new Map()
   const executionAsyncResultsMap = new Map()
-  const executionAsyncCallChainMap = new Map()
   const executionFinishListenersMap = new Map()
 
   const templatingEnginesEvaluate = async (mainCall, { engine, content, helpers, data }, { entity, entitySet }, req) => {
@@ -50,7 +51,6 @@ module.exports = (reporter) => {
       }
 
       executionAsyncResultsMap.delete(executionId)
-      executionAsyncCallChainMap.delete(executionId)
       executionFinishListenersMap.delete(executionId)
     }
   }
@@ -59,6 +59,8 @@ module.exports = (reporter) => {
     return templatingEnginesEvaluate(true, executionInfo, entityInfo, req)
   }
 
+  reporter.closeListeners.add('engineHelperCaller', () => helperCallerAsyncLocalStorage.disable())
+
   reporter.extendProxy((proxy, req, {
     runInSandbox,
     context,
@@ -90,10 +92,23 @@ module.exports = (reporter) => {
             const matchedPart = matchResult[0]
             const asyncResultId = matchResult[1]
             const result = await asyncResultMap.get(asyncResultId)
-            content = `${content.slice(0, matchResult.index)}${result}${content.slice(matchResult.index + matchedPart.length)}`
+            const isFullMatch = content === matchedPart
+
+            if (typeof result !== 'string' && isFullMatch) {
+              // this allows consuming async helper that returns a value other than string
+              // like an async helper that returns object and it is received as
+              // parameter of another helper
+              content = result
+            } else {
+              content = `${content.slice(0, matchResult.index)}${result}${content.slice(matchResult.index + matchedPart.length)}`
+            }
           }
 
-          matchResult = content.match(asyncHelperResultRegExp)
+          if (typeof content === 'string') {
+            matchResult = content.match(asyncHelperResultRegExp)
+          } else {
+            matchResult = null
+          }
         } while (matchResult != null)
 
         return content
@@ -103,13 +118,14 @@ module.exports = (reporter) => {
         const executionId = executionChain[executionChain.length - 1]
 
         if (executionId != null && executionAsyncResultsMap.has(executionId)) {
-          const asyncCallChainSet = executionAsyncCallChainMap.get(executionId)
-          const lastAsyncCall = [...asyncCallChainSet].pop()
-
           const asyncResultMap = executionAsyncResultsMap.get(executionId)
-          // we should exclude the last async call because if it exists it represents the parent
-          // async call that called .waitForAsyncHelpers, it is not going to be resolved at this point
-          const targetAsyncResultKeys = [...asyncResultMap.keys()].filter((key) => key !== lastAsyncCall)
+
+          const callerId = helperCallerAsyncLocalStorage.getStore()
+
+          // we must exclude the caller helper because if it exists it represents some parent
+          // sync/async call that called .waitForAsyncHelpers, it is not going to be resolved at this point
+          // so we should skip it in order for the execution to not hang
+          const targetAsyncResultKeys = [...asyncResultMap.keys()].filter((key) => key !== callerId)
 
           return Promise.all(targetAsyncResultKeys.map((k) => asyncResultMap.get(k)))
         }
@@ -157,7 +173,6 @@ module.exports = (reporter) => {
     } finally {
       executionFnParsedParamsMap.delete(req.context.id)
       executionAsyncResultsMap.delete(executionId)
-      executionAsyncCallChainMap.delete(executionId)
     }
   }
 
@@ -216,7 +231,6 @@ module.exports = (reporter) => {
     const executionFn = async ({ require, console, topLevelFunctions, context }) => {
       sandboxId = context.__sandboxId
       const asyncResultMap = new Map()
-      const asyncCallChainSet = new Set()
 
       if (!contextExecutionChainMap.has(sandboxId)) {
         contextExecutionChainMap.set(sandboxId, [])
@@ -225,7 +239,6 @@ module.exports = (reporter) => {
       contextExecutionChainMap.get(sandboxId).push(executionId)
 
       executionAsyncResultsMap.set(executionId, asyncResultMap)
-      executionAsyncCallChainMap.set(executionId, asyncCallChainSet)
       executionFinishListenersMap.set(executionId, reporter.createListenerCollection())
       executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions, context })
 
@@ -253,7 +266,7 @@ module.exports = (reporter) => {
           if (engine.getWrappingHelpersEnabled && engine.getWrappingHelpersEnabled(req) === false) {
             wrappedTopLevelFunctions[h] = engine.wrapHelper(wrappedTopLevelFunctions[h], { context })
           } else {
-            wrappedTopLevelFunctions[h] = wrapHelperForAsyncSupport(wrappedTopLevelFunctions[h], asyncResultMap, asyncCallChainSet)
+            wrappedTopLevelFunctions[h] = wrapHelperForAsyncSupport(wrappedTopLevelFunctions[h], h, asyncResultMap)
           }
         }
 
@@ -269,7 +282,6 @@ module.exports = (reporter) => {
 
           await Promise.all(keysEvaluated.map(async (k) => {
             const result = await clonedMap.get(k)
-            asyncCallChainSet.delete(k)
             resolvedResultsMap.set(k, `${result}`)
             clonedMap.delete(k)
           }))
@@ -401,20 +413,25 @@ module.exports = (reporter) => {
     }
   }
 
-  function wrapHelperForAsyncSupport (fn, asyncResultMap, asyncCallChainSet) {
+  function wrapHelperForAsyncSupport (fn, helperName, asyncResultMap) {
     return function (...args) {
-      // important to call the helper with the current this to preserve the same behavior
-      const fnResult = fn.call(this, ...args)
+      const resultId = nanoid(7)
+
+      let fnResult
+
+      // make the result id available for all calls inside the helper
+      helperCallerAsyncLocalStorage.run(resultId, () => {
+        // important to call the helper with the current this to preserve the same behavior
+        fnResult = fn.call(this, ...args)
+      })
 
       if (fnResult == null || typeof fnResult.then !== 'function') {
         return fnResult
       }
 
-      const asyncResultId = nanoid(7)
-      asyncResultMap.set(asyncResultId, fnResult)
-      asyncCallChainSet.add(asyncResultId)
+      asyncResultMap.set(resultId, fnResult)
 
-      return `{#asyncHelperResult ${asyncResultId}}`
+      return `{#asyncHelperResult ${resultId}}`
     }
   }
 

package/lib/worker/reporter.js

@@ -54,7 +54,7 @@ class WorkerReporter extends Reporter {
     await this.extensionsManager.init()
 
     this.documentStore = DocumentStore(this._documentStoreData, this.executeMainAction.bind(this))
-    this.blobStorage = BlobStorage(this.executeMainAction.bind(this))
+    this.blobStorage = BlobStorage(this.executeMainAction.bind(this), { writeTempFile: this.writeTempFile.bind(this), readTempFile: this.readTempFile.bind(this) })
 
     this.addRequestContextMetaConfig('rootId', { sandboxReadOnly: true })
     this.addRequestContextMetaConfig('id', { sandboxReadOnly: true })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jsreport/jsreport-core",
-  "version": "4.4.0",
+  "version": "4.5.0",
   "description": "javascript based business reporting",
   "keywords": [
     "report",
@@ -78,7 +78,7 @@
   },
   "devDependencies": {
     "@node-rs/jsonwebtoken": "0.2.0",
-    "jsdom": "17.0.0",
+    "jsdom": "24.1.0",
     "mocha": "10.1.0",
     "should": "13.2.3",
     "standard": "16.0.4",