@jsreport/jsreport-core 4.0.1 → 4.2.0

package/lib/shared/tempFilesHandler.js

@@ -13,8 +13,34 @@ module.exports.ensureTempDirectoryExists = async function (tempDirectory) {
   }
 }
 
+module.exports.getTempFilePath = getTempFilePath
+
+module.exports.readTempFileSync = function readTempFileSync (tempDirectory, filename, opts = {}) {
+  const { pathToFile } = getTempFilePath(tempDirectory, filename)
+
+  const content = fs.readFileSync(pathToFile, opts)
+
+  return {
+    pathToFile,
+    filename,
+    content
+  }
+}
+
+module.exports.openTempFile = async function (tempDirectory, filenameFn, flags) {
+  const { pathToFile, filename } = getTempFilePath(tempDirectory, filenameFn)
+
+  const fileHandle = await fsAsync.open(pathToFile, flags)
+
+  return {
+    pathToFile,
+    filename,
+    fileHandle
+  }
+}
+
 module.exports.readTempFile = async function readTempFile (tempDirectory, filename, opts = {}) {
-  const pathToFile = path.join(tempDirectory, filename)
+  const { pathToFile } = getTempFilePath(tempDirectory, filename)
 
   const content = await fsAsync.readFile(pathToFile, opts)
 
@@ -25,51 +51,91 @@ module.exports.readTempFile = async function readTempFile (tempDirectory, filena
   }
 }
 
-module.exports.readTempFileStream = async function readTempFileStream (tempDirectory, filename, opts = {}) {
-  const pathToFile = path.join(tempDirectory, filename)
+module.exports.readTempFileStream = function readTempFileStream (tempDirectory, filename, opts = {}) {
+  const { pathToFile } = getTempFilePath(tempDirectory, filename)
 
-  return new Promise((resolve) => {
-    const stream = fs.createReadStream(pathToFile, opts)
+  const stream = fs.createReadStream(pathToFile, opts)
 
-    resolve({
-      pathToFile,
-      filename,
-      stream
-    })
-  })
+  return {
+    pathToFile,
+    filename,
+    stream
+  }
+}
+
+module.exports.writeTempFileSync = function writeTempFileSync (tempDirectory, filenameOrFn, content, opts = {}) {
+  return writeFileSync(tempDirectory, filenameOrFn, content, opts)
 }
 
-module.exports.writeTempFile = async function writeTempFile (tempDirectory, filenameFn, content, opts = {}) {
-  return writeFile(tempDirectory, filenameFn, content, opts)
+module.exports.writeTempFile = async function writeTempFile (tempDirectory, filenameOrFn, content, opts = {}) {
+  return writeFile(tempDirectory, filenameOrFn, content, opts)
 }
 
-module.exports.writeTempFileStream = async function writeTempFileStream (tempDirectory, filenameFn, opts = {}) {
-  return writeFile(tempDirectory, filenameFn, undefined, opts, true)
+module.exports.writeTempFileStream = async function writeTempFileStream (tempDirectory, filenameOrFn, opts = {}) {
+  return writeFile(tempDirectory, filenameOrFn, undefined, opts, true)
 }
 
-async function writeFile (tempDirectory, filenameFn, content, opts, asStream = false) {
-  const filename = filenameFn(uuidv4())
+module.exports.copyFileToTempFile = async function copyFileToTempFile (tempDirectory, srcFilePath, destFilenameOrFn, mode) {
+  const { pathToFile, filename } = getTempFilePath(tempDirectory, destFilenameOrFn)
 
-  if (filename == null || filename === '') {
-    throw new Error('No valid filename was returned from filenameFn')
+  await fsAsync.mkdir(tempDirectory, {
+    recursive: true
+  })
+
+  await fsAsync.copyFile(srcFilePath, pathToFile, mode)
+
+  return {
+    pathToFile,
+    filename
+  }
+}
+
+function getTempFilePath (tempDirectory, filenameOrFn) {
+  const filenameResult = typeof filenameOrFn === 'function' ? filenameOrFn(uuidv4()) : filenameOrFn
+
+  if (filenameResult == null || filenameResult === '') {
+    throw new Error('No valid filename')
+  }
+
+  const pathToFile = path.isAbsolute(filenameResult) ? filenameResult : path.join(tempDirectory, filenameResult)
+  const filename = path.basename(pathToFile)
+
+  return {
+    pathToFile,
+    filename
+  }
+}
+
+function writeFileSync (tempDirectory, filenameOrFn, content, opts) {
+  const { pathToFile, filename } = getTempFilePath(tempDirectory, filenameOrFn)
+
+  fs.mkdirSync(tempDirectory, {
+    recursive: true
+  })
+
+  fs.writeFileSync(pathToFile, content, opts)
+
+  return {
+    pathToFile,
+    filename
   }
+}
 
-  const pathToFile = path.join(tempDirectory, filename)
+async function writeFile (tempDirectory, filenameOrFn, content, opts, asStream = false) {
+  const { pathToFile, filename } = getTempFilePath(tempDirectory, filenameOrFn)
 
   await fsAsync.mkdir(tempDirectory, {
     recursive: true
   })
 
   if (asStream === true) {
-    return new Promise((resolve) => {
-      const stream = fs.createWriteStream(pathToFile, opts)
-
-      resolve({
-        pathToFile,
-        filename,
-        stream
-      })
-    })
+    const stream = fs.createWriteStream(pathToFile, opts)
+
+    return {
+      pathToFile,
+      filename,
+      stream
+    }
   } else {
     await fsAsync.writeFile(pathToFile, content, opts)
 

package/lib/worker/defaultProxyExtend.js

@@ -12,10 +12,8 @@ module.exports = (reporter) => (proxy, req) => {
       context: {}
     }, req)
 
-    return {
-      content: res.content,
-      meta: res.meta
-    }
+    // expose the response api
+    return res
   }
 
   proxy.documentStore = {

package/lib/worker/render/engineStream.js

@@ -0,0 +1,97 @@
+const bytes = require('bytes')
+const { Readable } = require('stream')
+/*
+This adds jsreport.templatingEngines.createStream to the helpers proxy allowing to write giant texts to output
+which would otherwise hit nodejs max string size limit.
+
+Example usage
+===================
+async function myEach(items, options) {
+    const stream = await jsreport.templatingEngines.createStream()
+    for (let i = 0; i < items.length; i++) {
+        await stream.write(options.fn())
+    }
+    return await stream.toResult()
+}
+*/
+
+module.exports = (reporter) => {
+  reporter.afterTemplatingEnginesExecutedListeners.add('streamedEach', async (req, res) => {
+    if (req.context.engineStreamEnabled !== true) {
+      return
+    }
+
+    const content = (await res.output.getBuffer()).toString()
+
+    const matches = [...content.matchAll(/{#stream ([^{}]{0,500})}/g)]
+
+    async function * transform () {
+      if (matches.length) {
+        yield content.substring(0, matches[0].index)
+
+        for (let i = 0; i < matches.length; i++) {
+          const { stream } = reporter.readTempFileStream(matches[i][1])
+
+          for await (const content of stream) {
+            yield content
+          }
+
+          if (i < matches.length - 1) {
+            yield content.substring(matches[i].index + matches[i][0].length, matches[i + 1].index)
+          } else {
+            yield content.substring(matches[i].index + matches[i][0].length)
+          }
+        }
+      } else {
+        yield content
+      }
+    }
+
+    await res.output.update(Readable.from(transform()))
+  })
+
+  reporter.extendProxy((proxy, req, {
+    runInSandbox,
+    context,
+    getTopLevelFunctions
+  }) => {
+    if (proxy.templatingEngines) {
+      proxy.templatingEngines.createStream = async (opts = {}) => {
+        // limiting the number of temp files to avoid breaking server, otherwise I see no reason why having more than 1000 calls per req should be valid usecase
+        const counter = reporter.reqStorage.get('engine-stream-counter', req) || 0
+        if (counter > 1000) {
+          throw reporter.createError('Reached maximum limit of templatingEngine.createStream calls', {
+            weak: true,
+            statusCode: 400
+          })
+        }
+        reporter.reqStorage.set('engine-stream-counter', counter + 1, req)
+
+        req.context.engineStreamEnabled = true
+
+        const bufferSize = bytes(opts.bufferSize || '10mb')
+        let buf = ''
+
+        const { fileHandle, filename } = await reporter.openTempFile((uuid) => `${uuid}.stream`, 'a')
+        proxy.templatingEngines.addFinishListener(() => fileHandle.close().catch((e) => reporter.logger.error('Failed to close temp file handle', e, req)))
+
+        return {
+          write: async (text) => {
+            const realText = await proxy.templatingEngines.waitForAsyncHelper(text)
+
+            buf += realText
+
+            if (buf.length > bufferSize) {
+              await fileHandle.appendFile(buf)
+              buf = ''
+            }
+          },
+          toResult: async () => {
+            await fileHandle.appendFile(buf)
+            return `{#stream ${filename}}`
+          }
+        }
+      }
+    }
+  })
+}

package/lib/worker/render/executeEngine.js

@@ -16,6 +16,7 @@ module.exports = (reporter) => {
 
   const executionFnParsedParamsMap = new Map()
   const executionAsyncResultsMap = new Map()
+  const executionFinishListenersMap = new Map()
 
   const templatingEnginesEvaluate = async (mainCall, { engine, content, helpers, data }, { entity, entitySet }, req) => {
     const engineImpl = reporter.extensionsManager.engines.find((e) => e.name === engine)
@@ -47,6 +48,7 @@ module.exports = (reporter) => {
       }
 
       executionAsyncResultsMap.delete(executionId)
+      executionFinishListenersMap.delete(executionId)
     }
   }
 
@@ -94,6 +96,11 @@ module.exports = (reporter) => {
           return Promise.all([...asyncResultMap.keys()].map((k) => asyncResultMap.get(k)))
         }
       },
+      addFinishListener: (fn) => {
+        if (executionFinishListenersMap.has(context.__executionId)) {
+          executionFinishListenersMap.get(context.__executionId).add('finish', fn)
+        }
+      },
       createAsyncHelperResult: (v) => {
         const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
         const asyncResultId = nanoid(7)
@@ -190,9 +197,12 @@ module.exports = (reporter) => {
       context.__executionId = executionId
 
       executionAsyncResultsMap.set(executionId, asyncResultMap)
+      executionFinishListenersMap.set(executionId, reporter.createListenerCollection())
       executionFnParsedParamsMap.get(req.context.id).get(executionFnParsedParamsKey).resolve({ require, console, topLevelFunctions, context })
 
-      const key = `template:${content}:${engine.name}`
+      const key = engine.buildTemplateCacheKey
+        ? engine.buildTemplateCacheKey({ content }, req)
+        : `template:${content}:${engine.name}`
 
       if (!templatesCache.has(key)) {
         try {
@@ -208,7 +218,7 @@ module.exports = (reporter) => {
 
       for (const h of Object.keys(topLevelFunctions)) {
         // extra wrapping for enhance the error with the helper name
-        wrappedTopLevelFunctions[h] = wrapHelperForHelperNameWhenError(topLevelFunctions[h], h)
+        wrappedTopLevelFunctions[h] = wrapHelperForHelperNameWhenError(topLevelFunctions[h], h, () => executionFnParsedParamsMap.has(req.context.id))
 
         if (engine.getWrappingHelpersEnabled && engine.getWrappingHelpersEnabled(req) === false) {
           wrappedTopLevelFunctions[h] = engine.wrapHelper(wrappedTopLevelFunctions[h], { context })
@@ -221,19 +231,33 @@ module.exports = (reporter) => {
 
       const resolvedResultsMap = new Map()
 
-      while (asyncResultMap.size > 0) {
-        await Promise.all([...asyncResultMap.keys()].map(async (k) => {
-          resolvedResultsMap.set(k, `${await asyncResultMap.get(k)}`)
-          asyncResultMap.delete(k)
+      // we need to use the cloned map, becuase there can be a waitForAsyncHelper pending that needs the asyncResultMap values
+      const clonedMap = new Map(asyncResultMap)
+      while (clonedMap.size > 0) {
+        await Promise.all([...clonedMap.keys()].map(async (k) => {
+          resolvedResultsMap.set(k, `${await clonedMap.get(k)}`)
+          clonedMap.delete(k)
         }))
       }
+      asyncResultMap.clear()
 
       while (contentResult.includes('{#asyncHelperResult')) {
         contentResult = contentResult.replace(/{#asyncHelperResult ([^{}]+)}/g, (str, p1) => {
           const asyncResultId = p1
+          // this can happen if a child jsreport.templatingEngines.evaluate receives an async value from outer scope
+          // because every evaluate uses a unique map of async resuts
+          // example is the case when component receives as a value async thing
+          // instead of returning "undefined" we let the outer eval to do the replace
+          if (!resolvedResultsMap.has(asyncResultId)) {
+            // returning asyncUnresolvedHelperResult just to avoid endless loop, after replace we put it back to asyncHelperResult
+            return `{#asyncUnresolvedHelperResult ${asyncResultId}}`
+          }
           return `${resolvedResultsMap.get(asyncResultId)}`
         })
       }
+      contentResult = contentResult.replace(/asyncUnresolvedHelperResult/g, 'asyncHelperResult')
+
+      await executionFinishListenersMap.get(context.__executionId).fire()
 
       return {
         // handlebars escapes single brackets before execution to prevent errors on {#asset}
@@ -355,7 +379,7 @@ module.exports = (reporter) => {
     }
   }
 
-  function wrapHelperForHelperNameWhenError (fn, helperName) {
+  function wrapHelperForHelperNameWhenError (fn, helperName, isMainEvalStillRunningFn) {
     return function (...args) {
       let fnResult
 
@@ -373,6 +397,10 @@ module.exports = (reporter) => {
       }
 
       return fnResult.catch((asyncError) => {
+        if (!isMainEvalStillRunningFn()) {
+          // main exec already finished on some error, we just ignore errors of the hanging async calls
+          return
+        }
         throw getEnhancedHelperError(asyncError)
       })
     }

package/lib/worker/render/profiler.js

@@ -72,7 +72,15 @@ class Profiler {
     }
 
     if (m.doDiffs !== false && req.context.profiling.mode === 'full' && (m.type === 'operationStart' || m.type === 'operationEnd')) {
-      let content = res.content
+      let originalResContent = res.content
+
+      // if content is empty assume null to keep old logic working without major changes
+      // (here and in studio)
+      if (originalResContent != null && originalResContent.length === 0) {
+        originalResContent = null
+      }
+
+      let content = originalResContent
 
       if (content != null) {
         if (content.length > this.reporter.options.profiler.maxDiffSize) {
@@ -82,12 +90,12 @@ class Profiler {
         } else {
           if (isbinaryfile(content)) {
             content = {
-              content: res.content.toString('base64'),
+              content: originalResContent.toString('base64'),
               encoding: 'base64'
             }
           } else {
             content = {
-              content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', res.content.toString(), 0),
+              content: createPatch('res', req.context.profiling.resLastVal ? req.context.profiling.resLastVal.toString() : '', originalResContent.toString(), 0),
               encoding: 'diff'
             }
           }
@@ -107,7 +115,7 @@ class Profiler {
         m.req.diff = createPatch('req', req.context.profiling.reqLastVal || '', stringifiedReq, 0)
       }
 
-      req.context.profiling.resLastVal = (res.content == null || isbinaryfile(res.content) || content.tooLarge) ? null : res.content.toString()
+      req.context.profiling.resLastVal = (originalResContent == null || isbinaryfile(originalResContent) || content.tooLarge) ? null : originalResContent.toString()
       req.context.profiling.resMetaLastVal = stringifiedResMeta
       req.context.profiling.reqLastVal = stringifiedReq
     }

package/lib/worker/render/render.js

@@ -3,11 +3,10 @@
  *
  * Orchestration of the rendering process
  */
-const { Readable } = require('stream')
 const extend = require('node.extend.without.arrays')
 const ExecuteEngine = require('./executeEngine')
 const Request = require('../../shared/request')
-const generateRequestId = require('../../shared/generateRequestId')
+const Response = require('../../shared/response')
 const resolveReferences = require('./resolveReferences.js')
 const moduleHelper = require('./moduleHelper')
 
@@ -60,8 +59,7 @@ module.exports = (reporter) => {
     reporter.logger.debug(`Rendering engine ${engine.name}`, request)
 
     const engineRes = await executeEngine(engine, request)
-
-    response.content = Buffer.from(engineRes.content != null ? engineRes.content : '')
+    await response.output.update(Buffer.from(engineRes.content != null ? engineRes.content : ''))
 
     reporter.profiler.emit({
       type: 'operationEnd',
@@ -93,6 +91,7 @@ module.exports = (reporter) => {
     reporter.logger.debug('Executing recipe ' + request.template.recipe, request)
 
     await recipe.execute(request, response)
+
     reporter.profiler.emit({
       type: 'operationEnd',
       operationId: recipeProfilerEvent.operationId
@@ -101,22 +100,24 @@ module.exports = (reporter) => {
 
   async function afterRender (reporter, request, response) {
     await reporter.afterRenderListeners.fire(request, response)
-
-    response.stream = Readable.from(response.content)
-    response.result = response.stream
-
     return response
   }
 
   return async (req, parentReq) => {
     const request = Request(req, parentReq)
-    const response = { meta: {} }
+
+    if (request.context.id == null) {
+      request.context.id = reporter.generateRequestId()
+    }
+    if (parentReq == null) {
+      reporter.reqStorage.registerReq(request)
+    }
+
+    const response = Response(reporter, request.context.id)
+
     let renderStartProfilerEvent
-    try {
-      if (request.context.id == null) {
-        request.context.id = generateRequestId()
-      }
 
+    try {
       renderStartProfilerEvent = await reporter.profiler.renderStart(request, parentReq, response)
       request.data = resolveReferences(request.data) || {}
 
@@ -194,6 +195,7 @@ module.exports = (reporter) => {
     } finally {
       if (parentReq == null) {
         reporter.requestModulesCache.delete(request.context.rootId)
+        reporter.reqStorage.unregisterReq(request)
       }
     }
   }

package/lib/worker/reporter.js

@@ -10,6 +10,7 @@ const Reporter = require('../shared/reporter')
 const BlobStorage = require('./blobStorage.js')
 const Render = require('./render/render')
 const Profiler = require('./render/profiler.js')
+const engineStream = require('./render/engineStream.js')
 
 class WorkerReporter extends Reporter {
   constructor (workerData, executeMain) {
@@ -79,6 +80,8 @@ class WorkerReporter extends Reporter {
       execute: htmlRecipe
     })
 
+    engineStream(this)
+
     await this.initializeListeners.fire()
 
     if (!this._lockedDown && this.options.trustUserCode === false) {
@@ -169,8 +172,8 @@ class WorkerReporter extends Reporter {
     return proxyInstance
   }
 
-  render (req, parentReq) {
-    return this._render(req, parentReq)
+  async render (req, parentReq) {
+    return await this._render(req, parentReq)
   }
 
   async executeMainAction (actionName, data, req) {
@@ -221,17 +224,8 @@ class WorkerReporter extends Reporter {
 
   _registerRenderAction () {
     this.registerWorkerAction('render', async (data, req) => {
-      const res = await this.render(req)
-
-      const sharedBuf = new SharedArrayBuffer(res.content.byteLength)
-      const buf = Buffer.from(sharedBuf)
-
-      res.content.copy(buf)
-
-      return {
-        meta: res.meta,
-        content: buf
-      }
+      const response = await this._render(req)
+      return response.serialize()
     })
   }