@jsreport/jsreport-core 3.6.1 → 3.8.0

package/lib/main/logger.js

@@ -5,10 +5,12 @@ const winston = require('winston')
 const Transport = require('winston-transport')
 const debug = require('debug')('jsreport')
 const createDefaultLoggerFormat = require('./createDefaultLoggerFormat')
-const normalizeMetaFromLogs = require('../shared/normalizeMetaFromLogs')
+const createNormalizeMetaLoggerFormat = require('./createNormalizeMetaLoggerFormat')
+const Request = require('./request')
 
 const defaultLoggerFormat = createDefaultLoggerFormat()
 const defaultLoggerFormatWithTimestamp = createDefaultLoggerFormat({ timestamp: true })
+const normalizeMetaLoggerFormat = createNormalizeMetaLoggerFormat()
 
 function createLogger () {
   const logger = winston.createLogger(getConfigurationOptions())
@@ -180,25 +182,30 @@ function configureLogger (logger, _transports) {
     logger.add(transportInstance)
   }
 
-  logger.__configured__ = true
-}
+  const originalLog = logger.log
 
-function getConfigurationOptions () {
-  const normalizeMeta = winston.format((info) => {
-    const { level, message, ...meta } = info
-    const newMeta = normalizeMetaFromLogs(level, message, meta)
-
-    if (newMeta != null) {
-      return {
-        level,
-        message,
-        ...newMeta
-      }
+  // we want to normalize the req has httpIncomingRequest early
+  // otherwise we will get serialization issues when trying to
+  // log http.IncomingRequest
+  logger.log = function (level, msg, ...splat) {
+    const [meta] = splat
+
+    if (
+      typeof meta === 'object' &&
+      meta !== null &&
+      meta.context != null &&
+      meta.socket != null
+    ) {
+      splat[0] = Request(meta)
     }
 
-    return info
-  })
+    return originalLog.call(this, level, msg, ...splat)
+  }
+
+  logger.__configured__ = true
+}
 
+function getConfigurationOptions () {
   return {
     levels: {
       error: 0,
@@ -207,7 +214,7 @@ function getConfigurationOptions () {
       debug: 3
     },
     format: winston.format.combine(
-      normalizeMeta(),
+      normalizeMetaLoggerFormat(),
       defaultLoggerFormatWithTimestamp()
     ),
     transports: [new DebugTransport()]
@@ -231,6 +238,7 @@ class DebugTransport extends Transport {
 
     this.format = options.format || winston.format.combine(
       winston.format.colorize(),
+      normalizeMetaLoggerFormat(),
       defaultLoggerFormat()
     )
 

package/lib/main/profiler.js

@@ -1,4 +1,5 @@
 const EventEmitter = require('events')
+const winston = require('winston')
 const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
 const fs = require('fs/promises')
@@ -20,20 +21,44 @@ module.exports = (reporter) => {
   })
 
   const profilersMap = new Map()
-
   const profilerOperationsChainsMap = new Map()
-  function runInProfilerChain (fn, req) {
+  const profilerRequestMap = new Map()
+  const profilerLogRequestMap = new Map()
+
+  function runInProfilerChain (fnOrOptions, req) {
     if (req.context.profiling.mode === 'disabled') {
       return
     }
 
+    let fn
+    let cleanFn
+
+    if (typeof fnOrOptions === 'function') {
+      fn = fnOrOptions
+    } else {
+      fn = fnOrOptions.fn
+      cleanFn = fnOrOptions.cleanFn
+    }
+
+    // this only happens when rendering remote delegated requests on docker workers
+    // there won't be operations chain because the request started from another server
+    if (!profilerOperationsChainsMap.has(req.context.rootId)) {
+      return
+    }
+
     profilerOperationsChainsMap.set(req.context.rootId, profilerOperationsChainsMap.get(req.context.rootId).then(async () => {
+      if (cleanFn) {
+        cleanFn()
+      }
+
       if (req.context.profiling.chainFailed) {
         return
       }
 
       try {
-        await fn()
+        if (fn) {
+          await fn()
+        }
       } catch (e) {
         reporter.logger.warn('Failed persist profile', e)
         req.context.profiling.chainFailed = true
@@ -54,7 +79,7 @@ module.exports = (reporter) => {
     return m
   }
 
-  function emitProfiles (events, req) {
+  function emitProfiles ({ events, log = true }, req) {
     if (events.length === 0) {
       return
     }
@@ -63,7 +88,9 @@ module.exports = (reporter) => {
 
     for (const m of events) {
       if (m.type === 'log') {
-        reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp })
+        if (log) {
+          reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp, fromEmitProfile: true })
+        }
       } else {
         lastOperation = m
       }
@@ -78,19 +105,37 @@ module.exports = (reporter) => {
     }
 
     runInProfilerChain(() => {
-      if (req.context.profiling.logFilePath) {
-        return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
-      }
-
-      return reporter.blobStorage.append(
-        req.context.profiling.entity.blobName,
-        Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req
-      )
+      return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
     }, req)
   }
 
-  reporter.registerMainAction('profile', async (events, req) => {
-    return emitProfiles(events, req)
+  reporter.registerMainAction('profile', async (eventsOrOptions, _req) => {
+    let req = _req
+
+    // if there is request stored here then take it, this is needed
+    // for docker workers remote requests, so the emitProfile can work
+    // with the real render request object
+    if (profilerRequestMap.has(req.context.rootId) && req.__isJsreportRequest__ == null) {
+      req = profilerRequestMap.get(req.context.rootId)
+    }
+
+    let events
+    let log
+
+    if (Array.isArray(eventsOrOptions)) {
+      events = eventsOrOptions
+    } else {
+      events = eventsOrOptions.events
+      log = eventsOrOptions.log
+    }
+
+    const params = { events }
+
+    if (log != null) {
+      params.log = log
+    }
+
+    return emitProfiles(params, req)
   })
 
   reporter.attachProfiler = (req, profileMode) => {
@@ -118,20 +163,15 @@ module.exports = (reporter) => {
 
     req.context.profiling.lastOperation = null
 
-    const blobName = `profiles/${req.context.rootId}.log`
-
     const profile = {
       _id: reporter.documentStore.generateId(),
       timestamp: new Date(),
       state: 'queued',
-      mode: req.context.profiling.mode,
-      blobName
+      mode: req.context.profiling.mode
     }
 
-    if (!reporter.blobStorage.supportsAppend) {
-      const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
-      req.context.profiling.logFilePath = pathToFile
-    }
+    const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
+    req.context.profiling.logFilePath = pathToFile
 
     runInProfilerChain(async () => {
       req.context.skipValidationFor = profile
@@ -149,14 +189,16 @@ module.exports = (reporter) => {
 
     req.context.profiling.profileStartOperationId = profileStartOperation.operationId
 
-    emitProfiles([profileStartOperation], req)
+    emitProfiles({ events: [profileStartOperation] }, req)
 
-    emitProfiles([createProfileMessage({
-      type: 'log',
-      level: 'info',
-      message: `Render request ${req.context.reportCounter} queued for execution and waiting for availible worker`,
-      previousOperationId: profileStartOperation.operationId
-    }, req)], req)
+    emitProfiles({
+      events: [createProfileMessage({
+        type: 'log',
+        level: 'info',
+        message: `Render request ${req.context.reportCounter} queued for execution and waiting for available worker`,
+        previousOperationId: profileStartOperation.operationId
+      }, req)]
+    }, req)
   })
 
   reporter.beforeRenderListeners.add('profiler', async (req, res) => {
@@ -164,28 +206,20 @@ module.exports = (reporter) => {
       state: 'running'
     }
 
+    // we set the request here because this listener will container the req which
+    // the .render() starts
+    profilerRequestMap.set(req.context.rootId, req)
+
     const template = await reporter.templates.resolveTemplate(req)
     if (template && template._id) {
       req.context.resolvedTemplate = extend(true, {}, template)
-      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
-      const blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
-      update.templateShortid = template.shortid
-
-      const originalBlobName = req.context.profiling.entity.blobName
-      // we want to store the profile into blobName path reflecting the template path so we need to copy the blob to new path now
-      runInProfilerChain(async () => {
-        if (req.context.profiling.logFilePath == null) {
-          const content = await reporter.blobStorage.read(originalBlobName, req)
-          await reporter.blobStorage.write(blobName, content, req)
-          return reporter.blobStorage.remove(originalBlobName, req)
-        }
-      }, req)
 
-      update.blobName = blobName
+      update.templateShortid = template.shortid
     }
 
     runInProfilerChain(() => {
       req.context.skipValidationFor = update
+
       return reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
@@ -197,28 +231,38 @@ module.exports = (reporter) => {
   })
 
   reporter.afterRenderListeners.add('profiler', async (req, res) => {
-    emitProfiles([createProfileMessage({
-      type: 'operationEnd',
-      doDiffs: false,
-      previousEventId: req.context.profiling.lastEventId,
-      previousOperationId: req.context.profiling.lastOperationId,
-      operationId: req.context.profiling.profileStartOperationId
-    }, req)], req)
+    emitProfiles({
+      events: [createProfileMessage({
+        type: 'operationEnd',
+        doDiffs: false,
+        previousEventId: req.context.profiling.lastEventId,
+        previousOperationId: req.context.profiling.lastOperationId,
+        operationId: req.context.profiling.profileStartOperationId
+      }, req)]
+    }, req)
 
     res.meta.profileId = req.context.profiling?.entity?._id
 
     runInProfilerChain(async () => {
-      if (req.context.profiling.logFilePath != null) {
-        const content = await fs.readFile(req.context.profiling.logFilePath)
-        await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
-        await fs.unlink(req.context.profiling.logFilePath)
+      let blobName = `profiles/${req.context.rootId}.log`
+
+      if (req.context.resolvedTemplate) {
+        const templatePath = await reporter.folders.resolveEntityPath(req.context.resolvedTemplate, 'templates', req)
+        blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
       }
 
+      const content = await fs.readFile(req.context.profiling.logFilePath)
+      blobName = await reporter.blobStorage.write(blobName, content, req)
+      await fs.unlink(req.context.profiling.logFilePath)
+
       const update = {
         state: 'success',
-        finishedOn: new Date()
+        finishedOn: new Date(),
+        blobName
       }
+
       req.context.skipValidationFor = update
+
       await reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
@@ -226,61 +270,128 @@ module.exports = (reporter) => {
       }, req)
     }, req)
 
-    // we don't remove from profiler requests map, because the renderErrorListeners are invoked if the afterRenderListener fails
+    // we don't clean the profiler maps here, we do it later in main reporter .render,
+    // because the renderErrorListeners can be invoked if the afterRenderListener fails
   })
 
   reporter.renderErrorListeners.add('profiler', async (req, res, e) => {
-    try {
-      res.meta.profileId = req.context.profiling?.entity?._id
+    res.meta.profileId = req.context.profiling?.entity?._id
 
-      if (req.context.profiling?.entity != null) {
-        emitProfiles([{
+    if (req.context.profiling?.entity != null) {
+      emitProfiles({
+        events: [{
           type: 'error',
           timestamp: new Date().getTime(),
           ...e,
           id: generateRequestId(),
           stack: e.stack,
           message: e.message
-        }], req)
-        runInProfilerChain(async () => {
-          if (req.context.profiling.logFilePath != null) {
-            const content = await fs.readFile(req.context.profiling.logFilePath, 'utf8')
-            await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
-            await fs.unlink(req.context.profiling.logFilePath)
-          }
+        }]
+      }, req)
+
+      runInProfilerChain(async () => {
+        const update = {
+          state: 'error',
+          finishedOn: new Date(),
+          error: e.toString()
+        }
+
+        if (req.context.profiling.logFilePath != null) {
+          let blobName = `profiles/${req.context.rootId}.log`
 
-          const update = {
-            state: 'error',
-            finishedOn: new Date(),
-            error: e.toString()
+          if (req.context.resolvedTemplate) {
+            const templatePath = await reporter.folders.resolveEntityPath(req.context.resolvedTemplate, 'templates', req)
+            blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
           }
-          req.context.skipValidationFor = update
-          await reporter.documentStore.collection('profiles').update({
-            _id: req.context.profiling.entity._id
-          }, {
-            $set: update
-          }, req)
+
+          const content = await fs.readFile(req.context.profiling.logFilePath)
+          blobName = await reporter.blobStorage.write(blobName, content, req)
+          await fs.unlink(req.context.profiling.logFilePath)
+          update.blobName = blobName
+        }
+
+        req.context.skipValidationFor = update
+
+        await reporter.documentStore.collection('profiles').update({
+          _id: req.context.profiling.entity._id
+        }, {
+          $set: update
         }, req)
-      }
-    } finally {
-      profilersMap.delete(req.context.rootId)
-      profilerOperationsChainsMap.delete(req.context.rootId)
+      }, req)
+
+      // we don't clean the profiler maps here, we do it later in main reporter .render,
+      // we do this to ensure a single and clear order
     }
   })
 
+  const configuredPreviously = reporter.logger.__profilerConfigured__ === true
+
+  if (!configuredPreviously) {
+    const originalLog = reporter.logger.log
+
+    // we want to catch the original request
+    reporter.logger.log = function (level, msg, ...splat) {
+      const [meta] = splat
+
+      if (typeof meta === 'object' && meta !== null && meta.context?.rootId != null) {
+        profilerLogRequestMap.set(meta.context.rootId, meta)
+      }
+
+      return originalLog.call(this, level, msg, ...splat)
+    }
+
+    const mainLogsToProfile = winston.format((info) => {
+      // propagate the request logs occurring on main to the profile
+      if (info.rootId != null && info.fromEmitProfile == null && profilerLogRequestMap.has(info.rootId)) {
+        const req = profilerLogRequestMap.get(info.rootId)
+
+        emitProfiles({
+          events: [createProfileMessage({
+            type: 'log',
+            level: info.level,
+            message: info.message,
+            previousOperationId: req.context.profiling.lastOperationId
+          }, req)],
+          log: false
+        }, req)
+      }
+
+      if (info.fromEmitProfile != null) {
+        delete info.fromEmitProfile
+      }
+
+      return info
+    })
+
+    reporter.logger.format = winston.format.combine(
+      reporter.logger.format,
+      mainLogsToProfile()
+    )
+
+    reporter.logger.__profilerConfigured__ = true
+  }
+
   let profilesCleanupInterval
+
   reporter.initializeListeners.add('profiler', async () => {
     reporter.documentStore.collection('profiles').beforeRemoveListeners.add('profiles', async (query, req) => {
       const profiles = await reporter.documentStore.collection('profiles').find(query, req)
 
       for (const profile of profiles) {
-        await reporter.blobStorage.remove(profile.blobName)
+        if (profile.blobName != null) {
+          await reporter.blobStorage.remove(profile.blobName)
+        }
       }
     })
 
-    profilesCleanupInterval = setInterval(profilesCleanup, reporter.options.profiler.cleanupInterval)
+    function profilesCleanupExec () {
+      return reporter._profilesCleanup()
+    }
+
+    profilesCleanupInterval = setInterval(profilesCleanupExec, reporter.options.profiler.cleanupInterval)
     profilesCleanupInterval.unref()
-    await profilesCleanup()
+
+    await reporter._profilesCleanup()
   })
 
   reporter.closeListeners.add('profiler', async () => {
@@ -294,15 +405,24 @@ module.exports = (reporter) => {
         await profileAppendPromise
       }
     }
+
+    profilersMap.clear()
+    profilerOperationsChainsMap.clear()
+    profilerRequestMap.clear()
+    profilerLogRequestMap.clear()
   })
 
   let profilesCleanupRunning = false
-  async function profilesCleanup () {
+
+  reporter._profilesCleanup = async function profilesCleanup () {
     if (profilesCleanupRunning) {
       return
     }
+
     profilesCleanupRunning = true
+
     let lastRemoveError
+
     try {
       const profiles = await reporter.documentStore.collection('profiles').find({}).sort({ timestamp: -1 })
       const profilesToRemove = profiles.slice(reporter.options.profiler.maxProfilesHistory)
@@ -330,4 +450,28 @@ module.exports = (reporter) => {
       reporter.logger.warn('Profile cleanup failed for some entities, last error:', lastRemoveError)
     }
   }
+
+  return function cleanProfileInRequest (req) {
+    // - req.context.profiling is empty only on an early error
+    // that happens before setting the profiler.
+    // - when profiling.mode is "disabled" there is no profiler chain to append
+    // in both cases we want the clean code to happen immediately
+    if (req.context.profiling?.entity == null || req.context.profiling?.mode === 'disabled') {
+      profilersMap.delete(req.context.rootId)
+      profilerOperationsChainsMap.delete(req.context.rootId)
+      profilerRequestMap.delete(req.context.rootId)
+      profilerLogRequestMap.delete(req.context.rootId)
+      return
+    }
+
+    // this will get executed always even if some fn in the chain fails
+    runInProfilerChain({
+      cleanFn: () => {
+        profilersMap.delete(req.context.rootId)
+        profilerOperationsChainsMap.delete(req.context.rootId)
+        profilerRequestMap.delete(req.context.rootId)
+        profilerLogRequestMap.delete(req.context.rootId)
+      }
+    }, req)
+  }
 }

package/lib/main/reporter.js

@@ -196,7 +196,8 @@ class MainReporter extends Reporter {
       this.blobStorage = BlobStorage(this, this.options)
       blobStorageActions(this)
       Templates(this)
-      Profiler(this)
+
+      this._cleanProfileInRequest = Profiler(this)
 
       this.folders = Object.assign(this.folders, Folders(this))
 
@@ -472,11 +473,17 @@ class MainReporter extends Reporter {
       }, req)
 
       Object.assign(res, responseResult)
+
       await this.afterRenderListeners.fire(req, res)
+
       res.stream = Readable.from(res.content)
+
+      this._cleanProfileInRequest(req)
+
       return res
     } catch (err) {
       await this._handleRenderError(req, res, err)
+      this._cleanProfileInRequest(req)
       throw err
     } finally {
       if (worker && !workerAborted && !dontCloseProcessing) {

package/lib/main/validateEntityName.js

@@ -58,5 +58,11 @@ module.exports = (name) => {
     })
   }
 
+  if (name === 'config.json') {
+    throw createError('Entity name "config.json" is reserved and can\'t be used', {
+      statusCode: 400
+    })
+  }
+
   return true
 }

package/lib/worker/render/executeEngine.js

@@ -93,6 +93,12 @@ module.exports = (reporter) => {
           const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
           return Promise.all([...asyncResultMap.keys()].map((k) => asyncResultMap.get(k)))
         }
+      },
+      createAsyncHelperResult: (v) => {
+        const asyncResultMap = executionAsyncResultsMap.get(context.__executionId)
+        const asyncResultId = nanoid(7)
+        asyncResultMap.set(asyncResultId, v)
+        return `{#asyncHelperResult ${asyncResultId}}`
       }
     }
   })
@@ -197,7 +203,11 @@ module.exports = (reporter) => {
       const wrappedTopLevelFunctions = {}
 
       for (const h of Object.keys(topLevelFunctions)) {
-        wrappedTopLevelFunctions[h] = wrapHelperForAsyncSupport(topLevelFunctions[h], asyncResultMap)
+        if (engine.getWrappingHelpersEnabled && engine.getWrappingHelpersEnabled(req) === false) {
+          wrappedTopLevelFunctions[h] = engine.wrapHelper(topLevelFunctions[h], { context })
+        } else {
+          wrappedTopLevelFunctions[h] = wrapHelperForAsyncSupport(topLevelFunctions[h], asyncResultMap)
+        }
       }
 
       let contentResult = await engine.execute(compiledTemplate, wrappedTopLevelFunctions, data, { require })
@@ -249,7 +259,7 @@ module.exports = (reporter) => {
     try {
       return await reporter.runInSandbox({
         context: {
-          ...(engine.createContext ? engine.createContext() : {})
+          ...(engine.createContext ? engine.createContext(req) : {})
         },
         userCode: normalizedHelpers,
         initFn,

package/lib/worker/render/profiler.js

@@ -36,7 +36,10 @@ class Profiler {
       if (profilingInfo) {
         const batch = profilingInfo.batch
         profilingInfo.batch = []
-        await this.reporter.executeMainAction('profile', batch, profilingInfo.req).catch((e) => this.reporter.logger.error(e, profilingInfo.req))
+
+        if (batch.length > 0) {
+          await this.reporter.executeMainAction('profile', batch, profilingInfo.req).catch((e) => this.reporter.logger.error(e, profilingInfo.req))
+        }
       }
     }
   }
@@ -158,7 +161,10 @@ class Profiler {
       const profilingInfo = this.profiledRequestsMap.get(req.context.rootId)
       if (profilingInfo) {
         this.profiledRequestsMap.delete(req.context.rootId)
-        await this.reporter.executeMainAction('profile', profilingInfo.batch, req)
+
+        if (profilingInfo.batch.length > 0) {
+          await this.reporter.executeMainAction('profile', profilingInfo.batch, req)
+        }
       }
     }
   }

package/lib/worker/render/render.js

@@ -130,7 +130,7 @@ module.exports = (reporter) => {
         reporter.requestModulesCache.set(request.context.rootId, Object.create(null))
       }
 
-      reporter.logger.info(`Starting rendering request ${request.context.reportCounter} (user: ${(request.context.user ? request.context.user.username : 'null')})`, request)
+      reporter.logger.info(`Starting rendering request ${request.context.reportCounter} (user: ${(request.context.user ? request.context.user.name : 'null')})`, request)
 
       // TODO
       /* if (reporter.entityTypeValidator.getSchema('TemplateType') != null) {

package/lib/worker/sandbox/runInSandbox.js

@@ -34,7 +34,9 @@ module.exports = (reporter) => {
 
     const { sourceFilesInfo, run, compileScript, restore, sandbox, sandboxRequire } = createSandbox(context, {
       onLog: (log) => {
-        reporter.logger[log.level](log.message, { ...req, timestamp: log.timestamp })
+        // we mark any log done in sandbox as userLevel: true, this allows us to detect which logs belongs to user
+        // and can potentially contain sensitive information
+        reporter.logger[log.level](log.message, { ...req, timestamp: log.timestamp, userLevel: true })
       },
       formatError: (error, moduleName) => {
         error.message += ` To be able to require custom modules you need to add to configuration { "trustUserCode": true } or enable just specific module using { sandbox: { allowedModules": ["${moduleName}"] }`