@jsreport/jsreport-core 3.5.0 → 3.7.0

package/lib/main/logger.js

@@ -5,10 +5,12 @@ const winston = require('winston')
 const Transport = require('winston-transport')
 const debug = require('debug')('jsreport')
 const createDefaultLoggerFormat = require('./createDefaultLoggerFormat')
-const normalizeMetaFromLogs = require('../shared/normalizeMetaFromLogs')
+const createNormalizeMetaLoggerFormat = require('./createNormalizeMetaLoggerFormat')
+const Request = require('./request')
 
 const defaultLoggerFormat = createDefaultLoggerFormat()
 const defaultLoggerFormatWithTimestamp = createDefaultLoggerFormat({ timestamp: true })
+const normalizeMetaLoggerFormat = createNormalizeMetaLoggerFormat()
 
 function createLogger () {
   const logger = winston.createLogger(getConfigurationOptions())
@@ -166,6 +168,9 @@ function configureLogger (logger, _transports) {
   }
 
   for (const { TransportClass, options } of transportsToAdd) {
+    if (options.silent) {
+      continue
+    }
     const transportInstance = new TransportClass(options)
 
     const existingTransport = logger.transports.find((t) => t.name === transportInstance.name)
@@ -177,25 +182,30 @@ function configureLogger (logger, _transports) {
     logger.add(transportInstance)
   }
 
-  logger.__configured__ = true
-}
+  const originalLog = logger.log
 
-function getConfigurationOptions () {
-  const normalizeMeta = winston.format((info) => {
-    const { level, message, ...meta } = info
-    const newMeta = normalizeMetaFromLogs(level, message, meta)
-
-    if (newMeta != null) {
-      return {
-        level,
-        message,
-        ...newMeta
-      }
+  // we want to normalize the req has httpIncomingRequest early
+  // otherwise we will get serialization issues when trying to
+  // log http.IncomingRequest
+  logger.log = function (level, msg, ...splat) {
+    const [meta] = splat
+
+    if (
+      typeof meta === 'object' &&
+      meta !== null &&
+      meta.context != null &&
+      meta.socket != null
+    ) {
+      splat[0] = Request(meta)
     }
 
-    return info
-  })
+    return originalLog.call(this, level, msg, ...splat)
+  }
 
+  logger.__configured__ = true
+}
+
+function getConfigurationOptions () {
   return {
     levels: {
       error: 0,
@@ -204,7 +214,7 @@ function getConfigurationOptions () {
       debug: 3
     },
     format: winston.format.combine(
-      normalizeMeta(),
+      normalizeMetaLoggerFormat(),
       defaultLoggerFormatWithTimestamp()
     ),
     transports: [new DebugTransport()]
@@ -228,6 +238,7 @@ class DebugTransport extends Transport {
 
     this.format = options.format || winston.format.combine(
       winston.format.colorize(),
+      normalizeMetaLoggerFormat(),
       defaultLoggerFormat()
     )
 

package/lib/main/optionsLoad.js

@@ -39,6 +39,7 @@ async function optionsLoad ({
     loadConfigResult = await loadConfig(defaults, options, false)
   }
 
+  const explicitOptions = loadConfigResult[0]
   const appliedConfigFile = loadConfigResult[1]
 
   options.loadConfig = shouldLoadExternalConfig
@@ -79,9 +80,16 @@ async function optionsLoad ({
   options.store = options.store || { provider: 'memory' }
 
   options.sandbox = options.sandbox || {}
-  if (options.allowLocalFilesAccess === true) {
+
+  // NOTE: handling back-compatible introduction of "trustUserCode" option, "allowLocalFilesAccess" is deprecated
+  if (explicitOptions.allowLocalFilesAccess === true && explicitOptions.trustUserCode == null) {
+    options.trustUserCode = true
+  }
+
+  if (options.trustUserCode === true) {
     options.sandbox.allowedModules = '*'
   }
+
   options.sandbox.nativeModules = options.sandbox.nativeModules || []
   options.sandbox.modules = options.sandbox.modules || []
   options.sandbox.allowedModules = options.sandbox.allowedModules || []
@@ -98,7 +106,7 @@ async function optionsLoad ({
     fs.mkdirSync(options.tempCoreDirectory, { recursive: true })
   }
 
-  return appliedConfigFile
+  return [explicitOptions, appliedConfigFile]
 }
 
 /**

package/lib/main/optionsSchema.js

@@ -59,6 +59,7 @@ module.exports.getRootSchemaOptions = () => ({
       default: '2s'
     },
     enableRequestReportTimeout: { type: 'boolean', default: false, description: 'option that enables passing a custom report timeout per request using req.options.timeout. this enables that the caller of the report generation control the report timeout so enable it only when you trust the caller' },
+    trustUserCode: { type: 'boolean', default: false, description: 'option that control whether code sandboxing is enabled or not, code sandboxing has an impact on performance when rendering large reports. when true code sandboxing will be disabled meaning that users can potentially penetrate the local system if you allow code from external users to be part of your reports' },
     allowLocalFilesAccess: { type: 'boolean', default: false },
     encryption: {
       type: 'object',
@@ -77,6 +78,7 @@ module.exports.getRootSchemaOptions = () => ({
     },
     sandbox: {
       type: 'object',
+      default: {},
       properties: {
         allowedModules: {
           anyOf: [{
@@ -89,9 +91,10 @@ module.exports.getRootSchemaOptions = () => ({
         },
         cache: {
           type: 'object',
+          default: {},
           properties: {
-            max: { type: 'number' },
-            enabled: { type: 'boolean' }
+            max: { type: 'number', default: 100 },
+            enabled: { type: 'boolean', default: true }
           }
         }
       }
@@ -182,7 +185,7 @@ module.exports.getRootSchemaOptions = () => ({
           '$jsreport-acceptsDuration': true,
           default: '1m'
         },
-        maxResponseSize: {
+        maxDiffSize: {
           type: ['string', 'number'],
           '$jsreport-acceptsSize': true,
           default: '50mb'

package/lib/main/profiler.js

@@ -1,4 +1,5 @@
 const EventEmitter = require('events')
+const winston = require('winston')
 const extend = require('node.extend.without.arrays')
 const generateRequestId = require('../shared/generateRequestId')
 const fs = require('fs/promises')
@@ -20,20 +21,44 @@ module.exports = (reporter) => {
   })
 
   const profilersMap = new Map()
-
   const profilerOperationsChainsMap = new Map()
-  function runInProfilerChain (fn, req) {
+  const profilerRequestMap = new Map()
+  const profilerLogRequestMap = new Map()
+
+  function runInProfilerChain (fnOrOptions, req) {
     if (req.context.profiling.mode === 'disabled') {
       return
     }
 
+    let fn
+    let cleanFn
+
+    if (typeof fnOrOptions === 'function') {
+      fn = fnOrOptions
+    } else {
+      fn = fnOrOptions.fn
+      cleanFn = fnOrOptions.cleanFn
+    }
+
+    // this only happens when rendering remote delegated requests on docker workers
+    // there won't be operations chain because the request started from another server
+    if (!profilerOperationsChainsMap.has(req.context.rootId)) {
+      return
+    }
+
     profilerOperationsChainsMap.set(req.context.rootId, profilerOperationsChainsMap.get(req.context.rootId).then(async () => {
+      if (cleanFn) {
+        cleanFn()
+      }
+
       if (req.context.profiling.chainFailed) {
         return
       }
 
       try {
-        await fn()
+        if (fn) {
+          await fn()
+        }
       } catch (e) {
         reporter.logger.warn('Failed persist profile', e)
         req.context.profiling.chainFailed = true
@@ -54,7 +79,7 @@ module.exports = (reporter) => {
     return m
   }
 
-  function emitProfiles (events, req) {
+  function emitProfiles ({ events, log = true }, req) {
     if (events.length === 0) {
       return
     }
@@ -63,7 +88,9 @@ module.exports = (reporter) => {
 
     for (const m of events) {
       if (m.type === 'log') {
-        reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp })
+        if (log) {
+          reporter.logger[m.level](m.message, { ...req, ...m.meta, timestamp: m.timestamp, fromEmitProfile: true })
+        }
       } else {
         lastOperation = m
       }
@@ -78,19 +105,37 @@ module.exports = (reporter) => {
     }
 
     runInProfilerChain(() => {
-      if (req.context.profiling.logFilePath) {
-        return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
-      }
-
-      return reporter.blobStorage.append(
-        req.context.profiling.entity.blobName,
-        Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'), req
-      )
+      return fs.appendFile(req.context.profiling.logFilePath, Buffer.from(events.map(m => JSON.stringify(m)).join('\n') + '\n'))
     }, req)
   }
 
-  reporter.registerMainAction('profile', async (events, req) => {
-    return emitProfiles(events, req)
+  reporter.registerMainAction('profile', async (eventsOrOptions, _req) => {
+    let req = _req
+
+    // if there is request stored here then take it, this is needed
+    // for docker workers remote requests, so the emitProfile can work
+    // with the real render request object
+    if (profilerRequestMap.has(req.context.rootId) && req.__isJsreportRequest__ == null) {
+      req = profilerRequestMap.get(req.context.rootId)
+    }
+
+    let events
+    let log
+
+    if (Array.isArray(eventsOrOptions)) {
+      events = eventsOrOptions
+    } else {
+      events = eventsOrOptions.events
+      log = eventsOrOptions.log
+    }
+
+    const params = { events }
+
+    if (log != null) {
+      params.log = log
+    }
+
+    return emitProfiles(params, req)
   })
 
   reporter.attachProfiler = (req, profileMode) => {
@@ -118,20 +163,15 @@ module.exports = (reporter) => {
 
     req.context.profiling.lastOperation = null
 
-    const blobName = `profiles/${req.context.rootId}.log`
-
     const profile = {
       _id: reporter.documentStore.generateId(),
       timestamp: new Date(),
       state: 'queued',
-      mode: req.context.profiling.mode,
-      blobName
+      mode: req.context.profiling.mode
     }
 
-    if (!reporter.blobStorage.supportsAppend) {
-      const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
-      req.context.profiling.logFilePath = pathToFile
-    }
+    const { pathToFile } = await reporter.writeTempFile((uuid) => `${uuid}.log`, '')
+    req.context.profiling.logFilePath = pathToFile
 
     runInProfilerChain(async () => {
       req.context.skipValidationFor = profile
@@ -149,14 +189,16 @@ module.exports = (reporter) => {
 
     req.context.profiling.profileStartOperationId = profileStartOperation.operationId
 
-    emitProfiles([profileStartOperation], req)
+    emitProfiles({ events: [profileStartOperation] }, req)
 
-    emitProfiles([createProfileMessage({
-      type: 'log',
-      level: 'info',
-      message: `Render request ${req.context.reportCounter} queued for execution and waiting for availible worker`,
-      previousOperationId: profileStartOperation.operationId
-    }, req)], req)
+    emitProfiles({
+      events: [createProfileMessage({
+        type: 'log',
+        level: 'info',
+        message: `Render request ${req.context.reportCounter} queued for execution and waiting for available worker`,
+        previousOperationId: profileStartOperation.operationId
+      }, req)]
+    }, req)
   })
 
   reporter.beforeRenderListeners.add('profiler', async (req, res) => {
@@ -164,28 +206,20 @@ module.exports = (reporter) => {
       state: 'running'
     }
 
+    // we set the request here because this listener will container the req which
+    // the .render() starts
+    profilerRequestMap.set(req.context.rootId, req)
+
     const template = await reporter.templates.resolveTemplate(req)
     if (template && template._id) {
       req.context.resolvedTemplate = extend(true, {}, template)
-      const templatePath = await reporter.folders.resolveEntityPath(template, 'templates', req)
-      const blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
-      update.templateShortid = template.shortid
-
-      const originalBlobName = req.context.profiling.entity.blobName
-      // we want to store the profile into blobName path reflecting the template path so we need to copy the blob to new path now
-      runInProfilerChain(async () => {
-        if (req.context.profiling.logFilePath == null) {
-          const content = await reporter.blobStorage.read(originalBlobName, req)
-          await reporter.blobStorage.write(blobName, content, req)
-          return reporter.blobStorage.remove(originalBlobName, req)
-        }
-      }, req)
 
-      update.blobName = blobName
+      update.templateShortid = template.shortid
     }
 
     runInProfilerChain(() => {
       req.context.skipValidationFor = update
+
       return reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
@@ -197,28 +231,38 @@ module.exports = (reporter) => {
   })
 
   reporter.afterRenderListeners.add('profiler', async (req, res) => {
-    emitProfiles([createProfileMessage({
-      type: 'operationEnd',
-      doDiffs: false,
-      previousEventId: req.context.profiling.lastEventId,
-      previousOperationId: req.context.profiling.lastOperationId,
-      operationId: req.context.profiling.profileStartOperationId
-    }, req)], req)
+    emitProfiles({
+      events: [createProfileMessage({
+        type: 'operationEnd',
+        doDiffs: false,
+        previousEventId: req.context.profiling.lastEventId,
+        previousOperationId: req.context.profiling.lastOperationId,
+        operationId: req.context.profiling.profileStartOperationId
+      }, req)]
+    }, req)
 
     res.meta.profileId = req.context.profiling?.entity?._id
 
     runInProfilerChain(async () => {
-      if (req.context.profiling.logFilePath != null) {
-        const content = await fs.readFile(req.context.profiling.logFilePath)
-        await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
-        await fs.unlink(req.context.profiling.logFilePath)
+      let blobName = `profiles/${req.context.rootId}.log`
+
+      if (req.context.resolvedTemplate) {
+        const templatePath = await reporter.folders.resolveEntityPath(req.context.resolvedTemplate, 'templates', req)
+        blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
       }
 
+      const content = await fs.readFile(req.context.profiling.logFilePath)
+      blobName = await reporter.blobStorage.write(blobName, content, req)
+      await fs.unlink(req.context.profiling.logFilePath)
+
       const update = {
         state: 'success',
-        finishedOn: new Date()
+        finishedOn: new Date(),
+        blobName
       }
+
       req.context.skipValidationFor = update
+
       await reporter.documentStore.collection('profiles').update({
         _id: req.context.profiling.entity._id
       }, {
@@ -226,61 +270,128 @@ module.exports = (reporter) => {
       }, req)
     }, req)
 
-    // we don't remove from profiler requests map, because the renderErrorListeners are invoked if the afterRenderListener fails
+    // we don't clean the profiler maps here, we do it later in main reporter .render,
+    // because the renderErrorListeners can be invoked if the afterRenderListener fails
   })
 
   reporter.renderErrorListeners.add('profiler', async (req, res, e) => {
-    try {
-      res.meta.profileId = req.context.profiling?.entity?._id
+    res.meta.profileId = req.context.profiling?.entity?._id
 
-      if (req.context.profiling?.entity != null) {
-        emitProfiles([{
+    if (req.context.profiling?.entity != null) {
+      emitProfiles({
+        events: [{
           type: 'error',
           timestamp: new Date().getTime(),
           ...e,
           id: generateRequestId(),
           stack: e.stack,
           message: e.message
-        }], req)
-        runInProfilerChain(async () => {
-          if (req.context.profiling.logFilePath != null) {
-            const content = await fs.readFile(req.context.profiling.logFilePath, 'utf8')
-            await reporter.blobStorage.write(req.context.profiling.entity.blobName, content, req)
-            await fs.unlink(req.context.profiling.logFilePath)
-          }
+        }]
+      }, req)
+
+      runInProfilerChain(async () => {
+        const update = {
+          state: 'error',
+          finishedOn: new Date(),
+          error: e.toString()
+        }
+
+        if (req.context.profiling.logFilePath != null) {
+          let blobName = `profiles/${req.context.rootId}.log`
 
-          const update = {
-            state: 'error',
-            finishedOn: new Date(),
-            error: e.toString()
+          if (req.context.resolvedTemplate) {
+            const templatePath = await reporter.folders.resolveEntityPath(req.context.resolvedTemplate, 'templates', req)
+            blobName = `profiles/${templatePath.substring(1)}/${req.context.rootId}.log`
           }
-          req.context.skipValidationFor = update
-          await reporter.documentStore.collection('profiles').update({
-            _id: req.context.profiling.entity._id
-          }, {
-            $set: update
-          }, req)
+
+          const content = await fs.readFile(req.context.profiling.logFilePath)
+          blobName = await reporter.blobStorage.write(blobName, content, req)
+          await fs.unlink(req.context.profiling.logFilePath)
+          update.blobName = blobName
+        }
+
+        req.context.skipValidationFor = update
+
+        await reporter.documentStore.collection('profiles').update({
+          _id: req.context.profiling.entity._id
+        }, {
+          $set: update
         }, req)
-      }
-    } finally {
-      profilersMap.delete(req.context.rootId)
-      profilerOperationsChainsMap.delete(req.context.rootId)
+      }, req)
+
+      // we don't clean the profiler maps here, we do it later in main reporter .render,
+      // we do this to ensure a single and clear order
     }
   })
 
+  const configuredPreviously = reporter.logger.__profilerConfigured__ === true
+
+  if (!configuredPreviously) {
+    const originalLog = reporter.logger.log
+
+    // we want to catch the original request
+    reporter.logger.log = function (level, msg, ...splat) {
+      const [meta] = splat
+
+      if (typeof meta === 'object' && meta !== null && meta.context?.rootId != null) {
+        profilerLogRequestMap.set(meta.context.rootId, meta)
+      }
+
+      return originalLog.call(this, level, msg, ...splat)
+    }
+
+    const mainLogsToProfile = winston.format((info) => {
+      // propagate the request logs occurring on main to the profile
+      if (info.rootId != null && info.fromEmitProfile == null && profilerLogRequestMap.has(info.rootId)) {
+        const req = profilerLogRequestMap.get(info.rootId)
+
+        emitProfiles({
+          events: [createProfileMessage({
+            type: 'log',
+            level: info.level,
+            message: info.message,
+            previousOperationId: req.context.profiling.lastOperationId
+          }, req)],
+          log: false
+        }, req)
+      }
+
+      if (info.fromEmitProfile != null) {
+        delete info.fromEmitProfile
+      }
+
+      return info
+    })
+
+    reporter.logger.format = winston.format.combine(
+      reporter.logger.format,
+      mainLogsToProfile()
+    )
+
+    reporter.logger.__profilerConfigured__ = true
+  }
+
   let profilesCleanupInterval
+
   reporter.initializeListeners.add('profiler', async () => {
     reporter.documentStore.collection('profiles').beforeRemoveListeners.add('profiles', async (query, req) => {
       const profiles = await reporter.documentStore.collection('profiles').find(query, req)
 
       for (const profile of profiles) {
-        await reporter.blobStorage.remove(profile.blobName)
+        if (profile.blobName != null) {
+          await reporter.blobStorage.remove(profile.blobName)
+        }
       }
     })
 
-    profilesCleanupInterval = setInterval(profilesCleanup, reporter.options.profiler.cleanupInterval)
+    function profilesCleanupExec () {
+      return reporter._profilesCleanup()
+    }
+
+    profilesCleanupInterval = setInterval(profilesCleanupExec, reporter.options.profiler.cleanupInterval)
     profilesCleanupInterval.unref()
-    await profilesCleanup()
+
+    await reporter._profilesCleanup()
   })
 
   reporter.closeListeners.add('profiler', async () => {
@@ -294,15 +405,24 @@ module.exports = (reporter) => {
         await profileAppendPromise
       }
     }
+
+    profilersMap.clear()
+    profilerOperationsChainsMap.clear()
+    profilerRequestMap.clear()
+    profilerLogRequestMap.clear()
   })
 
   let profilesCleanupRunning = false
-  async function profilesCleanup () {
+
+  reporter._profilesCleanup = async function profilesCleanup () {
     if (profilesCleanupRunning) {
       return
     }
+
     profilesCleanupRunning = true
+
     let lastRemoveError
+
     try {
       const profiles = await reporter.documentStore.collection('profiles').find({}).sort({ timestamp: -1 })
       const profilesToRemove = profiles.slice(reporter.options.profiler.maxProfilesHistory)
@@ -330,4 +450,28 @@ module.exports = (reporter) => {
       reporter.logger.warn('Profile cleanup failed for some entities, last error:', lastRemoveError)
     }
   }
+
+  return function cleanProfileInRequest (req) {
+    // - req.context.profiling is empty only on an early error
+    // that happens before setting the profiler.
+    // - when profiling.mode is "disabled" there is no profiler chain to append
+    // in both cases we want the clean code to happen immediately
+    if (req.context.profiling?.entity == null || req.context.profiling?.mode === 'disabled') {
+      profilersMap.delete(req.context.rootId)
+      profilerOperationsChainsMap.delete(req.context.rootId)
+      profilerRequestMap.delete(req.context.rootId)
+      profilerLogRequestMap.delete(req.context.rootId)
+      return
+    }
+
+    // this will get executed always even if some fn in the chain fails
+    runInProfilerChain({
+      cleanFn: () => {
+        profilersMap.delete(req.context.rootId)
+        profilerOperationsChainsMap.delete(req.context.rootId)
+        profilerRequestMap.delete(req.context.rootId)
+        profilerLogRequestMap.delete(req.context.rootId)
+      }
+    }, req)
+  }
 }