newrelic 8.11.2 → 8.13.1

package/NEWS.md

@@ -1,3 +1,27 @@
+### v8.13.1 (2022-05-27)
+
+* Fixed passing undefined as a formatter options to `winston.format.combine`
+
+### v8.13.0 (2022-05-26)
+
+* Moved log forwarding logic to a transport so customer transports are not polluted with NR linking metadata and timestamp and error manipulations.
+
+* Prevented transmitting logs when application level logging has been disabled.
+
+### v8.12.0 (2022-05-24)
+
+* Added instrumentation to pino to support application logging use cases: forwarding, local decorating, and metrics.
+
+* Added supportability metrics about the data usage bytes of harvested data to the collector endpoints.
+
+* Added an optional way to avoid wrapping browser agent script with <script> tag when using `api.getBrowserTimingHeader`.  This will ease usage with Component based libraries like React.
+
+  Thanks to @github-dd-nicolas for the contribution. 🎉
+
+* Upgraded `@grpc/proto-loader` to fix a [CVE](https://security.snyk.io/vuln/SNYK-JS-PROTOBUFJS-2441248) with `protobufjs`.
+
+* Upgraded `@newrelic/test-utilities` to resolve a dev-only audit warning.
+
 ### v8.11.2 (2022-05-23)
 
 * Fixed winston instrumentation to no longer coerce every log line to be json.

package/THIRD_PARTY_NOTICES.md

@@ -26,6 +26,7 @@ code, the source code can be found at [https://github.com/newrelic/node-newrelic
 * [json-stringify-safe](#json-stringify-safe)
 * [readable-stream](#readable-stream)
 * [semver](#semver)
+* [winston-transport](#winston-transport)
 
 **[devDependencies](#devDependencies)**
 
@@ -295,7 +296,7 @@ This product includes source derived from [@grpc/grpc-js](https://github.com/grp
 
 ### @grpc/proto-loader
 
-This product includes source derived from [@grpc/proto-loader](https://github.com/grpc/grpc-node) ([v0.6.9](https://github.com/grpc/grpc-node/tree/v0.6.9)), distributed under the [Apache-2.0 License](https://github.com/grpc/grpc-node/blob/v0.6.9/LICENSE):
+This product includes source derived from [@grpc/proto-loader](https://github.com/grpc/grpc-node) ([v0.6.12](https://github.com/grpc/grpc-node/tree/v0.6.12)), distributed under the [Apache-2.0 License](https://github.com/grpc/grpc-node/blob/v0.6.12/LICENSE):
 
 ```
                                  Apache License
@@ -1330,6 +1331,36 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
 IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+```
+
+### winston-transport
+
+This product includes source derived from [winston-transport](https://github.com/winstonjs/winston-transport) ([v4.5.0](https://github.com/winstonjs/winston-transport/tree/v4.5.0)), distributed under the [MIT License](https://github.com/winstonjs/winston-transport/blob/v4.5.0/LICENSE):
+
+```
+The MIT License (MIT)
+
+Copyright (c) 2015 Charlie Robbins & the contributors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
 ```
 
 
@@ -1770,7 +1801,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 ### @newrelic/test-utilities
 
-This product includes source derived from [@newrelic/test-utilities](https://github.com/newrelic/node-test-utilities) ([v6.5.2](https://github.com/newrelic/node-test-utilities/tree/v6.5.2)), distributed under the [Apache-2.0 License](https://github.com/newrelic/node-test-utilities/blob/v6.5.2/LICENSE):
+This product includes source derived from [@newrelic/test-utilities](https://github.com/newrelic/node-test-utilities) ([v6.5.4](https://github.com/newrelic/node-test-utilities/tree/v6.5.4)), distributed under the [Apache-2.0 License](https://github.com/newrelic/node-test-utilities/blob/v6.5.4/LICENSE):
 
 ```
                                  Apache License

package/api.js

@@ -27,8 +27,8 @@ const NAMES = require('./lib/metrics/names')
  * CONSTANTS
  *
  */
-const RUM_STUB =
-  "<script type='text/javascript' %s>window.NREUM||(NREUM={});" + 'NREUM.info = %s; %s</script>'
+const RUM_STUB = 'window.NREUM||(NREUM={});NREUM.info = %s;'
+const RUM_STUB_SHELL = `<script type='text/javascript' %s>${RUM_STUB} %s</script>`
 
 // these messages are used in the _gracefail() method below in getBrowserTimingHeader
 const RUM_ISSUES = [
@@ -489,19 +489,24 @@ API.prototype.addIgnoringRule = function addIgnoringRule(pattern) {
 }
 
 /**
- * Get the <script>...</script> header necessary for Browser Monitoring
+ * Get the script header necessary for Browser Monitoring
  * This script must be manually injected into your templates, as high as possible
  * in the header, but _after_ any X-UA-COMPATIBLE HTTP-EQUIV meta tags.
  * Otherwise you may hurt IE!
  *
+ * By default this method will return a script wrapped by `<script>` tags, but with
+ * option `hasToRemoveScriptWrapper` it can send back only the script content
+ * without the `<script>` wrapper. Useful for React component based frontend.
+ *
  * This method must be called _during_ a transaction, and must be called every
  * time you want to generate the headers.
  *
  * Do *not* reuse the headers between users, or even between requests.
  *
  * @param {string} [options.nonce] - Nonce to inject into `<script>` header.
+ * @param {boolean} [options.hasToRemoveScriptWrapper] - Used to import agent script without `<script>` tag wrapper.
  * @param options
- * @returns {string} The `<script>` header to be injected.
+ * @returns {string} The script content to be injected in `<head>` or put inside `<script>` tag (depending on options)
  */
 API.prototype.getBrowserTimingHeader = function getBrowserTimingHeader(options) {
   const metric = this.agent.metrics.getOrCreateMetric(
@@ -650,9 +655,10 @@ API.prototype.getBrowserTimingHeader = function getBrowserTimingHeader(options)
 
   // set nonce attribute if passed in options
   const nonce = options && options.nonce ? 'nonce="' + options.nonce + '"' : ''
+  const script = options && options.hasToRemoveScriptWrapper ? RUM_STUB : RUM_STUB_SHELL
 
   // the complete header to be written to the browser
-  const out = util.format(RUM_STUB, nonce, json, jsAgentLoader)
+  const out = util.format(script, nonce, json, jsAgentLoader)
 
   logger.trace('generating RUM header', out)
 

package/lib/aggregators/log-aggregator.js

@@ -34,8 +34,21 @@ class LogAggregator extends EventAggregator {
       return
     }
 
-    const eventData = events.toArray()
-    return [{ logs: eventData }]
+    const logs = events.toArray()
+
+    /**
+     *  Due to logging library implementation details
+     *  some log lines are already JSON. We must de-serialize
+     *  so it can be serialized in RemoteMethod. I know, this sucks.
+     */
+    const formattedLogs = logs.map((logLine) => {
+      if (typeof logLine === 'string') {
+        return JSON.parse(logLine)
+      }
+
+      return logLine
+    })
+    return [{ logs: formattedLogs }]
   }
 
   add(logLine) {
@@ -52,6 +65,17 @@ class LogAggregator extends EventAggregator {
       super.add(logLine, priority)
     })
   }
+
+  reconfigure(config) {
+    const oldLimit = this.limit
+    super.reconfigure(config)
+    if (this.limit <= 0 && oldLimit > 0) {
+      logger.debug('This New Relic account has disabled APM logs.')
+      this.clear()
+    } else if (oldLimit <= 0 && this.limit > 0) {
+      logger.debug('This New Relic account has re-enabled APM logs.')
+    }
+  }
 }
 
 module.exports = LogAggregator

package/lib/collector/api.js

@@ -68,20 +68,24 @@ function CollectorAPI(agent) {
   /* RemoteMethods can be reused and have little per-object state, so why not
    * save some GC time?
    */
-  this._methods = {
-    preconnect: new RemoteMethod('preconnect', agent.config, initialEndpoint),
-    connect: new RemoteMethod('connect', agent.config, initialEndpoint),
-    settings: new RemoteMethod('agent_settings', agent.config, initialEndpoint),
-    errors: new RemoteMethod('error_data', agent.config, initialEndpoint),
-    metrics: new RemoteMethod('metric_data', agent.config, initialEndpoint),
-    traces: new RemoteMethod('transaction_sample_data', agent.config, initialEndpoint),
-    shutdown: new RemoteMethod('shutdown', agent.config, initialEndpoint),
-    events: new RemoteMethod('analytic_event_data', agent.config, initialEndpoint),
-    customEvents: new RemoteMethod('custom_event_data', agent.config, initialEndpoint),
-    queryData: new RemoteMethod('sql_trace_data', agent.config, initialEndpoint),
-    errorEvents: new RemoteMethod('error_event_data', agent.config, initialEndpoint),
-    spanEvents: new RemoteMethod('span_event_data', agent.config, initialEndpoint),
-    logEvents: new RemoteMethod('log_event_data', agent.config, initialEndpoint)
+  this._methods = {}
+  for (const [accessor, name] of [
+    ['preconnect', 'preconnect'],
+    ['connect', 'connect'],
+    ['settings', 'agent_settings'],
+    ['errors', 'error_data'],
+    ['metrics', 'metric_data'],
+    ['traces', 'transaction_sample_data'],
+    ['shutdown', 'shutdown'],
+    ['events', 'analytic_event_data'],
+    ['customEvents', 'custom_event_data'],
+    ['queryData', 'sql_trace_data'],
+    ['errorEvents', 'error_event_data'],
+    ['spanEvents', 'span_event_data'],
+    ['logEvents', 'log_event_data']
+  ]) {
+    const method = new RemoteMethod(name, agent, initialEndpoint)
+    this._methods[accessor] = method
   }
 }
 

package/lib/collector/remote-method.js

@@ -16,6 +16,11 @@ const Sink = require('../util/stream-sink')
 const agents = require('./http-agents')
 const certificates = require('./ssl/certificates')
 const isValidLength = require('../util/byte-limit').isValidLength
+const { DATA_USAGE } = require('../metrics/names')
+
+function getMetricName(name) {
+  return `${DATA_USAGE.PREFIX}/${name}/${DATA_USAGE.SUFFIX}`
+}
 
 /*
  *
@@ -29,13 +34,17 @@ const USER_AGENT_FORMAT = 'NewRelic-NodeAgent/%s (nodejs %s %s-%s)'
 const ENCODING_HEADER = 'CONTENT-ENCODING'
 const DEFAULT_ENCODING = 'identity'
 
-function RemoteMethod(name, config, endpoint) {
+function RemoteMethod(name, agent, endpoint) {
   if (!name) {
     throw new TypeError('Must include name of method to invoke on collector.')
   }
+  if (!agent) {
+    throw new TypeError('Must include an agent instance.')
+  }
 
   this.name = name
-  this._config = config
+  this._config = agent.config
+  this._agent = agent
   this._protocolVersion = 17
 
   this.endpoint = endpoint
@@ -62,6 +71,21 @@ RemoteMethod.prototype.serialize = function serialize(payload, callback) {
   return callback(null, res)
 }
 
+/**
+ * This records the amount of data usage per function call and sends
+ * it to the metrics.
+ *
+ * @param {string} sent        The serialized object we sent
+ * @param {object} received    The raw object we got back as a response
+ */
+RemoteMethod.prototype._reportDataUsage = function reportDataUsage(sent, received) {
+  const payloadByteLength = byteLength(sent)
+  const responseByteLength = received ? byteLength(JSON.stringify(received)) : 0
+  const measurement = [payloadByteLength, responseByteLength, true]
+  this._agent.metrics.measureBytes(DATA_USAGE.COLLECTOR, ...measurement)
+  this._agent.metrics.measureBytes(getMetricName(this.name), ...measurement)
+}
+
 /**
  * The primary operation on RemoteMethod objects. If you're calling anything on
  * RemoteMethod objects aside from invoke (and you're not writing test code),
@@ -88,7 +112,14 @@ RemoteMethod.prototype.invoke = function invoke(payload, nrHeaders, callback) {
       if (err) {
         return callback(err)
       }
-      this._post(serialized, nrHeaders, callback)
+      this._post(
+        serialized,
+        nrHeaders,
+        function onResponse(error, res) {
+          this._reportDataUsage(serialized, res && res.payload)
+          callback(error, res)
+        }.bind(this)
+      )
     }.bind(this)
   )
 }
@@ -251,6 +282,7 @@ RemoteMethod.prototype._safeRequest = function _safeRequest(options) {
  *                                      original callback given to .send).
  * @param {Function} options.onResponse Response handler for this request (created by
  *                                      ._post).
+ * @param options
  */
 RemoteMethod.prototype._request = function _request(options) {
   const requestOptions = {
@@ -375,6 +407,8 @@ RemoteMethod.prototype._headers = function _headers(options) {
  * load on the collector down.
  *
  * FIXME: come up with a better heuristic
+ *
+ * @param data
  */
 RemoteMethod.prototype._shouldCompress = function _shouldCompress(data) {
   return data && byteLength(data) > 65536

package/lib/instrumentation/nr-winston-transport.js

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2022 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const TransportStream = require('winston-transport')
+const logger = require('../logger').child({ component: 'nr-winston-transport' })
+const { truncate } = require('../util/application-logging')
+
+/**
+ * Transport used to prepare a log line and add to the new relic agent
+ * log aggregator.
+ *
+ * Note*: This copies the log line so no other transports will get the
+ * mutated data.
+ */
+class NrTransport extends TransportStream {
+  constructor(opts = {}) {
+    // set this option to have winston handle uncaught exceptions
+    // See: https://github.com/winstonjs/winston#handling-uncaught-exceptions-with-winston
+    opts.handleExceptions = true
+    super(opts)
+    this.name = 'newrelic'
+    this.agent = opts.agent
+    this.config = opts.agent.config
+  }
+
+  /**
+   * Executed on every log line. We will get the linking metadata
+   * and add this, along with reformatting of timestamp and error
+   * to a copy of the log line
+   *
+   * @param {object} logLine a winston log line
+   * @param {Function} callback callback to invoke once we are done
+   */
+  log(logLine, callback) {
+    const metadata = this.agent.getLinkingMetadata()
+    const formattedLine = reformatLogLine(logLine, metadata, this.agent)
+    this.agent.logs.add(formattedLine)
+    callback()
+  }
+}
+
+module.exports = NrTransport
+
+/**
+ * Reformats a log line by reformatting errors, timestamp and adding
+ * new relic linking metadata(context). When uncaught exceptions exist
+ * an exception property will exist on the log line.  This will tell us
+ * that we need to reformat the error
+ *
+ * @param {object} logLine log line
+ * @param {object} metadata linking metadata
+ * @param {object} agent NR agent
+ * @returns {object} copy of log line with NR linking metadata
+ */
+function reformatLogLine(logLine, metadata, agent) {
+  // Add the metadata to a copy of the logLine
+  const formattedLine = Object.assign({}, logLine, metadata)
+
+  if (formattedLine.exception === true) {
+    reformatError(formattedLine)
+  }
+
+  reformatTimestamp(formattedLine, agent)
+
+  return formattedLine
+}
+
+/**
+ * Decorates the log line with  truncated error.message, error.class, and error.stack and removes
+ * trace and stack
+ *
+ * @param {object} logLine a log line
+ */
+function reformatError(logLine) {
+  // Due to Winston internals sometimes the error on the logLine object is a string or an
+  // empty object, and so the message property is all we have
+  const errorMessage = logLine.error.message || logLine.message || ''
+
+  logLine['error.message'] = truncate(errorMessage)
+  logLine['error.class'] =
+    logLine.error.name === 'Error' ? logLine.error.constructor.name : logLine.error.name
+  logLine['error.stack'] = truncate(logLine.error.stack)
+  logLine.message = truncate(logLine.message)
+
+  // Removes additional capture of stack to reduce overall payload/log-line size.
+  // The server has a maximum of ~4k characters per line allowed.
+  delete logLine.trace
+  delete logLine.stack
+}
+
+/**
+ * Turns timestamp into unix timestamp. If timestamp existed it will move original
+ * to `original_timestamp` key
+ *
+ * @param {object} logLine a log line
+ */
+function reformatTimestamp(logLine) {
+  if (logLine.timestamp) {
+    logger.traceOnce(
+      'Overwriting `timestamp` key; assigning original value to `original_timestamp`.'
+    )
+    logLine.original_timestamp = logLine.timestamp
+  }
+  logLine.timestamp = Date.now()
+}

package/lib/instrumentation/pino/nr-hooks.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2022 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const pino = require('./pino')
+
+/**
+ * Need to use nr-hooks style for pino because we're using the onResolved hook
+ * to register instrumentation.
+ */
+module.exports = [
+  {
+    type: 'generic',
+    moduleName: 'pino',
+    onResolved: pino
+  }
+]

package/lib/instrumentation/pino/pino.js

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2022 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const {
+  createModuleUsageMetric,
+  isApplicationLoggingEnabled,
+  isLogForwardingEnabled,
+  isMetricsEnabled,
+  isLocalDecoratingEnabled,
+  incrementLoggingLinesMetrics,
+  truncate
+} = require('../../util/application-logging')
+const semver = require('semver')
+
+module.exports = function instrument(shim) {
+  const pinoVersion = shim.require('./package.json').version
+
+  if (semver.lt(pinoVersion, '7.0.0')) {
+    shim.logger.debug('Instrumentation only supported on pino >=7.0.0.')
+    return
+  }
+
+  const tools = shim.require('./lib/tools')
+  const agent = shim.agent
+  const config = agent.config
+
+  if (!isApplicationLoggingEnabled(config)) {
+    shim.logger.debug('Application logging not enabled. Not instrumenting pino.')
+    return
+  }
+
+  const metrics = agent.metrics
+  createModuleUsageMetric('pino', metrics)
+
+  const levelUtils = shim.require('./lib/levels')
+
+  /**
+   * Creates an object where the keys are the level labels and values are the level number.
+   * Pino passes in level as number but our spec needs it to be the label
+   */
+  const levelMap = Object.entries(levelUtils.levels).reduce((levels, level) => {
+    const [label, number] = level
+    levels[number] = label
+    return levels
+  }, {})
+
+  const symbols = shim.require('./lib/symbols')
+
+  /**
+   * Wraps the level cache so we can properly set a formatter to return the
+   * label as level in log line instead of number
+   */
+  shim.wrap(levelUtils, 'genLsCache', function genLsCache(shim, genLevelsCache) {
+    return function wrappedGenLsCache() {
+      const args = shim.argsToArray.apply(shim, arguments)
+      args[0][symbols.formattersSym].level = function nrLevelMapping(label) {
+        return { level: label }
+      }
+      return genLevelsCache.apply(this, args)
+    }
+  })
+
+  shim.wrap(tools, 'asJson', function wrapJson(shim, asJson) {
+    /**
+     * Wraps function in pino that is used to construct/serialize a log
+     * line to json
+     *
+     * @param {object} obj data from mixins
+     * @param {string} msg message of log line
+     * @param {number} num log level as number
+     * @param {string} time formatted snippet of json with time(`,"time":<unix time>`)
+     * @returns {string} serialized log line
+     */
+    return function wrappedAsJson() {
+      // overriding the symbol that defines the key for message(pino defaults this to `msg`)
+      this[symbols.messageKeySym] = 'message'
+
+      const args = shim.argsToArray.apply(shim, arguments)
+
+      /**
+       * changing the label from time to timestamp
+       * and assign unix timestamp to match our specification
+       */
+      args[3] = `,"timestamp":${Date.now()}`
+
+      if (isMetricsEnabled(config)) {
+        const level = args[2]
+        incrementLoggingLinesMetrics(levelMap[level], metrics)
+      }
+
+      if (isLogForwardingEnabled(config, agent)) {
+        const metadata = agent.getLinkingMetadata()
+        const chindings = this[symbols.chindingsSym]
+        reformatLogLine(args[0], metadata, chindings)
+      } else if (isLocalDecoratingEnabled(config)) {
+        args[1] += agent.getNRLinkingMetadata()
+      }
+
+      /**
+       * must call original asJson to allow pino
+       * to construct the entire log line before we
+       * add to the log aggregator
+       */
+      const logLine = asJson.apply(this, args)
+
+      if (isLogForwardingEnabled(config, agent)) {
+        agent.logs.add(logLine)
+      }
+
+      return logLine
+    }
+  })
+}
+
+/**
+ * reformats error and assigns NR context data
+ * to log line
+ *
+ * @param {object} logLine log line
+ * @param {object} metadata NR context data
+ * @param {string} chindings serialized string of all common log line data
+ */
+function reformatLogLine(logLine, metadata, chindings = '') {
+  if (logLine.err) {
+    reformatError(logLine)
+  }
+
+  /**
+   * pino adds this already for us at times
+   * since asJson manually constructs the json string,
+   * it will have hostname twice if we do not delete ours.
+   */
+  if (chindings.includes('hostname')) {
+    delete metadata.hostname
+  }
+
+  Object.assign(logLine, metadata)
+}
+
+/**
+ * Truncates the message, stack and class of error
+ * and reassigns to a `error.*` keyspace. Also removes the `err`
+ * key on a log line
+ *
+ * @param {object} logLine log line
+ */
+function reformatError(logLine) {
+  logLine['error.message'] = truncate(logLine.err.message)
+  logLine['error.stack'] = truncate(logLine.err.stack)
+  logLine['error.class'] =
+    logLine.err.name === 'Error' ? logLine.err.constructor.name : logLine.err.name
+  delete logLine.err
+}

package/lib/instrumentation/winston.js

@@ -12,10 +12,11 @@ const {
   isLocalDecoratingEnabled,
   isMetricsEnabled,
   createModuleUsageMetric,
-  incrementLoggingLinesMetrics,
-  truncate
+  incrementLoggingLinesMetrics
 } = require('../util/application-logging')
+
 const logger = require('../logger').child({ component: 'winston' })
+const NrTransport = require('./nr-winston-transport')
 
 module.exports = function instrument(agent, winston, _, shim) {
   const config = agent.config
@@ -33,40 +34,38 @@ module.exports = function instrument(agent, winston, _, shim) {
         return createLogger.apply(this, args)
       }
 
-      registerFormatter({ opts, config, agent, winston })
+      createModuleUsageMetric('winston', agent.metrics)
+
+      if (isLocalDecoratingEnabled(config) || isMetricsEnabled(config)) {
+        registerFormatter({ opts, agent, winston })
+      }
+
+      const winstonLogger = createLogger.apply(this, args)
 
-      return createLogger.apply(this, args)
+      if (isLogForwardingEnabled(config, agent)) {
+        winstonLogger.add(new NrTransport({ agent }))
+      }
+
+      return winstonLogger
     }
   })
 }
 
 /**
- * There is no right way to do this.  But since we are automagical
- * it is hard to predict how a customer uses winston.  We will iterate over the formatters specified on the logger and add last if forwarder or first if local decorating.
- * This is because we want all the customizations of previous formatters before adding log log to log aggregator.  But in the case of local decorating we want to do this first so any formatter that is transforming data will have the changes.
- *
- * Note: The logic explained above does not apply if a customer specifies multiple formats for a given transport.
- * We cannot instrument the formats on the transport because if a customer has multiple transports we would be duplicating logs when forwaring.
+ * Apply a formatter to keep track of logging metrics, and in the case of local decorating appending
+ * the NR-LINKING metadata to message.  We want to do this first so any formatter that is transforming
+ * data will have the changes.
  *
  * @param {object} params object passed to function
  * @param {object} params.opts options from winston.createLogger
- * @param {object} params.config agent config
  * @param {object} params.agent NR agent
  * @param {object} params.winston exported winston package
  */
-function registerFormatter({ opts, config, agent, winston }) {
+function registerFormatter({ opts, agent, winston }) {
   const instrumentedFormatter = nrWinstonFormatter(agent, winston)
 
-  if ('format' in opts) {
-    const formatters = [opts.format]
-
-    if (isLogForwardingEnabled(config, agent)) {
-      formatters.push(instrumentedFormatter())
-    } else {
-      formatters.unshift(instrumentedFormatter())
-    }
-
-    opts.format = winston.format.combine(...formatters)
+  if (opts.format) {
+    opts.format = winston.format.combine(instrumentedFormatter(), opts.format)
   } else {
     opts.format = instrumentedFormatter()
   }
@@ -74,39 +73,26 @@ function registerFormatter({ opts, config, agent, winston }) {
 
 /**
  * This formatter is being used to facilitate
- * the application logging use cases.
- * It is worth noting that the features below are mutually
- * exclusive.
- *
- * The application logging use cases are local log decorating
- * and log forwarding.
+ * the two application logging use cases: metrics and local log decorating.
  *
  * Local log decorating appends `NR-LINKING` piped metadata to
  * the message key in log line. You must configure a log forwarder to get
  * this data to NR1.
  *
- * Log forwarding includes the linking metadata as keys on logging
- * object as well as adds the log line to the agent log aggregator.
- *
  * @param {object} agent NR agent
  * @param {object} winston exported winston package
- * @returns {object} log line with NR context or NR-LINKING metadata on message
+ * @returns {object} log line NR-LINKING metadata on message when local log decorating is enabled
  */
 function nrWinstonFormatter(agent, winston) {
   const config = agent.config
   const metrics = agent.metrics
-  createModuleUsageMetric('winston', metrics)
 
   return winston.format((logLine) => {
     if (isMetricsEnabled(config)) {
       incrementLoggingLinesMetrics(logLine.level, metrics)
     }
 
-    if (isLogForwardingEnabled(config, agent)) {
-      const metadata = agent.getLinkingMetadata()
-      reformatLogLine(logLine, metadata, agent)
-      agent.logs.add(logLine)
-    } else if (isLocalDecoratingEnabled(config)) {
+    if (isLocalDecoratingEnabled(config)) {
       logLine.message += agent.getNRLinkingMetadata()
     }
 
@@ -114,64 +100,6 @@ function nrWinstonFormatter(agent, winston) {
   })
 }
 
-/**
- * Reformats a log line by reformatting errors, timestamp and adding
- * new relic linking metadata(context)
- *
- * @param {object} logLine log line
- * @param {object} metadata linking metadata
- * @param {object} agent NR agent
- */
-function reformatLogLine(logLine, metadata, agent) {
-  if (logLine.exception === true) {
-    reformatError(logLine)
-  }
-
-  reformatTimestamp(logLine, agent)
-
-  // Add the metadata to the logLine object being logged
-  Object.assign(logLine, metadata)
-}
-
-/**
- * Decorates the log line with  truncated error.message, error.class, and error.stack and removes
- * trace and stack
- *
- * @param {object} logLine a log line
- */
-function reformatError(logLine) {
-  // Due to Winston internals sometimes the error on the logLine object is a string or an
-  // empty object, and so the message property is all we have
-  const errorMessage = logLine.error.message || logLine.message || ''
-
-  logLine['error.message'] = truncate(errorMessage)
-  logLine['error.class'] =
-    logLine.error.name === 'Error' ? logLine.error.constructor.name : logLine.error.name
-  logLine['error.stack'] = truncate(logLine.error.stack)
-  logLine.message = truncate(logLine.message)
-
-  // Removes additional capture of stack to reduce overall payload/log-line size.
-  // The server has a maximum of ~4k characters per line allowed.
-  delete logLine.trace
-  delete logLine.stack
-}
-
-/**
- * Turns timestamp into unix timestamp. If timestamp existed it will move original
- * to `original_timestamp` key
- *
- * @param {object} logLine a log line
- */
-function reformatTimestamp(logLine) {
-  if (logLine.timestamp) {
-    logger.traceOnce(
-      'Overwriting `timestamp` key; assigning original value to `original_timestamp`.'
-    )
-    logLine.original_timestamp = logLine.timestamp
-  }
-  logLine.timestamp = Date.now()
-}
-
 /**
  * winston allows you to compose a logger
  * from an instantiated logger.  Through a series

package/lib/instrumentations.js

@@ -26,6 +26,7 @@ module.exports = function instrumentations() {
     'memcached': { type: MODULE_TYPE.DATASTORE },
     'mongodb': { type: MODULE_TYPE.DATASTORE },
     'mysql': { module: './instrumentation/mysql' },
+    'pino': { module: './instrumentation/pino' },
     'pg': { type: MODULE_TYPE.DATASTORE },
     'q': { type: null },
     'redis': { type: MODULE_TYPE.DATASTORE },

package/lib/metrics/index.js

@@ -41,9 +41,10 @@ const FROM_MILLIS = 1e-3
  * at serialization time, which allows the mappings from name to ID to happen
  * on the fly.
  *
- * @param {Number} apdexT The apdex-tolerating value, for use in creating apdex
+ * @param {number} apdexT The apdex-tolerating value, for use in creating apdex
  *                        statistics.
  * @param {MetricMapper} mapper The mapper that turns metric names into IDs.
+ * @param normalizer
  */
 function Metrics(apdexT, mapper, normalizer) {
   if (apdexT == null || apdexT === '') {
@@ -74,10 +75,10 @@ function Metrics(apdexT, mapper, normalizer) {
  *
  * @param {string} name The name of the metric.
  * @param {string} scope (Optional) The scope to which the metric belongs.
- * @param {Number} duration The duration of the related operation, in milliseconds.
- * @param {Number} exclusive (Optional) The portion of the operation specific to this
+ * @param {number} duration The duration of the related operation, in milliseconds.
+ * @param {number} exclusive (Optional) The portion of the operation specific to this
  *                           metric.
- * @return {Stats} The aggregated data related to this metric.
+ * @returns {Stats} The aggregated data related to this metric.
  */
 Metrics.prototype.measureMilliseconds = measureMilliseconds
 
@@ -92,12 +93,14 @@ function measureMilliseconds(name, scope, duration, exclusive) {
  * the collection will create a set of data before recording the measurement.
  *
  * @param {string} name The name of the metric.
- * @param {Number} size The size of the related operation, in bytes.
- * @return {Stats} The aggregated data related to this metric.
+ * @param {number} size The size of the related operation, in bytes.
+ * @param {number} exclusiveSize The exclusive size of the related operation, in megabytes.
+ * @param {boolean} exact If true, size is interpreted as bytes rather than megabytes
+ * @returns {Stats} The aggregated data related to this metric.
  */
-Metrics.prototype.measureBytes = function measureBytes(name, size) {
+Metrics.prototype.measureBytes = function measureBytes(name, size, exclusiveSize, exact) {
   const stats = this.getOrCreateMetric(name)
-  stats.recordValueInBytes(size)
+  stats.recordValueInBytes(size, exclusiveSize, exact)
   return stats
 }
 
@@ -107,7 +110,7 @@ Metrics.prototype.measureBytes = function measureBytes(name, size) {
  *
  * @param {string} name The name of the requested metric.
  * @param {string} scope (Optional) The scope to which the metric is bound.
- * @return {Stats} The aggregated data for that name.
+ * @returns {Stats} The aggregated data for that name.
  */
 Metrics.prototype.getOrCreateMetric = function getOrCreateMetric(name, scope) {
   const resolved = this._resolve(scope)
@@ -131,8 +134,7 @@ Metrics.prototype.getOrCreateMetric = function getOrCreateMetric(name, scope) {
  *                               a given run, because key transaction metrics
  *                               are set at connect time via server-side
  *                               configuration.
- *
- * @return {ApdexStats} The aggregated data for that name.
+ * @returns {ApdexStats} The aggregated data for that name.
  */
 Metrics.prototype.getOrCreateApdexMetric = getOrCreateApdexMetric
 
@@ -163,7 +165,7 @@ function getOrCreateApdexMetric(name, scope, overrideApdex) {
  * @param {string} name Metric name.
  * @param {string} scope (Optional) The scope, if any, to which the metric
  *                       belongs.
- * @return {object} Either a stats aggregate, an apdex stats aggregate, or
+ * @returns {object} Either a stats aggregate, an apdex stats aggregate, or
  *                  undefined.
  */
 Metrics.prototype.getMetric = function getMetric(name, scope) {
@@ -179,7 +181,7 @@ Metrics.prototype.getMetric = function getMetric(name, scope) {
  * by JSON.stringify and delivery to the collector. Hope you like nested
  * arrays!
  *
- * @return {Object} Set of nested arrays containing metric information.
+ * @returns {object} Set of nested arrays containing metric information.
  */
 Metrics.prototype.toJSON = function toJSON() {
   return this._toUnscopedData().concat(this._toScopedData())
@@ -195,7 +197,6 @@ Metrics.prototype.toJSON = function toJSON() {
  *
  * @param {Metrics} other
  *  The collection to be folded into this one.
- *
  * @param {boolean} adjustStartTime
  *  If the start time for the timeslice should be adjusted.
  */
@@ -228,7 +229,7 @@ function _merge(a, b) {
  * already exist.
  *
  * @param {string} scope (Optional) The scope to look up.
- * @return {object} The namespace associated with the provided scope, or the
+ * @returns {object} The namespace associated with the provided scope, or the
  *                  un-scoped metrics if the scope isn't set.
  */
 Metrics.prototype._resolve = function _resolve(scope) {
@@ -268,6 +269,7 @@ Metrics.prototype._getUnscopedData = function _getUnscopedData(name) {
  * mappings along the way. Split from _getUnscopedData for performance.
  *
  * @param {string} name The string to look up.
+ * @param scope
  */
 Metrics.prototype._getScopedData = function _getScopedData(name, scope) {
   if (!this.scoped[scope][name]) {
@@ -283,7 +285,7 @@ Metrics.prototype._getScopedData = function _getScopedData(name, scope) {
 }
 
 /**
- * @return {object} A serializable version of the unscoped metrics. Intended
+ * @returns {object} A serializable version of the unscoped metrics. Intended
  *                  for use by toJSON.
  */
 Metrics.prototype._toUnscopedData = function _toUnscopedData() {
@@ -300,7 +302,7 @@ Metrics.prototype._toUnscopedData = function _toUnscopedData() {
 }
 
 /**
- * @return {object} A serializable version of the scoped metrics. Intended for
+ * @returns {object} A serializable version of the scoped metrics. Intended for
  *                  use by toJSON.
  */
 Metrics.prototype._toScopedData = function _toScopedData() {

package/lib/metrics/metric-aggregator.js

@@ -75,7 +75,7 @@ class MetricAggregator extends Aggregator {
    *
    * @param {string} name The name of the requested metric.
    * @param {string} scope (Optional) The scope to which the metric is bound.
-   * @return {Stats} The aggregated data for that name.
+   * @returns {Stats} The aggregated data for that name.
    */
   getOrCreateMetric(name, scope) {
     return this._metrics.getOrCreateMetric(name, scope)
@@ -90,10 +90,10 @@ class MetricAggregator extends Aggregator {
    *
    * @param {string} name The name of the metric.
    * @param {string} scope (Optional) The scope to which the metric belongs.
-   * @param {Number} duration The duration of the related operation, in milliseconds.
-   * @param {Number} exclusive (Optional) The portion of the operation specific to this
+   * @param {number} duration The duration of the related operation, in milliseconds.
+   * @param {number} exclusive (Optional) The portion of the operation specific to this
    *                           metric.
-   * @return {Stats} The aggregated data related to this metric.
+   * @returns {Stats} The aggregated data related to this metric.
    */
   measureMilliseconds(name, scope, duration, exclusive) {
     return this._metrics.measureMilliseconds(name, scope, duration, exclusive)
@@ -102,13 +102,18 @@ class MetricAggregator extends Aggregator {
   /**
    * Set the size of an operation. If there are no data for the name existing,
    * the collection will create a set of data before recording the measurement.
+   * If data do exist for the given name, the value will be incremented by the
+   * given size. Use `exclusiveSize` to set the size of this specific operation,
+   * if it is different from the overall size of the operation.
    *
    * @param {string} name The name of the metric.
-   * @param {Number} size The size of the related operation, in bytes.
-   * @return {Stats} The aggregated data related to this metric.
+   * @param {number} size The size of the related operation, in megabytes.
+   * @param {number} exclusiveSize The exclusive size of the related operation, in megabytes.
+   * @param {boolean} exact If true, size is interpreted as bytes rather than megabytes
+   * @returns {Stats} The aggregated data related to this metric.
    */
-  measureBytes(name, size) {
-    return this._metrics.measureBytes(name, size)
+  measureBytes(name, size, exclusiveSize, exact) {
+    return this._metrics.measureBytes(name, size, exclusiveSize, exact)
   }
 
   /**
@@ -119,7 +124,7 @@ class MetricAggregator extends Aggregator {
    * @param {string} name Metric name.
    * @param {string} scope (Optional) The scope, if any, to which the metric
    *                       belongs.
-   * @return {object} Either a stats aggregate, an apdex stats aggregate, or
+   * @returns {object} Either a stats aggregate, an apdex stats aggregate, or
    *                  undefined.
    */
   getMetric(name, scope) {
@@ -138,8 +143,7 @@ class MetricAggregator extends Aggregator {
    *                               a given run, because key transaction metrics
    *                               are set at connect time via server-side
    *                               configuration.
-   *
-   * @return {ApdexStats} The aggregated data for that name.
+   * @returns {ApdexStats} The aggregated data for that name.
    */
   getOrCreateApdexMetric(name, scope, overrideApdex) {
     return this._metrics.getOrCreateApdexMetric(name, scope, overrideApdex)

package/lib/metrics/names.js

@@ -204,6 +204,15 @@ const EVENT_HARVEST = {
   }
 }
 
+const DATA_USAGE_PREFIX = `${SUPPORTABILITY.NODEJS}/Collector`
+const DATA_USAGE_SUFFIX = 'Output/Bytes'
+
+const DATA_USAGE = {
+  SUFFIX: DATA_USAGE_SUFFIX,
+  PREFIX: DATA_USAGE_PREFIX,
+  COLLECTOR: `${DATA_USAGE_PREFIX}/${DATA_USAGE_SUFFIX}`
+}
+
 const WEB = {
   RESPONSE_TIME: 'WebTransaction',
   FRAMEWORK_PREFIX: 'WebFrameworkUri',
@@ -295,6 +304,7 @@ module.exports = {
   CPU: CPU,
   CUSTOM: 'Custom',
   CUSTOM_EVENTS: CUSTOM_EVENTS,
+  DATA_USAGE,
   DB: DB,
   DISTRIBUTED_TRACE,
   ERRORS: ERRORS,

package/lib/stats/index.js

@@ -28,8 +28,8 @@ function Stats() {
 /**
  * Update the summary statistics with a new value.
  *
- * @param {Number} totalTime Time, in seconds, of the measurement.
- * @param {Number} exclusiveTime Time that was taken by only the
+ * @param {number} totalTime Time, in seconds, of the measurement.
+ * @param {number} exclusiveTime Time that was taken by only the
  *                               current measurement (optional).
  */
 Stats.prototype.recordValue = function recordValue(totalTime, exclusiveTime) {
@@ -63,9 +63,15 @@ function recordValueInMillis(totalTime, exclusiveTime) {
   this.recordValue(totalTime * FROM_MILLIS, exclusiveTime >= 0 ? exclusiveTime * FROM_MILLIS : null)
 }
 
-Stats.prototype.recordValueInBytes = function recordValueInBytes(bytes, exclusiveBytes) {
-  exclusiveBytes = exclusiveBytes || bytes
-  this.recordValue(bytes / BYTES_PER_MB, exclusiveBytes / BYTES_PER_MB)
+Stats.prototype.recordValueInBytes = function recordValueInBytes(bytes, exclusiveBytes, exact) {
+  exclusiveBytes = typeof exclusiveBytes === 'number' ? exclusiveBytes : bytes
+  if (!exact) {
+    // normally values are recorded in megabytes and so must be converted from bytes.
+    // set exact=true to set the byte value directly.
+    bytes = bytes / BYTES_PER_MB
+    exclusiveBytes = exclusiveBytes / BYTES_PER_MB
+  }
+  this.recordValue(bytes, exclusiveBytes)
 }
 
 Stats.prototype.incrementCallCount = function incrementCallCount(count) {
@@ -77,6 +83,8 @@ Stats.prototype.incrementCallCount = function incrementCallCount(count) {
 
 /**
  * Fold another summary's statistics into this one.
+ *
+ * @param other
  */
 Stats.prototype.merge = function merge(other) {
   if (other.count && !other.callCount) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "newrelic",
-  "version": "8.11.2",
+  "version": "8.13.1",
   "author": "New Relic Node.js agent team <nodejs@newrelic.com>",
   "license": "Apache-2.0",
   "contributors": [
@@ -160,7 +160,7 @@
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.5.5",
-    "@grpc/proto-loader": "^0.6.9",
+    "@grpc/proto-loader": "^0.6.12",
     "@newrelic/aws-sdk": "^4.1.1",
     "@newrelic/koa": "^6.1.1",
     "@newrelic/superagent": "^5.1.0",
@@ -170,7 +170,8 @@
     "https-proxy-agent": "^5.0.0",
     "json-stringify-safe": "^5.0.0",
     "readable-stream": "^3.6.0",
-    "semver": "^5.3.0"
+    "semver": "^5.3.0",
+    "winston-transport": "^4.5.0"
   },
   "optionalDependencies": {
     "@newrelic/native-metrics": "^8.0.0"
@@ -179,7 +180,7 @@
     "@newrelic/eslint-config": "^0.0.3",
     "@newrelic/newrelic-oss-cli": "^0.1.2",
     "@newrelic/proxy": "^2.0.0",
-    "@newrelic/test-utilities": "^6.5.2",
+    "@newrelic/test-utilities": "^6.5.4",
     "@octokit/rest": "^18.0.15",
     "@slack/bolt": "^3.7.0",
     "ajv": "^6.12.6",