newrelic 9.1.0 → 9.2.0

package/NEWS.md

@@ -1,27 +1,43 @@
-### v9.1.0 (2022-09-22)
-
-* Added [experimental loader](https://nodejs.org/api/esm.html#loaders) to support instrumentation of CommonJS packages in ECMAScript Module(ESM) applications.
-  * It only supports versions of Node.js >= `16.12.0`.
-  * It is subject to change due to its experimental stability.
-
-* Enhanced supportability metrics for ESM support.
-  * Added new metrics to track usage of ESM loader(`Supportability/Features/ESM/Loader` and `Supportability/Features/ESM/UnsupportedLoader`).
-  * Updated instrumentation map to include an optional "friendly name" for tracking metrics.
-
-* Enabled re-throwing ESM import errors of `newrelic.js` so that the user is informed to rename it to `newrelic.cjs`
-
-* Fixed an issue with mongodb instrumentation where IPv6 address([::1]) was not getting mapped to localhost when setting the host attribute on the segment.
-
-* Added a test ESM loader to properly mock out agent in versioned tests.
-
-* Added ESM versioned tests for: `express`, `pg`, `mongodb`, and `@grpc/grpc-js`.
-
-### v9.0.3 (2022-09-06)
-
-* Updated gRPC client instrumentation to respect `grpc.record_errors` when deciding to log errors on gRPC client requests.
-
-* Fixed transaction name finalization to properly copy the appropriate transaction name to root segment.
-
+### v9.2.0 (2022-10-06)
+
+* Added ability to instrument ES Modules with the New Relic ESM Loader.
+  * [Example ESM application](https://github.com/newrelic/newrelic-node-examples/tree/main/esm-app)
+
+* Added support for custom ESM instrumentation.
+  * There is structure to registering custom ESM instrumentation.  Set the relative path to the instrumentation entry point via `api.esm.custom_instrumentation_entrypoint`
+  * [Sample custom ESM instrumentation entrypoint](https://github.com/newrelic/newrelic-node-examples/blob/main/esm-app/custom-instrumentation/index.js)
+  * All the `newrelic.instrument*` methods will still work except `newrelic.instrumentLoadedModule`.  This is because it is geared toward CommonJS modules. 
+
+* Added test for asserting ESM loader functionality on ESM-only package
+
+* Added supportability metric of `Supportability/Nodejs/Collector/MaxPayloadSizeLimit/<endpoint>` when `max_payload_size_in_bytes` configuration value is exceeded.
+
+* Removed `application_logging.forwarding.enabled` stanza from sample config as the feature is now enabled by default.
+
+### v9.1.0 (2022-09-22)
+
+* Added [experimental loader](https://nodejs.org/api/esm.html#loaders) to support instrumentation of CommonJS packages in ECMAScript Module(ESM) applications.
+  * It only supports versions of Node.js >= `16.12.0`.
+  * It is subject to change due to its experimental stability.
+
+* Enhanced supportability metrics for ESM support.
+  * Added new metrics to track usage of ESM loader(`Supportability/Features/ESM/Loader` and `Supportability/Features/ESM/UnsupportedLoader`).
+  * Updated instrumentation map to include an optional "friendly name" for tracking metrics.
+
+* Enabled re-throwing ESM import errors of `newrelic.js` so that the user is informed to rename it to `newrelic.cjs`
+
+* Fixed an issue with mongodb instrumentation where IPv6 address([::1]) was not getting mapped to localhost when setting the host attribute on the segment.
+
+* Added a test ESM loader to properly mock out agent in versioned tests.
+
+* Added ESM versioned tests for: `express`, `pg`, `mongodb`, and `@grpc/grpc-js`.
+
+### v9.0.3 (2022-09-06)
+
+* Updated gRPC client instrumentation to respect `grpc.record_errors` when deciding to log errors on gRPC client requests.
+
+* Fixed transaction name finalization to properly copy the appropriate transaction name to root segment.
+
 ### v9.0.2 (2022-08-23)
 
 * Added unit test suite for `lib/logger.js`.

package/THIRD_PARTY_NOTICES.md

@@ -1334,7 +1334,7 @@ SOFTWARE.
 
 ### @newrelic/eslint-config
 
-This product includes source derived from [@newrelic/eslint-config](https://github.com/newrelic/eslint-config-newrelic) ([v0.1.0](https://github.com/newrelic/eslint-config-newrelic/tree/v0.1.0)), distributed under the [Apache-2.0 License](https://github.com/newrelic/eslint-config-newrelic/blob/v0.1.0/LICENSE):
+This product includes source derived from [@newrelic/eslint-config](https://github.com/newrelic/eslint-config-newrelic) ([v0.2.0](https://github.com/newrelic/eslint-config-newrelic/tree/v0.2.0)), distributed under the [Apache-2.0 License](https://github.com/newrelic/eslint-config-newrelic/blob/v0.2.0/LICENSE):
 
 ```
                                  Apache License
@@ -2224,7 +2224,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 ### eslint-plugin-jsdoc
 
-This product includes source derived from [eslint-plugin-jsdoc](https://github.com/gajus/eslint-plugin-jsdoc) ([v36.1.1](https://github.com/gajus/eslint-plugin-jsdoc/tree/v36.1.1)), distributed under the [BSD-3-Clause License](https://github.com/gajus/eslint-plugin-jsdoc/blob/v36.1.1/LICENSE):
+This product includes source derived from [eslint-plugin-jsdoc](https://github.com/gajus/eslint-plugin-jsdoc) ([v39.3.6](https://github.com/gajus/eslint-plugin-jsdoc/tree/v39.3.6)), distributed under the [BSD-3-Clause License](https://github.com/gajus/eslint-plugin-jsdoc/blob/v39.3.6/LICENSE):
 
 ```
 Copyright (c) 2018, Gajus Kuizinas (http://gajus.com/)
@@ -2318,10 +2318,10 @@ OTHER DEALINGS IN THE SOFTWARE.
 
 ### eslint
 
-This product includes source derived from [eslint](https://github.com/eslint/eslint) ([v7.32.0](https://github.com/eslint/eslint/tree/v7.32.0)), distributed under the [MIT License](https://github.com/eslint/eslint/blob/v7.32.0/LICENSE):
+This product includes source derived from [eslint](https://github.com/eslint/eslint) ([v8.24.0](https://github.com/eslint/eslint/tree/v8.24.0)), distributed under the [MIT License](https://github.com/eslint/eslint/blob/v8.24.0/LICENSE):
 
 ```
-Copyright JS Foundation and other contributors, https://js.foundation
+Copyright OpenJS Foundation and other contributors, <www.openjsf.org>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/api.js

@@ -1298,7 +1298,9 @@ API.prototype.instrumentMessages = function instrumentMessages(moduleName, onReq
 }
 
 /**
- * Applies an instrumentation to an already loaded module.
+ * Applies an instrumentation to an already loaded CommonJs module.
+ *
+ * Note: This function will not work for ESM packages.
  *
  *    // oh no, express was loaded before newrelic
  *    const express   = require('express')

package/esm-loader.mjs

@@ -8,14 +8,30 @@ import shimmer from './lib/shimmer.js'
 import loggingModule from './lib/logger.js'
 import NAMES from './lib/metrics/names.js'
 import semver from 'semver'
+import path from 'node:path'
 
 const isSupportedVersion = () => semver.gte(process.version, 'v16.12.0')
+// This check will prevent resolve hooks executing from within this file
+// If I do `import('foo')` in here it'll hit the resolve hook multiple times
+const isFromEsmLoader = (context) =>
+  context && context.parentURL && context.parentURL.includes('newrelic/esm-loader.mjs')
+
 const logger = loggingModule.child({ component: 'esm-loader' })
+const esmShimPath = new URL('./lib/esm-shim.mjs', import.meta.url)
+const customEntryPoint = newrelic?.agent?.config?.api.esm.custom_instrumentation_entrypoint
 
-if (newrelic.agent) {
-  addESMSupportabilityMetrics(newrelic.agent)
+// Hook point within agent for customers to register their custom instrumentation.
+if (customEntryPoint) {
+  const resolvedEntryPoint = path.resolve(customEntryPoint)
+  logger.debug('Registering custom ESM instrumentation at %s', resolvedEntryPoint)
+  await import(resolvedEntryPoint)
 }
 
+addESMSupportabilityMetrics(newrelic.agent)
+
+// exporting for testing purposes
+export const registeredSpecifiers = new Map()
+
 /**
  * Hook chain responsible for resolving a file URL for a given module specifier
  *
@@ -24,17 +40,14 @@ if (newrelic.agent) {
  *
  * Docs: https://nodejs.org/api/esm.html#resolvespecifier-context-nextresolve
  *
- * @param {string} specifier
- *        String identifier in an import statement or import() expression
- * @param {object} context
- *        Metadata about the specifier, including url of the parent module and any import assertions
+ * @param {string} specifier string identifier in an import statement or import() expression
+ * @param {object} context metadata about the specifier, including url of the parent module and any import assertions
  *        Optional argument that only needs to be passed when changed
- * @param {Function} nextResolve
- *        The Node.js default resolve hook
+ * @param {Function} nextResolve The subsequent resolve hook in the chain, or the Node.js default resolve hook after the last user-supplied resolve hook
  * @returns {Promise} Promise object representing the resolution of a given specifier
  */
 export async function resolve(specifier, context, nextResolve) {
-  if (!newrelic.agent || !isSupportedVersion()) {
+  if (!newrelic.agent || !isSupportedVersion() || isFromEsmLoader(context)) {
     return nextResolve(specifier, context, nextResolve)
   }
 
@@ -49,14 +62,15 @@ export async function resolve(specifier, context, nextResolve) {
   const instrumentationDefinition = shimmer.registeredInstrumentations[instrumentationName]
 
   if (instrumentationDefinition) {
-    logger.debug(`Instrumentation exists for ${specifier}`)
+    const { url, format } = resolvedModule
+    logger.debug(`Instrumentation exists for ${specifier} ${format} package.`)
 
-    if (resolvedModule.format === 'commonjs') {
+    if (format === 'commonjs') {
       // ES Modules translate import statements into fully qualified filepaths, so we create a copy of our instrumentation under this filepath
       const instrumentationDefinitionCopy = Object.assign({}, instrumentationDefinition)
 
       // Stripping the prefix is necessary because the code downstream gets this url without it
-      instrumentationDefinitionCopy.moduleName = resolvedModule.url.replace('file://', '')
+      instrumentationDefinitionCopy.moduleName = url.replace('file://', '')
 
       // Added to keep our Supportability metrics from exploding/including customer info via full filepath
       instrumentationDefinitionCopy.specifier = specifier
@@ -66,14 +80,72 @@ export async function resolve(specifier, context, nextResolve) {
       logger.debug(
         `Registered CommonJS instrumentation for ${specifier} under ${instrumentationDefinitionCopy.moduleName}`
       )
+    } else if (format === 'module') {
+      registeredSpecifiers.set(url, specifier)
+      const modifiedUrl = new URL(url)
+      // add a query param to the resolved url so the load hook below knows
+      // to rewrite and wrap the source code
+      modifiedUrl.searchParams.set('hasNrInstrumentation', 'true')
+      resolvedModule.url = modifiedUrl.href
     } else {
-      logger.debug(`${specifier} is not a CommonJS module, skipping for now`)
+      logger.debug(`${specifier} is not a CommonJS nor ESM package, skipping for now.`)
     }
   }
 
   return resolvedModule
 }
 
+/**
+ * Hook chain responsible for determining how a URL should be interpreted, retrieved, and parsed.
+ *
+ * Our loader has to be the last user-supplied loader if chaining is happening,
+ * as we rely on `nextLoad` being the default Node.js resolve hook to load the ESM.
+ *
+ * Docs: https://nodejs.org/dist/latest-v18.x/docs/api/esm.html#loadurl-context-nextload
+ *
+ * @param {string} url the URL returned by the resolve chain
+ * @param {object} context metadata about the url, including conditions, format and import assertions
+ * @param {Function} nextLoad the subsequent load hook in the chain, or the Node.js default load hook after the last user-supplied load hook
+ * @returns {Promise} Promise object representing the load of a given url
+ */
+export async function load(url, context, nextLoad) {
+  if (!newrelic.agent || !isSupportedVersion()) {
+    return nextLoad(url, context, nextLoad)
+  }
+
+  let parsedUrl
+
+  try {
+    parsedUrl = new URL(url)
+  } catch (err) {
+    logger.error('Unable to parse url: %s, msg: %s', url, err.message)
+    return nextLoad(url, context, nextLoad)
+  }
+
+  const hasNrInstrumentation = parsedUrl.searchParams.get('hasNrInstrumentation')
+
+  if (!hasNrInstrumentation) {
+    return nextLoad(url, context, nextLoad)
+  }
+
+  /**
+   * undo the work we did in the resolve hook above
+   * so we can properly rewrite source and not get in an infinite loop
+   */
+  parsedUrl.searchParams.delete('hasNrInstrumentation')
+
+  const originalUrl = parsedUrl.href
+  const specifier = registeredSpecifiers.get(originalUrl)
+  const rewrittenSource = await wrapEsmSource(originalUrl, specifier)
+  logger.debug(`Registered module instrumentation for ${specifier}.`)
+
+  return {
+    format: 'module',
+    source: rewrittenSource,
+    shortCircuit: true
+  }
+}
+
 /**
  * Helper function for determining which of our Supportability metrics to use for the current loader invocation
  *
@@ -81,7 +153,11 @@ export async function resolve(specifier, context, nextResolve) {
  *        instantiation of the New Relic agent
  * @returns {void}
  */
-export function addESMSupportabilityMetrics(agent) {
+function addESMSupportabilityMetrics(agent) {
+  if (!agent) {
+    return
+  }
+
   if (isSupportedVersion()) {
     agent.metrics.getOrCreateMetric(NAMES.FEATURES.ESM.LOADER).incrementCallCount()
   } else {
@@ -91,4 +167,38 @@ export function addESMSupportabilityMetrics(agent) {
     )
     agent.metrics.getOrCreateMetric(NAMES.FEATURES.ESM.UNSUPPORTED_LOADER).incrementCallCount()
   }
+
+  if (customEntryPoint) {
+    agent.metrics.getOrCreateMetric(NAMES.FEATURES.ESM.CUSTOM_INSTRUMENTATION).incrementCallCount()
+  }
+}
+
+/**
+ * Rewrites the source code of a ES module we want to instrument.
+ * This is done by injecting the ESM shim which proxies every property on the exported
+ * module and registers the module with shimmer so instrumentation can be registered properly.
+ *
+ * @param {string} url the URL returned by the resolve chain
+ * @param {string} specifier string identifier in an import statement or import() expression
+ * @returns {string} source code rewritten to wrap with our esm-shim
+ */
+async function wrapEsmSource(url, specifier) {
+  const pkg = await import(url)
+  const props = Object.keys(pkg)
+  const trimmedUrl = url.replace('file://', '')
+
+  return `
+    import wrapModule from '${esmShimPath.href}'
+    import * as _originalModule from '${url}'
+    const _wrappedModule = wrapModule(_originalModule, '${specifier}', '${trimmedUrl}')
+    ${props
+      .map((propName) => {
+        const propertyExportSource = `
+    let _${propName} = _wrappedModule.${propName}
+    export { _${propName} as ${propName} }`
+
+        return propertyExportSource
+      })
+      .join('\n')}
+  `
 }

package/lib/collector/remote-method.js

@@ -235,6 +235,9 @@ RemoteMethod.prototype._safeRequest = function _safeRequest(options) {
   let level = 'trace'
 
   if (!isValidLength(options.body, maxPayloadSize)) {
+    this._agent.metrics
+      .getOrCreateMetric(`${DATA_USAGE.PREFIX}/MaxPayloadSizeLimit/${this.name}`)
+      .incrementCallCount()
     logger.warn(
       'The payload size %d being sent to method %s exceeded the maximum size of %d',
       Buffer.byteLength(options.body, 'utf8'),

package/lib/config/default.js

@@ -26,7 +26,7 @@ exports.config = () => ({
   license_key: '',
   /**
    *
-   * Enables/Disables security politices.  Paste your security policies
+   * Enables/Disables security policies.  Paste your security policies
    * token from the New Relic Admin below.
    *
    * @env NEW_RELIC_SECURITY_POLICIES_TOKEN
@@ -623,7 +623,15 @@ exports.config = () => ({
      *
      * @env NEW_RELIC_API_NOTICE_ERROR
      */
-    notice_error_enabled: true
+    notice_error_enabled: true,
+    /**
+     * Sets the path to custom ESM instrumentation
+     *
+     * @env NEW_RELIC_API_ESM_CUSTOM_INSTRUMENTATION_ENTRYPOINT
+     */
+    esm: {
+      custom_instrumentation_entrypoint: null
+    }
   },
   /**
    * Transaction Events

package/lib/config/env.js

@@ -114,7 +114,10 @@ const ENV_MAPPING = {
   api: {
     custom_attributes_enabled: 'NEW_RELIC_API_CUSTOM_ATTRIBUTES',
     custom_events_enabled: 'NEW_RELIC_API_CUSTOM_EVENTS',
-    notice_error_enabled: 'NEW_RELIC_API_NOTICE_ERROR'
+    notice_error_enabled: 'NEW_RELIC_API_NOTICE_ERROR',
+    esm: {
+      custom_instrumentation_entrypoint: 'NEW_RELIC_API_ESM_CUSTOM_INSTRUMENTATION_ENTRYPOINT'
+    }
   },
   datastore_tracer: {
     instance_reporting: {

package/lib/esm-shim.mjs

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2022 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import shimmer from './shimmer.js'
+import newrelic from '../index.js'
+const { agent } = newrelic
+
+export default function wrapModule(module, moduleName, resolvedPath) {
+  const proxiedProps = Object.assign(Object.create(null), module)
+  const proxiedModule = new Proxy(module, {
+    get: (target, key) => {
+      return proxiedProps[key] || target[key]
+    },
+    set: (target, key, val) => {
+      return (proxiedProps[key] = val)
+    },
+    /**
+     * This is normally not allowed.
+     *  We have some things that define property, unsure if at the module level.
+     * Ideally, we can get away from doing this sort of thing at all in the future
+     * in a post AsyncLocalStorage world, etc.
+     * If we never do at the module level, perhaps we can not define this or defer upstream
+     *
+     * @param target
+     * @param key
+     * @param descriptor
+     */
+    defineProperty: (target, key, descriptor) => {
+      return Object.defineProperty(proxiedProps, key, descriptor)
+    }
+  })
+
+  return shimmer.instrumentPostLoad(agent, proxiedModule, moduleName, resolvedPath, true)
+}

package/lib/metrics/names.js

@@ -259,7 +259,8 @@ const INFINITE_TRACING = {
 const FEATURES = {
   ESM: {
     LOADER: `${SUPPORTABILITY.FEATURES}/ESM/Loader`,
-    UNSUPPORTED_LOADER: `${SUPPORTABILITY.FEATURES}/ESM/UnsupportedLoader`
+    UNSUPPORTED_LOADER: `${SUPPORTABILITY.FEATURES}/ESM/UnsupportedLoader`,
+    CUSTOM_INSTRUMENTATION: `${SUPPORTABILITY.FEATURES}/ESM/CustomInstrumentation`
   },
   CERTIFICATES: SUPPORTABILITY.FEATURES + '/Certificates',
   INSTRUMENTATION: {

package/lib/shimmer.js

@@ -467,12 +467,12 @@ const shimmer = (module.exports = {
     return instrumentation
   },
 
-  instrumentPostLoad(agent, module, moduleName, resolvedName) {
+  instrumentPostLoad(agent, module, moduleName, resolvedName, returnModule = false) {
     const result = _postLoad(agent, module, moduleName, resolvedName)
     // This is to not break the public API
     // previously it would just call instrumentation
     // and not check the result
-    return !!result.__NR_instrumented
+    return returnModule ? result : !!result.__NR_instrumented
   }
 })
 

package/newrelic.js

@@ -28,14 +28,6 @@ exports.config = {
    * attributes include/exclude lists.
    */
   allow_all_headers: true,
-  application_logging: {
-    forwarding: {
-      /**
-       * Toggles whether the agent gathers log records for sending to New Relic.
-       */
-      enabled: true
-    }
-  },
   attributes: {
     /**
      * Prefix of attributes to exclude from all destinations. Allows * as wildcard

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "newrelic",
-  "version": "9.1.0",
+  "version": "9.2.0",
   "author": "New Relic Node.js agent team <nodejs@newrelic.com>",
   "license": "Apache-2.0",
   "contributors": [
@@ -191,7 +191,7 @@
     "@newrelic/native-metrics": "^9.0.0"
   },
   "devDependencies": {
-    "@newrelic/eslint-config": "^0.1.0",
+    "@newrelic/eslint-config": "^0.2.0",
     "@newrelic/newrelic-oss-cli": "^0.1.2",
     "@newrelic/proxy": "^2.0.0",
     "@newrelic/test-utilities": "^7.1.1",
@@ -201,11 +201,11 @@
     "async": "^3.2.4",
     "chai": "^4.1.2",
     "commander": "^7.0.0",
-    "eslint": "^7.32.0",
+    "eslint": "^8.24.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-disable": "^2.0.1",
     "eslint-plugin-header": "^3.1.1",
-    "eslint-plugin-jsdoc": "^36.1.0",
+    "eslint-plugin-jsdoc": "^39.3.6",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prettier": "^3.4.0",
     "express": "*",