newrelic 11.4.0 → 11.6.0

package/lib/agent.js

@@ -197,6 +197,8 @@ function Agent(config) {
   // Used by shutdown code as well as entity tracking stats
   this.activeTransactions = 0
 
+  this.llm = {}
+
   // Finally, add listeners for the agent's own events.
   this.on('transactionFinished', this._transactionFinished.bind(this))
 }

package/lib/collector/remote-method.js

@@ -360,7 +360,7 @@ RemoteMethod.prototype._path = function _path() {
  * @param {object} options
  * @param {number} options.body       - Data to be sent.
  * @param {object} options.nrHeaders  - NR request headers from the connect response.
- * @param {bool}   options.compressed - The compression method used, if any.
+ * @param {boolean}   options.compressed - The compression method used, if any.
  */
 RemoteMethod.prototype._headers = function _headers(options) {
   const agent = this._userAgent()

package/lib/config/default.js

@@ -1345,6 +1345,20 @@ defaultConfig.definition = () => ({
       formatter: boolean,
       default: false
     }
+  },
+
+  /**
+   * When enabled, instrumentation of supported AI libraries will be in
+   * effect.
+   */
+  ai_monitoring: {
+    /**
+     * Toggles the generation of AI monitoring events by the agent.
+     */
+    enabled: {
+      formatter: boolean,
+      default: false
+    }
   }
 })
 

package/lib/config/hsm.js

@@ -55,6 +55,9 @@ const HIGH_SECURITY_SETTINGS = {
     forwarding: {
       enabled: false
     }
+  },
+  ai_monitoring: {
+    enabled: false
   }
 }
 

package/lib/instrumentation/@elastic/elasticsearch.js

@@ -19,7 +19,7 @@ const { isNotEmpty } = require('../../util/objects')
  * @returns {void}
  */
 module.exports = function initialize(_agent, elastic, _moduleName, shim) {
-  const pkgVersion = shim.require('./package.json').version
+  const pkgVersion = shim.pkgVersion
   if (semver.lt(pkgVersion, '7.13.0')) {
     shim &&
       shim.logger.debug(
@@ -67,7 +67,7 @@ function queryParser(params) {
   // let body or bulkBody override querystring, as some requests have both
   if (isNotEmpty(params.body)) {
     queryParam = params.body
-  } else if (isNotEmpty(params.bulkBody)) {
+  } else if (Array.isArray(params.bulkBody) && params.bulkBody.length) {
     queryParam = params.bulkBody
   }
 

package/lib/instrumentation/@nestjs/core.js

@@ -8,7 +8,7 @@ const logger = require('../../logger').child({ component: 'nestjs' })
 const semver = require('semver')
 
 module.exports = function initialize(agent, core, moduleName, shim) {
-  const nestJsVersion = shim.require('./package.json').version
+  const nestJsVersion = shim.pkgVersion
   shim.setFramework(shim.NEST)
   // Earliest version that runs in the tests
   if (semver.lt(nestJsVersion, '8.0.0')) {

package/lib/instrumentation/@prisma/client.js

@@ -154,7 +154,7 @@ function extractPrismaDatasource(client) {
  * @param {object} shim New Relic shim
  */
 module.exports = async function initialize(_agent, prisma, _moduleName, shim) {
-  const pkgVersion = shim.require('./package.json').version
+  const pkgVersion = shim.pkgVersion
   if (semver.lt(pkgVersion, '4.0.0')) {
     logger.warn(
       'Skipping instrumentation of @prisma/client.  Minimum supported version of library is 4.0.0, actual version %s',

package/lib/instrumentation/cassandra-driver.js

@@ -17,7 +17,7 @@ const semver = require('semver')
  * @param {object} shim - shim for instrumentation
  */
 module.exports = function initialize(_agent, cassandra, _moduleName, shim) {
-  const cassandraVersion = shim.require('./package.json').version
+  const cassandraVersion = shim.pkgVersion
 
   shim.setDatastore(shim.CASSANDRA)
 

package/lib/instrumentation/fastify.js

@@ -86,7 +86,7 @@ const setupRouteHandler = (shim, fastify) => {
 module.exports = function initialize(agent, fastify, moduleName, shim) {
   shim.setFramework(shim.FASTIFY)
 
-  const fastifyVersion = shim.require('./package.json').version
+  const fastifyVersion = shim.pkgVersion
   const isv3Plus = semver.satisfies(fastifyVersion, '>=3.0.0')
 
   /**

package/lib/instrumentation/grpc-js/grpc.js

@@ -20,7 +20,7 @@ module.exports.wrapStartCall = function wrappedClient(shim, callStream) {
 }
 
 module.exports.wrapServer = function wrapServer(shim, server) {
-  const grpcVersion = shim.require('./package.json').version
+  const grpcVersion = shim.pkgVersion
   if (semver.lt(grpcVersion, '1.4.0')) {
     shim.logger.debug('gRPC server-side instrumentation only supported on grpc-js >=1.4.0')
     return

package/lib/instrumentation/mongodb.js

@@ -36,7 +36,7 @@ function initialize(agent, mongodb, moduleName, shim) {
 
   shim.setDatastore(shim.MONGODB)
 
-  const mongoVersion = shim.require('./package.json').version
+  const mongoVersion = shim.pkgVersion
   if (semver.satisfies(mongoVersion, '>=4.0.0')) {
     instrumentV4(shim, mongodb)
   } else if (semver.satisfies(mongoVersion, '>=3.0.6')) {

package/lib/instrumentation/openai.js

@@ -0,0 +1,329 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const { openAiHeaders, openAiApiKey } = require('../../lib/symbols')
+const {
+  LlmChatCompletionMessage,
+  LlmChatCompletionSummary,
+  LlmEmbedding,
+  LlmErrorMessage
+} = require('../../lib/llm-events/openai')
+const LlmTrackedIds = require('../../lib/llm-events/tracked-ids')
+
+const MIN_VERSION = '4.0.0'
+const MIN_STREAM_VERSION = '4.12.2'
+const { AI } = require('../../lib/metrics/names')
+const semver = require('semver')
+
+let TRACKING_METRIC = AI.TRACKING_PREFIX
+
+/**
+ * Checks if we should skip instrumentation.
+ * Currently it checks if `ai_monitoring.enabled` is true
+ * and the package version >= 4.0.0
+ *
+ * @param {object} config agent config
+ * @param {Shim} shim instance of shim
+ * @returns {boolean} flag if instrumentation should be skipped
+ */
+function shouldSkipInstrumentation(config, shim) {
+  if (config?.ai_monitoring?.enabled !== true) {
+    shim.logger.debug('config.ai_monitoring.enabled is set to false.')
+    return true
+  }
+
+  const { pkgVersion } = shim
+  return semver.lt(pkgVersion, MIN_VERSION)
+}
+
+// eslint-disable-next-line sonarjs/cognitive-complexity
+module.exports = function initialize(agent, openai, moduleName, shim) {
+  if (shouldSkipInstrumentation(agent.config, shim)) {
+    shim.logger.debug(
+      `${moduleName} instrumentation support is for versions >=${MIN_VERSION}. Skipping instrumentation.`
+    )
+    return
+  }
+
+  // Update the tracking metric name with the version of the library
+  // being instrumented. We do not have access to the version when
+  // initially declaring the variable.
+  TRACKING_METRIC = `${TRACKING_METRIC}OpenAI/${shim.pkgVersion}`
+
+  /**
+   * Adds apiKey and response headers to the active segment
+   * on symbols
+   *
+   * @param {object} result from openai request
+   * @param {string} apiKey api key from openai client
+   */
+  function decorateSegment(result, apiKey) {
+    const segment = shim.getActiveSegment()
+
+    if (segment) {
+      segment[openAiApiKey] = apiKey
+
+      // If the result is an error, which is an OpenAI client error, then
+      // the headers are provided via a proxy attached to `result.headers`.
+      // Otherwise, result is a typical response-like object.
+      const headers = result?.response?.headers
+        ? Object.fromEntries(result.response.headers)
+        : { ...result?.headers }
+      segment[openAiHeaders] = headers
+    }
+  }
+
+  /**
+   * Enqueues a LLM event to the custom event aggregator
+   *
+   * @param {string} type of LLM event
+   * @param {object} msg LLM event
+   */
+  function recordEvent(type, msg) {
+    agent.metrics.getOrCreateMetric(TRACKING_METRIC).incrementCallCount()
+    msg = agent?.llm?.metadata ? { ...agent.llm.metadata, ...msg } : msg
+    agent.customEventAggregator.add([{ type, timestamp: Date.now() }, msg])
+  }
+
+  /**
+   * Instrumentation is only done to get the response headers and attach
+   * to the active segment as openai hides the headers from the functions we are
+   * trying to instrument
+   */
+  shim.wrap(openai.prototype, 'makeRequest', function wrapRequest(shim, makeRequest) {
+    return function wrappedRequest() {
+      const apiKey = this.apiKey
+      const result = makeRequest.apply(this, arguments)
+      result.then(
+        (data) => {
+          // add headers on resolve
+          decorateSegment(data, apiKey)
+        },
+        (data) => {
+          // add headers on reject
+          decorateSegment(data, apiKey)
+        }
+      )
+      return result
+    }
+  })
+
+  /**
+   * Assigns requestId, conversationId and messageIds for a given
+   * chat completion response on the active transaction.
+   * This is used for generating LlmFeedbackEvent via `api.recordLlmFeedbackEvent`
+   *
+   * @param {object} params input params
+   * @param {Transaction} params.tx active transaction
+   * @param {LlmChatCompletionMessage} params.completionMsg chat completion message
+   * @param {string} params.responseId id of response
+   */
+  function assignIdsToTx({ tx, completionMsg, responseId }) {
+    const tracker = tx.llm.responses
+    const trackedIds =
+      tracker.get(responseId) ??
+      new LlmTrackedIds({
+        requestId: completionMsg.request_id,
+        conversationId: completionMsg.conversation_id
+      })
+    trackedIds.message_ids.push(completionMsg.id)
+    tracker.set(responseId, trackedIds)
+  }
+
+  /**
+   * Generates LlmChatCompletionSummary for a chat completion creation.
+   * Also iterates over both input messages and the first response message
+   * and creates LlmChatCompletionMessage.
+   *
+   * Also assigns relevant ids by response id for LlmFeedbackEvent creation
+   *
+   * @param {object} params input params
+   * @param {TraceSegment} params.segment active segment from chat completion
+   * @param {object} params.request chat completion params
+   * @param {object} params.response chat completion response
+   * @param {boolean} [params.err] err if it exists
+   */
+  function recordChatCompletionMessages({ segment, request, response, err }) {
+    if (!response) {
+      // If we get an error, it is possible that `response = null`.
+      // In that case, we define it to be an empty object.
+      response = {}
+    }
+
+    response.headers = segment[openAiHeaders]
+    response.api_key = segment[openAiApiKey]
+    const tx = segment.transaction
+    // explicitly end segment to consistent duration
+    // for both LLM events and the segment
+    segment.end()
+    const completionSummary = new LlmChatCompletionSummary({
+      agent,
+      segment,
+      request,
+      response,
+      withError: err != null
+    })
+
+    // Only take the first response message and append to input messages
+    const messages = [...request.messages, response?.choices?.[0]?.message]
+    messages.forEach((message, index) => {
+      const completionMsg = new LlmChatCompletionMessage({
+        agent,
+        segment,
+        request,
+        response,
+        index,
+        completionId: completionSummary.id,
+        message
+      })
+
+      assignIdsToTx({ tx, completionMsg, responseId: response.id })
+      recordEvent('LlmChatCompletionMessage', completionMsg)
+    })
+
+    recordEvent('LlmChatCompletionSummary', completionSummary)
+
+    if (err) {
+      const llmError = new LlmErrorMessage({ cause: err, summary: completionSummary, response })
+      shim.agent.errors.add(segment.transaction, err, llmError)
+    }
+
+    delete response.headers
+    delete response.api_key
+  }
+
+  /*
+   * Chat completions create can return a stream once promise resolves
+   * This wraps the iterator which is a generator function
+   * We will call the original iterator, intercept chunks and yield
+   * to the original. On complete we will construct the new message object
+   * with what we have seen in the stream and create the chat completion
+   * messages
+   *
+   */
+  function instrumentStream({ request, response, segment }) {
+    shim.wrap(response, 'iterator', function wrapIterator(shim, orig) {
+      return async function* wrappedIterator() {
+        let content = ''
+        let role = ''
+        let chunk
+        let err
+        try {
+          const iterator = orig.apply(this, arguments)
+
+          for await (chunk of iterator) {
+            if (chunk.choices?.[0]?.delta?.role) {
+              role = chunk.choices[0].delta.role
+            }
+
+            content += chunk.choices?.[0]?.delta?.content ?? ''
+            yield chunk
+          }
+        } catch (streamErr) {
+          err = streamErr
+          throw err
+        } finally {
+          chunk.choices[0].message = { role, content }
+          // update segment duration since we want to extend the time it took to
+          // handle the stream
+          segment.touch()
+
+          recordChatCompletionMessages({
+            segment,
+            request,
+            response: chunk,
+            err
+          })
+        }
+      }
+    })
+  }
+
+  /**
+   * Instruments chat completion creation
+   * and creates the LLM events
+   *
+   * **Note**: Currently only for promises. streams will come later
+   */
+  shim.record(
+    openai.Chat.Completions.prototype,
+    'create',
+    function wrapCreate(shim, create, name, args) {
+      const [request] = args
+      if (request.stream && semver.lt(shim.pkgVersion, MIN_STREAM_VERSION)) {
+        shim.logger.warn(
+          `Instrumenting chat completion streams is only supported with openai version ${MIN_STREAM_VERSION}+.`
+        )
+        return
+      }
+
+      return {
+        name: `${AI.OPEN_AI}/Chat/Completions/Create`,
+        promise: true,
+        // eslint-disable-next-line max-params
+        after(_shim, _fn, _name, err, response, segment) {
+          if (request.stream) {
+            instrumentStream({ request, response, segment })
+          } else {
+            recordChatCompletionMessages({
+              segment,
+              request,
+              response,
+              err
+            })
+          }
+        }
+      }
+    }
+  )
+
+  /**
+   * Instruments embedding creation
+   * and creates LlmEmbedding event
+   */
+  shim.record(
+    openai.Embeddings.prototype,
+    'create',
+    function wrapEmbeddingCreate(shim, embeddingCreate, name, args) {
+      const [request] = args
+      return {
+        name: `${AI.OPEN_AI}/Embeddings/Create`,
+        promise: true,
+        // eslint-disable-next-line max-params
+        after(_shim, _fn, _name, err, response, segment) {
+          if (!response) {
+            // If we get an error, it is possible that `response = null`.
+            // In that case, we define it to be an empty object.
+            response = {}
+          }
+          response.headers = segment[openAiHeaders]
+          response.api_key = segment[openAiApiKey]
+          // explicitly end segment to get consistent duration
+          // for both LLM events and the segment
+          segment.end()
+          const embedding = new LlmEmbedding({
+            agent,
+            segment,
+            request,
+            response,
+            withError: err != null
+          })
+
+          recordEvent('LlmEmbedding', embedding)
+
+          if (err) {
+            const llmError = new LlmErrorMessage({ cause: err, embedding, response })
+            shim.agent.errors.add(segment.transaction, err, llmError)
+          }
+
+          // cleanup keys on response before returning to user code
+          delete response.api_key
+          delete response.headers
+        }
+      }
+    }
+  )
+}

package/lib/instrumentation/pino/pino.js

@@ -16,7 +16,7 @@ const {
 const semver = require('semver')
 
 module.exports = function instrument(shim, tools) {
-  const pinoVersion = shim.require('./package.json').version
+  const pinoVersion = shim.pkgVersion
 
   if (semver.lt(pinoVersion, '7.0.0')) {
     shim.logger.debug('Instrumentation only supported on pino >=7.0.0.')

package/lib/instrumentations.js

@@ -28,6 +28,7 @@ module.exports = function instrumentations() {
     'memcached': { type: MODULE_TYPE.DATASTORE },
     'mongodb': { type: MODULE_TYPE.DATASTORE },
     'mysql': { module: './instrumentation/mysql' },
+    'openai': { type: MODULE_TYPE.GENERIC },
     '@nestjs/core': { type: MODULE_TYPE.WEB_FRAMEWORK },
     'pino': { module: './instrumentation/pino' },
     'pg': { type: MODULE_TYPE.DATASTORE },

package/lib/llm-events/openai/chat-completion-message.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const LlmEvent = require('./event')
+
+module.exports = class LlmChatCompletionMessage extends LlmEvent {
+  constructor({ agent, segment, request = {}, response = {}, index = 0, message, completionId }) {
+    super({ agent, segment, request, response })
+    this.id = `${response.id}-${index}`
+    this.conversation_id = this.conversationId(agent)
+    this.content = message?.content
+    this.role = message?.role
+    this.sequence = index
+    this.completion_id = completionId
+    this.is_response = response?.choices?.[0]?.message?.content === this.content
+  }
+}

package/lib/llm-events/openai/chat-completion-summary.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const LlmEvent = require('./event')
+
+module.exports = class LlmChatCompletionSummary extends LlmEvent {
+  constructor({ agent, segment, request = {}, response = {}, withError = false }) {
+    super({ agent, segment, request, response, responseAttrs: true })
+    this.error = withError
+    this.conversation_id = this.conversationId(agent)
+    this['request.max_tokens'] = request.max_tokens
+    this['request.temperature'] = request.temperature
+    this['response.number_of_messages'] = request?.messages?.length + response?.choices?.length
+    this['response.usage.completion_tokens'] = response?.usage?.completion_tokens
+    this['response.choices.finish_reason'] = response?.choices?.[0]?.finish_reason
+  }
+}

package/lib/llm-events/openai/embedding.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const LlmEvent = require('./event')
+
+module.exports = class LlmEmbedding extends LlmEvent {
+  constructor({ agent, segment, request = {}, response = {}, withError = false }) {
+    super({ agent, segment, request, response, responseAttrs: true })
+    this.error = withError
+    this.input = request.input?.toString()
+  }
+}

package/lib/llm-events/openai/error-message.js

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+
+/**
+ * Represents an error object, to be tracked via `agent.errors`, that is the
+ * result of some error returned from the OpenAI API.
+ */
+module.exports = class LlmErrorMessage {
+  /**
+   * @param {object} params Constructor parameters
+   * @param {object} [params.response] Instance of an incoming message.
+   * @param {object} [params.cause] An instance of the OpenAI error object.
+   * @param {LlmChatCompletionSummary} [params.summary] Details about the
+   * conversation if it was a chat completion conversation.
+   * @param {LlmEmbedding} [params.embedding] Details about the conversation
+   * if it was an embedding conversation.
+   */
+  constructor({ response, cause, summary, embedding } = {}) {
+    this['http.statusCode'] = response?.status ?? cause?.status
+    this['error.message'] = cause?.message
+    this['error.code'] = response?.code ?? cause?.error?.code
+    this['error.param'] = response?.param ?? cause?.error?.param
+    this.completion_id = summary?.id
+    this.embedding_id = embedding?.id
+  }
+}

package/lib/llm-events/openai/event.js

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const { makeId } = require('../../util/hashes')
+const { DESTINATIONS } = require('../../../lib/config/attribute-filter')
+
+module.exports = class LlmEvent {
+  constructor({ agent, segment, request, response, responseAttrs = false }) {
+    this.id = makeId(36)
+    this.appName = agent.config.applications()[0]
+    this.request_id = response?.headers?.['x-request-id']
+    this.trace_id = segment?.transaction?.traceId
+    this.span_id = segment?.id
+    this.transaction_id = segment?.transaction?.id
+    this['response.model'] = response.model
+    this.vendor = 'openAI'
+    this.ingest_source = 'Node'
+
+    /**
+     * Used in embedding, and chat completion summary.
+     * The flag will include response attributes but also
+     * other attributes from request like model, and api key.
+     * Lastly, it includes the active span's duration.
+     */
+    if (responseAttrs) {
+      this['request.model'] = request.model || request.engine
+      this.duration = segment?.getDurationInMillis()
+      this.api_key_last_four_digits = response?.api_key && `sk-${response.api_key.slice(-4)}`
+      this.responseAttrs(response)
+    }
+  }
+
+  responseAttrs(response) {
+    this['response.organization'] = response?.headers?.['openai-organization']
+    this['response.usage.total_tokens'] = response?.usage?.total_tokens
+    this['response.usage.prompt_tokens'] = response?.usage?.prompt_tokens
+    this['response.headers.llmVersion'] = response?.headers?.['openai-version']
+    this['response.headers.ratelimitLimitRequests'] =
+      response?.headers?.['x-ratelimit-limit-requests']
+    this['response.headers.ratelimitLimitTokens'] = response?.headers?.['x-ratelimit-limit-tokens']
+    this['response.headers.ratelimitResetTokens'] = response?.headers?.['x-ratelimit-reset-tokens']
+    this['response.headers.ratelimitRemainingTokens'] =
+      response?.headers?.['x-ratelimit-remaining-tokens']
+    this['response.headers.ratelimitRemainingRequests'] =
+      response?.headers?.['x-ratelimit-remaining-requests']
+  }
+
+  conversationId(agent) {
+    const transaction = agent.tracer.getTransaction()
+    const attrs = transaction?.trace?.custom.get(DESTINATIONS.TRANS_SCOPE)
+    return attrs?.conversation_id
+  }
+}

package/lib/llm-events/openai/feedback-message.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+const { makeId } = require('../../util/hashes')
+
+module.exports = class LlmFeedbackMessage {
+  constructor(opts) {
+    this.id = makeId(32)
+    this.conversation_id = opts.conversationId
+    this.request_id = opts.requestId
+    this.message_id = opts.messageId
+    this.category = opts.category
+    this.rating = opts.rating
+    this.message = opts.message
+    this.ingest_source = 'Node'
+  }
+}

package/lib/llm-events/openai/index.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2023 New Relic Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+'use strict'
+
+const LlmChatCompletionSummary = require('./chat-completion-summary')
+const LlmChatCompletionMessage = require('./chat-completion-message')
+const LlmEmbedding = require('./embedding')
+const LlmFeedbackMessage = require('./feedback-message')
+const LlmErrorMessage = require('./error-message')
+
+module.exports = {
+  LlmChatCompletionMessage,
+  LlmChatCompletionSummary,
+  LlmEmbedding,
+  LlmErrorMessage,
+  LlmFeedbackMessage
+}