dd-trace 5.31.0 → 5.32.0

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/embedding.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const LangChainLLMObsHandler = require('.')
+const LLMObsTagger = require('../../../tagger')
+const { spanHasError } = require('../../../util')
+
+class LangChainLLMObsEmbeddingHandler extends LangChainLLMObsHandler {
+  setMetaTags ({ span, inputs, results }) {
+    const isWorkflow = LLMObsTagger.getSpanKind(span) === 'workflow'
+    let embeddingInput, embeddingOutput
+
+    if (isWorkflow) {
+      embeddingInput = this.formatIO(inputs)
+    } else {
+      const input = Array.isArray(inputs) ? inputs : [inputs]
+      embeddingInput = input.map(doc => ({ text: doc }))
+    }
+
+    if (spanHasError(span) || !results) {
+      embeddingOutput = ''
+    } else {
+      let embeddingDimensions, embeddingsCount
+      if (typeof results[0] === 'number') {
+        embeddingsCount = 1
+        embeddingDimensions = results.length
+      } else {
+        embeddingsCount = results.length
+        embeddingDimensions = results[0].length
+      }
+
+      embeddingOutput = `[${embeddingsCount} embedding(s) returned with size ${embeddingDimensions}]`
+    }
+
+    if (isWorkflow) {
+      this._tagger.tagTextIO(span, embeddingInput, embeddingOutput)
+    } else {
+      this._tagger.tagEmbeddingIO(span, embeddingInput, embeddingOutput)
+    }
+  }
+}
+
+module.exports = LangChainLLMObsEmbeddingHandler

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js

@@ -0,0 +1,102 @@
+'use strict'
+
+const ROLE_MAPPINGS = {
+  human: 'user',
+  ai: 'assistant',
+  system: 'system'
+}
+
+class LangChainLLMObsHandler {
+  constructor (tagger) {
+    this._tagger = tagger
+  }
+
+  setMetaTags () {}
+
+  formatIO (messages) {
+    if (messages.constructor.name === 'Object') { // plain JSON
+      const formatted = {}
+      for (const [key, value] of Object.entries(messages)) {
+        formatted[key] = this.formatIO(value)
+      }
+
+      return formatted
+    } else if (Array.isArray(messages)) {
+      return messages.map(message => this.formatIO(message))
+    } else { // either a BaseMesage type or a string
+      return this.getContentFromMessage(messages)
+    }
+  }
+
+  getContentFromMessage (message) {
+    if (typeof message === 'string') {
+      return message
+    } else {
+      try {
+        const messageContent = {}
+        messageContent.content = message.content || ''
+
+        const role = this.getRole(message)
+        if (role) messageContent.role = role
+
+        return messageContent
+      } catch {
+        return JSON.stringify(message)
+      }
+    }
+  }
+
+  checkTokenUsageChatOrLLMResult (results) {
+    const llmOutput = results.llmOutput
+    const tokens = {
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0
+    }
+    if (!llmOutput) return tokens
+    const tokenUsage = llmOutput.tokenUsage || llmOutput.usageMetadata || llmOutput.usage || {}
+    if (!tokenUsage) return tokens
+
+    tokens.inputTokens = tokenUsage.promptTokens || tokenUsage.inputTokens || 0
+    tokens.outputTokens = tokenUsage.completionTokens || tokenUsage.outputTokens || 0
+    tokens.totalTokens = tokenUsage.totalTokens || tokens.inputTokens + tokens.outputTokens
+
+    return tokens
+  }
+
+  checkTokenUsageFromAIMessage (message) {
+    let usage = message.usage_metadata || message.additional_kwargs?.usage
+    const runId = message.run_id || message.id || ''
+    const runIdBase = runId ? runId.split('-').slice(0, -1).join('-') : ''
+
+    const responseMetadata = message.response_metadata || {}
+    usage = usage || responseMetadata.usage || responseMetadata.tokenUsage || {}
+
+    const inputTokens = usage.promptTokens || usage.inputTokens || usage.prompt_tokens || usage.input_tokens || 0
+    const outputTokens =
+      usage.completionTokens || usage.outputTokens || usage.completion_tokens || usage.output_tokens || 0
+    const totalTokens = usage.totalTokens || inputTokens + outputTokens
+
+    return {
+      tokens: {
+        inputTokens,
+        outputTokens,
+        totalTokens
+      },
+      runId: runIdBase
+    }
+  }
+
+  getRole (message) {
+    if (message.role) return ROLE_MAPPINGS[message.role] || message.role
+
+    const type = (
+      (typeof message.getType === 'function' && message.getType()) ||
+      (typeof message._getType === 'function' && message._getType())
+    )
+
+    return ROLE_MAPPINGS[type] || type
+  }
+}
+
+module.exports = LangChainLLMObsHandler

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/llm.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const LangChainLLMObsHandler = require('.')
+const LLMObsTagger = require('../../../tagger')
+const { spanHasError } = require('../../../util')
+
+class LangChainLLMObsLlmHandler extends LangChainLLMObsHandler {
+  setMetaTags ({ span, inputs, results }) {
+    const isWorkflow = LLMObsTagger.getSpanKind(span) === 'workflow'
+    const prompts = Array.isArray(inputs) ? inputs : [inputs]
+
+    let outputs
+    if (spanHasError(span)) {
+      outputs = [{ content: '' }]
+    } else {
+      outputs = results.generations.map(completion => ({ content: completion[0].text }))
+
+      if (!isWorkflow) {
+        const tokens = this.checkTokenUsageChatOrLLMResult(results)
+        this._tagger.tagMetrics(span, tokens)
+      }
+    }
+
+    if (isWorkflow) {
+      this._tagger.tagTextIO(span, prompts, outputs)
+    } else {
+      this._tagger.tagLLMIO(span, prompts, outputs)
+    }
+  }
+}
+
+module.exports = LangChainLLMObsLlmHandler

package/packages/dd-trace/src/llmobs/plugins/langchain/index.js

@@ -0,0 +1,131 @@
+'use strict'
+
+const log = require('../../../log')
+const LLMObsPlugin = require('../base')
+
+const pluginManager = require('../../../../../..')._pluginManager
+
+const ANTHROPIC_PROVIDER_NAME = 'anthropic'
+const BEDROCK_PROVIDER_NAME = 'amazon_bedrock'
+const OPENAI_PROVIDER_NAME = 'openai'
+
+const SUPPORTED_INTEGRATIONS = ['openai']
+const LLM_SPAN_TYPES = ['llm', 'chat_model', 'embedding']
+const LLM = 'llm'
+const WORKFLOW = 'workflow'
+const EMBEDDING = 'embedding'
+
+const ChainHandler = require('./handlers/chain')
+const ChatModelHandler = require('./handlers/chat_model')
+const LlmHandler = require('./handlers/llm')
+const EmbeddingHandler = require('./handlers/embedding')
+
+class LangChainLLMObsPlugin extends LLMObsPlugin {
+  static get prefix () {
+    return 'tracing:apm:langchain:invoke'
+  }
+
+  constructor () {
+    super(...arguments)
+
+    this._handlers = {
+      chain: new ChainHandler(this._tagger),
+      chat_model: new ChatModelHandler(this._tagger),
+      llm: new LlmHandler(this._tagger),
+      embedding: new EmbeddingHandler(this._tagger)
+    }
+  }
+
+  getLLMObsSpanRegisterOptions (ctx) {
+    const span = ctx.currentStore?.span
+    const tags = span?.context()._tags || {}
+
+    const modelProvider = tags['langchain.request.provider'] // could be undefined
+    const modelName = tags['langchain.request.model'] // could be undefined
+    const kind = this.getKind(ctx.type, modelProvider)
+    const name = tags['resource.name']
+
+    return {
+      modelProvider,
+      modelName,
+      kind,
+      name
+    }
+  }
+
+  setLLMObsTags (ctx) {
+    const span = ctx.currentStore?.span
+    const type = ctx.type // langchain operation type (oneof chain,chat_model,llm,embedding)
+
+    if (!Object.keys(this._handlers).includes(type)) {
+      log.warn(`Unsupported LangChain operation type: ${type}`)
+      return
+    }
+
+    const provider = span?.context()._tags['langchain.request.provider']
+    const integrationName = this.getIntegrationName(type, provider)
+    this.setMetadata(span, provider)
+
+    const inputs = ctx.args?.[0]
+    const options = ctx.args?.[1]
+    const results = ctx.result
+
+    this._handlers[type].setMetaTags({ span, inputs, results, options, integrationName })
+  }
+
+  setMetadata (span, provider) {
+    if (!provider) return
+
+    const metadata = {}
+
+    // these fields won't be set for non model-based operations
+    const temperature =
+      span?.context()._tags[`langchain.request.${provider}.parameters.temperature`] ||
+      span?.context()._tags[`langchain.request.${provider}.parameters.model_kwargs.temperature`]
+
+    const maxTokens =
+      span?.context()._tags[`langchain.request.${provider}.parameters.max_tokens`] ||
+      span?.context()._tags[`langchain.request.${provider}.parameters.maxTokens`] ||
+      span?.context()._tags[`langchain.request.${provider}.parameters.model_kwargs.max_tokens`]
+
+    if (temperature) {
+      metadata.temperature = parseFloat(temperature)
+    }
+
+    if (maxTokens) {
+      metadata.maxTokens = parseInt(maxTokens)
+    }
+
+    this._tagger.tagMetadata(span, metadata)
+  }
+
+  getKind (type, provider) {
+    if (LLM_SPAN_TYPES.includes(type)) {
+      const llmobsIntegration = this.getIntegrationName(type, provider)
+
+      if (!this.isLLMIntegrationEnabled(llmobsIntegration)) {
+        return type === 'embedding' ? EMBEDDING : LLM
+      }
+    }
+
+    return WORKFLOW
+  }
+
+  getIntegrationName (type, provider = 'custom') {
+    if (provider.startsWith(BEDROCK_PROVIDER_NAME)) {
+      return 'bedrock'
+    } else if (provider.startsWith(OPENAI_PROVIDER_NAME)) {
+      return 'openai'
+    } else if (type === 'chat_model' && provider.startsWith(ANTHROPIC_PROVIDER_NAME)) {
+      return 'anthropic'
+    }
+
+    return provider
+  }
+
+  isLLMIntegrationEnabled (integration) {
+    return SUPPORTED_INTEGRATIONS.includes(integration) && pluginManager?._pluginsByName[integration]?.llmobs?._enabled
+  }
+}
+
+module.exports = LangChainLLMObsPlugin

package/packages/dd-trace/src/llmobs/plugins/openai.js

@@ -7,7 +7,7 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
     return 'tracing:apm:openai:request'
   }
 
-  getLLMObsSPanRegisterOptions (ctx) {
+  getLLMObsSpanRegisterOptions (ctx) {
     const resource = ctx.methodName
     const methodName = gateResource(normalizeOpenAIResourceName(resource))
     if (!methodName) return // we will not trace all openai methods for llmobs

package/packages/dd-trace/src/llmobs/tagger.js

@@ -40,6 +40,10 @@ class LLMObsTagger {
     return registry
   }
 
+  static getSpanKind (span) {
+    return registry.get(span)?.[SPAN_KIND]
+  }
+
   registerLLMObsSpan (span, {
     modelName,
     modelProvider,
@@ -60,10 +64,10 @@ class LLMObsTagger {
     if (modelName) this._setTag(span, MODEL_NAME, modelName)
     if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider)
 
-    sessionId = sessionId || parent?.context()._tags[SESSION_ID]
+    sessionId = sessionId || registry.get(parent)?.[SESSION_ID]
     if (sessionId) this._setTag(span, SESSION_ID, sessionId)
 
-    if (!mlApp) mlApp = parent?.context()._tags[ML_APP] || this._config.llmobs.mlApp
+    if (!mlApp) mlApp = registry.get(parent)?.[ML_APP] || this._config.llmobs.mlApp
     this._setTag(span, ML_APP, mlApp)
 
     const parentId =
@@ -136,6 +140,10 @@ class LLMObsTagger {
     this._setTag(span, TAGS, tags)
   }
 
+  changeKind (span, newKind) {
+    this._setTag(span, SPAN_KIND, newKind)
+  }
+
   _tagText (span, data, key) {
     if (data) {
       if (typeof data === 'string') {
@@ -310,7 +318,7 @@ class LLMObsTagger {
   _setTag (span, key, value) {
     if (!this._config.llmobs.enabled) return
     if (!registry.has(span)) {
-      this._handleFailure('Span must be an LLMObs generated span.')
+      this._handleFailure(`Span "${span._name}" must be an LLMObs generated span.`)
       return
     }
 

package/packages/dd-trace/src/llmobs/util.js

@@ -169,8 +169,14 @@ function getFunctionArguments (fn, args = []) {
   }
 }
 
+function spanHasError (span) {
+  const tags = span.context()._tags
+  return !!(tags.error || tags['error.type'])
+}
+
 module.exports = {
   encodeUnicode,
   validateKind,
-  getFunctionArguments
+  getFunctionArguments,
+  spanHasError
 }

package/packages/dd-trace/src/llmobs/writers/spans/agentProxy.js

@@ -10,10 +10,10 @@ const LLMObsBaseSpanWriter = require('./base')
 class LLMObsAgentProxySpanWriter extends LLMObsBaseSpanWriter {
   constructor (config) {
     super({
-      intake: config.hostname || 'localhost',
-      protocol: 'http:',
+      intake: config.url?.hostname || config.hostname || 'localhost',
+      protocol: config.url?.protocol || 'http:',
       endpoint: EVP_PROXY_AGENT_ENDPOINT,
-      port: config.port
+      port: config.url?.port || config.port
     })
 
     this._headers[EVP_SUBDOMAIN_HEADER_NAME] = EVP_SUBDOMAIN_HEADER_VALUE

package/packages/dd-trace/src/opentelemetry/context_manager.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { storage } = require('../../../datadog-core')
-const { trace, ROOT_CONTEXT } = require('@opentelemetry/api')
+const { trace, ROOT_CONTEXT, propagation } = require('@opentelemetry/api')
 const DataDogSpanContext = require('../opentracing/span_context')
 
 const SpanContext = require('./span_context')
@@ -18,17 +18,40 @@ class ContextManager {
     const context = (activeSpan && activeSpan.context()) || store || ROOT_CONTEXT
 
     if (!(context instanceof DataDogSpanContext)) {
+      const span = trace.getSpan(context)
+      // span instanceof NonRecordingSpan
+      if (span && span._spanContext && span._spanContext._ddContext && span._spanContext._ddContext._baggageItems) {
+        const baggages = span._spanContext._ddContext._baggageItems
+        const entries = {}
+        for (const [key, value] of Object.entries(baggages)) {
+          entries[key] = { value }
+        }
+        const otelBaggages = propagation.createBaggage(entries)
+        return propagation.setBaggage(context, otelBaggages)
+      }
       return context
     }
 
+    const baggages = JSON.parse(activeSpan.getAllBaggageItems())
+    const entries = {}
+    for (const [key, value] of Object.entries(baggages)) {
+      entries[key] = { value }
+    }
+    const otelBaggages = propagation.createBaggage(entries)
+
     if (!context._otelSpanContext) {
       const newSpanContext = new SpanContext(context)
       context._otelSpanContext = newSpanContext
     }
     if (store && trace.getSpanContext(store) === context._otelSpanContext) {
-      return store
+      return otelBaggages
+        ? propagation.setBaggage(store, otelBaggages)
+        : store
     }
-    return trace.setSpanContext(store || ROOT_CONTEXT, context._otelSpanContext)
+    const wrappedContext = trace.setSpanContext(store || ROOT_CONTEXT, context._otelSpanContext)
+    return otelBaggages
+      ? propagation.setBaggage(wrappedContext, otelBaggages)
+      : wrappedContext
   }
 
   with (context, fn, thisArg, ...args) {
@@ -38,9 +61,26 @@ class ContextManager {
       const cb = thisArg == null ? fn : fn.bind(thisArg)
       return this._store.run(context, cb, ...args)
     }
+    const baggages = propagation.getBaggage(context)
+    let baggageItems = []
+    if (baggages) {
+      baggageItems = baggages.getAllEntries()
+    }
     if (span && span._ddSpan) {
+      // does otel always override datadog?
+      span._ddSpan.removeAllBaggageItems()
+      for (const baggage of baggageItems) {
+        span._ddSpan.setBaggageItem(baggage[0], baggage[1].value)
+      }
       return ddScope.activate(span._ddSpan, run)
     }
+    // span instanceof NonRecordingSpan
+    if (span && span._spanContext && span._spanContext._ddContext && span._spanContext._ddContext._baggageItems) {
+      span._spanContext._ddContext._baggageItems = {}
+      for (const baggage of baggageItems) {
+        span._spanContext._ddContext._baggageItems[baggage[0]] = baggage[1].value
+      }
+    }
     return run()
   }
 

package/packages/dd-trace/src/plugins/ci_plugin.js

@@ -23,7 +23,11 @@ const {
   TEST_LEVEL_EVENT_TYPES,
   TEST_SUITE,
   getFileAndLineNumberFromError,
-  getTestSuitePath
+  DI_ERROR_DEBUG_INFO_CAPTURED,
+  DI_DEBUG_ERROR_PREFIX,
+  DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
+  DI_DEBUG_ERROR_FILE_SUFFIX,
+  DI_DEBUG_ERROR_LINE_SUFFIX
 } = require('./util/test')
 const Plugin = require('./plugin')
 const { COMPONENT } = require('../constants')
@@ -180,14 +184,18 @@ module.exports = class CiPlugin extends Plugin {
     }
   }
 
-  configure (config) {
+  configure (config, shouldGetEnvironmentData = true) {
     super.configure(config)
 
-    if (config.isTestDynamicInstrumentationEnabled) {
+    if (config.isTestDynamicInstrumentationEnabled && !this.di) {
       const testVisibilityDynamicInstrumentation = require('../ci-visibility/dynamic-instrumentation')
       this.di = testVisibilityDynamicInstrumentation
     }
 
+    if (!shouldGetEnvironmentData) {
+      return
+    }
+
     this.testEnvironmentMetadata = getTestEnvironmentMetadata(this.constructor.id, this.config)
 
     const {
@@ -292,37 +300,59 @@ module.exports = class CiPlugin extends Plugin {
     return testSpan
   }
 
-  // TODO: If the test finishes and the probe is not hit, we should remove the breakpoint
-  addDiProbe (err, probe) {
-    const [file, line] = getFileAndLineNumberFromError(err)
+  onDiBreakpointHit ({ snapshot }) {
+    if (!this.activeTestSpan || this.activeTestSpan.context()._isFinished) {
+      // This is unexpected and is caused by a race condition.
+      log.warn('Breakpoint snapshot could not be attached to the active test span')
+      return
+    }
 
-    const relativePath = getTestSuitePath(file, this.repositoryRoot)
+    const stackIndex = this.testErrorStackIndex
+
+    this.activeTestSpan.setTag(DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
+    this.activeTestSpan.setTag(
+      `${DI_DEBUG_ERROR_PREFIX}.${stackIndex}.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`,
+      snapshot.id
+    )
+    this.activeTestSpan.setTag(
+      `${DI_DEBUG_ERROR_PREFIX}.${stackIndex}.${DI_DEBUG_ERROR_FILE_SUFFIX}`,
+      snapshot.probe.location.file
+    )
+    this.activeTestSpan.setTag(
+      `${DI_DEBUG_ERROR_PREFIX}.${stackIndex}.${DI_DEBUG_ERROR_LINE_SUFFIX}`,
+      Number(snapshot.probe.location.lines[0])
+    )
+
+    const activeTestSpanContext = this.activeTestSpan.context()
+
+    this.tracer._exporter.exportDiLogs(this.testEnvironmentMetadata, {
+      debugger: { snapshot },
+      dd: {
+        trace_id: activeTestSpanContext.toTraceId(),
+        span_id: activeTestSpanContext.toSpanId()
+      }
+    })
+  }
 
-    const [
-      snapshotId,
-      setProbePromise,
-      hitProbePromise
-    ] = this.di.addLineProbe({ file: relativePath, line })
+  removeDiProbe (probeId) {
+    return this.di.removeProbe(probeId)
+  }
+
+  addDiProbe (err) {
+    const [file, line, stackIndex] = getFileAndLineNumberFromError(err, this.repositoryRoot)
 
-    if (probe) { // not all frameworks may sync with the set probe promise
-      probe.setProbePromise = setProbePromise
+    if (!file || !Number.isInteger(line)) {
+      log.warn('Could not add breakpoint for dynamic instrumentation')
+      return
     }
 
-    hitProbePromise.then(({ snapshot }) => {
-      // TODO: handle race conditions for this.retriedTestIds
-      const { traceId, spanId } = this.retriedTestIds
-      this.tracer._exporter.exportDiLogs(this.testEnvironmentMetadata, {
-        debugger: { snapshot },
-        dd: {
-          trace_id: traceId,
-          span_id: spanId
-        }
-      })
-    })
+    const [probeId, setProbePromise] = this.di.addLineProbe({ file, line }, this.onDiBreakpointHit.bind(this))
 
     return {
-      snapshotId,
-      file: relativePath,
+      probeId,
+      setProbePromise,
+      stackIndex,
+      file,
       line
     }
   }

package/packages/dd-trace/src/plugins/util/test.js

@@ -88,6 +88,7 @@ const TEST_BROWSER_VERSION = 'test.browser.version'
 // jest worker variables
 const JEST_WORKER_TRACE_PAYLOAD_CODE = 60
 const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61
+const JEST_WORKER_LOGS_PAYLOAD_CODE = 62
 
 // cucumber worker variables
 const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70
@@ -108,10 +109,10 @@ const TEST_LEVEL_EVENT_TYPES = [
 
 // Dynamic instrumentation - Test optimization integration tags
 const DI_ERROR_DEBUG_INFO_CAPTURED = 'error.debug_info_captured'
-// TODO: for the moment we'll only use a single snapshot id, so `0` is hardcoded
-const DI_DEBUG_ERROR_SNAPSHOT_ID = '_dd.debug.error.0.snapshot_id'
-const DI_DEBUG_ERROR_FILE = '_dd.debug.error.0.file'
-const DI_DEBUG_ERROR_LINE = '_dd.debug.error.0.line'
+const DI_DEBUG_ERROR_PREFIX = '_dd.debug.error'
+const DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX = 'snapshot_id'
+const DI_DEBUG_ERROR_FILE_SUFFIX = 'file'
+const DI_DEBUG_ERROR_LINE_SUFFIX = 'line'
 
 module.exports = {
   TEST_CODE_OWNERS,
@@ -134,6 +135,7 @@ module.exports = {
   LIBRARY_VERSION,
   JEST_WORKER_TRACE_PAYLOAD_CODE,
   JEST_WORKER_COVERAGE_PAYLOAD_CODE,
+  JEST_WORKER_LOGS_PAYLOAD_CODE,
   CUCUMBER_WORKER_TRACE_PAYLOAD_CODE,
   MOCHA_WORKER_TRACE_PAYLOAD_CODE,
   TEST_SOURCE_START,
@@ -191,9 +193,11 @@ module.exports = {
   getNumFromKnownTests,
   getFileAndLineNumberFromError,
   DI_ERROR_DEBUG_INFO_CAPTURED,
-  DI_DEBUG_ERROR_SNAPSHOT_ID,
-  DI_DEBUG_ERROR_FILE,
-  DI_DEBUG_ERROR_LINE
+  DI_DEBUG_ERROR_PREFIX,
+  DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
+  DI_DEBUG_ERROR_FILE_SUFFIX,
+  DI_DEBUG_ERROR_LINE_SUFFIX,
+  getFormattedError
 }
 
 // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -650,13 +654,30 @@ function getNumFromKnownTests (knownTests) {
   return totalNumTests
 }
 
-function getFileAndLineNumberFromError (error) {
+const DEPENDENCY_FOLDERS = [
+  'node_modules',
+  'node:',
+  '.pnpm',
+  '.yarn',
+  '.pnp'
+]
+
+function getFileAndLineNumberFromError (error, repositoryRoot) {
   // Split the stack trace into individual lines
   const stackLines = error.stack.split('\n')
 
-  // The top frame is usually the second line
-  const topFrame = stackLines[1]
+  // Remove potential messages on top of the stack that are not frames
+  const frames = stackLines.filter(line => line.includes('at ') && line.includes(repositoryRoot))
+
+  const topRelevantFrameIndex = frames.findIndex(line =>
+    line.includes(repositoryRoot) && !DEPENDENCY_FOLDERS.some(pattern => line.includes(pattern))
+  )
+
+  if (topRelevantFrameIndex === -1) {
+    return []
+  }
 
+  const topFrame = frames[topRelevantFrameIndex]
   // Regular expression to match the file path, line number, and column number
   const regex = /\s*at\s+(?:.*\()?(.+):(\d+):(\d+)\)?/
   const match = topFrame.match(regex)
@@ -664,9 +685,18 @@ function getFileAndLineNumberFromError (error) {
   if (match) {
     const filePath = match[1]
     const lineNumber = Number(match[2])
-    const columnNumber = Number(match[3])
 
-    return [filePath, lineNumber, columnNumber]
+    return [filePath, lineNumber, topRelevantFrameIndex]
   }
   return []
 }
+
+function getFormattedError (error, repositoryRoot) {
+  const newError = new Error(error.message)
+  if (error.stack) {
+    newError.stack = error.stack.split('\n').filter(line => line.includes(repositoryRoot)).join('\n')
+  }
+  newError.name = error.name
+
+  return newError
+}