@mastra/datadog 1.1.1-alpha.0 → 1.2.0-alpha.2

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @mastra/datadog
 
+## 1.2.0-alpha.2
+
+### Minor Changes
+
+- Mastra Eval results are now forwarded to Datadog. ([#16185](https://github.com/mastra-ai/mastra/pull/16185))
+
+- Mapped `MODEL_INFERENCE` spans to Datadog's `llm` kind (with token usage and model/provider attached) and `MODEL_STEP` to `workflow`. Falls back to the previous mapping when paired with an older `@mastra/core` that does not emit `MODEL_INFERENCE`. ([#16363](https://github.com/mastra-ai/mastra/pull/16363))
+
+### Patch Changes
+
+- Updated dependencies [[`6742347`](https://github.com/mastra-ai/mastra/commit/6742347d71955d7639adc9ddf6ff8282de7ee3ba), [`7b0ad1f`](https://github.com/mastra-ai/mastra/commit/7b0ad1f5c53dc118c6da12ae82ae2587037dc2b8), [`62666c3`](https://github.com/mastra-ai/mastra/commit/62666c367eaeac3941ead454b1d38810cc855721), [`4af2160`](https://github.com/mastra-ai/mastra/commit/4af2160322f4718cac421930cce85641e9512389), [`b2fd6be`](https://github.com/mastra-ai/mastra/commit/b2fd6beef989f5e463c9a33d8a6c20ac1800e011), [`136c959`](https://github.com/mastra-ai/mastra/commit/136c9592fb0eeb0cd212f28629d8a29b7557a2fc), [`00106be`](https://github.com/mastra-ai/mastra/commit/00106bede59b81e5b0e9cd6aad8d3b5dbc336387), [`4df7cc7`](https://github.com/mastra-ai/mastra/commit/4df7cc79342fd065fe7fdeef93c094db14b12bcd), [`aca3121`](https://github.com/mastra-ai/mastra/commit/aca31211233dac25459f140ea4fcfb3a5af64c18), [`7b0ad1f`](https://github.com/mastra-ai/mastra/commit/7b0ad1f5c53dc118c6da12ae82ae2587037dc2b8), [`9cdf38e`](https://github.com/mastra-ai/mastra/commit/9cdf38e58506e1109c8b38f97cd7770978a4218e), [`990851e`](https://github.com/mastra-ai/mastra/commit/990851edcb0e30be5c2c18b6532f1a876cc2d335), [`6068a6c`](https://github.com/mastra-ai/mastra/commit/6068a6c42950fad3ebfc92346417896ba60803d2), [`00106be`](https://github.com/mastra-ai/mastra/commit/00106bede59b81e5b0e9cd6aad8d3b5dbc336387), [`e2a079c`](https://github.com/mastra-ai/mastra/commit/e2a079cc3755b1895f7bd5dc36e9be81b11c7c22), [`534a456`](https://github.com/mastra-ai/mastra/commit/534a456a25e4df1e5407e7e632f4cb3b1fa14f9d), [`36bae07`](https://github.com/mastra-ai/mastra/commit/36bae07c0e70b1b3006f2fd20830e8883dcbd066)]:
+  - @mastra/core@1.33.0-alpha.7
+  - @mastra/observability@1.12.0-alpha.1
+
+## 1.1.1-alpha.1
+
+### Patch Changes
+
+- Updated dependencies [[`00ef282`](https://github.com/mastra-ai/mastra/commit/00ef2826034d006b984b3f19cd33ba0bba14d6c6)]:
+  - @mastra/observability@1.12.0-alpha.0
+
 ## 1.1.1-alpha.0
 
 ### Patch Changes

package/dist/bridge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bridge.d.ts","sourceRoot":"","sources":["../src/bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EACV,YAAY,EAIZ,mBAAmB,EACnB,iBAAiB,EACjB,QAAQ,EACR,OAAO,EACR,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAuB,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AA8DhE;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,kBAAkB;IAC7D;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,qBAAa,aAAc,SAAQ,YAAa,YAAW,mBAAmB;IAC5E,IAAI,SAAoB;IAExB,OAAO,CAAC,MAAM,CAA8E;IAC5F,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,YAAY,CAAmC;IACvD,OAAO,CAAC,cAAc,CAA6B;gBAEvC,MAAM,GAAE,mBAAwB;IAyC5C;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,iBAAiB,CAAC,QAAQ,CAAC,GAAG,OAAO,GAAG,SAAS;IAgErE;;OAEG;IACH,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAIrE;;OAEG;IACH,oBAAoB,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;IAIvD,OAAO,CAAC,sBAAsB;IAmB9B;;OAEG;cACa,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BvE,OAAO,CAAC,kBAAkB;IA0C1B,OAAO,CAAC,eAAe;IASvB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAwD7B,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;IAW3B,OAAO,CAAC,gBAAgB;IA2FxB,OAAO,CAAC,YAAY;IASd,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BhC"}
+{"version":3,"file":"bridge.d.ts","sourceRoot":"","sources":["../src/bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EACV,YAAY,EAKZ,mBAAmB,EACnB,iBAAiB,EACjB,QAAQ,EACR,OAAO,EACR,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAuB,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AA+DhE;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,kBAAkB;IAC7D;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,qBAAa,aAAc,SAAQ,YAAa,YAAW,mBAAmB;IAC5E,IAAI,SAAoB;IAExB,OAAO,CAAC,MAAM,CAA8E;IAC5F,OAAO,CAAC,SAAS,CAA0B;IAC3C,OAAO,CAAC,YAAY,CAAmC;IACvD,OAAO,CAAC,cAAc,CAA6B;gBAEvC,MAAM,GAAE,mBAAwB;IAyC5C;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,iBAAiB,CAAC,QAAQ,CAAC,GAAG,OAAO,GAAG,SAAS;IAgErE;;OAEG;IACH,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAIrE;;OAEG;IACH,oBAAoB,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;IAIvD,OAAO,CAAC,sBAAsB;IAmB9B;;OAEG;cACa,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BvE,OAAO,CAAC,kBAAkB;IA0C1B,OAAO,CAAC,eAAe;IASvB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAwD7B,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;IAW3B,OAAO,CAAC,gBAAgB;IA8FxB,OAAO,CAAC,YAAY;IASd,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBtB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BhC"}

package/dist/features.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Feature detection for paired @mastra/core and @mastra/observability versions.
+ *
+ * `coreFeatures` is a hard peer dependency, so it can be imported statically.
+ * `observabilityFeatures` is loaded via dynamic import so this exporter degrades
+ * gracefully against older `@mastra/observability` versions that don't export it
+ * (per the pattern documented in `observability/mastra/src/features.ts`).
+ *
+ * Detection runs once at module load. Callers consume the result through the
+ * sync `isModelInferenceEnabled()` accessor, which conservatively returns
+ * `false` until detection settles — long enough only to cover the microtask
+ * window before any spans are emitted in practice.
+ */
+/**
+ * Returns true when both packages report the `model-inference-span` feature,
+ * meaning MODEL_INFERENCE spans are emitted by the tracker. Drives the
+ * Datadog kind mapping and usage-source switch (legacy: MODEL_STEP carries
+ * 'llm' kind + usage; new: MODEL_INFERENCE does).
+ */
+export declare function isModelInferenceEnabled(): boolean;
+/**
+ * @internal Test-only override. Allows tests to simulate a paired older or
+ * newer `@mastra/observability` without juggling dynamic imports.
+ */
+export declare function __setObservabilityFeaturesForTest(features: ReadonlySet<string> | undefined): void;
+//# sourceMappingURL=features.d.ts.map

package/dist/features.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"features.d.ts","sourceRoot":"","sources":["../src/features.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AA0BH;;;;;GAKG;AACH,wBAAgB,uBAAuB,IAAI,OAAO,CAEjD;AAED;;;GAGG;AACH,wBAAgB,iCAAiC,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,SAAS,GAAG,IAAI,CAIjG"}

package/dist/index.cjs

@@ -4,12 +4,29 @@ var observability$1 = require('@mastra/core/observability');
 var utils = require('@mastra/core/utils');
 var observability = require('@mastra/observability');
 var tracer3 = require('dd-trace');
+var features = require('@mastra/core/features');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
 var tracer3__default = /*#__PURE__*/_interopDefault(tracer3);
 
 // src/bridge.ts
+var FEATURE = "model-inference-span";
+var observabilityFeatures;
+var featureLoadPromise;
+function loadObservabilityFeatures() {
+  if (!featureLoadPromise) {
+    featureLoadPromise = import('@mastra/observability').then((mod) => {
+      observabilityFeatures = mod.observabilityFeatures;
+    }).catch(() => {
+    });
+  }
+  return featureLoadPromise;
+}
+void loadObservabilityFeatures();
+function isModelInferenceEnabled() {
+  return observabilityFeatures?.has(FEATURE) === true && features.coreFeatures.has(FEATURE);
+}
 
 // src/metrics.ts
 function formatUsageMetrics(usage) {
@@ -35,18 +52,26 @@ function formatUsageMetrics(usage) {
   }
   return Object.keys(result).length > 0 ? result : void 0;
 }
-var SPAN_TYPE_TO_KIND = {
+var SPAN_TYPE_TO_KIND_LEGACY = {
   [observability$1.SpanType.AGENT_RUN]: "agent",
-  // MODEL_GENERATION is the wrapper around 1..N MODEL_STEPs (the actual API calls).
-  // It maps to 'workflow' so Datadog doesn't double-count it as an LLM call.
   [observability$1.SpanType.MODEL_GENERATION]: "workflow",
-  // MODEL_STEP is "Single model execution step within a generation (one API call)"
-  // per packages/core/src/observability/types/tracing.ts, so it is the real LLM span.
   [observability$1.SpanType.MODEL_STEP]: "llm",
   [observability$1.SpanType.TOOL_CALL]: "tool",
   [observability$1.SpanType.MCP_TOOL_CALL]: "tool",
   [observability$1.SpanType.WORKFLOW_RUN]: "workflow"
 };
+var SPAN_TYPE_TO_KIND_INFERENCE = {
+  [observability$1.SpanType.AGENT_RUN]: "agent",
+  [observability$1.SpanType.MODEL_GENERATION]: "workflow",
+  [observability$1.SpanType.MODEL_STEP]: "workflow",
+  [observability$1.SpanType.MODEL_INFERENCE]: "llm",
+  [observability$1.SpanType.TOOL_CALL]: "tool",
+  [observability$1.SpanType.MCP_TOOL_CALL]: "tool",
+  [observability$1.SpanType.WORKFLOW_RUN]: "workflow"
+};
+function getSpanTypeToKind() {
+  return isModelInferenceEnabled() ? SPAN_TYPE_TO_KIND_INFERENCE : SPAN_TYPE_TO_KIND_LEGACY;
+}
 var tracerInitFlag = { done: false };
 function ensureTracer(config) {
   if (tracerInitFlag.done) return;
@@ -72,7 +97,7 @@ function ensureTracer(config) {
   tracerInitFlag.done = true;
 }
 function kindFor(spanType) {
-  return SPAN_TYPE_TO_KIND[spanType] || "task";
+  return getSpanTypeToKind()[spanType] || "task";
 }
 function toDate(value) {
   return value instanceof Date ? value : new Date(value);
@@ -447,7 +472,8 @@ var DatadogBridge = class extends observability.BaseExporter {
     if (span.output !== void 0) {
       annotations.outputData = formatOutput(span.output, span.type);
     }
-    if (span.type === observability$1.SpanType.MODEL_STEP) {
+    const usageSpanType = isModelInferenceEnabled() ? observability$1.SpanType.MODEL_INFERENCE : observability$1.SpanType.MODEL_STEP;
+    if (span.type === usageSpanType) {
       const usage = span.attributes?.usage;
       const metrics = formatUsageMetrics(usage);
       if (metrics) {
@@ -713,7 +739,8 @@ var DatadogExporter = class extends observability.BaseExporter {
     if (span.output !== void 0) {
       annotations.outputData = formatOutput(span.output, span.type);
     }
-    if (span.type === observability$1.SpanType.MODEL_STEP) {
+    const usageSpanType = isModelInferenceEnabled() ? observability$1.SpanType.MODEL_INFERENCE : observability$1.SpanType.MODEL_STEP;
+    if (span.type === usageSpanType) {
       const usage = span.attributes?.usage;
       const metrics = formatUsageMetrics(usage);
       if (metrics) {
@@ -783,6 +810,63 @@ var DatadogExporter = class extends observability.BaseExporter {
     }
     return annotations;
   }
+  /**
+   * Submit an eval score to Datadog LLM Observability for the matching ddSpan.
+   *
+   * Ordering constraint: the matching span must have already been emitted to dd-trace
+   * (i.e. its `SPAN_ENDED` event must have been processed and the trace tree flushed).
+   * On Mastra's normal scoring path this is always true — scorer hooks fire after the
+   * scored entity completes, so the root span has ended by the time `onScoreEvent` runs.
+   *
+   * If a score arrives for an unexported span (either before `SPAN_ENDED` or after the
+   * `traceState` entry has been cleaned up), the event is dropped and a warning is logged
+   * so the misuse is observable. Scores must therefore only be submitted for spans whose
+   * lifecycle has completed.
+   */
+  async onScoreEvent(event) {
+    if (this.isDisabled || !tracer3__default.default.llmobs?.submitEvaluation) return;
+    const { score } = event;
+    if (!score.traceId || !score.spanId) {
+      this.logger.warn("Datadog exporter: dropping score with no traceId/spanId", {
+        scorerId: score.scorerId
+      });
+      return;
+    }
+    const ctx = this.traceState.get(score.traceId)?.contexts.get(score.spanId);
+    const exported = ctx?.exported;
+    if (!exported) {
+      this.logger.warn(
+        "Datadog exporter: dropping score for span that has not been emitted to dd-trace yet (span_ended must be processed before submitting a score for it)",
+        {
+          traceId: score.traceId,
+          spanId: score.spanId,
+          scorerId: score.scorerId
+        }
+      );
+      return;
+    }
+    try {
+      tracer3__default.default.llmobs.submitEvaluation(
+        { traceId: exported.traceId, spanId: exported.spanId },
+        {
+          label: score.scorerName ?? score.scorerId,
+          value: score.score,
+          metricType: "score",
+          mlApp: this.config.mlApp,
+          timestampMs: score.timestamp instanceof Date ? score.timestamp.getTime() : Date.now(),
+          ...score.reason ? { reasoning: score.reason } : {},
+          ...score.metadata ? { metadata: score.metadata } : {}
+        }
+      );
+    } catch (err) {
+      this.logger.error("Datadog exporter: Failed to submit evaluation", {
+        error: err,
+        traceId: score.traceId,
+        spanId: score.spanId,
+        scorerId: score.scorerId
+      });
+    }
+  }
   /**
    * Force flush any buffered spans without shutting down the exporter.
    * This is useful in serverless environments where you need to ensure spans