@jaypie/llm 1.2.39 → 1.2.40

package/dist/cjs/observability/llmobs.d.ts

@@ -0,0 +1,72 @@
+/**
+ * LLM Observability span kinds supported by Datadog's `llmobs` SDK.
+ * @see https://docs.datadoghq.com/llm_observability/setup/sdk/nodejs/
+ */
+export type LlmObsSpanKind = "agent" | "embedding" | "llm" | "retrieval" | "task" | "tool" | "workflow";
+export interface LlmObsSpanOptions {
+    kind: LlmObsSpanKind;
+    modelName?: string;
+    modelProvider?: string;
+    name: string;
+}
+export interface LlmObsAnnotation {
+    inputData?: unknown;
+    metadata?: Record<string, unknown>;
+    metrics?: Record<string, number>;
+    outputData?: unknown;
+    tags?: Record<string, unknown>;
+}
+/** Manual span handle for streaming, where a callback cannot enclose the work. */
+export interface LlmObsSpanHandle {
+    annotate: (annotation: LlmObsAnnotation) => void;
+    finish: () => void;
+}
+/** Minimal shape of the dd-trace `llmobs` SDK surface this module uses. */
+interface LlmObsSdk {
+    annotate: (span: unknown, annotation: LlmObsAnnotation) => void;
+    trace: (options: LlmObsSpanOptions, fn: (span: unknown) => unknown) => unknown;
+}
+/** Reset the cached SDK resolution. Exposed for tests. */
+export declare function _resetLlmObs(): void;
+/**
+ * Inject an SDK to bypass dd-trace resolution. Test-only: dd-trace is not a
+ * dependency, so the enabled path cannot otherwise be exercised in unit tests.
+ */
+export declare function _setLlmObs(sdk: LlmObsSdk | null): void;
+/**
+ * True when LLM Observability emission is opted in via `DD_LLMOBS_ENABLED`.
+ * Accepts any truthy value except "false"/"0".
+ */
+export declare function isLlmObsEnabled(): boolean;
+/**
+ * Run `fn` inside an LLM Observability span. When emission is disabled or
+ * dd-trace is absent, `fn` is invoked directly with no overhead. The active
+ * span is established for the duration of `fn`, so nested `withLlmObsSpan`
+ * calls (and `annotateLlmObs`) attach as children/annotations automatically.
+ */
+export declare function withLlmObsSpan<T>(options: LlmObsSpanOptions, fn: () => Promise<T>): Promise<T>;
+/**
+ * Annotate the currently active LLM Observability span. No-op when disabled.
+ */
+export declare function annotateLlmObs(annotation: LlmObsAnnotation): void;
+/**
+ * Open a manual span that is finished later. Used by streaming, where work is
+ * yielded incrementally and cannot be enclosed in a single callback. The span
+ * is kept open via a deferred promise resolved by `finish()`. Returns null when
+ * emission is disabled or dd-trace is absent.
+ *
+ * Note: because the span is held open across `yield` boundaries (outside the
+ * SDK's AsyncLocalStorage scope), manual spans are emitted flat rather than
+ * nested. Annotations are applied to the explicit span reference.
+ */
+export declare function openLlmObsSpan(options: LlmObsSpanOptions): LlmObsSpanHandle | null;
+/**
+ * Sum an `LlmUsage` array into LLM Observability metric keys. Returns undefined
+ * when there is no usage data so callers can omit the `metrics` field.
+ */
+export declare function usageToLlmObsMetrics(usage?: Array<{
+    input?: number;
+    output?: number;
+    total?: number;
+}>): Record<string, number> | undefined;
+export {};

package/dist/esm/index.js

@@ -5,6 +5,8 @@ import { placeholders, JAYPIE, resolveValue, sleep } from '@jaypie/kit';
 import RandomLib from 'random';
 import { RateLimitError, APIConnectionError, APIConnectionTimeoutError, InternalServerError, APIUserAbortError, AuthenticationError, BadRequestError, ConflictError, NotFoundError, PermissionDeniedError, UnprocessableEntityError, OpenAI } from 'openai';
 import { zodResponseFormat } from 'openai/helpers/zod';
+import { createRequire } from 'module';
+import { pathToFileURL } from 'url';
 import { PDFDocument } from 'pdf-lib';
 import { readFile } from 'fs/promises';
 import { resolve } from 'path';
@@ -4835,6 +4837,198 @@ class Toolkit {
     }
 }
 
+//
+//
+// Constants
+//
+const ENV = {
+    DD_LLMOBS_ENABLED: "DD_LLMOBS_ENABLED",
+};
+const MODULE = {
+    // Computed at runtime so bundlers (esbuild) do not attempt to include
+    // dd-trace, which is provided by the Datadog Lambda layer at runtime.
+    DD_TRACE: ["dd", "trace"].join("-"),
+};
+//
+//
+// Helpers
+//
+// CJS/ESM compatible require - handles bundling to CJS where import.meta.url
+// becomes undefined (mirrors packages/testkit/src/mock/original.ts).
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore - __filename exists in CJS context when ESM is bundled to CJS
+const requireModule = typeof __filename !== "undefined"
+    ? createRequire(pathToFileURL(__filename).href)
+    : createRequire(import.meta.url);
+let resolved = false;
+let cachedSdk$4 = null;
+/**
+ * Lazily resolve the dd-trace `llmobs` SDK. Returns null (and never throws)
+ * when dd-trace is absent or the SDK surface is unexpected. Cached after the
+ * first attempt.
+ */
+function resolveLlmObs() {
+    if (resolved) {
+        return cachedSdk$4;
+    }
+    resolved = true;
+    try {
+        const ddTrace = requireModule(MODULE.DD_TRACE);
+        const tracer = ddTrace?.default ?? ddTrace;
+        const llmobs = tracer?.llmobs;
+        if (llmobs &&
+            typeof llmobs.trace === "function" &&
+            typeof llmobs.annotate === "function") {
+            cachedSdk$4 = llmobs;
+        }
+    }
+    catch {
+        cachedSdk$4 = null;
+    }
+    return cachedSdk$4;
+}
+//
+//
+// Main
+//
+/**
+ * True when LLM Observability emission is opted in via `DD_LLMOBS_ENABLED`.
+ * Accepts any truthy value except "false"/"0".
+ */
+function isLlmObsEnabled() {
+    const value = process.env[ENV.DD_LLMOBS_ENABLED];
+    if (!value) {
+        return false;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized !== "" && normalized !== "false" && normalized !== "0";
+}
+/** Resolve the SDK only when enabled and available. */
+function activeSdk() {
+    if (!isLlmObsEnabled()) {
+        return null;
+    }
+    return resolveLlmObs();
+}
+/**
+ * Run `fn` inside an LLM Observability span. When emission is disabled or
+ * dd-trace is absent, `fn` is invoked directly with no overhead. The active
+ * span is established for the duration of `fn`, so nested `withLlmObsSpan`
+ * calls (and `annotateLlmObs`) attach as children/annotations automatically.
+ */
+async function withLlmObsSpan(options, fn) {
+    const sdk = activeSdk();
+    if (!sdk) {
+        return fn();
+    }
+    let started = false;
+    try {
+        return (await sdk.trace(options, () => {
+            started = true;
+            return fn();
+        }));
+    }
+    catch (error) {
+        // If `fn` ran, propagate its error (the SDK re-throws after finishing the
+        // span). Only swallow failures from the instrumentation itself, re-running
+        // `fn` so observability never breaks the underlying call.
+        if (started) {
+            throw error;
+        }
+        getLogger$6().warn("[llmobs] span emission failed");
+        getLogger$6().var({ error });
+        return fn();
+    }
+}
+/**
+ * Annotate the currently active LLM Observability span. No-op when disabled.
+ */
+function annotateLlmObs(annotation) {
+    const sdk = activeSdk();
+    if (!sdk) {
+        return;
+    }
+    try {
+        sdk.annotate(undefined, annotation);
+    }
+    catch (error) {
+        getLogger$6().warn("[llmobs] annotate failed");
+        getLogger$6().var({ error });
+    }
+}
+/**
+ * Open a manual span that is finished later. Used by streaming, where work is
+ * yielded incrementally and cannot be enclosed in a single callback. The span
+ * is kept open via a deferred promise resolved by `finish()`. Returns null when
+ * emission is disabled or dd-trace is absent.
+ *
+ * Note: because the span is held open across `yield` boundaries (outside the
+ * SDK's AsyncLocalStorage scope), manual spans are emitted flat rather than
+ * nested. Annotations are applied to the explicit span reference.
+ */
+function openLlmObsSpan(options) {
+    const sdk = activeSdk();
+    if (!sdk) {
+        return null;
+    }
+    try {
+        let capturedSpan;
+        let release;
+        const held = new Promise((resolve) => {
+            release = resolve;
+        });
+        // trace() runs the callback synchronously, capturing the span; the returned
+        // pending promise keeps the span open until finish() resolves it.
+        void sdk.trace(options, (span) => {
+            capturedSpan = span;
+            return held;
+        });
+        let finished = false;
+        return {
+            annotate: (annotation) => {
+                try {
+                    sdk.annotate(capturedSpan, annotation);
+                }
+                catch (error) {
+                    getLogger$6().warn("[llmobs] annotate failed");
+                    getLogger$6().var({ error });
+                }
+            },
+            finish: () => {
+                if (finished) {
+                    return;
+                }
+                finished = true;
+                release?.();
+            },
+        };
+    }
+    catch (error) {
+        getLogger$6().warn("[llmobs] span emission failed");
+        getLogger$6().var({ error });
+        return null;
+    }
+}
+//
+//
+// Mapping Helpers
+//
+/**
+ * Sum an `LlmUsage` array into LLM Observability metric keys. Returns undefined
+ * when there is no usage data so callers can omit the `metrics` field.
+ */
+function usageToLlmObsMetrics(usage) {
+    if (!usage || usage.length === 0) {
+        return undefined;
+    }
+    const metrics = usage.reduce((sum, item) => ({
+        input_tokens: sum.input_tokens + (item.input ?? 0),
+        output_tokens: sum.output_tokens + (item.output ?? 0),
+        total_tokens: sum.total_tokens + (item.total ?? 0),
+    }), { input_tokens: 0, output_tokens: 0, total_tokens: 0 });
+    return metrics;
+}
+
 //
 //
 // Main
@@ -5706,30 +5900,46 @@ class OperateLoop {
         // Initialize state
         const state = await this.initializeState(input, options);
         const context = this.createContext(options);
-        // Build initial request
-        let request = this.buildInitialRequest(state, options);
-        // Multi-turn loop
-        while (state.currentTurn < state.maxTurns) {
-            state.currentTurn++;
-            // Execute one turn with retry logic
-            const shouldContinue = await this.executeOneTurn(request, state, context, options);
-            if (!shouldContinue) {
-                break;
+        const modelName = options.model ?? this.adapter.defaultModel;
+        // Enclosing LLM Observability span (no-op when DD_LLMOBS_ENABLED is unset).
+        // Child llm/tool spans nest under it via the SDK's active-span context.
+        return withLlmObsSpan({
+            kind: state.toolkit ? "agent" : "llm",
+            modelName,
+            modelProvider: this.adapter.name,
+            name: "jaypie.llm.operate",
+        }, async () => {
+            // Build initial request
+            let request = this.buildInitialRequest(state, options);
+            // Multi-turn loop
+            while (state.currentTurn < state.maxTurns) {
+                state.currentTurn++;
+                // Execute one turn with retry logic
+                const shouldContinue = await this.executeOneTurn(request, state, context, options);
+                if (!shouldContinue) {
+                    break;
+                }
+                // Rebuild request with updated history for next turn
+                request = {
+                    format: state.formattedFormat,
+                    instructions: options.instructions,
+                    messages: state.currentInput,
+                    model: modelName,
+                    providerOptions: options.providerOptions,
+                    system: options.system,
+                    temperature: options.temperature,
+                    tools: state.formattedTools,
+                    user: options.user,
+                };
             }
-            // Rebuild request with updated history for next turn
-            request = {
-                format: state.formattedFormat,
-                instructions: options.instructions,
-                messages: state.currentInput,
-                model: options.model ?? this.adapter.defaultModel,
-                providerOptions: options.providerOptions,
-                system: options.system,
-                temperature: options.temperature,
-                tools: state.formattedTools,
-                user: options.user,
-            };
-        }
-        return state.responseBuilder.build();
+            const response = state.responseBuilder.build();
+            annotateLlmObs({
+                inputData: input,
+                metrics: usageToLlmObsMetrics(response.usage),
+                outputData: response.content,
+            });
+            return response;
+        });
     }
     //
     // Private Methods
@@ -5831,20 +6041,34 @@ class OperateLoop {
             options,
             providerRequest,
         });
-        // Execute with retry (RetryExecutor handles error hooks and throws appropriate errors)
-        const response = await retryExecutor.execute((signal) => this.adapter.executeRequest(this.client, providerRequest, signal), {
-            context: {
-                input: state.currentInput,
-                options,
-                providerRequest,
-            },
-            hooks: context.hooks,
+        // Execute with retry inside a child llm span (no-op when llmobs disabled).
+        // RetryExecutor handles error hooks and throws appropriate errors.
+        const { parsed, response } = await withLlmObsSpan({
+            kind: "llm",
+            modelName: options.model ?? this.adapter.defaultModel,
+            modelProvider: this.adapter.name,
+            name: "jaypie.llm.model",
+        }, async () => {
+            const response = await retryExecutor.execute((signal) => this.adapter.executeRequest(this.client, providerRequest, signal), {
+                context: {
+                    input: state.currentInput,
+                    options,
+                    providerRequest,
+                },
+                hooks: context.hooks,
+            });
+            // Log what was returned from the model
+            log.trace("[operate] Model response received");
+            log.var({ "operate.response": response });
+            // Parse response
+            const parsed = this.adapter.parseResponse(response, options);
+            annotateLlmObs({
+                inputData: state.currentInput,
+                metrics: usageToLlmObsMetrics(parsed.usage ? [parsed.usage] : undefined),
+                outputData: parsed.content ?? "",
+            });
+            return { parsed, response };
         });
-        // Log what was returned from the model
-        log.trace("[operate] Model response received");
-        log.var({ "operate.response": response });
-        // Parse response
-        const parsed = this.adapter.parseResponse(response, options);
         // Track usage
         if (parsed.usage) {
             state.responseBuilder.addUsage(parsed.usage);
@@ -5889,11 +6113,19 @@ class OperateLoop {
                             args: toolCall.arguments,
                             toolName: toolCall.name,
                         });
-                        // Call the tool
+                        // Call the tool inside a child tool span (no-op when disabled)
                         log.trace(`[operate] Calling tool - ${toolCall.name}`);
-                        const result = await state.toolkit.call({
-                            arguments: toolCall.arguments,
-                            name: toolCall.name,
+                        const result = await withLlmObsSpan({ kind: "tool", name: toolCall.name }, async () => {
+                            const result = await state.toolkit.call({
+                                arguments: toolCall.arguments,
+                                name: toolCall.name,
+                            });
+                            annotateLlmObs({
+                                inputData: toolCall.arguments,
+                                metadata: { tool: toolCall.name },
+                                outputData: result,
+                            });
+                            return result;
                         });
                         // Execute afterEachTool hook
                         await this.hookRunnerInstance.runAfterTool(context.hooks, {
@@ -6143,7 +6375,7 @@ class StreamLoop {
             }
             // If we have tool calls, process them
             if (toolCalls && toolCalls.length > 0 && state.toolkit) {
-                yield* this.processToolCalls(toolCalls, state, context, options);
+                yield* this.processToolCalls(toolCalls, state, context);
                 // Check if we've reached max turns
                 if (state.currentTurn >= state.maxTurns) {
                     const error = new TooManyRequestsError();
@@ -6250,6 +6482,16 @@ class StreamLoop {
             options,
             providerRequest,
         });
+        // Open a manual llm span held open across the streamed turn. Flat (not
+        // nested) because it spans yield boundaries; no-op when llmobs disabled.
+        const llmSpan = openLlmObsSpan({
+            kind: "llm",
+            modelName: options.model ?? this.adapter.defaultModel,
+            modelProvider: this.adapter.name,
+            name: "jaypie.llm.model",
+        });
+        const inputSnapshot = [...state.currentInput];
+        let streamedText = "";
         // Collect tool calls from the stream
         const collectedToolCalls = [];
         // Retry loop for connection-level failures
@@ -6269,6 +6511,7 @@ class StreamLoop {
                         // Pass through text chunks
                         if (chunk.type === LlmStreamChunkType.Text) {
                             chunksYielded = true;
+                            streamedText += chunk.content ?? "";
                             yield chunk;
                         }
                         // Collect tool calls
@@ -6315,6 +6558,12 @@ class StreamLoop {
                                 title: "Stream Error",
                             },
                         };
+                        llmSpan?.annotate({
+                            inputData: inputSnapshot,
+                            metrics: usageToLlmObsMetrics(state.usageItems),
+                            outputData: streamedText,
+                        });
+                        llmSpan?.finish();
                         return { shouldContinue: false };
                     }
                     // Check if we've exhausted retries or error is not retryable
@@ -6323,6 +6572,7 @@ class StreamLoop {
                         log.error(`Stream request failed after ${this.retryPolicy.maxRetries} retries`);
                         log.var({ error });
                         const errorMessage = error instanceof Error ? error.message : String(error);
+                        llmSpan?.finish();
                         throw new BadGatewayError(errorMessage);
                     }
                     const delay = this.retryPolicy.getDelayForAttempt(attempt);
@@ -6336,6 +6586,13 @@ class StreamLoop {
         finally {
             guard.remove();
         }
+        // Annotate and finish the streamed llm span (no-op when disabled)
+        llmSpan?.annotate({
+            inputData: inputSnapshot,
+            metrics: usageToLlmObsMetrics(state.usageItems),
+            outputData: streamedText,
+        });
+        llmSpan?.finish();
         // Execute afterEachModelResponse hook
         await this.hookRunnerInstance.runAfterModelResponse(context.hooks, {
             content: "",
@@ -6370,7 +6627,7 @@ class StreamLoop {
         }
         return { shouldContinue: false };
     }
-    async *processToolCalls(toolCalls, state, context, _options) {
+    async *processToolCalls(toolCalls, state, context) {
         const log = getLogger$6();
         for (const toolCall of toolCalls) {
             try {
@@ -6379,11 +6636,19 @@ class StreamLoop {
                     args: toolCall.arguments,
                     toolName: toolCall.name,
                 });
-                // Call the tool
+                // Call the tool inside a child tool span (no-op when disabled)
                 log.trace(`[stream] Calling tool - ${toolCall.name}`);
-                const result = await state.toolkit.call({
-                    arguments: toolCall.arguments,
-                    name: toolCall.name,
+                const result = await withLlmObsSpan({ kind: "tool", name: toolCall.name }, async () => {
+                    const result = await state.toolkit.call({
+                        arguments: toolCall.arguments,
+                        name: toolCall.name,
+                    });
+                    annotateLlmObs({
+                        inputData: toolCall.arguments,
+                        metadata: { tool: toolCall.name },
+                        outputData: result,
+                    });
+                    return result;
                 });
                 // Execute afterEachTool hook
                 await this.hookRunnerInstance.runAfterTool(context.hooks, {