@jaypie/llm 1.2.39 → 1.2.41

package/dist/esm/index.js

@@ -5,6 +5,8 @@ import { placeholders, JAYPIE, resolveValue, sleep } from '@jaypie/kit';
 import RandomLib from 'random';
 import { RateLimitError, APIConnectionError, APIConnectionTimeoutError, InternalServerError, APIUserAbortError, AuthenticationError, BadRequestError, ConflictError, NotFoundError, PermissionDeniedError, UnprocessableEntityError, OpenAI } from 'openai';
 import { zodResponseFormat } from 'openai/helpers/zod';
+import { createRequire } from 'module';
+import { pathToFileURL } from 'url';
 import { PDFDocument } from 'pdf-lib';
 import { readFile } from 'fs/promises';
 import { resolve } from 'path';
@@ -570,6 +572,154 @@ function extractReasoning(history) {
     return reasoningTexts;
 }
 
+function naturalZodSchema(definition) {
+    if (Array.isArray(definition)) {
+        if (definition.length === 0) {
+            // Handle empty array - accept any[]
+            return z.array(z.any());
+        }
+        else if (definition.length === 1) {
+            // Handle array types
+            const itemType = definition[0];
+            switch (itemType) {
+                case String:
+                    return z.array(z.string());
+                case Number:
+                    return z.array(z.number());
+                case Boolean:
+                    return z.array(z.boolean());
+                default:
+                    if (typeof itemType === "object") {
+                        // Handle array of objects
+                        return z.array(naturalZodSchema(itemType));
+                    }
+                    // Handle enum arrays
+                    return z.enum(definition);
+            }
+        }
+        else {
+            // Handle enum arrays
+            return z.enum(definition);
+        }
+    }
+    else if (definition && typeof definition === "object") {
+        if (Object.keys(definition).length === 0) {
+            // Handle empty object - accept any key-value pairs
+            return z.record(z.string(), z.any());
+        }
+        else {
+            // Handle object with properties
+            const schemaShape = {};
+            for (const [key, value] of Object.entries(definition)) {
+                schemaShape[key] = naturalZodSchema(value);
+            }
+            return z.object(schemaShape);
+        }
+    }
+    else {
+        switch (definition) {
+            case String:
+                return z.string();
+            case Number:
+                return z.number();
+            case Boolean:
+                return z.boolean();
+            case Object:
+                return z.record(z.string(), z.any());
+            case Array:
+                return z.array(z.any());
+            default:
+                throw new Error(`Unsupported type: ${definition}`);
+        }
+    }
+}
+
+//
+//
+// Helpers
+//
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
+ * Convert a `format` declaration (Zod schema, NaturalSchema, or a JSON Schema
+ * JsonObject) into a plain JSON Schema we can walk. Mirrors the conversion the
+ * provider adapters perform in `formatOutputSchema`, but without any
+ * provider-specific sanitization.
+ */
+function formatToJsonSchema(format) {
+    if (format instanceof z.ZodType) {
+        return z.toJSONSchema(format);
+    }
+    if (isPlainObject(format) && format.type === "json_schema") {
+        const clone = structuredClone(format);
+        clone.type = "object";
+        return clone;
+    }
+    try {
+        return z.toJSONSchema(naturalZodSchema(format));
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Walk a JSON Schema alongside a parsed value, filling any declared array field
+ * that is absent (`undefined`/`null`) with `[]`. Recurses into object
+ * properties and array items so nested declared arrays are also backfilled.
+ */
+function fillFromSchema(schema, value) {
+    if (!isPlainObject(schema)) {
+        return value;
+    }
+    const type = schema.type;
+    const isArray = type === "array" || (type === undefined && "items" in schema);
+    if (isArray) {
+        if (value === undefined || value === null) {
+            return [];
+        }
+        const items = schema.items;
+        if (Array.isArray(value) && isPlainObject(items)) {
+            return value.map((entry) => fillFromSchema(items, entry));
+        }
+        return value;
+    }
+    const isObject = type === "object" || (type === undefined && "properties" in schema);
+    if (isObject) {
+        if (!isPlainObject(value)) {
+            return value;
+        }
+        const properties = schema.properties;
+        if (isPlainObject(properties)) {
+            for (const [key, propSchema] of Object.entries(properties)) {
+                value[key] = fillFromSchema(propSchema, value[key]);
+            }
+        }
+        return value;
+    }
+    return value;
+}
+//
+//
+// Main
+//
+/**
+ * Ensure every array field declared in `format` is present in `content` as an
+ * array. A declared `format` is a schema contract: an empty list should surface
+ * as `[]`, not be dropped from the response. Some providers/models omit empty
+ * array fields entirely, leaving consumers to read `.length` on `undefined`.
+ *
+ * Only mutates a (cloned) structured object; strings and non-objects pass
+ * through untouched.
+ */
+function fillFormatArrays({ content, format, }) {
+    const schema = formatToJsonSchema(format);
+    if (!schema) {
+        return content;
+    }
+    return fillFromSchema(schema, structuredClone(content));
+}
+
 /**
  * Converts a string to a standardized LlmInputMessage
  * @param input - String to format
@@ -697,68 +847,6 @@ function maxTurnsFromOptions(options) {
     return 1;
 }
 
-function naturalZodSchema(definition) {
-    if (Array.isArray(definition)) {
-        if (definition.length === 0) {
-            // Handle empty array - accept any[]
-            return z.array(z.any());
-        }
-        else if (definition.length === 1) {
-            // Handle array types
-            const itemType = definition[0];
-            switch (itemType) {
-                case String:
-                    return z.array(z.string());
-                case Number:
-                    return z.array(z.number());
-                case Boolean:
-                    return z.array(z.boolean());
-                default:
-                    if (typeof itemType === "object") {
-                        // Handle array of objects
-                        return z.array(naturalZodSchema(itemType));
-                    }
-                    // Handle enum arrays
-                    return z.enum(definition);
-            }
-        }
-        else {
-            // Handle enum arrays
-            return z.enum(definition);
-        }
-    }
-    else if (definition && typeof definition === "object") {
-        if (Object.keys(definition).length === 0) {
-            // Handle empty object - accept any key-value pairs
-            return z.record(z.string(), z.any());
-        }
-        else {
-            // Handle object with properties
-            const schemaShape = {};
-            for (const [key, value] of Object.entries(definition)) {
-                schemaShape[key] = naturalZodSchema(value);
-            }
-            return z.object(schemaShape);
-        }
-    }
-    else {
-        switch (definition) {
-            case String:
-                return z.string();
-            case Number:
-                return z.number();
-            case Boolean:
-                return z.boolean();
-            case Object:
-                return z.record(z.string(), z.any());
-            case Array:
-                return z.array(z.any());
-            default:
-                throw new Error(`Unsupported type: ${definition}`);
-        }
-    }
-}
-
 //
 // Constants
 //
@@ -4835,6 +4923,198 @@ class Toolkit {
     }
 }
 
+//
+//
+// Constants
+//
+const ENV = {
+    DD_LLMOBS_ENABLED: "DD_LLMOBS_ENABLED",
+};
+const MODULE = {
+    // Computed at runtime so bundlers (esbuild) do not attempt to include
+    // dd-trace, which is provided by the Datadog Lambda layer at runtime.
+    DD_TRACE: ["dd", "trace"].join("-"),
+};
+//
+//
+// Helpers
+//
+// CJS/ESM compatible require - handles bundling to CJS where import.meta.url
+// becomes undefined (mirrors packages/testkit/src/mock/original.ts).
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore - __filename exists in CJS context when ESM is bundled to CJS
+const requireModule = typeof __filename !== "undefined"
+    ? createRequire(pathToFileURL(__filename).href)
+    : createRequire(import.meta.url);
+let resolved = false;
+let cachedSdk$4 = null;
+/**
+ * Lazily resolve the dd-trace `llmobs` SDK. Returns null (and never throws)
+ * when dd-trace is absent or the SDK surface is unexpected. Cached after the
+ * first attempt.
+ */
+function resolveLlmObs() {
+    if (resolved) {
+        return cachedSdk$4;
+    }
+    resolved = true;
+    try {
+        const ddTrace = requireModule(MODULE.DD_TRACE);
+        const tracer = ddTrace?.default ?? ddTrace;
+        const llmobs = tracer?.llmobs;
+        if (llmobs &&
+            typeof llmobs.trace === "function" &&
+            typeof llmobs.annotate === "function") {
+            cachedSdk$4 = llmobs;
+        }
+    }
+    catch {
+        cachedSdk$4 = null;
+    }
+    return cachedSdk$4;
+}
+//
+//
+// Main
+//
+/**
+ * True when LLM Observability emission is opted in via `DD_LLMOBS_ENABLED`.
+ * Accepts any truthy value except "false"/"0".
+ */
+function isLlmObsEnabled() {
+    const value = process.env[ENV.DD_LLMOBS_ENABLED];
+    if (!value) {
+        return false;
+    }
+    const normalized = value.trim().toLowerCase();
+    return normalized !== "" && normalized !== "false" && normalized !== "0";
+}
+/** Resolve the SDK only when enabled and available. */
+function activeSdk() {
+    if (!isLlmObsEnabled()) {
+        return null;
+    }
+    return resolveLlmObs();
+}
+/**
+ * Run `fn` inside an LLM Observability span. When emission is disabled or
+ * dd-trace is absent, `fn` is invoked directly with no overhead. The active
+ * span is established for the duration of `fn`, so nested `withLlmObsSpan`
+ * calls (and `annotateLlmObs`) attach as children/annotations automatically.
+ */
+async function withLlmObsSpan(options, fn) {
+    const sdk = activeSdk();
+    if (!sdk) {
+        return fn();
+    }
+    let started = false;
+    try {
+        return (await sdk.trace(options, () => {
+            started = true;
+            return fn();
+        }));
+    }
+    catch (error) {
+        // If `fn` ran, propagate its error (the SDK re-throws after finishing the
+        // span). Only swallow failures from the instrumentation itself, re-running
+        // `fn` so observability never breaks the underlying call.
+        if (started) {
+            throw error;
+        }
+        getLogger$6().warn("[llmobs] span emission failed");
+        getLogger$6().var({ error });
+        return fn();
+    }
+}
+/**
+ * Annotate the currently active LLM Observability span. No-op when disabled.
+ */
+function annotateLlmObs(annotation) {
+    const sdk = activeSdk();
+    if (!sdk) {
+        return;
+    }
+    try {
+        sdk.annotate(undefined, annotation);
+    }
+    catch (error) {
+        getLogger$6().warn("[llmobs] annotate failed");
+        getLogger$6().var({ error });
+    }
+}
+/**
+ * Open a manual span that is finished later. Used by streaming, where work is
+ * yielded incrementally and cannot be enclosed in a single callback. The span
+ * is kept open via a deferred promise resolved by `finish()`. Returns null when
+ * emission is disabled or dd-trace is absent.
+ *
+ * Note: because the span is held open across `yield` boundaries (outside the
+ * SDK's AsyncLocalStorage scope), manual spans are emitted flat rather than
+ * nested. Annotations are applied to the explicit span reference.
+ */
+function openLlmObsSpan(options) {
+    const sdk = activeSdk();
+    if (!sdk) {
+        return null;
+    }
+    try {
+        let capturedSpan;
+        let release;
+        const held = new Promise((resolve) => {
+            release = resolve;
+        });
+        // trace() runs the callback synchronously, capturing the span; the returned
+        // pending promise keeps the span open until finish() resolves it.
+        void sdk.trace(options, (span) => {
+            capturedSpan = span;
+            return held;
+        });
+        let finished = false;
+        return {
+            annotate: (annotation) => {
+                try {
+                    sdk.annotate(capturedSpan, annotation);
+                }
+                catch (error) {
+                    getLogger$6().warn("[llmobs] annotate failed");
+                    getLogger$6().var({ error });
+                }
+            },
+            finish: () => {
+                if (finished) {
+                    return;
+                }
+                finished = true;
+                release?.();
+            },
+        };
+    }
+    catch (error) {
+        getLogger$6().warn("[llmobs] span emission failed");
+        getLogger$6().var({ error });
+        return null;
+    }
+}
+//
+//
+// Mapping Helpers
+//
+/**
+ * Sum an `LlmUsage` array into LLM Observability metric keys. Returns undefined
+ * when there is no usage data so callers can omit the `metrics` field.
+ */
+function usageToLlmObsMetrics(usage) {
+    if (!usage || usage.length === 0) {
+        return undefined;
+    }
+    const metrics = usage.reduce((sum, item) => ({
+        input_tokens: sum.input_tokens + (item.input ?? 0),
+        output_tokens: sum.output_tokens + (item.output ?? 0),
+        total_tokens: sum.total_tokens + (item.total ?? 0),
+    }), { input_tokens: 0, output_tokens: 0, total_tokens: 0 });
+    return metrics;
+}
+
 //
 //
 // Main
@@ -5706,30 +5986,46 @@ class OperateLoop {
         // Initialize state
         const state = await this.initializeState(input, options);
         const context = this.createContext(options);
-        // Build initial request
-        let request = this.buildInitialRequest(state, options);
-        // Multi-turn loop
-        while (state.currentTurn < state.maxTurns) {
-            state.currentTurn++;
-            // Execute one turn with retry logic
-            const shouldContinue = await this.executeOneTurn(request, state, context, options);
-            if (!shouldContinue) {
-                break;
+        const modelName = options.model ?? this.adapter.defaultModel;
+        // Enclosing LLM Observability span (no-op when DD_LLMOBS_ENABLED is unset).
+        // Child llm/tool spans nest under it via the SDK's active-span context.
+        return withLlmObsSpan({
+            kind: state.toolkit ? "agent" : "llm",
+            modelName,
+            modelProvider: this.adapter.name,
+            name: "jaypie.llm.operate",
+        }, async () => {
+            // Build initial request
+            let request = this.buildInitialRequest(state, options);
+            // Multi-turn loop
+            while (state.currentTurn < state.maxTurns) {
+                state.currentTurn++;
+                // Execute one turn with retry logic
+                const shouldContinue = await this.executeOneTurn(request, state, context, options);
+                if (!shouldContinue) {
+                    break;
+                }
+                // Rebuild request with updated history for next turn
+                request = {
+                    format: state.formattedFormat,
+                    instructions: options.instructions,
+                    messages: state.currentInput,
+                    model: modelName,
+                    providerOptions: options.providerOptions,
+                    system: options.system,
+                    temperature: options.temperature,
+                    tools: state.formattedTools,
+                    user: options.user,
+                };
             }
-            // Rebuild request with updated history for next turn
-            request = {
-                format: state.formattedFormat,
-                instructions: options.instructions,
-                messages: state.currentInput,
-                model: options.model ?? this.adapter.defaultModel,
-                providerOptions: options.providerOptions,
-                system: options.system,
-                temperature: options.temperature,
-                tools: state.formattedTools,
-                user: options.user,
-            };
-        }
-        return state.responseBuilder.build();
+            const response = state.responseBuilder.build();
+            annotateLlmObs({
+                inputData: input,
+                metrics: usageToLlmObsMetrics(response.usage),
+                outputData: response.content,
+            });
+            return response;
+        });
     }
     //
     // Private Methods
@@ -5831,20 +6127,34 @@ class OperateLoop {
             options,
             providerRequest,
         });
-        // Execute with retry (RetryExecutor handles error hooks and throws appropriate errors)
-        const response = await retryExecutor.execute((signal) => this.adapter.executeRequest(this.client, providerRequest, signal), {
-            context: {
-                input: state.currentInput,
-                options,
-                providerRequest,
-            },
-            hooks: context.hooks,
+        // Execute with retry inside a child llm span (no-op when llmobs disabled).
+        // RetryExecutor handles error hooks and throws appropriate errors.
+        const { parsed, response } = await withLlmObsSpan({
+            kind: "llm",
+            modelName: options.model ?? this.adapter.defaultModel,
+            modelProvider: this.adapter.name,
+            name: "jaypie.llm.model",
+        }, async () => {
+            const response = await retryExecutor.execute((signal) => this.adapter.executeRequest(this.client, providerRequest, signal), {
+                context: {
+                    input: state.currentInput,
+                    options,
+                    providerRequest,
+                },
+                hooks: context.hooks,
+            });
+            // Log what was returned from the model
+            log.trace("[operate] Model response received");
+            log.var({ "operate.response": response });
+            // Parse response
+            const parsed = this.adapter.parseResponse(response, options);
+            annotateLlmObs({
+                inputData: state.currentInput,
+                metrics: usageToLlmObsMetrics(parsed.usage ? [parsed.usage] : undefined),
+                outputData: parsed.content ?? "",
+            });
+            return { parsed, response };
         });
-        // Log what was returned from the model
-        log.trace("[operate] Model response received");
-        log.var({ "operate.response": response });
-        // Parse response
-        const parsed = this.adapter.parseResponse(response, options);
         // Track usage
         if (parsed.usage) {
             state.responseBuilder.addUsage(parsed.usage);
@@ -5865,7 +6175,7 @@ class OperateLoop {
         if (this.adapter.hasStructuredOutput(response)) {
             const structuredOutput = this.adapter.extractStructuredOutput(response);
             if (structuredOutput) {
-                state.responseBuilder.setContent(structuredOutput);
+                state.responseBuilder.setContent(this.applyFormatArrayDefaults(structuredOutput, options));
                 state.responseBuilder.complete();
                 return false; // Stop loop
             }
@@ -5889,11 +6199,19 @@ class OperateLoop {
                             args: toolCall.arguments,
                             toolName: toolCall.name,
                         });
-                        // Call the tool
+                        // Call the tool inside a child tool span (no-op when disabled)
                         log.trace(`[operate] Calling tool - ${toolCall.name}`);
-                        const result = await state.toolkit.call({
-                            arguments: toolCall.arguments,
-                            name: toolCall.name,
+                        const result = await withLlmObsSpan({ kind: "tool", name: toolCall.name }, async () => {
+                            const result = await state.toolkit.call({
+                                arguments: toolCall.arguments,
+                                name: toolCall.name,
+                            });
+                            annotateLlmObs({
+                                inputData: toolCall.arguments,
+                                metadata: { tool: toolCall.name },
+                                outputData: result,
+                            });
+                            return result;
                         });
                         // Execute afterEachTool hook
                         await this.hookRunnerInstance.runAfterTool(context.hooks, {
@@ -5982,7 +6300,7 @@ class OperateLoop {
             }
         }
         // No tool calls or no toolkit - we're done
-        state.responseBuilder.setContent(parsed.content);
+        state.responseBuilder.setContent(this.applyFormatArrayDefaults(parsed.content, options));
         state.responseBuilder.complete();
         // Add final history items
         const historyItems = this.adapter.responseToHistoryItems(parsed.raw);
@@ -5991,6 +6309,20 @@ class OperateLoop {
         }
         return false; // Stop loop
     }
+    /**
+     * Backfill declared array fields when a `format` is supplied. A declared
+     * `format` is a schema contract: an empty array field should surface as `[]`
+     * rather than be dropped by a provider/model that omits empty lists.
+     */
+    applyFormatArrayDefaults(content, options) {
+        if (!options.format) {
+            return content;
+        }
+        if (typeof content !== "object" || content === null) {
+            return content;
+        }
+        return fillFormatArrays({ content, format: options.format });
+    }
     /**
      * Sync the current input state from the updated provider request.
      * This is necessary because appendToolResult modifies the provider-specific request,
@@ -6143,7 +6475,7 @@ class StreamLoop {
             }
             // If we have tool calls, process them
             if (toolCalls && toolCalls.length > 0 && state.toolkit) {
-                yield* this.processToolCalls(toolCalls, state, context, options);
+                yield* this.processToolCalls(toolCalls, state, context);
                 // Check if we've reached max turns
                 if (state.currentTurn >= state.maxTurns) {
                     const error = new TooManyRequestsError();
@@ -6250,6 +6582,16 @@ class StreamLoop {
             options,
             providerRequest,
         });
+        // Open a manual llm span held open across the streamed turn. Flat (not
+        // nested) because it spans yield boundaries; no-op when llmobs disabled.
+        const llmSpan = openLlmObsSpan({
+            kind: "llm",
+            modelName: options.model ?? this.adapter.defaultModel,
+            modelProvider: this.adapter.name,
+            name: "jaypie.llm.model",
+        });
+        const inputSnapshot = [...state.currentInput];
+        let streamedText = "";
         // Collect tool calls from the stream
         const collectedToolCalls = [];
         // Retry loop for connection-level failures
@@ -6269,6 +6611,7 @@ class StreamLoop {
                         // Pass through text chunks
                         if (chunk.type === LlmStreamChunkType.Text) {
                             chunksYielded = true;
+                            streamedText += chunk.content ?? "";
                             yield chunk;
                         }
                         // Collect tool calls
@@ -6315,6 +6658,12 @@ class StreamLoop {
                                 title: "Stream Error",
                             },
                         };
+                        llmSpan?.annotate({
+                            inputData: inputSnapshot,
+                            metrics: usageToLlmObsMetrics(state.usageItems),
+                            outputData: streamedText,
+                        });
+                        llmSpan?.finish();
                         return { shouldContinue: false };
                     }
                     // Check if we've exhausted retries or error is not retryable
@@ -6323,6 +6672,7 @@ class StreamLoop {
                         log.error(`Stream request failed after ${this.retryPolicy.maxRetries} retries`);
                         log.var({ error });
                         const errorMessage = error instanceof Error ? error.message : String(error);
+                        llmSpan?.finish();
                         throw new BadGatewayError(errorMessage);
                     }
                     const delay = this.retryPolicy.getDelayForAttempt(attempt);
@@ -6336,6 +6686,13 @@ class StreamLoop {
         finally {
             guard.remove();
         }
+        // Annotate and finish the streamed llm span (no-op when disabled)
+        llmSpan?.annotate({
+            inputData: inputSnapshot,
+            metrics: usageToLlmObsMetrics(state.usageItems),
+            outputData: streamedText,
+        });
+        llmSpan?.finish();
         // Execute afterEachModelResponse hook
         await this.hookRunnerInstance.runAfterModelResponse(context.hooks, {
             content: "",
@@ -6370,7 +6727,7 @@ class StreamLoop {
         }
         return { shouldContinue: false };
     }
-    async *processToolCalls(toolCalls, state, context, _options) {
+    async *processToolCalls(toolCalls, state, context) {
         const log = getLogger$6();
         for (const toolCall of toolCalls) {
             try {
@@ -6379,11 +6736,19 @@ class StreamLoop {
                     args: toolCall.arguments,
                     toolName: toolCall.name,
                 });
-                // Call the tool
+                // Call the tool inside a child tool span (no-op when disabled)
                 log.trace(`[stream] Calling tool - ${toolCall.name}`);
-                const result = await state.toolkit.call({
-                    arguments: toolCall.arguments,
-                    name: toolCall.name,
+                const result = await withLlmObsSpan({ kind: "tool", name: toolCall.name }, async () => {
+                    const result = await state.toolkit.call({
+                        arguments: toolCall.arguments,
+                        name: toolCall.name,
+                    });
+                    annotateLlmObs({
+                        inputData: toolCall.arguments,
+                        metadata: { tool: toolCall.name },
+                        outputData: result,
+                    });
+                    return result;
                 });
                 // Execute afterEachTool hook
                 await this.hookRunnerInstance.runAfterTool(context.hooks, {