@nodetool-ai/runtime 0.8.1 → 0.8.3

package/dist/providers/base-provider.js

@@ -1,8 +1,10 @@
+import { stringifyTraceContent } from "../run-trace-serialization.js";
+import { tracedProviderLoop, tracedToolCall, beginProviderRound, providerRoundContext, traceContentAllowed, messageTraceContent, traceTextContent } from "./loop-tracing.js";
 import { isProviderSessionUpdate, isProviderMessageEvent, isProviderStop, isProviderToolErrorResult, PROVIDER_STOP_ABORTED } from "./types.js";
 import { ProviderGenerationsUnsupportedError } from "./provider-generations.js";
 import { CostCalculator } from "./cost-calculator.js";
 import { getTracer } from "../telemetry.js";
-import { withUsageCapture, withModalityCapture, setLastUsage, setLastRequest, consumeLastUsage, peekLastUsage, peekLastRequest, createUsageSlot } from "../tracing-helpers.js";
+import { withUsageCapture, withModalityCapture, setLastUsage, setLastRequest, consumeLastUsage, peekLastUsage, peekLastRequest, createUsageSlot, withSpan, withSpanGen } from "../tracing-helpers.js";
 import { isRunBudget } from "../turn-budget.js";
 import { countTokens } from "../token-counter.js";
 import { isCallable, isObjectLike, isRecord, isString } from "@nodetool-ai/protocol";
@@ -11,7 +13,7 @@ import { annotateProviderError, markContextExceeded } from "./provider-error.js"
 import { buildProviderCallFailure } from "./provider-call-failure.js";
 import { applyEntityReferences } from "./entity-references.js";
 import { expandVideoContentAsFrames, videoFrameFallbackEnabled } from "./video-frame-fallback.js";
-import { SpanStatusCode } from "@opentelemetry/api";
+import { SpanStatusCode, context, trace } from "@opentelemetry/api";
 import { createLogger, getDefaultAssetsPath } from "@nodetool-ai/config";
 const log = createLogger("nodetool.runtime.provider");
 /**
@@ -72,117 +74,6 @@ export function splitToolResultImages(result) {
         imageMessage: { role: "user", content: [...texts, ...images] }
     };
 }
-/**
- * Derive a provider's capability set by checking which optional `getAvailable*`
- * methods it overrides on its prototype. Every provider can always generate
- * messages; image/video/TTS/ASR/embedding are advertised only when the concrete
- * class overrides the matching base method.
- *
- * Shared by `packages/websocket/src/models-api.ts` (REST leftover) and the
- * `models` tRPC router so both report identical capabilities for the same
- * provider instance.
- */
-export function providerCapabilities(instance) {
-    const capabilities = [
-        "generate_message",
-        "generate_messages"
-    ];
-    if (instance.getAvailableImageModels !==
-        BaseProvider.prototype.getAvailableImageModels) {
-        capabilities.push("text_to_image", "image_to_image");
-    }
-    if (instance.getAvailableVideoModels !==
-        BaseProvider.prototype.getAvailableVideoModels) {
-        capabilities.push("text_to_video", "image_to_video");
-    }
-    if (instance.referenceToVideo !== BaseProvider.prototype.referenceToVideo) {
-        capabilities.push("reference_to_video");
-    }
-    // The remaining task types don't have their own model-discovery method —
-    // they reuse Image/VideoModel (advertised via each model's `supportedTasks`).
-    // Advertise the provider-level capability when the concrete class overrides
-    // the matching task method.
-    if (instance.inpaint !== BaseProvider.prototype.inpaint) {
-        capabilities.push("inpainting");
-    }
-    if (instance.outpaintImage !== BaseProvider.prototype.outpaintImage) {
-        capabilities.push("outpaint_image");
-    }
-    if (instance.upscaleImage !== BaseProvider.prototype.upscaleImage) {
-        capabilities.push("upscale_image");
-    }
-    if (instance.removeBackground !== BaseProvider.prototype.removeBackground) {
-        capabilities.push("remove_background");
-    }
-    if (instance.relightImage !== BaseProvider.prototype.relightImage) {
-        capabilities.push("relight_image");
-    }
-    if (instance.segmentImage !== BaseProvider.prototype.segmentImage) {
-        capabilities.push("segment_image");
-    }
-    if (instance.vectorizeImage !== BaseProvider.prototype.vectorizeImage) {
-        capabilities.push("vectorize_image");
-    }
-    if (instance.videoToVideo !== BaseProvider.prototype.videoToVideo) {
-        capabilities.push("video_to_video");
-    }
-    if (instance.videoToAudio !== BaseProvider.prototype.videoToAudio) {
-        capabilities.push("video_to_audio");
-    }
-    if (instance.trackObject !== BaseProvider.prototype.trackObject) {
-        capabilities.push("track_object");
-    }
-    if (instance.extendVideo !== BaseProvider.prototype.extendVideo) {
-        capabilities.push("extend_video");
-    }
-    if (instance.upscaleVideo !== BaseProvider.prototype.upscaleVideo) {
-        capabilities.push("upscale_video");
-    }
-    if (instance.interpolateVideo !== BaseProvider.prototype.interpolateVideo) {
-        capabilities.push("interpolate_video");
-    }
-    if (instance.outpaintVideo !== BaseProvider.prototype.outpaintVideo) {
-        capabilities.push("outpaint_video");
-    }
-    if (instance.lipSync !== BaseProvider.prototype.lipSync) {
-        capabilities.push("lip_sync");
-    }
-    if (instance.getAvailableTTSModels !==
-        BaseProvider.prototype.getAvailableTTSModels) {
-        capabilities.push("text_to_speech");
-    }
-    if (instance.getAvailableMusicModels !==
-        BaseProvider.prototype.getAvailableMusicModels) {
-        capabilities.push("text_to_music");
-    }
-    if (instance.getAvailableAudioToAudioModels !==
-        BaseProvider.prototype.getAvailableAudioToAudioModels) {
-        capabilities.push("audio_to_audio");
-    }
-    if (instance.getAvailableASRModels !==
-        BaseProvider.prototype.getAvailableASRModels) {
-        capabilities.push("automatic_speech_recognition");
-    }
-    if (instance.getAvailableEmbeddingModels !==
-        BaseProvider.prototype.getAvailableEmbeddingModels) {
-        capabilities.push("generate_embedding");
-    }
-    if (instance.getAvailable3DModels !==
-        BaseProvider.prototype.getAvailable3DModels) {
-        // Providers that expose 3D models are assumed to support both task types;
-        // individual `textTo3D` / `imageTo3D` calls will throw if not implemented.
-        capabilities.push("text_to_3d", "image_to_3d");
-    }
-    // The provider's own record of what it ran, exposed only by providers with
-    // a history API behind it (see provider-generations.ts).
-    if (instance.listGenerations !== BaseProvider.prototype.listGenerations) {
-        capabilities.push("list_generations");
-    }
-    if (instance.getGeneration !== BaseProvider.prototype.getGeneration) {
-        capabilities.push("get_generation");
-    }
-    return capabilities;
-}
 /**
  * What one attached medium costs a vision/audio model, per block. These are
  * modality constants, not measurements of the bytes: by the time a turn is
@@ -308,6 +199,35 @@ export class BaseProvider {
         if (this._emitMessage)
             this._emitMessage(msg);
     }
+    /**
+     * Report one non-chat provider call (image, video, speech, ASR, embedding…)
+     * as an `llm_call` so the trace shows every provider call, not only chat.
+     * Carries no token counts: those modalities bill per unit, not per token.
+     * @internal
+     */
+    recordModalityCall(input) {
+        if (!this._emitMessage)
+            return;
+        const prompt = input.args
+            .map((arg) => isObjectLike(arg) ? arg.prompt : undefined)
+            .find(isString);
+        this.emitMessage({
+            type: "llm_call",
+            node_id: "",
+            provider: this.provider,
+            operation: input.operation,
+            model: extractModelId(input.args),
+            messages: prompt ? [{ role: "user", content: prompt }] : [],
+            response: null,
+            tool_calls: null,
+            tokens_input: null,
+            tokens_output: null,
+            cost: null,
+            duration_ms: Date.now() - input.startedAt,
+            error: input.error === undefined ? null : String(input.error),
+            timestamp: new Date(input.startedAt).toISOString()
+        });
+    }
     /**
      * Record a failed provider call: a server-side log with the exact request,
      * and a `provider_call_failed` message so the surface showing the failure
@@ -379,8 +299,23 @@ export class BaseProvider {
                 // images onto the source list, before the concrete provider runs.
                 const args = applyEntityReferences(name, rawArgs);
                 const startedAt = Date.now();
+                const call = () => original.apply(this, args);
                 try {
-                    return await original.apply(this, args);
+                    // A nested call (a batch helper delegating to its singular method)
+                    // stays inside the outer call's span.
+                    const result = alreadyActive
+                        ? await call()
+                        : await withSpan(`provider.${name}`, modalitySpanAttributes(this.provider, name, args), async (span) => {
+                            const value = await call();
+                            if (span) {
+                                applyUsageAttributes(span, peekLastUsage());
+                            }
+                            return value;
+                        });
+                    if (!alreadyActive) {
+                        this.recordModalityCall({ operation: name, args, startedAt });
+                    }
+                    return result;
                 }
                 catch (err) {
                     annotateProviderError(err, {
@@ -388,6 +323,12 @@ export class BaseProvider {
                         model: extractModelId(args)
                     });
                     if (!alreadyActive) {
+                        this.recordModalityCall({
+                            operation: name,
+                            args,
+                            startedAt,
+                            error: err
+                        });
                         const failureArgs = name === "referenceToVideo"
                             ? [referenceVideoDiagnostics(args)]
                             : args;
@@ -410,7 +351,7 @@ export class BaseProvider {
                 continue;
             }
             const original = fn;
-            Reflect.set(this, name, (...args) => wrapModalityGenerator(this, original, args, name));
+            Reflect.set(this, name, (...args) => withSpanGen(`provider.${name}`, modalitySpanAttributes(this.provider, name, args), () => wrapModalityGenerator(this, original, args, name)));
         }
     }
     /**
@@ -483,32 +424,25 @@ export class BaseProvider {
         return null;
     }
     /**
-     * Explicit capability declaration. Returns `null` by default, in which case
-     * {@link getCapabilities} derives capabilities by reflecting on which
-     * optional methods the concrete class overrides. Providers may override this
-     * to declare their capabilities directly — an alternative to method
-     * reflection when methods are wrapped, bound, or composed via mixins.
+     * The operations beyond chat that this provider implements. Each concrete
+     * provider lists its own; the base implements none. A capability belongs
+     * here only when the matching method has a working implementation.
      */
     declaredCapabilities() {
-        return null;
+        return [];
     }
     /**
-     * The capabilities this provider exposes. Prefers an explicit declaration
-     * from {@link declaredCapabilities}; otherwise falls back to reflecting over
-     * overridden methods via {@link providerCapabilities}. Callers should use
-     * this instead of calling `providerCapabilities()` directly so explicit
-     * declarations are honored.
+     * The capabilities this provider exposes: chat generation, which every
+     * provider has, plus its {@link declaredCapabilities}.
      */
     getCapabilities() {
-        const declared = this.declaredCapabilities();
-        if (declared) {
-            // Message generation is always available; ensure it is present.
-            const set = new Set(declared);
-            set.add("generate_message");
-            set.add("generate_messages");
-            return [...set];
-        }
-        return providerCapabilities(this);
+        return [
+            ...new Set([
+                "generate_message",
+                "generate_messages",
+                ...this.declaredCapabilities()
+            ])
+        ];
     }
     /**
      * The provider's own record of the generations this account ran, newest
@@ -679,6 +613,10 @@ export class BaseProvider {
     async getAvailableEmbeddingModels() {
         return [];
     }
+    /** Reranking models exposed by this provider. */
+    async getAvailableRerankModels() {
+        return [];
+    }
     /**
      * 3D **generation** models exposed by this provider (e.g. Meshy's `meshy-4`,
      * Rodin's `rodin-regular`). Override on providers that can synthesize 3D
@@ -700,12 +638,14 @@ export class BaseProvider {
                     const result = await this.generateMessage(args);
                     const content = isString(result.content)
                         ? result.content
-                        : JSON.stringify(result.content);
+                        : stringifyTraceContent(result.content);
                     span.setAttributes({
                         "llm.response.role": result.role,
-                        "llm.response.content": content.slice(0, 2000),
                         "llm.response.tool_calls_count": result.toolCalls?.length ?? 0
                     });
+                    if (traceContentAllowed()) {
+                        span.setAttribute("llm.response.content", traceTextContent(content));
+                    }
                     span.setStatus({ code: SpanStatusCode.OK });
                     return result;
                 }
@@ -793,6 +733,10 @@ export class BaseProvider {
         }
     }
     applyLlmRequestAttributes(span, args, streaming) {
+        if (traceContentAllowed()) {
+            span.setAttribute("llm.request.messages", messageTraceContent(args.messages.filter((message) => typeof message === "object" && message !== null && "role" in message)));
+            span.setAttribute("llm.tool.names", stringifyTraceContent((args.tools ?? []).map((tool) => typeof tool === "object" && tool !== null && "name" in tool ? tool.name : null)));
+        }
         span.setAttributes({
             "llm.provider": this.provider,
             "llm.model": args.model,
@@ -953,8 +897,11 @@ export class BaseProvider {
         });
     }
     async *generateLoop(args) {
+        return yield* tracedProviderLoop(this.provider, args.model, () => BaseProvider.prototype.generateBaseLoop.call(this, args));
+    }
+    async *generateBaseLoop(args) {
         const maxIterations = args.maxIterations ?? 25;
-        const { executeTool, maxIterations: _omitMax, sequentialTools, turnBudget: budgetArg, workspaceDir: _omitWorkspaceDir, skills: _omitSkills, resolveMedia, ...turnArgs } = args;
+        const { executeTool, maxIterations: _omitMax, sequentialTools, providedToolsOnly: _omitProvidedToolsOnly, turnBudget: budgetArg, workspaceDir: _omitWorkspaceDir, skills: _omitSkills, resolveMedia, ...turnArgs } = args;
         const { runBudget, turnBudget } = resolveTurnBudget(budgetArg);
         const messages = [...args.messages];
         // What the provider is actually sent: `messages` with every media
@@ -983,11 +930,8 @@ export class BaseProvider {
         let toolCallSeq = 0;
         const onToolCall = executeTool
             ? async (name, toolArgs) => {
-                const result = await executeTool({
-                    id: `call_${++toolCallSeq}`,
-                    name,
-                    args: toolArgs
-                });
+                const call = { id: `call_${++toolCallSeq}`, name, args: toolArgs };
+                const result = await tracedToolCall(call, () => executeTool(call));
                 if (isProviderToolErrorResult(result)) {
                     return toolResultToText(result.content);
                 }
@@ -1011,6 +955,7 @@ export class BaseProvider {
                     return;
                 }
             }
+            beginProviderRound(this.provider, args.model);
             const costBeforeTurn = turnBudget ? this.getTotalCost() : 0;
             let assistantText = "";
             let finalizedAssistant;
@@ -1131,10 +1076,12 @@ export class BaseProvider {
             const runTool = async (tc) => {
                 const tool = toolMap.get(tc.name);
                 try {
-                    if (tool?.execute)
-                        return await tool.execute(tc.args ?? {}, tc.id);
+                    if (tool?.execute) {
+                        const execute = tool.execute;
+                        return await tracedToolCall(tc, () => execute(tc.args ?? {}, tc.id));
+                    }
                     if (executeTool)
-                        return await executeTool(tc);
+                        return await tracedToolCall(tc, () => executeTool(tc));
                     return `Tool "${tc.name}" is not available`;
                 }
                 catch (err) {
@@ -1223,15 +1170,31 @@ export class BaseProvider {
     }
     /** Internal: wrap generateMessages with OTel span */
     async *_tracedStream(args, tracer) {
-        const span = tracer.startSpan(`llm.stream ${this.provider}/${args.model}`);
+        const span = tracer.startSpan(`llm.stream ${this.provider}/${args.model}`, {}, providerRoundContext());
         this.applyLlmRequestAttributes(span, args, true);
         let chunkCount = 0;
+        let response = "";
+        const stream = this.generateMessages(args);
+        const active = trace.setSpan(providerRoundContext(), span);
+        let exhausted = false;
         try {
-            for await (const item of this.generateMessages(args)) {
+            while (true) {
+                const next = await context.with(active, () => stream.next());
+                if (next.done) {
+                    exhausted = true;
+                    break;
+                }
+                const item = next.value;
+                if (traceContentAllowed() && "content" in item && typeof item.content === "string" && !("thinking" in item && item.thinking) && (!("content_type" in item) || item.content_type === "text")) {
+                    response = traceTextContent(response + item.content);
+                }
                 chunkCount++;
                 yield item;
             }
-            span.setAttributes({ "llm.response.chunk_count": chunkCount });
+            span.setAttribute("llm.response.chunk_count", chunkCount);
+            if (traceContentAllowed()) {
+                span.setAttribute("llm.response.content", response);
+            }
             span.setStatus({ code: SpanStatusCode.OK });
         }
         catch (err) {
@@ -1240,6 +1203,9 @@ export class BaseProvider {
             throw err;
         }
         finally {
+            if (!exhausted) {
+                await context.with(active, () => stream.return(undefined));
+            }
             applyUsageAttributes(span, peekLastUsage());
             span.end();
         }
@@ -1324,6 +1290,13 @@ export class BaseProvider {
     async removeBackground(_image, _params) {
         throw new Error(`${this.provider} does not support removeBackground`);
     }
+    /**
+     * Estimate per-pixel depth. Returns a grayscale PNG at the input's size,
+     * where brighter pixels are nearer to the camera.
+     */
+    async estimateDepth(_image, _params) {
+        throw new Error(`${this.provider} does not support estimateDepth`);
+    }
     /**
      * Segment an image into object masks. Returns one entry per object the model
      * found, in the model's own order; an empty array means it found nothing,
@@ -1455,6 +1428,10 @@ export class BaseProvider {
     async generateEmbedding(_args) {
         throw new Error(`${this.provider} does not support generateEmbedding`);
     }
+    /** Score each document against the query, most relevant first. */
+    async rerank(_params) {
+        throw new Error(`${this.provider} does not support rerank`);
+    }
     /**
      * Whether `error` is this provider refusing the request because it exceeds
      * the model's context window. Default `false`: a provider whose signal has
@@ -1615,6 +1592,7 @@ const MODALITY_PROMISE_METHODS = [
     "outpaintImage",
     "upscaleImage",
     "removeBackground",
+    "estimateDepth",
     "relightImage",
     "vectorizeImage",
     "segmentImage",
@@ -1634,7 +1612,8 @@ const MODALITY_PROMISE_METHODS = [
     "lipSync",
     "textTo3D",
     "imageTo3D",
-    "generateEmbedding"
+    "generateEmbedding",
+    "rerank"
 ];
 /** Async-generator non-chat modality methods wrapped with failure logging. */
 const MODALITY_GENERATOR_METHODS = ["textToSpeech"];
@@ -1669,6 +1648,7 @@ async function* wrapModalityGenerator(provider, original, args, operationName) {
     const startedAt = Date.now();
     const source = original.apply(provider, args);
     let exhausted = false;
+    let failure;
     try {
         while (true) {
             const result = await runInSlot(() => source.next());
@@ -1680,6 +1660,7 @@ async function* wrapModalityGenerator(provider, original, args, operationName) {
         }
     }
     catch (err) {
+        failure = err;
         annotateProviderError(err, {
             provider: provider.provider,
             model: extractModelId(args)
@@ -1702,8 +1683,22 @@ async function* wrapModalityGenerator(provider, original, args, operationName) {
                     value: undefined
                 })).catch(() => { });
         }
+        provider.recordModalityCall({
+            operation: operationName,
+            args,
+            startedAt,
+            error: failure
+        });
     }
 }
+/** Metadata for a non-chat modality span. The prompt and media stay out of it. */
+function modalitySpanAttributes(provider, operation, args) {
+    return {
+        "gen_ai.system": provider,
+        "gen_ai.operation.name": operation,
+        "gen_ai.request.model": extractModelId(args)
+    };
+}
 /** Attach gen_ai usage attributes to a span (no-op if usage is null). */
 function applyUsageAttributes(span, usage) {
     if (!usage)