nexus-ai-pro 2.0.0 → 2.2.0

package/dist-cjs/testing/record.js

@@ -36,7 +36,10 @@ function recordingFetch(options) {
         const captured = await captureRequest(copyOf(input, init), options);
         const real = options.fetch ?? globalThis.fetch;
         const response = await real(input, init);
-        const bytes = new Uint8Array(await response.arrayBuffer());
+        const { bytes, chunks } = options.timing
+            ? await readTimed(response)
+            : { bytes: new Uint8Array(await response.arrayBuffer()), chunks: undefined };
+        const contentType = response.headers.get('content-type');
         const index = counters.get(captured.key) ?? 0;
         counters.set(captured.key, index + 1);
         let exchange = {
@@ -47,7 +50,10 @@ function recordingFetch(options) {
                 status: response.status,
                 ...(response.statusText ? { statusText: response.statusText } : {}),
                 headers: withoutTransportHeaders(redactHeaders(response.headers, options.redactHeaders)),
-                ...encodeBody(bytes, response.headers.get('content-type')),
+                ...encodeBody(bytes, contentType),
+                ...(chunks
+                    ? { chunks: chunks.map((chunk) => ({ atMs: chunk.atMs, ...encodeBody(chunk.bytes, contentType) })) }
+                    : {}),
             },
             recordedAt: now().toISOString(),
         };
@@ -83,9 +89,13 @@ function replayFetch(options) {
         const position = served.get(captured.key) ?? 0;
         served.set(captured.key, position + 1);
         const exchange = recordings[Math.min(position, recordings.length - 1)];
-        const body = exchange.response.bodyEncoding === 'base64'
-            ? Buffer.from(exchange.response.body, 'base64')
-            : exchange.response.body;
+        const pace = options.pace ?? 'instant';
+        const chunks = exchange.response.chunks;
+        const body = pace !== 'instant' && chunks?.length
+            ? pacedBody(chunks, pace, request.signal)
+            : exchange.response.bodyEncoding === 'base64'
+                ? Buffer.from(exchange.response.body, 'base64')
+                : exchange.response.body;
         return new Response(isNullBodyStatus(exchange.response.status) ? null : body, {
             status: exchange.response.status,
             statusText: exchange.response.statusText ?? '',
@@ -193,6 +203,48 @@ function encodeBody(bytes, contentType) {
     }
     return { body: Buffer.from(bytes).toString('base64'), bodyEncoding: 'base64' };
 }
+async function readTimed(response) {
+    const chunks = [];
+    if (!response.body)
+        return { bytes: new Uint8Array(), chunks };
+    const started = performance.now();
+    const reader = response.body.getReader();
+    for (;;) {
+        const { value, done } = await reader.read();
+        if (done)
+            break;
+        chunks.push({ atMs: Math.round(performance.now() - started), bytes: value });
+    }
+    const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.bytes.length, 0));
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk.bytes, offset);
+        offset += chunk.bytes.length;
+    }
+    return { bytes, chunks };
+}
+function pacedBody(chunks, pace, signal) {
+    let index = 0;
+    return new ReadableStream({
+        async pull(controller) {
+            const chunk = chunks[index];
+            if (!chunk || signal.aborted) {
+                controller.close();
+                return;
+            }
+            const delayMs = Math.max(0, chunk.atMs - (index === 0 ? chunk.atMs : chunks[index - 1].atMs));
+            if (typeof pace === 'function')
+                await pace({ index, count: chunks.length, atMs: chunk.atMs, delayMs });
+            else {
+                const wait = pace === 'original' ? delayMs : delayMs * pace;
+                if (wait > 0)
+                    await new Promise((resolve) => setTimeout(resolve, wait));
+            }
+            index += 1;
+            controller.enqueue(chunk.bodyEncoding === 'base64' ? Buffer.from(chunk.body, 'base64') : new TextEncoder().encode(chunk.body));
+        },
+    });
+}
 function isNullBodyStatus(status) {
     return status === 101 || status === 204 || status === 205 || status === 304;
 }

package/dist-cjs/tracing/index.d.ts

@@ -1,6 +1,6 @@
 export { type AlertEvent, AlertEvaluator, type AlertMetric, type AlertNotifier, type AlertRule, createWebhookNotifier, measure, type WebhookNotifierOptions, } from './alerts.js';
 export { compareTraces, formatTree, type RunDifference, type TraceComparison } from './compare.js';
 export { type GraphTracing, type GraphTracingOptions, type ModelClientLike, traceGraph, traceModelClient, } from './instrument.js';
-export { applyQuery, assembleTree, JsonlTraceStore, type JsonlTraceStoreOptions, MemoryTraceStore, type MemoryTraceStoreOptions, } from './stores.js';
-export { type FinishRunOptions, type RunHandle, type StartRunOptions, stripFields, Tracer, type TracerOptions, } from './tracer.js';
-export type { RedactionPolicy, Run, RunFeedback, RunKind, RunQuery, RunStatus, RunTree, SamplingPolicy, TraceStore, } from '../types/tracing.js';
+export { applyQuery, assembleTree, closeAbandonedRuns, type CloseAbandonedRunsOptions, JsonlTraceStore, type JsonlTraceStoreOptions, MemoryTraceStore, type MemoryTraceStoreOptions, } from './stores.js';
+export { type FinishRunOptions, type RunHandle, type StartRunOptions, stripFields, Tracer, type TracerOptions, w3cSpanId, w3cTraceId, } from './tracer.js';
+export type { RedactionPolicy, Run, RunFeedback, RunKind, RunQuery, RunStatus, RunTree, SamplingPolicy, TraceExporter, TraceStore, } from '../types/tracing.js';

package/dist-cjs/tracing/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Tracer = exports.stripFields = exports.MemoryTraceStore = exports.JsonlTraceStore = exports.assembleTree = exports.applyQuery = exports.traceModelClient = exports.traceGraph = exports.formatTree = exports.compareTraces = exports.measure = exports.createWebhookNotifier = exports.AlertEvaluator = void 0;
+exports.w3cTraceId = exports.w3cSpanId = exports.Tracer = exports.stripFields = exports.MemoryTraceStore = exports.JsonlTraceStore = exports.closeAbandonedRuns = exports.assembleTree = exports.applyQuery = exports.traceModelClient = exports.traceGraph = exports.formatTree = exports.compareTraces = exports.measure = exports.createWebhookNotifier = exports.AlertEvaluator = void 0;
 var alerts_js_1 = require("./alerts.js");
 Object.defineProperty(exports, "AlertEvaluator", { enumerable: true, get: function () { return alerts_js_1.AlertEvaluator; } });
 Object.defineProperty(exports, "createWebhookNotifier", { enumerable: true, get: function () { return alerts_js_1.createWebhookNotifier; } });
@@ -14,8 +14,11 @@ Object.defineProperty(exports, "traceModelClient", { enumerable: true, get: func
 var stores_js_1 = require("./stores.js");
 Object.defineProperty(exports, "applyQuery", { enumerable: true, get: function () { return stores_js_1.applyQuery; } });
 Object.defineProperty(exports, "assembleTree", { enumerable: true, get: function () { return stores_js_1.assembleTree; } });
+Object.defineProperty(exports, "closeAbandonedRuns", { enumerable: true, get: function () { return stores_js_1.closeAbandonedRuns; } });
 Object.defineProperty(exports, "JsonlTraceStore", { enumerable: true, get: function () { return stores_js_1.JsonlTraceStore; } });
 Object.defineProperty(exports, "MemoryTraceStore", { enumerable: true, get: function () { return stores_js_1.MemoryTraceStore; } });
 var tracer_js_1 = require("./tracer.js");
 Object.defineProperty(exports, "stripFields", { enumerable: true, get: function () { return tracer_js_1.stripFields; } });
 Object.defineProperty(exports, "Tracer", { enumerable: true, get: function () { return tracer_js_1.Tracer; } });
+Object.defineProperty(exports, "w3cSpanId", { enumerable: true, get: function () { return tracer_js_1.w3cSpanId; } });
+Object.defineProperty(exports, "w3cTraceId", { enumerable: true, get: function () { return tracer_js_1.w3cTraceId; } });

package/dist-cjs/tracing/instrument.js

@@ -13,6 +13,7 @@ function traceGraph(tracer, options = {}) {
     const active = new Map();
     const retries = new Map();
     const emitted = new Map();
+    const tools = new Map();
     const onEvent = (event) => {
         switch (event.type) {
             case 'task_start': {
@@ -29,6 +30,29 @@ function traceGraph(tracer, options = {}) {
                 retries.set(event.taskId, event.attempt);
                 return;
             }
+            case 'tool': {
+                const key = `${event.taskId}:${event.tool.id}`;
+                if (event.tool.phase === 'start') {
+                    const parent = active.get(event.taskId) ?? root;
+                    tools.set(key, parent.child({
+                        name: event.tool.name,
+                        kind: 'tool',
+                        inputs: event.tool.args,
+                        metadata: { callId: event.tool.id },
+                    }));
+                    return;
+                }
+                if (event.tool.phase === 'progress')
+                    return;
+                const handle = tools.get(key);
+                if (!handle)
+                    return;
+                tools.delete(key);
+                void handle.finish(event.tool.phase === 'error'
+                    ? { error: new Error(event.tool.error ?? 'The tool failed') }
+                    : { outputs: event.tool.result });
+                return;
+            }
             case 'custom': {
                 emitted.set(event.taskId, [...(emitted.get(event.taskId) ?? []), event.data]);
                 return;
@@ -57,6 +81,10 @@ function traceGraph(tracer, options = {}) {
         root,
         runFor: (node) => [...active.values()].find((handle) => handle.id && nameOf(active, handle) === node),
         async finish(result) {
+            for (const [key, handle] of tools) {
+                tools.delete(key);
+                await handle.finish({ metadata: { unfinished: true } });
+            }
             for (const [taskId, handle] of active) {
                 active.delete(taskId);
                 await handle.finish({ metadata: { unfinished: true } });
@@ -116,6 +144,8 @@ function traceModelClient(client, tracer, options = {}) {
                         model: response.meta?.modelUsed,
                         finishReason: response.finishReason,
                         ...(request.metadata?.prompt === undefined ? {} : { prompt: request.metadata.prompt }),
+                        ...(response.meta?.cache ? { cache: response.meta.cache } : {}),
+                        ...(request.tenantId ? { tenantId: request.tenantId } : {}),
                     },
                 });
                 return response;

package/dist-cjs/tracing/otlp.d.ts

@@ -0,0 +1,91 @@
+import type { Run, TraceExporter } from '../types/tracing.js';
+/** Options for `OtlpTraceExporter`. */
+export interface OtlpTraceExporterOptions {
+    /**
+     * The collector's OTLP/HTTP base URL, such as `http://localhost:4318`. Spans are posted to
+     * `/v1/traces` under it, as JSON.
+     */
+    endpoint: string;
+    /** Headers on every request, such as an API key a hosted backend wants. */
+    headers?: Record<string, string>;
+    /** `service.name` on every span. Defaults to `nexus-ai-pro`. */
+    serviceName?: string;
+    /** More resource attributes, such as `deployment.environment.name` or `service.version`. */
+    resource?: Record<string, string | number | boolean>;
+    /** Spans sent per request. Defaults to 100. */
+    maxBatch?: number;
+    /** How long a span waits for others to batch with, in milliseconds. Defaults to 2,000. */
+    flushIntervalMs?: number;
+    /** Replaces the global `fetch`, for tests or a proxy. */
+    fetch?: typeof globalThis.fetch;
+    /** Called when a batch cannot be sent. The batch is dropped; a trace is never worth a retry storm. */
+    onError?: (error: unknown) => void;
+}
+type AttributeValue = {
+    stringValue: string;
+} | {
+    intValue: string;
+} | {
+    doubleValue: number;
+} | {
+    boolValue: boolean;
+};
+/** One span in OTLP's JSON encoding. */
+export interface OtlpSpan {
+    /** 32 hex digits. */
+    traceId: string;
+    /** 16 hex digits. */
+    spanId: string;
+    /** 16 hex digits, absent for a root span. */
+    parentSpanId?: string;
+    /** The span's name. */
+    name: string;
+    /** 1 internal, 3 client. */
+    kind: number;
+    /** Nanoseconds since the epoch, as a decimal string. */
+    startTimeUnixNano: string;
+    /** Nanoseconds since the epoch, as a decimal string. */
+    endTimeUnixNano: string;
+    /** Typed attributes. */
+    attributes: Array<{
+        key: string;
+        value: AttributeValue;
+    }>;
+    /** 1 ok, 2 error. */
+    status: {
+        code: number;
+        message?: string;
+    };
+}
+/**
+ * Turns one finished run into an OpenTelemetry span, with the GenAI semantic conventions.
+ *
+ * A model run is a client span named `chat <model>` with `gen_ai.operation.name`,
+ * `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`, and
+ * `gen_ai.usage.input_tokens` and `output_tokens`. A tool run carries `gen_ai.tool.name`, an agent
+ * run `gen_ai.agent.name`. What only Nexus knows goes under `nexus.*`: the run kind, a graph node and
+ * its step, the cost, and the run id. A failed run has `error.type` and an error status.
+ */
+export declare function runToOtlpSpan(run: Run): OtlpSpan;
+/**
+ * Sends finished runs to any OpenTelemetry collector as OTLP/HTTP JSON, with no SDK to install.
+ *
+ * Pass it to a tracer's `exporters`, and every run the tracer keeps arrives as a span with the GenAI
+ * semantic conventions, so Grafana Tempo, Jaeger, Honeycomb, Datadog, New Relic, or any OTLP backend
+ * reads it as it reads the rest of the application. Spans are batched; `flush()` sends what is
+ * waiting. Trace ids are the W3C ids `RunHandle.traceparent()` propagates, so a trace that crossed
+ * processes arrives as one.
+ */
+export declare class OtlpTraceExporter implements TraceExporter {
+    private readonly options;
+    private readonly queue;
+    private timer;
+    private sending;
+    constructor(options: OtlpTraceExporterOptions);
+    /** Queues a run, sending a batch when one is full. Runs still in progress are skipped. */
+    export(run: Run): void;
+    /** Sends every span waiting. */
+    flush(): Promise<void>;
+    private send;
+}
+export {};

package/dist-cjs/tracing/otlp.js

@@ -0,0 +1,129 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OtlpTraceExporter = void 0;
+exports.runToOtlpSpan = runToOtlpSpan;
+const tracer_js_1 = require("./tracer.js");
+const OPERATIONS = {
+    model: 'chat',
+    embedding: 'embeddings',
+    tool: 'execute_tool',
+    agent: 'invoke_agent',
+};
+function runToOtlpSpan(run) {
+    const attributes = { 'nexus.run.kind': run.kind, 'nexus.run.id': run.id };
+    const operation = OPERATIONS[run.kind];
+    if (operation)
+        attributes['gen_ai.operation.name'] = operation;
+    if (run.provider)
+        attributes['gen_ai.provider.name'] = run.provider;
+    if (run.kind === 'model' || run.kind === 'embedding') {
+        const requested = run.metadata?.requestedModel;
+        if (typeof requested === 'string')
+            attributes['gen_ai.request.model'] = requested;
+        else if (run.model)
+            attributes['gen_ai.request.model'] = run.model;
+        if (run.model)
+            attributes['gen_ai.response.model'] = run.model;
+        const input = run.usage?.inputTokens ?? run.usage?.promptTokens;
+        const output = run.usage?.outputTokens ?? run.usage?.completionTokens;
+        if (input !== undefined)
+            attributes['gen_ai.usage.input_tokens'] = input;
+        if (output !== undefined)
+            attributes['gen_ai.usage.output_tokens'] = output;
+    }
+    if (run.kind === 'tool')
+        attributes['gen_ai.tool.name'] = run.name;
+    if (run.kind === 'agent')
+        attributes['gen_ai.agent.name'] = run.name;
+    if (run.kind === 'node') {
+        attributes['nexus.graph.node'] = run.name;
+        if (typeof run.metadata?.step === 'number')
+            attributes['nexus.graph.step'] = run.metadata.step;
+    }
+    if (run.cost !== undefined)
+        attributes['nexus.cost.usd'] = run.cost;
+    if (run.error)
+        attributes['error.type'] = run.error.name;
+    for (const tag of run.tags ?? [])
+        attributes[`nexus.tag.${tag}`] = true;
+    const started = Date.parse(run.startedAt);
+    const ended = run.endedAt ? Date.parse(run.endedAt) : started + (run.latencyMs ?? 0);
+    const name = run.kind === 'model' && run.model ? `chat ${run.model}` : operation ? `${operation} ${run.name}` : run.name;
+    return {
+        traceId: (0, tracer_js_1.w3cTraceId)(run.traceId),
+        spanId: (0, tracer_js_1.w3cSpanId)(run.id),
+        ...(run.parentId ? { parentSpanId: (0, tracer_js_1.w3cSpanId)(run.parentId) } : {}),
+        name,
+        kind: run.kind === 'model' || run.kind === 'embedding' ? 3 : 1,
+        startTimeUnixNano: nanos(started),
+        endTimeUnixNano: nanos(ended),
+        attributes: Object.entries(attributes).map(([key, value]) => ({ key, value: typed(value) })),
+        status: run.status === 'error' ? { code: 2, ...(run.error ? { message: run.error.message } : {}) } : { code: 1 },
+    };
+}
+class OtlpTraceExporter {
+    options;
+    queue = [];
+    timer;
+    sending = Promise.resolve();
+    constructor(options) {
+        this.options = options;
+    }
+    export(run) {
+        if (run.status === 'running')
+            return;
+        this.queue.push(runToOtlpSpan(run));
+        if (this.queue.length >= (this.options.maxBatch ?? 100)) {
+            void this.flush();
+            return;
+        }
+        if (!this.timer) {
+            this.timer = setTimeout(() => void this.flush(), this.options.flushIntervalMs ?? 2_000);
+            this.timer.unref?.();
+        }
+    }
+    async flush() {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = undefined;
+        while (this.queue.length > 0) {
+            const batch = this.queue.splice(0, this.options.maxBatch ?? 100);
+            this.sending = this.sending.then(() => this.send(batch));
+        }
+        await this.sending;
+    }
+    async send(spans) {
+        const resource = { 'service.name': this.options.serviceName ?? 'nexus-ai-pro', ...this.options.resource };
+        const body = {
+            resourceSpans: [
+                {
+                    resource: { attributes: Object.entries(resource).map(([key, value]) => ({ key, value: typed(value) })) },
+                    scopeSpans: [{ scope: { name: 'nexus-ai-pro' }, spans }],
+                },
+            ],
+        };
+        try {
+            const response = await (this.options.fetch ?? globalThis.fetch)(`${this.options.endpoint.replace(/\/+$/, '')}/v1/traces`, {
+                method: 'POST',
+                headers: { 'content-type': 'application/json', ...this.options.headers },
+                body: JSON.stringify(body),
+            });
+            if (!response.ok)
+                throw new Error(`The collector answered ${response.status}`);
+        }
+        catch (error) {
+            this.options.onError?.(error);
+        }
+    }
+}
+exports.OtlpTraceExporter = OtlpTraceExporter;
+function nanos(milliseconds) {
+    return `${BigInt(Math.round(milliseconds)) * 1000000n}`;
+}
+function typed(value) {
+    if (typeof value === 'boolean')
+        return { boolValue: value };
+    if (typeof value === 'number')
+        return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
+    return { stringValue: value };
+}

package/dist-cjs/tracing/rollups.d.ts

@@ -0,0 +1,100 @@
+import type { Run, RunKind, TraceStore } from '../types/tracing.js';
+/**
+ * Hourly totals of runs, so a dashboard reads a few rows instead of every trace.
+ *
+ * Traces are the forensic record: everything about one run. A chart of cost or errors over a week
+ * needs none of that, only sums, and reading 100,000 traces to draw it is how a dashboard gets slow
+ * as the service grows. A rollup keeps one row per hour, kind, model, provider, and tenant, updated
+ * as runs finish.
+ */
+/** What one rollup row is counted by. */
+export interface RollupKey {
+    /** The hour, as ISO-8601 truncated to the hour: `2026-10-04T13:00:00.000Z`. */
+    hour: string;
+    /** The run kind. */
+    kind: RunKind;
+    /** The run's name, for root runs, such as a graph or an assistant. Empty for others. */
+    name: string;
+    /** The model, or an empty string. */
+    model: string;
+    /** The provider, or an empty string. */
+    provider: string;
+    /** `metadata.tenantId`, or an empty string. */
+    tenant: string;
+}
+/** The totals of one rollup row. */
+export interface RollupTotals {
+    /** Runs counted. */
+    runs: number;
+    /** Of those, runs that failed. */
+    errors: number;
+    /** Their cost. */
+    cost: number;
+    /** Input tokens reported. */
+    inputTokens: number;
+    /** Output tokens reported. */
+    outputTokens: number;
+    /** Latency counts per bucket of `ROLLUP_LATENCY_BOUNDS_MS`, for percentiles without the runs. */
+    latency: number[];
+}
+/** One rollup row: its key and its totals. */
+export interface RollupRow extends RollupKey, RollupTotals {
+}
+/** Which rows to read. Every field narrows; the hour range is inclusive. */
+export interface RollupQuery {
+    /** The first hour, ISO-8601. */
+    since?: string;
+    /** The last hour, ISO-8601. */
+    until?: string;
+    /** Rows of these kinds. */
+    kind?: RunKind | RunKind[];
+    /** Rows of this model. */
+    model?: string;
+    /** Rows of this provider. */
+    provider?: string;
+    /** Rows of this tenant. */
+    tenant?: string;
+    /** Rows of this name. */
+    name?: string;
+}
+/** Where rollup rows live. Adding to a row that does not exist creates it. */
+export interface RollupStore {
+    /** Adds totals to the row for `key`. */
+    add(key: RollupKey, totals: RollupTotals): Promise<void> | void;
+    /** Rows matching a query, oldest hour first. */
+    query(query?: RollupQuery): Promise<RollupRow[]> | RollupRow[];
+}
+/**
+ * Upper bounds of the latency buckets, in milliseconds; a last bucket holds everything slower. A
+ * percentile read from them is exact to the bucket, which is close enough for a chart.
+ */
+export declare const ROLLUP_LATENCY_BOUNDS_MS: readonly [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, 300000, 600000];
+/** The latency at percentile `p`, 0 to 1, from rollup buckets: the bound of the bucket it falls in. */
+export declare function rollupPercentile(latency: readonly number[], p: number): number | undefined;
+/** Rows summed across their keys: a total over a day, a model, or a whole range. */
+export declare function sumRollups(rows: readonly RollupTotals[]): RollupTotals;
+/** Rollup rows in process memory. Rows are small, so a year of hourly rows for a few models fits easily. */
+export declare class MemoryRollupStore implements RollupStore {
+    private readonly rows;
+    /** Adds totals to a row. */
+    add(key: RollupKey, totals: RollupTotals): void;
+    /** Rows matching a query, oldest hour first. */
+    query(query?: RollupQuery): RollupRow[];
+    /** Rows held. */
+    size(): number;
+}
+/** Options for `rollupTraceStore()`. */
+export interface RollupTraceStoreOptions {
+    /** Which finished runs are counted. Defaults to every root run, and every model and embedding run. */
+    include?: (run: Run) => boolean;
+}
+/**
+ * Wraps a trace store so every run that finishes is also added to a rollup.
+ *
+ * Reads go to the wrapped store unchanged. A run is counted once, when it is saved finished; a save
+ * of a run still running, which an `incremental` tracer makes, is not counted, and neither is
+ * feedback added later.
+ */
+export declare function rollupTraceStore(store: TraceStore, rollups: RollupStore, options?: RollupTraceStoreOptions): TraceStore;
+/** The rollup row a finished run belongs to. */
+export declare function rollupKeyOf(run: Run): RollupKey;

package/dist-cjs/tracing/rollups.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryRollupStore = exports.ROLLUP_LATENCY_BOUNDS_MS = void 0;
+exports.rollupPercentile = rollupPercentile;
+exports.sumRollups = sumRollups;
+exports.rollupTraceStore = rollupTraceStore;
+exports.rollupKeyOf = rollupKeyOf;
+exports.ROLLUP_LATENCY_BOUNDS_MS = [
+    10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 120_000, 300_000, 600_000,
+];
+function rollupPercentile(latency, p) {
+    const total = latency.reduce((sum, count) => sum + count, 0);
+    if (total === 0)
+        return undefined;
+    const target = Math.max(1, Math.ceil(total * Math.min(1, Math.max(0, p))));
+    let seen = 0;
+    for (const [index, count] of latency.entries()) {
+        seen += count;
+        if (seen >= target)
+            return exports.ROLLUP_LATENCY_BOUNDS_MS[index] ?? Number.POSITIVE_INFINITY;
+    }
+    return Number.POSITIVE_INFINITY;
+}
+function sumRollups(rows) {
+    const total = emptyTotals();
+    for (const row of rows) {
+        total.runs += row.runs;
+        total.errors += row.errors;
+        total.cost += row.cost;
+        total.inputTokens += row.inputTokens;
+        total.outputTokens += row.outputTokens;
+        for (const [index, count] of row.latency.entries())
+            total.latency[index] = (total.latency[index] ?? 0) + count;
+    }
+    return total;
+}
+class MemoryRollupStore {
+    rows = new Map();
+    add(key, totals) {
+        const id = `${key.hour}|${key.kind}|${key.name}|${key.model}|${key.provider}|${key.tenant}`;
+        const row = this.rows.get(id);
+        if (!row) {
+            this.rows.set(id, { ...key, ...totals, latency: [...totals.latency] });
+            return;
+        }
+        row.runs += totals.runs;
+        row.errors += totals.errors;
+        row.cost += totals.cost;
+        row.inputTokens += totals.inputTokens;
+        row.outputTokens += totals.outputTokens;
+        for (const [index, count] of totals.latency.entries())
+            row.latency[index] = (row.latency[index] ?? 0) + count;
+    }
+    query(query = {}) {
+        return [...this.rows.values()]
+            .filter((row) => matches(row, query))
+            .sort((a, b) => a.hour.localeCompare(b.hour))
+            .map((row) => ({ ...row, latency: [...row.latency] }));
+    }
+    size() {
+        return this.rows.size;
+    }
+}
+exports.MemoryRollupStore = MemoryRollupStore;
+function rollupTraceStore(store, rollups, options = {}) {
+    const include = options.include ?? ((run) => !run.parentId || run.kind === 'model' || run.kind === 'embedding');
+    return {
+        async save(run) {
+            await store.save(run);
+            if (run.status === 'running' || !include(run))
+                return;
+            await rollups.add(rollupKeyOf(run), totalsOf(run));
+        },
+        get: (runId) => store.get(runId),
+        query: (query) => store.query(query),
+        tree: (traceId) => store.tree(traceId),
+        async addFeedback(runId, feedback) {
+            if (store.addFeedback) {
+                await store.addFeedback(runId, feedback);
+                return;
+            }
+            const run = await store.get(runId);
+            if (run)
+                await store.save({ ...run, feedback: [...(run.feedback ?? []), feedback] });
+        },
+        ...(store.prune
+            ? { prune: (before) => store.prune(before) }
+            : {}),
+    };
+}
+function rollupKeyOf(run) {
+    const started = new Date(run.startedAt);
+    started.setUTCMinutes(0, 0, 0);
+    const tenant = run.metadata?.tenantId;
+    return {
+        hour: started.toISOString(),
+        kind: run.kind,
+        name: run.parentId ? '' : run.name,
+        model: run.model ?? '',
+        provider: run.provider ?? '',
+        tenant: typeof tenant === 'string' ? tenant : '',
+    };
+}
+function totalsOf(run) {
+    const totals = emptyTotals();
+    totals.runs = 1;
+    totals.errors = run.status === 'error' ? 1 : 0;
+    totals.cost = run.cost ?? 0;
+    totals.inputTokens = run.usage?.inputTokens ?? run.usage?.promptTokens ?? 0;
+    totals.outputTokens = run.usage?.outputTokens ?? run.usage?.completionTokens ?? 0;
+    const latency = run.latencyMs ?? 0;
+    const bucket = exports.ROLLUP_LATENCY_BOUNDS_MS.findIndex((bound) => latency <= bound);
+    totals.latency[bucket === -1 ? exports.ROLLUP_LATENCY_BOUNDS_MS.length : bucket] = 1;
+    return totals;
+}
+function emptyTotals() {
+    return {
+        runs: 0,
+        errors: 0,
+        cost: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        latency: Array.from({ length: exports.ROLLUP_LATENCY_BOUNDS_MS.length + 1 }, () => 0),
+    };
+}
+function matches(row, query) {
+    if (query.since && row.hour < hourOf(query.since))
+        return false;
+    if (query.until && row.hour > query.until)
+        return false;
+    if (query.kind !== undefined && !(Array.isArray(query.kind) ? query.kind : [query.kind]).includes(row.kind))
+        return false;
+    if (query.model !== undefined && row.model !== query.model)
+        return false;
+    if (query.provider !== undefined && row.provider !== query.provider)
+        return false;
+    if (query.tenant !== undefined && row.tenant !== query.tenant)
+        return false;
+    if (query.name !== undefined && row.name !== query.name)
+        return false;
+    return true;
+}
+function hourOf(iso) {
+    const date = new Date(iso);
+    date.setUTCMinutes(0, 0, 0);
+    return date.toISOString();
+}

package/dist-cjs/tracing/stores.d.ts

@@ -34,6 +34,23 @@ export declare class MemoryTraceStore implements TraceStore {
     /** Runs held. */
     size(): number;
 }
+/** Options for `closeAbandonedRuns()`. */
+export interface CloseAbandonedRunsOptions {
+    /** A run still running after this long is taken to belong to a process that ended. */
+    olderThanMs: number;
+    /** Replaces the system clock, for tests. */
+    now?: () => Date;
+}
+/**
+ * Closes runs a process left running when it ended, so a crash shows as a failure rather than as
+ * work that is still going.
+ *
+ * Only a tracer with `incremental` writes a run before it finishes, so only its runs can be left
+ * behind. Each one older than `olderThanMs` is saved as an error named `RunAbandoned`, with
+ * `metadata.abandoned`, keeping everything it recorded. Call it when a worker starts, or on a
+ * schedule. Returns how many runs it closed.
+ */
+export declare function closeAbandonedRuns(store: TraceStore, options: CloseAbandonedRunsOptions): Promise<number>;
 /** Options for the JSONL trace store. */
 export interface JsonlTraceStoreOptions {
     /** File runs are appended to, one JSON object per line. */