nexus-ai-pro 2.0.0 → 2.2.0

package/dist/testing/record.d.ts

@@ -32,6 +32,40 @@ export interface RecordedResponse {
     body: string;
     /** How `body` is encoded. */
     bodyEncoding: 'utf8' | 'base64';
+    /**
+     * The body as it arrived, chunk by chunk, with when each came, when recorded with `timing`. A
+     * paced replay sends these; an instant one sends `body`, which holds the same bytes.
+     */
+    chunks?: RecordedChunk[];
+}
+/** One piece of a streamed response, and when it arrived. */
+export interface RecordedChunk {
+    /** Milliseconds after the response started. */
+    atMs: number;
+    /** The bytes, as text or base64. */
+    body: string;
+    /** How `body` is encoded. */
+    bodyEncoding: 'utf8' | 'base64';
+}
+/**
+ * How a replayed stream is paced, for a recording made with `timing`.
+ *
+ * - `instant` (the default) sends the whole body at once, as a recording without timing does.
+ * - `original` sends each chunk when it arrived, relative to the first.
+ * - A number scales the gaps: `0.1` is ten times faster, `2` twice as slow.
+ * - A function is awaited before each chunk, so a test can release them one at a time.
+ */
+export type ReplayPace = 'instant' | 'original' | number | ((chunk: ReplayChunkInfo) => Promise<void> | void);
+/** What a pacing function is told about the chunk it releases. */
+export interface ReplayChunkInfo {
+    /** Its position, from 0. */
+    index: number;
+    /** How many chunks the response has. */
+    count: number;
+    /** When it arrived when recorded, in milliseconds after the response started. */
+    atMs: number;
+    /** The recorded gap since the chunk before it, in milliseconds. */
+    delayMs: number;
 }
 /** One request and the response it got, as written to a fixture file. */
 export interface RecordedExchange {
@@ -86,6 +120,12 @@ export interface RecordOptions extends FixtureOptions {
     redact?: (exchange: RecordedExchange) => RecordedExchange;
     /** Replaces the system clock, for `recordedAt`. */
     now?: () => Date;
+    /**
+     * Keeps each chunk of a response and when it arrived, so a replay can reproduce a stream's pace
+     * as well as its bytes: for testing idle timeouts, cancellation, backpressure, and streaming UIs.
+     * Defaults to false.
+     */
+    timing?: boolean;
 }
 /** Options for replaying. */
 export interface ReplayOptions extends FixtureOptions {
@@ -96,6 +136,8 @@ export interface ReplayOptions extends FixtureOptions {
     onMissing?: 'throw' | 'live';
     /** The real fetch, for `onMissing: 'live'`. */
     fetch?: typeof globalThis.fetch;
+    /** How a stream recorded with `timing` is paced. Defaults to `instant`. */
+    pace?: ReplayPace;
 }
 /** A fetch that also lets a test wait for every fixture it has written. */
 export type RecordingFetch = typeof globalThis.fetch & {

package/dist/testing/record.js

@@ -24,7 +24,10 @@ export function recordingFetch(options) {
         const captured = await captureRequest(copyOf(input, init), options);
         const real = options.fetch ?? globalThis.fetch;
         const response = await real(input, init);
-        const bytes = new Uint8Array(await response.arrayBuffer());
+        const { bytes, chunks } = options.timing
+            ? await readTimed(response)
+            : { bytes: new Uint8Array(await response.arrayBuffer()), chunks: undefined };
+        const contentType = response.headers.get('content-type');
         const index = counters.get(captured.key) ?? 0;
         counters.set(captured.key, index + 1);
         let exchange = {
@@ -35,7 +38,10 @@ export function recordingFetch(options) {
                 status: response.status,
                 ...(response.statusText ? { statusText: response.statusText } : {}),
                 headers: withoutTransportHeaders(redactHeaders(response.headers, options.redactHeaders)),
-                ...encodeBody(bytes, response.headers.get('content-type')),
+                ...encodeBody(bytes, contentType),
+                ...(chunks
+                    ? { chunks: chunks.map((chunk) => ({ atMs: chunk.atMs, ...encodeBody(chunk.bytes, contentType) })) }
+                    : {}),
             },
             recordedAt: now().toISOString(),
         };
@@ -71,9 +77,13 @@ export function replayFetch(options) {
         const position = served.get(captured.key) ?? 0;
         served.set(captured.key, position + 1);
         const exchange = recordings[Math.min(position, recordings.length - 1)];
-        const body = exchange.response.bodyEncoding === 'base64'
-            ? Buffer.from(exchange.response.body, 'base64')
-            : exchange.response.body;
+        const pace = options.pace ?? 'instant';
+        const chunks = exchange.response.chunks;
+        const body = pace !== 'instant' && chunks?.length
+            ? pacedBody(chunks, pace, request.signal)
+            : exchange.response.bodyEncoding === 'base64'
+                ? Buffer.from(exchange.response.body, 'base64')
+                : exchange.response.body;
         return new Response(isNullBodyStatus(exchange.response.status) ? null : body, {
             status: exchange.response.status,
             statusText: exchange.response.statusText ?? '',
@@ -181,6 +191,48 @@ function encodeBody(bytes, contentType) {
     }
     return { body: Buffer.from(bytes).toString('base64'), bodyEncoding: 'base64' };
 }
+async function readTimed(response) {
+    const chunks = [];
+    if (!response.body)
+        return { bytes: new Uint8Array(), chunks };
+    const started = performance.now();
+    const reader = response.body.getReader();
+    for (;;) {
+        const { value, done } = await reader.read();
+        if (done)
+            break;
+        chunks.push({ atMs: Math.round(performance.now() - started), bytes: value });
+    }
+    const bytes = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.bytes.length, 0));
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk.bytes, offset);
+        offset += chunk.bytes.length;
+    }
+    return { bytes, chunks };
+}
+function pacedBody(chunks, pace, signal) {
+    let index = 0;
+    return new ReadableStream({
+        async pull(controller) {
+            const chunk = chunks[index];
+            if (!chunk || signal.aborted) {
+                controller.close();
+                return;
+            }
+            const delayMs = Math.max(0, chunk.atMs - (index === 0 ? chunk.atMs : chunks[index - 1].atMs));
+            if (typeof pace === 'function')
+                await pace({ index, count: chunks.length, atMs: chunk.atMs, delayMs });
+            else {
+                const wait = pace === 'original' ? delayMs : delayMs * pace;
+                if (wait > 0)
+                    await new Promise((resolve) => setTimeout(resolve, wait));
+            }
+            index += 1;
+            controller.enqueue(chunk.bodyEncoding === 'base64' ? Buffer.from(chunk.body, 'base64') : new TextEncoder().encode(chunk.body));
+        },
+    });
+}
 function isNullBodyStatus(status) {
     return status === 101 || status === 204 || status === 205 || status === 304;
 }

package/dist/tracing/index.d.ts

@@ -1,6 +1,6 @@
 export { type AlertEvent, AlertEvaluator, type AlertMetric, type AlertNotifier, type AlertRule, createWebhookNotifier, measure, type WebhookNotifierOptions, } from './alerts.js';
 export { compareTraces, formatTree, type RunDifference, type TraceComparison } from './compare.js';
 export { type GraphTracing, type GraphTracingOptions, type ModelClientLike, traceGraph, traceModelClient, } from './instrument.js';
-export { applyQuery, assembleTree, JsonlTraceStore, type JsonlTraceStoreOptions, MemoryTraceStore, type MemoryTraceStoreOptions, } from './stores.js';
-export { type FinishRunOptions, type RunHandle, type StartRunOptions, stripFields, Tracer, type TracerOptions, } from './tracer.js';
-export type { RedactionPolicy, Run, RunFeedback, RunKind, RunQuery, RunStatus, RunTree, SamplingPolicy, TraceStore, } from '../types/tracing.js';
+export { applyQuery, assembleTree, closeAbandonedRuns, type CloseAbandonedRunsOptions, JsonlTraceStore, type JsonlTraceStoreOptions, MemoryTraceStore, type MemoryTraceStoreOptions, } from './stores.js';
+export { type FinishRunOptions, type RunHandle, type StartRunOptions, stripFields, Tracer, type TracerOptions, w3cSpanId, w3cTraceId, } from './tracer.js';
+export type { RedactionPolicy, Run, RunFeedback, RunKind, RunQuery, RunStatus, RunTree, SamplingPolicy, TraceExporter, TraceStore, } from '../types/tracing.js';

package/dist/tracing/index.js

@@ -1,5 +1,5 @@
 export { AlertEvaluator, createWebhookNotifier, measure, } from './alerts.js';
 export { compareTraces, formatTree } from './compare.js';
 export { traceGraph, traceModelClient, } from './instrument.js';
-export { applyQuery, assembleTree, JsonlTraceStore, MemoryTraceStore, } from './stores.js';
-export { stripFields, Tracer, } from './tracer.js';
+export { applyQuery, assembleTree, closeAbandonedRuns, JsonlTraceStore, MemoryTraceStore, } from './stores.js';
+export { stripFields, Tracer, w3cSpanId, w3cTraceId, } from './tracer.js';

package/dist/tracing/instrument.js

@@ -9,6 +9,7 @@ export function traceGraph(tracer, options = {}) {
     const active = new Map();
     const retries = new Map();
     const emitted = new Map();
+    const tools = new Map();
     const onEvent = (event) => {
         switch (event.type) {
             case 'task_start': {
@@ -25,6 +26,29 @@ export function traceGraph(tracer, options = {}) {
                 retries.set(event.taskId, event.attempt);
                 return;
             }
+            case 'tool': {
+                const key = `${event.taskId}:${event.tool.id}`;
+                if (event.tool.phase === 'start') {
+                    const parent = active.get(event.taskId) ?? root;
+                    tools.set(key, parent.child({
+                        name: event.tool.name,
+                        kind: 'tool',
+                        inputs: event.tool.args,
+                        metadata: { callId: event.tool.id },
+                    }));
+                    return;
+                }
+                if (event.tool.phase === 'progress')
+                    return;
+                const handle = tools.get(key);
+                if (!handle)
+                    return;
+                tools.delete(key);
+                void handle.finish(event.tool.phase === 'error'
+                    ? { error: new Error(event.tool.error ?? 'The tool failed') }
+                    : { outputs: event.tool.result });
+                return;
+            }
             case 'custom': {
                 emitted.set(event.taskId, [...(emitted.get(event.taskId) ?? []), event.data]);
                 return;
@@ -53,6 +77,10 @@ export function traceGraph(tracer, options = {}) {
         root,
         runFor: (node) => [...active.values()].find((handle) => handle.id && nameOf(active, handle) === node),
         async finish(result) {
+            for (const [key, handle] of tools) {
+                tools.delete(key);
+                await handle.finish({ metadata: { unfinished: true } });
+            }
             for (const [taskId, handle] of active) {
                 active.delete(taskId);
                 await handle.finish({ metadata: { unfinished: true } });
@@ -112,6 +140,8 @@ export function traceModelClient(client, tracer, options = {}) {
                         model: response.meta?.modelUsed,
                         finishReason: response.finishReason,
                         ...(request.metadata?.prompt === undefined ? {} : { prompt: request.metadata.prompt }),
+                        ...(response.meta?.cache ? { cache: response.meta.cache } : {}),
+                        ...(request.tenantId ? { tenantId: request.tenantId } : {}),
                     },
                 });
                 return response;

package/dist/tracing/otlp.d.ts

@@ -0,0 +1,91 @@
+import type { Run, TraceExporter } from '../types/tracing.js';
+/** Options for `OtlpTraceExporter`. */
+export interface OtlpTraceExporterOptions {
+    /**
+     * The collector's OTLP/HTTP base URL, such as `http://localhost:4318`. Spans are posted to
+     * `/v1/traces` under it, as JSON.
+     */
+    endpoint: string;
+    /** Headers on every request, such as an API key a hosted backend wants. */
+    headers?: Record<string, string>;
+    /** `service.name` on every span. Defaults to `nexus-ai-pro`. */
+    serviceName?: string;
+    /** More resource attributes, such as `deployment.environment.name` or `service.version`. */
+    resource?: Record<string, string | number | boolean>;
+    /** Spans sent per request. Defaults to 100. */
+    maxBatch?: number;
+    /** How long a span waits for others to batch with, in milliseconds. Defaults to 2,000. */
+    flushIntervalMs?: number;
+    /** Replaces the global `fetch`, for tests or a proxy. */
+    fetch?: typeof globalThis.fetch;
+    /** Called when a batch cannot be sent. The batch is dropped; a trace is never worth a retry storm. */
+    onError?: (error: unknown) => void;
+}
+type AttributeValue = {
+    stringValue: string;
+} | {
+    intValue: string;
+} | {
+    doubleValue: number;
+} | {
+    boolValue: boolean;
+};
+/** One span in OTLP's JSON encoding. */
+export interface OtlpSpan {
+    /** 32 hex digits. */
+    traceId: string;
+    /** 16 hex digits. */
+    spanId: string;
+    /** 16 hex digits, absent for a root span. */
+    parentSpanId?: string;
+    /** The span's name. */
+    name: string;
+    /** 1 internal, 3 client. */
+    kind: number;
+    /** Nanoseconds since the epoch, as a decimal string. */
+    startTimeUnixNano: string;
+    /** Nanoseconds since the epoch, as a decimal string. */
+    endTimeUnixNano: string;
+    /** Typed attributes. */
+    attributes: Array<{
+        key: string;
+        value: AttributeValue;
+    }>;
+    /** 1 ok, 2 error. */
+    status: {
+        code: number;
+        message?: string;
+    };
+}
+/**
+ * Turns one finished run into an OpenTelemetry span, with the GenAI semantic conventions.
+ *
+ * A model run is a client span named `chat <model>` with `gen_ai.operation.name`,
+ * `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`, and
+ * `gen_ai.usage.input_tokens` and `output_tokens`. A tool run carries `gen_ai.tool.name`, an agent
+ * run `gen_ai.agent.name`. What only Nexus knows goes under `nexus.*`: the run kind, a graph node and
+ * its step, the cost, and the run id. A failed run has `error.type` and an error status.
+ */
+export declare function runToOtlpSpan(run: Run): OtlpSpan;
+/**
+ * Sends finished runs to any OpenTelemetry collector as OTLP/HTTP JSON, with no SDK to install.
+ *
+ * Pass it to a tracer's `exporters`, and every run the tracer keeps arrives as a span with the GenAI
+ * semantic conventions, so Grafana Tempo, Jaeger, Honeycomb, Datadog, New Relic, or any OTLP backend
+ * reads it as it reads the rest of the application. Spans are batched; `flush()` sends what is
+ * waiting. Trace ids are the W3C ids `RunHandle.traceparent()` propagates, so a trace that crossed
+ * processes arrives as one.
+ */
+export declare class OtlpTraceExporter implements TraceExporter {
+    private readonly options;
+    private readonly queue;
+    private timer;
+    private sending;
+    constructor(options: OtlpTraceExporterOptions);
+    /** Queues a run, sending a batch when one is full. Runs still in progress are skipped. */
+    export(run: Run): void;
+    /** Sends every span waiting. */
+    flush(): Promise<void>;
+    private send;
+}
+export {};

package/dist/tracing/otlp.js

@@ -0,0 +1,124 @@
+import { w3cSpanId, w3cTraceId } from './tracer.js';
+const OPERATIONS = {
+    model: 'chat',
+    embedding: 'embeddings',
+    tool: 'execute_tool',
+    agent: 'invoke_agent',
+};
+export function runToOtlpSpan(run) {
+    const attributes = { 'nexus.run.kind': run.kind, 'nexus.run.id': run.id };
+    const operation = OPERATIONS[run.kind];
+    if (operation)
+        attributes['gen_ai.operation.name'] = operation;
+    if (run.provider)
+        attributes['gen_ai.provider.name'] = run.provider;
+    if (run.kind === 'model' || run.kind === 'embedding') {
+        const requested = run.metadata?.requestedModel;
+        if (typeof requested === 'string')
+            attributes['gen_ai.request.model'] = requested;
+        else if (run.model)
+            attributes['gen_ai.request.model'] = run.model;
+        if (run.model)
+            attributes['gen_ai.response.model'] = run.model;
+        const input = run.usage?.inputTokens ?? run.usage?.promptTokens;
+        const output = run.usage?.outputTokens ?? run.usage?.completionTokens;
+        if (input !== undefined)
+            attributes['gen_ai.usage.input_tokens'] = input;
+        if (output !== undefined)
+            attributes['gen_ai.usage.output_tokens'] = output;
+    }
+    if (run.kind === 'tool')
+        attributes['gen_ai.tool.name'] = run.name;
+    if (run.kind === 'agent')
+        attributes['gen_ai.agent.name'] = run.name;
+    if (run.kind === 'node') {
+        attributes['nexus.graph.node'] = run.name;
+        if (typeof run.metadata?.step === 'number')
+            attributes['nexus.graph.step'] = run.metadata.step;
+    }
+    if (run.cost !== undefined)
+        attributes['nexus.cost.usd'] = run.cost;
+    if (run.error)
+        attributes['error.type'] = run.error.name;
+    for (const tag of run.tags ?? [])
+        attributes[`nexus.tag.${tag}`] = true;
+    const started = Date.parse(run.startedAt);
+    const ended = run.endedAt ? Date.parse(run.endedAt) : started + (run.latencyMs ?? 0);
+    const name = run.kind === 'model' && run.model ? `chat ${run.model}` : operation ? `${operation} ${run.name}` : run.name;
+    return {
+        traceId: w3cTraceId(run.traceId),
+        spanId: w3cSpanId(run.id),
+        ...(run.parentId ? { parentSpanId: w3cSpanId(run.parentId) } : {}),
+        name,
+        kind: run.kind === 'model' || run.kind === 'embedding' ? 3 : 1,
+        startTimeUnixNano: nanos(started),
+        endTimeUnixNano: nanos(ended),
+        attributes: Object.entries(attributes).map(([key, value]) => ({ key, value: typed(value) })),
+        status: run.status === 'error' ? { code: 2, ...(run.error ? { message: run.error.message } : {}) } : { code: 1 },
+    };
+}
+export class OtlpTraceExporter {
+    options;
+    queue = [];
+    timer;
+    sending = Promise.resolve();
+    constructor(options) {
+        this.options = options;
+    }
+    export(run) {
+        if (run.status === 'running')
+            return;
+        this.queue.push(runToOtlpSpan(run));
+        if (this.queue.length >= (this.options.maxBatch ?? 100)) {
+            void this.flush();
+            return;
+        }
+        if (!this.timer) {
+            this.timer = setTimeout(() => void this.flush(), this.options.flushIntervalMs ?? 2_000);
+            this.timer.unref?.();
+        }
+    }
+    async flush() {
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = undefined;
+        while (this.queue.length > 0) {
+            const batch = this.queue.splice(0, this.options.maxBatch ?? 100);
+            this.sending = this.sending.then(() => this.send(batch));
+        }
+        await this.sending;
+    }
+    async send(spans) {
+        const resource = { 'service.name': this.options.serviceName ?? 'nexus-ai-pro', ...this.options.resource };
+        const body = {
+            resourceSpans: [
+                {
+                    resource: { attributes: Object.entries(resource).map(([key, value]) => ({ key, value: typed(value) })) },
+                    scopeSpans: [{ scope: { name: 'nexus-ai-pro' }, spans }],
+                },
+            ],
+        };
+        try {
+            const response = await (this.options.fetch ?? globalThis.fetch)(`${this.options.endpoint.replace(/\/+$/, '')}/v1/traces`, {
+                method: 'POST',
+                headers: { 'content-type': 'application/json', ...this.options.headers },
+                body: JSON.stringify(body),
+            });
+            if (!response.ok)
+                throw new Error(`The collector answered ${response.status}`);
+        }
+        catch (error) {
+            this.options.onError?.(error);
+        }
+    }
+}
+function nanos(milliseconds) {
+    return `${BigInt(Math.round(milliseconds)) * 1000000n}`;
+}
+function typed(value) {
+    if (typeof value === 'boolean')
+        return { boolValue: value };
+    if (typeof value === 'number')
+        return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
+    return { stringValue: value };
+}

package/dist/tracing/rollups.d.ts

@@ -0,0 +1,100 @@
+import type { Run, RunKind, TraceStore } from '../types/tracing.js';
+/**
+ * Hourly totals of runs, so a dashboard reads a few rows instead of every trace.
+ *
+ * Traces are the forensic record: everything about one run. A chart of cost or errors over a week
+ * needs none of that, only sums, and reading 100,000 traces to draw it is how a dashboard gets slow
+ * as the service grows. A rollup keeps one row per hour, kind, model, provider, and tenant, updated
+ * as runs finish.
+ */
+/** What one rollup row is counted by. */
+export interface RollupKey {
+    /** The hour, as ISO-8601 truncated to the hour: `2026-10-04T13:00:00.000Z`. */
+    hour: string;
+    /** The run kind. */
+    kind: RunKind;
+    /** The run's name, for root runs, such as a graph or an assistant. Empty for others. */
+    name: string;
+    /** The model, or an empty string. */
+    model: string;
+    /** The provider, or an empty string. */
+    provider: string;
+    /** `metadata.tenantId`, or an empty string. */
+    tenant: string;
+}
+/** The totals of one rollup row. */
+export interface RollupTotals {
+    /** Runs counted. */
+    runs: number;
+    /** Of those, runs that failed. */
+    errors: number;
+    /** Their cost. */
+    cost: number;
+    /** Input tokens reported. */
+    inputTokens: number;
+    /** Output tokens reported. */
+    outputTokens: number;
+    /** Latency counts per bucket of `ROLLUP_LATENCY_BOUNDS_MS`, for percentiles without the runs. */
+    latency: number[];
+}
+/** One rollup row: its key and its totals. */
+export interface RollupRow extends RollupKey, RollupTotals {
+}
+/** Which rows to read. Every field narrows; the hour range is inclusive. */
+export interface RollupQuery {
+    /** The first hour, ISO-8601. */
+    since?: string;
+    /** The last hour, ISO-8601. */
+    until?: string;
+    /** Rows of these kinds. */
+    kind?: RunKind | RunKind[];
+    /** Rows of this model. */
+    model?: string;
+    /** Rows of this provider. */
+    provider?: string;
+    /** Rows of this tenant. */
+    tenant?: string;
+    /** Rows of this name. */
+    name?: string;
+}
+/** Where rollup rows live. Adding to a row that does not exist creates it. */
+export interface RollupStore {
+    /** Adds totals to the row for `key`. */
+    add(key: RollupKey, totals: RollupTotals): Promise<void> | void;
+    /** Rows matching a query, oldest hour first. */
+    query(query?: RollupQuery): Promise<RollupRow[]> | RollupRow[];
+}
+/**
+ * Upper bounds of the latency buckets, in milliseconds; a last bucket holds everything slower. A
+ * percentile read from them is exact to the bucket, which is close enough for a chart.
+ */
+export declare const ROLLUP_LATENCY_BOUNDS_MS: readonly [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, 300000, 600000];
+/** The latency at percentile `p`, 0 to 1, from rollup buckets: the bound of the bucket it falls in. */
+export declare function rollupPercentile(latency: readonly number[], p: number): number | undefined;
+/** Rows summed across their keys: a total over a day, a model, or a whole range. */
+export declare function sumRollups(rows: readonly RollupTotals[]): RollupTotals;
+/** Rollup rows in process memory. Rows are small, so a year of hourly rows for a few models fits easily. */
+export declare class MemoryRollupStore implements RollupStore {
+    private readonly rows;
+    /** Adds totals to a row. */
+    add(key: RollupKey, totals: RollupTotals): void;
+    /** Rows matching a query, oldest hour first. */
+    query(query?: RollupQuery): RollupRow[];
+    /** Rows held. */
+    size(): number;
+}
+/** Options for `rollupTraceStore()`. */
+export interface RollupTraceStoreOptions {
+    /** Which finished runs are counted. Defaults to every root run, and every model and embedding run. */
+    include?: (run: Run) => boolean;
+}
+/**
+ * Wraps a trace store so every run that finishes is also added to a rollup.
+ *
+ * Reads go to the wrapped store unchanged. A run is counted once, when it is saved finished; a save
+ * of a run still running, which an `incremental` tracer makes, is not counted, and neither is
+ * feedback added later.
+ */
+export declare function rollupTraceStore(store: TraceStore, rollups: RollupStore, options?: RollupTraceStoreOptions): TraceStore;
+/** The rollup row a finished run belongs to. */
+export declare function rollupKeyOf(run: Run): RollupKey;