@lanefoundry/gatelane-engine 0.0.0-canary-b3f6165

package/dist/otel-export.js

@@ -0,0 +1,162 @@
+/**
+ * OTel export — convert Turn[] traces to OpenTelemetry spans and send
+ * to any OTLP-compatible backend (Langfuse, Jaeger, Datadog, Grafana Tempo, etc.).
+ *
+ * Uses GenAI semantic conventions for LLM-specific attributes.
+ *
+ * @example
+ * ```ts
+ * import { exportTurnsToOTel } from '@lanefoundry/gatelane-engine';
+ *
+ * await exportTurnsToOTel(turns, {
+ *   endpoint: 'https://cloud.langfuse.com/api/public/otel',
+ *   headers: { Authorization: 'Basic <base64(publicKey:secretKey)>' },
+ * });
+ * ```
+ */
+import { SpanKind, SpanStatusCode } from '@opentelemetry/api';
+import { NodeTracerProvider, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+const SPAN_KIND_MAP = {
+    llm: SpanKind.CLIENT,
+    tool: SpanKind.INTERNAL,
+    agent: SpanKind.INTERNAL,
+    retriever: SpanKind.CLIENT,
+    guardrail: SpanKind.INTERNAL,
+};
+/**
+ * Export Turn[] traces to an OTLP-compatible backend.
+ *
+ * Creates a temporary TracerProvider with an OTLP/HTTP exporter, converts
+ * each Turn with a span_id into an OTel span (using GenAI semantic
+ * conventions), exports them, and shuts down.
+ */
+export async function exportTurnsToOTel(turns, config) {
+    const spannedTurns = turns.filter((t) => t.span_id);
+    if (spannedTurns.length === 0) {
+        return { exported: 0 };
+    }
+    const exporter = new OTLPTraceExporter({
+        url: config.endpoint.replace(/\/$/, '') + '/v1/traces',
+        headers: config.headers,
+    });
+    const provider = new NodeTracerProvider({
+        resource: resourceFromAttributes({
+            'service.name': config.service_name ?? 'gatelane',
+        }),
+        spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    const tracer = provider.getTracer('gatelane-otel-export');
+    // Build a map of span_id → Turn for parent lookup
+    const spanMap = new Map();
+    for (const turn of spannedTurns) {
+        spanMap.set(turn.span_id, turn);
+    }
+    // Process root spans first, then children, to maintain parent-child relationships.
+    // OTel SDK handles span linking via context, so we process in order and
+    // create spans with explicit start/end times.
+    const roots = spannedTurns.filter((t) => !t.parent_span_id);
+    const children = spannedTurns.filter((t) => t.parent_span_id);
+    let exported = 0;
+    for (const turn of [...roots, ...children]) {
+        const spanKind = SPAN_KIND_MAP[turn.span_kind ?? 'agent'] ?? SpanKind.INTERNAL;
+        const spanName = buildSpanName(turn);
+        const startTime = turn.started_at ? new Date(turn.started_at) : new Date();
+        const span = tracer.startSpan(spanName, {
+            kind: spanKind,
+            startTime,
+        });
+        // GenAI semantic conventions
+        if (turn.model) {
+            span.setAttribute('gen_ai.request.model', turn.model);
+        }
+        if (turn.tokens_in !== undefined) {
+            span.setAttribute('gen_ai.usage.input_tokens', turn.tokens_in);
+        }
+        if (turn.tokens_out !== undefined) {
+            span.setAttribute('gen_ai.usage.output_tokens', turn.tokens_out);
+        }
+        if (turn.cost_usd !== undefined) {
+            span.setAttribute('gen_ai.usage.cost', turn.cost_usd);
+        }
+        // gatelane-specific attributes
+        span.setAttribute('gatelane.role', turn.role);
+        if (turn.span_kind) {
+            span.setAttribute('gatelane.span_kind', turn.span_kind);
+        }
+        if (turn.span_id) {
+            span.setAttribute('gatelane.span_id', turn.span_id);
+        }
+        if (turn.parent_span_id) {
+            span.setAttribute('gatelane.parent_span_id', turn.parent_span_id);
+        }
+        if (turn.latency_ms !== undefined) {
+            span.setAttribute('gatelane.latency_ms', turn.latency_ms);
+        }
+        // Content (opt-in via GenAI conventions — content is a span event)
+        if (turn.content) {
+            span.addEvent(turn.role === 'assistant' ? 'gen_ai.completion' : 'gen_ai.prompt', {
+                'gen_ai.content': turn.content.slice(0, 4096),
+            });
+        }
+        // Tool calls
+        if (turn.tool_calls) {
+            for (const tc of turn.tool_calls) {
+                span.addEvent('gen_ai.tool_call', {
+                    'gen_ai.tool.name': tc.name,
+                    'gen_ai.tool.call_id': tc.id,
+                    'gen_ai.tool.arguments': tc.arguments.slice(0, 4096),
+                });
+            }
+        }
+        // Tool result (for role=tool turns)
+        if (turn.role === 'tool' && turn.name) {
+            span.setAttribute('gen_ai.tool.name', turn.name);
+            if (turn.tool_call_id) {
+                span.setAttribute('gen_ai.tool.call_id', turn.tool_call_id);
+            }
+        }
+        // Error status
+        if (turn.status === 'error' || turn.status === 'timeout') {
+            span.setStatus({
+                code: SpanStatusCode.ERROR,
+                message: turn.error?.message ?? turn.status,
+            });
+            if (turn.error) {
+                span.recordException({
+                    name: turn.error.type,
+                    message: turn.error.message,
+                    stack: turn.error.stack,
+                });
+            }
+        }
+        else {
+            span.setStatus({ code: SpanStatusCode.OK });
+        }
+        const endTime = turn.completed_at
+            ? new Date(turn.completed_at)
+            : turn.latency_ms !== undefined
+                ? new Date(startTime.getTime() + turn.latency_ms)
+                : new Date();
+        span.end(endTime);
+        exported++;
+    }
+    // Flush and shutdown
+    await provider.forceFlush();
+    await provider.shutdown();
+    return { exported };
+}
+function buildSpanName(turn) {
+    if (turn.span_kind === 'llm' && turn.model) {
+        return `llm.${turn.model}`;
+    }
+    if (turn.span_kind === 'tool' && turn.name) {
+        return `tool.${turn.name}`;
+    }
+    if (turn.span_kind) {
+        return turn.span_kind;
+    }
+    return `${turn.role}`;
+}
+//# sourceMappingURL=otel-export.js.map

package/dist/otel-export.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"otel-export.js","sourceRoot":"","sources":["../src/otel-export.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACxF,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAC;AAC5E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAYlE,MAAM,aAAa,GAA6B;IAC9C,GAAG,EAAE,QAAQ,CAAC,MAAM;IACpB,IAAI,EAAE,QAAQ,CAAC,QAAQ;IACvB,KAAK,EAAE,QAAQ,CAAC,QAAQ;IACxB,SAAS,EAAE,QAAQ,CAAC,MAAM;IAC1B,SAAS,EAAE,QAAQ,CAAC,QAAQ;CAC7B,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAA0B,EAC1B,MAAwB;IAExB,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,YAAY;QACtD,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QACtC,QAAQ,EAAE,sBAAsB,CAAC;YAC/B,cAAc,EAAE,MAAM,CAAC,YAAY,IAAI,UAAU;SAClD,CAAC;QACF,cAAc,EAAE,CAAC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAE1D,kDAAkD;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgB,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAQ,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,mFAAmF;IACnF,wEAAwE;IACxE,8CAA8C;IAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAE9D,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;QAC/E,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAE3E,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE;YACtC,IAAI,EAAE,QAAQ;YACd,SAAS;SACV,CAAC,CAAC;QAEH,6BAA6B;QAC7B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,YAAY,CAAC,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,CAAC,2BAA2B,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,YAAY,CAAC,4BAA4B,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,yBAAyB,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,YAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,CAAC;QAED,mEAAmE;QACnE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,eAAe,EAAE;gBAC/E,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;aAC9C,CAAC,CAAC;QACL,CAAC;QAED,aAAa;QACb,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE;oBAChC,kBAAkB,EAAE,EAAE,CAAC,IAAI;oBAC3B,qBAAqB,EAAE,EAAE,CAAC,EAAE;oBAC5B,uBAAuB,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;iBACrD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,eAAe;QACf,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACzD,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,MAAM;aAC5C,CAAC,CAAC;YACH,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,eAAe,CAAC;oBACnB,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;oBACrB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;oBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY;YAC/B,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS;gBAC7B,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;gBACjD,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,QAAQ,EAAE,CAAC;IACb,CAAC;IAED,qBAAqB;IACrB,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;IAC5B,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAE1B,OAAO,EAAE,QAAQ,EAAE,CAAC;AACtB,CAAC;AAED,SAAS,aAAa,CAAC,IAAU;IAC/B,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,OAAO,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IACD,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC3C,OAAO,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC"}

package/dist/replay.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Replay engine — runs each dataset item against each candidate via LLMCaller.
+ *
+ * Two modes:
+ * - Single-turn: one prompt → one response (legacy, item.input only)
+ * - Multi-turn: replay full agent trace with mocked tool results (item.turns)
+ *
+ * Multi-turn replay feeds user/system turns to the candidate, lets the model
+ * decide tool calls, injects the ORIGINAL tool results from the captured trace,
+ * and records the complete new trace. This isolates model behavior changes
+ * from tool behavior changes.
+ *
+ * Determinism: a stable seed per (candidate, item) produces reproducible output.
+ *
+ * @see docs/prd.md §5.1 — The gate
+ */
+import type { CapturedCall, Turn } from '@lanefoundry/gatelane-sdk/capture';
+import type { DatasetItem, FrozenDataset } from '@lanefoundry/gatelane-sdk/dataset';
+import type { LLMCaller, ChatMessage, LLMResponse } from './llm.js';
+export type ReplayRow = {
+    item_id: string;
+    candidate_ref: string;
+    response: LLMResponse;
+    captured: CapturedCall;
+    /** Full multi-turn trace when replaying an agent trace. */
+    turns?: Turn[];
+};
+export type ReplayResult = {
+    rows: ReplayRow[];
+    /** 0.0–1.0 across all (candidate, item) pairs. Higher = more reproducible. */
+    determinism_score: number;
+    total_cost_usd: number;
+    total_latency_ms: number;
+};
+export type ReplayArgs = {
+    dataset: FrozenDataset;
+    candidates: ReadonlyArray<{
+        ref: string;
+        model: string;
+        messages_from_item?: (item: DatasetItem) => Array<ChatMessage>;
+    }>;
+    caller: LLMCaller;
+    /** Master seed; per (candidate, item) seed is derived deterministically. */
+    seed?: number;
+    /** If true, replay twice to measure determinism. Doubles cost. */
+    measureDeterminism?: boolean;
+    /** Max LLM round-trips per multi-turn replay (prevents infinite loops). */
+    maxTurns?: number;
+};
+export declare function replay(args: ReplayArgs): Promise<ReplayResult>;
+//# sourceMappingURL=replay.d.ts.map

package/dist/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAEpF,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEpE,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,YAAY,CAAC;IACvB,2DAA2D;IAC3D,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,8EAA8E;IAC9E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,aAAa,CAAC;QACxB,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC;KAChE,CAAC,CAAC;IACH,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AA0KF,wBAAsB,MAAM,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,CAsFpE"}

package/dist/replay.js

@@ -0,0 +1,260 @@
+/**
+ * Replay engine — runs each dataset item against each candidate via LLMCaller.
+ *
+ * Two modes:
+ * - Single-turn: one prompt → one response (legacy, item.input only)
+ * - Multi-turn: replay full agent trace with mocked tool results (item.turns)
+ *
+ * Multi-turn replay feeds user/system turns to the candidate, lets the model
+ * decide tool calls, injects the ORIGINAL tool results from the captured trace,
+ * and records the complete new trace. This isolates model behavior changes
+ * from tool behavior changes.
+ *
+ * Determinism: a stable seed per (candidate, item) produces reproducible output.
+ *
+ * @see docs/prd.md §5.1 — The gate
+ */
+function deriveSeed(master, candidateRef, itemId) {
+    let h = 0x811c9dc5 ^ master;
+    for (const ch of `${candidateRef}|${itemId}`) {
+        h ^= ch.charCodeAt(0);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+function itemId(item, idx) {
+    return item.id ?? `<item-${idx}>`;
+}
+/** Build a tool-result lookup from original turns: tool_call_id → tool content. */
+function buildToolResultMap(turns) {
+    const map = new Map();
+    for (const t of turns) {
+        if (t.role === 'tool' && t.tool_call_id) {
+            map.set(t.tool_call_id, t);
+        }
+    }
+    return map;
+}
+/** Convert Turn[] into ChatMessage[] for the LLMCaller (which only takes role+content). */
+function turnsToMessages(turns) {
+    return turns
+        .filter((t) => t.content !== null)
+        .map((t) => ({
+        role: t.role === 'tool' ? 'user' : t.role,
+        content: t.role === 'tool'
+            ? `[Tool result from ${t.name ?? 'unknown'}]: ${t.content}`
+            : t.content,
+    }));
+}
+/**
+ * Multi-turn replay: replay a full agent trace with mocked tool results.
+ *
+ * The model makes fresh decisions (may call different tools), but tool results
+ * come from the original trace. If the model calls a tool that wasn't in the
+ * original trace, the tool result is "Tool not available in replay".
+ */
+async function replayMultiTurn(args) {
+    const { itemTurns, candidateRef, model, caller, seed, maxTurns } = args;
+    const toolResultMap = buildToolResultMap(itemTurns);
+    const replayTurns = [];
+    let totalCost = 0;
+    let totalLatency = 0;
+    let lastResponse = null;
+    // Start with system and user turns from the original trace
+    const initialTurns = itemTurns.filter((t) => t.role === 'system' || t.role === 'user');
+    for (const t of initialTurns) {
+        replayTurns.push({ ...t });
+    }
+    let roundTrips = 0;
+    while (roundTrips < maxTurns) {
+        roundTrips++;
+        const messages = turnsToMessages(replayTurns);
+        const startedAt = new Date().toISOString();
+        const response = await caller.call({
+            candidate_ref: candidateRef,
+            model,
+            messages,
+            seed,
+        });
+        const completedAt = new Date().toISOString();
+        totalCost += response.cost_usd;
+        totalLatency += response.latency_ms;
+        lastResponse = response;
+        // Parse tool_calls from the raw response if the provider returned them
+        const rawToolCalls = extractToolCalls(response);
+        if (rawToolCalls.length > 0) {
+            // Model wants to call tools
+            replayTurns.push({
+                role: 'assistant',
+                content: response.content || null,
+                tool_calls: rawToolCalls,
+                status: 'ok',
+                span_kind: 'llm',
+                model,
+                started_at: startedAt,
+                completed_at: completedAt,
+                cost_usd: response.cost_usd,
+                latency_ms: response.latency_ms,
+                tokens_in: response.tokens_in,
+                tokens_out: response.tokens_out,
+            });
+            // Inject original tool results (mocked)
+            for (const tc of rawToolCalls) {
+                const originalResult = toolResultMap.get(tc.id);
+                if (originalResult) {
+                    replayTurns.push({
+                        ...originalResult,
+                        status: originalResult.status ?? 'ok',
+                    });
+                }
+                else {
+                    replayTurns.push({
+                        role: 'tool',
+                        tool_call_id: tc.id,
+                        name: tc.name,
+                        content: JSON.stringify({ error: 'Tool not available in replay', tool: tc.name }),
+                        status: 'error',
+                        error: { type: 'ReplayError', message: `Tool "${tc.name}" was not called in the original trace` },
+                    });
+                }
+            }
+        }
+        else {
+            // Model gave a final text response — done
+            replayTurns.push({
+                role: 'assistant',
+                content: response.content,
+                status: response.finish_reason === 'error' ? 'error' : 'ok',
+                span_kind: 'llm',
+                model,
+                started_at: startedAt,
+                completed_at: completedAt,
+                cost_usd: response.cost_usd,
+                latency_ms: response.latency_ms,
+                tokens_in: response.tokens_in,
+                tokens_out: response.tokens_out,
+                ...(response.finish_reason === 'error' ? {
+                    error: { type: 'LLMError', message: 'Model returned error finish_reason' },
+                } : {}),
+            });
+            break;
+        }
+    }
+    if (!lastResponse) {
+        throw new Error(`Multi-turn replay produced no response for ${candidateRef}`);
+    }
+    return { response: lastResponse, replayTurns, totalCost, totalLatency };
+}
+/** Extract tool_calls from a raw LLM response (OpenAI format). */
+function extractToolCalls(response) {
+    if (response.finish_reason !== 'tool_calls')
+        return [];
+    const raw = response.raw;
+    if (!raw)
+        return [];
+    // OpenAI format: choices[0].message.tool_calls
+    const choices = raw.choices;
+    const toolCalls = choices?.[0]?.message?.tool_calls;
+    if (!toolCalls || !Array.isArray(toolCalls))
+        return [];
+    return toolCalls.map((tc) => ({
+        id: tc.id,
+        name: tc.function.name,
+        arguments: tc.function.arguments,
+    }));
+}
+export async function replay(args) {
+    const items = args.dataset.items ?? [];
+    if (items.length === 0) {
+        return { rows: [], determinism_score: 1, total_cost_usd: 0, total_latency_ms: 0 };
+    }
+    const master = args.seed ?? 0xC0FFEE;
+    const maxTurns = args.maxTurns ?? 10;
+    const rows = [];
+    let totalCost = 0;
+    let totalLatency = 0;
+    let matches = 0;
+    let attempts = 0;
+    for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        if (!item)
+            continue;
+        const id = itemId(item, i);
+        for (const cand of args.candidates) {
+            const seed = deriveSeed(master, cand.ref, id);
+            // Multi-turn replay when item has turns
+            if (item.turns && item.turns.length > 0) {
+                const result = await replayMultiTurn({
+                    item,
+                    itemTurns: item.turns,
+                    candidateRef: cand.ref,
+                    model: cand.model,
+                    caller: args.caller,
+                    seed,
+                    maxTurns,
+                });
+                const captured = result.response.toCapturedCall({
+                    id: `r-${id}-${cand.ref}`,
+                    started_at: new Date().toISOString(),
+                    completed_at: new Date().toISOString(),
+                    span_kind: 'gate.replay',
+                });
+                rows.push({
+                    item_id: id,
+                    candidate_ref: cand.ref,
+                    response: result.response,
+                    captured,
+                    turns: result.replayTurns,
+                });
+                totalCost += result.totalCost;
+                totalLatency += result.totalLatency;
+            }
+            else {
+                // Single-turn replay (legacy path)
+                const messages = cand.messages_from_item?.(item) ?? deriveMessages(item);
+                const response = await args.caller.call({
+                    candidate_ref: cand.ref,
+                    model: cand.model,
+                    messages,
+                    seed,
+                });
+                const captured = response.toCapturedCall({
+                    id: `r-${id}-${cand.ref}`,
+                    started_at: new Date().toISOString(),
+                    completed_at: new Date().toISOString(),
+                    span_kind: 'gate.replay',
+                });
+                rows.push({ item_id: id, candidate_ref: cand.ref, response, captured });
+                totalCost += response.cost_usd;
+                totalLatency += response.latency_ms;
+                if (args.measureDeterminism === true) {
+                    const second = await args.caller.call({
+                        candidate_ref: cand.ref,
+                        model: cand.model,
+                        messages,
+                        seed,
+                    });
+                    attempts += 1;
+                    if (second.content === response.content)
+                        matches += 1;
+                }
+            }
+        }
+    }
+    const determinism_score = attempts === 0 ? 1 : matches / attempts;
+    return {
+        rows,
+        determinism_score,
+        total_cost_usd: totalCost,
+        total_latency_ms: totalLatency,
+    };
+}
+function deriveMessages(item) {
+    if (Array.isArray(item.input)) {
+        return item.input
+            .filter((m) => typeof m === 'object' && m !== null && 'role' in m && 'content' in m)
+            .map((m) => ({ role: m.role, content: m.content }));
+    }
+    return [{ role: 'user', content: String(item.input ?? '') }];
+}
+//# sourceMappingURL=replay.js.map

package/dist/replay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.js","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAwCH,SAAS,UAAU,CAAC,MAAc,EAAE,YAAoB,EAAE,MAAc;IACtE,IAAI,CAAC,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5B,KAAK,MAAM,EAAE,IAAI,GAAG,YAAY,IAAI,MAAM,EAAE,EAAE,CAAC;QAC7C,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,SAAS,MAAM,CAAC,IAAiB,EAAE,GAAW;IAC5C,OAAO,IAAI,CAAC,EAAE,IAAI,SAAS,GAAG,GAAG,CAAC;AACpC,CAAC;AAED,mFAAmF;AACnF,SAAS,kBAAkB,CAAC,KAA0B;IACpD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAgB,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YACxC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,2FAA2F;AAC3F,SAAS,eAAe,CAAC,KAA0B;IACjD,OAAO,KAAK;SACT,MAAM,CAAC,CAAC,CAAC,EAAmC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC;SAClE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAe,CAAC,CAAC,CAAC,CAAC,CAAC,IAA2B;QACzE,OAAO,EAAE,CAAC,CAAC,IAAI,KAAK,MAAM;YACxB,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,IAAI,SAAS,MAAM,CAAC,CAAC,OAAO,EAAE;YAC3D,CAAC,CAAC,CAAC,CAAC,OAAO;KACd,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,eAAe,CAAC,IAQ9B;IACC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAExE,MAAM,aAAa,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IACpD,MAAM,WAAW,GAAW,EAAE,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAuB,IAAI,CAAC;IAE5C,2DAA2D;IAC3D,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IACvF,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,WAAW,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,UAAU,GAAG,QAAQ,EAAE,CAAC;QAC7B,UAAU,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;YACjC,aAAa,EAAE,YAAY;YAC3B,KAAK;YACL,QAAQ;YACR,IAAI;SACL,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7C,SAAS,IAAI,QAAQ,CAAC,QAAQ,CAAC;QAC/B,YAAY,IAAI,QAAQ,CAAC,UAAU,CAAC;QACpC,YAAY,GAAG,QAAQ,CAAC;QAExB,uEAAuE;QACvE,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAEhD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,4BAA4B;YAC5B,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,IAAI;gBACjC,UAAU,EAAE,YAAY;gBACxB,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,KAAK;gBAChB,KAAK;gBACL,UAAU,EAAE,SAAS;gBACrB,YAAY,EAAE,WAAW;gBACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;YAEH,wCAAwC;YACxC,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;gBAC9B,MAAM,cAAc,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAChD,IAAI,cAAc,EAAE,CAAC;oBACnB,WAAW,CAAC,IAAI,CAAC;wBACf,GAAG,cAAc;wBACjB,MAAM,EAAE,cAAc,CAAC,MAAM,IAAI,IAAI;qBACtC,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,IAAI,CAAC;wBACf,IAAI,EAAE,MAAM;wBACZ,YAAY,EAAE,EAAE,CAAC,EAAE;wBACnB,IAAI,EAAE,EAAE,CAAC,IAAI;wBACb,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;wBACjF,MAAM,EAAE,OAAO;wBACf,KAAK,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,IAAI,wCAAwC,EAAE;qBAClG,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,MAAM,EAAE,QAAQ,CAAC,aAAa,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;gBAC3D,SAAS,EAAE,KAAK;gBAChB,KAAK;gBACL,UAAU,EAAE,SAAS;gBACrB,YAAY,EAAE,WAAW;gBACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,GAAG,CAAC,QAAQ,CAAC,aAAa,KAAK,OAAO,CAAC,CAAC,CAAC;oBACvC,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,oCAAoC,EAAE;iBAC3E,CAAC,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CAAC;YACH,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,YAAY,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;AAC1E,CAAC;AAED,kEAAkE;AAClE,SAAS,gBAAgB,CAAC,QAAqB;IAC7C,IAAI,QAAQ,CAAC,aAAa,KAAK,YAAY;QAAE,OAAO,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAA0C,CAAC;IAChE,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IAEpB,+CAA+C;IAC/C,MAAM,OAAO,GAAG,GAAG,CAAC,OAAiI,CAAC;IACtJ,MAAM,SAAS,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC;IACpD,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5B,EAAE,EAAE,EAAE,CAAC,EAAE;QACT,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;QACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAgB;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACpF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IACrC,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAE9C,wCAAwC;YACxC,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;oBACnC,IAAI;oBACJ,SAAS,EAAE,IAAI,CAAC,KAAK;oBACrB,YAAY,EAAE,IAAI,CAAC,GAAG;oBACtB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,IAAI;oBACJ,QAAQ;iBACT,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAiB,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC;oBAC5D,EAAE,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE;oBACzB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACtC,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC;oBACR,OAAO,EAAE,EAAE;oBACX,aAAa,EAAE,IAAI,CAAC,GAAG;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,QAAQ;oBACR,KAAK,EAAE,MAAM,CAAC,WAAW;iBAC1B,CAAC,CAAC;gBACH,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;gBAC9B,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,mCAAmC;gBACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;gBACzE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBACtC,aAAa,EAAE,IAAI,CAAC,GAAG;oBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ;oBACR,IAAI;iBACL,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAiB,QAAQ,CAAC,cAAc,CAAC;oBACrD,EAAE,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE;oBACzB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACtC,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACxE,SAAS,IAAI,QAAQ,CAAC,QAAQ,CAAC;gBAC/B,YAAY,IAAI,QAAQ,CAAC,UAAU,CAAC;gBAEpC,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,EAAE,CAAC;oBACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;wBACpC,aAAa,EAAE,IAAI,CAAC,GAAG;wBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,QAAQ;wBACR,IAAI;qBACL,CAAC,CAAC;oBACH,QAAQ,IAAI,CAAC,CAAC;oBACd,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,OAAO;wBAAE,OAAO,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,QAAQ,CAAC;IAClE,OAAO;QACL,IAAI;QACJ,iBAAiB;QACjB,cAAc,EAAE,SAAS;QACzB,gBAAgB,EAAE,YAAY;KAC/B,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAiB;IACvC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,KAAK;aACd,MAAM,CAAC,CAAC,CAAC,EAA0C,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,MAAM,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,CAAC;aAC3H,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAuC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;AAC/D,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Real runGate runner — orchestrates replay + judge + compare + sign + evaluate.
+ *
+ * This is what gets installed into the SDK via `setGateRunner()`. Decoupled
+ * from the SDK shape so the engine can evolve without breaking SDK consumers.
+ *
+ * Flow:
+ *   1. Resolve each candidate ref → { ref, model, messages_from_item }.
+ *   2. Replay each (candidate, item) via LLMCaller.
+ *   3. For each (candidate, item, judge), call LLMJudge and parse verdict.
+ *   4. Compare → per-candidate metrics + judge matrix.
+ *   5. Sign the resulting GateReport via HMAC-SHA256.
+ *   6. Evaluate policy → GateDecision.
+ *
+ * @see docs/prd.md §5.1 — The gate
+ */
+import type { DatasetItem } from '@lanefoundry/gatelane-sdk/dataset';
+import { type GateRunner } from '@lanefoundry/gatelane-sdk/gate';
+import { type LLMCaller, type LLMRequest } from './llm.js';
+import type { D1DatabaseLike } from '@lanefoundry/gatelane-sdk';
+export type RunnerOptions = {
+    /** Caller used for both candidate invocations and judge calls. */
+    caller: LLMCaller;
+    /** Per-judge LLMCallers (optional; defaults to `caller`). */
+    judge_callers?: Record<string, LLMCaller>;
+    /** HMAC signing key. Required. */
+    signing_key: string;
+    /** Master seed for replay determinism. */
+    seed?: number;
+    /** Replay twice to compute determinism_score. Doubles cost. */
+    measureDeterminism?: boolean;
+    /** How to extract chat messages from a dataset item. */
+    messages_from_item?: (item: DatasetItem) => Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+    /** Optional D1 database for audit log persistence. */
+    audit_db?: D1DatabaseLike;
+    /** Service name for OTel tracing. */
+    tracing_service_name?: string;
+    /** Max concurrent LLM calls during judge stage (default: 5). */
+    concurrency?: number;
+};
+export declare function createRunner(opts: RunnerOptions): GateRunner;
+export type { LLMRequest };
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAGL,KAAK,UAAU,EAChB,MAAM,gCAAgC,CAAC;AAKxC,OAAO,EAAsB,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAQ/E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAEhE,MAAM,MAAM,aAAa,GAAG;IAC1B,kEAAkE;IAClE,MAAM,EAAE,SAAS,CAAC;IAClB,6DAA6D;IAC7D,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC1C,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,wDAAwD;IACxD,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,WAAW,KAAK,KAAK,CAAC;QAAE,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9G,sDAAsD;IACtD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,qCAAqC;IACrC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gEAAgE;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,wBAAgB,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CAuQ5D;AAED,YAAY,EAAE,UAAU,EAAE,CAAC"}