melchizedek-agents 0.9.2 → 0.9.3

package/dist/lib/observability/supabaseSpanExporter.js

@@ -1,35 +1,297 @@
 /**
- * lib/observability/supabaseSpanExporter.ts — optional OpenTelemetry sink
- * that persists model-request telemetry to Supabase.
+ * lib/observability/supabaseSpanExporter.ts — the observability ledger sink.
  *
  * WHY this file exists:
  *   The console exporter in tracer.ts prints every span as an
- *   [OTEL_SPAN_JSON] line — good for a terminal, gone when it scrolls.
- *   Operators who want durable usage data (tokens per model, per provider,
- *   over time) can opt in with TELEMETRY_SUPABASE=true, and every
- *   `llm.request` and `Syndicate Execution:*` span is also inserted into
- *   the adk_telemetry table (schema: db/telemetry.sql, RLS: db/hardening.sql).
+ *   [OTEL_SPAN_JSON] line — good for a terminal, gone when it scrolls. With
+ *   TELEMETRY_SUPABASE=true (and Supabase credentials) every turn and every
+ *   model call is also written to Supabase, in three tiers with three
+ *   lifetimes (schema: db/telemetry.sql, lockdown: db/hardening.sql):
+ *
+ *     adk_turns      one row per turn — the system of record: input, output,
+ *                    route, agent, errors, tokens, model-vs-tool time, the
+ *                    tool calls WITH their responses, the ids that join it to
+ *                    the session (session_id, invocation_id) and the task,
+ *                    and provenance (config_hash, engine_version).
+ *     adk_telemetry  one row per span — llm.request and root spans, raw.
+ *     adk_payloads   full request/response per model call, captured by
+ *                    POLICY and expiring. ADK's own call_llm spans carry the
+ *                    assembled prompt and raw response; they are 10-100x a
+ *                    turn row, so: errors and fallbacks always, a
+ *                    deterministic sample of the rest, 30-day TTL.
  *
  * DESIGN NOTES:
- *   - Fire-and-forget: an insert failure is logged once and never thrown
- *     into the tracing pipeline — telemetry must not break inference.
- *   - Only spans that carry usage are exported (name filter below); ADK's
- *     internal spans stay console-only to keep row volume sane.
+ *   - Never on the hot path: inserts are fire-and-forget behind a
+ *     BatchSpanProcessor; a failure is logged once per table and the rows
+ *     are spooled to a dead-letter file (OUTPUTS_DIR/telemetry-deadletter.ndjson)
+ *     so a sink outage loses nothing silently. `npm run telemetry:stats`
+ *     shows what landed; `telemetry:replay` re-sends the spool.
+ *   - Payload decisions need the TURN's outcome (error? fallback?), which is
+ *     only known when the root span ends — after its call_llm children. So
+ *     call_llm spans wait in a per-trace buffer until their root arrives,
+ *     then the decision is applied to all of them at once. Children that
+ *     arrive after the root (a later batch) find the decision cached.
+ *   - Sampling is deterministic in the trace id: a trace keeps all of its
+ *     payloads or none, and a re-run of the same policy keeps the same set.
  */
+import { appendFileSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
 import core from '@opentelemetry/core';
 const { ExportResultCode } = core;
 import { hasSupabaseCredentials } from "../persistence/supabaseProvider.js";
-/** True when the operator has opted into the Supabase telemetry sink. */
+import { TELEMETRY_SCHEMA_VERSION } from "./lineage.js";
+/**
+ * TELEMETRY_PAYLOADS: off | errors | sample (default) | all
+ * TELEMETRY_PAYLOAD_SAMPLE: 0..1, default 0.10
+ * TELEMETRY_PAYLOAD_TTL_DAYS: default 30
+ */
+export function payloadPolicyFromEnv(env = process.env) {
+    const raw = (env.TELEMETRY_PAYLOADS ?? 'sample').trim().toLowerCase();
+    const mode = raw === 'off' || raw === 'errors' || raw === 'all' ? raw : 'sample';
+    const rate = Number(env.TELEMETRY_PAYLOAD_SAMPLE ?? '0.10');
+    const ttl = Number(env.TELEMETRY_PAYLOAD_TTL_DAYS ?? '30');
+    return {
+        mode,
+        sampleRate: Number.isFinite(rate) ? Math.min(1, Math.max(0, rate)) : 0.1,
+        ttlDays: Number.isFinite(ttl) && ttl > 0 ? ttl : 30,
+    };
+}
+/** Deterministic 0..1 value from a trace id (first 8 hex chars). */
+export function traceSampleValue(traceId) {
+    const head = parseInt(traceId.slice(0, 8), 16);
+    return Number.isFinite(head) ? head / 0x100000000 : 1;
+}
+/** Which payloads of a turn to keep, given the policy and the turn's outcome. */
+export function payloadDecision(policy, traceId, facts) {
+    if (policy.mode === 'off')
+        return null;
+    if (facts.error)
+        return 'error';
+    if (facts.fallback)
+        return 'fallback';
+    if (policy.mode === 'errors')
+        return null;
+    if (policy.mode === 'all')
+        return 'all';
+    return traceSampleValue(traceId) < policy.sampleRate ? 'sample' : null;
+}
+// ── Span → row mapping (pure; tested offline) ───────────────────────────────
+const ROOT_PREFIX = 'Syndicate Execution: ';
+const EXPORTED_SPAN_NAMES = /^(llm\.request$|Syndicate Execution: )/;
+const ADK_SCOPE = 'gcp.vertex.agent';
+function attrsOf(span) {
+    return span.attributes;
+}
+function num(attrs, key) {
+    return typeof attrs[key] === 'number' ? attrs[key] : null;
+}
+function str(attrs, key) {
+    return typeof attrs[key] === 'string' && attrs[key] !== '' ? attrs[key] : null;
+}
+function bool(attrs, key) {
+    return typeof attrs[key] === 'boolean' ? attrs[key] : null;
+}
+function spanTs(span) {
+    return new Date(span.startTime[0] * 1000 + span.startTime[1] / 1e6).toISOString();
+}
+function durationMs(span) {
+    return span.duration[0] * 1000 + span.duration[1] / 1e6;
+}
+function scopeName(span) {
+    const s = span;
+    return s.instrumentationScope?.name ?? s.instrumentationLibrary?.name ?? '';
+}
+export function isRootSpan(span) {
+    return span.name.startsWith(ROOT_PREFIX);
+}
+export function isPayloadSpan(span) {
+    return span.name === 'call_llm' && scopeName(span) === ADK_SCOPE;
+}
+/** `syndicate.input` is JSON.stringify of either a string or A2A parts. */
+export function parseInputText(raw) {
+    if (raw === null || raw === undefined)
+        return '';
+    if (typeof raw !== 'string')
+        return String(raw);
+    let data;
+    try {
+        data = JSON.parse(raw);
+    }
+    catch {
+        return raw;
+    }
+    if (Array.isArray(data)) {
+        return data
+            .map((p) => (p && typeof p === 'object' ? String(p.text ?? '') : String(p)))
+            .join('\n');
+    }
+    return typeof data === 'string' ? data : JSON.stringify(data);
+}
+function parseLoose(value) {
+    if (typeof value !== 'string')
+        return value ?? null;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return value;
+    }
+}
+function syndicateOf(span) {
+    const attrs = attrsOf(span);
+    return str(attrs, 'syndicate.name') ?? (isRootSpan(span) ? span.name.slice(ROOT_PREFIX.length) : null);
+}
+function spanJson(span) {
+    return {
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        name: span.name,
+        durationMs: durationMs(span),
+        attributes: attrsOf(span),
+        events: span.events.map((e) => ({ name: e.name, attributes: e.attributes ?? {} })),
+        status: span.status,
+    };
+}
+/** Flatten an llm.request or root span into one adk_telemetry row. */
+export function toTelemetryRow(span) {
+    const attrs = attrsOf(span);
+    return {
+        ts: spanTs(span),
+        trace_id: span.spanContext().traceId,
+        span_id: span.spanContext().spanId,
+        span_name: span.name,
+        syndicate: syndicateOf(span),
+        agent: str(attrs, 'llm.agent') ?? str(attrs, 'syndicate.agent'),
+        provider: str(attrs, 'llm.provider'),
+        model: str(attrs, 'llm.model'),
+        input_tokens: num(attrs, 'llm.tokens.input') ?? num(attrs, 'syndicate.tokens.input'),
+        output_tokens: num(attrs, 'llm.tokens.output') ?? num(attrs, 'syndicate.tokens.output'),
+        thinking_tokens: num(attrs, 'llm.tokens.thinking') ?? num(attrs, 'syndicate.tokens.thinking'),
+        latency_ms: durationMs(span),
+        session_id: str(attrs, 'session.id'),
+        user_id: str(attrs, 'user.id'),
+        task_id: str(attrs, 'a2a.task_id'),
+        invocation_id: str(attrs, 'adk.invocation_id'),
+        schema_version: TELEMETRY_SCHEMA_VERSION,
+        span: spanJson(span),
+    };
+}
+/** Project a root span into the adk_turns system-of-record row. */
+export function toTurnRow(span) {
+    const attrs = attrsOf(span);
+    const toolEvents = span.events
+        .filter((e) => e.name === 'ToolCall' || e.name === 'ToolResponse')
+        .map((e) => {
+        const a = (e.attributes ?? {});
+        return e.name === 'ToolCall'
+            ? { name: 'ToolCall', tool: a['tool.name'] ?? null, args: parseLoose(a['tool.args']) }
+            : { name: 'ToolResponse', tool: a['tool.name'] ?? null, data: parseLoose(a['tool.data_gathered']) };
+    });
+    const models = (str(attrs, 'syndicate.models') ?? '').split(',').filter(Boolean);
+    const evalTrial = attrs['eval.trial'];
+    return {
+        ts: spanTs(span),
+        trace_id: span.spanContext().traceId,
+        span_id: span.spanContext().spanId,
+        session_id: str(attrs, 'session.id'),
+        user_id: str(attrs, 'user.id'),
+        task_id: str(attrs, 'a2a.task_id'),
+        invocation_id: str(attrs, 'adk.invocation_id'),
+        syndicate: syndicateOf(span) ?? 'unknown',
+        stage: str(attrs, 'syndicate.stage') ?? 'delegate',
+        agent: str(attrs, 'syndicate.agent'),
+        route: str(attrs, 'syndicate.route'),
+        route_reason: str(attrs, 'syndicate.route.reason'),
+        route_fell_back: bool(attrs, 'syndicate.route.fell_back'),
+        route_via_override: bool(attrs, 'syndicate.route.via_override'),
+        relay_fallback: bool(attrs, 'syndicate.relay_fallback') ?? false,
+        input: parseInputText(attrs['syndicate.input']),
+        output: str(attrs, 'syndicate.output') ?? '',
+        error_code: str(attrs, 'syndicate.error.code'),
+        error_message: str(attrs, 'syndicate.error.message'),
+        input_tokens: num(attrs, 'syndicate.tokens.input'),
+        output_tokens: num(attrs, 'syndicate.tokens.output'),
+        thinking_tokens: num(attrs, 'syndicate.tokens.thinking'),
+        latency_ms: durationMs(span),
+        model_ms: num(attrs, 'syndicate.latency.model_ms'),
+        tool_ms: num(attrs, 'syndicate.latency.tool_ms'),
+        llm_calls: num(attrs, 'syndicate.llm_calls'),
+        tool_calls: toolEvents.filter((e) => e.name === 'ToolCall').length,
+        models,
+        tool_events: toolEvents,
+        config_hash: str(attrs, 'syndicate.config_hash'),
+        engine_version: str(attrs, 'engine.version'),
+        eval_run: str(attrs, 'eval.run'),
+        eval_suite: str(attrs, 'eval.suite'),
+        eval_case: str(attrs, 'eval.case'),
+        eval_variant: str(attrs, 'eval.variant'),
+        eval_trial: typeof evalTrial === 'number' ? evalTrial : evalTrial ? Number(evalTrial) || null : null,
+        attributes: attrs,
+        schema_version: TELEMETRY_SCHEMA_VERSION,
+    };
+}
+/** The facts a payload decision needs, read off a root span. */
+export function turnFacts(span) {
+    const attrs = attrsOf(span);
+    const statusError = span.status?.code === 2; // SpanStatusCode.ERROR
+    return {
+        error: !!str(attrs, 'syndicate.error.code') || statusError,
+        fallback: bool(attrs, 'syndicate.relay_fallback') === true || bool(attrs, 'syndicate.route.fell_back') === true,
+    };
+}
+/** Map an ADK call_llm span into an adk_payloads row. */
+export function toPayloadRow(span, reason, ttlDays, turn) {
+    const attrs = attrsOf(span);
+    const request = str(attrs, 'gcp.vertex.agent.llm_request') ?? '';
+    const response = str(attrs, 'gcp.vertex.agent.llm_response') ?? '';
+    const started = span.startTime[0] * 1000 + span.startTime[1] / 1e6;
+    return {
+        ts: new Date(started).toISOString(),
+        trace_id: span.spanContext().traceId,
+        span_id: span.spanContext().spanId,
+        session_id: turn?.sessionId ?? null,
+        invocation_id: str(attrs, 'gcp.vertex.agent.invocation_id') ?? turn?.invocationId ?? null,
+        agent: str(attrs, 'llm.agent') ?? str(attrs, 'gen_ai.agent.name'),
+        provider: str(attrs, 'gen_ai.system'),
+        model: str(attrs, 'gen_ai.request.model'),
+        reason,
+        request: parseLoose(request),
+        response: parseLoose(response),
+        request_chars: request.length,
+        response_chars: response.length,
+        expires_at: new Date(started + ttlDays * 86_400_000).toISOString(),
+        schema_version: TELEMETRY_SCHEMA_VERSION,
+    };
+}
+// ── The exporter ─────────────────────────────────────────────────────────────
+/** True when the operator has opted into the Supabase ledger. */
 export function telemetrySinkEnabled() {
     return process.env.TELEMETRY_SUPABASE === 'true' && hasSupabaseCredentials();
 }
-const EXPORTED_SPAN_NAMES = /^(llm\.request$|Syndicate Execution: )/;
+export function deadLetterPath() {
+    return join(process.env.OUTPUTS_DIR ?? join(process.cwd(), 'outputs'), 'telemetry-deadletter.ndjson');
+}
+const PENDING_TTL_MS = 10 * 60 * 1000;
+const DECISION_TTL_MS = 30 * 60 * 1000;
+const PENDING_CAP = 5_000;
 export class SupabaseSpanExporter {
-    /** Lazily created on first export so constructing the exporter is free
-     *  and synchronous (NodeTracerProvider wants processors at construction). */
     clientPromise;
-    insertFailureLogged = false;
+    policy;
+    deadLetterFile;
+    failureLogged = new Set();
     pendingInserts = new Set();
+    /** call_llm spans waiting for their root span's outcome, by trace id. */
+    pendingPayloads = new Map();
+    /** Decisions already taken for a trace, for children arriving late. */
+    decisions = new Map();
+    pendingCount = 0;
+    stats = { inserted: {}, failed: {}, deadLettered: 0, payloadsDropped: 0 };
+    constructor(options = {}) {
+        this.policy = options.policy ?? payloadPolicyFromEnv();
+        this.deadLetterFile = options.deadLetterFile ?? deadLetterPath();
+        if (options.client)
+            this.clientPromise = Promise.resolve(options.client);
+    }
     getClient() {
         if (!this.clientPromise) {
             this.clientPromise = import('@supabase/supabase-js').then(({ createClient }) => createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY));
@@ -37,36 +299,109 @@ export class SupabaseSpanExporter {
         return this.clientPromise;
     }
     export(spans, resultCallback) {
-        const rows = spans
-            .filter((span) => EXPORTED_SPAN_NAMES.test(span.name))
-            .map((span) => this.toRow(span));
-        if (rows.length === 0) {
-            resultCallback({ code: ExportResultCode.SUCCESS });
-            return;
-        }
-        // Fire-and-forget: report success to the pipeline immediately; log the
-        // first insert failure so a misconfigured table is visible without
-        // spamming every span batch.
-        const insert = this.getClient()
-            .then((client) => client.from('adk_telemetry').insert(rows))
-            .then(({ error }) => {
-            if (error && !this.insertFailureLogged) {
-                this.insertFailureLogged = true;
-                console.warn(`[TELEMETRY] Supabase insert failed (${error.message}). ` +
-                    'Has db/telemetry.sql been applied? Telemetry rows are being dropped; ' +
-                    'console [OTEL_SPAN_JSON] output is unaffected.');
+        const telemetryRows = [];
+        const turnRows = [];
+        const payloadRows = [];
+        for (const span of spans) {
+            if (EXPORTED_SPAN_NAMES.test(span.name))
+                telemetryRows.push(toTelemetryRow(span));
+            if (isRootSpan(span)) {
+                const turn = toTurnRow(span);
+                turnRows.push(turn);
+                const traceId = span.spanContext().traceId;
+                const reason = payloadDecision(this.policy, traceId, turnFacts(span));
+                const ctx = { sessionId: turn.session_id, invocationId: turn.invocation_id };
+                this.decisions.set(traceId, { at: Date.now(), reason, ...ctx });
+                const waiting = this.pendingPayloads.get(traceId);
+                if (waiting) {
+                    this.pendingPayloads.delete(traceId);
+                    this.pendingCount -= waiting.spans.length;
+                    if (reason) {
+                        for (const child of waiting.spans)
+                            payloadRows.push(toPayloadRow(child, reason, this.policy.ttlDays, ctx));
+                    }
+                    else {
+                        this.stats.payloadsDropped += waiting.spans.length;
+                    }
+                }
+                continue;
             }
-        })
-            .catch((err) => {
-            if (!this.insertFailureLogged) {
-                this.insertFailureLogged = true;
-                console.warn(`[TELEMETRY] Supabase sink error: ${err instanceof Error ? err.message : err}`);
+            if (this.policy.mode !== 'off' && isPayloadSpan(span)) {
+                const traceId = span.spanContext().traceId;
+                const decided = this.decisions.get(traceId);
+                if (decided) {
+                    if (decided.reason)
+                        payloadRows.push(toPayloadRow(span, decided.reason, this.policy.ttlDays, decided));
+                    else
+                        this.stats.payloadsDropped += 1;
+                }
+                else if (this.pendingCount < PENDING_CAP) {
+                    const entry = this.pendingPayloads.get(traceId) ?? { at: Date.now(), spans: [] };
+                    entry.spans.push(span);
+                    this.pendingPayloads.set(traceId, entry);
+                    this.pendingCount += 1;
+                }
+                else {
+                    this.stats.payloadsDropped += 1;
+                }
             }
-        })
-            .finally(() => this.pendingInserts.delete(insert));
-        this.pendingInserts.add(insert);
+        }
+        this.pruneBuffers();
+        if (telemetryRows.length)
+            this.insert('adk_telemetry', telemetryRows);
+        if (turnRows.length)
+            this.insert('adk_turns', turnRows);
+        if (payloadRows.length)
+            this.insert('adk_payloads', payloadRows);
         resultCallback({ code: ExportResultCode.SUCCESS });
     }
+    pruneBuffers() {
+        const now = Date.now();
+        for (const [traceId, entry] of this.pendingPayloads) {
+            if (now - entry.at > PENDING_TTL_MS) {
+                this.pendingPayloads.delete(traceId);
+                this.pendingCount -= entry.spans.length;
+                this.stats.payloadsDropped += entry.spans.length;
+            }
+        }
+        for (const [traceId, d] of this.decisions) {
+            if (now - d.at > DECISION_TTL_MS)
+                this.decisions.delete(traceId);
+        }
+    }
+    /** Fire-and-forget insert; failures are logged once per table and spooled. */
+    insert(table, rows) {
+        const attempt = this.getClient()
+            .then((client) => client.from(table).insert(rows))
+            .then(({ error }) => {
+            if (error)
+                this.onFailure(table, rows, error.message);
+            else
+                this.stats.inserted[table] = (this.stats.inserted[table] ?? 0) + rows.length;
+        })
+            .catch((err) => this.onFailure(table, rows, err instanceof Error ? err.message : String(err)))
+            .finally(() => this.pendingInserts.delete(attempt));
+        this.pendingInserts.add(attempt);
+    }
+    onFailure(table, rows, message) {
+        this.stats.failed[table] = (this.stats.failed[table] ?? 0) + rows.length;
+        if (!this.failureLogged.has(table)) {
+            this.failureLogged.add(table);
+            console.warn(`[TELEMETRY] insert into ${table} failed (${message}). ` +
+                `Has db/telemetry.sql been applied? Rows are spooled to ${this.deadLetterFile || '(spool disabled)'}; ` +
+                'console [OTEL_SPAN_JSON] output is unaffected.');
+        }
+        if (!this.deadLetterFile)
+            return;
+        try {
+            mkdirSync(join(this.deadLetterFile, '..'), { recursive: true });
+            appendFileSync(this.deadLetterFile, `${JSON.stringify({ ts: new Date().toISOString(), table, error: message, rows })}\n`);
+            this.stats.deadLettered += rows.length;
+        }
+        catch {
+            /* the spool is best-effort; the warning above already fired */
+        }
+    }
     /** Waits for in-flight inserts so short-lived scripts don't drop rows. */
     async shutdown() {
         await Promise.allSettled([...this.pendingInserts]);
@@ -74,42 +409,4 @@ export class SupabaseSpanExporter {
     async forceFlush() {
         await Promise.allSettled([...this.pendingInserts]);
     }
-    /** Flatten a span into one adk_telemetry row. */
-    toRow(span) {
-        const attrs = span.attributes;
-        const num = (key) => typeof attrs[key] === 'number' ? attrs[key] : null;
-        const str = (key) => typeof attrs[key] === 'string' ? attrs[key] : null;
-        // Root spans are named "Syndicate Execution: <name>"; llm.request spans
-        // carry syndicate context only if the caller set syndicate.name.
-        const syndicate = str('syndicate.name') ??
-            (span.name.startsWith('Syndicate Execution: ')
-                ? span.name.slice('Syndicate Execution: '.length)
-                : null);
-        return {
-            ts: new Date(span.startTime[0] * 1000 + span.startTime[1] / 1e6).toISOString(),
-            trace_id: span.spanContext().traceId,
-            span_id: span.spanContext().spanId,
-            span_name: span.name,
-            syndicate,
-            agent: str('llm.agent'),
-            provider: str('llm.provider'),
-            model: str('llm.model'),
-            input_tokens: num('llm.tokens.input') ?? num('syndicate.tokens.input'),
-            output_tokens: num('llm.tokens.output') ?? num('syndicate.tokens.output'),
-            thinking_tokens: num('llm.tokens.thinking') ?? num('syndicate.tokens.thinking'),
-            latency_ms: span.duration[0] * 1000 + span.duration[1] / 1e6,
-            span: {
-                traceId: span.spanContext().traceId,
-                spanId: span.spanContext().spanId,
-                name: span.name,
-                durationMs: span.duration[0] * 1000 + span.duration[1] / 1e6,
-                attributes: attrs,
-                events: span.events.map((e) => ({
-                    name: e.name,
-                    attributes: e.attributes ?? {},
-                })),
-                status: span.status,
-            },
-        };
-    }
 }

package/dist/lib/observability/tracer.d.ts

@@ -1,5 +1,15 @@
 import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import type { Event, LlmResponse } from '@google/adk';
+/**
+ * Optional OTLP export for a live trace viewer (Phoenix, Langfuse, Grafana
+ * Tempo, Jaeger — anything that speaks OTLP/HTTP). Set
+ * OTEL_EXPORTER_OTLP_ENDPOINT (e.g. http://localhost:6006 for Phoenix) and
+ * every span — ADK's included, so the viewer gets the waterfall — is also
+ * sent there. OTEL_EXPORTER_OTLP_HEADERS is honoured by the exporter itself
+ * (api keys); OTEL_SERVICE_NAME names the service (default "melchizedek").
+ * Supabase stays the system of record; this is a viewer.
+ */
+export declare function otlpEndpoint(): string | undefined;
 /**
  * Flushes buffered span processors (console is synchronous; the Supabase
  * batch sink is not). Call before process exit in short-lived scripts so
@@ -8,11 +18,74 @@ import type { Event, LlmResponse } from '@google/adk';
 export declare function flushTracing(): Promise<void>;
 /** Subscribe to finished spans. Returns an unsubscribe function. */
 export declare function onSpanEnd(listener: (span: ReadableSpan) => void): () => void;
+/**
+ * Per-trace accumulators, filled as child spans END and read by the root
+ * span in its own finally: how much of a turn was model time vs tool time,
+ * how many model calls it took, which models answered. The root turns
+ * these into `syndicate.latency.model_ms` / `.tool_ms` / `syndicate.llm_calls`
+ * / `syndicate.models`, so a p95 regression is attributable without
+ * reassembling child spans at query time.
+ */
+interface TraceStats {
+    modelMs: number;
+    toolMs: number;
+    llmCalls: number;
+    models: Set<string>;
+    startedAt: number;
+}
+/** Read-and-forget the stats for a finished trace. */
+export declare function takeTraceStats(traceId: string): TraceStats | undefined;
+/** Name of the nearest enclosing ADK `invoke_agent` span, or null. */
+export declare function agentForSpan(spanId: string | undefined): string | null;
 export declare function initializeTracing(): void;
 export interface TraceMetadata {
     syndicateName: string;
     bindings?: Record<string, any>;
     input?: any;
+    /**
+     * The plan-dispatch resolution this turn runs under, recorded on the root
+     * span as `syndicate.route*` attributes. Without it the chosen route was
+     * only ever a console line, so routing accuracy could not be scored from
+     * stored telemetry.
+     */
+    route?: {
+        route: string;
+        reason?: string;
+        fellBack?: boolean;
+        viaOverride?: boolean;
+    };
+    /**
+     * Extra root-span attributes. The observatory tags eval traffic with
+     * `eval.suite` / `eval.case` / `eval.variant` / `eval.trial` / `eval.run`
+     * so post-deployment suites can exclude it from production traces.
+     */
+    attributes?: Record<string, string | number | boolean>;
+    /** Receives the root span's ids as soon as it starts (before any child). */
+    onSpanStart?: (ids: {
+        traceId: string;
+        spanId: string;
+    }) => void;
+    /**
+     * Identity of the turn. These are what make a stored turn joinable to
+     * its session row (`adk_sessions.id` = app:user:session) and to the ADK
+     * events of the same invocation; without them a trace and a session can
+     * only be matched by timestamp and luck.
+     */
+    sessionId?: string;
+    userId?: string;
+    taskId?: string;
+    /** 'delegate' | 'dispatch' | 'classify' | 'judge' — which kind of turn. */
+    stage?: string;
+    /** Provenance (lib/observability/lineage.ts): the resolved-config hash. */
+    configHash?: string;
+    /**
+     * Called once, after the last event and before the span ends, to add
+     * attributes only the consumer can know at that point — the A2A executor
+     * reports whether the DELEGATE relay fallback fired, a fact decided from
+     * the drained text and tool names. Runs inside the generator's finally,
+     * so every event has been processed by the time it is asked.
+     */
+    onEnd?: () => Record<string, string | number | boolean | undefined>;
 }
 /**
  * Wraps an ADK `Runner.runAsync` stream with a root OpenTelemetry span.
@@ -42,3 +115,4 @@ export declare function traceLlmGeneration(meta: LlmCallMeta, inner: AsyncGenera
  * Used by adapters for per-request flags like llm.web_search.omitted.
  */
 export declare function setLlmSpanAttribute(key: string, value: string | number | boolean): void;
+export {};