melchizedek-agents 0.9.2 → 0.9.4

package/dist/lib/observability/tracer.js

@@ -1,4 +1,6 @@
 import { trace, context } from '@opentelemetry/api';
+import { createRequire } from 'node:module';
+import { TELEMETRY_SCHEMA_VERSION, engineVersion } from "./lineage.js";
 import sdkNode from '@opentelemetry/sdk-trace-node';
 const { NodeTracerProvider } = sdkNode;
 import sdkBase from '@opentelemetry/sdk-trace-base';
@@ -6,6 +8,38 @@ const { SimpleSpanProcessor } = sdkBase;
 import core from '@opentelemetry/core';
 const { ExportResultCode } = core;
 import { SupabaseSpanExporter, telemetrySinkEnabled, } from "./supabaseSpanExporter.js";
+/**
+ * Optional OTLP export for a live trace viewer (Phoenix, Langfuse, Grafana
+ * Tempo, Jaeger — anything that speaks OTLP/HTTP). Set
+ * OTEL_EXPORTER_OTLP_ENDPOINT (e.g. http://localhost:6006 for Phoenix) and
+ * every span — ADK's included, so the viewer gets the waterfall — is also
+ * sent there. OTEL_EXPORTER_OTLP_HEADERS is honoured by the exporter itself
+ * (api keys); OTEL_SERVICE_NAME names the service (default "melchizedek").
+ * Supabase stays the system of record; this is a viewer.
+ */
+export function otlpEndpoint() {
+    const raw = process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim();
+    if (!raw)
+        return undefined;
+    return /\/v1\/traces\/?$/.test(raw) ? raw : `${raw.replace(/\/$/, '')}/v1/traces`;
+}
+function otlpProcessor() {
+    const url = otlpEndpoint();
+    if (!url)
+        return undefined;
+    try {
+        // Synchronous load: NodeTracerProvider wants its processors at construction.
+        const mod = createRequire(import.meta.url)('@opentelemetry/exporter-trace-otlp-http');
+        const OTLPTraceExporter = mod.OTLPTraceExporter ?? mod.default?.OTLPTraceExporter;
+        const { BatchSpanProcessor } = sdkBase;
+        console.log(`[TELEMETRY] OTLP export enabled → ${url}`);
+        return new BatchSpanProcessor(new OTLPTraceExporter({ url }));
+    }
+    catch (err) {
+        console.warn(`[TELEMETRY] OTLP exporter unavailable (${err instanceof Error ? err.message : err}); spans not sent to ${url}`);
+        return undefined;
+    }
+}
 let isOtelInitialized = false;
 let tracerProvider;
 let telemetryExporter;
@@ -27,6 +61,24 @@ export function onSpanEnd(listener) {
     spanEndListeners.add(listener);
     return () => spanEndListeners.delete(listener);
 }
+// ADK's own spans (scope "gcp.vertex.agent": invocation, invoke_agent,
+// call_llm, execute_tool) carry the FULL model request and response as
+// attributes. They reach the in-process listeners — that is how per-agent
+// attribution and the observatory work — but are not printed unless asked:
+// on a server they would put every prompt into the log stream.
+const ADK_SCOPE = 'gcp.vertex.agent';
+const PRINT_ALL_SPANS = process.env.OTEL_CONSOLE_ALL_SPANS === 'true';
+// OTEL_CONSOLE_SPANS=false silences the [OTEL_SPAN_JSON] lines entirely.
+// The in-process listeners below still fire and the Supabase sink still
+// records, so this costs no telemetry — it only unclutters a chat session.
+// Read at export time rather than module load, so an entry point can set
+// its own default in main() (syndicate_chat.ts defaults to silent) after
+// loadEnv() has run and before the first span ends.
+const printConsoleSpans = () => process.env.OTEL_CONSOLE_SPANS !== 'false';
+function spanScopeName(span) {
+    const s = span;
+    return s.instrumentationScope?.name ?? s.instrumentationLibrary?.name ?? '';
+}
 class JsonConsoleExporter {
     export(spans, resultCallback) {
         for (const span of spans) {
@@ -38,6 +90,10 @@ class JsonConsoleExporter {
                     /* a listener bug must not break the export pipeline */
                 }
             }
+            if (!printConsoleSpans())
+                continue;
+            if (!PRINT_ALL_SPANS && spanScopeName(span) === ADK_SCOPE)
+                continue;
             // Create a clean JSON representation of the span
             const jsonSpan = {
                 traceId: span.spanContext().traceId,
@@ -54,7 +110,7 @@ class JsonConsoleExporter {
                 })),
                 status: span.status,
             };
-            // Print to stdout with a specific prefix so test_local.py can easily extract it
+            // Print to stdout with a fixed prefix so a wrapper can extract spans from a transcript
             console.log(`[OTEL_SPAN_JSON] ${JSON.stringify(jsonSpan)}`);
         }
         resultCallback({ code: ExportResultCode.SUCCESS });
@@ -63,6 +119,131 @@ class JsonConsoleExporter {
         return Promise.resolve();
     }
 }
+// ── Span lineage (per-agent attribution) ─────────────────────────────────────
+// ADK wraps every agent turn in an `invoke_agent <name>` span and every model
+// call in a `call_llm` child of it; our llm.request span is a child of THAT.
+// A span cannot read its parent's name through the OTEL API, so a processor
+// records each span's name and parent at start, and llm.request walks up the
+// chain to find the agent it belongs to. This is what fills the `agent`
+// column of adk_telemetry — per-subagent cost and latency attribution — and
+// what lets the observatory say which agent made which call.
+const spanLineage = new Map();
+const SPAN_LINEAGE_CAP = 10_000;
+const traceStats = new Map();
+const TRACE_STATS_TTL_MS = 60 * 60 * 1000;
+function statsFor(traceId) {
+    let s = traceStats.get(traceId);
+    if (!s) {
+        s = { modelMs: 0, toolMs: 0, llmCalls: 0, models: new Set(), startedAt: Date.now() };
+        traceStats.set(traceId, s);
+        if (traceStats.size > 2_000) {
+            const cutoff = Date.now() - TRACE_STATS_TTL_MS;
+            for (const [id, st] of traceStats)
+                if (st.startedAt < cutoff)
+                    traceStats.delete(id);
+        }
+    }
+    return s;
+}
+/** Read-and-forget the stats for a finished trace. */
+export function takeTraceStats(traceId) {
+    const s = traceStats.get(traceId);
+    traceStats.delete(traceId);
+    return s;
+}
+function spanDurationMs(span) {
+    return span.duration[0] * 1000 + span.duration[1] / 1e6;
+}
+class SpanLineageProcessor {
+    onStart(span) {
+        const ctx = span.spanContext();
+        const parent = span.parentSpanContext?.spanId ?? span.parentSpanId ?? undefined;
+        spanLineage.set(ctx.spanId, { name: span.name, parent });
+        // ADK's call_llm span (the one carrying the full request/response
+        // payloads) learns its agent the same way llm.request does, so a
+        // payload row can be attributed without re-walking the tree later.
+        if (span.name === 'call_llm') {
+            const agent = agentForSpan(parent);
+            if (agent)
+                span.setAttribute('llm.agent', agent);
+        }
+        if (spanLineage.size > SPAN_LINEAGE_CAP) {
+            // A runaway process must never grow this without bound; parents end
+            // after their children, so evicting the oldest entry is safe.
+            const oldest = spanLineage.keys().next().value;
+            if (oldest !== undefined)
+                spanLineage.delete(oldest);
+        }
+    }
+    onEnd(span) {
+        spanLineage.delete(span.spanContext().spanId);
+        const traceId = span.spanContext().traceId;
+        if (span.name === 'llm.request') {
+            const s = statsFor(traceId);
+            s.modelMs += spanDurationMs(span);
+            s.llmCalls += 1;
+            const model = span.attributes['llm.model'];
+            if (typeof model === 'string' && model)
+                s.models.add(model);
+        }
+        else if (span.name.startsWith('execute_tool ')) {
+            statsFor(traceId).toolMs += spanDurationMs(span);
+        }
+    }
+    shutdown() { return Promise.resolve(); }
+    forceFlush() { return Promise.resolve(); }
+}
+/** Name of the nearest enclosing ADK `invoke_agent` span, or null. */
+export function agentForSpan(spanId) {
+    let cursor = spanId;
+    for (let depth = 0; cursor && depth < 12; depth++) {
+        const entry = spanLineage.get(cursor);
+        if (!entry)
+            return null;
+        if (entry.name.startsWith('invoke_agent '))
+            return entry.name.slice('invoke_agent '.length);
+        cursor = entry.parent;
+    }
+    return null;
+}
+/**
+ * Hands the provider to ADK's private copy of @opentelemetry/api.
+ *
+ * WHY: @google/adk pins its own @opentelemetry/api (nested under its
+ * node_modules) and creates its tracer at module-import time —
+ * `trace.getTracer("gcp.vertex.agent")` in telemetry/tracing.js. That call
+ * runs BEFORE any entrypoint can register a provider, so it binds to that
+ * copy's ProxyTracerProvider, whose delegate is only ever set by a
+ * `setGlobalTracerProvider` call made through the SAME copy. Ours goes
+ * through the top-level copy; the global registration is shared (it lives
+ * on a well-known Symbol), but the proxy's delegate is not. Result: every
+ * ADK span — invoke_agent, call_llm, execute_tool — was a no-op, and
+ * llm.request spans had no parent. Setting the delegate on ADK's proxy
+ * fixes both: ADK spans now export, and the context manager (global,
+ * shared) threads our llm.request spans under ADK's call_llm.
+ *
+ * `_proxyTracerProvider` is a private field of TraceAPI; this is the only
+ * place that touches it, it is guarded, and a failure just means ADK spans
+ * stay invisible — as they were.
+ */
+function adoptAdkTracerApi(provider) {
+    try {
+        const requireHere = createRequire(import.meta.url);
+        const adkEntry = requireHere.resolve('@google/adk');
+        const adkApi = createRequire(adkEntry)('@opentelemetry/api');
+        if (adkApi?.trace === trace)
+            return true;
+        const proxy = adkApi?.trace?._proxyTracerProvider;
+        if (proxy && typeof proxy.setDelegate === 'function') {
+            proxy.setDelegate(provider);
+            return true;
+        }
+    }
+    catch {
+        /* ADK not installed or its api copy changed shape — see WHY above */
+    }
+    return false;
+}
 export function initializeTracing() {
     if (isOtelInitialized)
         return;
@@ -70,6 +251,7 @@ export function initializeTracing() {
     // Since we register this globally, any internal ADK spans (e.g. tool calls, model calls)
     // will also automatically route to this provider and show up in the console.
     const spanProcessors = [
+        new SpanLineageProcessor(),
         new SimpleSpanProcessor(new JsonConsoleExporter()),
     ];
     // Optional durable sink (TELEMETRY_SUPABASE=true + Supabase credentials):
@@ -82,12 +264,27 @@ export function initializeTracing() {
         spanProcessors.push(new BatchSpanProcessor(telemetryExporter));
         console.log('[TELEMETRY] Supabase sink enabled → adk_telemetry');
     }
-    const provider = new NodeTracerProvider({ spanProcessors });
+    const otlp = otlpProcessor();
+    if (otlp)
+        spanProcessors.push(otlp);
+    let resource;
+    try {
+        const res = createRequire(import.meta.url)('@opentelemetry/resources');
+        const make = res.resourceFromAttributes ?? res.Resource?.default?.bind(res.Resource);
+        resource = make ? make({ 'service.name': process.env.OTEL_SERVICE_NAME ?? 'melchizedek' }) : undefined;
+    }
+    catch {
+        resource = undefined;
+    }
+    const provider = new NodeTracerProvider({ spanProcessors, ...(resource ? { resource } : {}) });
     provider.register();
+    adoptAdkTracerApi(provider);
     tracerProvider = provider;
     isOtelInitialized = true;
 }
 const tracer = trace.getTracer('melchizedek-tracer');
+/** Turn identity by trace id, so llm.request children can carry it too. */
+const turnContexts = new Map();
 /**
  * Wraps an ADK `Runner.runAsync` stream with a root OpenTelemetry span.
  * Accumulates token counts, records variables (bindings) and the raw input/output.
@@ -95,16 +292,53 @@ const tracer = trace.getTracer('melchizedek-tracer');
 export async function* traceAgentRun(stream, metadata) {
     initializeTracing();
     const span = tracer.startSpan(`Syndicate Execution: ${metadata.syndicateName}`);
+    turnContexts.set(span.spanContext().traceId, {
+        sessionId: metadata.sessionId,
+        userId: metadata.userId,
+        taskId: metadata.taskId,
+    });
     if (metadata.bindings) {
         span.setAttribute('syndicate.bindings', JSON.stringify(metadata.bindings));
     }
     if (metadata.input) {
         span.setAttribute('syndicate.input', JSON.stringify(metadata.input));
     }
+    if (metadata.route) {
+        span.setAttribute('syndicate.route', metadata.route.route);
+        if (metadata.route.reason)
+            span.setAttribute('syndicate.route.reason', metadata.route.reason);
+        span.setAttribute('syndicate.route.fell_back', metadata.route.fellBack === true);
+        span.setAttribute('syndicate.route.via_override', metadata.route.viaOverride === true);
+    }
+    if (metadata.sessionId) {
+        span.setAttribute('session.id', metadata.sessionId);
+        span.setAttribute('gen_ai.conversation.id', metadata.sessionId);
+    }
+    if (metadata.userId)
+        span.setAttribute('user.id', metadata.userId);
+    if (metadata.taskId)
+        span.setAttribute('a2a.task_id', metadata.taskId);
+    if (metadata.stage)
+        span.setAttribute('syndicate.stage', metadata.stage);
+    if (metadata.configHash)
+        span.setAttribute('syndicate.config_hash', metadata.configHash);
+    span.setAttribute('engine.version', engineVersion());
+    span.setAttribute('telemetry.schema_version', TELEMETRY_SCHEMA_VERSION);
+    for (const [key, value] of Object.entries(metadata.attributes ?? {})) {
+        span.setAttribute(key, value);
+    }
+    metadata.onSpanStart?.({
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+    });
     let totalInputTokens = 0;
     let totalOutputTokens = 0;
     let totalThinkingTokens = 0;
     let outputText = '';
+    let invocationId = '';
+    let respondingAgent = '';
+    let errorCode = '';
+    let errorMessage = '';
     const ctx = trace.setSpan(context.active(), span);
     try {
         const wrappedStream = {
@@ -117,6 +351,13 @@ export async function* traceAgentRun(stream, metadata) {
             }
         };
         for await (const event of wrappedStream) {
+            const ev = event;
+            if (!invocationId && typeof ev.invocationId === 'string')
+                invocationId = ev.invocationId;
+            if ((ev.errorCode || ev.errorMessage) && ev.errorCode !== 'STOP') {
+                errorCode = String(ev.errorCode ?? 'ERROR');
+                errorMessage = String(ev.errorMessage ?? '');
+            }
             if (event.usageMetadata) {
                 // Aggregation caveat: a multi-turn run repeats the growing prompt on
                 // every model call, so Math.max under-reports true billed input. The
@@ -135,8 +376,13 @@ export async function* traceAgentRun(stream, metadata) {
             if (event.content && event.content.parts) {
                 for (const part of event.content.parts) {
                     const p = part;
-                    if (p.text && !p.thought) {
+                    // Under SSE the reply streams as partials and is then repeated
+                    // whole on the final event. Count it once, from the final, or
+                    // syndicate.output records every answer twice.
+                    if (p.text && !p.thought && !ev.partial) {
                         outputText += p.text;
+                        if (ev.author)
+                            respondingAgent = String(ev.author);
                     }
                     if (p.functionCall) {
                         span.addEvent('ToolCall', {
@@ -169,10 +415,66 @@ export async function* traceAgentRun(stream, metadata) {
         span.setAttribute('syndicate.tokens.output', totalOutputTokens);
         span.setAttribute('syndicate.tokens.thinking', totalThinkingTokens);
         span.setAttribute('syndicate.output', outputText);
+        if (invocationId)
+            span.setAttribute('adk.invocation_id', invocationId);
+        if (respondingAgent)
+            span.setAttribute('syndicate.agent', respondingAgent);
+        if (errorCode) {
+            span.setAttribute('syndicate.error.code', errorCode);
+            span.setAttribute('syndicate.error.message', errorMessage.slice(0, 4000));
+        }
+        turnContexts.delete(span.spanContext().traceId);
+        const stats = takeTraceStats(span.spanContext().traceId);
+        if (stats) {
+            span.setAttribute('syndicate.latency.model_ms', Math.round(stats.modelMs));
+            span.setAttribute('syndicate.latency.tool_ms', Math.round(stats.toolMs));
+            span.setAttribute('syndicate.llm_calls', stats.llmCalls);
+            span.setAttribute('syndicate.models', [...stats.models].sort().join(','));
+        }
+        try {
+            for (const [key, value] of Object.entries(metadata.onEnd?.() ?? {})) {
+                if (value !== undefined)
+                    span.setAttribute(key, value);
+            }
+        }
+        catch {
+            /* a consumer's onEnd must never break the span */
+        }
         span.end();
     }
 }
 const THINKING_EVENT_MAX_CHARS = 600;
+const ERROR_PAYLOAD_MAX_CHARS = 200_000;
+const ERROR_MESSAGE_MAX_CHARS = 4_000;
+/** JSON.stringify that never throws and never exceeds the cap. */
+function safePayloadJson(value) {
+    let json;
+    try {
+        json = JSON.stringify(value) ?? String(value);
+    }
+    catch {
+        json = String(value);
+    }
+    return json.length > ERROR_PAYLOAD_MAX_CHARS
+        ? json.slice(0, ERROR_PAYLOAD_MAX_CHARS)
+        : json;
+}
+/**
+ * Providers throw errors whose message IS the API's JSON error body
+ * (Gemini: {"error":{"code":503,"message":...,"status":...}}). Pull the
+ * numeric code out so llm.error_code reads "503", not "THROWN".
+ */
+function refineErrorCode(code, message) {
+    try {
+        const parsed = JSON.parse(message);
+        if (parsed?.error?.code)
+            return String(parsed.error.code);
+    }
+    catch {
+        /* not a JSON body — keep the code we have */
+    }
+    return code;
+}
 /**
  * Wraps one adapter `generateContentAsync` invocation in an `llm.request`
  * span — one span per model call, for EVERY provider. Records provider,
@@ -189,10 +491,33 @@ export async function* traceLlmGeneration(meta, inner) {
     const span = tracer.startSpan('llm.request');
     span.setAttribute('llm.provider', meta.provider);
     span.setAttribute('llm.model', meta.model);
+    // OpenTelemetry GenAI semantic conventions alongside the llm.* names, so
+    // an OTLP backend (Phoenix, Langfuse, Tempo) reads these spans natively.
+    span.setAttribute('gen_ai.system', meta.provider);
+    span.setAttribute('gen_ai.request.model', meta.model);
+    const agent = agentForSpan(span.spanContext().spanId);
+    if (agent) {
+        span.setAttribute('llm.agent', agent);
+        span.setAttribute('gen_ai.agent.name', agent);
+    }
+    const turn = turnContexts.get(span.spanContext().traceId);
+    if (turn) {
+        if (turn.sessionId) {
+            span.setAttribute('session.id', turn.sessionId);
+            span.setAttribute('gen_ai.conversation.id', turn.sessionId);
+        }
+        if (turn.userId)
+            span.setAttribute('user.id', turn.userId);
+        if (turn.taskId)
+            span.setAttribute('a2a.task_id', turn.taskId);
+    }
     let inputTokens = 0;
     let outputTokens = 0;
     let thinkingTokens = 0;
     let thinkingPreview = '';
+    let errorCode = '';
+    let errorMessage = '';
+    let errorResponse;
     const ctx = trace.setSpan(context.active(), span);
     try {
         while (true) {
@@ -207,7 +532,9 @@ export async function* traceLlmGeneration(meta, inner) {
                 thinkingTokens = resp.usageMetadata.thoughtsTokenCount ?? thinkingTokens;
             }
             if (resp.errorCode) {
-                span.setAttribute('llm.error_code', resp.errorCode);
+                errorCode = String(resp.errorCode);
+                errorMessage = String(resp.errorMessage ?? '');
+                errorResponse = resp;
             }
             for (const part of resp.content?.parts ?? []) {
                 const p = part;
@@ -220,15 +547,32 @@ export async function* traceLlmGeneration(meta, inner) {
     }
     catch (error) {
         span.recordException(error);
+        // Gemini (and any adapter that throws instead of yielding an error
+        // response) lands here; the message is often the raw API error body.
+        errorMessage = error instanceof Error ? error.message : String(error);
+        errorCode = refineErrorCode(errorCode || 'THROWN', errorMessage);
+        errorResponse = { errorCode, errorMessage };
         throw error;
     }
     finally {
         if (thinkingPreview) {
             span.addEvent('llm.thinking', { 'thinking.preview': thinkingPreview });
         }
+        if (errorCode) {
+            span.setAttribute('llm.error_code', errorCode);
+            span.setAttribute('llm.error_message', errorMessage.slice(0, ERROR_MESSAGE_MAX_CHARS));
+            // Make this span a payload candidate (see supabaseSpanExporter's
+            // isPayloadSpan): the request as sent and the error as received.
+            if (meta.llmRequest !== undefined) {
+                span.setAttribute('llm.payload.request', safePayloadJson(meta.llmRequest));
+            }
+            span.setAttribute('llm.payload.response', safePayloadJson(errorResponse ?? { errorCode, errorMessage }));
+        }
         span.setAttribute('llm.tokens.input', inputTokens);
         span.setAttribute('llm.tokens.output', outputTokens);
         span.setAttribute('llm.tokens.thinking', thinkingTokens);
+        span.setAttribute('gen_ai.usage.input_tokens', inputTokens);
+        span.setAttribute('gen_ai.usage.output_tokens', outputTokens);
         // llm.latency_ms mirrors the span duration for the flat adk_telemetry row.
         span.end();
     }

package/dist/scripts/a2a_server.d.ts

@@ -1,2 +1,12 @@
 #!/usr/bin/env node
+/** One human-readable line naming WHY a turn failed, for the failed task's
+ *  status message — the text the Discord surface renders verbatim. Provider
+ *  errors sometimes arrive as a raw JSON blob (Gemini ApiError:
+ *  `{"error":{"code":503,"message":"...","status":"UNAVAILABLE"}}`), so the
+ *  inner message is dug out before falling back to the string itself. An
+ *  empty message keeps the old pointer at the server logs. */
+export declare function describeTurnError(error: {
+    code: string;
+    message: string;
+}): string;
 export declare function startServer(syndicateName?: string): Promise<void>;