melchizedek-agents 1.7.0 → 1.10.0

package/dist/lib/observability/supabaseSpanExporter.js

@@ -13,7 +13,11 @@
  *                    tool calls WITH their responses, the ids that join it to
  *                    the session (session_id, invocation_id) and the task,
  *                    and provenance (config_hash, engine_version).
- *     adk_telemetry  one row per span — llm.request and root spans, raw.
+ *     adk_telemetry  one row per span the ledger keeps — the turn's root, each
+ *                    agent run, model step and provider call, tool call,
+ *                    script run and call, workflow walk and node run — each
+ *                    with its parent among them and its kind, so a turn's
+ *                    span tree is stored (ADR 0139).
  *     adk_payloads   full request/response per model call, captured by
  *                    POLICY and expiring. ADK's own call_llm spans carry the
  *                    assembled prompt and raw response, and the native
@@ -41,7 +45,7 @@ import { join } from 'node:path';
 import core from '@opentelemetry/core';
 const { ExportResultCode } = core;
 import { hasSupabaseCredentials } from '../persistence/supabaseProvider.js';
-import { TELEMETRY_SCHEMA_VERSION, isModelCallSpan } from './lineage.js';
+import { LEDGER_PARENT_ATTRIBUTE, TELEMETRY_SCHEMA_VERSION, isModelCallSpan, ledgerSpanKind } from './lineage.js';
 import { redactRow, telemetryRedactor } from './redact.js';
 import { outputsDir } from '../paths.js';
 /**
@@ -81,8 +85,17 @@ export function payloadDecision(policy, traceId, facts) {
 }
 // ── Span → row mapping (pure; tested offline) ───────────────────────────────
 const ROOT_PREFIX = 'Syndicate Execution: ';
-// A script's own spans (ADR 0137 item 16) carry its run, its calls' kinds, names, sequence and status, never an argument or a result.
-const EXPORTED_SPAN_NAMES = /^(llm\.request$|Syndicate Execution: |script\.(invoke|call) )/;
+/**
+ * The spans that get an adk_telemetry row: every span ledgerSpanKind names
+ * (ADR 0139). A model step's row (`model.call`) leaves its payload to
+ * adk_payloads (spanJson drops `llm.payload.*`) and its tokens to its
+ * llm.request rows, so a trace's rows sum without counting a call twice. A
+ * script's own spans (ADR 0137 item 16) carry its run, its calls' kinds,
+ * names, sequence and status, never an argument or a result.
+ */
+export function isLedgerSpan(span) {
+    return ledgerSpanKind(span.name) !== null;
+}
 function attrsOf(span) {
     return span.attributes;
 }
@@ -169,6 +182,25 @@ function spanJson(span) {
         status: span.status,
     };
 }
+/** The span's parent in the ledger: the ancestor the tracer recorded at start, else its direct parent. */
+export function ledgerParentOf(span) {
+    const recorded = attrsOf(span)[LEDGER_PARENT_ATTRIBUTE];
+    if (typeof recorded === 'string' && recorded)
+        return recorded;
+    const s = span;
+    const parent = s.parentSpanContext?.spanId ?? s.parentSpanId;
+    return typeof parent === 'string' && parent ? parent : null;
+}
+/** Whether a span failed: its status, an error code it records, or a tool's error response. */
+export function spanFailed(span) {
+    const attrs = attrsOf(span);
+    return (span.status?.code === 2 || // SpanStatusCode.ERROR
+        !!str(attrs, 'llm.error_code') ||
+        !!str(attrs, 'syndicate.error.code') ||
+        attrs['tool.error'] === true ||
+        attrs['script.call.status'] === 'error' ||
+        attrs['adk.node.status'] === 'failed');
+}
 /** Span attributes minus the payload capture (`llm.payload.*`). */
 export function withoutPayloadAttributes(attrs) {
     const out = {};
@@ -177,7 +209,7 @@ export function withoutPayloadAttributes(attrs) {
             out[k] = v;
     return out;
 }
-/** Flatten an llm.request or root span into one adk_telemetry row. */
+/** Flatten a span the ledger keeps (isLedgerSpan) into one adk_telemetry row. */
 export function toTelemetryRow(span) {
     const attrs = attrsOf(span);
     return {
@@ -192,11 +224,16 @@ export function toTelemetryRow(span) {
         input_tokens: num(attrs, 'llm.tokens.input') ?? num(attrs, 'syndicate.tokens.input'),
         output_tokens: num(attrs, 'llm.tokens.output') ?? num(attrs, 'syndicate.tokens.output'),
         thinking_tokens: num(attrs, 'llm.tokens.thinking') ?? num(attrs, 'syndicate.tokens.thinking'),
+        cache_read_tokens: num(attrs, 'llm.tokens.cache_read') ?? num(attrs, 'syndicate.tokens.cache_read'),
+        cache_write_tokens: num(attrs, 'llm.tokens.cache_write') ?? num(attrs, 'syndicate.tokens.cache_write'),
         latency_ms: durationMs(span),
         session_id: str(attrs, 'session.id'),
         user_id: str(attrs, 'user.id'),
         task_id: str(attrs, 'a2a.task_id'),
         invocation_id: str(attrs, 'adk.invocation_id'),
+        parent_span_id: ledgerParentOf(span),
+        kind: ledgerSpanKind(span.name),
+        error: spanFailed(span),
         schema_version: TELEMETRY_SCHEMA_VERSION,
         span: spanJson(span),
     };
@@ -241,6 +278,8 @@ export function toTurnRow(span) {
         input_tokens: num(attrs, 'syndicate.tokens.input'),
         output_tokens: num(attrs, 'syndicate.tokens.output'),
         thinking_tokens: num(attrs, 'syndicate.tokens.thinking'),
+        cache_read_tokens: num(attrs, 'syndicate.tokens.cache_read'),
+        cache_write_tokens: num(attrs, 'syndicate.tokens.cache_write'),
         latency_ms: durationMs(span),
         model_ms: num(attrs, 'syndicate.latency.model_ms'),
         tool_ms: num(attrs, 'syndicate.latency.tool_ms'),
@@ -371,7 +410,7 @@ export class SupabaseSpanExporter {
         const turnRows = [];
         const payloadRows = [];
         for (const span of spans) {
-            if (EXPORTED_SPAN_NAMES.test(span.name))
+            if (isLedgerSpan(span))
                 telemetryRows.push(toTelemetryRow(span));
             if (isRootSpan(span)) {
                 const turn = toTurnRow(span);

package/dist/lib/observability/tracer.d.ts

@@ -2,7 +2,7 @@ import type { Context, Span, SpanContext, SpanOptions } from '@opentelemetry/api
 import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import type { LlmResponse } from '../models/genaiMapping.ts';
 import type { TurnEvent as Event } from '../runtime/events.ts';
-import type { ModelRequest } from '../models/contract.ts';
+import type { ModelRequest, ModelResponse } from '../models/contract.ts';
 import type { ExporterOptions, LedgerClient } from './supabaseSpanExporter.ts';
 /**
  * Optional OTLP export for a live trace viewer (Phoenix, Langfuse, Grafana
@@ -45,6 +45,9 @@ interface TraceStats {
     toolMs: number;
     llmCalls: number;
     models: Set<string>;
+    /** Cache tokens summed over every call (`syndicate.tokens.cache_read` / `.cache_write`). */
+    cacheRead: number;
+    cacheWrite: number;
     /** Each model's calls and tokens, input summed per call (what a provider bills), for `syndicate.usage` (ADR 0136). */
     usage: Map<string, ModelUsage>;
     startedAt: number;
@@ -52,9 +55,13 @@ interface TraceStats {
 /** One model's share of a turn: what the ledger prices it from (model_pricing, ADR 0136). */
 export interface ModelUsage {
     calls: number;
+    /** Every input token, the cache's included (the contract's meaning, lib/models/contract.ts). */
     input: number;
     output: number;
     thinking: number;
+    /** The parts of `input` read from and written to the provider's prompt cache, priced at their own rates (ADR 0139). */
+    cache_read: number;
+    cache_write: number;
 }
 /** Read-and-forget the stats for a finished trace. */
 export declare function takeTraceStats(traceId: string): TraceStats | undefined;
@@ -171,6 +178,15 @@ export interface LlmCallMeta {
      * for an adapter written before the model contract.
      */
     llmRequest?: unknown;
+    /**
+     * The contract response a yielded LlmResponse was mapped from, when the
+     * caller holds it (the native step, the summarizer, the memory extractor).
+     * The span reads what an LlmResponse cannot carry from it: the cache
+     * write count and the contract's finish reason. Without it the span
+     * records the cache reads Gemini's usage names and no finish reason
+     * unless the call failed.
+     */
+    source?: (response: LlmResponse) => ModelResponse | undefined;
 }
 /**
  * Wraps one adapter `generateContentAsync` invocation in an `llm.request`

package/dist/lib/observability/tracer.js

@@ -1,6 +1,6 @@
 import { trace, context, defaultTextMapGetter, ROOT_CONTEXT, INVALID_SPAN_CONTEXT } from '@opentelemetry/api';
 import { createRequire } from 'node:module';
-import { ADK_SPAN_SCOPE, RUNTIME_SPAN_SCOPE, TELEMETRY_SCHEMA_VERSION, agentOfSpanName, engineVersion, isModelCallSpan, isToolSpanName } from './lineage.js';
+import { ADK_SPAN_SCOPE, LEDGER_PARENT_ATTRIBUTE, RUNTIME_SPAN_SCOPE, TELEMETRY_SCHEMA_VERSION, agentOfSpanName, engineVersion, isModelCallSpan, isToolSpanName, ledgerSpanKind } from './lineage.js';
 import { FilteringSpanExporter, otlpContentMode } from './otlpFilter.js';
 import { telemetryRedactor } from './redact.js';
 import { chargeLlmCall, chargeTokens, turnUntraced } from '../runtime/turnControl.js';
@@ -194,7 +194,7 @@ const TRACE_STATS_TTL_MS = 60 * 60 * 1000;
 function statsFor(traceId) {
     let s = traceStats.get(traceId);
     if (!s) {
-        s = { modelMs: 0, toolMs: 0, llmCalls: 0, models: new Set(), usage: new Map(), startedAt: Date.now() };
+        s = { modelMs: 0, toolMs: 0, llmCalls: 0, models: new Set(), cacheRead: 0, cacheWrite: 0, usage: new Map(), startedAt: Date.now() };
         traceStats.set(traceId, s);
         if (traceStats.size > 2_000) {
             const cutoff = Date.now() - TRACE_STATS_TTL_MS;
@@ -219,6 +219,15 @@ class SpanLineageProcessor {
         const ctx = span.spanContext();
         const parent = span.parentSpanContext?.spanId ?? span.parentSpanId ?? undefined;
         spanLineage.set(ctx.spanId, { name: span.name, parent });
+        // The ledger's parent is the nearest ancestor it keeps a row for, read
+        // now: an ancestor's entry is gone once it ends, and the exporter sees
+        // the span only after that (ADR 0139). Recorded only when it differs
+        // from the direct parent, which the span carries itself.
+        if (ledgerSpanKind(span.name) !== null) {
+            const ledgerParent = ledgerAncestor(parent);
+            if (ledgerParent && ledgerParent !== parent)
+                span.setAttribute(LEDGER_PARENT_ATTRIBUTE, ledgerParent);
+        }
         // The model-call span (the loop's model.call, ADK's call_llm before 1.0.0: the one
         // carrying the full request/response payloads) learns its agent the same
         // way llm.request does, so a payload row can be attributed without
@@ -250,13 +259,21 @@ class SpanLineageProcessor {
                     const v = span.attributes[key];
                     return typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0;
                 };
-                const u = s.usage.get(model) ?? { calls: 0, input: 0, output: 0, thinking: 0 };
+                const u = s.usage.get(model) ?? { calls: 0, input: 0, output: 0, thinking: 0, cache_read: 0, cache_write: 0 };
                 u.calls += 1;
                 u.input += tokens('llm.tokens.input');
                 u.output += tokens('llm.tokens.output');
                 u.thinking += tokens('llm.tokens.thinking');
+                u.cache_read += tokens('llm.tokens.cache_read');
+                u.cache_write += tokens('llm.tokens.cache_write');
                 s.usage.set(model, u);
             }
+            const cache = (key) => {
+                const v = span.attributes[key];
+                return typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0;
+            };
+            s.cacheRead += cache('llm.tokens.cache_read');
+            s.cacheWrite += cache('llm.tokens.cache_write');
         }
         else if (isToolSpanName(span.name)) {
             statsFor(traceId).toolMs += spanDurationMs(span);
@@ -265,6 +282,23 @@ class SpanLineageProcessor {
     shutdown() { return Promise.resolve(); }
     forceFlush() { return Promise.resolve(); }
 }
+/**
+ * The nearest of `spanId` and its ancestors that the ledger keeps a row for
+ * (ledgerSpanKind), or `spanId` itself when the chain leaves the lineage
+ * (a remote parent, an evicted entry) or holds no such span.
+ */
+function ledgerAncestor(spanId) {
+    let cursor = spanId;
+    for (let depth = 0; cursor && depth < 32; depth++) {
+        const entry = spanLineage.get(cursor);
+        if (!entry)
+            return depth === 0 ? spanId : cursor;
+        if (ledgerSpanKind(entry.name) !== null)
+            return cursor;
+        cursor = entry.parent;
+    }
+    return spanId;
+}
 /** Name of the agent of the nearest enclosing agent span (ADK's `invoke_agent`, the loop's `agent.invoke`), or null. */
 export function agentForSpan(spanId) {
     let cursor = spanId;
@@ -557,6 +591,9 @@ export async function* traceAgentRun(stream, metadata) {
             span.setAttribute('syndicate.latency.tool_ms', Math.round(stats.toolMs));
             span.setAttribute('syndicate.llm_calls', stats.llmCalls);
             span.setAttribute('syndicate.models', [...stats.models].sort().join(','));
+            // Summed over every call, as a provider bills them (adk_turns.cache_read_tokens / cache_write_tokens).
+            span.setAttribute('syndicate.tokens.cache_read', stats.cacheRead);
+            span.setAttribute('syndicate.tokens.cache_write', stats.cacheWrite);
             // The ledger prices the turn from this, per model (melchizedek_usage_cost, ADR 0136).
             span.setAttribute('syndicate.usage', JSON.stringify(Object.fromEntries([...stats.usage].sort(([a], [b]) => a.localeCompare(b)))));
         }
@@ -673,6 +710,9 @@ export async function* traceLlmGeneration(meta, inner) {
     let inputTokens = 0;
     let outputTokens = 0;
     let thinkingTokens = 0;
+    let cacheRead;
+    let cacheWrite;
+    let finishReason;
     let thinkingPreview = '';
     let errorCode = '';
     let errorMessage = '';
@@ -689,6 +729,15 @@ export async function* traceLlmGeneration(meta, inner) {
                 inputTokens = resp.usageMetadata.promptTokenCount ?? inputTokens;
                 outputTokens = resp.usageMetadata.candidatesTokenCount ?? outputTokens;
                 thinkingTokens = resp.usageMetadata.thoughtsTokenCount ?? thinkingTokens;
+                cacheRead = resp.usageMetadata.cachedContentTokenCount ?? cacheRead;
+            }
+            const source = meta.source?.(resp);
+            if (source && !source.partial) {
+                finishReason = source.finishReason;
+                if (source.usage?.cacheReadTokens !== undefined)
+                    cacheRead = source.usage.cacheReadTokens;
+                if (source.usage?.cacheWriteTokens !== undefined)
+                    cacheWrite = source.usage.cacheWriteTokens;
             }
             if (resp.errorCode) {
                 errorCode = String(resp.errorCode);
@@ -733,6 +782,15 @@ export async function* traceLlmGeneration(meta, inner) {
         span.setAttribute('llm.tokens.input', inputTokens);
         span.setAttribute('llm.tokens.output', outputTokens);
         span.setAttribute('llm.tokens.thinking', thinkingTokens);
+        // Parts of llm.tokens.input, set only when the provider reported them (ADR 0139).
+        if (cacheRead !== undefined)
+            span.setAttribute('llm.tokens.cache_read', cacheRead);
+        if (cacheWrite !== undefined)
+            span.setAttribute('llm.tokens.cache_write', cacheWrite);
+        if (errorCode)
+            finishReason = 'error';
+        if (finishReason)
+            span.setAttribute('llm.finish_reason', finishReason);
         span.setAttribute('gen_ai.usage.input_tokens', inputTokens);
         span.setAttribute('gen_ai.usage.output_tokens', outputTokens);
         // llm.latency_ms mirrors the span duration for the flat adk_telemetry row.

package/dist/lib/project.d.ts

@@ -0,0 +1,116 @@
+/**
+ * lib/project.ts — the project manifest, `melchizedek.json` (ADR 0138).
+ *
+ * A project is a folder, usually a git repository, with a `melchizedek.json`
+ * at its root. The manifest records what the engine cannot infer from the
+ * files: which folder holds the syndicates, which engine version the project
+ * is written for, and how each file ships. It holds names, never values: an
+ * `env` entry is a variable name, and no field may carry a secret.
+ *
+ * `findProject()` walks up from a directory to the nearest manifest and stops
+ * at the repository root (the first directory holding `.git`) or the home
+ * directory, so a manifest outside the repository never steers it.
+ * `resolveAgentsDir()` is the one agents-directory chain every loader and bin
+ * uses: explicit option → MELCHIZEDEK_AGENTS_DIR → the manifest's `agents` →
+ * <cwd>/config/agents. A deployed server sets the variable, so the manifest
+ * only changes what a bare command run inside a project finds.
+ *
+ * Imports only node built-ins, zod and lib/paths.ts.
+ */
+import { z } from 'zod';
+export declare const PROJECT_MANIFEST = "melchizedek.json";
+export declare const MANIFEST_VERSION = 1;
+export declare const projectManifestSchema: z.ZodObject<{
+    manifest: z.ZodNumber;
+    engine: z.ZodString;
+    agents: z.ZodString;
+    ignore: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    tests: z.ZodOptional<z.ZodString>;
+    env: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    syndicates: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+        deliverable: z.ZodOptional<z.ZodEnum<{
+            library: "library";
+            mcp: "mcp";
+            service: "service";
+        }>>;
+        publish: z.ZodOptional<z.ZodObject<{
+            kind: z.ZodLiteral<"registry">;
+            id: z.ZodString;
+        }, z.core.$loose>>;
+    }, z.core.$loose>>>;
+    registry: z.ZodOptional<z.ZodObject<{
+        kind: z.ZodLiteral<"supabase">;
+        url_env: z.ZodOptional<z.ZodString>;
+        restart: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
+    production: z.ZodOptional<z.ZodObject<{
+        ledger: z.ZodOptional<z.ZodObject<{
+            kind: z.ZodLiteral<"postgres">;
+            url_env: z.ZodString;
+        }, z.core.$loose>>;
+    }, z.core.$loose>>;
+    tools: z.ZodOptional<z.ZodObject<{
+        remote: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+            mcp_env: z.ZodString;
+        }, z.core.$loose>>>;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
+export type ProjectManifest = z.infer<typeof projectManifestSchema>;
+export interface Project {
+    /** The folder holding `melchizedek.json`. */
+    root: string;
+    /** Absolute path of the manifest. */
+    file: string;
+    manifest: ProjectManifest;
+    /** `agents`, resolved under the root. */
+    agentsDir: string;
+    /** `tests`, resolved under the root (default `tests/agents`). */
+    testsDir: string;
+}
+/** Thrown for a manifest that does not parse or validate; names the file and fields, never a value. */
+export declare class ProjectManifestError extends Error {
+    readonly file: string;
+    readonly issues: string[];
+    constructor(file: string, issues: string[]);
+}
+/**
+ * The nearest folder at or above `from` holding a manifest, without crossing
+ * the repository root or the home directory. Undefined when there is none.
+ */
+export declare function findProjectRoot(from?: string): string | undefined;
+/** Parse and validate a manifest's text. Throws `ProjectManifestError`. */
+export declare function parseProjectManifest(text: string, file?: string): ProjectManifest;
+/** Load the project rooted at `root` (the folder holding the manifest). Throws `ProjectManifestError`. */
+export declare function loadProject(root: string): Project;
+/** The project at or above `from`, or undefined when there is none. Throws on a broken manifest. */
+export declare function findProject(from?: string): Project | undefined;
+/** Forget cached manifest lookups (tests, and an embedder that edits the manifest in-process). */
+export declare function clearProjectCache(): void;
+/**
+ * The agents directory: `explicit` → MELCHIZEDEK_AGENTS_DIR → the nearest
+ * manifest's `agents` → <cwd>/config/agents. Always absolute.
+ */
+export declare function resolveAgentsDir(explicit?: string, env?: NodeJS.ProcessEnv): string;
+/** True when `relFromRoot` (a `/`-separated path relative to the project root) matches an `ignore` pattern. */
+export declare function isIgnored(project: Pick<Project, 'manifest'>, relFromRoot: string): boolean;
+/** The syndicate files directly in the agents folder, as names relative to it, minus `ignore`. */
+export declare function listProjectSyndicates(project: Project): string[];
+/** The installed engine's package version (no commit suffix). */
+export declare function enginePackageVersion(): string;
+export interface ProjectCheck {
+    root: string;
+    agentsDir: string;
+    /** The manifest's engine pin, and the engine running now. */
+    engine: {
+        pinned: string;
+        running: string;
+        matches: boolean;
+    };
+    syndicates: string[];
+    /** Stop `melchizedek-doctor --check`. */
+    problems: string[];
+    /** Shown, never fatal. */
+    warnings: string[];
+}
+/** What is wrong with a project: engine pin, missing folders, stale ignores, unset env names. Never reads a value. */
+export declare function checkProject(project: Project, running?: string, env?: NodeJS.ProcessEnv): ProjectCheck;