open-multi-agent-kit 0.79.1 → 0.80.0

package/dist/replay/inspector.js

@@ -0,0 +1,264 @@
+/**
+ * ReplayInspector — forensic inspection of a single run.
+ */
+import { readFile } from "fs/promises";
+import { join } from "path";
+import { createManifestBuilder } from "./manifest-builder.js";
+import { getRunPath } from "../util/run-store.js";
+import { style, status, label } from "../util/theme.js";
+export async function inspectRun(runId, options = {}) {
+    const builder = createManifestBuilder({ runsDir: options.runsDir });
+    const validation = await builder.validate(runId);
+    if (!validation.valid) {
+        console.error(status.error(`Run ${runId} is not replayable:`));
+        for (const err of validation.errors)
+            console.error(`  ${style.red("✕")} ${err}`);
+        process.exit(1);
+    }
+    const manifest = await builder.build(runId);
+    // Load deep forensic data
+    const evidenceEntries = options.evidence ? await loadEvidenceEntries(runId, options.runsDir) : [];
+    const allDecisions = options.decisions || options.repair ? await loadDecisions(runId, options.runsDir) : [];
+    if (options.json) {
+        const payload = { manifest };
+        if (options.nodeId) {
+            const node = manifest.nodes.find((n) => n.nodeId === options.nodeId);
+            payload.node = node ?? null;
+        }
+        if (options.attemptId) {
+            const attempt = manifest.nodes
+                .flatMap((n) => n.attempts)
+                .find((a) => a.attemptId === options.attemptId);
+            payload.attempt = attempt ?? null;
+        }
+        if (options.evidence)
+            payload.evidence = evidenceEntries;
+        if (options.decisions || options.repair)
+            payload.decisions = allDecisions;
+        console.log(JSON.stringify(payload, null, 2));
+        return;
+    }
+    if (options.nodeId) {
+        const node = manifest.nodes.find((n) => n.nodeId === options.nodeId);
+        if (!node) {
+            console.error(status.error(`Node ${options.nodeId} not found in run ${runId}`));
+            process.exit(1);
+        }
+        await renderNode(node, manifest.runId, options);
+        return;
+    }
+    if (options.attemptId) {
+        const attempt = manifest.nodes
+            .flatMap((n) => n.attempts)
+            .find((a) => a.attemptId === options.attemptId);
+        if (!attempt) {
+            console.error(status.error(`Attempt ${options.attemptId} not found in run ${runId}`));
+            process.exit(1);
+        }
+        await renderAttempt(attempt, manifest.runId, false, options);
+        return;
+    }
+    renderManifest(manifest, options);
+}
+function renderManifest(m, options = {}) {
+    console.log(style.purpleBold(`🔍 Replay Manifest — ${m.runId}`));
+    console.log(label("OMK Version", m.omkVersion));
+    console.log(label("Started", m.startedAt));
+    if (m.completedAt)
+        console.log(label("Completed", m.completedAt));
+    console.log(label("DAG Hash", m.dagHash));
+    console.log("");
+    console.log(style.pinkBold("Summary"));
+    console.log(`  Nodes:        ${m.summary.totalNodes}`);
+    console.log(`  Attempts:     ${m.summary.totalAttempts}`);
+    console.log(`  Success:      ${style.mint(String(m.summary.successCount))}`);
+    console.log(`  Failed:       ${m.summary.failureCount > 0 ? style.red(String(m.summary.failureCount)) : "0"}`);
+    console.log(`  Skipped:      ${m.summary.skippedCount}`);
+    console.log(`  Latency:      ${formatMs(m.summary.totalLatencyMs)}`);
+    console.log(`  Cost (est.):  $${m.summary.totalCostUsd.toFixed(4)}`);
+    console.log("");
+    console.log(style.pinkBold("Nodes"));
+    for (const node of m.nodes) {
+        const statusIcon = node.finalStatus === "success" ? style.mint("✓") : node.finalStatus === "skipped" ? style.gray("⊘") : style.red("✕");
+        console.log(`  ${statusIcon} ${style.cream(node.nodeId)} ${style.gray(`(${node.attempts.length} attempts, ${formatMs(node.totalLatencyMs)})`)}`);
+        for (const a of node.attempts) {
+            const aIcon = a.status === "success" ? style.mint("✓") : style.red("✕");
+            console.log(`    ${aIcon} ${style.gray(a.attemptId)} ${a.runtime}${a.model ? ` / ${a.model}` : ""} ${formatMs(a.latencyMs ?? 0)}`);
+            if (a.decisionTrace.length > 0) {
+                console.log(`      ${style.gray("decisions:")} ${a.decisionTrace.length}`);
+            }
+        }
+    }
+    // Forensic extras
+    if (options.decisions || options.repair) {
+        const repairDecisions = m.nodes.flatMap((n) => n.attempts.flatMap((a) => a.decisionTrace.filter((d) => d.component === "repair-policy")));
+        if (repairDecisions.length > 0) {
+            console.log("");
+            console.log(style.pinkBold("Repair Decisions"));
+            for (const d of repairDecisions) {
+                console.log(`  ${style.gray(d.at)} [${style.orange(d.nodeId ?? "?")}] ${d.outputDecision}`);
+                console.log(`    ${style.gray(d.reason)}`);
+            }
+        }
+    }
+}
+async function renderNode(node, runId, options = {}) {
+    console.log(style.purpleBold(`🔍 Node — ${node.nodeId}`));
+    console.log(label("Run", runId));
+    console.log(label("Final Status", node.finalStatus));
+    console.log(label("Attempts", String(node.attempts.length)));
+    console.log(label("Latency", formatMs(node.totalLatencyMs)));
+    console.log("");
+    for (const a of node.attempts) {
+        await renderAttempt(a, runId, /* compact */ true, options);
+        console.log("");
+    }
+}
+async function renderAttempt(a, runId, compact = false, options = {}) {
+    if (!compact)
+        console.log(style.purpleBold(`🔍 Attempt — ${a.attemptId}`));
+    else
+        console.log(style.creamBold(a.attemptId));
+    if (!compact)
+        console.log(label("Run", runId));
+    console.log(label("Runtime", a.runtime));
+    if (a.model)
+        console.log(label("Model", a.model));
+    if (a.provider)
+        console.log(label("Provider", a.provider));
+    console.log(label("Status", a.status));
+    console.log(label("Latency", formatMs(a.latencyMs ?? 0)));
+    console.log(label("Tokens", `${a.inputTokensEstimated} in / ${a.outputTokensEstimated} out`));
+    if (a.error)
+        console.log(label("Error", style.red(a.error)));
+    // Decision trace
+    if (a.decisionTrace.length > 0 && (options.decisions || options.repair || (!options.context && !options.evidence))) {
+        const traces = options.repair
+            ? a.decisionTrace.filter((d) => d.component === "repair-policy")
+            : a.decisionTrace;
+        if (traces.length > 0) {
+            console.log(style.gray("Decision Trace:"));
+            for (const d of traces) {
+                const scores = d.scores ? ` ${style.gray(JSON.stringify(d.scores))}` : "";
+                console.log(`  ${style.gray(d.at)} [${style.purple(d.component)}] ${d.outputDecision}${scores}`);
+                console.log(`    ${style.gray(d.reason)}`);
+            }
+        }
+    }
+    // Context capsule deep-dive
+    if (options.context) {
+        const capsule = await loadContextCapsule(runId, a.attemptId.split("__")[0] ?? "", a.attemptId, options.runsDir);
+        if (capsule) {
+            console.log(style.gray("Context Capsule:"));
+            const c = capsule;
+            console.log(`  ${style.gray("Tokens:")} ${c._snapshot?.estimatedTokens ?? "?"}`);
+            if (c.budget)
+                console.log(`  ${style.gray("Budget:")} ${c.budget.maxInputTokens} max-input, ${c.budget.compression}`);
+            if (c.goal)
+                console.log(`  ${style.gray("Goal:")} ${c.goal.slice(0, 120).replace(/\n/g, " ")}${c.goal.length > 120 ? "…" : ""}`);
+            if (c.relevantFiles?.length) {
+                console.log(`  ${style.gray("Files:")} ${c.relevantFiles.length}`);
+                for (const f of c.relevantFiles.slice(0, 5)) {
+                    console.log(`    ${style.cream(f.path)} ${style.gray(`(${f.content.length} chars)`)}`);
+                }
+                if (c.relevantFiles.length > 5)
+                    console.log(`    ${style.gray(`... and ${c.relevantFiles.length - 5} more`)}`);
+            }
+            if (c.graphMemory?.length) {
+                console.log(`  ${style.gray("Memory:")} ${c.graphMemory.length} facts`);
+                for (const m of c.graphMemory.slice(0, 3)) {
+                    console.log(`    ${style.cream(m.key)} = ${m.value.slice(0, 60)}${m.value.length > 60 ? "…" : ""}`);
+                }
+                if (c.graphMemory.length > 3)
+                    console.log(`    ${style.gray(`... and ${c.graphMemory.length - 3} more`)}`);
+            }
+        }
+        else {
+            console.log(style.gray("Context Capsule: (missing)"));
+        }
+    }
+    // Evidence results
+    if (options.evidence) {
+        const entries = await loadEvidenceEntriesForAttempt(runId, a.attemptId.split("__")[0] ?? "", a.attemptId, options.runsDir);
+        if (entries.length > 0) {
+            console.log(style.gray("Evidence Results:"));
+            for (const ev of entries) {
+                const icon = ev.passed ? style.mint("✓") : style.red("✕");
+                console.log(`  ${icon} ${ev.gate}`);
+                if (ev.message)
+                    console.log(`    ${style.gray(ev.message)}`);
+                if (ev.ref)
+                    console.log(`    ${style.gray("Ref: " + ev.ref)}`);
+            }
+        }
+        else {
+            console.log(style.gray("Evidence Results: (none recorded)"));
+        }
+    }
+}
+function formatMs(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    if (ms < 60_000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const m = Math.floor(ms / 60_000);
+    const s = Math.floor((ms % 60_000) / 1000);
+    return `${m}m${s}s`;
+}
+async function loadContextCapsule(runId, nodeId, attemptId, runsDir) {
+    const dir = getRunPath(runId, undefined, runsDir);
+    const path = join(dir, "context-capsules", `${nodeId}-${attemptId}.json`);
+    try {
+        const content = await readFile(path, "utf-8");
+        return JSON.parse(content);
+    }
+    catch {
+        return null;
+    }
+}
+async function loadEvidenceEntriesForAttempt(runId, nodeId, attemptId, runsDir) {
+    const dir = getRunPath(runId, undefined, runsDir);
+    const path = join(dir, "evidence.jsonl");
+    try {
+        const content = await readFile(path, "utf-8");
+        return content
+            .trim()
+            .split("\n")
+            .filter((l) => l.length > 0)
+            .map((line) => JSON.parse(line))
+            .filter((e) => e.nodeId === nodeId && e.attemptId === attemptId);
+    }
+    catch {
+        return [];
+    }
+}
+async function loadEvidenceEntries(runId, runsDir) {
+    const dir = getRunPath(runId, undefined, runsDir);
+    const path = join(dir, "evidence.jsonl");
+    try {
+        const content = await readFile(path, "utf-8");
+        return content
+            .trim()
+            .split("\n")
+            .filter((l) => l.length > 0)
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return [];
+    }
+}
+async function loadDecisions(runId, runsDir) {
+    const dir = getRunPath(runId, undefined, runsDir);
+    const path = join(dir, "decisions.jsonl");
+    try {
+        const content = await readFile(path, "utf-8");
+        return content
+            .trim()
+            .split("\n")
+            .filter((l) => l.length > 0)
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return [];
+    }
+}

package/dist/replay/manifest-builder.d.ts

@@ -0,0 +1,11 @@
+/**
+ * ReplayManifestBuilder — constructs a ReplayManifest from run artifacts.
+ */
+import type { ReplayManifest, ReplayValidationResult } from "../contracts/replay.js";
+export interface ManifestBuilderOptions {
+    runsDir?: string;
+}
+export declare function createManifestBuilder(options?: ManifestBuilderOptions): {
+    build: (runId: string) => Promise<ReplayManifest>;
+    validate: (runId: string) => Promise<ReplayValidationResult>;
+};

package/dist/replay/manifest-builder.js

@@ -0,0 +1,184 @@
+/**
+ * ReplayManifestBuilder — constructs a ReplayManifest from run artifacts.
+ */
+import { readFile } from "fs/promises";
+import { readFileSync } from "fs";
+import { join } from "path";
+import { createDecisionTraceStore } from "../evidence/decision-trace.js";
+import { validateRunId } from "../util/run-store.js";
+export function createManifestBuilder(options) {
+    const runsDir = options?.runsDir ?? ".omk/runs";
+    const decisionStore = createDecisionTraceStore(runsDir);
+    async function build(runId) {
+        const validRunId = validateRunId(runId);
+        const runDir = join(runsDir, validRunId);
+        // Load run meta
+        const metaRaw = await readFile(join(runDir, "run.json"), "utf-8").catch(() => null);
+        const meta = metaRaw ? JSON.parse(metaRaw) : {};
+        // Load DAG snapshot for hash (use content hash as proxy)
+        const dagRaw = await readFile(join(runDir, "dag.json"), "utf-8").catch(() => "{}");
+        const dagHash = hashString(dagRaw);
+        // Load attempts
+        const attemptsPath = join(runDir, "attempts.jsonl");
+        const attemptsRaw = await readFile(attemptsPath, "utf-8").catch(() => "");
+        const attempts = attemptsRaw
+            .trim()
+            .split("\n")
+            .filter((l) => l.length > 0)
+            .map((line) => JSON.parse(line));
+        // Load all decision traces
+        const allDecisions = decisionStore.load(runId);
+        // Group by node
+        const nodeMap = new Map();
+        for (const a of attempts) {
+            const list = nodeMap.get(a.nodeId) ?? [];
+            list.push(a);
+            nodeMap.set(a.nodeId, list);
+        }
+        const nodes = [];
+        let totalAttempts = 0;
+        let successCount = 0;
+        let failureCount = 0;
+        let skippedCount = 0;
+        let totalLatencyMs = 0;
+        let totalCostUsd = 0;
+        let totalInputTokens = 0;
+        let totalOutputTokens = 0;
+        for (const [nodeId, nodeAttempts] of nodeMap) {
+            // Sort by attempt number
+            nodeAttempts.sort((a, b) => {
+                const aNum = parseInt(a.attemptId.split("__").pop() ?? "0", 10);
+                const bNum = parseInt(b.attemptId.split("__").pop() ?? "0", 10);
+                return aNum - bNum;
+            });
+            const lastAttempt = nodeAttempts[nodeAttempts.length - 1];
+            const finalStatus = lastAttempt?.status === "success"
+                ? "success"
+                : lastAttempt?.status === "cancelled"
+                    ? "cancelled"
+                    : nodeAttempts.length === 0
+                        ? "skipped"
+                        : "failed";
+            if (finalStatus === "success")
+                successCount++;
+            else if (finalStatus === "skipped")
+                skippedCount++;
+            else
+                failureCount++;
+            const replayAttempts = nodeAttempts.map((a) => {
+                const attemptDecisions = allDecisions.filter((d) => d.attemptId === a.attemptId);
+                const basePath = join(runsDir, runId);
+                return {
+                    attemptId: a.attemptId,
+                    runtime: a.runtime,
+                    model: a.model,
+                    provider: a.provider,
+                    contextCapsulePath: join(basePath, "context-capsules", `${a.nodeId}-${a.attemptId}.json`),
+                    promptSnapshotPath: join(basePath, "prompts", `${a.nodeId}-${a.attemptId}.txt`),
+                    toolEventsPath: join(basePath, "tool-events", `${a.nodeId}-${a.attemptId}.jsonl`),
+                    evidencePath: join(basePath, "evidence.jsonl"),
+                    decisionTracePath: join(basePath, "decisions.jsonl"),
+                    startedAt: a.startedAt,
+                    endedAt: a.endedAt,
+                    latencyMs: a.latencyMs,
+                    status: a.status,
+                    error: a.error,
+                    inputTokensEstimated: a.inputTokensEstimated,
+                    outputTokensEstimated: a.outputTokensEstimated,
+                    toolResultTokensEstimated: a.toolResultTokensEstimated,
+                    costUsdEstimated: a.costUsdEstimated,
+                    decisionTrace: attemptDecisions,
+                };
+            });
+            const nodeLatency = nodeAttempts.reduce((s, a) => s + (a.latencyMs ?? 0), 0);
+            const nodeCost = nodeAttempts.reduce((s, a) => s + (a.costUsdEstimated ?? 0), 0);
+            nodes.push({
+                nodeId,
+                nodeName: nodeId,
+                attempts: replayAttempts,
+                finalStatus,
+                totalLatencyMs: nodeLatency,
+                totalCostUsd: nodeCost,
+            });
+            totalAttempts += nodeAttempts.length;
+            totalLatencyMs += nodeLatency;
+            totalCostUsd += nodeCost;
+            totalInputTokens += nodeAttempts.reduce((s, a) => s + (a.inputTokensEstimated ?? 0), 0);
+            totalOutputTokens += nodeAttempts.reduce((s, a) => s + (a.outputTokensEstimated ?? 0), 0);
+        }
+        // Sort nodes deterministically
+        nodes.sort((a, b) => a.nodeId.localeCompare(b.nodeId));
+        return {
+            runId,
+            omkVersion: getOmkVersion(),
+            dagHash,
+            policyHash: "pending", // populated by external policy engine
+            routerPolicyHash: "pending",
+            repairPolicyHash: "pending",
+            contextPolicyHash: "pending",
+            startedAt: meta.startedAt ?? new Date().toISOString(),
+            completedAt: meta.completedAt,
+            nodes,
+            summary: {
+                totalNodes: nodes.length,
+                totalAttempts,
+                successCount,
+                failureCount,
+                skippedCount,
+                totalLatencyMs,
+                totalCostUsd,
+                totalInputTokens,
+                totalOutputTokens,
+            },
+        };
+    }
+    async function validate(runId) {
+        const errors = [];
+        const warnings = [];
+        const validRunId = validateRunId(runId);
+        const runDir = join(runsDir, validRunId);
+        const required = ["run.json", "dag.json", "attempts.jsonl"];
+        for (const file of required) {
+            const exists = await readFile(join(runDir, file), "utf-8")
+                .then(() => true)
+                .catch(() => false);
+            if (!exists)
+                errors.push(`Missing required artifact: ${file}`);
+        }
+        // Check that every attempt has a context capsule
+        const attemptsPath = join(runDir, "attempts.jsonl");
+        const attemptsRaw = await readFile(attemptsPath, "utf-8").catch(() => "");
+        const attempts = attemptsRaw
+            .trim()
+            .split("\n")
+            .filter((l) => l.length > 0)
+            .map((line) => JSON.parse(line));
+        for (const a of attempts) {
+            const capsulePath = join(runDir, "context-capsules", `${a.nodeId}-${a.attemptId}.json`);
+            const exists = await readFile(capsulePath, "utf-8")
+                .then(() => true)
+                .catch(() => false);
+            if (!exists) {
+                warnings.push(`Missing context capsule for ${a.nodeId}/${a.attemptId}`);
+            }
+        }
+        return { valid: errors.length === 0, errors, warnings };
+    }
+    return { build, validate };
+}
+function hashString(input) {
+    let h = 0;
+    for (let i = 0; i < input.length; i++) {
+        h = ((h << 5) - h + input.charCodeAt(i)) | 0;
+    }
+    return Math.abs(h).toString(36);
+}
+function getOmkVersion() {
+    try {
+        const pkg = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8"));
+        return pkg.version ?? "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}

package/dist/replay/replay-engine.d.ts

@@ -0,0 +1,18 @@
+/**
+ * ReplayEngine — restores and replays a run's full execution timeline.
+ *
+ * Reads context capsules, evidence results, and decision traces to reconstruct
+ * the exact sequence of events: provider selection, context delivery,
+ * evidence gates, and repair decisions.
+ */
+export interface ReplayOptions {
+    runsDir?: string;
+    json?: boolean;
+    context?: boolean;
+    evidence?: boolean;
+    decisions?: boolean;
+    repair?: boolean;
+    nodeId?: string;
+    attemptId?: string;
+}
+export declare function replayRun(runId: string, options?: ReplayOptions): Promise<void>;