@deksden-com/dd-flow-cli 0.9.0-beta.13 → 0.9.0-beta.134

package/dist/harness-runtime/lib/runner-events.mjs

@@ -0,0 +1,200 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open, readFile, rename, unlink } from "node:fs/promises";
+import path from "node:path";
+import { withRunnerLock as withLock } from "./runner-lock.mjs";
+import { isObservationLoss, errorRecord } from "./operation-errors.mjs";
+import { operationContext } from "./operation-context.mjs";
+
+const appendQueues = new Map();
+
+export function canonicalJson(value) {
+  const normalize = (item) => {
+    if (Array.isArray(item)) return item.map(normalize);
+    if (item && typeof item === "object") return Object.fromEntries(Object.keys(item).sort().map((key) => [key, normalize(item[key])]));
+    return item;
+  };
+  return `${JSON.stringify(normalize(value))}\n`;
+}
+
+export function sha256(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+export function hashJson(value) { return sha256(canonicalJson(value)); }
+
+export async function writeJsonAtomic(file, value) {
+  await mkdir(path.dirname(file), { recursive: true });
+  const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    const payload = await open(temporary, "wx", 0o600);
+    try { await payload.writeFile(`${JSON.stringify(value, null, 2)}\n`); await payload.sync(); }
+    finally { await payload.close(); }
+    await rename(temporary, file);
+    const directory = await open(path.dirname(file), "r");
+    try { await directory.sync(); }
+    finally { await directory.close(); }
+  } finally { await unlink(temporary).catch(error => { if (error.code !== "ENOENT") throw error; }); }
+}
+
+export async function readJsonLines(file, label = "JSONL") {
+  try {
+    const source = await readFile(file, "utf8");
+    return source.split("\n").filter(Boolean).map((line, index) => {
+      try { return JSON.parse(line); }
+      catch { throw new Error(`invalid ${label} record at ${file}:${index + 1}`); }
+    });
+  } catch (error) {
+    if (error?.code === "ENOENT") return [];
+    throw error;
+  }
+}
+
+export async function readEvents(file) {
+  const records = await readJsonLines(file, "runner event");
+  return records.map((record, index) => {
+    try { return validateRunnerEvent(record); }
+    catch { throw new Error(`invalid runner event at ${file}:${index + 1}`); }
+  });
+}
+
+export function validateRunnerEvent(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value) || value.specversion !== "1.0" || typeof value.id !== "string" || typeof value.source !== "string" || typeof value.type !== "string" || typeof value.time !== "string" || !value.data || typeof value.data !== "object" || !Number.isInteger(value.data.sequence) || value.data.sequence < 1) {
+    throw new Error("invalid normalized runner event");
+  }
+  return value;
+}
+
+export async function appendEvent(file, input) {
+  const prior = appendQueues.get(file) ?? Promise.resolve();
+  const current = prior.then(() => withLock(file, async () => {
+    const events = await readEvents(file);
+    // Internal callers can validate a dispatch or bind its generation under
+    // the same cross-process lock that orders cancellation and completion.
+    if (input.deduplicate && input.id) { const existing = events.find(event => event.id === input.id); if (existing) return existing; }
+    if (input.beforeAppend?.(events) === false) return null;
+    const sequence = events.length + 1;
+    const event = {
+      specversion: "1.0",
+      id: input.id ?? `EVT-${randomUUID()}`,
+      source: input.source,
+      type: input.type,
+      time: input.time ?? new Date().toISOString(),
+      subject: input.subject ?? "runner",
+      datacontenttype: "application/json",
+      runid: input.runId,
+      executionid: input.executionId,
+      traceid: input.traceId,
+      spanid: input.spanId ?? randomUUID(),
+      ...(input.parentSpanId ? { parentspanid: input.parentSpanId } : {}),
+      data: { sequence, ...input.data }
+    };
+    const context = operationContext.getStore();
+    if (context?.eventsFile === file && context.executionId === input.executionId && context.operationId && input.executionId) {
+      event.data.execution_operation_id ??= context.operationId;
+    }
+    validateRunnerEvent(event);
+    await mkdir(path.dirname(file), { recursive: true });
+    const handle = await open(file, "a");
+    try { await handle.write(`${JSON.stringify(event)}\n`); await handle.sync(); }
+    finally { await handle.close(); }
+    return event;
+  }));
+  const tail = current.catch(() => {}); appendQueues.set(file, tail);
+  try { return await current; }
+  finally { if (appendQueues.get(file) === tail) appendQueues.delete(file); }
+}
+
+export function reduceEvents(events) {
+  const operations = new Map();
+  let state = "planned";
+  for (const event of events) {
+    const data = event?.data ?? {};
+    if (["dev.dd.eval.planned", "dev.dd.eval.completed"].includes(event.type) && typeof data.state === "string") state = data.state;
+    if (typeof data.operation_id !== "string") continue;
+    const prior = operations.get(data.operation_id) ?? { id: data.operation_id, requested: false, started: false, terminal: null };
+    if (event.type.endsWith(".requested")) prior.requested = true;
+    if (event.type.endsWith(".started")) prior.started = true;
+    if (event.type.endsWith(".observation_lost")) prior.observation_lost = data.error ?? { code: "observation_lost" };
+    if (event.type.endsWith(".completed") || event.type.endsWith(".failed") || event.type.endsWith(".cancelled")) {
+      const terminal = event.type.split(".").at(-1);
+      const result = terminal === "completed" ? data.result : { error: data.error ?? null };
+      const resultHash = hashJson({ terminal, result });
+      if (prior.terminal && (prior.terminal !== terminal || prior.result_hash !== resultHash)) {
+        throw Object.assign(new Error(`runner operation ${data.operation_id} has conflicting terminal events`), { code: "journal_conflict", operation_id: data.operation_id });
+      }
+      prior.terminal = terminal;
+      prior.result = result;
+      prior.result_hash = resultHash;
+    }
+    operations.set(data.operation_id, prior);
+  }
+  return { state, operations: Object.fromEntries([...operations].map(([id, value]) => [id, value])) };
+}
+
+export async function recordOperation({ eventsFile, source, runId, executionId, traceId, operationId, operation, subject = "runner", action }) {
+  const existing = await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
+    const current = reduceEvents(await readEvents(eventsFile)).operations[operationId];
+    if (current?.terminal === "completed") return current;
+    if (current?.terminal) throw Object.assign(new Error(`operation ${operationId} is already ${current.terminal}; use explicit recovery instead of replaying it`), { code: "operation_terminal", operation_id: operationId, terminal: current.terminal });
+    if (current?.started && !current.terminal) {
+      const error = new Error(current.observation_lost
+        ? `operation ${operationId} has no confirmed outcome; reconcile its provider state before retrying`
+        : `operation ${operationId} is already in progress; reconcile before retrying`);
+      error.code = current.observation_lost ? "operation_observation_lost" : "operation_in_progress";
+      error.operation_id = operationId;
+      throw error;
+    }
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.requested", data: { operation_id: operationId, operation, status: "requested" } });
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.started", data: { operation_id: operationId, operation, status: "started" } });
+    return null;
+  });
+  if (existing) return { reused: true, operation_id: operationId, result: existing.result };
+  try {
+    const result = await operationContext.run({ operationId, eventsFile, runId, executionId }, action);
+    await completeOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, result });
+    return { reused: false, operation_id: operationId, result };
+  } catch (error) {
+    if (isObservationLoss(error)) await observationLost({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, error });
+    else await failOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, error });
+    throw error;
+  }
+}
+
+export async function completeOperation({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, result }) {
+  return await settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal: "completed", result });
+}
+
+export async function failOperation({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, error }) {
+  return await settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal: "failed", result: { error: errorRecord(error) } });
+}
+
+/** A lost observer response is deliberately non-terminal: replay could duplicate a provider Turn. */
+export async function observationLost({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, error }) {
+  return await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
+    const existing = reduceEvents(await readEvents(eventsFile)).operations[operationId];
+    if (existing?.terminal) return { reused: true, operation_id: operationId, result: existing.result };
+    if (!existing?.started) throw Object.assign(new Error(`operation ${operationId} has no started receipt`), { code: "operation_not_started" });
+    if (existing.observation_lost) return { reused: true, operation_id: operationId, observation_lost: existing.observation_lost };
+    const recorded = errorRecord(error);
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.observation_lost", data: { operation_id: operationId, operation, status: "observation_lost", error: recorded } });
+    return { reused: false, operation_id: operationId, observation_lost: recorded };
+  });
+}
+
+async function settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal, result }) {
+  return await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
+    const existing = reduceEvents(await readEvents(eventsFile)).operations[operationId];
+    const expectedHash = hashJson({ terminal, result });
+    if (existing?.terminal) {
+      if (existing.terminal === terminal && existing.result_hash === expectedHash) return { reused: true, operation_id: operationId, result: existing.result };
+      throw Object.assign(new Error(`runner operation ${operationId} has conflicting terminal result`), { code: "journal_conflict", operation_id: operationId });
+    }
+    if (!existing?.started) throw Object.assign(new Error(`operation ${operationId} has no started receipt`), { code: "operation_not_started" });
+    const type = terminal === "completed" ? "dev.dd.eval.operation.completed" : "dev.dd.eval.operation.failed";
+    const data = terminal === "completed"
+      ? { operation_id: operationId, operation, status: terminal, result }
+      : { operation_id: operationId, operation, status: terminal, error: result.error };
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type, data });
+    return { reused: false, operation_id: operationId, result };
+  });
+}

package/dist/harness-runtime/lib/runner-lock.mjs

@@ -0,0 +1,42 @@
+import { randomUUID } from "node:crypto";
+import { mkdirSync, readdirSync, readFileSync, writeFileSync, unlinkSync, rmdirSync } from "node:fs";
+import path from "node:path";
+
+function release(lock, owner) {
+  try { unlinkSync(path.join(lock, owner)); } catch (error) { if (error.code === "ENOENT") return; throw error; }
+  // Never recursively remove a directory: another owner may already have acquired it.
+  try { rmdirSync(lock); } catch (error) { if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes(error.code)) throw error; }
+}
+
+function reclaimDeadOwner(lock) {
+  let entries;
+  try { entries = readdirSync(lock); } catch (error) { if (error.code === "ENOENT") return; throw error; }
+  if (entries.length !== 1 || !/^owner-.*\.json$/.test(entries[0])) return;
+  let owner;
+  try { owner = JSON.parse(readFileSync(path.join(lock, entries[0]), "utf8")); } catch { return; }
+  if (!Number.isInteger(owner.pid) || owner.pid <= 0) return;
+  try { process.kill(owner.pid, 0); }
+  catch (error) { if (error.code === "ESRCH") release(lock, entries[0]); }
+  // A live/reused PID or unknown owner is conservative: age never proves death.
+}
+
+export async function withRunnerLock(file, action, { timeoutMs = 30_000 } = {}) {
+  const lock = `${file}.lock`;
+  const owner = `owner-${process.pid}-${randomUUID()}.json`;
+  const deadline = performance.now() + timeoutMs;
+  for (;;) {
+    let acquired = false;
+    try { mkdirSync(lock); acquired = true; }
+    catch (error) { if (error.code !== "EEXIST") throw error; }
+    if (acquired) {
+      // No await between acquisition and identifying the owner.
+      try { writeFileSync(path.join(lock, owner), JSON.stringify({ pid: process.pid }), { flag: "wx" }); }
+      catch (error) { release(lock, owner); throw error; }
+      try { return await action(); }
+      finally { release(lock, owner); }
+    }
+    reclaimDeadOwner(lock);
+    if (performance.now() >= deadline) throw Object.assign(new Error(`runner lock has a live or unconfirmed owner: ${lock}; inspect its owner before recovery`), { code: "runner_lock_timeout" });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+}

package/dist/harness-runtime/lib/session-settlement.mjs

@@ -0,0 +1,30 @@
+import { errorRecord } from "./operation-errors.mjs";
+
+/** Diagnostic failure never prevents cancellation of already owned Sessions.
+ * Only a fresh successful observation can establish settlement. */
+export async function waitForSettlement({ observe, cancel, knownSessions = () => [], timeoutMs = 5_000 }) {
+  const deadline = performance.now() + timeoutMs;
+  const cancelled = new Set(), errors = [];
+  if (cancel) for (const id of knownSessions()) {
+    cancelled.add(id);
+    try { await cancel(id); } catch (error) { errors.push({ phase: "cancel", session_id: id, ...errorRecord(error) }); }
+  }
+  for (;;) {
+    let state;
+    try { state = await observe(); }
+    catch (error) {
+      if (!cancel) throw error;
+      errors.push({ phase: "observe", ...errorRecord(error) });
+      state = { sessions: knownSessions(), active: true, observation_error: errorRecord(error) };
+    }
+    if (!state.active && state.sessions.length === 0) return { ...state, cleanup_errors: errors };
+    // The observation budget must not skip a newly discovered owned target.
+    // Attempt its cancellation once, then leave physical settlement pending.
+    for (const id of state.sessions) if (cancel && !cancelled.has(id)) {
+      cancelled.add(id);
+      try { await cancel(id); } catch (error) { errors.push({ phase: "cancel", session_id: id, ...errorRecord(error) }); }
+    }
+    if (!cancel || performance.now() >= deadline) throw Object.assign(new Error("daemon still owns an unsettled Session tree"), { code: "tree_not_settled", details: { ...state, cleanup_errors: errors } });
+    await new Promise(resolve => setTimeout(resolve, 50));
+  }
+}

package/dist/harness-runtime/lib/setup-progress.d.mts

@@ -0,0 +1,9 @@
+export const SETUP_STARTUP_MS: number;
+export const SETUP_QUIET_MS: number;
+export const setupPhases: Set<string>;
+export function setupReporter(operationId?: string): (phase: string, step?: string | number | null) => void;
+export const publishSetup: ReturnType<typeof setupReporter>;
+export function isSetupPhase(value: unknown): boolean;
+export function startupObserver(stateDir: string, daemonId: string): () => Promise<number>;
+export function setupObserver(operationId: string): (value: unknown) => boolean;
+export function observationDetails(clock: import('./observation-clock.mjs').ObservationClock, phase: string | null, pid: number | undefined): Record<string, unknown>;

package/dist/harness-runtime/lib/setup-progress.mjs

@@ -0,0 +1,51 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+export const SETUP_STARTUP_MS = 120_000;
+export const SETUP_QUIET_MS = 30_000;
+export const setupPhases = new Set(['ingress_entered', 'imports_ready', 'probe_started', 'probe_completed', 'dependency_check_completed', 'transfer_progress', 'identity_digest_progress', 'identity_digest_completed', 'native_initialized', 'ready']);
+export function setupReporter(operationId = process.env.DD_EVAL_OPERATION_ID) {
+  if (!operationId) return () => {};
+  let sequence = 0;
+  return (phase, step = null) => {
+    if (!setupPhases.has(phase)) throw new Error('Invalid setup phase');
+    process.stderr.write(JSON.stringify({ kind: 'setup_phase', operation_id: operationId, phase, step, sequence: ++sequence, observed_at: new Date().toISOString() }) + '\n');
+  };
+}
+export const publishSetup = setupReporter();
+export function isSetupPhase(value) {
+  return value?.kind === 'setup_phase' && typeof value.operation_id === 'string' && value.operation_id.length > 0 && setupPhases.has(value.phase)
+    && Number.isSafeInteger(value.sequence) && value.sequence > 0 && value.sequence <= 1_000_000
+    && (value.step === null || typeof value.step === 'string' && value.step.length <= 4096 || Number.isSafeInteger(value.step) && value.step >= 0)
+    && typeof value.observed_at === 'string' && Number.isFinite(Date.parse(value.observed_at))
+    && !Object.hasOwn(value, 'error') && !Object.hasOwn(value, 'code');
+}
+export function startupObserver(stateDir, daemonId) {
+  const seen = new Set(); let sequence = 0, binding;
+  return async () => {
+    let state;
+    try { state = JSON.parse(await readFile(path.join(stateDir, 'daemon.json'), 'utf8')); }
+    catch (error) { if (error.code === 'ENOENT') return sequence; throw error; }
+    if (state.daemon_id !== daemonId) throw Object.assign(new Error('Startup daemon identity changed'), { code: 'operation_binding_conflict' });
+    const owner = state.config?.runtime_owner;
+    const identity = JSON.stringify([state.started_at ?? null, owner?.owner_id ?? null, owner?.operation_id ?? null, state.generation ?? null]);
+    binding ??= identity;
+    if (identity !== binding) throw Object.assign(new Error('Startup ownership binding changed'), { code: 'operation_binding_conflict' });
+    const marker = JSON.stringify([state.pid ?? null, state.provider_pid ?? null, state.shutdown_state ?? null, state.startup_phase ?? null]);
+    if (!seen.has(marker)) { seen.add(marker); sequence++; publishSetup(state.shutdown_state === 'running' ? 'native_initialized' : 'probe_started', `${daemonId}:${sequence}`); }
+    return sequence;
+  };
+}
+export function setupObserver(operationId) {
+  const startedAt = Date.now(); let sequence = 0; const markers = new Set();
+  return value => {
+    if (typeof operationId !== 'string' || markers.size >= 4096 || !isSetupPhase(value) || value.operation_id !== operationId
+      || value.sequence <= sequence || Date.parse(value.observed_at) < startedAt || Date.parse(value.observed_at) > Date.now()) return false;
+    const marker = JSON.stringify([value.phase, value.step]);
+    if (markers.has(marker)) return false;
+    markers.add(marker); sequence = value.sequence; return true;
+  };
+}
+export function observationDetails(clock, phase, pid) {
+  return { reason: clock.observationLost ? 'observation_gap' : 'inactivity', phase, pid,
+    timeout_ms: clock.timeoutMs, observation_lost: clock.observationLost, observation_clock: clock.state(), ...(clock.setupProgress ? { last_setup: clock.setupProgress } : {}), effect: 'unknown' };
+}

package/dist/harness-runtime/lib/tool-observations.d.mts

@@ -0,0 +1,21 @@
+export interface ToolEvent {
+  session?: { harness_id: string; session_id: string };
+  call_id?: string;
+  status?: string;
+  name?: string;
+  reason?: string;
+  [key: string]: unknown;
+}
+export function classifyCodexItem(type: unknown): "tool" | "non_tool" | "unknown";
+export class ToolObservations {
+  calls: Map<string, ToolEvent>;
+  turns: Map<string, { session: { harness_id: string; session_id: string }; turn_id: string; started: boolean; completed: boolean }>;
+  gaps: Set<string>;
+  sources: Map<string, ToolEvent>;
+  add(event: ToolEvent | null): void;
+  observeTurn(event: { session: { harness_id: string; session_id: string }; turn_id: string; started?: boolean; completed?: boolean }): void;
+  summary(sessionId?: string, cursor?: Set<string>): { total: number; failures: number; pending: number; by_tool: Record<string, number>; completeness: string; outcome_completeness: string; sessions: Array<{ session: { harness_id: string; session_id: string }; total: number; failures: number; pending: number; by_tool: Record<string, number> }>; reasons: string[] };
+  cursor(): Set<string>;
+}
+export function codexToolObservation(record: Record<string, unknown>, sessionId: string): ToolEvent | null;
+export function replayToolJournal(file: string, options: { harness: string; rootSessionId: string; adapterSessionId?: string; text?: string; expectedSessionIds?: string[] }, reducer?: ToolObservations): ToolObservations;

package/dist/harness-runtime/lib/tool-observations.mjs

@@ -0,0 +1,228 @@
+import fs from "node:fs";
+
+/** Canonical physical-session tool accounting. Native decoding lives below;
+ * consumers never have to inspect provider envelopes or aggregate tokens. */
+export class ToolObservations {
+  constructor() { this.calls = new Map(); this.turns = new Map(); this.gaps = new Set(); this.transcriptSessions = new Set(); this.sources = new Map(); }
+  observeTurn(event) {
+    if (!event?.session?.harness_id || !event.session.session_id || !event.turn_id) { this.gaps.add("turn_identity_missing"); return; }
+    const key = JSON.stringify([event.session, event.turn_id]);
+    const old = this.turns.get(key);
+    this.turns.set(key, { ...old, ...event, started: old?.started || event.started || false,
+      completed: old?.completed || event.completed || false });
+  }
+  add(event) {
+    if (!event) return;
+    if (event.reason) this.gaps.add(event.reason);
+    if (event.source_observed && event.session?.harness_id && event.session?.session_id) { this.sources.set(JSON.stringify(event.session), event); return; }
+    if (event.reason && !event.call_id) return;
+    const { harness_id, session_id } = event.session ?? {};
+    if (!harness_id || !session_id || !event.call_id) { this.gaps.add("tool_identity_missing"); return; }
+    const key = JSON.stringify([harness_id, session_id, event.epoch ?? "", event.call_id]);
+    const physical = JSON.stringify([harness_id, session_id]);
+    if (event.source?.channel === "agy.native_transcript") {
+      this.transcriptSessions.add(physical);
+      for (const [oldKey, call] of this.calls) if (JSON.stringify([call.session.harness_id, call.session.session_id]) === physical && call.source?.channel !== "agy.native_transcript") {
+        this.calls.delete(oldKey); this.gaps.add("agy_live_transcript_correlation_unproven");
+      }
+    } else if (this.transcriptSessions.has(physical)) { this.gaps.add("agy_live_transcript_correlation_unproven"); return; }
+    const old = this.calls.get(key);
+    // Replay and late starts cannot undo a terminal outcome. Conflicting
+    // terminal facts remain visible rather than silently choosing success.
+    const proven = value => ["completed", "failed"].includes(value);
+    if (old && proven(old.status) && proven(event.status) && old.status !== event.status) this.gaps.add("conflicting_tool_outcomes");
+    this.calls.set(key, { ...old, ...event, name: event.name ?? old?.name ?? "unknown",
+      status: old?.status === "failed" || event.status === "failed" ? "failed"
+        : old?.status === "completed" || event.status === "completed" ? "completed"
+        : old?.status === "unknown" || event.status === "unknown" ? "unknown" : "running" });
+  }
+  summary(sessionId, cursor) {
+    const by_tool = Object.create(null), sessions = new Map(), reasons = new Set(this.gaps); let total = 0, failures = 0, pending = 0;
+    for (const [key, value] of this.calls) {
+      if (sessionId !== undefined && value.session.session_id !== sessionId || cursor?.has(key)) continue;
+      const id = JSON.stringify(value.session);
+      if (value.name === "unknown") reasons.add("tool_start_missing");
+      const session = sessions.get(id) ?? { session: value.session, parent_session: value.parent_session ?? null, total: 0, failures: 0, pending: 0, by_tool: Object.create(null) };
+      total++; session.total++; by_tool[value.name] = (by_tool[value.name] ?? 0) + 1; session.by_tool[value.name] = (session.by_tool[value.name] ?? 0) + 1;
+      if (value.status === "failed") { failures++; session.failures++; }
+      if (!["completed", "failed"].includes(value.status)) { pending++; session.pending++; }
+      sessions.set(id, session);
+    }
+    if (!cursor) for (const turn of this.turns.values()) {
+      if (sessionId !== undefined && turn.session.session_id !== sessionId) continue;
+      const id = JSON.stringify(turn.session);
+      sessions.set(id, sessions.get(id) ?? { session: turn.session, parent_session: null, total: 0, failures: 0, pending: 0, by_tool: {} });
+      if (!turn.started || !turn.completed) reasons.add("turn_lifecycle_incomplete");
+    }
+    if (!cursor) for (const [id, event] of this.sources) {
+      if (sessionId !== undefined && event.session.session_id !== sessionId) continue;
+      sessions.set(id, sessions.get(id) ?? { session: event.session, parent_session: event.parent_session ?? null, total: 0, failures: 0, pending: 0, by_tool: {} });
+    }
+    return { schema_id: "dd-flow/tool-summary@1", scope: "physical_sessions", measurement: "observed_invocations", origin: "native_tool_events", aggregation: cursor ? "delta" : "cumulative", total, failures, pending, by_tool,
+      sessions: [...sessions.values()], completeness: reasons.size ? "partial" : "complete", outcome_completeness: pending ? "partial" : "complete", reasons: [...reasons] };
+  }
+  cursor() { return new Set(this.calls.keys()); }
+}
+
+/** For adapters whose raw transcript is not portable, retain only normalized
+ * tool facts in the existing owned journal. Repeated snapshots are idempotent. */
+export function journalToolObservations(file, events, previous) {
+  const batch = new ToolObservations();
+  for (const event of events) batch.add(event);
+  const records = [...batch.calls.values(), ...batch.sources.values(), ...[...batch.gaps].map(reason => ({ reason }))];
+  for (const event of records) {
+    const key = JSON.stringify([event.session, event.epoch, event.call_id, event.reason]);
+    const encoded = JSON.stringify(event);
+    if (previous.get(key) === encoded) continue;
+    fs.appendFileSync(file, `${JSON.stringify({ kind: "tool_observation", schema_id: "dd-flow/tool-observation@1", observed_at: new Date().toISOString(), event })}\n`, { mode: 0o600 });
+    previous.set(key, encoded);
+  }
+}
+
+const identity = (harness_id, session_id) => ({ harness_id, session_id });
+const status = value => ["error", "failed", "ERROR"].includes(value) ? "failed" : ["completed", "success", "DONE"].includes(value) ? "completed"
+  : value === undefined || value === null || ["running", "pending", "in_progress", "inProgress", "RUNNING", "PENDING", "IN_PROGRESS"].includes(value) ? "running" : "unknown";
+const nativeStatus = value => ({ status: status(value), ...(status(value) === "unknown" ? { reason: "tool_status_unsupported" } : {}) });
+export function zcodePhysicalSession(update, rootSessionId) {
+  const native = update?._meta?.zcodeRuntime ?? {};
+  if (native.source === "subagent" && !native.childSessionId) return null;
+  return native.childSessionId ?? rootSessionId;
+}
+
+export function acpToolObservation(message, { harness = "zcode-acp", rootSessionId, adapterSessionId } = {}) {
+  if (!["session/update", "_x.ai/session_notification"].includes(message?.method)) return null;
+  const update = message.params?.update;
+  if (!["tool_call", "tool_call_update"].includes(update?.sessionUpdate)) return null;
+  const transport = message.params?.sessionId;
+  if (adapterSessionId && transport !== adapterSessionId) return null;
+  const root = rootSessionId ?? transport;
+  const session = harness === "zcode-acp" ? zcodePhysicalSession(update, root) : harness === "grok-acp" ? transport : root;
+  if (!session || !update.toolCallId) return { reason: "tool_identity_missing" };
+  return { session: identity(harness, session), parent_session: session !== root ? identity(harness, root) : null,
+    transport_session_id: transport, call_id: update.toolCallId, name: update._meta?.["x.ai/tool"]?.name ?? update._meta?.claudeCode?.toolName ?? update.title?.split(":", 1)[0], ...nativeStatus(update.status) };
+}
+
+export function droidToolObservations(record, sessionId) {
+  return (Array.isArray(record.message?.content) ? record.message.content : []).flatMap(block => {
+    if (block.type === "tool_use") return [{ session: identity("droid-cli", sessionId), call_id: block.id, name: block.name, status: "running" }];
+    if (block.type === "tool_result") return [{ session: identity("droid-cli", sessionId), call_id: block.tool_use_id, status: block.is_error ? "failed" : "completed" }];
+    return [];
+  });
+}
+
+export function codexToolObservation(record, sessionId) {
+  if (record.type !== "response_item") return null;
+  const p = record.payload ?? {};
+  if (["function_call", "custom_tool_call"].includes(p.type)) return { session: identity("codex-desktop", sessionId), call_id: p.call_id, name: p.name, status: "running" };
+  if (["function_call_output", "custom_tool_call_output"].includes(p.type)) {
+    return { session: identity("codex-desktop", sessionId), call_id: p.call_id, status: structuredFailure(p.output) ? "failed" : "unknown" };
+  }
+  return null;
+}
+
+function structuredFailure(value) {
+  if (!value || typeof value !== "object") return false;
+  return value.ok === false || value.is_error === true || value.isError === true || value.error === true
+    || typeof value.exit_code === "number" && value.exit_code !== 0;
+}
+
+const codexTools = new Set(["commandExecution", "fileChange", "mcpToolCall", "dynamicToolCall", "collabAgentToolCall", "webSearch", "imageView", "imageGeneration", "sleep"]);
+const codexNonTools = new Set(["userMessage", "hookPrompt", "agentMessage", "functionCallOutput", "plan", "reasoning", "subAgentActivity", "enteredReviewMode", "exitedReviewMode", "contextCompaction"]);
+export function classifyCodexItem(type) {
+  return codexTools.has(type) ? "tool" : codexNonTools.has(type) ? "non_tool" : "unknown";
+}
+export function codexAppServerObservation(message) {
+  if (!["item/started", "item/completed"].includes(message?.method)) return null;
+  const { threadId, turnId, item } = message.params ?? {};
+  if (!item || classifyCodexItem(item.type) === "non_tool") return null;
+  if (classifyCodexItem(item.type) === "unknown") return { reason: "codex_item_type_unsupported" };
+  if (!threadId || !item.id) return { reason: "tool_identity_missing" };
+  const native = item.status;
+  const failed = ["failed", "declined", "interrupted"].includes(native) || item.type === "commandExecution" && typeof item.exitCode === "number" && item.exitCode !== 0;
+  const completed = message.method === "item/completed" && (native === undefined || native === "completed");
+  const known = native === undefined || ["inProgress", "completed", "failed", "declined", "interrupted"].includes(native);
+  return { session: identity("codex-desktop", threadId), epoch: turnId ?? "", call_id: item.id, name: item.type,
+    status: !known ? "unknown" : failed ? "failed" : completed ? "completed" : "running", ...(!known ? { reason: "codex_item_status_unsupported" } : {}) };
+}
+
+export function openCodeToolObservation(part, sessionId, messageId) {
+  if (part.type !== "tool") return null;
+  return { session: identity("opencode-server", part.sessionID ?? sessionId), call_id: part.callID ?? part.id,
+    epoch: messageId, name: part.tool, ...nativeStatus(part.state?.status) };
+}
+
+export function agyToolObservation(step, rootSessionId) {
+  if (step.step_type !== "tool" && !(step.step_type === "subagent" && step.tool_name === "invoke_subagent")) return null;
+  const sessionId = step.conversation_id ?? rootSessionId;
+  return { session: identity("antigravity-cli", sessionId), parent_session: sessionId !== rootSessionId ? identity("antigravity-cli", rootSessionId) : null,
+    call_id: step.step_id ?? step.stepId ?? step.id ?? step.tool_call_id ?? step.toolCallId ?? (Number.isInteger(step.step_index) ? String(step.step_index) : null),
+    name: step.tool_name ?? step.tool_info?.name, ...(step.tool_info?.error ? { status: "failed" } : nativeStatus(step.state)) };
+}
+
+/** Native AGY JSONL records have planner tool invocations, not matching live
+ * step IDs. GENERIC result text is deliberately not guessed into success. */
+export function agyTranscriptObservations(text, sessionId, parentSessionId, source = {}) {
+  const events = [], lines = text.split("\n"); let recognized = false;
+  const session = identity("antigravity-cli", sessionId);
+  for (const [line, bytes] of lines.entries()) {
+    if (!bytes.trim()) continue;
+    let record; try { record = JSON.parse(bytes); } catch { events.push({ session, reason: line === lines.length - 1 ? "agy_transcript_partial_tail" : "agy_transcript_invalid_record" }); continue; }
+    if (!record || typeof record !== "object" || Array.isArray(record)) { events.push({ session, reason: "agy_transcript_shape_unsupported" }); continue; }
+    if (record.conversation_id && record.conversation_id !== sessionId) { events.push({ session, reason: "agy_transcript_session_mismatch" }); continue; }
+    if (!["SYSTEM_MESSAGE", "USER_INPUT", "PLANNER_RESPONSE", "GENERIC", "TOOL_RESPONSE", "CHECKPOINT"].includes(record.type)) { events.push({ session, reason: "agy_transcript_shape_unsupported" }); continue; }
+    recognized = true;
+    if (record.tool_calls !== undefined && !Array.isArray(record.tool_calls)) events.push({ session, reason: "agy_transcript_shape_unsupported" });
+    if (record.truncated || record.truncation || typeof record.content === "string" && /\[.*truncat|<.*truncat/i.test(record.content)) events.push({ session, reason: "agy_transcript_truncated" });
+    for (const [index, call] of (Array.isArray(record.tool_calls) ? record.tool_calls : []).entries()) {
+      if (!Number.isInteger(record.step_index) || !call || typeof call.name !== "string" || !call.name) { events.push({ session, reason: "agy_transcript_tool_identity_missing" }); continue; }
+      const id = call.id ?? call.tool_call_id;
+      events.push({ session, parent_session: identity("antigravity-cli", parentSessionId), epoch: "native_transcript", call_id: id ?? `${record.step_index}:${index}`, name: call.name,
+        status: "unknown", reason: "agy_transcript_tool_outcome_unproven", source: { ...source, channel: "agy.native_transcript", line: line + 1, step_index: record.step_index } });
+    }
+  }
+  if (!recognized) events.push({ session, reason: "agy_transcript_shape_unsupported" });
+  else events.push({ session, parent_session: identity("antigravity-cli", parentSessionId), source_observed: true, source: { ...source, channel: "agy.native_transcript" } });
+  return events;
+}
+
+/** Read-only replay, shared by accounting and offline diagnosis. A journal
+ * carries physical identities; an ACP connection identity never crosses out. */
+export function replayToolJournal(file, options, reducer = new ToolObservations()) {
+  let bytes;
+  try { bytes = options.text ?? fs.readFileSync(file, "utf8"); } catch (error) { reducer.gaps.add(error.code === "ENOENT" ? "journal_missing" : "journal_unreadable"); return reducer; }
+  const lines = bytes.split("\n"); let session = options.rootSessionId, transport = options.adapterSessionId;
+  for (const [index, line] of lines.entries()) {
+    if (!line.trim()) continue;
+    let e; try { e = JSON.parse(line); } catch { reducer.gaps.add(index === lines.length - 1 ? "journal_partial_tail" : "journal_invalid_record"); continue; }
+    const add = event => { if (event) reducer.add({ ...event, source: { ...event.source, journal: file, journal_line: index + 1 }, observed_at: e.observed_at ?? e.timestamp ?? e.at ?? e.ts ?? null }); };
+    if (e.kind === "tool_observation" && e.schema_id === "dd-flow/tool-observation@1") { add(e.event); continue; }
+    if (e.kind === "notification_failure") reducer.gaps.add("notification_failure");
+    const m = e.payload ?? e;
+    if (options.harness === "zcode-acp" || options.harness === "grok-acp") {
+      if (m.method === "zcode/session/allocated" && m.params?.provider_session_id === session) transport = m.params.adapter_session_id;
+      if (e.kind === "inbound" || m.method) add(acpToolObservation(m, { ...options, rootSessionId: session, adapterSessionId: transport }));
+    } else if (options.harness === "codex-desktop") {
+      if (e.type === "session_meta" && e.payload?.id !== session) { reducer.gaps.add("session_identity_mismatch"); break; }
+      if (m.params?.threadId && options.expectedSessionIds && !options.expectedSessionIds.includes(m.params.threadId)) continue;
+      if (["turn/started", "turn/completed"].includes(m.method)) {
+        const params = m.params ?? {};
+        if (params.threadId && params.turn?.id) reducer.observeTurn({ session: identity("codex-desktop", params.threadId), turn_id: params.turn.id,
+          started: m.method === "turn/started", completed: m.method === "turn/completed" && params.turn.status === "completed" });
+        else reducer.gaps.add("turn_identity_missing");
+      }
+      add(codexAppServerObservation(m) ?? codexToolObservation(e, session));
+    } else if (options.harness === "droid-cli") {
+      if (e.type === "session_start" && e.id !== session) { reducer.gaps.add("session_identity_mismatch"); break; }
+      for (const event of droidToolObservations(e, session)) add(event);
+    } else if (options.harness === "antigravity-cli") {
+      const native = m.event ?? e;
+      if (native.event === "step_update") add(agyToolObservation(native.step_update ?? {}, session));
+    } else if (options.harness === "opencode-server") {
+      // Native SSE message.part.updated, retained in the adapter journal.
+      const event = m.event ?? m;
+      const part = event.properties?.part;
+      if (part) add(openCodeToolObservation(part, session, part.messageID));
+    } else reducer.gaps.add("tool_observation_unsupported");
+  }
+  return reducer;
+}