@askexenow/exe-os 0.9.292 → 0.9.293

package/dist/chunk-B5ZCGIYB.js

@@ -0,0 +1,630 @@
+import {
+  getClient
+} from "./chunk-IDJCLLJF.js";
+
+// src/lib/orchestration-events.ts
+import crypto from "crypto";
+var ORCHESTRATION_COUNTER_KEYS = [
+  "duplicateNudges",
+  "missedReviews",
+  "tmuxCommandTimeouts",
+  "claimCollisions",
+  "crossSessionBlocks",
+  "dispatchUnclaimed",
+  "sessionExits",
+  "sessionBorns",
+  "sessionIdleKills",
+  "sessionTtlKills",
+  "sessionContextKills",
+  "sessionResumed",
+  "eventLoopBlocks",
+  "shardCircuitOpens",
+  "taskBlocked",
+  "taskCancelled",
+  "checkpoints",
+  "messagesSent",
+  "reviewsCreated",
+  "reviewsApproved",
+  "reviewsRejected",
+  "dispatches",
+  "signalsCreated",
+  "signalsConsumed",
+  "workerInstancesSelected",
+  "contextPressure",
+  "mcpUnreachable",
+  "dispatchAttempted",
+  "dispatchFailed",
+  "reviewStarted",
+  "messagesDelivered",
+  "messagesRead",
+  "signalsStale",
+  "tmuxNudgeAttempted",
+  "tmuxNudgeCompleted",
+  "tmuxSpawnAttempted",
+  "tmuxSpawnCompleted",
+  "consistencyMismatches",
+  "consistencyRepaired",
+  "fileRowsResynced",
+  "fileOrphansCleaned",
+  "worktreeCreated",
+  "worktreeReused",
+  "worktreeFailed",
+  "worktreePruned",
+  "worktreeSkipped",
+  "registryMissingTmux",
+  "registryMissingEntry",
+  "taskGroupsCreated",
+  "taskGroupBarriersFired",
+  "taskGroupTimeouts",
+  "taskGroupPartialFailures"
+];
+var ORCHESTRATION_LATENCY_METRIC_KEYS = [
+  "createToClaimMs",
+  "dispatchToClaimMs",
+  "doneToReviewMs",
+  "reviewToStartedMs",
+  "reviewToApprovedMs",
+  "signalToConsumedMs",
+  "blockedDurationMs",
+  "sessionDurationMs",
+  "daemonTickMs"
+];
+var SENSITIVE_PAYLOAD_KEY = /(context|prompt|secret|token|password|passphrase|recovery|raw|key)/i;
+var BEST_EFFORT_QUEUE_LIMIT = 200;
+var schemaInitializedClients = /* @__PURE__ */ new WeakSet();
+var schemaInitializingClients = /* @__PURE__ */ new WeakMap();
+var bestEffortQueue = Promise.resolve();
+var bestEffortPending = 0;
+async function ensureOrchestrationEventsSchema(client = getClient()) {
+  if (schemaInitializedClients.has(client)) return;
+  const existing = schemaInitializingClients.get(client);
+  if (existing) return existing;
+  const initialization = createOrchestrationEventsSchema(client).then(() => {
+    schemaInitializedClients.add(client);
+    schemaInitializingClients.delete(client);
+  }).catch((err) => {
+    schemaInitializingClients.delete(client);
+    throw err;
+  });
+  schemaInitializingClients.set(client, initialization);
+  return initialization;
+}
+async function createOrchestrationEventsSchema(client) {
+  await client.execute({
+    sql: `CREATE TABLE IF NOT EXISTS orchestration_events (
+      id TEXT PRIMARY KEY,
+      ts TEXT NOT NULL,
+      event_type TEXT NOT NULL,
+      source TEXT NOT NULL,
+      severity TEXT NOT NULL DEFAULT 'info',
+      task_id TEXT,
+      agent_id TEXT,
+      reviewer TEXT,
+      session_scope TEXT,
+      project_name TEXT,
+      instance_id TEXT,
+      tmux_session TEXT,
+      runtime TEXT,
+      correlation_id TEXT,
+      attempt INTEGER,
+      duration_ms INTEGER,
+      result TEXT,
+      error_code TEXT,
+      payload_json TEXT,
+      created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+    )`,
+    args: []
+  });
+  await client.execute({ sql: "CREATE INDEX IF NOT EXISTS idx_orch_events_ts ON orchestration_events(ts)", args: [] });
+  await client.execute({ sql: "CREATE INDEX IF NOT EXISTS idx_orch_events_task ON orchestration_events(task_id, ts)", args: [] });
+  await client.execute({ sql: "CREATE INDEX IF NOT EXISTS idx_orch_events_agent ON orchestration_events(agent_id, session_scope, ts)", args: [] });
+  await client.execute({ sql: "CREATE INDEX IF NOT EXISTS idx_orch_events_type ON orchestration_events(event_type, ts)", args: [] });
+}
+async function recordOrchestrationEvent(input) {
+  try {
+    const client = getClient();
+    await ensureOrchestrationEventsSchema(client);
+    const payloadJson = sanitizePayload(input.payload);
+    await client.execute({
+      sql: `INSERT INTO orchestration_events (
+        id, ts, event_type, source, severity,
+        task_id, agent_id, reviewer, session_scope, project_name,
+        instance_id, tmux_session, runtime, correlation_id, attempt,
+        duration_ms, result, error_code, payload_json
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        crypto.randomUUID(),
+        input.timestamp ?? (/* @__PURE__ */ new Date()).toISOString(),
+        input.eventType,
+        input.source,
+        input.severity ?? "info",
+        input.taskId ?? null,
+        input.agentId ?? null,
+        input.reviewer ?? null,
+        input.sessionScope ?? null,
+        input.projectName ?? null,
+        input.instanceId ?? null,
+        input.tmuxSession ?? null,
+        input.runtime ?? null,
+        input.correlationId ?? null,
+        input.attempt ?? null,
+        input.durationMs ?? null,
+        input.result ?? null,
+        input.errorCode ?? null,
+        payloadJson
+      ]
+    });
+    return true;
+  } catch (err) {
+    if (process.env.EXE_DEBUG === "1") {
+      process.stderr.write(
+        `[orchestration-events] write skipped: ${err instanceof Error ? err.message : String(err)}
+`
+      );
+    }
+    return false;
+  }
+}
+function recordOrchestrationEventBestEffort(input) {
+  if (bestEffortPending >= BEST_EFFORT_QUEUE_LIMIT) {
+    if (process.env.EXE_DEBUG === "1") {
+      process.stderr.write(
+        `[orchestration-events] best-effort queue saturated; dropped ${input.eventType}
+`
+      );
+    }
+    return;
+  }
+  bestEffortPending++;
+  bestEffortQueue = bestEffortQueue.then(async () => {
+    try {
+      await recordOrchestrationEvent(input);
+    } finally {
+      bestEffortPending = Math.max(0, bestEffortPending - 1);
+    }
+  }).catch((err) => {
+    if (process.env.EXE_DEBUG === "1") {
+      process.stderr.write(
+        `[orchestration-events] best-effort write skipped: ${err instanceof Error ? err.message : String(err)}
+`
+      );
+    }
+  });
+}
+async function getOrchestrationMetrics(input = {}) {
+  const client = getClient();
+  await ensureOrchestrationEventsSchema(client);
+  const sinceIso = input.sinceIso ?? null;
+  const result = sinceIso ? await client.execute({
+    sql: `SELECT event_type, ts, task_id, tmux_session, agent_id, reviewer, attempt, duration_ms, result, error_code
+            FROM orchestration_events
+            WHERE ts >= ?
+            ORDER BY ts ASC`,
+    args: [sinceIso]
+  }) : await client.execute({
+    sql: `SELECT event_type, ts, task_id, tmux_session, agent_id, reviewer, attempt, duration_ms, result, error_code
+            FROM orchestration_events
+            ORDER BY ts ASC`,
+    args: []
+  });
+  const rows = result.rows;
+  const taskEvents = /* @__PURE__ */ new Map();
+  const sessionEvents = /* @__PURE__ */ new Map();
+  const daemonTickDurations = [];
+  const resumeTimestamps = /* @__PURE__ */ new Map();
+  const reviewLatenciesByReviewer = /* @__PURE__ */ new Map();
+  const counts = {
+    duplicateNudges: 0,
+    missedReviews: 0,
+    tmuxCommandTimeouts: 0,
+    claimCollisions: 0,
+    crossSessionBlocks: 0,
+    dispatchUnclaimed: 0,
+    sessionExits: 0,
+    sessionBorns: 0,
+    sessionIdleKills: 0,
+    sessionTtlKills: 0,
+    sessionContextKills: 0,
+    sessionResumed: 0,
+    eventLoopBlocks: 0,
+    shardCircuitOpens: 0,
+    taskBlocked: 0,
+    taskCancelled: 0,
+    checkpoints: 0,
+    messagesSent: 0,
+    reviewsCreated: 0,
+    reviewsApproved: 0,
+    reviewsRejected: 0,
+    dispatches: 0,
+    signalsCreated: 0,
+    signalsConsumed: 0,
+    workerInstancesSelected: 0,
+    contextPressure: 0,
+    mcpUnreachable: 0,
+    dispatchAttempted: 0,
+    dispatchFailed: 0,
+    reviewStarted: 0,
+    messagesDelivered: 0,
+    messagesRead: 0,
+    signalsStale: 0,
+    tmuxNudgeAttempted: 0,
+    tmuxNudgeCompleted: 0,
+    tmuxSpawnAttempted: 0,
+    tmuxSpawnCompleted: 0,
+    consistencyMismatches: 0,
+    consistencyRepaired: 0,
+    fileRowsResynced: 0,
+    fileOrphansCleaned: 0,
+    worktreeCreated: 0,
+    worktreeReused: 0,
+    worktreeFailed: 0,
+    worktreePruned: 0,
+    worktreeSkipped: 0,
+    registryMissingTmux: 0,
+    registryMissingEntry: 0,
+    taskGroupsCreated: 0,
+    taskGroupBarriersFired: 0,
+    taskGroupTimeouts: 0,
+    taskGroupPartialFailures: 0
+  };
+  for (const row of rows) {
+    const type = String(row.event_type ?? "");
+    const taskId = row.task_id == null ? "" : String(row.task_id);
+    const ts = Date.parse(String(row.ts ?? ""));
+    const eventCount = eventWeight(row);
+    if (type === "nudge.duplicate") counts.duplicateNudges++;
+    if (type === "review.missed") counts.missedReviews++;
+    if (type === "tmux.command.timeout") counts.tmuxCommandTimeouts++;
+    if (type === "claim.collision") counts.claimCollisions++;
+    if (type === "session.scope_violation") counts.crossSessionBlocks++;
+    if (type === "dispatch.unclaimed") counts.dispatchUnclaimed++;
+    if (type === "session.exit") counts.sessionExits++;
+    if (type === "session.born") counts.sessionBorns++;
+    if (type === "session.idle_kill") counts.sessionIdleKills++;
+    if (type === "session.ttl_kill") counts.sessionTtlKills++;
+    if (type === "session.context_kill") counts.sessionContextKills++;
+    if (type === "session.resumed") counts.sessionResumed++;
+    if (type === "daemon.event_loop_blocked") counts.eventLoopBlocks++;
+    if (type === "shard.circuit_open") counts.shardCircuitOpens++;
+    if (type === "task.blocked") counts.taskBlocked++;
+    if (type === "task.cancelled") counts.taskCancelled++;
+    if (type === "task.checkpoint") counts.checkpoints++;
+    if (type === "message.sent") counts.messagesSent++;
+    if (type === "review.created") counts.reviewsCreated++;
+    if (type === "review.approved") counts.reviewsApproved++;
+    if (type === "review.rejected") counts.reviewsRejected++;
+    if (type === "dispatch.completed") counts.dispatches++;
+    if (type === "signal.created") counts.signalsCreated++;
+    if (type === "signal.consumed") counts.signalsConsumed++;
+    if (type === "worker.instance.selected") counts.workerInstancesSelected++;
+    if (type === "context.pressure") counts.contextPressure++;
+    if (type === "mcp.unreachable") counts.mcpUnreachable++;
+    if (type === "dispatch.attempted") counts.dispatchAttempted++;
+    if (type === "dispatch.failed") counts.dispatchFailed++;
+    if (type === "review.started") counts.reviewStarted++;
+    if (type === "message.delivered") counts.messagesDelivered++;
+    if (type === "message.read") counts.messagesRead++;
+    if (type === "signal.stale") counts.signalsStale++;
+    if (type === "tmux.nudge.attempted") counts.tmuxNudgeAttempted++;
+    if (type === "tmux.nudge.completed") {
+      counts.tmuxNudgeCompleted++;
+      if (String(row.result ?? "") === "debounced") counts.duplicateNudges++;
+    }
+    if (type === "tmux.spawn.attempted") counts.tmuxSpawnAttempted++;
+    if (type === "tmux.spawn.completed") counts.tmuxSpawnCompleted++;
+    if (type === "consistency.mismatch") counts.consistencyMismatches++;
+    if (type === "consistency.repaired") counts.consistencyRepaired++;
+    if (type === "task_file.resynced") counts.fileRowsResynced++;
+    if (type === "task_file.orphan_cleaned") counts.fileOrphansCleaned++;
+    if (type === "worktree.created") counts.worktreeCreated++;
+    if (type === "worktree.reused") counts.worktreeReused++;
+    if (type === "worktree.failed") counts.worktreeFailed++;
+    if (type === "worktree.pruned") counts.worktreePruned++;
+    if (type === "worktree.skipped") counts.worktreeSkipped++;
+    if (type === "registry.missing_tmux") counts.registryMissingTmux += eventCount;
+    if (type === "registry.missing_entry") counts.registryMissingEntry += eventCount;
+    if (type === "task_group.created") counts.taskGroupsCreated++;
+    if (type === "task_group.barrier_fired") counts.taskGroupBarriersFired++;
+    if (type === "task_group.timeout") counts.taskGroupTimeouts++;
+    if (type === "task_group.partial_failure") counts.taskGroupPartialFailures++;
+    if (type.startsWith("tmux.") && String(row.error_code ?? "").toLowerCase().includes("timeout")) {
+      counts.tmuxCommandTimeouts++;
+    }
+    if (type === "session.resumed" && Number.isFinite(ts)) {
+      const agentId = row.agent_id == null ? "" : String(row.agent_id);
+      if (agentId) {
+        const list = resumeTimestamps.get(agentId) ?? [];
+        list.push(ts);
+        resumeTimestamps.set(agentId, list);
+      }
+    }
+    if (type === "daemon.tick.completed") {
+      const duration = Number(row.duration_ms ?? NaN);
+      if (Number.isFinite(duration)) daemonTickDurations.push(duration);
+    }
+    const tmuxSess = row.tmux_session == null ? "" : String(row.tmux_session);
+    if (tmuxSess && Number.isFinite(ts)) {
+      if (type === "session.born") sessionEvents.set(tmuxSess, { ...sessionEvents.get(tmuxSess) ?? {}, born: ts });
+      if ((type === "session.exit" || type === "session.idle_kill" || type === "session.ttl_kill" || type === "session.context_kill") && !sessionEvents.get(tmuxSess)?.exited) {
+        const entry = sessionEvents.get(tmuxSess) ?? {};
+        entry.exited = ts;
+        const hookDuration = Number(row.duration_ms ?? NaN);
+        if (Number.isFinite(hookDuration)) entry.durationMs = hookDuration;
+        sessionEvents.set(tmuxSess, entry);
+      }
+    }
+    if (!taskId || !Number.isFinite(ts)) continue;
+    const bucket = taskEvents.get(taskId) ?? {};
+    if (type === "task.created" && bucket.created === void 0) bucket.created = ts;
+    if (type === "task.claimed" && bucket.claimed === void 0) bucket.claimed = ts;
+    if (type === "task.blocked" && bucket.blocked === void 0) bucket.blocked = ts;
+    if (type === "task.unblocked" && bucket.unblocked === void 0) bucket.unblocked = ts;
+    if (type === "task.done" && bucket.done === void 0) bucket.done = ts;
+    if (type === "review.ready" && bucket.reviewReady === void 0) {
+      bucket.reviewReady = ts;
+      if (row.reviewer != null) bucket.reviewer = String(row.reviewer);
+    }
+    if (type === "review.started" && bucket.reviewStarted === void 0) bucket.reviewStarted = ts;
+    if (type === "review.approved" && bucket.reviewApproved === void 0) bucket.reviewApproved = ts;
+    if (type === "review.rejected" && bucket.reviewRejected === void 0) bucket.reviewRejected = ts;
+    if (type === "signal.created" && bucket.signalCreated === void 0) bucket.signalCreated = ts;
+    if (type === "signal.consumed" && bucket.signalConsumed === void 0) bucket.signalConsumed = ts;
+    if (type === "dispatch.attempted" && bucket.dispatched === void 0) bucket.dispatched = ts;
+    taskEvents.set(taskId, bucket);
+  }
+  const createToClaim = [];
+  const doneToReview = [];
+  const reviewToApproved = [];
+  const reviewToStarted = [];
+  const signalToConsumed = [];
+  const dispatchToClaim = [];
+  const blockedDuration = [];
+  for (const eventSet of taskEvents.values()) {
+    if (eventSet.created !== void 0 && eventSet.claimed !== void 0 && eventSet.claimed >= eventSet.created) {
+      createToClaim.push(eventSet.claimed - eventSet.created);
+    }
+    if (eventSet.done !== void 0 && eventSet.reviewReady !== void 0 && eventSet.reviewReady >= eventSet.done) {
+      doneToReview.push(eventSet.reviewReady - eventSet.done);
+    }
+    if (eventSet.reviewReady !== void 0 && eventSet.reviewApproved !== void 0 && eventSet.reviewApproved >= eventSet.reviewReady) {
+      reviewToApproved.push(eventSet.reviewApproved - eventSet.reviewReady);
+    }
+    if (eventSet.reviewReady !== void 0 && eventSet.reviewStarted !== void 0 && eventSet.reviewStarted >= eventSet.reviewReady) {
+      reviewToStarted.push(eventSet.reviewStarted - eventSet.reviewReady);
+    }
+    if (eventSet.signalCreated !== void 0 && eventSet.signalConsumed !== void 0 && eventSet.signalConsumed >= eventSet.signalCreated) {
+      signalToConsumed.push(eventSet.signalConsumed - eventSet.signalCreated);
+    }
+    if (eventSet.dispatched !== void 0 && eventSet.claimed !== void 0 && eventSet.claimed >= eventSet.dispatched) {
+      dispatchToClaim.push(eventSet.claimed - eventSet.dispatched);
+    }
+    if (eventSet.blocked !== void 0 && eventSet.unblocked !== void 0 && eventSet.unblocked >= eventSet.blocked) {
+      blockedDuration.push(eventSet.unblocked - eventSet.blocked);
+    }
+    if (eventSet.dispatched !== void 0 && eventSet.claimed === void 0 && Date.now() - eventSet.dispatched >= 5 * 6e4) {
+      counts.dispatchUnclaimed++;
+    }
+  }
+  const STORM_WINDOW_MS = 30 * 6e4;
+  const STORM_THRESHOLD = 3;
+  const resumeStorms = [];
+  for (const [agentId, timestamps] of resumeTimestamps) {
+    if (timestamps.length < STORM_THRESHOLD) continue;
+    const sorted = [...timestamps].sort((a, b) => a - b);
+    let maxInWindow = 0;
+    for (let i = 0; i < sorted.length; i++) {
+      let j = i;
+      while (j < sorted.length && sorted[j] - sorted[i] <= STORM_WINDOW_MS) j++;
+      maxInWindow = Math.max(maxInWindow, j - i);
+    }
+    if (maxInWindow >= STORM_THRESHOLD) {
+      resumeStorms.push({ agentId, count: maxInWindow, windowMinutes: 30 });
+    }
+  }
+  for (const row of rows) {
+    const type = String(row.event_type ?? "");
+    if (type !== "review.approved" && type !== "review.rejected") continue;
+    const reviewerStr = row.reviewer == null ? "" : String(row.reviewer);
+    const taskId = row.task_id == null ? "" : String(row.task_id);
+    if (!reviewerStr || !taskId) continue;
+    const bucket = taskEvents.get(taskId);
+    if (bucket?.reviewReady !== void 0) {
+      const ts = Date.parse(String(row.ts ?? ""));
+      if (Number.isFinite(ts) && ts >= bucket.reviewReady) {
+        const latencies = reviewLatenciesByReviewer.get(reviewerStr) ?? [];
+        latencies.push(ts - bucket.reviewReady);
+        reviewLatenciesByReviewer.set(reviewerStr, latencies);
+      }
+    }
+  }
+  const BOTTLENECK_P95_MS = 30 * 6e4;
+  const reviewerBottlenecks = [];
+  for (const [reviewer, latencies] of reviewLatenciesByReviewer) {
+    if (latencies.length < 2) continue;
+    const p95 = percentile([...latencies].sort((a, b) => a - b), 95);
+    if (p95 > BOTTLENECK_P95_MS) {
+      reviewerBottlenecks.push({ reviewer, pendingCount: latencies.length, p95Ms: p95 });
+    }
+  }
+  const BACKLOG_THRESHOLD_MS = 30 * 6e4;
+  const reviewerBacklogMap = /* @__PURE__ */ new Map();
+  const nowMs = Date.now();
+  for (const eventSet of taskEvents.values()) {
+    if (eventSet.reviewReady === void 0 || eventSet.reviewApproved !== void 0 || eventSet.reviewRejected !== void 0) continue;
+    const ageMs = nowMs - eventSet.reviewReady;
+    if (ageMs < BACKLOG_THRESHOLD_MS) continue;
+    const reviewer = eventSet.reviewer ?? "unknown";
+    const current = reviewerBacklogMap.get(reviewer) ?? { pendingCount: 0, oldestMs: 0 };
+    current.pendingCount++;
+    current.oldestMs = Math.max(current.oldestMs, ageMs);
+    reviewerBacklogMap.set(reviewer, current);
+  }
+  const reviewerBacklogs = [...reviewerBacklogMap.entries()].map(([reviewer, value]) => ({ reviewer, ...value })).sort((a, b) => b.oldestMs - a.oldestMs);
+  const sessionDurations = [];
+  for (const sess of sessionEvents.values()) {
+    if (sess.durationMs !== void 0) {
+      sessionDurations.push(sess.durationMs);
+    } else if (sess.born !== void 0 && sess.exited !== void 0 && sess.exited >= sess.born) {
+      sessionDurations.push(sess.exited - sess.born);
+    }
+  }
+  return {
+    sinceIso,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    totalEvents: rows.length,
+    counts,
+    createToClaimMs: percentileMetric(createToClaim),
+    doneToReviewMs: percentileMetric(doneToReview),
+    reviewToApprovedMs: percentileMetric(reviewToApproved),
+    reviewToStartedMs: percentileMetric(reviewToStarted),
+    signalToConsumedMs: percentileMetric(signalToConsumed),
+    dispatchToClaimMs: percentileMetric(dispatchToClaim),
+    blockedDurationMs: percentileMetric(blockedDuration),
+    sessionDurationMs: percentileMetric(sessionDurations),
+    daemonTickMs: percentileMetric(daemonTickDurations),
+    resumeStorms,
+    reviewerBottlenecks,
+    reviewerBacklogs
+  };
+}
+function formatOrchestrationMetrics(metrics) {
+  const lines = [
+    "# Orchestration Metrics",
+    "",
+    `Generated: ${metrics.generatedAt}`,
+    `Since: ${metrics.sinceIso ?? "beginning"}`,
+    `Total events: ${metrics.totalEvents}`,
+    "",
+    "## Latency",
+    `- create\u2192claim: ${formatMetric(metrics.createToClaimMs)}`,
+    `- dispatch\u2192claim: ${formatMetric(metrics.dispatchToClaimMs)}`,
+    `- done\u2192review.ready: ${formatMetric(metrics.doneToReviewMs)}`,
+    `- review.ready\u2192started: ${formatMetric(metrics.reviewToStartedMs)}`,
+    `- review.ready\u2192approved: ${formatMetric(metrics.reviewToApprovedMs)}`,
+    `- signal.created\u2192consumed: ${formatMetric(metrics.signalToConsumedMs)}`,
+    `- task blocked duration: ${formatMetric(metrics.blockedDurationMs)}`,
+    `- session duration (born\u2192exit): ${formatMetric(metrics.sessionDurationMs)}`,
+    `- daemon tick: ${formatMetric(metrics.daemonTickMs)}`,
+    "",
+    "## Task Lifecycle",
+    `- dispatches: ${metrics.counts.dispatches}`,
+    `- checkpoints: ${metrics.counts.checkpoints}`,
+    `- reviews created: ${metrics.counts.reviewsCreated}`,
+    `- reviews started: ${metrics.counts.reviewStarted}`,
+    `- reviews approved: ${metrics.counts.reviewsApproved}`,
+    `- reviews rejected: ${metrics.counts.reviewsRejected}`,
+    `- tasks blocked: ${metrics.counts.taskBlocked}`,
+    `- tasks cancelled: ${metrics.counts.taskCancelled}`,
+    `- messages sent: ${metrics.counts.messagesSent}`,
+    `- messages delivered: ${metrics.counts.messagesDelivered}`,
+    `- messages read: ${metrics.counts.messagesRead}`,
+    "",
+    "## Sessions & Workers",
+    `- sessions born: ${metrics.counts.sessionBorns}`,
+    `- session exits: ${metrics.counts.sessionExits}`,
+    `- idle kills: ${metrics.counts.sessionIdleKills}`,
+    `- TTL kills: ${metrics.counts.sessionTtlKills}`,
+    `- context kills: ${metrics.counts.sessionContextKills}`,
+    `- session resumes: ${metrics.counts.sessionResumed}`,
+    `- worker instances selected: ${metrics.counts.workerInstancesSelected}`,
+    `- tmux spawn attempted: ${metrics.counts.tmuxSpawnAttempted}`,
+    `- tmux spawn completed: ${metrics.counts.tmuxSpawnCompleted}`,
+    `- worktrees created: ${metrics.counts.worktreeCreated}`,
+    `- worktrees reused: ${metrics.counts.worktreeReused}`,
+    `- worktrees failed: ${metrics.counts.worktreeFailed}`,
+    `- worktrees pruned: ${metrics.counts.worktreePruned}`,
+    `- worktrees skipped: ${metrics.counts.worktreeSkipped}`,
+    "",
+    "## Dispatch & Signals",
+    `- dispatch attempted: ${metrics.counts.dispatchAttempted}`,
+    `- dispatch failed: ${metrics.counts.dispatchFailed}`,
+    `- tmux nudges attempted: ${metrics.counts.tmuxNudgeAttempted}`,
+    `- tmux nudges completed: ${metrics.counts.tmuxNudgeCompleted}`,
+    `- signals created: ${metrics.counts.signalsCreated}`,
+    `- signals consumed: ${metrics.counts.signalsConsumed}`,
+    `- signals stale (cleaned): ${metrics.counts.signalsStale}`,
+    `- context pressure (>70%): ${metrics.counts.contextPressure}`,
+    "",
+    "## Errors & Anomalies",
+    `- claim collisions: ${metrics.counts.claimCollisions}`,
+    `- duplicate nudges: ${metrics.counts.duplicateNudges}`,
+    `- missed reviews: ${metrics.counts.missedReviews}`,
+    `- dispatch unclaimed: ${metrics.counts.dispatchUnclaimed}`,
+    `- DB/file consistency mismatches: ${metrics.counts.consistencyMismatches}`,
+    `- file rows resynced into DB: ${metrics.counts.fileRowsResynced}`,
+    `- file orphans cleaned: ${metrics.counts.fileOrphansCleaned}`,
+    `- registry missing tmux sessions: ${metrics.counts.registryMissingTmux}`,
+    `- registry missing entries: ${metrics.counts.registryMissingEntry}`,
+    `- cross-session blocks: ${metrics.counts.crossSessionBlocks}`,
+    `- tmux command timeouts: ${metrics.counts.tmuxCommandTimeouts}`,
+    `- MCP unreachable: ${metrics.counts.mcpUnreachable}`,
+    "",
+    "## Task Groups (Fan-out/Fan-in)",
+    `- groups created: ${metrics.counts.taskGroupsCreated}`,
+    `- barriers fired: ${metrics.counts.taskGroupBarriersFired}`,
+    `- group timeouts: ${metrics.counts.taskGroupTimeouts}`,
+    `- partial failures: ${metrics.counts.taskGroupPartialFailures}`,
+    "",
+    "## Daemon Health",
+    `- event loop blocks: ${metrics.counts.eventLoopBlocks}`,
+    `- shard circuit opens: ${metrics.counts.shardCircuitOpens}`,
+    "",
+    "## Derived Analysis",
+    `- RESUME storms: ${metrics.resumeStorms.length === 0 ? "none" : metrics.resumeStorms.map((s) => `${s.agentId} (${s.count}x in ${s.windowMinutes}min)`).join(", ")}`,
+    `- reviewer bottlenecks: ${metrics.reviewerBottlenecks.length === 0 ? "none" : metrics.reviewerBottlenecks.map((b) => `${b.reviewer} (p95=${formatMs(b.p95Ms)}, n=${b.pendingCount})`).join(", ")}`,
+    `- reviewer backlogs: ${metrics.reviewerBacklogs.length === 0 ? "none" : metrics.reviewerBacklogs.map((b) => `${b.reviewer} (${b.pendingCount} pending, oldest=${formatMs(b.oldestMs)})`).join(", ")}`
+  ];
+  return lines.join("\n");
+}
+function sanitizePayload(payload) {
+  if (!payload) return null;
+  const safe = {};
+  for (const [key, value] of Object.entries(payload)) {
+    if (SENSITIVE_PAYLOAD_KEY.test(key)) continue;
+    if (value == null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+      safe[key] = value;
+    } else if (Array.isArray(value)) {
+      safe[key] = value.filter((item) => ["string", "number", "boolean"].includes(typeof item)).slice(0, 20);
+    }
+  }
+  return Object.keys(safe).length > 0 ? JSON.stringify(safe) : null;
+}
+function percentileMetric(values) {
+  if (values.length === 0) return { count: 0, p50: null, p95: null, p99: null };
+  const sorted = [...values].sort((a, b) => a - b);
+  return {
+    count: sorted.length,
+    p50: percentile(sorted, 50),
+    p95: percentile(sorted, 95),
+    p99: percentile(sorted, 99)
+  };
+}
+function percentile(sorted, percentileValue) {
+  if (sorted.length === 0) return 0;
+  const rank = Math.ceil(percentileValue / 100 * sorted.length) - 1;
+  return sorted[Math.min(sorted.length - 1, Math.max(0, rank))] ?? 0;
+}
+function eventWeight(row) {
+  const value = Number(row.attempt ?? 1);
+  if (!Number.isFinite(value) || value < 1) return 1;
+  return Math.floor(value);
+}
+function formatMetric(metric) {
+  if (metric.count === 0) return "n=0";
+  return `n=${metric.count}, p50=${formatMs(metric.p50)}, p95=${formatMs(metric.p95)}, p99=${formatMs(metric.p99)}`;
+}
+function formatMs(value) {
+  if (value == null) return "n/a";
+  if (value < 1e3) return `${value}ms`;
+  return `${Math.round(value / 1e3 * 10) / 10}s`;
+}
+
+export {
+  ORCHESTRATION_COUNTER_KEYS,
+  ORCHESTRATION_LATENCY_METRIC_KEYS,
+  ensureOrchestrationEventsSchema,
+  recordOrchestrationEvent,
+  recordOrchestrationEventBestEffort,
+  getOrchestrationMetrics,
+  formatOrchestrationMetrics
+};