@llblab/pi-actors 0.43.1 → 0.44.0

package/dist/lib/tools-inspect.js

@@ -3,11 +3,13 @@
  * Zones: Run Recipe/Trace/Control views and runtime/recipe/tool diagnostics
  * Owns exact inspect target/view dispatch; source projection stays in domain modules.
  */
+import { statSync } from "node:fs";
 import { join } from "node:path";
 import * as AsyncRuns from "./async-runs.js";
 import * as ControlProjection from "./control-projection.js";
 import * as Inspector from "./inspector.js";
 import * as Limits from "./limits.js";
+import { computeRunControlCapacity } from "./run-evidence-policy.js";
 import * as Paths from "./paths.js";
 import * as RecipesDiscovery from "./recipes-discovery.js";
 import * as ReviewDiagnostics from "./review-diagnostics.js";
@@ -55,7 +57,9 @@ function runtimeTriage(ctx, deps) {
     const failed = runs.filter((run) => run.status === "failed");
     const pendingControls = [];
     const staleControls = [];
+    const backpressuredRuns = [];
     const controlDiagnostics = [];
+    const traceDiagnostics = [];
     const nowMs = Date.now();
     for (const run of runs) {
         const stateDir = typeof run.state_dir === "string" ? run.state_dir : undefined;
@@ -63,13 +67,19 @@ function runtimeTriage(ctx, deps) {
             continue;
         const runId = String(run.run ?? "unknown");
         const journal = RunsControls.readRunControlJournalFromStateDir(stateDir);
+        const capacity = computeRunControlCapacity(journal.records);
         controlDiagnostics.push(...journal.diagnostics.map((diagnostic) => ({
             ...(diagnostic.line === undefined ? {} : { line: diagnostic.line }),
-            reason: diagnostic.line === undefined
-                ? "unreadable_control_journal"
-                : "invalid_control_json",
-            run: runId,
+            reason: diagnostic.line === undefined ? "unreadable_control_journal" : "invalid_control_json", run: runId,
         })));
+        if (capacity.backpressured)
+            backpressuredRuns.push({
+                run: runId, pending: capacity.pending, pending_limit: capacity.limit,
+                journal_bytes: statSync(RunsControls.runControlsFile(stateDir)).size,
+            });
+        const traceSummary = RunsTrace.summarizeRunTraceJournal(RunsTrace.readRunTraceJournal(stateDir));
+        if (!traceSummary.history_complete)
+            traceDiagnostics.push({ run: runId, ...traceSummary });
         for (const control of journal.records) {
             const classified = RuntimeTriage.classifyRuntimeControl({
                 run: runId,
@@ -114,12 +124,13 @@ function runtimeTriage(ctx, deps) {
         pending_controls: boundedPendingControls,
         stale_control_count: staleControls.length,
         stale_controls: boundedStaleControls,
-        control_diagnostic_count: controlDiagnostics.length,
-        control_diagnostics: controlDiagnostics.slice(0, 40),
+        backpressured_run_count: backpressuredRuns.length, backpressured_runs: backpressuredRuns.slice(0, 40),
+        control_diagnostic_count: controlDiagnostics.length, control_diagnostics: controlDiagnostics.slice(0, 40),
+        trace_diagnostic_count: traceDiagnostics.length, trace_diagnostics: traceDiagnostics.slice(0, 40),
         attention_events: attentionEvents.slice(-40).reverse(),
         next_actions: [
             ...(failed.length ? ["inspect target=runtime view=runs status=failed"] : []),
-            ...staleControls.slice(0, 3).map((entry) => `inspect target=run:${entry.run} view=control`),
+            ...new Set([...staleControls, ...backpressuredRuns].slice(0, 3).map((entry) => `inspect target=run:${entry.run} view=control`)),
             ...(attentionEvents.length
                 ? attentionEvents.slice(-3).map((entry) => `inspect target=run:${String(entry.run)} view=trace source=runtime`)
                 : []),
@@ -131,7 +142,7 @@ function compactRuntime(view, details) {
         return `\nruntime runs=${details.runs?.length ?? 0}`;
     }
     if (view === "triage") {
-        return `\nruntime failed=${details.failed_runs?.length ?? 0} pending_controls=${Number(details.pending_control_count ?? 0)} stale_controls=${Number(details.stale_control_count ?? 0)} attention=${details.attention_events?.length ?? 0}`;
+        return `\nruntime failed=${details.failed_runs?.length ?? 0} pending_controls=${Number(details.pending_control_count ?? 0)} stale_controls=${Number(details.stale_control_count ?? 0)} backpressured=${Number(details.backpressured_run_count ?? 0)} incomplete_trace=${Number(details.trace_diagnostic_count ?? 0)} attention=${details.attention_events?.length ?? 0}`;
     }
     return `\nruntime version=${String(details.version)} state_schema=${String(details.state_schema)} automatic_review=${String(details.automatic_review)}`;
 }
@@ -199,8 +210,8 @@ function inspectRun(run, view, input, ctx, deps) {
             throw new Error(`unsupported Trace source: ${source}`);
         }
         return {
-            run,
-            source,
+            run, source,
+            summary: RunsTrace.summarizeRunTraceJournal(RunsTrace.readRunTraceJournal(stateDir)),
             items: TraceProjection.projectRunTrace(stateDir, {
                 artifacts: status.artifacts,
                 limit: Number(input.lines || Limits.DEFAULT_INSPECT_LINES),
@@ -209,19 +220,42 @@ function inspectRun(run, view, input, ctx, deps) {
         };
     }
     const runInstanceId = String(status.run_instance_id ?? "");
+    const journal = RunsControls.readRunControlJournalFromStateDir(stateDir);
+    const capacity = computeRunControlCapacity(journal.records);
+    const structuralDiagnostics = journal.records.flatMap((value) => {
+        const classified = RuntimeTriage.classifyRuntimeControl({ run,
+            runInstanceId: runInstanceId || undefined, status: String(status.status ?? "unknown") }, value, Date.now());
+        return classified.diagnostic ? [classified.diagnostic.reason] : [];
+    });
+    let journalBytes = 0;
+    try {
+        journalBytes = statSync(RunsControls.runControlsFile(stateDir)).size;
+    }
+    catch { }
+    const diagnostics = [...journal.diagnostics.map((item) => ({
+            ...(item.line === undefined ? {} : { line: item.line }),
+            reason: item.line === undefined ? "unreadable_control_journal" : "invalid_control_json",
+        })), ...structuralDiagnostics.map((reason) => ({ reason }))].slice(0, 40);
+    const nowMs = Date.now();
+    const stalePending = journal.records.reduce((count, control) => count + Number(Boolean(RuntimeTriage.classifyRuntimeControl({ run, runInstanceId: runInstanceId || undefined,
+        status: String(status.status ?? "unknown") }, control, nowMs).stale)), 0);
     return {
         run,
         run_instance_id: runInstanceId,
         status: status.status,
+        pending: capacity.pending, pending_limit: capacity.limit,
+        available: capacity.available, backpressured: capacity.backpressured,
+        journal_bytes: journalBytes, journal_limit: Limits.RUN_CONTROL_JOURNAL_MAX_BYTES,
+        ...(capacity.oldest_pending_at ? { oldest_pending_at: capacity.oldest_pending_at } : {}),
+        stale_pending: stalePending,
+        diagnostics,
         actor_actions: Array.isArray(status.control) ? status.control : [],
         runtime_actions: status.status === "running" ? ["kill"] : ["archive", "prune"],
         endpoint: runInstanceId
             ? RunControlDelivery.readRunControlEndpoint(stateDir, runInstanceId)
             : undefined,
-        recent_controls: RunsControls.readRunControlsFromStateDir(stateDir)
-            .slice(-Math.max(1, Number(input.lines || 20)))
-            .reverse()
-            .map(ControlProjection.projectRunControl),
+        recent_controls: journal.records.slice(-Math.max(1, Number(input.lines || 20))).reverse()
+            .map((control) => ControlProjection.projectRunControl(control)),
     };
 }
 function inspectTool(name, view, deps) {

package/dist/lib/trace-projection.js

@@ -9,11 +9,21 @@ import * as ControlProjection from "./control-projection.js";
 import * as ExecutionSessions from "./execution-sessions.js";
 import * as Limits from "./limits.js";
 import { resolveArtifactManifest, } from "./runs-artifacts.js";
+import * as RunsControls from "./runs-controls.js";
 import { tailFile } from "./runs-status.js";
+import * as RunsTrace from "./runs-trace.js";
 import * as SessionEvidence from "./session-evidence.js";
-import { readJsonFileResilient, readJsonlFileResilient, } from "./state-readers.js";
+import { readJsonFileResilient, } from "./state-readers.js";
 const LIFECYCLE_PREFIXES = ["run.", "command."];
 const SOURCE_LIMIT = 100;
+const SOURCE_RANK = {
+    lifecycle: 6,
+    control: 5,
+    process: 4,
+    agent: 3,
+    artifact: 2,
+    runtime: 1,
+};
 function redact(value) {
     return SessionEvidence.redactSessionEvidenceValue(value);
 }
@@ -22,6 +32,21 @@ function sourceForEvent(event) {
         ? "lifecycle"
         : "runtime";
 }
+function ordered(item, sourceKey, ordinal) {
+    return { item, ordinal, sourceKey };
+}
+function compareTraceItems(left, right) {
+    const timestamp = right.item.ts.localeCompare(left.item.ts);
+    if (timestamp)
+        return timestamp;
+    if (left.sourceKey === right.sourceKey) {
+        const ordinal = right.ordinal - left.ordinal;
+        if (ordinal)
+            return ordinal;
+    }
+    return SOURCE_RANK[right.item.source] - SOURCE_RANK[left.item.source] ||
+        right.item.id.localeCompare(left.item.id);
+}
 function latestControlTimestamp(control) {
     return (control.failed_at ??
         control.handled_at ??
@@ -57,39 +82,57 @@ function controlDiagnosticItems(diagnostics) {
     }));
 }
 function traceEventItems(stateDir) {
-    const read = readJsonlFileResilient(join(stateDir, "trace.jsonl"));
-    return [
-        ...read.records.slice(-SOURCE_LIMIT).map((event) => ({
-            id: event.id,
-            ts: event.ts,
-            source: sourceForEvent(event),
-            kind: event.kind,
-            summary: event.summary ?? event.kind,
-            ...(event.level ? { level: event.level } : {}),
-            detail: redact({
-                ...(event.attention ? { attention: event.attention } : {}),
-                ...(event.data !== undefined ? { data: event.data } : {}),
-            }),
-        })),
-        ...diagnosticItems(read.diagnostics),
-    ];
+    const read = RunsTrace.readRunTraceJournal(stateDir);
+    const retained = read.events.slice(-SOURCE_LIMIT);
+    const projected = retained.map(({ event, ordinal }) => ordered({
+        id: event.id,
+        ts: event.ts,
+        source: sourceForEvent(event),
+        kind: event.kind,
+        summary: event.summary ?? event.kind,
+        ...(event.level ? { level: event.level } : {}),
+        detail: redact({
+            ...(event.attention ? { attention: event.attention } : {}),
+            ...(event.data !== undefined ? { data: event.data } : {}),
+        }),
+    }, "trace", ordinal));
+    const diagnostics = diagnosticItems(read.diagnostics).map((item, index) => ordered(item, "trace", read.events.length + index + 1));
+    const hasMarker = read.events.some(({ event }) => event.kind === "runtime.trace_compacted");
+    if (read.omittedPrefixBytes > 0 && !hasMarker) {
+        diagnostics.push(ordered({
+            id: "runtime:trace-history-incomplete",
+            ts: new Date(0).toISOString(),
+            source: "runtime",
+            kind: "runtime.trace_history_incomplete",
+            summary: "Trace history is incomplete",
+            level: "warning",
+            detail: {
+                omitted_prefix_bytes: read.omittedPrefixBytes,
+                read_bytes: read.readBytes,
+                retained_events: read.events.length,
+            },
+        }, "trace", 0));
+    }
+    return [...projected, ...diagnostics];
 }
 function controlItems(stateDir) {
-    const read = readJsonlFileResilient(join(stateDir, "controls.jsonl"));
+    const read = RunsControls.readRunControlJournalFromStateDir(stateDir);
+    const start = Math.max(0, read.records.length - SOURCE_LIMIT);
+    const projected = read.records.slice(start).map((control, index) => {
+        const value = ControlProjection.projectRunControl(control);
+        return ordered({
+            id: `control:${value.id}`,
+            ts: latestControlTimestamp(value),
+            source: "control",
+            kind: `control.${value.status}`,
+            summary: `${value.action} ${value.status}`,
+            ...(value.status === "failed" ? { level: "error" } : {}),
+            detail: value,
+        }, "control", start + index + 1);
+    });
     return [
-        ...read.records.slice(-SOURCE_LIMIT).map((control) => {
-            const projected = ControlProjection.projectRunControl(control);
-            return {
-                id: `control:${projected.id}`,
-                ts: latestControlTimestamp(projected),
-                source: "control",
-                kind: `control.${projected.status}`,
-                summary: `${projected.action} ${projected.status}`,
-                ...(projected.status === "failed" ? { level: "error" } : {}),
-                detail: projected,
-            };
-        }),
-        ...controlDiagnosticItems(read.diagnostics),
+        ...projected,
+        ...controlDiagnosticItems(read.diagnostics).map((item, index) => ordered(item, "control", read.records.length + index + 1)),
     ];
 }
 function agentItems(stateDir) {
@@ -98,7 +141,7 @@ function agentItems(stateDir) {
         .map((turn) => {
         const text = turn.assistantText ?? turn.userText ?? "Agent turn";
         const summary = text.replaceAll(/\s+/g, " ").trim().slice(0, 160);
-        return {
+        return ordered({
             id: `agent:${turn.commandId}:${turn.sessionFile}:${turn.index}`,
             ts: turn.timestamp ?? new Date(0).toISOString(),
             source: "agent",
@@ -106,7 +149,7 @@ function agentItems(stateDir) {
             summary: summary || "Agent turn",
             ...(turn.error ? { level: "error" } : {}),
             detail: redact(turn),
-        };
+        }, "agent", turn.index);
     });
 }
 function processItems(stateDir) {
@@ -118,7 +161,7 @@ function processItems(stateDir) {
         const tail = tailFile(path, Limits.DEFAULT_INSPECT_LINES);
         if (!tail && stat.size === 0)
             return [];
-        return [{
+        return [ordered({
                 id: `process:${stream}`,
                 ts: stat.mtime.toISOString(),
                 source: "process",
@@ -126,17 +169,18 @@ function processItems(stateDir) {
                 summary: `${stream} (${stat.size} bytes)`,
                 ...(stream === "stderr" ? { level: "warning" } : {}),
                 detail: redact({ bytes: stat.size, path, tail }),
-            }];
+            }, `process:${stream}`, 1)];
     });
 }
 function resultItems(stateDir) {
     const path = join(stateDir, "result.json");
     const read = readJsonFileResilient(path, {});
-    if (Object.keys(read.value).length === 0)
-        return diagnosticItems(read.diagnostics);
+    if (Object.keys(read.value).length === 0) {
+        return diagnosticItems(read.diagnostics).map((item, index) => ordered(item, "process:result", index));
+    }
     const stat = statSync(path);
     const failed = Number(read.value.code ?? 0) !== 0;
-    return [{
+    return [ordered({
             id: "process:result",
             ts: typeof read.value.completedAt === "string"
                 ? read.value.completedAt
@@ -146,13 +190,13 @@ function resultItems(stateDir) {
             summary: failed ? "Run process failed" : "Run process completed",
             ...(failed ? { level: "error" } : {}),
             detail: redact(read.value),
-        }, ...diagnosticItems(read.diagnostics)];
+        }, "process:result", 1), ...diagnosticItems(read.diagnostics).map((item, index) => ordered(item, "process:result", index + 2))];
 }
 function artifactItems(stateDir, declarations) {
     const manifest = resolveArtifactManifest(declarations);
     if (!manifest)
         return [];
-    return Object.entries(manifest).slice(0, SOURCE_LIMIT).map(([name, artifact]) => ({
+    return Object.entries(manifest).slice(0, SOURCE_LIMIT).map(([name, artifact], index) => ordered({
         id: `artifact:${name}`,
         ts: artifact.exists && existsSync(artifact.path)
             ? statSync(artifact.path).mtime.toISOString()
@@ -167,7 +211,7 @@ function artifactItems(stateDir, declarations) {
             required: artifact.required,
             ...(artifact.size !== undefined ? { size: artifact.size } : {}),
         }),
-    }));
+    }, "artifact", index + 1));
 }
 export function projectRunTrace(stateDir, options = {}) {
     const source = options.source ?? "all";
@@ -179,7 +223,9 @@ export function projectRunTrace(stateDir, options = {}) {
         ...resultItems(stateDir),
         ...artifactItems(stateDir, options.artifacts),
     ]
-        .filter((item) => source === "all" || item.source === source)
-        .sort((left, right) => right.ts.localeCompare(left.ts) || right.id.localeCompare(left.id));
-    return items.slice(0, Math.max(1, Math.min(options.limit ?? 100, 200)));
+        .filter(({ item }) => source === "all" || item.source === source)
+        .sort(compareTraceItems);
+    return items
+        .slice(0, Math.max(1, Math.min(options.limit ?? 100, 200)))
+        .map(({ item }) => item);
 }

package/dist/scripts/conformance.mjs

@@ -15,6 +15,11 @@ const conformanceSuites = [
   "tests/control.test.ts",
   "tests/runs-controls.test.ts",
   "tests/runs-trace.test.ts",
+  "tests/observability.test.ts",
+  "tests/async-run-control.test.ts",
+  "tests/run-kernel-dogfood.test.ts",
+  "tests/installed.scripts.test.ts",
+  "tests/inspector-overlay.test.ts",
   "tests/tools-inspect-kernel.test.ts",
   "tests/recipes-discovery.test.ts",
   "tests/review-swarm-dogfood.test.ts",

package/dist/scripts/locker.mjs

@@ -11,7 +11,6 @@
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import {
-  appendFileSync,
   existsSync,
   mkdirSync,
   readFileSync,
@@ -32,8 +31,12 @@ async function importRuntimeModule(name) {
   return await import(pathToFileURL(existsSync(compiled) ? compiled : source).href);
 }
 
-const { acquireFileMutationLock, writeJsonAtomic, writeTextAtomic } =
-  await importRuntimeModule("file-state");
+const { acquireFileMutationLock, writeJsonAtomic, writeTextAtomic } = await importRuntimeModule("file-state");
+const { readJsonlFileResilient } = await importRuntimeModule("state-readers");
+const LOCKER_JOURNAL_MAX_RECORDS = 512;
+const LOCKER_JOURNAL_MAX_BYTES = 1024 * 1024;
+const { claimRunControlByIdInStateDir, updateRunControlStatusInStateDir } =
+  await importRuntimeModule("runs-controls");
 const { appendRunTraceEvent } = await importRuntimeModule("runs-trace");
 
 function parseArgs(argv) {
@@ -59,7 +62,6 @@ async function runLocker(argv = process.argv.slice(2)) {
   const queuePath = join(stateDir, "queue.json");
   const locksPath = join(stateDir, "locks.json");
   const journalPath = join(stateDir, "journal.jsonl");
-  const controlsPath = join(stateDir, "controls.jsonl");
   const controlPath = join(stateDir, "control.fifo");
   let runInstanceId;
   mkdirSync(stateDir, { recursive: true });
@@ -96,10 +98,22 @@ async function runLocker(argv = process.argv.slice(2)) {
   }
 
   function journal(event, data = {}) {
-    appendFileSync(
-      journalPath,
-      `${JSON.stringify({ event, ts: new Date().toISOString(), ...data })}\n`,
-    );
+    const release = acquireFileMutationLock(journalPath);
+    try {
+      const records = [...readJsonlFileResilient(journalPath).records,
+        { event, ts: new Date().toISOString(), ...data }]
+        .slice(-LOCKER_JOURNAL_MAX_RECORDS);
+      let content = encodeJournal(records);
+      while (records.length && Buffer.byteLength(content) > LOCKER_JOURNAL_MAX_BYTES) {
+        records.shift();
+        content = encodeJournal(records);
+      }
+      writeTextAtomic(journalPath, content);
+    } finally { release(); }
+  }
+
+  function encodeJournal(records) {
+    return records.length ? `${records.map((record) => JSON.stringify(record)).join("\n")}\n` : "";
   }
 
   function emitTrace(kind, summary, data = {}, level = "info") {
@@ -147,69 +161,6 @@ async function runLocker(argv = process.argv.slice(2)) {
     }
   }
 
-  function acquireControlsLock() {
-    return acquireFileMutationLock(controlsPath);
-  }
-
-  function readControls() {
-    if (!existsSync(controlsPath)) return [];
-    return readFileSync(controlsPath, "utf8")
-      .split("\n")
-      .filter((line) => line.trim())
-      .map((line) => {
-        try {
-          return JSON.parse(line);
-        } catch {
-          return undefined;
-        }
-      })
-      .filter(Boolean);
-  }
-
-  function writeControls(controls) {
-    writeTextAtomic(
-      controlsPath,
-      controls.length
-        ? `${controls.map((control) => JSON.stringify(control)).join("\n")}\n`
-        : "",
-    );
-  }
-
-  function claimControl(id) {
-    const release = acquireControlsLock();
-    try {
-      const controls = readControls();
-      const control = controls.find((item) => item.id === id);
-      if (
-        !control ||
-        control.run_instance_id !== runInstanceId ||
-        (control.status !== "queued" && control.status !== "delivered")
-      ) {
-        return undefined;
-      }
-      control.claimed_at = new Date().toISOString();
-      control.status = "claimed";
-      writeControls(controls);
-      return { ...control };
-    } finally {
-      release();
-    }
-  }
-
-  function finalizeControl(id, status, error) {
-    const release = acquireControlsLock();
-    try {
-      const controls = readControls();
-      const control = controls.find((item) => item.id === id);
-      if (!control || control.status !== "claimed") return;
-      control.status = status;
-      control[`${status}_at`] = new Date().toISOString();
-      if (error) control.error = error;
-      writeControls(controls);
-    } finally {
-      release();
-    }
-  }
 
   function tailJournal(count) {
     if (!existsSync(journalPath)) return [];
@@ -397,7 +348,9 @@ async function runLocker(argv = process.argv.slice(2)) {
   function handleLine(line) {
     const control = normalizeControl(line);
     if (!control) return false;
-    const claimed = claimControl(control.id);
+    const claimed = claimRunControlByIdInStateDir(
+      stateDir, runInstanceId, control.id,
+    );
     if (!claimed) {
       emitTrace(
         "lock.control_rejected",
@@ -409,11 +362,15 @@ async function runLocker(argv = process.argv.slice(2)) {
     }
     try {
       const stopping = handle(claimed) === true;
-      finalizeControl(claimed.id, "handled");
+      updateRunControlStatusInStateDir(
+        stateDir, claimed.id, "handled", {}, ["claimed"],
+      );
       return stopping;
     } catch (error) {
       const text = error instanceof Error ? error.message : String(error);
-      finalizeControl(claimed.id, "failed", text);
+      updateRunControlStatusInStateDir(
+        stateDir, claimed.id, "failed", { error: text }, ["claimed"],
+      );
       journal("lock.error", { error: text });
       emitTrace("lock.error", text, { error: text }, "error");
       return false;