@llblab/pi-actors 0.22.5 → 0.24.0

package/lib/async-runner.ts

@@ -0,0 +1,173 @@
+/**
+ * Detached async-runner entrypoint logic.
+ * Zones: async run process, command-template execution telemetry
+ */
+
+import { appendFileSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { appendRecipeContextToPiArgs } from "./actor-recipe-context.ts";
+import { execCommandTemplate } from "./command-templates.ts";
+import { executeRegisteredTool, type ToolExecOptions } from "./execution.ts";
+import { writeJsonAtomic } from "./file-state.ts";
+
+function quoteCommandDetailPart(value: string): string {
+  if (value === "") return "''";
+  if (/^[A-Za-z0-9_/:=.,@%+\-]+$/.test(value)) return value;
+  return `'${String(value).replaceAll("'", "'\\''")}'`;
+}
+
+function formatCommandDetail(command: string, args: string[]): string {
+  return [command, ...args].map(quoteCommandDetailPart).join(" ");
+}
+
+function summarizeCommandDetail(commandDetail: string): string {
+  return commandDetail.length > 160
+    ? `${commandDetail.slice(0, 157)}...`
+    : commandDetail;
+}
+
+export async function runAsyncRunner(stateDir = process.argv[2]): Promise<void> {
+  if (!stateDir) throw new Error("missing state dir");
+
+  const runPath = join(stateDir, "run.json");
+  const progressPath = join(stateDir, "progress.json");
+  const resultPath = join(stateDir, "result.json");
+  const eventsPath = join(stateDir, "events.jsonl");
+  const outboxPath = join(stateDir, "outbox.jsonl");
+  const stdoutPath = join(stateDir, "stdout.log");
+  const stderrPath = join(stateDir, "stderr.log");
+  const meta = JSON.parse(readFileSync(runPath, "utf8"));
+
+  function event(name: string, data: Record<string, unknown> = {}): void {
+    appendFileSync(
+      eventsPath,
+      `${JSON.stringify({ event: name, ts: new Date().toISOString(), ...data })}\n`,
+    );
+  }
+
+  function outbox(
+    name: string,
+    summary: string,
+    data: Record<string, unknown> = {},
+    delivery = "log",
+    level = "info",
+  ): void {
+    appendFileSync(
+      outboxPath,
+      `${JSON.stringify({ body: data, data, delivery, event: name, from: `run:${meta.run}`, level, summary, to: "coordinator", ts: new Date().toISOString(), type: name })}\n`,
+    );
+  }
+
+  function progress(phase: string, extra: Record<string, unknown> = {}): void {
+    writeJsonAtomic(progressPath, {
+      phase,
+      updatedAt: new Date().toISOString(),
+      ...extra,
+    });
+  }
+
+  let activeSubagents = 0;
+  let completedSubagents = 0;
+  const subagentFailures: Record<string, unknown>[] = [];
+
+  function getCommandDoneDelivery(result: { code: number }): string {
+    return result.code !== 0 || activeSubagents > 0 ? "followup" : "log";
+  }
+
+  function progressRunning(): void {
+    progress("running", {
+      activeSubagents,
+      completed: completedSubagents,
+      failures: subagentFailures,
+    });
+  }
+
+  async function observedExec(command: string, args: string[], options?: ToolExecOptions) {
+    const commandDetail = formatCommandDetail(command, args);
+    const execArgs = appendRecipeContextToPiArgs(
+      command,
+      args,
+      meta.recipe_context_records,
+      options?.actorRecipeContext,
+    );
+    activeSubagents += 1;
+    event("command.start", { activeSubagents, command: commandDetail });
+    progressRunning();
+    const result = await execCommandTemplate(command, execArgs, options);
+    activeSubagents = Math.max(0, activeSubagents - 1);
+    completedSubagents += 1;
+    if (result.code !== 0) {
+      subagentFailures.push({
+        code: result.code,
+        command: commandDetail,
+        killed: result.killed,
+      });
+    }
+    event("command.done", {
+      activeSubagents,
+      code: result.code,
+      command: commandDetail,
+      killed: result.killed,
+    });
+    outbox(
+      "command.done",
+      `Command ${summarizeCommandDetail(commandDetail)} completed with code ${result.code}`,
+      {
+        activeSubagents,
+        ...(meta.artifacts ? { artifacts: meta.artifacts } : {}),
+        run_files: [stdoutPath, stderrPath, resultPath, eventsPath, outboxPath],
+        code: result.code,
+        command: commandDetail,
+        killed: result.killed,
+      },
+      getCommandDoneDelivery(result),
+      result.code === 0 ? "info" : "error",
+    );
+    progressRunning();
+    return result;
+  }
+
+  try {
+    event("run.runner.start", { pid: process.pid });
+    progressRunning();
+    const result = await executeRegisteredTool(
+      {
+        name: "run_actor",
+        description: "Detached command-template run actor",
+        template: meta.template,
+        args: [],
+        defaults: {},
+      },
+      meta.values || {},
+      observedExec,
+      meta.cwd,
+    );
+    const text = result.content?.[0]?.text || "";
+    appendFileSync(stdoutPath, text);
+    writeJsonAtomic(resultPath, {
+      code: result.details.code,
+      command: result.details.command,
+      killed: result.details.killed,
+      truncated: result.details.truncated,
+      completedAt: new Date().toISOString(),
+    });
+    progress("done", {
+      completed: 1,
+      failures: result.details.nonCriticalFailures || [],
+    });
+    event("run.done", { code: result.details.code });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    appendFileSync(stderrPath, `${message}\n`);
+    writeJsonAtomic(resultPath, {
+      code: 1,
+      error: message,
+      killed: false,
+      completedAt: new Date().toISOString(),
+    });
+    progress("failed", { completed: 0, failures: [{ message }] });
+    event("run.failed", { error: message });
+    throw error;
+  }
+}

package/lib/async-runs.ts

@@ -5,7 +5,7 @@
  */
 
 import { spawn, spawnSync } from "node:child_process";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import {
   closeSync,
   constants,
@@ -14,7 +14,9 @@ import {
   openSync,
   readdirSync,
   readFileSync,
+  cpSync,
   readlinkSync,
+  renameSync,
   rmSync,
   statSync,
   writeFileSync,
@@ -31,6 +33,7 @@ import type {
 } from "./command-templates.ts";
 import { substituteCommandTemplateToken } from "./command-templates.ts";
 import { writeJsonAtomic } from "./file-state.ts";
+import { readJsonFileResilient, readJsonlFileResilient } from "./state-readers.ts";
 import * as Paths from "./paths.ts";
 import * as RecipeReferences from "./recipe-references.ts";
 import * as RecipeUsage from "./recipe-usage.ts";
@@ -65,7 +68,7 @@ export interface AsyncRunStartParams {
   timeout?: number | string;
   delay?: number | string;
   output?: string;
-  artifacts?: Record<string, string>;
+  artifacts?: Record<string, RunArtifactDeclaration>;
   mailbox?: RecipeReferences.TemplateRecipeMailbox;
   retire_when?: "children_terminal";
   retry?: number | string;
@@ -121,7 +124,7 @@ export interface AsyncRunMeta {
   tool?: string;
   template: CommandTemplateValue;
   values: Record<string, unknown>;
-  artifacts?: Record<string, string>;
+  artifacts?: Record<string, RunArtifactDeclaration>;
   control?: AsyncRunControlEndpoint;
   mailbox?: RecipeReferences.TemplateRecipeMailbox;
   recipe_context_records?: RecipeReferences.TemplateRecipeContextRecord[];
@@ -160,23 +163,93 @@ function safeRunId(value: string | undefined): string {
   return run;
 }
 
+export type RunArtifactDeclaration =
+  | string
+  | { path: string; kind?: string; media_type?: string; required?: boolean };
+
+export interface RunArtifactManifestEntry {
+  exists: boolean;
+  kind?: string;
+  media_type?: string;
+  path: string;
+  required?: boolean;
+  sha256?: string;
+  size?: number;
+}
+
 function resolveArtifactPaths(
-  artifacts: Record<string, string> | undefined,
+  artifacts: Record<string, RunArtifactDeclaration> | undefined,
   values: Record<string, unknown>,
-): Record<string, string> | undefined {
+): Record<string, RunArtifactDeclaration> | undefined {
   if (!artifacts) return undefined;
-  const resolved: Record<string, string> = {};
+  const resolved: Record<string, RunArtifactDeclaration> = {};
   for (const [key, value] of Object.entries(artifacts)) {
     if (!key.trim()) continue;
-    resolved[key] = substituteCommandTemplateToken(
-      value,
-      values,
-      `recipe artifacts.${key}`,
-    );
+    if (typeof value === "string") {
+      resolved[key] = substituteCommandTemplateToken(
+        value,
+        values,
+        `recipe artifacts.${key}`,
+      );
+    } else if (
+      value &&
+      typeof value === "object" &&
+      typeof value.path === "string"
+    ) {
+      resolved[key] = {
+        ...value,
+        path: substituteCommandTemplateToken(
+          value.path,
+          values,
+          `recipe artifacts.${key}.path`,
+        ),
+      };
+    }
   }
   return Object.keys(resolved).length > 0 ? resolved : undefined;
 }
 
+export function resolveArtifactManifest(
+  artifacts: Record<string, RunArtifactDeclaration> | undefined,
+): Record<string, RunArtifactManifestEntry> | undefined {
+  if (!artifacts) return undefined;
+  const manifest: Record<string, RunArtifactManifestEntry> = {};
+  for (const [name, artifact] of Object.entries(artifacts)) {
+    const declaration =
+      typeof artifact === "string" ? { path: artifact } : artifact;
+    if (!declaration?.path) continue;
+    try {
+      const content = readFileSync(declaration.path);
+      manifest[name] = {
+        exists: true,
+        ...(declaration.kind ? { kind: declaration.kind } : {}),
+        ...(declaration.media_type
+          ? { media_type: declaration.media_type }
+          : {}),
+        path: declaration.path,
+        ...(declaration.required !== undefined
+          ? { required: declaration.required }
+          : {}),
+        sha256: createHash("sha256").update(content).digest("hex"),
+        size: content.byteLength,
+      };
+    } catch {
+      manifest[name] = {
+        exists: false,
+        ...(declaration.kind ? { kind: declaration.kind } : {}),
+        ...(declaration.media_type
+          ? { media_type: declaration.media_type }
+          : {}),
+        path: declaration.path,
+        ...(declaration.required !== undefined
+          ? { required: declaration.required }
+          : {}),
+      };
+    }
+  }
+  return Object.keys(manifest).length > 0 ? manifest : undefined;
+}
+
 function resolveRunTemplate(params: AsyncRunStartParams): {
   template: CommandTemplateValue;
 } {
@@ -273,11 +346,10 @@ function resolveStartParams(params: AsyncRunStartParams): AsyncRunStartParams {
 }
 
 function readJson(path: string): Record<string, unknown> | undefined {
-  try {
-    return JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
-  } catch {
-    return undefined;
-  }
+  return readJsonFileResilient<Record<string, unknown> | undefined>(
+    path,
+    undefined,
+  ).value;
 }
 
 function isAlive(pid: number): boolean {
@@ -320,16 +392,12 @@ function tailLines(path: string, lines: number): string[] {
 function getInterruptedRunStatus(
   stateDir: string,
 ): "cancelled" | "killed" | undefined {
-  const events = tailFile(join(stateDir, "events.jsonl"), 200);
-  if (!events) return undefined;
-  for (const line of events.split("\n").reverse()) {
-    try {
-      const event = JSON.parse(line) as Record<string, unknown>;
-      if (event.event === "run.kill") return "killed";
-      if (event.event === "run.cancel") return "cancelled";
-    } catch {
-      // Ignore malformed event lines.
-    }
+  const events = readJsonlFileResilient<Record<string, unknown>>(
+    join(stateDir, "events.jsonl"),
+  ).records.slice(-200);
+  for (const event of events.reverse()) {
+    if (event.event === "run.kill") return "killed";
+    if (event.event === "run.cancel") return "cancelled";
   }
   return undefined;
 }
@@ -426,7 +494,11 @@ export function startRun(
         ? RecipeReferences.buildRecipeContextRecords(recipeFile)
         : undefined;
     if (recipeFile && isMutableUsageRecipeFile(recipeFile)) {
-      RecipeUsage.recordRecipeLaunch(recipeFile);
+      RecipeUsage.recordRecipeLaunch(
+        recipeFile,
+        new Date(),
+        startParams.launch_source === "tool" ? "tool" : "spawn",
+      );
     }
     const outFd = openSync(stdout, "a");
     const errFd = openSync(stderr, "a");
@@ -611,6 +683,16 @@ export function getRunStatus(runOrDir: string): Record<string, unknown> {
   };
 }
 
+export interface RunStateIndexEntry {
+  ownerId?: string;
+  recipe?: string;
+  run: string;
+  state_dir: string;
+  status: AsyncRunStatus;
+  tool?: string;
+  updated_at?: string;
+}
+
 function matchesStatusFilter(
   status: unknown,
   filter: string | undefined,
@@ -621,30 +703,125 @@ function matchesStatusFilter(
   return status === filter;
 }
 
+export function listRunStateDirs(
+  stateRoot = DEFAULT_STATE_ROOT,
+  depth = 0,
+  seen = new Set<string>(),
+): string[] {
+  if (!existsSync(stateRoot) || seen.has(resolve(stateRoot))) return [];
+  seen.add(resolve(stateRoot));
+  let entries: import("node:fs").Dirent[];
+  try {
+    entries = readdirSync(stateRoot, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  const result: string[] = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const child = join(stateRoot, entry.name);
+    if (existsSync(join(child, "run.json"))) result.push(child);
+    if (depth + 1 < 8) result.push(...listRunStateDirs(child, depth + 1, seen));
+  }
+  return result;
+}
+
+function runIndexPath(stateRoot: string): string {
+  return join(stateRoot, "index.json");
+}
+
+function indexEntryFromStatus(
+  status: Record<string, unknown>,
+): RunStateIndexEntry {
+  const progress =
+    status.progress && typeof status.progress === "object"
+      ? (status.progress as Record<string, unknown>)
+      : {};
+  return {
+    ...(typeof status.ownerId === "string" ? { ownerId: status.ownerId } : {}),
+    ...(typeof status.recipe === "string" ? { recipe: status.recipe } : {}),
+    run: String(status.run),
+    state_dir: String(status.state_dir),
+    status: status.status as AsyncRunStatus,
+    ...(typeof status.tool === "string" ? { tool: status.tool } : {}),
+    updated_at:
+      typeof progress.updatedAt === "string"
+        ? progress.updatedAt
+        : typeof status.createdAt === "string"
+          ? status.createdAt
+          : new Date(0).toISOString(),
+  };
+}
+
+export function rebuildRunStateIndex(
+  stateRoot = DEFAULT_STATE_ROOT,
+): RunStateIndexEntry[] {
+  mkdirSync(stateRoot, { recursive: true });
+  const entries = listRunStateDirs(stateRoot)
+    .flatMap((stateDir) => {
+      try {
+        return [indexEntryFromStatus(getRunStatus(stateDir))];
+      } catch {
+        return [];
+      }
+    })
+    .sort((a, b) => (b.updated_at ?? "").localeCompare(a.updated_at ?? ""));
+  writeJsonAtomic(runIndexPath(stateRoot), {
+    entries,
+    rebuilt_at: new Date().toISOString(),
+  });
+  return entries;
+}
+
+export function readRunStateIndex(
+  stateRoot = DEFAULT_STATE_ROOT,
+): RunStateIndexEntry[] | undefined {
+  const index = readJson(runIndexPath(stateRoot));
+  if (!index || typeof index !== "object") return undefined;
+  const entries = (index as Record<string, unknown>).entries;
+  if (!Array.isArray(entries)) return undefined;
+  const valid = entries.filter((entry): entry is RunStateIndexEntry => {
+    const record =
+      entry && typeof entry === "object"
+        ? (entry as Record<string, unknown>)
+        : {};
+    return (
+      typeof record.run === "string" &&
+      typeof record.state_dir === "string" &&
+      typeof record.status === "string"
+    );
+  });
+  if (valid.some((entry) => !existsSync(join(entry.state_dir, "run.json"))))
+    return undefined;
+  return valid;
+}
+
 export function listRuns(
   stateRoot = DEFAULT_STATE_ROOT,
   statusFilter?: string,
 ): Array<Record<string, unknown>> {
   if (!existsSync(stateRoot)) return [];
-  const runs: Array<Record<string, unknown>> = [];
-  for (const entry of readdirSync(stateRoot, { withFileTypes: true })) {
-    if (!entry.isDirectory()) continue;
-    try {
-      const stateDir = join(stateRoot, entry.name);
-      const status = getRunStatus(stateDir);
-      if (!matchesStatusFilter(status.status, statusFilter)) continue;
-      runs.push({
-        run: status.run,
-        state_dir: stateDir,
-        status: status.status,
-        ...(typeof status.tool === "string" ? { tool: status.tool } : {}),
-        ...(typeof status.recipe === "string" ? { recipe: status.recipe } : {}),
-      });
-    } catch {
-      // Ignore malformed run dirs.
-    }
+  const indexed = readRunStateIndex(stateRoot);
+  if (indexed) {
+    return indexed
+      .filter((entry) => matchesStatusFilter(entry.status, statusFilter))
+      .map((entry) => ({
+        run: entry.run,
+        state_dir: entry.state_dir,
+        status: entry.status,
+        ...(entry.tool ? { tool: entry.tool } : {}),
+        ...(entry.recipe ? { recipe: entry.recipe } : {}),
+      }));
   }
-  return runs;
+  return rebuildRunStateIndex(stateRoot)
+    .filter((entry) => matchesStatusFilter(entry.status, statusFilter))
+    .map((entry) => ({
+      run: entry.run,
+      state_dir: entry.state_dir,
+      status: entry.status,
+      ...(entry.tool ? { tool: entry.tool } : {}),
+      ...(entry.recipe ? { recipe: entry.recipe } : {}),
+    }));
 }
 
 export function tailRun(runOrDir: string, lines = 40): string {
@@ -732,13 +909,7 @@ function parseRunInboxLine(line: string): RunInboxMessage | undefined {
 }
 
 function readRunInboxMessagesFromStateDir(stateDir: string): RunInboxMessage[] {
-  const file = runInboxFile(stateDir);
-  if (!existsSync(file)) return [];
-  return readFileSync(file, "utf8")
-    .split("\n")
-    .filter((line) => line.trim())
-    .map(parseRunInboxLine)
-    .filter((message): message is RunInboxMessage => Boolean(message));
+  return readJsonlFileResilient<RunInboxMessage>(runInboxFile(stateDir)).records;
 }
 
 function writeRunInboxMessages(
@@ -879,12 +1050,19 @@ export function appendRunOutboxEvent(
   const run = String(status.run ?? runOrDir);
   const type = event.type || event.event || "run.message";
   const to = event.to || "coordinator";
+  const metadata = event.metadata ?? {};
+  const requiresResponse = metadata.requires_response === true;
   const payload = {
     ...(event.body !== undefined ? { body: event.body } : {}),
     ...(event.correlation_id ? { correlation_id: event.correlation_id } : {}),
     ...(event.data !== undefined ? { data: event.data } : {}),
     delivery: normalizeRunOutboxDelivery(
-      event.delivery ?? (to === "coordinator" ? "followup" : "log"),
+      event.delivery ??
+        (requiresResponse
+          ? "followup"
+          : to === "coordinator"
+            ? "notify"
+            : "log"),
     ),
     event: type,
     from: event.from || `run:${run}`,
@@ -1242,6 +1420,77 @@ export function cancelRun(runOrDir: string): Record<string, unknown> {
     : result;
 }
 
+function assertTerminalRun(runOrDir: string): Record<string, unknown> {
+  const status = getRunStatus(runOrDir);
+  if (status.status === "running") {
+    throw new Error("Only terminal runs can be archived or pruned.");
+  }
+  return status;
+}
+
+function archivePathFor(run: string, stateDir: string): string {
+  const archiveRoot = join(dirname(stateDir), "archived");
+  mkdirSync(archiveRoot, { recursive: true });
+  return join(
+    archiveRoot,
+    `${safeRunId(run)}-${new Date().toISOString().replace(/[:.]/g, "-")}`,
+  );
+}
+
+export function archiveRun(runOrDir: string): Record<string, unknown> {
+  const status = assertTerminalRun(runOrDir);
+  const stateDir = String(status.state_dir);
+  const run = String(status.run ?? basename(stateDir));
+  const archiveDir = archivePathFor(run, stateDir);
+  renameSync(stateDir, archiveDir);
+  mkdirSync(stateDir, { recursive: true });
+  const tombstone = {
+    archived: true,
+    archive_dir: archiveDir,
+    original_state_dir: stateDir,
+    run,
+    status: status.status,
+    ts: new Date().toISOString(),
+  };
+  writeJsonAtomic(join(stateDir, "archive-tombstone.json"), tombstone);
+  return tombstone;
+}
+
+export function pruneRun(
+  runOrDir: string,
+  options: { preserveArtifacts?: boolean } = {},
+): Record<string, unknown> {
+  const status = assertTerminalRun(runOrDir);
+  const stateDir = String(status.state_dir);
+  const run = String(status.run ?? basename(stateDir));
+  const manifest = resolveArtifactManifest(
+    status.artifacts as Record<string, RunArtifactDeclaration> | undefined,
+  );
+  const preserved: Record<string, string> = {};
+  if (options.preserveArtifacts && manifest) {
+    const preserveRoot = join(
+      dirname(stateDir),
+      "preserved-artifacts",
+      safeRunId(run),
+    );
+    mkdirSync(preserveRoot, { recursive: true });
+    for (const [name, artifact] of Object.entries(manifest)) {
+      if (!artifact.exists) continue;
+      const target = join(preserveRoot, basename(artifact.path));
+      cpSync(artifact.path, target, { force: true });
+      preserved[name] = target;
+    }
+  }
+  rmSync(stateDir, { recursive: true, force: true });
+  return {
+    pruned: true,
+    preserved_artifacts: preserved,
+    run,
+    state_dir: stateDir,
+    ts: new Date().toISOString(),
+  };
+}
+
 export function killRun(runOrDir: string): Record<string, unknown> {
   const result = stopRun(runOrDir, "SIGKILL", "run.kill");
   return Object.hasOwn(result, "stopped")

package/lib/build-dist.ts

@@ -0,0 +1,30 @@
+/**
+ * Dist build pipeline entrypoint logic.
+ * Zones: packaging, JavaScript-only distributive tree
+ */
+
+import { spawnSync } from "node:child_process";
+import { cpSync, mkdirSync, readdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+
+function run(command: string, args: string[]): void {
+  const result = spawnSync(command, args, { stdio: "inherit" });
+  if (result.status !== 0) process.exit(result.status ?? 1);
+}
+
+export function buildDist(): void {
+  rmSync("dist", { recursive: true, force: true });
+  mkdirSync("dist", { recursive: true });
+
+  run("tsc", ["-p", "tsconfig.build.json"]);
+
+  for (const dir of ["scripts", "recipes", "fixtures", "skills"]) {
+    cpSync(dir, join("dist", dir), { recursive: true });
+  }
+
+  const builtScripts = readdirSync(join("dist", "scripts"))
+    .filter((name) => name.endsWith(".mjs"))
+    .map((name) => join("dist", "scripts", name));
+
+  run(process.execPath, ["--check", ...builtScripts]);
+}