@marcoscale98/pi-extensible-workflows 5.17.0-fork.1 → 6.2.1-fork.1

package/trajectory/src/index.ts

@@ -1,15 +1,16 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { mkdir, open, readFile, rm } from "node:fs/promises";
+import { mkdir, readFile, rm } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { getAgentDir, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { clearTrajectoryHost, setTrajectoryHost, type TrajectoryHost, type TrajectoryPublisherProvider } from "../../src/trajectory-host-handle.js";
-import { processAlive } from "../../src/session-lease.js";
+import { atomicJson } from "../../src/io.js";
 import { errorText, isNodeError, object, positiveInteger } from "../../src/utils.js";
-import { isTrajectoryAction, isTrajectoryTarget, trajectoryActionError, TRAJECTORY_MAX_TRANSCRIPT_BYTES, type TrajectoryPublisherInput, type TrajectoryPublisherMetadata, type TrajectoryTranscriptRequest, type TrajectoryTranscriptResult } from "../../src/trajectory.js";
+import { isTimingTranscriptEntry, isTrajectoryAction, isTrajectoryTarget, trajectoryActionError, TRAJECTORY_MAX_TRANSCRIPT_BYTES, type TrajectoryPublisherInput, type TrajectoryPublisherMetadata, type TrajectoryTranscriptRequest, type TrajectoryTranscriptResult } from "../../src/trajectory.js";
 import { shareTrajectoryRun } from "./export.js";
+import { identityOf, processConfirmed, processLive, processStart, recordStartupServer, releaseStartupMutex, tryAcquireStartupMutex } from "./startup-mutex.js";
 
 const DEFAULT_TRAJECTORY_PORT = 7432;
 const TRAJECTORY_IDLE_EXIT_MS = 5 * 60 * 1000;
@@ -24,7 +25,7 @@ type TrajectoryPublisherClient = {
 
 type TrajectoryPublisherConstructor = new (url: string) => TrajectoryPublisherClient;
 
-type TrajectoryLock = { pid: number; port: number; fingerprint?: string; startedAt?: number };
+type TrajectoryLock = { pid: number; port: number; fingerprint?: string; startedAt?: number; start?: string };
 export type TrajectoryController = {
   open(input: TrajectoryPublisherInput): Promise<{ port: number }>;
   close(): Promise<void>;
@@ -65,111 +66,134 @@ function signalProcess(pid: number, signal: NodeJS.Signals): void {
     if (!isNodeError(error, "ESRCH")) throw error;
   }
 }
-async function stopStaleServer(lock: TrajectoryLock): Promise<void> {
-  // During startup, the lock can name the current Pi process rather than the detached server.
-  if (lock.pid === process.pid) return;
-  // NOTE: after a reboot the pid can belong to an unrelated process; startedAt (checked against /proc ctime) is the only proof it is still ours.
-  if (!await processAlive(lock.pid, lock.startedAt)) return;
-  signalProcess(lock.pid, "SIGTERM");
+// Called right after /health named this pid; its start, read now, tells a later reuse of the pid apart before SIGKILL.
+// Returns only once the server is gone: a server that may still serve keeps its lock and port, so nothing replaces it.
+async function stopStaleServer(pid: number, port: number): Promise<void> {
+  const server = identityOf(pid);
+  if (!processLive(server)) return;
+  signalProcess(pid, "SIGTERM");
   const deadline = Date.now() + 3000;
   while (Date.now() < deadline) {
-    if (!await serverHealthy(lock.port)) break;
+    if (!await serverHealthy(port)) break;
     const remaining = deadline - Date.now();
     if (remaining <= 0) break;
     await delay(Math.min(50, remaining));
   }
-  if (await processAlive(lock.pid, lock.startedAt)) signalProcess(lock.pid, "SIGKILL");
+  // Without a confirmed identity the pid may now be an unrelated process, so SIGTERM is never escalated.
+  if (processConfirmed(server)) signalProcess(pid, "SIGKILL");
+  for (let attempt = 0; attempt < 50 && processLive(server); attempt += 1) await delay(20);
+  if (processLive(server)) throw new Error(`Stale Trajectory server ${String(pid)} on port ${String(port)} did not stop; stop it and retry`);
 }
 
+// Only a missing or malformed lock reads as absent; any other read error leaves its owner unknown, so it is thrown.
 async function readLock(path: string): Promise<TrajectoryLock | undefined> {
-  try {
-    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
-    if (!object(parsed) || !positiveInteger(parsed.pid) || !positiveInteger(parsed.port) || parsed.port > 65535) return undefined;
-    const fingerprint = typeof parsed.fingerprint === "string" ? parsed.fingerprint : undefined;
-    const startedAt = positiveInteger(parsed.startedAt) ? parsed.startedAt : undefined;
-    return { pid: parsed.pid, port: parsed.port, ...(fingerprint === undefined ? {} : { fingerprint }), ...(startedAt === undefined ? {} : { startedAt }) };
-  } catch (error) {
-    if (isNodeError(error, "ENOENT")) return undefined;
-    return undefined;
-  }
+  let text: string;
+  try { text = await readFile(path, "utf8"); }
+  catch (error) { if (isNodeError(error, "ENOENT")) return undefined; throw error; }
+  let parsed: unknown;
+  try { parsed = JSON.parse(text); } catch { return undefined; }
+  if (!object(parsed) || !positiveInteger(parsed.pid) || !positiveInteger(parsed.port) || parsed.port > 65535) return undefined;
+  const fingerprint = typeof parsed.fingerprint === "string" ? parsed.fingerprint : undefined;
+  const startedAt = positiveInteger(parsed.startedAt) ? parsed.startedAt : undefined;
+  const start = typeof parsed.start === "string" ? parsed.start : undefined;
+  return { pid: parsed.pid, port: parsed.port, ...(fingerprint === undefined ? {} : { fingerprint }), ...(startedAt === undefined ? {} : { startedAt }), ...(start === undefined ? {} : { start }) };
 }
 
-// Only a server reporting this fingerprint counts: an older one still bound to the port would otherwise be adopted forever.
-async function waitForServer(port: number, fingerprint: string): Promise<void> {
-  let foreign = false;
-  for (let attempt = 0; attempt < 60; attempt += 1) {
+// One holder runs at most two bounded health waits (60 × (300 + 50) ms each) and one stop (3 s, its final health check, and 1 s for its exit).
+const STARTUP_MUTEX_WAIT_MS = 2 * 60 * (300 + 50) + 3_000 + 1_000 + 1_000;
+async function acquireStartupMutex(lockPath: string): Promise<string> {
+  const deadline = Date.now() + STARTUP_MUTEX_WAIT_MS;
+  for (;;) {
+    const token = await tryAcquireStartupMutex(lockPath);
+    if (token !== undefined) return token;
+    if (Date.now() >= deadline) throw new Error("Another Pi is still starting Trajectory; retry");
+    // Contenders that saw each other both back off; a random delay keeps them from retrying in step.
+    await delay(25 + Math.floor(Math.random() * 50));
+  }
+}
+type IdentifiedHealth = ServerHealth & { pid: number; fingerprint: string };
+// Only a Trajectory server reports a fingerprint; any other service answering /health is never signalled.
+function isTrajectoryServer(health: ServerHealth | undefined): health is IdentifiedHealth { return health?.pid !== undefined && health.fingerprint !== undefined; }
+// A lock proves which process serves its port only when that process reports the same identity.
+function ownsLock(health: ServerHealth | undefined, lock: TrajectoryLock): health is IdentifiedHealth {
+  return isTrajectoryServer(health) && health.pid === lock.pid && (lock.startedAt === undefined || lock.startedAt === health.startedAt);
+}
+async function awaitHealth(port: number, accept: (health: ServerHealth) => boolean, stopped: () => boolean = () => false): Promise<ServerHealth | undefined> {
+  for (let attempt = 0; attempt < 60 && !stopped(); attempt += 1) {
     const health = await serverHealth(port);
-    if (health?.fingerprint === fingerprint) return;
-    foreign = health !== undefined;
+    if (health !== undefined && accept(health)) return health;
     await delay(50);
   }
-  throw new Error(foreign ? `Trajectory port ${String(port)} is held by another Trajectory server; stop it and retry` : `Trajectory server did not start on port ${String(port)}`);
+  return undefined;
+}
+
+async function startServer(lockPath: string, serverPath: string, fingerprint: string, port: number, startupToken: string): Promise<{ port: number }> {
+  // NOTE: under a Bun-compiled pi binary process.execPath is the pi CLI, and Bun's node:http never writes the WebSocket 101 upgrade (oven-sh/bun#28157), so the server must run on a real node from PATH.
+  const child = spawn(process.versions.bun ? "node" : process.execPath, [serverPath, "--port", String(port), "--lock", lockPath, "--fingerprint", fingerprint, "--startup", startupToken], { detached: true, stdio: "ignore" });
+  const startup: { ended: boolean; error?: Error } = { ended: false };
+  const exited = new Promise<void>((resolve) => {
+    child.once("error", (error) => { startup.error = error; startup.ended = true; resolve(); });
+    child.once("exit", () => { startup.ended = true; resolve(); });
+  });
+  try {
+    if (child.pid !== undefined) recordStartupServer(lockPath, startupToken, child.pid);
+    const health = await awaitHealth(port, (candidate) => candidate.pid === child.pid && candidate.fingerprint === fingerprint, () => startup.ended);
+    if (startup.error !== undefined) throw startup.error;
+    // The server publishes its lock before it answers, so readiness is reported only once both agree.
+    const lock = health === undefined ? undefined : await readLock(lockPath);
+    // The detached server outlives this Pi only once it is ready; until then this Pi keeps waiting for its exit.
+    if (lock !== undefined && ownsLock(health, lock) && lock.port === port && lock.fingerprint === fingerprint) { child.unref(); return { port }; }
+    throw new Error(`Trajectory server did not start on port ${String(port)}`);
+  } catch (error) {
+    // A server outliving its startup could later publish over another owner's lock, so it is stopped before the mutex is released.
+    if (!startup.ended) { child.kill("SIGKILL"); await exited; }
+    if ((await readLock(lockPath))?.pid === child.pid) await rm(lockPath, { force: true });
+    throw error;
+  }
 }
-async function resolveExistingServer(lockPath: string, existing: TrajectoryLock, fingerprint: string): Promise<TrajectoryLock | undefined> {
-  const health = await serverHealth(existing.port);
-  if (health) {
-    if (existing.fingerprint === fingerprint && health.fingerprint === fingerprint) return existing;
-    await stopStaleServer(existing);
+
+// Runs under the startup mutex, so no other Pi or idle server changes the lock meanwhile.
+async function startOrReuseServer(lockPath: string, serverPath: string, fingerprint: string, configuredPort: number, startupToken: string): Promise<{ port: number }> {
+  const existing = await readLock(lockPath);
+  if (existing) {
+    let health = await serverHealth(existing.port);
+    // A live lock owner can be a server still starting or one too loaded to answer in time.
+    if (health === undefined && processLive(existing)) health = await awaitHealth(existing.port, (candidate) => candidate.pid === existing.pid);
+    const lock = health === undefined ? existing : await readLock(lockPath) ?? existing;
+    if (ownsLock(health, lock)) {
+      if (lock.fingerprint === fingerprint && health.fingerprint === fingerprint) return { port: lock.port };
+      await stopStaleServer(health.pid, lock.port);
+    } else if (!isTrajectoryServer(health) && lock.pid !== process.pid && processLive(lock)) {
+      // Unidentified is not dead: a suspended or overloaded owner can resume serving, so its lock stays and no second server is started beside it.
+      // Only a lock naming this Pi, left by an older release's startup reservation, is known not to name a server.
+      throw new Error(health === undefined ? `Trajectory server ${String(lock.pid)} named by ${lockPath} is alive but not answering on port ${String(lock.port)}; retry once it answers, or remove the lock if that process is not a Trajectory server` : `Trajectory port ${String(lock.port)} is held by another Trajectory server; stop it and retry`);
+    }
+    // Any other owner is dead, or another Trajectory server holds its port, so it is never signalled.
     await rm(lockPath, { force: true });
-    return undefined;
   }
-  if (await processAlive(existing.pid, existing.startedAt)) {
-    try {
-      await waitForServer(existing.port, fingerprint);
-      return existing;
-    } catch {
-      // A live lock can still name the attaching process during startup; after the bounded wait, replace the unrecoverable startup owner and retry normally.
-      await stopStaleServer(existing);
-      await rm(lockPath, { force: true });
-      return undefined;
+  const occupant = await serverHealth(configuredPort);
+  if (occupant !== undefined) {
+    if (!isTrajectoryServer(occupant)) throw new Error(`Trajectory port ${String(configuredPort)} is held by another Trajectory server; stop it and retry`);
+    if (occupant.fingerprint === fingerprint && occupant.startedAt !== undefined) {
+      // This server lost its lock; restoring it keeps every healthy server named by the lock.
+      //NOTE: its start is read just after /health named the pid; a reuse of that pid in between is not excluded.
+      await atomicJson(lockPath, { pid: occupant.pid, port: configuredPort, fingerprint, startedAt: occupant.startedAt, start: processStart(occupant.pid) });
+      return { port: configuredPort };
     }
+    await stopStaleServer(occupant.pid, configuredPort);
   }
-  await rm(lockPath, { force: true });
-  return undefined;
+  return startServer(lockPath, serverPath, fingerprint, configuredPort, startupToken);
 }
 
+//NOTE: Pi releases without the startup mutex do not take it, so a concurrent start by one of them is not excluded.
 async function ensureTrajectoryServer(agentDir: string, configuredPort: number): Promise<{ port: number }> {
   const lockPath = trajectoryLockPath(agentDir);
   const serverPath = trajectoryServerPath();
   const fingerprint = await trajectoryFingerprint(serverPath);
   await mkdir(dirname(lockPath), { recursive: true, mode: 0o700 });
-  const existing = await readLock(lockPath);
-  if (existing) {
-    const reused = await resolveExistingServer(lockPath, existing, fingerprint);
-    if (reused) return reused;
-  }
-  let lockHandle: Awaited<ReturnType<typeof open>> | undefined;
-  try {
-    lockHandle = await open(lockPath, "wx", 0o600);
-    await lockHandle.writeFile(`${JSON.stringify({ pid: process.pid, port: configuredPort, fingerprint, startedAt: Date.now() })}\n`, "utf8");
-  } catch (error) {
-    if (isNodeError(error, "EEXIST")) {
-      const raced = await readLock(lockPath);
-      if (raced) {
-        const reused = await resolveExistingServer(lockPath, raced, fingerprint);
-        if (reused) return reused;
-      }
-      await rm(lockPath, { force: true });
-      lockHandle = await open(lockPath, "wx", 0o600);
-      await lockHandle.writeFile(`${JSON.stringify({ pid: process.pid, port: configuredPort, fingerprint, startedAt: Date.now() })}\n`, "utf8");
-    } else {
-      throw error;
-    }
-  } finally { await lockHandle?.close(); }
-  // A server can answer on the port without owning the lock, e.g. one whose lock was lost; it reports its pid, so replace it before spawning.
-  const occupant = await serverHealth(configuredPort);
-  if (occupant?.pid !== undefined && occupant.fingerprint !== fingerprint) await stopStaleServer({ pid: occupant.pid, port: configuredPort, ...(occupant.startedAt === undefined ? {} : { startedAt: occupant.startedAt }) });
-  try {
-    // NOTE: under a Bun-compiled pi binary process.execPath is the pi CLI, and Bun's node:http never writes the WebSocket 101 upgrade (oven-sh/bun#28157), so the server must run on a real node from PATH.
-    const child = spawn(process.versions.bun ? "node" : process.execPath, [serverPath, "--port", String(configuredPort), "--lock", lockPath, "--fingerprint", fingerprint], { detached: true, stdio: "ignore" });
-    const startupError = new Promise<never>((_resolve, reject) => { child.once("error", reject); });
-    child.unref();
-    await Promise.race([waitForServer(configuredPort, fingerprint), startupError]);
-    return { port: configuredPort };
-  } catch (error) {
-    await rm(lockPath, { force: true });
-    throw error;
-  }
+  const startupToken = await acquireStartupMutex(lockPath);
+  try { return await startOrReuseServer(lockPath, serverPath, fingerprint, configuredPort, startupToken); }
+  finally { await releaseStartupMutex(lockPath, startupToken); }
 }
 
 function trajectoryWebSocket(): TrajectoryPublisherConstructor | undefined {
@@ -202,7 +226,6 @@ const RECONNECT_MAX_DELAY_MS = 5_000;
 type LiveStateRecord = Record<string, unknown>;
 type TranscriptRevision = { signature: string; revision: number };
 type PendingTranscript = { generation: number; timer: ReturnType<typeof setTimeout>; publisherId: string; runId?: string; agentId?: string; subagentId?: string; revision?: number };
-function isTimingEntry(value: unknown): boolean { return object(value) && value.type === "custom" && value.customType === "pi-workflows:tool-timing"; }
 function boundedString(value: unknown, maxLength = 1024): unknown {
   if (typeof value !== "string") return value;
   const bytes = Buffer.from(value);
@@ -212,7 +235,7 @@ const MAX_LIVE_STRING_BYTES = 64 * 1024;
 const MAX_LIVE_ARRAY_ENTRIES = 256;
 const MAX_LIVE_OBJECT_KEYS = 64;
 const LIVE_METADATA_ARRAY_KEYS = new Set(["agents", "runs", "subagents"]);
-const LIVE_TRUNCATABLE_ARRAY_KEYS = new Set(["events", "phaseHistory"]);
+const LIVE_TRUNCATABLE_ARRAY_KEYS = new Set(["events", "phaseHistory", "scriptCalls"]);
 const LIVE_METADATA_OBJECT_KEYS = new Set(["transcripts"]);
 function liveValueWillBeBounded(value: unknown, key = "", depth = 0): boolean {
   if (typeof value === "string") return Buffer.byteLength(value) > MAX_LIVE_STRING_BYTES;
@@ -231,7 +254,7 @@ function liveValueWillBeBounded(value: unknown, key = "", depth = 0): boolean {
 const MAX_LIVE_TIMING_BYTES = 64 * 1024;
 /** Keeps the newest timing entries: dropping the oldest leaves a live agent's gantt growing instead of frozen. */
 function boundedTiming(value: unknown): unknown[] {
-  const entries = Array.isArray(value) ? value.filter(isTimingEntry) : [];
+  const entries = Array.isArray(value) ? value.filter(isTimingTranscriptEntry) : [];
   const retained: unknown[] = [];
   let bytes = 2;
   for (let index = entries.length - 1; index >= 0; index -= 1) {
@@ -446,11 +469,9 @@ export function createTrajectoryController(agentDir: string): TrajectoryControll
     }
   };
   const transcriptFromFallback = async (input: TrajectoryPublisherInput, request: TrajectoryTranscriptRequest): Promise<TrajectoryTranscriptResult> => {
-    const boundedTranscriptResult = (result: TrajectoryTranscriptResult): TrajectoryTranscriptResult => {
-      try { if (result.status === "available" && Buffer.byteLength(JSON.stringify(result.entries)) > TRAJECTORY_MAX_TRANSCRIPT_BYTES) return { status: "oversized", revision: result.revision, entries: [], error: "Transcript is too large" }; } catch { return { status: "failed", revision: result.revision, entries: [], error: "Transcript failed" }; }
-      return result;
-    };
-    const boundedResult = (entries: readonly unknown[], revision: number): TrajectoryTranscriptResult => { try { if (Buffer.byteLength(JSON.stringify(entries)) > TRAJECTORY_MAX_TRANSCRIPT_BYTES) return { status: "oversized", revision, entries: [], error: "Transcript is too large" }; } catch { return { status: "failed", revision, entries: [], error: "Transcript failed" }; } return { status: entries.length ? "available" : "empty", revision, entries }; };
+    const oversized = (entries: readonly unknown[], revision: number): TrajectoryTranscriptResult | undefined => { try { return Buffer.byteLength(JSON.stringify(entries)) > TRAJECTORY_MAX_TRANSCRIPT_BYTES ? { status: "oversized", revision, entries: [], error: "Transcript is too large" } : undefined; } catch { return { status: "failed", revision, entries: [], error: "Transcript failed" }; } };
+    const boundedTranscriptResult = (result: TrajectoryTranscriptResult): TrajectoryTranscriptResult => (result.status === "available" ? oversized(result.entries, result.revision) : undefined) ?? result;
+    const boundedResult = (entries: readonly unknown[], revision: number): TrajectoryTranscriptResult => oversized(entries, revision) ?? { status: entries.length ? "available" : "empty", revision, entries };
     if (input.loadTranscript) {
       const result = await input.loadTranscript(request);
       const revisionChanged = request.revision !== undefined && result.revision !== request.revision && result.status !== "missing" && result.status !== "failed" && result.status !== "oversized" && result.status !== "disconnected";

package/trajectory/src/server.ts

@@ -1,10 +1,14 @@
 import { createHash, randomUUID } from "node:crypto";
 import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
-import { readFile, rm, writeFile } from "node:fs/promises";
+import { readFile, rm } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import { URL } from "node:url";
 import { isTrajectoryAction, isTrajectoryTarget, trajectoryActionError } from "../../src/trajectory-contracts.js";
+import { atomicWriteFile } from "../../src/io.js";
 import { sameFilesystemPath } from "../../src/paths.js";
+import { isNodeError, object } from "../../src/utils.js";
+import { processStart, readStartupHolder, releaseStartupMutex, tryAcquireStartupMutex } from "./startup-mutex.js";
+import { TOOL_TIMING_ENTRY_TYPE } from "../../src/tool-timing.js";
 const TRAJECTORY_IDLE_EXIT_MS = 5 * 60 * 1000;
 
 type Socket = import("node:stream").Duplex;
@@ -17,9 +21,8 @@ const MAX_FRAME_BYTES = 32 * 1024 * 1024;
 const MAX_PENDING_REQUESTS = 128;
 const TRANSCRIPT_REQUEST_TIMEOUT_MS = 10_000;
 const ACTION_REQUEST_TIMEOUT_MS = 5 * 60 * 1000;
-const TIMING_ENTRY_TYPE = "pi-workflows:tool-timing";
 function isTimingEntry(value: unknown): boolean {
-  return Boolean(value && typeof value === "object" && !Array.isArray(value) && (value as { type?: unknown }).type === "custom" && (value as { customType?: unknown }).customType === TIMING_ENTRY_TYPE);
+  return Boolean(value && typeof value === "object" && !Array.isArray(value) && (value as { type?: unknown }).type === "custom" && (value as { customType?: unknown }).customType === TOOL_TIMING_ENTRY_TYPE);
 }
 function compactTranscript(value: unknown): unknown {
   if (Array.isArray(value)) return value.filter(isTimingEntry);
@@ -270,17 +273,31 @@ export function createTrajectoryServer(port: number, lockPath: string, options:
     if (!requests.size) pending.delete(browser);
     emit(browser, errorResponse(requestId, request, "failed", error));
   };
+  const busy = () => closed || [...publishers.values()].some(({ value }) => value.connected === true);
   const scheduleIdleExit = () => {
-    if (closed || [...publishers.values()].some(({ value }) => value.connected === true) || idleTimer !== undefined) return;
+    if (busy() || idleTimer !== undefined) return;
     idleTimer = setTimeout(() => {
-      server.closeAllConnections();
-      for (const client of clients) client.socket.destroy();
-      server.close(() => {
-        void rm(lockPath, { force: true }).then(() => { process.exit(0); }).catch(() => { process.exit(1); });
-      });
+      idleTimer = undefined;
+      void tryAcquireStartupMutex(lockPath).then(async (token) => {
+        // A Pi holding the startup mutex may be about to reuse this server, so the exit waits for another idle period.
+        if (token === undefined) { scheduleIdleExit(); return; }
+        if (busy() || idleTimer !== undefined) { await releaseStartupMutex(lockPath, token); return; }
+        server.closeAllConnections();
+        for (const client of clients) client.socket.destroy();
+        server.close(() => {
+          void removeOwnLock().then(() => releaseStartupMutex(lockPath, token)).then(() => { process.exit(0); }).catch(() => { process.exit(1); });
+        });
+      }).catch(() => { process.exit(1); });
     }, TRAJECTORY_IDLE_EXIT_MS);
     idleTimer.unref();
   };
+  // Runs under the startup mutex; a Pi may have replaced the lock while this server was idle.
+  const removeOwnLock = async (): Promise<void> => {
+    let current: unknown;
+    try { current = JSON.parse(await readFile(lockPath, "utf8")); }
+    catch (error) { if (isNodeError(error, "ENOENT")) return; throw error; }
+    if (object(current) && current.pid === process.pid && current.startedAt === startedAt) await rm(lockPath, { force: true });
+  };
   const cancelIdleExit = () => {
     if (idleTimer !== undefined) { clearTimeout(idleTimer); idleTimer = undefined; }
   };
@@ -528,18 +545,37 @@ export function createTrajectoryServer(port: number, lockPath: string, options:
     // A dropped tab only sends FIN, and an upgraded socket stays half-open until this side closes it.
     socket.on("end", () => { socket.destroy(); });
   });
-  server.once("listening", () => { startedAt = Date.now(); void writeFile(lockPath, `${JSON.stringify({ pid: process.pid, port, fingerprint: serverFingerprint, startedAt })}\n`, { mode: 0o600 }).catch(() => { process.exitCode = 1; }); scheduleIdleExit(); });
+  // Listening is emitted before the event loop accepts a connection, so the synchronous atomic write publishes the lock before /health can answer.
+  server.once("listening", () => {
+    startedAt = Date.now();
+    try { atomicWriteFile(lockPath, `${JSON.stringify({ pid: process.pid, port, fingerprint: serverFingerprint, startedAt, start: processStart(process.pid) })}\n`, true); }
+    catch (error) { server.close(); server.emit("error", error); return; }
+    scheduleIdleExit();
+  });
   server.on("close", () => { closed = true; if (idleTimer !== undefined) { clearTimeout(idleTimer); idleTimer = undefined; } });
   return server;
 }
 
+// The spawning Pi records this server in its startup holder, which then stays held until this server publishes or exits; without that record the startup was abandoned.
+async function awaitStartupRecord(lockPath: string, token: string): Promise<void> {
+  for (let attempt = 0; attempt < 60; attempt += 1) {
+    const holder = readStartupHolder(lockPath, token);
+    if (holder === undefined) break;
+    if (holder.server?.pid === process.pid) return;
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  throw new Error("Trajectory startup was abandoned before the server listened");
+}
+
 async function main(): Promise<void> {
   const args = new Map<string, string>();
   for (let index = 2; index + 1 < process.argv.length; index += 2) args.set(process.argv[index] ?? "", process.argv[index + 1] ?? "");
   const port = Number(args.get("--port"));
   const lockPath = args.get("--lock");
   const fingerprint = args.get("--fingerprint");
+  const startupToken = args.get("--startup");
   if (!Number.isSafeInteger(port) || port < 1 || port > 65535 || !lockPath || !fingerprint) throw new Error("Invalid Trajectory server arguments");
+  if (startupToken !== undefined) await awaitStartupRecord(lockPath, startupToken);
   const server = createTrajectoryServer(port, lockPath, { fingerprint });
   await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(port, "127.0.0.1", () => { resolve(); }); });
 }

package/trajectory/src/startup-mutex.ts

@@ -0,0 +1,127 @@
+import { randomUUID } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { mkdir, readdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { atomicWriteFile } from "../../src/io.js";
+import { isNodeError, object, positiveInteger } from "../../src/utils.js";
+
+// Every change to the Trajectory lock happens while holding this mutex: a Pi checking, replacing, or starting a server, and a server publishing or removing its lock.
+// Each contender writes its own holder file, then looks for any other live holder and backs off if it finds one. Of two contenders at least one sees the other, so both never proceed,
+// and a dead holder's file is unique to it, so removing it needs no compare-and-delete.
+type ProcessIdentity = { pid: number; start?: string };
+type StartupHolder = ProcessIdentity & { token: string; server?: ProcessIdentity };
+
+let bootId: string | undefined;
+// The boot id and the start time in clock ticks never change for a process and differ for any later process reusing its pid.
+// The /proc/<pid> ctime does not qualify: it records when that entry was looked up, and the kernel can recreate the entry.
+export function processStart(pid: number): string | undefined {
+  if (process.platform !== "linux") return undefined;
+  try {
+    bootId ??= readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
+    const stat = readFileSync(`/proc/${String(pid)}/stat`, "utf8");
+    // The command name can contain spaces and parentheses, so fields are counted after its last closing parenthesis: field 22 is the start time.
+    const startTicks = stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19];
+    return startTicks === undefined || !/^\d+$/.test(startTicks) ? undefined : `${bootId}:${startTicks}`;
+  } catch { return undefined; }
+}
+// Unverified is not dead: an existing pid whose start was never recorded or cannot be read counts as alive.
+//NOTE: without /proc (macOS, Windows) only the pid is checked, so a pid reused by an unrelated process keeps blocking until that process exits.
+export function processLive(identity: ProcessIdentity): boolean {
+  try { process.kill(identity.pid, 0); } catch (error) { if (isNodeError(error, "ESRCH")) return false; }
+  if (identity.start === undefined) return true;
+  const current = processStart(identity.pid);
+  return current === undefined || current === identity.start;
+}
+// Only a recorded start that still matches proves the pid is that process; nothing less authorises a signal.
+export function processConfirmed(identity: ProcessIdentity): boolean {
+  return identity.start !== undefined && processStart(identity.pid) === identity.start;
+}
+
+function holderDirectory(lockPath: string): string { return `${lockPath}.startup`; }
+function holderPath(lockPath: string, token: string): string { return join(holderDirectory(lockPath), `${token}.json`); }
+export function identityOf(pid: number): ProcessIdentity {
+  const start = processStart(pid);
+  return start === undefined ? { pid } : { pid, start };
+}
+function processIdentity(value: unknown): ProcessIdentity | undefined {
+  if (!object(value) || !positiveInteger(value.pid)) return undefined;
+  return typeof value.start === "string" ? { pid: value.pid, start: value.start } : { pid: value.pid };
+}
+function decodeHolder(text: string): StartupHolder | undefined {
+  try {
+    const parsed: unknown = JSON.parse(text);
+    const holder = processIdentity(parsed);
+    if (holder === undefined || !object(parsed) || typeof parsed.token !== "string") return undefined;
+    const server = processIdentity(parsed.server);
+    return { ...holder, token: parsed.token, ...(server === undefined ? {} : { server }) };
+  } catch { return undefined; }
+}
+
+export function readStartupHolder(lockPath: string, token: string): StartupHolder | undefined {
+  try { return decodeHolder(readFileSync(holderPath(lockPath, token), "utf8")); }
+  catch (error) { if (isNodeError(error, "ENOENT")) return undefined; throw error; }
+}
+
+// A holder's server can still publish its lock until it has done so or exited; a lock that cannot be read leaves that unknown, so the error is thrown.
+async function serverPending(lockPath: string, server: ProcessIdentity): Promise<boolean> {
+  if (!processLive(server)) return false;
+  let text: string;
+  try { text = await readFile(lockPath, "utf8"); }
+  catch (error) { if (isNodeError(error, "ENOENT")) return true; throw error; }
+  try {
+    const lock: unknown = JSON.parse(text);
+    return !(object(lock) && lock.pid === server.pid);
+  } catch { return true; }
+}
+async function holderLive(lockPath: string, path: string, read: StartupHolder): Promise<boolean> {
+  if (processLive(read)) return true;
+  // The holder may have recorded its server after it was read; once its Pi is dead the file can no longer change, so this read is final.
+  let holder: StartupHolder | undefined;
+  try { holder = decodeHolder(await readFile(path, "utf8")); }
+  catch (error) { if (isNodeError(error, "ENOENT")) return false; throw error; }
+  if (holder?.server === undefined || !await serverPending(lockPath, holder.server)) return false;
+  // Its Pi died mid-startup; the server it recorded is stopped so it can never publish over the next holder's lock.
+  // An unconfirmed pid may now be an unrelated process, so it is left alone and the holder stays held until that pid exits.
+  if (!processConfirmed(holder.server)) return true;
+  try { process.kill(holder.server.pid, "SIGKILL"); }
+  catch (error) { if (!isNodeError(error, "ESRCH")) throw error; }
+  return true;
+}
+
+export async function tryAcquireStartupMutex(lockPath: string): Promise<string | undefined> {
+  const directory = holderDirectory(lockPath);
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+  const token = randomUUID();
+  const own = `${token}.json`;
+  await atomicWriteFile(join(directory, own), `${JSON.stringify({ ...identityOf(process.pid), token })}\n`);
+  let contended = false;
+  try {
+    for (const name of await readdir(directory)) {
+      if (name === own || !name.endsWith(".json")) continue;
+      const path = join(directory, name);
+      let holder: StartupHolder | undefined;
+      try { holder = decodeHolder(await readFile(path, "utf8")); }
+      catch (error) { if (isNodeError(error, "ENOENT")) continue; throw error; }
+      if (holder !== undefined && await holderLive(lockPath, path, holder)) contended = true;
+      else await rm(path, { force: true });
+    }
+  } catch (error) {
+    // A holder file left by a failed attempt would block every later start until this Pi exits.
+    await rm(join(directory, own), { force: true });
+    throw error;
+  }
+  if (!contended) return token;
+  await rm(join(directory, own), { force: true });
+  return undefined;
+}
+
+// Called right after spawning, so the mutex stays held while that server can still publish, even if this Pi dies.
+export function recordStartupServer(lockPath: string, token: string, pid: number): void {
+  const holder = readStartupHolder(lockPath, token);
+  if (holder === undefined) throw new Error("Trajectory startup mutex was lost");
+  atomicWriteFile(holderPath(lockPath, token), `${JSON.stringify({ ...holder, server: identityOf(pid) })}\n`, true);
+}
+
+export async function releaseStartupMutex(lockPath: string, token: string): Promise<void> {
+  await rm(holderPath(lockPath, token), { force: true });
+}

package/dist/src/roles.d.ts

@@ -1,70 +0,0 @@
-import type { AgentDefinition, AgentResourceSelectorSources, ContextFileScope, ModelSpec, WorkflowExtensionSettings, WorkflowRoleDirectoryRegistration } from "./types.js";
-export declare function parseRoleMarkdown(content: string, strict?: boolean, rolePath?: string): AgentDefinition;
-export declare function workflowRoleDirectories(agentDir?: string): readonly string[];
-export type WorkflowRoleDirectoryInput = string | WorkflowRoleDirectoryRegistration;
-export interface RoleDiscoveryOptions {
-    cwd: string;
-    agentDir?: string;
-    projectTrusted?: boolean;
-    extensionRoleDirectories?: readonly WorkflowRoleDirectoryInput[];
-}
-export interface RoleResourceCandidates {
-    skills?: readonly string[];
-    extensions?: readonly string[];
-    tools?: readonly string[];
-}
-export interface RoleResolutionOptions extends RoleDiscoveryOptions {
-    definition?: AgentDefinition;
-    definitions?: Readonly<Record<string, AgentDefinition>>;
-    model?: string;
-    modelOverride?: ModelSpec;
-    rootModel?: ModelSpec;
-    rootTools?: ReadonlySet<string> | readonly string[];
-    inheritedTools?: readonly string[];
-    availableModels?: ReadonlySet<string>;
-    knownModels?: ReadonlySet<string>;
-    modelAliases?: Readonly<Record<string, string>>;
-    blockedAliases?: ReadonlySet<string>;
-    blockedAliasTargets?: Readonly<Record<string, string>>;
-    settingsPath?: string;
-    selectorSources?: AgentResourceSelectorSources;
-    resources?: RoleResourceCandidates;
-    tools?: readonly string[];
-    skills?: readonly string[];
-    extensions?: readonly string[];
-    contextFiles?: readonly ContextFileScope[];
-    effectiveTools?: readonly string[];
-}
-export interface ResolvedRole {
-    name?: string;
-    definition?: AgentDefinition;
-    model?: ModelSpec;
-    requestedModel?: string;
-    tools?: readonly string[];
-    prompt: string;
-    overrideSystemPrompt: boolean;
-    systemPrompt: {
-        mode: "override" | "append";
-        text: string;
-    };
-    contextFiles?: readonly ContextFileScope[];
-    extensionSettings?: Readonly<WorkflowExtensionSettings>;
-    selectorSources: AgentResourceSelectorSources;
-    selectorLayers: {
-        skills: readonly (readonly string[] | undefined)[];
-        extensions: readonly (readonly string[] | undefined)[];
-        tools: readonly (readonly string[] | undefined)[];
-    };
-    selectedSkills?: readonly string[];
-    selectedExtensions?: readonly string[];
-    unmatchedSkills?: readonly string[];
-    unmatchedExtensions?: readonly string[];
-    unmatchedTools?: readonly string[];
-}
-export declare function loadProjectAgentDefinitions(cwd: string): Readonly<Record<string, AgentDefinition>>;
-export declare function discoverRoles(input: RoleDiscoveryOptions): Readonly<Record<string, AgentDefinition>>;
-export declare function loadRole(name: string, input: RoleDiscoveryOptions): AgentDefinition;
-export declare function loadAgentDefinitions(cwd: string, agentDir?: string, projectTrusted?: boolean, extensionRoleDirectories?: readonly WorkflowRoleDirectoryInput[]): Readonly<Record<string, AgentDefinition>>;
-export declare function canonicalExtensionSelector(selector: string, base?: string): string;
-/** Without a supplied root/tool/resource boundary, this API reports selector layers but does not infer available resources; tools and model remain undefined when no candidates are supplied. When selector sources are supplied, model aliases default to none and blocked aliases are not applied unless provided. */
-export declare function resolveRole(name: string | undefined, options: RoleResolutionOptions): ResolvedRole;