@amaster.ai/employee-runtime-connector 0.1.0-beta.21 → 0.1.0-beta.23

package/README.md

@@ -38,7 +38,7 @@ The daemon reports its connector contract, exact package/build, platform/archite
 
 `AMASTER_RUNTIME_ATTESTATION_MODE` defaults to `shadow`. Set it to `enforce` only after the expected version and build marker are configured and shadow diagnostics are clean; enforce mode refuses to create or lease Runtime V2 mutation commands without a current exact attestation. `AMASTER_RUNTIME_ATTESTATION_TTL_SECONDS` defaults to `300` and accepts `60`–`3600`. Invalid mode or TTL values fail server startup visibly.
 
-Threat model: these facts are self-reported by a connector authenticated with its existing connector credential. There is no hardware-backed signature, remote attestation, or independent trust root. The proof detects accidental version/schema/executor drift by an honest connector and correlates commands to the observed facts; it does not prove that a compromised or malicious connector is running the claimed code. Keep the default `shadow` mode observational. Do not treat this mechanism as a production security boundary or enable `enforce` until credential-epoch identity, transactional create/lease revalidation, batch-local rejection, and recovery from honest drift have passed review.
+Threat model: these facts are self-reported by a connector authenticated with its existing connector credential. There is no hardware-backed signature, remote attestation, or independent trust root. The proof detects accidental version/schema/executor drift by an honest connector and correlates commands to the observed facts; it does not prove that a compromised or malicious connector is running the claimed code. Proof identity is stable for the exact connector credential and fact set, create/lease revalidate it under the connector transaction lock, stale commands are isolated within a poll batch, and an active connector can recover after restoring exact facts. Keep the default `shadow` mode observational, and do not treat this mechanism as a standalone production security boundary or enable `enforce` before expected markers are configured and shadow diagnostics are clean.
 
 The AMaster remote stack derives its runtime image from the Pi base image with
 `docker/Dockerfile.amaster-employee-pi-cli-runtime`. The build installs one exact

package/dist/amaster-runtime-daemon.mjs

@@ -3,7 +3,7 @@
 
 // src/amaster-runtime-daemon.mjs
 import { createHash as createHash6 } from "node:crypto";
-import { chmodSync as chmodSync3, copyFileSync as copyFileSync2, existsSync as existsSync9, lstatSync as lstatSync4, mkdirSync as mkdirSync5, readFileSync as readFileSync7, readdirSync as readdirSync6, realpathSync as realpathSync3, renameSync as renameSync2, rmSync as rmSync4, statSync as statSync7, symlinkSync as symlinkSync2, unlinkSync, writeFileSync as writeFileSync5 } from "node:fs";
+import { chmodSync as chmodSync3, copyFileSync as copyFileSync2, existsSync as existsSync9, lstatSync as lstatSync4, mkdirSync as mkdirSync5, readFileSync as readFileSync7, readdirSync as readdirSync6, realpathSync as realpathSync3, renameSync as renameSync3, rmSync as rmSync5, statSync as statSync7, symlinkSync as symlinkSync2, unlinkSync, writeFileSync as writeFileSync5 } from "node:fs";
 import { arch as arch3, homedir as homedir3, hostname as hostname2, platform as platform3 } from "node:os";
 import { basename as basename6, delimiter as delimiter2, dirname as dirname6, extname as extname2, isAbsolute as isAbsolute6, join as join10, relative as relative6, resolve as resolve8 } from "node:path";
 import { spawn, spawnSync as spawnSync5 } from "node:child_process";
@@ -2144,6 +2144,27 @@ function selectExecutor(config, command) {
 function processGroupIdForChild(child, processPlatform = process.platform) {
   return processPlatform === "win32" || typeof child.pid !== "number" || child.pid <= 0 ? null : child.pid;
 }
+function isExecutorProcessAlive(child, processGroupId = processGroupIdForChild(child), processApi = process) {
+  if (processGroupId !== null) {
+    try {
+      processApi.kill(-processGroupId, 0);
+      return true;
+    } catch (error) {
+      if (error?.code === "ESRCH") return false;
+      return true;
+    }
+  }
+  return child.exitCode === null && child.signalCode === null;
+}
+function isProcessIdAlive(pid, processApi = process) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    processApi.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code !== "ESRCH";
+  }
+}
 function signalExecutorProcess(child, signal, processGroupId = processGroupIdForChild(child), processApi = process) {
   if (processGroupId !== null) {
     try {
@@ -2156,7 +2177,7 @@ function signalExecutorProcess(child, signal, processGroupId = processGroupIdFor
       }
     }
   }
-  if (!child.killed) {
+  if (child.exitCode === null && child.signalCode === null) {
     try {
       return child.kill(signal);
     } catch {
@@ -2435,7 +2456,7 @@ function compileCommandPromptWithManifest(input, options = {}) {
 }
 
 // src/amaster-runtime-daemon/config-state.mjs
-import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, writeFileSync as writeFileSync3 } from "node:fs";
+import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync, rmSync as rmSync3, writeFileSync as writeFileSync3 } from "node:fs";
 import { homedir as homedir2, hostname } from "node:os";
 import { dirname as dirname3, join as join4 } from "node:path";
 
@@ -2559,16 +2580,53 @@ function runtimeHome(env) {
 function stateFilePath(env) {
   return env.AMASTER_DAEMON_STATE_FILE ? expandHomePath(String(env.AMASTER_DAEMON_STATE_FILE)) : join4(runtimeHome(env), "runtime-connector-state.json");
 }
+function corruptStatePath(path) {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/\D/g, "").slice(0, 14);
+  const base = `${path}.corrupt-${timestamp}Z`;
+  let candidate = base;
+  for (let suffix = 1; existsSync3(candidate); suffix += 1) {
+    candidate = `${base}.${suffix}`;
+  }
+  return candidate;
+}
 function readState(env) {
   const path = stateFilePath(env);
   if (!existsSync3(path)) return {};
-  return JSON.parse(readFileSync3(path, "utf8"));
+  try {
+    const state = JSON.parse(readFileSync3(path, "utf8"));
+    if (!state || typeof state !== "object" || Array.isArray(state)) {
+      throw new TypeError("runtime connector state must be a JSON object");
+    }
+    return state;
+  } catch {
+    const quarantinePath = corruptStatePath(path);
+    try {
+      renameSync(path, quarantinePath);
+      process.stderr.write(`AMaster runtime state was invalid and quarantined at ${quarantinePath}
+`);
+    } catch (error) {
+      process.stderr.write(
+        `AMaster runtime state was invalid but could not be quarantined (${error?.code ?? "unknown error"})
+`
+      );
+    }
+    return {};
+  }
 }
 function writeState(env, state) {
   const path = stateFilePath(env);
   mkdirSync3(dirname3(path), { recursive: true });
-  writeFileSync3(path, `${JSON.stringify(state, null, 2)}
-`);
+  const temporaryPath = `${path}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  try {
+    writeFileSync3(temporaryPath, `${JSON.stringify(state, null, 2)}
+`, {
+      flag: "wx",
+      mode: 384
+    });
+    renameSync(temporaryPath, path);
+  } finally {
+    rmSync3(temporaryPath, { force: true });
+  }
 }
 function buildConfig(env = process.env, flags = {}) {
   const serverUrl = String(flags.serverUrl ?? env.AMASTER_EMPLOYEE_SERVER_URL ?? "http://127.0.0.1:3100").replace(/\/+$/, "");
@@ -3722,14 +3780,15 @@ function buildRuntimeConnectorAuthHeaders(config) {
   if (config.token) return { Authorization: `Bearer ${config.token}` };
   return {};
 }
-async function postRuntimeConnectorJson(config, path, payload) {
+async function postRuntimeConnectorJson(config, path, payload, options = {}) {
   const res = await fetch(`${config.serverUrl}${path}`, {
     method: "POST",
     headers: {
       "content-type": "application/json",
       ...buildRuntimeConnectorAuthHeaders(config)
     },
-    body: JSON.stringify(payload)
+    body: JSON.stringify(payload),
+    signal: options.signal
   });
   const text = await res.text();
   const body = text ? JSON.parse(text) : null;
@@ -3760,6 +3819,26 @@ async function postRuntimeConnectorBytes(config, path, body, headers = {}) {
   }
   return response;
 }
+function retryableRuntimeConnectorPost(error) {
+  const status = Number(error?.httpStatus);
+  return !Number.isInteger(status) || status === 408 || status === 429 || status >= 500;
+}
+async function postRuntimeConnectorJsonWithRetry(config, path, payload, options = {}) {
+  const maxAttempts = Math.max(1, Math.min(5, Number(options.maxAttempts ?? 3)));
+  const timeoutMs = Math.max(1, Number(options.timeoutMs ?? 5e3));
+  const delayMs = Math.max(0, Number(options.delayMs ?? 100));
+  let lastError;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await postRuntimeConnectorJson(config, path, payload, { signal: AbortSignal.timeout(timeoutMs) });
+    } catch (error) {
+      lastError = error;
+      if (attempt >= maxAttempts || !retryableRuntimeConnectorPost(error)) throw error;
+      if (delayMs > 0) await new Promise((resolve9) => setTimeout(resolve9, delayMs));
+    }
+  }
+  throw lastError;
+}
 async function bestEffortPostRuntimeConnectorJson(config, path, payload, options = {}) {
   try {
     return await postRuntimeConnectorJson(config, path, payload);
@@ -3775,6 +3854,7 @@ async function bestEffortPostRuntimeConnectorJson(config, path, payload, options
   }
 }
 var postJson = postRuntimeConnectorJson;
+var postJsonWithRetry = postRuntimeConnectorJsonWithRetry;
 var bestEffortPostJson = bestEffortPostRuntimeConnectorJson;
 
 // src/amaster-runtime-daemon/runtime-artifact-upload.mjs
@@ -3843,7 +3923,7 @@ import { existsSync as existsSync5, mkdirSync as mkdirSync4, realpathSync as rea
 import { basename as basename4, join as join6, isAbsolute as isAbsolute4, relative as relative4, resolve as resolve4 } from "node:path";
 
 // src/amaster-runtime-daemon/workspace-manifest.mjs
-import { existsSync as existsSync4, readFileSync as readFileSync5, renameSync, rmSync as rmSync3, writeFileSync as writeFileSync4 } from "node:fs";
+import { existsSync as existsSync4, readFileSync as readFileSync5, renameSync as renameSync2, rmSync as rmSync4, writeFileSync as writeFileSync4 } from "node:fs";
 import { basename as basename3, dirname as dirname4, join as join5 } from "node:path";
 var WORKSPACE_MANIFEST_FILENAME = ".amaster-runtime.json";
 function nowIso() {
@@ -3872,9 +3952,9 @@ function writeWorkspaceManifest(manifestPath, manifest) {
   try {
     writeFileSync4(tempPath, `${JSON.stringify(manifest, null, 2)}
 `, { mode: 384 });
-    renameSync(tempPath, manifestPath);
+    renameSync2(tempPath, manifestPath);
   } catch (err) {
-    rmSync3(tempPath, { force: true });
+    rmSync4(tempPath, { force: true });
     throw err;
   }
   return manifest;
@@ -3891,6 +3971,7 @@ function createWorkspaceManifest(workspace, input = {}) {
     executorKind: input.executorKind ?? null,
     executorHome: workspace.executorHome ?? null,
     commandId: input.commandId ?? null,
+    generation: input.generation ?? null,
     runId: input.runId ?? null,
     issueId: input.issueId ?? null,
     workspaceKey: workspace.workspaceKey ?? null,
@@ -4004,6 +4085,7 @@ function resolveExecutionWorkspace(config, command, opts = {}) {
   createWorkspaceManifest(workspace, {
     executorKind,
     commandId: readString(command.commandId) ?? null,
+    generation: Number.isInteger(command.generation) ? command.generation : null,
     runId,
     issueId
   });
@@ -4581,7 +4663,7 @@ function readWorkspaceStatus(cwd, opts = {}) {
 }
 
 // src/amaster-runtime-daemon.mjs
-var CONNECTOR_VERSION = "0.1.0-beta.21";
+var CONNECTOR_VERSION = "0.1.0-beta.23";
 var CONNECTOR_CONTRACT_VERSION = "2026-06-04.v1";
 var MAX_CHECKPOINT_BYTES = 20 * 1024 * 1024;
 var CHECKPOINT_TTL_MS = 24 * 60 * 60 * 1e3;
@@ -4662,6 +4744,40 @@ function resultOutboxActiveRunCommands(config) {
   }
   return entries;
 }
+function resultOutboxFailedRunCommands(config) {
+  const dir = resultOutboxInvalidDir(config);
+  if (!existsSync9(dir)) return [];
+  const entries = [];
+  for (const file of readdirSync6(dir).filter((name) => name.endsWith(".json")).sort().slice(-100)) {
+    let entry;
+    try {
+      entry = asRecord(JSON.parse(readFileSync7(join10(dir, file), "utf8")));
+    } catch {
+      continue;
+    }
+    const failureReason = readString(entry.invalidReason);
+    if (!failureReason || !["server_rejected_terminal", "retry_exhausted"].includes(failureReason)) continue;
+    const activeRun = asRecord(entry.activeRun);
+    const commandId = readString(activeRun.commandId) ?? readString(entry.commandId);
+    if (!commandId) continue;
+    entries.push({
+      commandId,
+      ...readString(activeRun.runId) ? { runId: readString(activeRun.runId) } : {},
+      ...readString(activeRun.issueId) ? { issueId: readString(activeRun.issueId) } : {},
+      ...readString(activeRun.executorKind) ? { executorKind: readString(activeRun.executorKind) } : {},
+      ...readString(activeRun.workspacePath) ? { workspacePath: readString(activeRun.workspacePath) } : {},
+      ...readString(activeRun.startedAt) ? { startedAt: readString(activeRun.startedAt) } : {},
+      ...readString(activeRun.executorCompletedAt) ? { executorCompletedAt: readString(activeRun.executorCompletedAt) } : {},
+      phase: "result_outbox_failed",
+      outboxPending: 0,
+      evidenceDeliveryStatus: "failed",
+      evidenceFailureReason: failureReason,
+      ...typeof entry.httpStatus === "number" ? { evidenceHttpStatus: entry.httpStatus } : {},
+      ...readString(entry.invalidAt) ? { evidenceFailedAt: readString(entry.invalidAt) } : {}
+    });
+  }
+  return entries;
+}
 function clearDeliveredResultOutboxCommand(config, entry) {
   const commandId = readString(entry?.commandId);
   if (commandId) pendingResultOutboxRunCommands.delete(commandId);
@@ -4964,8 +5080,10 @@ function buildHeartbeatPayload(config, options = {}) {
   const orphanReaperSummary = asRecord(options.orphanReaper ?? lastOrphanReaperSummary);
   const activeRunCommandById = /* @__PURE__ */ new Map();
   for (const entry of [
+    ...Array.from(localDispatchCommandIds, (commandId) => ({ commandId, phase: "preparing" })),
     ...Array.from(activeRunCommands.values()),
     ...Array.from(pendingResultOutboxRunCommands.values()),
+    ...resultOutboxFailedRunCommands(config),
     ...resultOutboxActiveRunCommands(config)
   ]) {
     const commandId = readString(entry.commandId);
@@ -5534,7 +5652,7 @@ function writeJsonFileAtomic(filePath, value) {
   const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`;
   writeFileSync5(tmpPath, `${JSON.stringify(value, null, 2)}
 `, { mode: 384 });
-  renameSync2(tmpPath, filePath);
+  renameSync3(tmpPath, filePath);
 }
 function isPlainRecord(value) {
   return value && typeof value === "object" && !Array.isArray(value);
@@ -5956,7 +6074,7 @@ function buildExecutorInvocation(executor, command = {}, workspace = null) {
   }
   return { command: executor.command, args: [], stdin: "prompt" };
 }
-async function executeModelCallCommand(config, command) {
+async function executeModelCallCommand(config, command, signal) {
   const executor = selectExecutor(config, command);
   const payload = asRecord(command.payload);
   const prompt = readString(payload.prompt) ?? "";
@@ -5972,6 +6090,7 @@ async function executeModelCallCommand(config, command) {
     config.executorMaxOutputBytes,
     readNumber(payload.maxOutputBytes, 512 * 1024)
   ));
+  await ackCommand(config, command, "spawned");
   await ingestLog(config, command, "system", "info", `Starting ${executor.kind} runtime model call`, {
     executorKind: executor.kind,
     args: invocation.args,
@@ -5984,7 +6103,8 @@ async function executeModelCallCommand(config, command) {
     stdin: invocation.stdin === "prompt" ? prompt : "",
     timeoutSeconds,
     maxOutputBytes,
-    executorKind: executor.kind
+    executorKind: executor.kind,
+    signal
   });
   if (execution.stdout) {
     await ingestLog(config, command, "stdout", "info", truncateText(execution.stdout, 4e3));
@@ -6022,6 +6142,17 @@ async function executeModelCallCommand(config, command) {
     } : {}
   }, error ?? void 0);
 }
+async function executeTrackedModelCallCommand(config, command) {
+  const abortController = new AbortController();
+  const forgetActiveModelCall = rememberActiveRunCommand(command, {});
+  const stopActiveModelCallHeartbeats = startActiveRunHeartbeats(config, command, abortController);
+  try {
+    return await executeModelCallCommand(config, command, abortController.signal);
+  } finally {
+    stopActiveModelCallHeartbeats();
+    forgetActiveModelCall();
+  }
+}
 function redactProtectedText(value, protectedValues = []) {
   let text = String(value ?? "");
   for (const protectedValue of protectedValues) {
@@ -6268,6 +6399,7 @@ function realOrResolvedPath(value) {
     return resolve8(value);
   }
 }
+var LSOF_COMMAND = process.platform === "darwin" && existsSync9("/usr/sbin/lsof") ? "/usr/sbin/lsof" : "lsof";
 function processCwdForPid(pid) {
   if (process.platform === "linux") {
     try {
@@ -6277,7 +6409,7 @@ function processCwdForPid(pid) {
     }
   }
   if (process.platform === "darwin") {
-    const result2 = spawnSync5("lsof", ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], {
+    const result2 = spawnSync5(LSOF_COMMAND, ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     });
@@ -6307,7 +6439,7 @@ function allProcessCwdsByPid() {
     return cwds;
   }
   if (process.platform === "darwin") {
-    const result2 = spawnSync5("lsof", ["-nP", "-d", "cwd", "-Fp", "-Fn"], {
+    const result2 = spawnSync5(LSOF_COMMAND, ["-nP", "-d", "cwd", "-Fp", "-Fn"], {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     });
@@ -6525,8 +6657,19 @@ function runExecutor(command, args, options) {
     let timer = null;
     let rssTimer = null;
     let stopKillTimer = null;
-    let stoppedExitDrainTimer = null;
+    let quiescenceTimer = null;
+    let stoppedResidentTimer = null;
     let completionOutputDrainTimer = null;
+    let stopReason = null;
+    let childExited = false;
+    let childClosed = false;
+    let closeCode = null;
+    let closeSignal = null;
+    let spawnError = null;
+    let outputDrainForcedClosed = false;
+    let killedWorkspaceResidents = [];
+    let stoppedResidentDrainUntil = 0;
+    const residentSignalledAt = /* @__PURE__ */ new Map();
     let piCompletionOutputLineBuffer = "";
     const outputBytes = { stdout: 0, stderr: 0 };
     const maxOutputBytes = parsePositiveInteger(options.maxOutputBytes, 50 * 1024 * 1024);
@@ -6545,11 +6688,10 @@ function runExecutor(command, args, options) {
       if (timer) clearTimeout(timer);
       if (rssTimer) clearInterval(rssTimer);
       if (stopKillTimer) clearTimeout(stopKillTimer);
-      if (stoppedExitDrainTimer) clearTimeout(stoppedExitDrainTimer);
+      if (quiescenceTimer) clearTimeout(quiescenceTimer);
+      if (stoppedResidentTimer) clearTimeout(stoppedResidentTimer);
       if (completionOutputDrainTimer) clearTimeout(completionOutputDrainTimer);
       options.signal?.removeEventListener?.("abort", abort);
-      const shouldCleanupWorkspaceResidents = Boolean(aborted || outputFlood || memoryLimit || result2.timedOut || completionOutputType);
-      const killedWorkspaceResidents = shouldCleanupWorkspaceResidents ? killWorkspaceResidentProcesses(options.cwd, processGroupId) : [];
       resolveRun({
         stdout,
         stderr,
@@ -6558,25 +6700,77 @@ function runExecutor(command, args, options) {
         memoryLimit,
         completionOutputType,
         killedWorkspaceResidents,
+        ...outputDrainForcedClosed ? { outputDrainForcedClosed: true } : {},
         ...result2
       });
     };
     const scheduleStopKill = () => {
-      if (stopKillTimer) clearTimeout(stopKillTimer);
+      if (stopKillTimer) return;
       stopKillTimer = setTimeout(() => signalExecutorProcess(child, "SIGKILL", processGroupId), 2e3);
-      stopKillTimer.unref?.();
     };
-    const destroyExecutorOutputPipes = () => {
-      child.stdout.destroy();
-      child.stderr.destroy();
+    const requestStop = (reason) => {
+      if (settled || stopReason) return;
+      stopReason = reason;
+      signalExecutorProcess(child, "SIGTERM", processGroupId);
+      scheduleStopKill();
+    };
+    const reapStoppedWorkspaceResidents = () => {
+      if (settled || !stopReason || stopReason === "completion_cleanup") return;
+      const residents = listWorkspaceResidentProcesses(options.cwd, processGroupId, {
+        processRows: allProcessRows(),
+        processCwdsByPid: allProcessCwdsByPid()
+      });
+      const seen = new Set(killedWorkspaceResidents.map((resident) => resident.pid));
+      const now = Date.now();
+      for (const resident of residents) {
+        if (!seen.has(resident.pid)) killedWorkspaceResidents.push(resident);
+        if (!residentSignalledAt.has(resident.pid)) {
+          residentSignalledAt.set(resident.pid, now);
+          try {
+            process.kill(resident.pid, "SIGTERM");
+          } catch {
+          }
+        }
+      }
+      for (const resident of killedWorkspaceResidents) {
+        const signalledAt = residentSignalledAt.get(resident.pid) ?? now;
+        if (now - signalledAt < 2e3 || !isProcessIdAlive(resident.pid)) continue;
+        try {
+          process.kill(resident.pid, "SIGKILL");
+        } catch {
+        }
+      }
+      if (isExecutorProcessAlive(child, processGroupId) || now < stoppedResidentDrainUntil || killedWorkspaceResidents.some((resident) => isProcessIdAlive(resident.pid))) {
+        stoppedResidentTimer = setTimeout(reapStoppedWorkspaceResidents, 100);
+      } else {
+        stoppedResidentTimer = null;
+      }
+      maybeFinishQuiescent();
     };
-    const finishStoppedExecutorAfterExit = (code, signal) => {
-      if (settled || !aborted && !outputFlood && !memoryLimit && !(completionOutputType && signal === "SIGTERM") || stoppedExitDrainTimer) return;
-      stoppedExitDrainTimer = setTimeout(() => {
-        destroyExecutorOutputPipes();
-        finish({ exitCode: code, signal, timedOut: false, spawnError: null, cancelled: aborted });
-      }, 100);
-      stoppedExitDrainTimer.unref?.();
+    const maybeFinishQuiescent = () => {
+      if (settled || !childExited && !childClosed) return;
+      if (isExecutorProcessAlive(child, processGroupId) || !childClosed && Date.now() < stoppedResidentDrainUntil || killedWorkspaceResidents.some((resident) => isProcessIdAlive(resident.pid))) {
+        requestStop("completion_cleanup");
+        if (!quiescenceTimer) {
+          quiescenceTimer = setTimeout(() => {
+            quiescenceTimer = null;
+            maybeFinishQuiescent();
+          }, 25);
+        }
+        return;
+      }
+      if (!childClosed) {
+        outputDrainForcedClosed = true;
+        child.stdout.destroy();
+        child.stderr.destroy();
+      }
+      finish({
+        exitCode: closeCode,
+        signal: closeSignal,
+        timedOut: stopReason === "timeout",
+        spawnError,
+        cancelled: aborted
+      });
     };
     const maybeSchedulePiCompletionDrain = (text) => {
       if (settled || outputFlood || options.executorKind !== "pi" || completionOutputType) return;
@@ -6601,8 +6795,7 @@ function runExecutor(command, args, options) {
             completionOutputType = stopType;
             completionOutputDrainTimer = setTimeout(() => {
               if (settled) return;
-              signalExecutorProcess(child, "SIGTERM", processGroupId);
-              scheduleStopKill();
+              requestStop("completion_output");
             }, PI_COMPLETION_OUTPUT_GRACE_MS);
             completionOutputDrainTimer.unref?.();
             return;
@@ -6618,8 +6811,7 @@ function runExecutor(command, args, options) {
         bytes: outputBytes[stream],
         limitBytes: maxOutputBytes
       };
-      signalExecutorProcess(child, "SIGTERM", processGroupId);
-      scheduleStopKill();
+      requestStop("output_flood");
     };
     const killForMemoryLimit = (rssBytes) => {
       if (settled || memoryLimit) return;
@@ -6627,23 +6819,17 @@ function runExecutor(command, args, options) {
         rssBytes,
         limitBytes: maxRssBytes
       };
-      signalExecutorProcess(child, "SIGTERM", processGroupId);
-      scheduleStopKill();
+      requestStop("memory_limit");
     };
     const abort = () => {
       if (settled) return;
       aborted = true;
-      signalExecutorProcess(child, "SIGTERM", processGroupId);
-      scheduleStopKill();
+      requestStop("cancel");
     };
     options.signal?.addEventListener?.("abort", abort, { once: true });
     if (options.signal?.aborted) abort();
     timer = setTimeout(() => {
-      signalExecutorProcess(child, "SIGTERM", processGroupId);
-      const timeoutKillTimer = setTimeout(() => signalExecutorProcess(child, "SIGKILL", processGroupId), 2e3);
-      timeoutKillTimer.unref?.();
-      destroyExecutorOutputPipes();
-      finish({ exitCode: null, signal: "SIGTERM", timedOut: true, spawnError: null });
+      requestStop("timeout");
     }, options.timeoutSeconds * 1e3);
     if (maxRssMb > 0 && processGroupId !== null) {
       rssTimer = setInterval(() => {
@@ -6670,13 +6856,27 @@ function runExecutor(command, args, options) {
       if (outputBytes.stderr > maxOutputBytes) killForOutputFlood("stderr");
     });
     child.on("error", (err) => {
-      finish({ exitCode: null, signal: null, timedOut: false, spawnError: err.message });
+      spawnError = err.message;
     });
     child.on("exit", (code, signal) => {
-      finishStoppedExecutorAfterExit(code, signal);
+      childExited = true;
+      closeCode = code;
+      closeSignal = signal;
+      stoppedResidentDrainUntil = Date.now() + 100;
+      if (!stopReason && isExecutorProcessAlive(child, processGroupId)) {
+        requestStop("completion_cleanup");
+      }
+      if (stopReason && stopReason !== "completion_cleanup") {
+        stoppedResidentDrainUntil = Date.now() + 100;
+        reapStoppedWorkspaceResidents();
+      }
+      maybeFinishQuiescent();
     });
     child.on("close", (code, signal) => {
-      finish({ exitCode: code, signal, timedOut: false, spawnError: null, cancelled: aborted });
+      childClosed = true;
+      closeCode = code;
+      closeSignal = signal;
+      maybeFinishQuiescent();
     });
     if (options.stdin) child.stdin.end(options.stdin);
     else child.stdin.end();
@@ -6869,7 +7069,7 @@ function moveResultOutboxEntryToInvalid(config, file, fullPath, reason, detail,
   const invalidPath = join10(invalidDir, file);
   if (original === void 0) {
     try {
-      renameSync2(fullPath, invalidPath);
+      renameSync3(fullPath, invalidPath);
     } catch {
       copyFileSync2(fullPath, invalidPath);
       unlinkSync(fullPath);
@@ -7240,7 +7440,7 @@ async function materializeRequiredArtifactInputs(config, command, workspace) {
     throw new Error("artifact_input_manifest_invalid: expected version 1 entries array");
   }
   const targetRoot = join10(workspace.cwd, "input-artifacts");
-  rmSync4(targetRoot, { recursive: true, force: true });
+  rmSync5(targetRoot, { recursive: true, force: true });
   mkdirSync5(targetRoot, { recursive: true });
   const usedPaths = /* @__PURE__ */ new Set();
   const materialized = [];
@@ -7316,7 +7516,7 @@ function issueCheckpointDir(workspace) {
 async function clearIssueCheckpoint(config, command, workspace, reason) {
   const checkpointDir = issueCheckpointDir(workspace);
   if (!existsSync9(checkpointDir)) return false;
-  rmSync4(checkpointDir, { recursive: true, force: true });
+  rmSync5(checkpointDir, { recursive: true, force: true });
   await ingestLog(config, command, "system", "info", "Cleared issue continuation checkpoint", { reason });
   return true;
 }
@@ -7339,12 +7539,12 @@ async function materializeIssueCheckpoint(config, command, workspace) {
   try {
     manifest = asRecord(JSON.parse(readFileSync7(manifestPath, "utf8")));
   } catch (err) {
-    rmSync4(checkpointDir, { recursive: true, force: true });
+    rmSync5(checkpointDir, { recursive: true, force: true });
     throw new Error(`invalid issue checkpoint manifest: ${err instanceof Error ? err.message : String(err)}`);
   }
   const expiresAt = Date.parse(readString(manifest.expiresAt) ?? "");
   if (!Number.isFinite(expiresAt) || expiresAt <= Date.now() || readString(manifest.issueId) !== commandIssueId(command)) {
-    rmSync4(checkpointDir, { recursive: true, force: true });
+    rmSync5(checkpointDir, { recursive: true, force: true });
     return [];
   }
   try {
@@ -7418,7 +7618,7 @@ async function materializeIssueCheckpoint(config, command, workspace) {
 async function saveIssueCheckpoint(config, command, workspace, candidates) {
   const checkpointDir = issueCheckpointDir(workspace);
   const filesDir = join10(checkpointDir, "files");
-  rmSync4(checkpointDir, { recursive: true, force: true });
+  rmSync5(checkpointDir, { recursive: true, force: true });
   mkdirSync5(filesDir, { recursive: true });
   const files = [];
   let totalBytes = 0;
@@ -7441,7 +7641,7 @@ async function saveIssueCheckpoint(config, command, workspace, candidates) {
     files.push({ path: relativePath, byteSize, sha256: hashFileSha256(ownedSource) });
   }
   if (files.length === 0) {
-    rmSync4(checkpointDir, { recursive: true, force: true });
+    rmSync5(checkpointDir, { recursive: true, force: true });
     return null;
   }
   const manifest = {
@@ -7479,10 +7679,12 @@ function piOutputValidationError(parsed, options = {}) {
   }
   return null;
 }
-async function ackCommand(config, command) {
+async function ackCommand(config, command, phase = "preparing") {
   const connectorId = requireConnectorId(config);
-  return await postJson(config, `/api/amaster/runtime-connectors/${connectorId}/commands/${command.commandId}/ack`, {
-    leaseId: command.leaseId
+  return await postJsonWithRetry(config, `/api/amaster/runtime-connectors/${connectorId}/commands/${command.commandId}/ack`, {
+    leaseId: command.leaseId,
+    generation: command.generation,
+    phase
   });
 }
 async function executeRunCommand(config, command) {
@@ -7571,6 +7773,7 @@ async function executeRunCommand(config, command) {
     presentationKind: "context_manifest",
     contextManifest
   });
+  await ackCommand(config, command, "spawned");
   await ingestLog(config, command, "system", "info", `Starting ${executor.kind} executor`, {
     executorKind: executor.kind,
     cwd,
@@ -7759,6 +7962,7 @@ async function executeRunCommand(config, command) {
       });
     }
     const result2 = {
+      evidenceContract: { version: 1 },
       executorKind: executor.kind,
       command: invocation.command,
       args: invocation.args,
@@ -7768,6 +7972,7 @@ async function executeRunCommand(config, command) {
       exitCode: execution.exitCode,
       signal: execution.signal,
       timedOut: execution.timedOut,
+      ...execution.outputDrainForcedClosed ? { outputDrainForcedClosed: true } : {},
       ...readString(execution.completionOutputType) ? { completionOutputType: readString(execution.completionOutputType) } : {},
       ...cancelled ? { cancelledByControlPlane: true } : {},
       ...invocation.nativeSession ? { nativeSession: invocation.nativeSession } : {},
@@ -7863,7 +8068,7 @@ async function processCommand(config, command) {
       return;
     }
     if (command.commandType === "model_call") {
-      await executeModelCallCommand(config, command);
+      await executeTrackedModelCallCommand(config, command);
       return;
     }
     await completeCommand(config, command, "succeeded", {

package/dist/amaster-runtime.mjs

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { spawn, spawnSync } from "node:child_process";
 import { dirname, join, resolve } from "node:path";
 import { homedir, hostname } from "node:os";
 import { fileURLToPath } from "node:url";
 
-const CONNECTOR_VERSION = "0.1.0-beta.21";
+const CONNECTOR_VERSION = "0.1.0-beta.23";
 
 const CAPABILITIES = [
   "remote_registration",
@@ -277,8 +277,26 @@ function readState(config) {
   const statePath = config.AMASTER_DAEMON_STATE_FILE || join(configDir, "runtime-connector-state.json");
   if (!existsSync(statePath)) return {};
   try {
-    return JSON.parse(readFileSync(statePath, "utf8"));
+    const state = JSON.parse(readFileSync(statePath, "utf8"));
+    if (!state || typeof state !== "object" || Array.isArray(state)) {
+      throw new TypeError("runtime connector state must be a JSON object");
+    }
+    return state;
   } catch {
+    const timestamp = new Date().toISOString().replace(/\D/g, "").slice(0, 14);
+    const base = `${statePath}.corrupt-${timestamp}Z`;
+    let quarantinePath = base;
+    for (let suffix = 1; existsSync(quarantinePath); suffix += 1) {
+      quarantinePath = `${base}.${suffix}`;
+    }
+    try {
+      renameSync(statePath, quarantinePath);
+      process.stderr.write(`AMaster runtime state was invalid and quarantined at ${quarantinePath}\n`);
+    } catch (error) {
+      process.stderr.write(
+        `AMaster runtime state was invalid but could not be quarantined (${error?.code ?? "unknown error"})\n`,
+      );
+    }
     return {};
   }
 }
@@ -286,7 +304,16 @@ function readState(config) {
 function writeState(config, state) {
   const statePath = withDefaults(config).AMASTER_DAEMON_STATE_FILE || join(configDir, "runtime-connector-state.json");
   mkdirSync(dirname(statePath), { recursive: true });
-  writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`);
+  const temporaryPath = `${statePath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  try {
+    writeFileSync(temporaryPath, `${JSON.stringify(state, null, 2)}\n`, {
+      flag: "wx",
+      mode: 0o600,
+    });
+    renameSync(temporaryPath, statePath);
+  } finally {
+    rmSync(temporaryPath, { force: true });
+  }
 }
 
 function readPidInfo() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amaster.ai/employee-runtime-connector",
-  "version": "0.1.0-beta.21",
+  "version": "0.1.0-beta.23",
   "description": "AMaster Employee runtime connector CLI and daemon",
   "license": "MIT",
   "type": "module",