machine-bridge-mcp 3.0.0-beta.21 → 3.0.0-beta.26

package/src/local/managed-job-runner-claim.mjs

@@ -0,0 +1,54 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { createMonotonicDeadline } from "./monotonic-deadline.mjs";
+import { createExclusiveFileSync, replaceFileAtomicallySync } from "./exclusive-file.mjs";
+import { ownerOnlyFile } from "./state.mjs";
+import { readBoundedFile } from "./managed-job-storage.mjs";
+
+const RUNNER_CLAIM_BYTES = 1024;
+const RUNNER_CLAIM_WAIT_MS = 30_000;
+
+export function publishProvisionalRunnerClaim(dir, pid, launchToken) {
+  const file = join(dir, "runner.pid");
+  const claim = { pid, startedAt: new Date().toISOString(), launchToken };
+  try {
+    createExclusiveFileSync(file, `${JSON.stringify(claim)}\n`, { mode: 0o600 });
+  } catch (error) {
+    if (error?.code !== "EEXIST") throw error;
+    const existing = readRunnerClaim(file, "managed job runner claim already exists but is unreadable");
+    if (Number(existing?.pid) !== pid) throw new Error("managed job runner claim is owned by another process");
+  }
+  ownerOnlyFile(file);
+}
+
+export async function confirmRunnerClaim({ file, pid, processStartedAt, launchToken }) {
+  const exact = { pid, startedAt: new Date().toISOString(), processStartedAt };
+  if (!launchToken) {
+    createExclusiveFileSync(file, `${JSON.stringify(exact)}\n`, { mode: 0o600 });
+    return;
+  }
+  if (!/^[a-f0-9]{32}$/.test(launchToken)) throw new Error("runner launch token is invalid");
+  const deadline = createMonotonicDeadline(RUNNER_CLAIM_WAIT_MS);
+  while (!deadline.expired()) {
+    if (!existsSync(file)) {
+      await new Promise((resolvePromise) => { setTimeout(resolvePromise, 10); });
+      continue;
+    }
+    const provisional = readRunnerClaim(file, "runner ownership claim is unreadable");
+    if (Number(provisional?.pid) !== pid || provisional?.launchToken !== launchToken) {
+      throw new Error("runner ownership claim does not match the spawned process");
+    }
+    exact.startedAt = typeof provisional.startedAt === "string" && provisional.startedAt
+      ? provisional.startedAt
+      : exact.startedAt;
+    replaceFileAtomicallySync(file, `${JSON.stringify(exact)}\n`, { mode: 0o600 });
+    return;
+  }
+  throw new Error("runner ownership claim was not published before startup deadline");
+}
+
+function readRunnerClaim(file, message) {
+  try { return JSON.parse(readBoundedFile(file, RUNNER_CLAIM_BYTES).toString("utf8")); } catch {
+    throw new Error(message);
+  }
+}

package/src/local/managed-job-runner.mjs

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
 import { closeSync } from "node:fs";
 import { basename, join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -6,10 +7,12 @@ import { inspectProcessInstance } from "./process-identity.mjs";
 import { classifyOperationalError } from "./log.mjs";
 import { ownerOnlyFile } from "./state.mjs";
 import { openPrivateAppendFile, readBoundedFile, trimDiagnosticFile } from "./managed-job-storage.mjs";
+import { publishProvisionalRunnerClaim } from "./managed-job-runner-claim.mjs";
 
 const RUNNER_PATH = fileURLToPath(new URL("./job-runner.mjs", import.meta.url));
 
 export function launchRunner(dir, recover = false, recoveryToken = "", options = {}) {
+  const launchToken = randomBytes(16).toString("hex");
   const args = [RUNNER_PATH, "--job-dir", dir];
   if (recover) args.push("--recover");
   const stdoutFile = join(dir, "runner.out.log");
@@ -28,7 +31,7 @@ export function launchRunner(dir, recover = false, recoveryToken = "", options =
       stdio: ["ignore", stdoutFd, stderrFd],
       windowsHide: true,
       shell: false,
-      env: managedRunnerEnvironment({ fullEnv: options.fullEnv === true, recoveryToken, source: options.env || process.env }),
+      env: managedRunnerEnvironment({ fullEnv: options.fullEnv === true, recoveryToken, launchToken, source: options.env || process.env }),
     });
   } finally {
     if (stdoutFd !== undefined) closeSync(stdoutFd);
@@ -46,6 +49,12 @@ export function launchRunner(dir, recover = false, recoveryToken = "", options =
   });
   const pid = Number(child.pid);
   if (!Number.isInteger(pid) || pid <= 0) throw new Error("managed job runner did not receive a process id");
+  try {
+    publishProvisionalRunnerClaim(dir, pid, launchToken);
+  } catch (error) {
+    try { child.kill?.("SIGKILL"); } catch {}
+    throw error;
+  }
   child.unref();
   return pid;
 }
@@ -72,7 +81,7 @@ function readRunnerOwner(dir, fallback = {}) {
   }
 }
 
-export function managedRunnerEnvironment({ fullEnv = false, recoveryToken = "", source = process.env } = {}) {
+export function managedRunnerEnvironment({ fullEnv = false, recoveryToken = "", launchToken = "", source = process.env } = {}) {
   const env = fullEnv ? { ...source } : {};
   if (!fullEnv) {
     for (const key of ["PATH", "HOME", "USERPROFILE", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "LANG", "LC_ALL", "LC_CTYPE", "TMPDIR", "TEMP", "TMP"]) {
@@ -81,5 +90,7 @@ export function managedRunnerEnvironment({ fullEnv = false, recoveryToken = "",
   }
   if (recoveryToken) env.MBM_RECOVERY_LOCK_TOKEN = recoveryToken;
   else delete env.MBM_RECOVERY_LOCK_TOKEN;
+  if (launchToken) env.MBM_RUNNER_LAUNCH_TOKEN = launchToken;
+  else delete env.MBM_RUNNER_LAUNCH_TOKEN;
   return env;
 }

package/src/local/process-execution.mjs

@@ -78,9 +78,9 @@ export class ProcessExecutionService {
     return publicProcessToolResult({ name: command.name, cwd: this.displayPath(cwd, context), timeout_seconds: timeoutSeconds, ...result });
   }
 
-  async probeShell(context = {}) {
+  async probeShell(context = {}, timeoutMs = 5_000) {
     const shell = workspaceShellCommand(process.platform === "win32" ? "cd" : "pwd");
-    return this.run(shell.cmd, shell.args, 5000, true, 64 * 1024, context);
+    return this.run(shell.cmd, shell.args, timeoutMs, true, 64 * 1024, context);
   }
 
   async runFixedInternal(cmd, args, timeoutMs, allowFailure = false, maxOutputBytes = DEFAULT_PROCESS_OUTPUT_BYTES, context = {}, cwd = this.workspace) {

package/src/local/process-identity.mjs

@@ -34,6 +34,16 @@ export function processStartTimeMs(pid) {
   return result.ok ? parseTime(result.stdout) : null;
 }
 
+export function processState(pid) {
+  const parsed = normalizePid(pid);
+  if (!parsed || process.platform === "win32") return "unknown";
+  const result = runBounded("ps", ["-p", String(parsed), "-o", "state="]);
+  if (!result.ok) return "unknown";
+  const state = result.stdout.trim().charAt(0).toUpperCase();
+  if (state === "Z") return "zombie";
+  return state ? "running" : "unknown";
+}
+
 export function processCommandLine(pid) {
   const parsed = normalizePid(pid);
   if (!parsed) return "";
@@ -123,6 +133,7 @@ function runBounded(command, args) {
   const result = spawnSync(command, args, {
     encoding: "utf8",
     timeout: COMMAND_TIMEOUT_MS,
+    killSignal: "SIGKILL",
     maxBuffer: COMMAND_OUTPUT_BYTES,
     windowsHide: true,
     env: process.platform === "win32" ? process.env : { ...process.env, LC_ALL: "C", LANG: "C" },

package/src/local/process-tree-ownership-types.d.ts

@@ -0,0 +1,37 @@
+export interface ChildProcessIdentity {
+  pid?: unknown;
+  exitCode?: unknown;
+  signalCode?: unknown;
+}
+
+export interface ProcessGroupEntry {
+  pid: number;
+  pgid: number;
+  startedAt: number;
+}
+
+export interface ProcessOwnershipMember {
+  pid: number;
+  startedAt: number | null;
+}
+
+export interface ProcessOwnershipSnapshot {
+  platform: string;
+  pid: number;
+  members: ProcessOwnershipMember[];
+}
+
+export interface ProcessSnapshotResult {
+  error?: unknown;
+  status?: number | null;
+  stdout?: string | Buffer;
+}
+
+export interface ProcessOwnershipOptions {
+  platform?: string;
+  listProcessGroups?: (options: ProcessOwnershipOptions, pid: number, timeoutMs: number) => ProcessGroupEntry[];
+  spawnSyncProcess?: (command: string, args: string[], options: Record<string, unknown>) => ProcessSnapshotResult;
+  ownershipCheckBudgetMs?: unknown;
+  processSnapshotTimeoutMs?: unknown;
+  monotonicNow?: () => number;
+}

package/src/local/process-tree-ownership.mjs

@@ -1,10 +1,16 @@
+// @ts-check
+
 import { spawnSync } from "node:child_process";
+import { createMonotonicDeadline } from "./monotonic-deadline.mjs";
 
-const PROCESS_SNAPSHOT_TIMEOUT_MS = 3000;
-const PROCESS_SNAPSHOT_BYTES = 512 * 1024;
-const START_TIME_TOLERANCE_MS = 1500;
+export const DEFAULT_PROCESS_OWNERSHIP_CHECK_BUDGET_MS = 3000;
+/** @typedef {import("./process-tree-ownership-types.d.ts").ChildProcessIdentity} ChildProcessIdentity */
+/** @typedef {import("./process-tree-ownership-types.d.ts").ProcessGroupEntry} ProcessGroupEntry */
+/** @typedef {import("./process-tree-ownership-types.d.ts").ProcessOwnershipMember} ProcessOwnershipMember */
+/** @typedef {import("./process-tree-ownership-types.d.ts").ProcessOwnershipSnapshot} ProcessOwnershipSnapshot */
+/** @typedef {import("./process-tree-ownership-types.d.ts").ProcessOwnershipOptions} ProcessOwnershipOptions */
 
-export function captureProcessTreeOwnership(child, options = {}) {
+export function captureProcessTreeOwnership(/** @type {ChildProcessIdentity} */ child, /** @type {ProcessOwnershipOptions} */ options = {}) {
   const pid = positivePid(child?.pid);
   const platform = String(options.platform || process.platform);
   if (!pid) return { platform, pid: 0, members: [] };
@@ -12,7 +18,7 @@ export function captureProcessTreeOwnership(child, options = {}) {
   return { platform, pid, members: processGroupMembers(pid, options) };
 }
 
-export function refreshProcessTreeOwnership(snapshot, options = {}) {
+export function refreshProcessTreeOwnership(/** @type {ProcessOwnershipSnapshot | null | undefined} */ snapshot, /** @type {ProcessOwnershipOptions} */ options = {}) {
   if (!snapshot?.pid || snapshot.platform === "win32") return snapshot;
   const members = [...(snapshot.members || [])];
   for (const observed of processGroupMembers(snapshot.pid, options)) {
@@ -21,51 +27,53 @@ export function refreshProcessTreeOwnership(snapshot, options = {}) {
   return { ...snapshot, members };
 }
 
-export function processTreeOwnershipStillCurrent(snapshot, child, options = {}) {
+export function processTreeOwnershipStillCurrent(/** @type {ProcessOwnershipSnapshot | null | undefined} */ snapshot, /** @type {ChildProcessIdentity} */ child, /** @type {ProcessOwnershipOptions} */ options = {}) {
   if (!snapshot?.pid) return false;
   if (snapshot.platform === "win32") return !childHasExited(child);
-  if (!Array.isArray(snapshot.members) || snapshot.members.length === 0) return !childHasExited(child);
-  const current = processGroupMembers(snapshot.pid, options);
-  if (snapshot.members.some((expected) => current.some((observed) => sameProcessIdentity(expected, observed)))) return true;
-  return snapshot.members.some((expected) => processGroupMembers(snapshot.pid, options, expected.pid)
-    .some((observed) => sameProcessIdentity(expected, observed)));
+  const members = Array.isArray(snapshot.members) ? snapshot.members : [];
+  if (members.length === 0) return false;
+  const budget = createSnapshotBudget(options);
+  const fullSnapshotTimeoutMs = budget.take(members.length + 1);
+  if (!fullSnapshotTimeoutMs) return false;
+  const current = processGroupMembers(snapshot.pid, options, 0, fullSnapshotTimeoutMs);
+  if (members.some((expected) => current.some((observed) => sameProcessIdentity(expected, observed)))) return true;
+  for (let index = 0; index < members.length; index += 1) {
+    const timeoutMs = budget.take(members.length - index);
+    if (!timeoutMs) return false;
+    const expected = members[index];
+    if (processGroupMembers(snapshot.pid, options, expected.pid, timeoutMs).some((observed) => sameProcessIdentity(expected, observed))) return true;
+  }
+  return false;
 }
 
-function processGroupMembers(groupId, options = {}, pid = 0) {
-  const list = typeof options.listProcessGroups === "function" ? options.listProcessGroups : listProcessGroups;
-  return list(options, pid).filter((entry) => entry.pgid === groupId).map(({ pid: memberPid, startedAt }) => ({ pid: memberPid, startedAt }));
+function processGroupMembers(/** @type {number} */ groupId, /** @type {ProcessOwnershipOptions} */ options = {}, /** @type {number} */ pid = 0, /** @type {number} */ timeoutMs = snapshotTimeout(options)) {
+  const entries = typeof options.listProcessGroups === "function"
+    ? options.listProcessGroups(options, pid, timeoutMs) : listProcessGroups(options, pid, timeoutMs, groupId);
+  return entries.filter((entry) => entry.pgid === groupId).map(({ pid: memberPid, startedAt }) => ({ pid: memberPid, startedAt }));
 }
 
-function listProcessGroups(options = {}, pid = 0) {
-  const run = typeof options.spawnSyncProcess === "function" ? options.spawnSyncProcess : spawnSync;
-  const args = pid ? ["-p", String(pid), "-o", "pid=,pgid=,lstart="] : ["-axo", "pid=,pgid=,lstart="];
+function listProcessGroups(/** @type {ProcessOwnershipOptions} */ options = {}, /** @type {number} */ pid = 0, /** @type {number} */ timeoutMs = snapshotTimeout(options), /** @type {number} */ groupId = 0) {
+  const run = typeof options.spawnSyncProcess === "function" ? options.spawnSyncProcess : defaultSpawnSyncProcess;
+  const platform = String(options.platform || process.platform);
+  const args = pid ? ["-p", String(pid), "-o", "pid=,pgid=,lstart="]
+    : platform === "darwin" ? ["-g", String(groupId), "-o", "pid=,pgid=,lstart="] : ["-axo", "pid=,pgid=,lstart="];
   const result = run("ps", args, {
-    encoding: "utf8",
-    timeout: PROCESS_SNAPSHOT_TIMEOUT_MS,
-    maxBuffer: PROCESS_SNAPSHOT_BYTES,
-    windowsHide: true,
-    env: { ...process.env, LC_ALL: "C", LANG: "C" },
-    stdio: ["ignore", "pipe", "ignore"],
+    encoding: "utf8", timeout: timeoutMs, killSignal: "SIGKILL", maxBuffer: 512 * 1024,
+    windowsHide: true, env: { ...process.env, LC_ALL: "C", LANG: "C" }, stdio: ["ignore", "pipe", "ignore"],
   });
   if (result?.error || result?.status !== 0) return [];
-  return String(result.stdout || "").split(/\r?\n/).map(parseProcessRow).filter(Boolean);
+  return String(result.stdout || "").split(/\r?\n/).map(parseProcessRow).filter(isProcessGroupEntry);
 }
 
-function parseProcessRow(line) {
-  const match = /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(String(line || ""));
-  if (!match) return null;
-  const pid = positivePid(match[1]);
-  const pgid = positivePid(match[2]);
-  const startedAt = Date.parse(match[3]);
-  return pid && pgid && Number.isFinite(startedAt) ? { pid, pgid, startedAt } : null;
-}
-
-function sameProcessIdentity(left, right) {
-  return left.pid === right.pid && Number.isFinite(left.startedAt) && Number.isFinite(right.startedAt)
-    && Math.abs(left.startedAt - right.startedAt) <= START_TIME_TOLERANCE_MS;
-}
-function positivePid(value) { const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : 0; }
-function childHasExited(child) {
-  return child?.exitCode !== null && child?.exitCode !== undefined
-    || child?.signalCode !== null && child?.signalCode !== undefined;
+function defaultSpawnSyncProcess(/** @type {string} */ command, /** @type {string[]} */ args, /** @type {Record<string, unknown>} */ options) {
+  const result = spawnSync(command, args, /** @type {import("node:child_process").SpawnSyncOptionsWithStringEncoding} */ (/** @type {unknown} */ (options)));
+  return { error: result.error, status: result.status, stdout: result.stdout };
 }
+function createSnapshotBudget(/** @type {ProcessOwnershipOptions} */ options) { const total = boundedPositive(options.ownershipCheckBudgetMs, DEFAULT_PROCESS_OWNERSHIP_CHECK_BUDGET_MS); const deadline = createMonotonicDeadline(total, options.monotonicNow); let unallocated = total; return { take(/** @type {number} */ slots) { const remaining = Math.min(unallocated, Math.floor(deadline.remainingMs())); if (remaining < 1) return 0; const value = Math.max(1, Math.min(DEFAULT_PROCESS_OWNERSHIP_CHECK_BUDGET_MS, Math.floor(remaining / Math.max(1, slots)))); unallocated -= value; return value; } }; }
+function snapshotTimeout(/** @type {ProcessOwnershipOptions} */ options) { return boundedPositive(options.processSnapshotTimeoutMs, DEFAULT_PROCESS_OWNERSHIP_CHECK_BUDGET_MS); }
+function boundedPositive(/** @type {unknown} */ value, /** @type {number} */ fallback) { const parsed = Number(value); return Number.isFinite(parsed) && parsed >= 1 ? Math.min(DEFAULT_PROCESS_OWNERSHIP_CHECK_BUDGET_MS, Math.floor(parsed)) : fallback; }
+function parseProcessRow(/** @type {unknown} */ line) { const match = /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(String(line || "")); if (!match) return null; const pid = positivePid(match[1]); const pgid = positivePid(match[2]); const startedAt = Date.parse(match[3]); return pid && pgid && Number.isFinite(startedAt) ? { pid, pgid, startedAt } : null; }
+function isProcessGroupEntry(/** @type {ProcessGroupEntry | null} */ value) { return value !== null; }
+function sameProcessIdentity(/** @type {ProcessOwnershipMember} */ left, /** @type {ProcessOwnershipMember} */ right) { return left.pid === right.pid && typeof left.startedAt === "number" && Number.isFinite(left.startedAt) && typeof right.startedAt === "number" && Number.isFinite(right.startedAt) && left.startedAt === right.startedAt; }
+function positivePid(/** @type {unknown} */ value) { const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : 0; }
+function childHasExited(/** @type {ChildProcessIdentity} */ child) { return child?.exitCode !== null && child?.exitCode !== undefined || child?.signalCode !== null && child?.signalCode !== undefined; }

package/src/local/process-tree.mjs

@@ -4,7 +4,7 @@ import {
   processTreeOwnershipStillCurrent,
   refreshProcessTreeOwnership,
 } from "./process-tree-ownership.mjs";
-export { captureProcessTreeOwnership, processTreeOwnershipStillCurrent, refreshProcessTreeOwnership } from "./process-tree-ownership.mjs";
+export { DEFAULT_PROCESS_OWNERSHIP_CHECK_BUDGET_MS, captureProcessTreeOwnership, processTreeOwnershipStillCurrent, refreshProcessTreeOwnership } from "./process-tree-ownership.mjs";
 
 export const DEFAULT_PROCESS_TERMINATION_GRACE_MS = 2000;
 

package/src/local/relay-call-recovery.mjs

@@ -43,28 +43,36 @@ export class RelayCallRecovery {
   /** @param {RelayResult} response */
   deliver(response) {
     const callId = String(response?.id || "");
-    if (this.send(response)) {
-      if (callId) this.pendingResults.delete(callId);
-      return true;
-    }
-    if (callId && this.isRecoverable()) {
-      this.pendingResults.set(callId, response);
-      this.scheduleExpiry();
-      this.logger.event?.("debug", "relay.tool_result.queued", {
-        call_id: shortCallId(callId), queued_results: this.pendingResults.size,
-      }, "Queued a completed tool result while the relay reconnects");
+    if (!callId) return this.send(response);
+    if (!this.isRecoverable()) {
+      this.logger.event?.("debug", "relay.tool_result.discarded", {
+        call_id: shortCallId(callId), reason: "transport_unavailable",
+      }, "Discarded a tool result because the relay is no longer recoverable");
       return false;
     }
-    this.logger.event?.("debug", "relay.tool_result.discarded", {
-      call_id: shortCallId(callId), reason: "transport_unavailable",
-    }, "Discarded a tool result because the relay is no longer recoverable");
+
+    // Retain sent results until the Worker commits and acknowledges them.
+    this.pendingResults.set(callId, response);
+    const sent = this.send(response);
+    if (sent) {
+      this.logger.event?.("debug", "relay.tool_result.awaiting_ack", {
+        call_id: shortCallId(callId), unacknowledged_results: this.pendingResults.size,
+      }, "Delivered a tool result and retained it until Worker acknowledgement");
+      return true;
+    }
+
+    this.scheduleExpiry();
+    this.logger.event?.("debug", "relay.tool_result.queued", {
+      call_id: shortCallId(callId), queued_results: this.pendingResults.size,
+    }, "Queued a completed tool result while the relay reconnects");
     return false;
   }
 
   /** @param {unknown} callId */
-  discard(callId) {
-    return this.pendingResults.delete(String(callId));
-  }
+  acknowledge(callId) { return this.pendingResults.delete(String(callId)); }
+
+  /** @param {unknown} callId */
+  discard(callId) { return this.pendingResults.delete(String(callId)); }
 
   /** @param {Iterable<string>} resumedCallIds @param {(callId: string) => boolean} cancelCall */
   reconcile(resumedCallIds, cancelCall) {
@@ -94,18 +102,29 @@ export class RelayCallRecovery {
 
   ready() {
     this.clearTimer();
+    this.retryUnacknowledged("reconnected");
+  }
+
+  pulse() {
+    this.retryUnacknowledged("heartbeat");
+  }
+
+  /** @param {string} reason */
+  retryUnacknowledged(reason) {
     let delivered = 0;
-    for (const [callId, response] of [...this.pendingResults]) {
+    for (const response of this.pendingResults.values()) {
       if (!this.send(response)) {
         this.scheduleExpiry();
         break;
       }
-      this.pendingResults.delete(callId);
       delivered += 1;
     }
     if (delivered > 0) {
-      this.logger.event?.("info", "relay.tool_results.replayed", { delivered_results: delivered },
-        "Delivered completed tool results after the relay reconnected");
+      this.logger.event?.(reason === "reconnected" ? "info" : "debug", "relay.tool_results.replayed", {
+        delivered_results: delivered,
+        reason,
+        unacknowledged_results: this.pendingResults.size,
+      }, "Replayed completed tool results awaiting Worker acknowledgement");
     }
   }
 
@@ -122,7 +141,7 @@ export class RelayCallRecovery {
       const cancelled = this.cancelOrigin("remote relay reconnect grace expired");
       const discarded = this.pendingResults.size;
       this.pendingResults.clear();
-      this.terminate();
+      if (cancelled > 0) this.terminate();
       if (cancelled > 0 || discarded > 0) {
         this.logger.warn?.(`remote relay did not recover within ${this.graceMs / 1000} seconds; cancelled ${cancelled} call(s) and discarded ${discarded} queued result(s)`);
       }