@the-open-engine/zeroshot 6.5.0 → 6.7.0

package/lib/detached-startup.js

@@ -3,10 +3,12 @@ const os = require('os');
 const path = require('path');
 const lockfile = require('proper-lockfile');
 const { resolveRunPlan } = require('./run-plan');
+const { isProcessRunning } = require('./process-liveness');
 
 const DEFAULT_WAIT_TIMEOUT_SECONDS = 180;
 const DEFAULT_WAIT_POLL_MS = 1000;
 const CLUSTERS_LOCK_STALE_MS = 5000;
+const DEFAULT_OWNERSHIP_TIMEOUT_MS = 10000;
 
 function getStorageDir(storageDir) {
   return storageDir || path.join(os.homedir(), '.zeroshot');
@@ -24,17 +26,8 @@ function resolveWaitTimeoutMs(waitTimeoutSeconds) {
   return Math.floor(parsed * 1000);
 }
 
-function isProcessAlive(pid) {
-  if (!pid || !Number.isInteger(pid) || pid <= 0) {
-    return false;
-  }
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (error) {
-    return error && error.code === 'EPERM';
-  }
-}
+// Kept as an alias: this module used to carry its own copy of this check.
+const isProcessAlive = isProcessRunning;
 
 function isClusterRegistered(clusterId, storageDir) {
   const clustersFile = getClustersFilePath(storageDir);
@@ -218,14 +211,137 @@ async function waitForClusterRegistration({
   );
 }
 
+/**
+ * Atomically claim the right to resume-daemon an *existing* cluster record.
+ *
+ * This deliberately does NOT touch cluster.pid/state - those are the fields
+ * orchestrator.resume()'s own eligibility guard inspects to decide "is
+ * someone else already running this?". If this function stamped its PID onto
+ * `cluster.pid` up front, the daemon's own subsequent resume() call would see
+ * state:'running' with a PID that (trivially) belongs to itself and is
+ * therefore alive, and reject itself as "still running" - which is exactly
+ * what happened in early revisions of this fix. Ownership for the handoff is
+ * tracked in a separate `resumeDaemonPid` field instead; orchestrator.resume()
+ * never looks at it, and _restartClusterAgents sets the real cluster.pid once
+ * resume() actually decides to proceed.
+ *
+ * Unlike registerDetachedSetupCluster (which writes a brand-new placeholder
+ * record for a cluster ID that doesn't exist yet), this only adds
+ * resumeDaemonPid to the existing record - config/agents/worktree/isolation
+ * survive untouched, or the daemon's orchestrator.resume() would reload a
+ * blank record and reject it as an unfinished setup cluster.
+ *
+ * The liveness re-check happens inside the same updateClustersFile lock that
+ * performs the write, so two concurrent resume --detach invocations can't both
+ * observe no live claimant and both win: whichever's write lands second sees
+ * either the first daemon's (still-live) resumeDaemonPid, or - once that gets
+ * overwritten by the first daemon's own early _saveClusters() call - its real
+ * state:'running' + live pid, and aborts instead of clobbering it.
+ */
+async function patchDetachedResumeCluster({ clusterId, daemonPid, storageDir }) {
+  await updateClustersFile(storageDir, (clusters) => {
+    const existing = clusters[clusterId];
+    if (!existing) {
+      throw new Error(`Cannot start resume daemon: cluster "${clusterId}" not found in registry`);
+    }
+    if (existing.resumeDaemonPid && isProcessRunning(existing.resumeDaemonPid)) {
+      throw new Error(
+        `Cluster "${clusterId}" already has a live resume daemon (PID ${existing.resumeDaemonPid}); refusing to start a second one`
+      );
+    }
+    // resumeDaemonPid only covers the window up to orchestrator.resume()
+    // actually proceeding - _restartClusterAgents sets the real cluster.pid,
+    // and _resumeFailedCluster/_resumeCleanCluster save that almost
+    // immediately (well before the resumed work itself finishes), which wipes
+    // resumeDaemonPid since it isn't in _saveClusters' persisted field list.
+    // Falling back to state+pid here closes that second window: a first
+    // daemon that has already progressed to actually running is just as much
+    // a live claimant as one still mid-handoff.
+    if (existing.state === 'running' && existing.pid && isProcessRunning(existing.pid)) {
+      throw new Error(
+        `Cluster "${clusterId}" is already running (PID ${existing.pid}); refusing to start a second resume daemon`
+      );
+    }
+    clusters[clusterId] = { ...existing, resumeDaemonPid: daemonPid };
+    return clusters;
+  });
+}
+
+/**
+ * Undo a resume handoff that didn't pan out (daemon died before completing
+ * ownership, or orchestrator.resume() itself rejected it) so the cluster
+ * doesn't end up stuck at state:'running' with a dead PID - the exact zombie
+ * this whole mechanism exists to prevent.
+ */
+async function revertDetachedResumeCluster({ clusterId, storageDir, error }) {
+  await updateClustersFile(storageDir, (clusters) => {
+    const existing = clusters[clusterId];
+    if (!existing) return clusters;
+    clusters[clusterId] = {
+      ...existing,
+      state: 'failed',
+      pid: null,
+      resumeDaemonPid: null,
+      failureInfo: {
+        type: 'resume-daemon',
+        error: error instanceof Error ? error.message : String(error),
+        timestamp: Date.now(),
+      },
+    };
+    return clusters;
+  });
+}
+
+function getRegisteredResumeDaemonPid(clusterId, storageDir) {
+  const clustersFile = getClustersFilePath(storageDir);
+  if (!fs.existsSync(clustersFile)) {
+    return null;
+  }
+  try {
+    const parsed = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
+    return parsed?.[clusterId]?.resumeDaemonPid ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Daemon-side half of the resume handoff: block until the registry shows
+ * *this process's* PID as resumeDaemonPid before touching the cluster.
+ * Closes the window between spawn() returning a PID and the parent's
+ * patchDetachedResumeCluster() write landing - without it, a losing daemon
+ * from a concurrent resume --detach could start running orchestrator.resume()
+ * before its parent's CAS check ever gets a chance to reject it.
+ */
+async function waitForResumeOwnership({
+  clusterId,
+  daemonPid,
+  storageDir,
+  timeoutMs = DEFAULT_OWNERSHIP_TIMEOUT_MS,
+  pollMs = DEFAULT_WAIT_POLL_MS,
+}) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (getRegisteredResumeDaemonPid(clusterId, storageDir) === daemonPid) {
+      return true;
+    }
+    await sleep(pollMs);
+  }
+  return getRegisteredResumeDaemonPid(clusterId, storageDir) === daemonPid;
+}
+
 module.exports = {
   DEFAULT_WAIT_TIMEOUT_SECONDS,
   getClustersFilePath,
+  getRegisteredResumeDaemonPid,
   isClusterRegistered,
   isProcessAlive,
   markDetachedSetupFailed,
+  patchDetachedResumeCluster,
   registerDetachedSetupCluster,
   removeDetachedSetupCluster,
   resolveWaitTimeoutMs,
+  revertDetachedResumeCluster,
   waitForClusterRegistration,
+  waitForResumeOwnership,
 };

package/lib/process-liveness.js

@@ -0,0 +1,26 @@
+/**
+ * Single source of truth for "is this PID actually alive?"
+ *
+ * Previously this check was reimplemented separately in src/orchestrator.js
+ * (_isProcessRunning) and lib/detached-startup.js (isProcessAlive), plus an
+ * inline copy in _getActiveClustersForIssue. Four of those call sites agreed
+ * on the answer; the resume() eligibility guard trusted persisted cluster
+ * state instead of asking the OS, which is what let a dead-PID cluster be
+ * simultaneously reported as "zombie" by status and "still running" by
+ * resume. Consolidating avoids a sixth call site reintroducing that split.
+ */
+function isProcessRunning(pid) {
+  if (!pid || !Number.isInteger(pid) || pid <= 0) {
+    return false;
+  }
+  try {
+    // Signal 0 doesn't kill, just checks if the process exists.
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // ESRCH = no such process, EPERM = process exists but we lack permission.
+    return Boolean(error && error.code === 'EPERM');
+  }
+}
+
+module.exports = { isProcessRunning };

package/lib/start-cluster.js

@@ -185,13 +185,17 @@ function resolveParameterizedConfigFile(config) {
   return resolver.resolveTemplate(config, {});
 }
 
-function loadClusterConfig(orchestrator, configPath, settings = {}, providerOverride) {
-  const config = resolveParameterizedConfigFile(orchestrator.loadConfig(configPath));
-  ensureConfigProviderDefaults(config, settings);
+function prepareClusterConfig(config, settings = {}, providerOverride) {
+  const prepared = resolveParameterizedConfigFile(config);
+  ensureConfigProviderDefaults(prepared, settings);
   if (providerOverride) {
-    applyProviderOverrideToConfig(config, providerOverride, settings);
+    applyProviderOverrideToConfig(prepared, providerOverride, settings);
   }
-  return config;
+  return prepared;
+}
+
+function loadClusterConfig(orchestrator, configPath, settings = {}, providerOverride) {
+  return prepareClusterConfig(orchestrator.loadConfig(configPath), settings, providerOverride);
 }
 
 function mergeRunOptions(options) {
@@ -252,29 +256,98 @@ function buildStartOptions({
 }) {
   const mergedOptions = mergeRunOptions(options);
   const plan = resolveEffectiveRunPlan(mergedOptions, settings);
-  return {
+  return buildStartOptionsFromPlan({
     clusterId,
-    cwd: resolveTargetCwd(),
+    plan,
+    options: mergedOptions,
+    settings,
+    providerOverride,
+    modelOverride,
+    forceProvider,
+    environment: true,
+  });
+}
+
+function buildStartOptionsFromPlan({
+  clusterId,
+  plan,
+  options,
+  settings,
+  providerOverride,
+  modelOverride,
+  forceProvider,
+  environment,
+}) {
+  const targetCwd = environment ? resolveTargetCwd() : options.cwd;
+  return Object.freeze({
+    clusterId,
+    cwd: targetCwd,
     isolation: plan.isolation === 'docker',
-    isolationImage: firstTruthy(mergedOptions.dockerImage, process.env.ZEROSHOT_DOCKER_IMAGE),
+    isolationImage: environment
+      ? firstTruthy(options.dockerImage, process.env.ZEROSHOT_DOCKER_IMAGE)
+      : optionalValue(options.dockerImage),
     worktree: plan.isolation === 'worktree',
     autoPr: plan.delivery !== 'none',
     autoMerge: plan.autoMerge,
-    autoPush: process.env.ZEROSHOT_PUSH === '1',
+    autoPush: environment ? process.env.ZEROSHOT_PUSH === '1' : options.autoPush === true,
     modelOverride: optionalValue(modelOverride),
     providerOverride: optionalValue(providerOverride),
-    noMounts: anyTruthy(mergedOptions.mounts === false, mergedOptions.noMounts === true),
-    mounts: resolveMounts(mergedOptions),
-    containerHome: optionalValue(mergedOptions.containerHome),
+    noMounts: anyTruthy(options.mounts === false, options.noMounts === true),
+    mounts: resolveMounts(options),
+    containerHome: optionalValue(options.containerHome),
     forceProvider: optionalValue(forceProvider),
-    prBase: resolvePrBase(mergedOptions),
-    mergeQueue: resolveMergeQueue(mergedOptions),
-    closeIssue: resolveCloseIssue(mergedOptions),
-    ship: mergedOptions.ship,
+    prBase: environment ? resolvePrBase(options) : optionalValue(options.prBase),
+    mergeQueue: environment ? resolveMergeQueue(options) : optionalValue(options.mergeQueue),
+    closeIssue: environment ? resolveCloseIssue(options) : optionalValue(options.closeIssue),
+    ship: plan.delivery === 'ship',
     runMode: runModeFromPlan(plan),
-    requiredQualityGates: mergedOptions.requiredQualityGates,
+    requiredQualityGates: options.requiredQualityGates,
+    settings,
+  });
+}
+
+/**
+ * Build start options from a registry-owned canonical plan. Unlike the CLI
+ * builder, this function never reads ZEROSHOT_RUN_OPTIONS or mode-related
+ * environment variables. The caller must resolve and validate the deployment
+ * profile before invoking it.
+ */
+function buildTrustedStartOptions({
+  clusterId,
+  plan,
+  options = {},
+  settings = {},
+  providerOverride,
+  forceProvider,
+}) {
+  if (!plan || !Object.isFrozen(plan)) {
+    throw new Error('Trusted start requires a frozen canonical run plan');
+  }
+  const canonical = resolveRunPlan({
+    docker: plan.isolation === 'docker',
+    worktree: plan.isolation === 'worktree',
+    pr: plan.delivery === 'pr',
+    ship: plan.delivery === 'ship',
+  });
+  if (
+    canonical.isolation !== plan.isolation ||
+    canonical.delivery !== plan.delivery ||
+    canonical.autoMerge !== plan.autoMerge
+  ) {
+    throw new Error('Trusted start plan is not canonical');
+  }
+  if (canonical.isolation === 'none') {
+    throw new Error('Trusted worker start requires worktree or docker isolation');
+  }
+  return buildStartOptionsFromPlan({
+    clusterId,
+    plan: canonical,
+    options,
     settings,
-  };
+    providerOverride,
+    forceProvider,
+    environment: false,
+  });
 }
 
 function resolveConfigOrThrow({
@@ -352,8 +425,10 @@ module.exports = {
   decodeStdinEnv,
   resolveProviderOverride,
   resolveConfigPath,
+  prepareClusterConfig,
   loadClusterConfig,
   buildStartOptions,
+  buildTrustedStartOptions,
   resolveEffectiveRunPlan,
   startClusterFromText,
   startClusterFromIssue,

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@the-open-engine/zeroshot",
-  "version": "6.5.0",
+  "version": "6.7.0",
   "description": "Multi-agent orchestration engine for Claude, Codex, and Gemini",
   "main": "src/orchestrator.js",
   "bin": {
     "zeroshot": "./cli/index.js",
-    "zeroshot-agent-provider": "./lib/agent-cli-provider/executable.js"
+    "zeroshot-agent-provider": "./lib/agent-cli-provider/executable.js",
+    "zeroshot-cluster-worker": "./bin/zeroshot-cluster-worker.js"
   },
   "directories": {
     "example": "examples",
@@ -63,6 +64,8 @@
     "validate:templates": "node scripts/validate-templates.js",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
+    "protocol:check": "cargo run -p openengine-cluster-testkit --bin generate-cluster-protocol -- --check",
+    "rust:check": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && cargo test --workspace",
     "deadcode": "ts-prune --skip node_modules",
     "deadcode:files": "unimported",
     "deadcode:deps": "depcheck",
@@ -71,6 +74,8 @@
     "check": "npm run typecheck && npm run lint",
     "check:all": "npm run check && npm run deadcode:all",
     "release": "semantic-release",
+    "release:preflight": "node scripts/release-preflight.js",
+    "release:assert-published": "node scripts/assert-release-published.js",
     "prepublishOnly": "npm run lint && npm run typecheck && npm run check:agent-cli-provider:ci",
     "prepare": "npm run build:agent-cli-provider && husky"
   },
@@ -121,12 +126,15 @@
   "files": [
     "src/",
     "lib/",
+    "bin/",
     "cli/",
     "task-lib/",
     "cluster-templates/",
     "cluster-hooks/",
     "docker/",
     "scripts/",
+    "protocol/openengine-cluster/v1/worker.schema.json",
+    "docs/openengine-cluster-protocol/v1/legacy-worker.md",
     "README.md",
     "LICENSE",
     "CHANGELOG.md"