@the-open-engine/zeroshot 6.25.0 → 6.26.0

package/task-lib/rpc-watcher.js

@@ -26,9 +26,9 @@ import { getTask, updateTask } from './store.js';
 import { createCommandSpecCleanup } from './command-spec-cleanup.js';
 import {
   commitOwnership,
-  markCleanupRequired,
   readOwnership,
   recordVerifiedMaterialization,
+  retireOmpOwnershipAtTerminalBoundary,
   transferOmpSessionOwnership,
 } from './omp-session-ownership.js';
 import {
@@ -98,14 +98,16 @@ function terminateOwnedProviderBoundary(exitObserved = false) {
   );
 }
 
+/** Every failed/cancelled/uncertain terminal boundary in this watcher routes through the shared
+ * durable-boundary retirement, so a post-transfer failure leaves the resumed row cleanup-required
+ * rather than provisional forever. Idempotent and never throwing, so it cannot itself prevent the
+ * task from reaching a terminal status. */
 function markOmpCleanupRequiredSafely() {
-  try {
-    markCleanupRequired(taskId);
-  } catch (error) {
+  retireOmpOwnershipAtTerminalBoundary(taskId, (error) =>
     emergencyLog(
       `[${Date.now()}][OMP-OWNERSHIP] Failed to mark cleanup-required: ${error.message}\n`
-    );
-  }
+    )
+  );
 }
 
 async function crashWithError(error, source) {
@@ -135,13 +137,14 @@ function identityText(identity) {
  * a re-minted session ID, a substituted inode, or an execution-contract change all fail here.
  *
  * `evidence` is null on the pre-spawn pass, where OMP has reported nothing yet; the structural and
- * identity halves still apply.
+ * identity halves still apply. When evidence *is* present this is the pre-prompt checkpoint, so
+ * the echoed session identity must be complete (see assertEchoedSessionMatches).
  */
 function assertNoResumeDrift(verified, evidence) {
   const expected = ompResumeExpectation;
   const drift = [];
 
-  assertEchoedSessionMatches(evidence, drift);
+  assertEchoedSessionMatches(evidence, drift, { requireComplete: evidence !== null });
 
   if (verified.sessionFileName !== expected.sessionFileName) {
     drift.push(`sessionFileName ${verified.sessionFileName} != ${expected.sessionFileName}`);
@@ -149,13 +152,16 @@ function assertNoResumeDrift(verified, evidence) {
   if (verified.sessionFilePath !== expected.sessionFilePath) {
     drift.push(`sessionFilePath ${verified.sessionFilePath} != ${expected.sessionFilePath}`);
   }
-  if (identityText(verified.partitionIdentity) !== identityText(expected.expectedPartitionIdentity)) {
+  if (
+    identityText(verified.partitionIdentity) !== identityText(expected.expectedPartitionIdentity)
+  ) {
     drift.push(
       `partitionIdentity ${identityText(verified.partitionIdentity)} != ${identityText(expected.expectedPartitionIdentity)}`
     );
   }
   if (
-    identityText(verified.sessionFileIdentity) !== identityText(expected.expectedSessionFileIdentity)
+    identityText(verified.sessionFileIdentity) !==
+    identityText(expected.expectedSessionFileIdentity)
   ) {
     drift.push(
       `sessionFileIdentity ${identityText(verified.sessionFileIdentity)} != ${identityText(expected.expectedSessionFileIdentity)}`
@@ -192,11 +198,35 @@ function assertNoResumeDrift(verified, evidence) {
  * transcript has legitimately grown — re-running the manifest/inode comparison there would reject
  * a perfectly healthy turn. The identity and fingerprint comparisons below have no such staleness,
  * so a mid-turn switch to a different session or a changed model/thinking level is still caught.
+ *
+ * `requireComplete` is the difference between the two callers, and it is the difference between
+ * "OMP agreed it opened exactly this session" and "OMP declined to say".
+ *
+ * At the pre-prompt checkpoint of a resume it is set, and both echoed values must be present and
+ * exactly equal: the full session ID and the full absolute session file. Without it, a `get_state`
+ * that simply omits `sessionId` (or `sessionFile`) would transfer a committed lineage and receive
+ * the prompt on the strength of the *disk* alone — and disk state cannot answer the only question
+ * that matters here, which is which session the running OMP process actually attached to. A prefix
+ * is never enough either: OMP resolves `--resume` IDs by prefix (session-manager.ts), so a shorter
+ * echoed ID is precisely the ambiguity this check exists to reject.
+ *
+ * `session_info_update` passes leave it unset: those frames legitimately carry only a subset
+ * (docs/rpc.md), and the driver merges them onto evidence whose complete form this checkpoint has
+ * already proven, so a partial later frame is checked against that proof rather than replacing it.
  */
-function assertEchoedSessionMatches(evidence, drift) {
+function assertEchoedSessionMatches(evidence, drift, { requireComplete = false } = {}) {
   if (!evidence) return;
   const expected = ompResumeExpectation;
 
+  if (requireComplete) {
+    if (!evidence.sessionId) {
+      drift.push('OMP reported no sessionId at the pre-prompt checkpoint');
+    }
+    if (!evidence.sessionFile) {
+      drift.push('OMP reported no sessionFile at the pre-prompt checkpoint');
+    }
+  }
+
   if (evidence.selectedProvider !== expected.expectedSelectedProvider) {
     drift.push(
       `selectedProvider ${evidence.selectedProvider} != ${expected.expectedSelectedProvider}`
@@ -263,10 +293,16 @@ function verifyOmpSessionBeforeSpawn() {
 /**
  * Owner-fenced ownership transfer, run from the `ready` hook after re-verification and strictly
  * before the prompt command is written. One transaction moves the prior committed owner's lineage
- * onto this task's provisional row and clears the prior row, so the partition is never owned by
- * two rows and never owned by none. A transfer that does not apply (the prior owner moved, this
- * row already advanced) throws, which fails the turn closed through the same cleanup-required path
- * as any other drift — the resumed session is never steered on an unresolved ownership claim.
+ * onto this task's provisional row and clears the prior row, so the partition never has two
+ * committed owners.
+ *
+ * From here to this turn's own success boundary the partition has *no* committed owner at all —
+ * the authoritative live claimant is this still-`provisional` row, by design, and every partition
+ * fence is written for that (see findAuthoritativeOwnersForPartition). A transfer that does not
+ * apply (the prior owner moved, this row already advanced) throws, which fails the turn closed
+ * through the same cleanup-required path as any other drift: the resumed session is never steered
+ * on an unresolved ownership claim, and a turn that dies after the transfer retires the row it now
+ * holds rather than stranding the lineage.
  */
 function transferResumedOwnershipBeforePrompt() {
   if (ownershipTransferred) return;

package/task-lib/runner.js

@@ -3,13 +3,17 @@ import { join, dirname, resolve as resolvePath } from 'path';
 import { fileURLToPath } from 'url';
 import { mkdirSync } from 'fs';
 import { LOGS_DIR } from './config.js';
-import { addTask, generateId, ensureDirs } from './store.js';
+import { addTask, generateId, ensureDirs, updateTask } from './store.js';
 import {
   isOmpSessionlessRun,
   resolveOmpStorageRoot,
   resolveOmpOwnerKind,
 } from './omp-storage-root.js';
-import { readOwnership, writeProvisionalOwnership } from './omp-session-ownership.js';
+import {
+  readOwnership,
+  retireOmpOwnershipAtTerminalBoundary,
+  writeProvisionalOwnership,
+} from './omp-session-ownership.js';
 import { createRequire } from 'module';
 
 const require = createRequire(import.meta.url);
@@ -202,6 +206,38 @@ function resolveOmpSessionPlan({ id, cwd, options }) {
   };
 }
 
+/**
+ * Close a spawn that failed after its row was written but before anything could own the task.
+ *
+ * Two durable transitions, in this order and both idempotent, so a retry or a crash-recovery
+ * replay converges on the same state:
+ *   1. the OMP ownership record is retired to `cleanup-required`, releasing the partition claim
+ *      that would otherwise block every future reclaim of that directory;
+ *   2. the task row reaches a terminal status, so nothing downstream (status, kill, resume, the
+ *      stuck-task recovery sweep) keeps treating it as a live run.
+ *
+ * The decision comes from this boundary alone. Whether the partition directory exists is not
+ * consulted and must not be: a partial mkdir and a clean failure are indistinguishable on disk,
+ * and the row is the only thing that knows a spawn was attempted at all.
+ */
+function failSpawnAtProvisionalBoundary(id, error) {
+  retireOmpOwnershipAtTerminalBoundary(id, (ownershipError) => {
+    console.warn(
+      `Warning: failed to retire the OMP session ownership of task ${id}: ${ownershipError.message}`
+    );
+  });
+  try {
+    updateTask(id, {
+      status: 'failed',
+      pid: null,
+      exitCode: 1,
+      error: `Task spawn failed before the provider started: ${error.message}`,
+    });
+  } catch (updateError) {
+    console.warn(`Warning: failed to mark task ${id} failed: ${updateError.message}`);
+  }
+}
+
 export function spawnTask(prompt, options = {}) {
   ensureDirs();
 
@@ -235,11 +271,22 @@ export function spawnTask(prompt, options = {}) {
   });
 
   // Row-before-directory: the SQL row is durable proof of an attempted allocation before the
-  // partition directory (or anything else OMP-owned) exists on disk. A crash between these two
+  // partition directory (or anything else OMP-owned) exists on disk. A *crash* between these two
   // lines leaves a provisional row pointing at a path with nothing there yet — cleanup safely
   // no-ops on a nonexistent path, and normal task-lifecycle recovery handles the row itself.
+  //
+  // A *thrown* materialization failure is different, and must not be left to recovery: this
+  // process is still alive and owns the row, so it has to close the boundary itself. Without this,
+  // an EACCES/ENOSPC mkdir left a row that looks forever like a live task holding a live
+  // provisional claim on a partition — permanently unreclaimable, because cleanup refuses to touch
+  // a partition any other row still claims provisionally.
   addTask(task);
-  ompPlan?.createDirectory();
+  try {
+    ompPlan?.createDirectory();
+  } catch (error) {
+    failSpawnAtProvisionalBoundary(id, error);
+    throw error;
+  }
 
   const watcherConfig = buildWatcherConfig(
     outputFormat,

package/task-lib/store.js

@@ -11,6 +11,7 @@ import Database from 'better-sqlite3';
 import { TASKS_DIR, LOGS_DIR } from './config.js';
 import { generateName } from './name-generator.js';
 import {
+  inspectStoredOmpSessionOwnership,
   parseOmpSessionOwnership,
   serializeOmpSessionOwnership,
 } from './omp-session-ownership-schema.js';
@@ -213,9 +214,22 @@ function rowToTask(row) {
     cancelRequested: Boolean(row.cancel_requested),
     spawnOwnershipToken: row.spawn_ownership_token,
     ompSessionOwnership: parseOmpSessionOwnership(row.omp_session_ownership),
+    // Raw-presence seam (see inspectStoredOmpSessionOwnership). `ompSessionOwnership: null` alone
+    // cannot distinguish "this task never had an OMP session" from "this task's owner record is
+    // unreadable", and those demand opposite handling: the first is nothing to clean, the second
+    // means a partition may exist that only this row still points at. The malformed bytes
+    // themselves are deliberately not exposed — nothing may act on them.
+    ompSessionOwnershipPresent: inspectStoredOmpSessionOwnership(row.omp_session_ownership).present,
   };
 }
 
+/** True when a task row carries an `omp_session_ownership` value that exists but cannot be read as
+ * the closed schema. Such a row is retained by every cleanup surface with a warning: deleting it
+ * would orphan whatever partition the unreadable record described. */
+export function hasUnreadableOmpSessionOwnership(task) {
+  return Boolean(task?.ompSessionOwnershipPresent) && !task?.ompSessionOwnership;
+}
+
 /**
  * Load all tasks as object keyed by id
  * @returns {Object.<string, Object>}
@@ -230,78 +244,6 @@ export function loadTasks() {
   return tasks;
 }
 
-/**
- * Save all tasks (replaces entire store - for migration compatibility)
- * @param {Object.<string, Object>} tasks
- */
-export function saveTasks(tasks) {
-  const database = getDb();
-  const insert = database.prepare(`
-    INSERT OR REPLACE INTO tasks (
-      id, prompt, full_prompt, cwd, status, pid, session_id, session_id_conflict, requested_resume_session_id, resume_identity_verified, log_file,
-      created_at, updated_at, exit_code, error, provider, model,
-      schedule_id, socket_path, attachable, process_group_id, termination_strategy,
-      command_cleanup, cancel_requested, spawn_ownership_token, omp_session_ownership
-    ) VALUES (
-      @id, @prompt, @fullPrompt, @cwd, @status, @pid, @sessionId, @sessionIdConflict, @requestedResumeSessionId, @resumeIdentityVerified, @logFile,
-      @createdAt, @updatedAt, @exitCode, @error, @provider, @model,
-      @scheduleId, @socketPath, @attachable, @processGroupId, @terminationStrategy,
-      @commandCleanup, @cancelRequested, @spawnOwnershipToken, @ompSessionOwnership
-    )
-  `);
-
-  const insertMany = database.transaction((tasksObj) => {
-    // Clear existing
-    database.prepare('DELETE FROM tasks').run();
-    // Insert all
-    for (const task of Object.values(tasksObj)) {
-      insert.run({
-        id: task.id,
-        prompt: task.prompt || null,
-        fullPrompt: task.fullPrompt || null,
-        cwd: task.cwd || null,
-        status: task.status || 'pending',
-        pid: task.pid || null,
-        sessionId: task.sessionId || null,
-        sessionIdConflict: task.sessionIdConflict ? 1 : 0,
-        requestedResumeSessionId: nullable(task.requestedResumeSessionId),
-        resumeIdentityVerified: task.resumeIdentityVerified ? 1 : 0,
-        logFile: task.logFile || null,
-        createdAt: task.createdAt || new Date().toISOString(),
-        updatedAt: task.updatedAt || new Date().toISOString(),
-        exitCode: task.exitCode ?? null,
-        error: task.error || null,
-        provider: task.provider || null,
-        model: task.model || null,
-        scheduleId: task.scheduleId || null,
-        socketPath: task.socketPath || null,
-        attachable: task.attachable ? 1 : 0,
-        processGroupId: task.processGroupId || null,
-        terminationStrategy: task.terminationStrategy || null,
-        commandCleanup: serializeCommandCleanup(task.commandCleanup),
-        cancelRequested: task.cancelRequested ? 1 : 0,
-        spawnOwnershipToken: task.spawnOwnershipToken || null,
-        ompSessionOwnership: serializeOmpSessionOwnership(task.ompSessionOwnership || null),
-      });
-    }
-  });
-
-  insertMany(tasks);
-}
-
-/**
- * For API compatibility - just runs the modifier synchronously
- * SQLite WAL handles concurrency, no lock needed
- * @param {Function} modifier
- * @returns {any}
- */
-export function withTasksLock(modifier) {
-  const tasks = loadTasks();
-  const result = modifier(tasks);
-  saveTasks(tasks);
-  return result;
-}
-
 /**
  * Get a single task by id
  * @param {string} id
@@ -487,6 +429,59 @@ export function removeTask(id) {
   getDb().prepare('DELETE FROM tasks WHERE id = ?').run(id);
 }
 
+/**
+ * Remove a task row only while it still matches the snapshot the caller validated.
+ *
+ * `clean`/`purge` decide what to remove from one `loadTasks()` snapshot and then do real work
+ * (partition staging, command cleanup, log deletion) before they get to the delete. A watcher,
+ * a resume's ownership transfer, or a kill can land in that window; deleting on the strength of a
+ * stale snapshot would destroy a row that is no longer the row that was examined. The status and
+ * the exact ownership bytes are the two fields that decide whether removal is still correct, so
+ * both are the fence. `store.js` writes the ownership column only through
+ * `serializeOmpSessionOwnership`, whose output is canonical per record, which is what makes a
+ * byte comparison an exact "same record" test.
+ *
+ * @param {string} id
+ * @param {{status: string, ompSessionOwnership: object|null}} expected snapshot values
+ * @returns {boolean} true when the row was removed
+ */
+export function removeTaskIfUnchanged(id, expected) {
+  const result = getDb()
+    .prepare(
+      `DELETE FROM tasks
+       WHERE id = ? AND status IS ? AND omp_session_ownership IS ?`
+    )
+    .run(
+      id,
+      expected?.status ?? null,
+      serializeOmpSessionOwnership(expected?.ompSessionOwnership || null)
+    );
+  return result.changes === 1;
+}
+
+/**
+ * Clear one row's persisted command-cleanup receipt, and nothing else, only if it is still the
+ * exact serialized receipt the caller processed.
+ *
+ * Deliberately narrower than updateTask(), which is a read-modify-write over every column and
+ * would write back whatever the caller's snapshot held for the rest of the row. This is used on
+ * the retained path of `clean`, where a concurrent writer may already have installed a new cleanup
+ * receipt that must survive.
+ *
+ * @param {string} id
+ * @param {object} expected exact command-cleanup receipt processed by the caller
+ * @returns {boolean} true when that exact receipt was cleared
+ */
+export function clearTaskCommandCleanup(id, expected) {
+  const result = getDb()
+    .prepare(
+      `UPDATE tasks SET command_cleanup = NULL, updated_at = ?
+       WHERE id = ? AND command_cleanup IS ?`
+    )
+    .run(new Date().toISOString(), id, serializeCommandCleanup(expected));
+  return result.changes === 1;
+}
+
 export function generateId() {
   return generateName('task');
 }