@the-open-engine/zeroshot 6.24.0 → 6.25.0

package/task-lib/omp-session-ownership.js

@@ -0,0 +1,332 @@
+// Owner-fenced transitions for task.ompSessionOwnership (schema v5).
+//
+// Every transition is a SQL compare-and-swap against the *exact* JSON currently persisted for that
+// row, so a duplicate or re-entrant completion call (two crash-recovery paths racing, a retried
+// hook, a watcher and its parent both reacting to the same terminal frame) can never clobber a
+// state a concurrent writer already advanced past. `store.js` always writes this column through
+// `serializeOmpSessionOwnership`, whose output is canonical for a given record, which is what makes
+// full-value CAS byte-stable.
+//
+// State machine, identical for fresh and resumed turns:
+//
+//   (row inserted)        provisional
+//   materialization       provisional + observation      recordVerifiedMaterialization / commitOwnership
+//   success boundary      committed                      commitOwnership / commitRecordedOwnership
+//   any other boundary    cleanup-required               markCleanupRequired
+//
+// A resumed turn additionally performs `transferOmpSessionOwnership` before its prompt is written:
+// in one transaction the prior committed owner's record is cleared and its lineage (partition
+// identity + session tuple) is moved onto the resumed task's still-`provisional` row. That keeps
+// exactly one *committed* owner — and keeps "committed" meaning what it means everywhere else,
+// this turn's own success boundary has passed — instead of publishing a half-finished
+// continuation as resumable.
+//
+// It does NOT keep exactly one row *referencing* the partition. A resumed row is inserted (and
+// therefore already names the partition) before its transfer runs, so two competing resumes of the
+// same committed session put three rows on one partition: the prior owner plus both candidates.
+// Only one transfer can win; the loser fails closed and is retired to `cleanup-required` holding a
+// record that still names the partition but carries no lineage of its own. Anything that acts on a
+// partition — cleanup above all — must therefore fence on *every* row that references it
+// (`findAuthoritativeOwnersForPartition`), never on the committed rows alone and never on the
+// assumption that its own row is the only claimant.
+import { basename } from 'path';
+import { statSync } from 'fs';
+import { getTask, getTaskStoreDatabase } from './store.js';
+import {
+  buildProvisionalOwnership,
+  computeExecutionFingerprint,
+  serializeOmpSessionOwnership,
+  validateOmpSessionOwnership,
+  validateOwnedByTask,
+} from './omp-session-ownership-schema.js';
+
+export { computeExecutionFingerprint };
+
+function statIdentity(targetPath) {
+  const stat = statSync(targetPath);
+  return { device: String(stat.dev), inode: String(stat.ino) };
+}
+
+/** Pure builder for the initial provisional record; embed the result in the task row passed to
+ * addTask() so the SQL row is durable before the partition directory is created on disk. */
+export function writeProvisionalOwnership({
+  partitionId,
+  storageRoot,
+  canonicalWorkspace,
+  owner,
+}) {
+  return buildProvisionalOwnership({
+    partitionId,
+    storageRoot,
+    storageRootIdentity: statIdentity(storageRoot),
+    canonicalWorkspace,
+    owner,
+  });
+}
+
+/** Read a row's ownership record, fenced to that row: a well-formed record whose `owner.taskId` is
+ * some other task is not this row's ownership and is never returned. */
+export function readOwnership(taskId) {
+  return validateOwnedByTask(getTask(taskId)?.ompSessionOwnership ?? null, taskId);
+}
+
+/**
+ * Compare-and-swap the whole record. `expected` must be the record as currently persisted; a
+ * concurrent writer that has already changed the row makes this a no-op returning false.
+ */
+function casOwnership(taskId, expected, next) {
+  const validatedNext = validateOwnedByTask(next, taskId);
+  if (!validatedNext) return false;
+  const expectedJson = serializeOmpSessionOwnership(expected);
+  const database = getTaskStoreDatabase();
+  const result = database
+    .prepare(
+      `UPDATE tasks SET omp_session_ownership = ?, updated_at = ?
+       WHERE id = ? AND omp_session_ownership = ?`
+    )
+    .run(
+      serializeOmpSessionOwnership(validatedNext),
+      new Date().toISOString(),
+      taskId,
+      expectedJson
+    );
+  return result.changes === 1;
+}
+
+/** Shape the observed materialization evidence. Returns null (never throws) if a stat fails. */
+function buildObservedEvidence(
+  current,
+  {
+    sessionId,
+    sessionFilePath,
+    partitionIdentity,
+    sessionFileIdentity,
+    artifactManifestDigest,
+    executionFingerprint,
+    selectedProvider,
+    selectedModel,
+  }
+) {
+  try {
+    return {
+      // The verifier hands back descriptor-pinned identities; fall back to a stat only when a
+      // caller (a test double, or a path that never ran the verifier) did not supply them.
+      partitionIdentity: partitionIdentity ?? statIdentity(current.partitionPath),
+      session: {
+        sessionId,
+        fileName: basename(sessionFilePath),
+        fileIdentity: sessionFileIdentity ?? statIdentity(sessionFilePath),
+        artifactManifestDigest,
+        executionFingerprint,
+        selectedProvider,
+        selectedModel,
+      },
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Persist owner-fenced verified materialization evidence WITHOUT advancing state. Used by the
+ * detached RPC watcher for cluster-agent owners: the watcher verifies the terminal session file
+ * itself (two-phase file contract) but must never decide "committed" on its own — that decision
+ * belongs to the parent agent process's post-hook success boundary (see commitRecordedOwnership).
+ * Fails closed (returns false, never throws) when the record is missing, is not this task's, has
+ * already left `provisional`, or the evidence does not validate.
+ */
+export function recordVerifiedMaterialization({ taskId, ...evidence }) {
+  const current = readOwnership(taskId);
+  if (!current || current.state !== 'provisional') return false;
+  const observed = buildObservedEvidence(current, evidence);
+  if (!observed) return false;
+  return casOwnership(taskId, current, { ...current, state: 'provisional', ...observed });
+}
+
+/**
+ * Commit a provisional record once the terminal boundary for this task's owner kind has actually
+ * succeeded (standalone: the watcher's own output validation; cluster-agent: logical/schema/
+ * onComplete hook success). Fails closed (returns false, never throws); the caller must treat a
+ * false return as "did not commit" and mark cleanup-required instead.
+ */
+export function commitOwnership({ taskId, ...evidence }) {
+  const current = readOwnership(taskId);
+  if (!current || current.state !== 'provisional') return false;
+  const observed = buildObservedEvidence(current, evidence);
+  if (!observed) return false;
+  return casOwnership(taskId, current, { ...current, state: 'committed', ...observed });
+}
+
+/**
+ * Commit using evidence the watcher already recorded via recordVerifiedMaterialization — never
+ * re-verifies the partition. This is the ONLY path that may advance a cluster-agent owner to
+ * `committed`, and only from the post-hook success boundary in src/agent/agent-lifecycle.js:
+ * committing earlier would let a later turn resume a turn whose logical/schema output or
+ * onComplete hook subsequently failed. Returns false when no verified evidence exists yet.
+ */
+export function commitRecordedOwnership(taskId) {
+  const current = readOwnership(taskId);
+  if (!current || current.state !== 'provisional') return false;
+  if (!current.partitionIdentity || !current.session) return false;
+  return casOwnership(taskId, current, { ...current, state: 'committed' });
+}
+
+/**
+ * Mark a still-provisional record cleanup-required on any failed, cancelled, or uncertain terminal
+ * boundary. No-op once a record has left `provisional`: a `committed` record is never downgraded,
+ * because commit is strictly the last action of an already-successful turn — and a resumed turn is
+ * provisional right up to that same boundary (see transferOmpSessionOwnership), so retiring a
+ * failed continuation needs no exception here.
+ */
+export function markCleanupRequired(taskId) {
+  const current = readOwnership(taskId);
+  if (!current || current.state !== 'provisional') return current;
+  const updated = { ...current, state: 'cleanup-required' };
+  return casOwnership(taskId, current, updated) ? updated : readOwnership(taskId);
+}
+
+/**
+ * The ownership states that constitute a live claim on a partition.
+ *
+ * `provisional` is authoritative because it is the state a turn occupies while it is *using* the
+ * partition — a fresh turn writing into it, and (post-transfer) a resumed turn continuing it right
+ * up to its own success boundary. `committed` is authoritative because the session is resumable.
+ *
+ * `cleanup-required` is deliberately NOT authoritative: it is the state of a turn that has already
+ * been retired and makes no further claim. Treating it as one would deadlock cleanup whenever two
+ * retired rows name the same partition (the third-owner residue below) — each would refuse forever
+ * on account of the other, and the partition could never be reclaimed by anybody.
+ */
+export const AUTHORITATIVE_OWNERSHIP_STATES = Object.freeze(['provisional', 'committed']);
+
+const AUTHORITATIVE_STATES = new Set(AUTHORITATIVE_OWNERSHIP_STATES);
+
+/** Every row other than `excludeTaskId` whose *own* valid record names this partition, as
+ * `{taskId, state}`. Rows whose stored JSON is unparseable, invalid, or owned by a different task
+ * id are not that row's ownership and are skipped. */
+function ownersForPartition(partitionId, excludeTaskId, database) {
+  const rows = database
+    .prepare(
+      `SELECT id, omp_session_ownership FROM tasks
+       WHERE omp_session_ownership IS NOT NULL
+         AND json_extract(omp_session_ownership, '$.partitionId') = ?`
+    )
+    .all(partitionId);
+  const owners = [];
+  for (const row of rows) {
+    if (row.id === excludeTaskId) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(row.omp_session_ownership);
+    } catch {
+      continue;
+    }
+    const record = validateOwnedByTask(parsed, row.id);
+    if (record) owners.push({ taskId: row.id, state: record.state });
+  }
+  return owners;
+}
+
+/** Rows other than `excludeTaskId` that still hold a committed record for this partition — i.e.
+ * the rows for which this partition is a *resumable* session. */
+export function findCommittedOwnersForPartition(
+  partitionId,
+  excludeTaskId = null,
+  database = getTaskStoreDatabase()
+) {
+  return ownersForPartition(partitionId, excludeTaskId, database)
+    .filter((owner) => owner.state === 'committed')
+    .map((owner) => owner.taskId);
+}
+
+/**
+ * Rows other than `excludeTaskId` holding an authoritative (`provisional` or `committed`) claim on
+ * this partition, as `{taskId, state}`.
+ *
+ * This is the owner fence cleanup runs on, and it is strictly wider than the committed owners:
+ * after a resume's atomic transfer the winning row is `provisional` — it carries the whole
+ * inherited lineage and is actively continuing the session, while *no* row is committed. A losing
+ * competing resume, retired to `cleanup-required`, still names the same partition and holds no
+ * lineage of its own, so a committed-only fence would see nothing and let it delete the winner's
+ * live partition out from under it.
+ *
+ * `database` is injectable so a caller can run this inside its own write transaction (see
+ * task-lib/omp-session-cleanup.js) rather than racing its own check.
+ */
+export function findAuthoritativeOwnersForPartition(
+  partitionId,
+  excludeTaskId = null,
+  database = getTaskStoreDatabase()
+) {
+  return ownersForPartition(partitionId, excludeTaskId, database).filter((owner) =>
+    AUTHORITATIVE_STATES.has(owner.state)
+  );
+}
+
+/**
+ * Atomically move a committed partition's ownership from `fromTaskId` to `toTaskId`, before the
+ * resumed turn's prompt is written.
+ *
+ * Both sides are fenced on their exact current JSON inside one transaction, so the outcome is all
+ * or nothing: either the prior owner's record is cleared *and* the resumed row carries the
+ * inherited lineage, or nothing changed and the caller must fail the turn closed. There is never a
+ * window in which two rows are committed owners of the same partition, nor one in which the
+ * partition has no owner row at all.
+ *
+ * Returns the transferred record, or null when the transfer did not apply (prior owner already
+ * moved, resumed row already advanced, lineage mismatch).
+ */
+export function transferOmpSessionOwnership({ fromTaskId, toTaskId }) {
+  if (!fromTaskId || !toTaskId || fromTaskId === toTaskId) return null;
+  const prior = readOwnership(fromTaskId);
+  const incoming = readOwnership(toTaskId);
+  if (!prior || prior.state !== 'committed' || !prior.session || !prior.partitionIdentity) {
+    return null;
+  }
+  if (!incoming || incoming.state !== 'provisional') return null;
+  if (
+    incoming.partitionId !== prior.partitionId ||
+    incoming.partitionPath !== prior.partitionPath ||
+    incoming.storageRoot !== prior.storageRoot
+  ) {
+    return null;
+  }
+
+  const transferred = {
+    ...incoming,
+    state: 'provisional',
+    partitionIdentity: prior.partitionIdentity,
+    session: prior.session,
+  };
+  if (!validateOwnedByTask(transferred, toTaskId)) return null;
+
+  const database = getTaskStoreDatabase();
+  const now = new Date().toISOString();
+  const priorJson = serializeOmpSessionOwnership(prior);
+  const incomingJson = serializeOmpSessionOwnership(incoming);
+  const transferredJson = serializeOmpSessionOwnership(transferred);
+
+  const apply = database.transaction(() => {
+    const released = database
+      .prepare(
+        `UPDATE tasks SET omp_session_ownership = NULL, updated_at = ?
+         WHERE id = ? AND omp_session_ownership = ?`
+      )
+      .run(now, fromTaskId, priorJson);
+    if (released.changes !== 1) throw new Error('prior-owner-moved');
+    const claimed = database
+      .prepare(
+        `UPDATE tasks SET omp_session_ownership = ?, updated_at = ?
+         WHERE id = ? AND omp_session_ownership = ?`
+      )
+      .run(transferredJson, now, toTaskId, incomingJson);
+    if (claimed.changes !== 1) throw new Error('resumed-owner-moved');
+  });
+
+  try {
+    apply();
+  } catch {
+    return null;
+  }
+  return validateOmpSessionOwnership(transferred);
+}

package/task-lib/omp-storage-root.js

@@ -0,0 +1,35 @@
+// Where OMP session partitions live on disk: the owning cluster's storageDir for cluster-agent
+// tasks (passed down via env since agent-task-executor.js spawns this CLI as a child process), or
+// the standalone TASKS_DIR otherwise. Never derived from prompt text or cwd.
+import { TASKS_DIR } from './config.js';
+
+export const OMP_STORAGE_ROOT_ENV = 'ZEROSHOT_OMP_STORAGE_ROOT';
+export const OMP_OWNER_CLUSTER_ID_ENV = 'ZEROSHOT_CLUSTER_ID';
+export const OMP_OWNER_AGENT_ID_ENV = 'ZEROSHOT_AGENT_ID';
+/**
+ * Set by the agent for a Docker-isolated OMP run. Issue #866 keeps Docker fresh-only, and the
+ * container is the reason: its filesystem is ephemeral, so a partition allocated inside it could
+ * never be resumed and an ownership row pointing at it would be unreclaimable the moment the
+ * container is removed. A task carrying this marker allocates no partition and persists no
+ * ownership row — the adapter launches `--no-session`.
+ */
+export const OMP_SESSIONLESS_ENV = 'ZEROSHOT_OMP_SESSIONLESS';
+
+/** True when this task must run without any session partition at all. */
+export function isOmpSessionlessRun(options = {}) {
+  return options.sessionless === true || process.env[OMP_SESSIONLESS_ENV] === '1';
+}
+
+export function resolveOmpStorageRoot(options = {}) {
+  return options.storageRoot || process.env[OMP_STORAGE_ROOT_ENV] || TASKS_DIR;
+}
+
+/** cluster-agent when both a cluster and agent id are known, standalone otherwise. */
+export function resolveOmpOwnerKind(options = {}) {
+  const clusterId = options.clusterId || process.env[OMP_OWNER_CLUSTER_ID_ENV] || null;
+  const agentId = options.agentId || process.env[OMP_OWNER_AGENT_ID_ENV] || null;
+  if (clusterId && agentId) {
+    return { kind: 'cluster-agent', clusterId, agentId };
+  }
+  return { kind: 'standalone', clusterId: null, agentId: null };
+}