chainlesschain 0.163.6 → 0.163.7

package/src/lib/agent-schedule-store.js

@@ -22,7 +22,7 @@
 import fs from "fs";
 import os from "os";
 import path from "path";
-import { randomUUID } from "crypto";
+import { createHash, randomUUID } from "crypto";
 import { isEntryExpired, effectiveFireAt } from "./schedule-planner.js";
 import { SUBAGENT_PERMISSION_MODES } from "./subagent-contract.js";
 import { parseGoalCondition } from "./goal-condition-engine.js";
@@ -30,6 +30,7 @@ import { normalizeActionClass } from "./unattended-action-policy.js";
 import { withFileLock } from "./with-file-lock.js";
 
 export const SCHEDULE_KINDS = Object.freeze(["wakeup", "cron", "monitor"]);
+export const AGENDA_SCHEDULER_MIGRATION_SCHEMA_VERSION = 1;
 
 /** A valid absolute-epoch-ms expiry, or null (never expires). */
 function normalizeExpiresAt(v) {
@@ -168,6 +169,87 @@ function hasLiveExecutionLease(entry, now) {
   return Number(entry.executionLease?.expiresAt) > Number(now);
 }
 
+function stableJsonValue(value) {
+  if (Array.isArray(value)) return value.map(stableJsonValue);
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.keys(value)
+        .sort()
+        .map((key) => [key, stableJsonValue(value[key])]),
+    );
+  }
+  return value;
+}
+
+function agendaMigrationSource(entry) {
+  const source = { ...entry };
+  delete source.executionLease;
+  delete source.schedulerExecution;
+  delete source.schedulerMigration;
+  delete source.firedAt;
+  delete source.lastRunAt;
+  delete source.runs;
+  delete source.nextAt;
+  delete source.lastCheckAt;
+  delete source.checks;
+  delete source.lastMtimeMs;
+  delete source.lastEventId;
+  delete source.lastAuthority;
+  delete source.status;
+  return source;
+}
+
+export function agendaMigrationSourceDigest(entry) {
+  return createHash("sha256")
+    .update("chainlesschain.agenda.scheduler-migration-source.v1\0", "utf8")
+    .update(
+      JSON.stringify(stableJsonValue(agendaMigrationSource(entry))),
+      "utf8",
+    )
+    .digest("hex");
+}
+
+export function agendaMigrationSourceSnapshot(entry) {
+  return stableJsonValue(agendaMigrationSource(entry));
+}
+
+function schedulerMigrationFence(entry, now) {
+  const migration = entry.schedulerMigration;
+  if (
+    migration?.schemaVersion !== AGENDA_SCHEDULER_MIGRATION_SCHEMA_VERSION ||
+    typeof migration.retirementToken !== "string" ||
+    migration.retirementToken.length === 0
+  ) {
+    return false;
+  }
+  return (
+    hasLiveExecutionLease(entry, now) &&
+    entry.executionLease?.owner ===
+      `scheduler-migration:${migration.retirementToken}`
+  );
+}
+
+function applySchedulerMigrationFence(entry, now) {
+  const token = entry.schedulerMigration?.retirementToken;
+  if (typeof token !== "string" || token.length === 0) return false;
+  entry.executionLease = {
+    owner: `scheduler-migration:${token}`,
+    claimedAt: now,
+    expiresAt: Number.MAX_SAFE_INTEGER,
+  };
+  return true;
+}
+
+function normalizeMigrationField(value, field) {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw agendaStoreError(
+      "AGENDA_SCHEDULER_MIGRATION_INVALID",
+      `Agenda scheduler migration ${field} is required`,
+    );
+  }
+  return value.trim();
+}
+
 function agendaStoreError(code, message, { retryable = false, retryAt } = {}) {
   const error = new Error(message);
   error.code = code;
@@ -820,15 +902,246 @@ export class AgentScheduleStore {
     return this.list().find((entry) => entry.id === id) || null;
   }
 
+  prepareSchedulerMigration(
+    id,
+    { migrationId, sourceDigest, targetJobId } = {},
+  ) {
+    const migration = normalizeMigrationField(migrationId, "migrationId");
+    const expectedDigest = normalizeMigrationField(
+      sourceDigest,
+      "sourceDigest",
+    );
+    const target = normalizeMigrationField(targetJobId, "targetJobId");
+    this._ensureDir();
+    return withFileLock(
+      path.join(this.dir, ".agenda-claims"),
+      () => {
+        for (const kind of SCHEDULE_KINDS) {
+          const entries = this._readAll(kind);
+          const entry = entries.find((candidate) => candidate.id === id);
+          if (!entry) continue;
+          if (!isSchedulableStatus(entry)) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_ENTRY_TERMINAL",
+              `Agenda entry is not schedulable: ${id}`,
+            );
+          }
+          const actualDigest = agendaMigrationSourceDigest(entry);
+          if (actualDigest !== expectedDigest) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_MIGRATION_SOURCE_CHANGED",
+              `Agenda entry changed before migration: ${id}`,
+            );
+          }
+          const prior = entry.schedulerMigration;
+          if (prior) {
+            if (
+              prior.schemaVersion !==
+                AGENDA_SCHEDULER_MIGRATION_SCHEMA_VERSION ||
+              prior.sourceDigest !== expectedDigest ||
+              prior.targetJobId !== target ||
+              prior.migrationId !== migration ||
+              prior.state !== "prepared"
+            ) {
+              throw agendaStoreError(
+                "AGENDA_SCHEDULER_MIGRATION_CONFLICT",
+                `Agenda entry already belongs to another scheduler migration: ${id}`,
+              );
+            }
+            return entry;
+          }
+          entry.schedulerMigration = {
+            schemaVersion: AGENDA_SCHEDULER_MIGRATION_SCHEMA_VERSION,
+            state: "prepared",
+            migrationId: migration,
+            sourceDigest: expectedDigest,
+            targetJobId: target,
+            preparedAt: this._now(),
+          };
+          this._writeAll(kind, entries);
+          return entry;
+        }
+        throw agendaStoreError(
+          "AGENDA_SCHEDULER_ENTRY_NOT_FOUND",
+          `Agenda entry does not exist: ${id}`,
+        );
+      },
+      { failIfUnavailable: true },
+    );
+  }
+
+  retireForSchedulerMigration(
+    id,
+    {
+      migrationId,
+      sourceDigest,
+      targetJobId,
+      retirementToken,
+      atMs = null,
+    } = {},
+  ) {
+    const migration = normalizeMigrationField(migrationId, "migrationId");
+    const expectedDigest = normalizeMigrationField(
+      sourceDigest,
+      "sourceDigest",
+    );
+    const target = normalizeMigrationField(targetJobId, "targetJobId");
+    const token = normalizeMigrationField(retirementToken, "retirementToken");
+    const now = atMs != null ? Number(atMs) : this._now();
+    this._ensureDir();
+    return withFileLock(
+      path.join(this.dir, ".agenda-claims"),
+      () => {
+        for (const kind of SCHEDULE_KINDS) {
+          const entries = this._readAll(kind);
+          const entry = entries.find((candidate) => candidate.id === id);
+          if (!entry) continue;
+          const prior = entry.schedulerMigration;
+          if (
+            prior?.schemaVersion !==
+              AGENDA_SCHEDULER_MIGRATION_SCHEMA_VERSION ||
+            prior.sourceDigest !== expectedDigest ||
+            prior.targetJobId !== target ||
+            prior.migrationId !== migration ||
+            !["prepared", "retired"].includes(prior.state)
+          ) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_MIGRATION_CONFLICT",
+              `Agenda migration preparation changed before retirement: ${id}`,
+            );
+          }
+          if (prior.state === "retired") {
+            if (
+              prior.retirementToken !== token ||
+              !schedulerMigrationFence(entry, now)
+            ) {
+              throw agendaStoreError(
+                "AGENDA_SCHEDULER_MIGRATION_CONFLICT",
+                `Agenda migration retirement fence changed: ${id}`,
+              );
+            }
+            return entry;
+          }
+          if (agendaMigrationSourceDigest(entry) !== expectedDigest) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_MIGRATION_SOURCE_CHANGED",
+              `Agenda entry changed before retirement: ${id}`,
+            );
+          }
+          if (hasLiveExecutionLease(entry, now)) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_LEGACY_CLAIM_ACTIVE",
+              `Agenda entry is already claimed by a legacy runner: ${id}`,
+              {
+                retryable: true,
+                retryAt: Number(entry.executionLease.expiresAt),
+              },
+            );
+          }
+          entry.schedulerMigration = {
+            ...prior,
+            state: "retired",
+            retirementToken: token,
+            retiredAt: now,
+          };
+          applySchedulerMigrationFence(entry, now);
+          this._writeAll(kind, entries);
+          return entry;
+        }
+        throw agendaStoreError(
+          "AGENDA_SCHEDULER_ENTRY_NOT_FOUND",
+          `Agenda entry does not exist: ${id}`,
+        );
+      },
+      { failIfUnavailable: true },
+    );
+  }
+
+  restoreFromSchedulerMigration(
+    id,
+    { migrationId, sourceDigest, targetJobId, retirementToken } = {},
+  ) {
+    const migrationIdValue = normalizeMigrationField(
+      migrationId,
+      "migrationId",
+    );
+    const expectedDigest = normalizeMigrationField(
+      sourceDigest,
+      "sourceDigest",
+    );
+    const target = normalizeMigrationField(targetJobId, "targetJobId");
+    const token =
+      retirementToken === null || retirementToken === undefined
+        ? null
+        : normalizeMigrationField(retirementToken, "retirementToken");
+    this._ensureDir();
+    return withFileLock(
+      path.join(this.dir, ".agenda-claims"),
+      () => {
+        for (const kind of SCHEDULE_KINDS) {
+          const entries = this._readAll(kind);
+          const entry = entries.find((candidate) => candidate.id === id);
+          if (!entry) continue;
+          const migration = entry.schedulerMigration;
+          if (!migration) {
+            if (
+              agendaMigrationSourceDigest(entry) !== expectedDigest ||
+              String(entry.executionLease?.owner || "").startsWith(
+                "scheduler-migration:",
+              )
+            ) {
+              throw agendaStoreError(
+                "AGENDA_SCHEDULER_MIGRATION_CONFLICT",
+                `Agenda source changed after rollback restoration: ${id}`,
+              );
+            }
+            return entry;
+          }
+          if (
+            migration?.schemaVersion !==
+              AGENDA_SCHEDULER_MIGRATION_SCHEMA_VERSION ||
+            migration.sourceDigest !== expectedDigest ||
+            agendaMigrationSourceDigest(entry) !== expectedDigest ||
+            migration.targetJobId !== target ||
+            migration.migrationId !== migrationIdValue ||
+            !["prepared", "retired"].includes(migration.state) ||
+            (migration.state === "retired" &&
+              (migration.retirementToken !== token ||
+                entry.executionLease?.owner !== `scheduler-migration:${token}`))
+          ) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_MIGRATION_CONFLICT",
+              `Agenda migration fence changed before rollback: ${id}`,
+            );
+          }
+          delete entry.schedulerMigration;
+          if (migration.state === "retired") delete entry.executionLease;
+          this._writeAll(kind, entries);
+          return entry;
+        }
+        throw agendaStoreError(
+          "AGENDA_SCHEDULER_ENTRY_NOT_FOUND",
+          `Agenda entry does not exist: ${id}`,
+        );
+      },
+      { failIfUnavailable: true },
+    );
+  }
+
   /** Entries whose time has come (pending wakeup / active cron|monitor). */
-  due(kind = null, atMs = null) {
+  due(kind = null, atMs = null, { schedulerMigration = false } = {}) {
     const now = atMs != null ? atMs : this._now();
     return this.list(kind).filter((entry) => {
       // An entry past its expiry never fires — even if it is not yet retired
       // (defense-in-depth: retireExpired need not have run first).
       if (isEntryExpired(entry, now)) return false;
       if (!isSchedulableStatus(entry)) return false;
-      if (hasLiveExecutionLease(entry, now)) return false;
+      if (
+        hasLiveExecutionLease(entry, now) &&
+        !(schedulerMigration && schedulerMigrationFence(entry, now))
+      ) {
+        return false;
+      }
       // The effective fire time applies the entry's OWN deterministic jitter
       // (`entry.jitterMs`, default 0 → base fire time, byte-identical to the
       // prior `dueAt`/`nextAt` comparison). A jittered entry fires slightly
@@ -946,7 +1259,8 @@ export class AgentScheduleStore {
           ) {
             return entry;
           }
-          if (hasLiveExecutionLease(entry, now)) {
+          const migrationFence = schedulerMigrationFence(entry, now);
+          if (hasLiveExecutionLease(entry, now) && !migrationFence) {
             throw agendaStoreError(
               "AGENDA_SCHEDULER_LEGACY_CLAIM_ACTIVE",
               `Agenda entry is already claimed by a legacy runner: ${id}`,
@@ -1051,6 +1365,102 @@ export class AgentScheduleStore {
             ...(outcome === "succeeded" ? { result: storedResult } : { error }),
           };
           delete entry.executionLease;
+          if (isSchedulableStatus(entry))
+            applySchedulerMigrationFence(entry, now);
+          this._writeAll(kind, entries);
+          return entry;
+        }
+        throw agendaStoreError(
+          "AGENDA_SCHEDULER_ENTRY_NOT_FOUND",
+          `Agenda entry does not exist: ${id}`,
+        );
+      },
+      { failIfUnavailable: true },
+    );
+  }
+
+  /**
+   * Resolve start-only scheduler evidence after an operator has verified the
+   * external outcome. This is intentionally separate from normal completion:
+   * it preserves the adjudication identity and can release the non-expiring
+   * legacy fence without pretending that a machine observed a terminal result.
+   */
+  adjudicateSchedulerExecution(
+    id,
+    {
+      occurrenceId,
+      snapshotDigest,
+      attempt,
+      decision,
+      requestId,
+      atMs = null,
+    } = {},
+  ) {
+    if (!["confirmed_applied", "confirmed_not_applied"].includes(decision)) {
+      throw agendaStoreError(
+        "AGENDA_SCHEDULER_ADJUDICATION_INVALID",
+        "Agenda scheduler adjudication decision is invalid",
+      );
+    }
+    const now = atMs != null ? Number(atMs) : this._now();
+    this._ensureDir();
+    return withFileLock(
+      path.join(this.dir, ".agenda-claims"),
+      () => {
+        for (const kind of SCHEDULE_KINDS) {
+          const entries = this._readAll(kind);
+          const entry = entries.find((candidate) => candidate.id === id);
+          if (!entry) continue;
+          const evidence = entry.schedulerExecution;
+          if (
+            evidence?.occurrenceId === occurrenceId &&
+            evidence.snapshotDigest === snapshotDigest &&
+            evidence.adjudication?.requestId === requestId &&
+            evidence.adjudication.decision === decision &&
+            evidence.status ===
+              (decision === "confirmed_applied"
+                ? "adjudicated_applied"
+                : "adjudicated_not_applied")
+          ) {
+            return entry;
+          }
+          if (
+            !evidence ||
+            evidence.occurrenceId !== occurrenceId ||
+            evidence.snapshotDigest !== snapshotDigest ||
+            !Number.isSafeInteger(evidence.attempt) ||
+            evidence.attempt < 1 ||
+            evidence.attempt > attempt ||
+            evidence.status !== "running"
+          ) {
+            throw agendaStoreError(
+              "AGENDA_SCHEDULER_ADJUDICATION_EVIDENCE_MISMATCH",
+              `Agenda scheduler adjudication evidence does not match: ${id}`,
+            );
+          }
+          if (decision === "confirmed_applied") {
+            if (entry.kind === "monitor") {
+              // A generic operator decision proves the prior check ran, but
+              // not whether its observation matched. Pause the monitor rather
+              // than inventing a domain result or rearming a duplicate check.
+              entry.status = "adjudicated";
+              entry.lastCheckAt = now;
+            } else {
+              applyScheduledAgentSuccess(entry, now);
+            }
+          }
+          entry.schedulerExecution = {
+            ...evidence,
+            status:
+              decision === "confirmed_applied"
+                ? "adjudicated_applied"
+                : "adjudicated_not_applied",
+            endedAt: now,
+            adjudication: { requestId, decision },
+          };
+          delete entry.executionLease;
+          if (isSchedulableStatus(entry))
+            applySchedulerMigrationFence(entry, now);
           this._writeAll(kind, entries);
           return entry;
         }

package/src/lib/agent-team/team-adjudication.js

@@ -70,6 +70,10 @@ const EVENT_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/;
 const CASE_ID_PATTERN = /^tadj_[a-f0-9]{64}$/;
 const CLAIM_ID_PATTERN =
   /^tadj_claim_[a-zA-Z0-9](?:[a-zA-Z0-9._-]{0,126}[a-zA-Z0-9])?$/;
+const WINDOWS_RENAME_TRANSIENT_CODES = new Set(["EACCES", "EBUSY", "EPERM"]);
+const WINDOWS_RENAME_RETRY_DELAYS_MS = Object.freeze([
+  5, 10, 20, 40, 80, 100, 100, 100,
+]);
 const DECISIONS = new Set(Object.values(TEAM_ADJUDICATION_DECISIONS));
 const HEADER_KEYS = new Set([
   "schema",
@@ -130,6 +134,19 @@ function adjudicationError(code, message, details = {}, cause = null) {
   return new TeamAdjudicationError(code, message, details, cause);
 }
 
+function sleepSync(milliseconds) {
+  const duration = Math.max(0, Number(milliseconds) || 0);
+  if (duration === 0) return;
+  try {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, duration);
+  } catch {
+    const deadline = Date.now() + duration;
+    while (Date.now() < deadline) {
+      // SharedArrayBuffer is unavailable; retain a bounded synchronous wait.
+    }
+  }
+}
+
 function isAdjudicationError(error) {
   return Object.values(TEAM_ADJUDICATION_ERROR_CODES).includes(error?.code);
 }
@@ -892,11 +909,13 @@ export class TeamAdjudicationStore {
     _lock = withFileLock,
     _randomUUID = randomUUID,
     _platform = process.platform,
+    _sleep = sleepSync,
   } = {}) {
     this._fs = _fs;
     this._lock = _lock;
     this._randomUUID = _randomUUID;
     this._platform = _platform;
+    this._sleep = typeof _sleep === "function" ? _sleep : sleepSync;
     this._now = typeof now === "function" ? now : () => Number(now);
     this.lockTimeoutMs = Math.max(0, Number(lockTimeoutMs) || 0);
     this.lockStaleMs = Math.max(1, Number(lockStaleMs) || 30_000);
@@ -1161,7 +1180,27 @@ export class TeamAdjudicationStore {
             descriptor = null;
 
             this._assertAuthorityPaths();
-            this._fs.renameSync(temporaryPath, canonicalPath);
+            let renameAttempt = 0;
+            for (;;) {
+              try {
+                this._fs.renameSync(temporaryPath, canonicalPath);
+                break;
+              } catch (cause) {
+                const delay = WINDOWS_RENAME_RETRY_DELAYS_MS[renameAttempt];
+                if (
+                  this._platform !== "win32" ||
+                  !WINDOWS_RENAME_TRANSIENT_CODES.has(cause?.code) ||
+                  delay === undefined
+                ) {
+                  throw cause;
+                }
+                renameAttempt += 1;
+                this._sleep(delay);
+                // Keep each retry bound to the same canonical authority after
+                // yielding to a transient Windows file-sharing holder.
+                this._assertAuthorityPaths();
+              }
+            }
             renamed = true;
             this._fs.chmodSync(canonicalPath, 0o600);
             const persisted = this._secureRead();