chainlesschain 0.163.6 → 0.163.7

package/src/lib/scheduler-kernel/routine-adapter.js

@@ -1,6 +1,11 @@
 import { createHash, randomUUID } from "node:crypto";
 import { nextCronTime } from "../agent-schedule-store.js";
-import { executeRoutine } from "../routine-store.js";
+import {
+  executeRoutine,
+  routineEffectiveSchedulerView,
+  routineMigrationSourceDigest,
+  routineMigrationSourceSnapshot,
+} from "../routine-store.js";
 import {
   SchedulerKernelError,
   canonicalJson,
@@ -8,10 +13,12 @@ import {
   normalizeIdentifier,
 } from "./contract.js";
 import { SchedulerRuntime } from "./runtime.js";
+import { schedulerMigrationSourceDigest } from "./store.js";
 import {
   bindSchedulerAuthorityPolicy,
   createSchedulerAuthorityResolver,
 } from "./authority-resolver.js";
+import { canonicalSchedulerSourcePath } from "./source-locator-path.js";
 
 export const ROUTINE_SCHEDULER_KIND = "routine";
 export const ROUTINE_SCHEDULER_CAPABILITY = "agent.execute";
@@ -389,6 +396,243 @@ export function syncRoutineSchedulerJob(schedulerStore, routine, options = {}) {
   }
 }
 
+export function migrateRoutineSchedule({
+  routineStore,
+  schedulerStore,
+  routine,
+}) {
+  const effective = routineEffectiveSchedulerView(routine);
+  const snapshot = routineDefinitionSnapshot(effective);
+  if (!["cron", "once", "github"].includes(snapshot.trigger.kind)) {
+    throw routineError(
+      "SCHEDULER_ROUTINE_MIGRATION_TRIGGER_UNSUPPORTED",
+      `Routine trigger has no legacy driver to migrate: ${snapshot.id}`,
+    );
+  }
+  const directory = canonicalSchedulerSourcePath(routineStore.dir);
+  const sourceScope = { store: "routines", directory };
+  const sourceLocator = {
+    schemaVersion: 1,
+    type: "routine-store",
+    directory,
+  };
+  let prepared = schedulerStore.getActiveDomainMigrationBySource({
+    domain: "routine",
+    sourceScope,
+    sourceId: snapshot.id,
+  });
+  if (!prepared && routineStore.dir !== directory) {
+    const ambiguousLegacy = schedulerStore.getActiveDomainMigrationBySource({
+      domain: "routine",
+      sourceScope: { store: "routines", directory: routineStore.dir },
+      sourceId: snapshot.id,
+    });
+    if (ambiguousLegacy) {
+      throw routineError(
+        "SCHEDULER_ROUTINE_MIGRATION_SOURCE_UNBOUND",
+        `Legacy Routine migration uses a non-canonical path scope and cannot be bound safely: ${snapshot.id}`,
+      );
+    }
+  }
+  const channel =
+    snapshot.trigger.kind === "github"
+      ? ROUTINE_SCHEDULER_CHANNELS.GITHUB
+      : ROUTINE_SCHEDULER_CHANNELS.SCHEDULED;
+  // GitHub occurrences use per-batch jobs. A disabled guard job owns the
+  // migration namespace while actual batches remain independently immutable.
+  const desired =
+    channel === ROUTINE_SCHEDULER_CHANNELS.GITHUB
+      ? {
+          ...buildRoutineSchedulerJob(effective, {
+            channel: ROUTINE_SCHEDULER_CHANNELS.SCHEDULED,
+          }),
+          id: routineSchedulerJobId(snapshot.id, channel),
+          trigger: {
+            source: "routine-store",
+            channel,
+            routineKind: "github",
+          },
+          enabled: true,
+        }
+      : buildRoutineSchedulerJob(effective, { channel });
+  desired.authority = bindSchedulerAuthorityPolicy(
+    schedulerStore,
+    desired.authority,
+  );
+  const sourceSnapshot = routineMigrationSourceSnapshot(routine);
+  const sourceDigest = routineMigrationSourceDigest(routine);
+  if (prepared) {
+    const migrationEntries = prepared.entries.filter(
+      (candidate) =>
+        candidate.domain === "routine" && candidate.sourceId === snapshot.id,
+    );
+    if (prepared.entries.length !== 1 || migrationEntries.length !== 1) {
+      throw routineError(
+        "SCHEDULER_ROUTINE_MIGRATION_DOMAIN_MISMATCH",
+        "Routine belongs to an invalid mixed-domain migration",
+      );
+    }
+    const [migrationEntry] = migrationEntries;
+    schedulerStore.bindDomainMigrationSourceLocator({
+      migrationId: prepared.id,
+      entryId: migrationEntry.entryId,
+      sourceLocator,
+      expectedSourceDigest: migrationEntry.sourceDigest,
+      expectedTargetJobId: migrationEntry.targetJobId,
+    });
+    prepared = schedulerStore.getDomainMigration(prepared.id);
+  } else {
+    prepared = schedulerStore.prepareDomainMigration({
+      entries: [
+        {
+          domain: "routine",
+          sourceId: snapshot.id,
+          sourceScope,
+          sourceLocator,
+          source: sourceSnapshot,
+          targetJob: desired,
+          rollbackStrategy: "disable",
+        },
+      ],
+    });
+  }
+  if (
+    prepared.entries.length !== 1 ||
+    prepared.entries[0].domain !== "routine"
+  ) {
+    throw routineError(
+      "SCHEDULER_ROUTINE_MIGRATION_DOMAIN_MISMATCH",
+      "Routine belongs to an invalid mixed-domain migration",
+    );
+  }
+  if (
+    prepared.entries[0].sourceDigest !==
+      schedulerMigrationSourceDigest(sourceSnapshot) ||
+    prepared.entries[0].targetJobId !== desired.id
+  ) {
+    throw routineError(
+      "SCHEDULER_ROUTINE_MIGRATION_SOURCE_CHANGED",
+      `Routine changed after migration started: ${snapshot.id}`,
+    );
+  }
+  let migration = prepared;
+  if (["rolling_back", "rolled_back"].includes(migration.state)) {
+    throw routineError(
+      "SCHEDULER_ROUTINE_MIGRATION_STATE_CONFLICT",
+      `Routine migration cannot resume from state ${migration.state}`,
+    );
+  }
+  if (migration.state === "retired") {
+    routineStore.retireForSchedulerMigration(snapshot.id, {
+      migrationId: migration.id,
+      sourceDigest,
+      targetJobId: desired.id,
+      retirementToken: migration.entries[0].retirementToken,
+    });
+    return migration;
+  }
+  if (migration.state === "prepared") {
+    routineStore.prepareSchedulerMigration(snapshot.id, {
+      migrationId: migration.id,
+      sourceDigest,
+      targetJobId: desired.id,
+    });
+    migration = schedulerStore.applyDomainMigration(migration.id);
+  }
+  if (migration.state === "applied") {
+    migration = schedulerStore.verifyDomainMigration(migration.id, {
+      sources: [
+        { entryId: migration.entries[0].entryId, source: sourceSnapshot },
+      ],
+    });
+  }
+  const retiring =
+    migration.state === "retiring"
+      ? migration
+      : schedulerStore.beginDomainMigrationRetirement(migration.id);
+  const entry = retiring.entries[0];
+  const retired = routineStore.retireForSchedulerMigration(snapshot.id, {
+    migrationId: migration.id,
+    sourceDigest,
+    targetJobId: desired.id,
+    retirementToken: entry.retirementToken,
+  });
+  schedulerStore.confirmDomainMigrationEntryRetired({
+    migrationId: migration.id,
+    entryId: entry.entryId,
+    retirementToken: entry.retirementToken,
+    source: retired,
+  });
+  return schedulerStore.getDomainMigration(migration.id);
+}
+
+export function rollbackRoutineMigration({
+  routineStore,
+  schedulerStore,
+  migrationId,
+}) {
+  const directory = canonicalSchedulerSourcePath(routineStore.dir);
+  const assertRollbackBinding = (migration) => {
+    if (
+      migration?.id !== migrationId ||
+      !Array.isArray(migration.entries) ||
+      migration.entries.length === 0 ||
+      migration.entries.some((entry) => entry.domain !== "routine")
+    ) {
+      throw routineError(
+        "SCHEDULER_ROUTINE_MIGRATION_DOMAIN_MISMATCH",
+        "Routine rollback cannot operate on a mixed-domain migration",
+      );
+    }
+    for (const entry of migration.entries) {
+      const locator = entry.sourceLocator;
+      if (
+        locator?.schemaVersion !== 1 ||
+        locator.type !== "routine-store" ||
+        locator.directory !== directory
+      ) {
+        throw routineError(
+          "SCHEDULER_ROUTINE_MIGRATION_SOURCE_UNBOUND",
+          `Routine rollback source locator does not match its store: ${entry.sourceId}`,
+        );
+      }
+    }
+    return migration;
+  };
+  const current = assertRollbackBinding(
+    schedulerStore.getDomainMigration(migrationId),
+  );
+  if (current.state === "rolled_back") {
+    return { ...current, deduplicated: true };
+  }
+  const migration = assertRollbackBinding(
+    schedulerStore.beginDomainMigrationRollback(migrationId),
+  );
+  if (migration.state === "rolled_back") return migration;
+  const entries = migration.entries;
+  for (const entry of entries) {
+    if (entry.state === "rolled_back") continue;
+    schedulerStore.restoreDomainMigrationEntrySource({
+      migrationId,
+      entryId: entry.entryId,
+      retirementToken: entry.retirementToken,
+      restoreSource: () => {
+        const current = routineStore.get(entry.sourceId);
+        const restored = current.schedulerMigration
+          ? routineStore.restoreFromSchedulerMigration(entry.sourceId, {
+              migrationId,
+              sourceDigest: routineMigrationSourceDigest(current),
+              targetJobId: entry.targetJobId,
+              retirementToken: entry.retirementToken,
+            })
+          : current;
+        return routineMigrationSourceSnapshot(restored);
+      },
+    });
+  }
+  return schedulerStore.getDomainMigration(migrationId);
+}
+
 export function scheduledRoutineFireAt(routine) {
   const snapshot = routineSnapshot(routine);
   if (snapshot.trigger.kind === "once") {
@@ -528,7 +772,47 @@ export function createRoutineSchedulerAdapter({ routineStore, runAgent } = {}) {
   }
   return {
     kind: ROUTINE_SCHEDULER_KIND,
-    async execute({ occurrence }) {
+    async adjudicate({ occurrence, adjudication }) {
+      const payload = occurrence.payload;
+      const expected = payload?.routine;
+      const runId = routineSchedulerRunId(occurrence.id);
+      if (
+        !expected ||
+        typeof routineStore.recordRunAdjudication !== "function"
+      ) {
+        throw routineError(
+          "SCHEDULER_ROUTINE_ADJUDICATION_UNSUPPORTED",
+          `Routine adjudication cannot be applied: ${occurrence.id}`,
+        );
+      }
+      routineStore.recordRunAdjudication(runId, {
+        decision: adjudication.decision,
+        requestId: adjudication.requestId,
+        occurrenceId: occurrence.id,
+        snapshotDigest: payload.snapshotDigest,
+      });
+      if (adjudication.decision === "confirmed_applied") {
+        const current = routineStore.get(expected.id);
+        if (current) {
+          routineStore.update(expected.id, {
+            lastFiredAt: occurrence.scheduledFor,
+            ...(current.trigger.kind === "once" || current.schedulerMigration
+              ? { enabled: false }
+              : {}),
+          });
+        }
+        return {
+          settled: true,
+          result: {
+            runId,
+            status: "adjudicated-applied",
+            adjudicationRequestId: adjudication.requestId,
+          },
+        };
+      }
+      return { continue: true };
+    },
+    async execute({ occurrence, adjudication }) {
       const payload = occurrence.payload;
       const expected = payload?.routine;
       if (!expected) {
@@ -579,10 +863,11 @@ export function createRoutineSchedulerAdapter({ routineStore, runAgent } = {}) {
           `Routine run evidence does not match occurrence: ${occurrence.id}`,
         );
       }
+      const effectiveCurrent = routineEffectiveSchedulerView(current);
       const currentDigest =
         payload.snapshotType === "definition"
-          ? routineDefinitionDigest(current)
-          : routineSnapshotDigest(current);
+          ? routineDefinitionDigest(effectiveCurrent)
+          : routineSnapshotDigest(effectiveCurrent);
       if (!existingRun && currentDigest !== payload.snapshotDigest) {
         throw routineError(
           "SCHEDULER_ROUTINE_STALE_SNAPSHOT",
@@ -592,26 +877,32 @@ export function createRoutineSchedulerAdapter({ routineStore, runAgent } = {}) {
       if (
         payload.channel !== ROUTINE_SCHEDULER_CHANNELS.MANUAL &&
         !existingRun &&
-        current.enabled !== true
+        effectiveCurrent.enabled !== true
       ) {
         throw routineError(
           "SCHEDULER_ROUTINE_DISABLED",
           `Routine was disabled before scheduled execution: ${expected.id}`,
         );
       }
-      const execution = await executeRoutine(routineStore, current, runAgent, {
-        trigger:
-          payload.channel === ROUTINE_SCHEDULER_CHANNELS.MANUAL
-            ? "manual"
-            : payload.channel === ROUTINE_SCHEDULER_CHANNELS.GITHUB
-              ? `github:${payload.github.events
-                  .map((event) => event.type)
-                  .join(",")}`
-              : current.trigger.kind,
-        runId: schedulerRunId,
-        schedulerOccurrenceId: occurrence.id,
-        schedulerSnapshotDigest: payload.snapshotDigest,
-      });
+      const execution = await executeRoutine(
+        routineStore,
+        effectiveCurrent,
+        runAgent,
+        {
+          trigger:
+            payload.channel === ROUTINE_SCHEDULER_CHANNELS.MANUAL
+              ? "manual"
+              : payload.channel === ROUTINE_SCHEDULER_CHANNELS.GITHUB
+                ? `github:${payload.github.events
+                    .map((event) => event.type)
+                    .join(",")}`
+                : current.trigger.kind,
+          runId: schedulerRunId,
+          schedulerOccurrenceId: occurrence.id,
+          schedulerSnapshotDigest: payload.snapshotDigest,
+          ...(adjudication ? { adjudication } : {}),
+        },
+      );
       if (execution.status !== "ok") {
         throw routineError(
           "SCHEDULER_ROUTINE_EXECUTION_FAILED",
@@ -647,6 +938,7 @@ export class RoutineSchedulerBridge {
     leaseMs,
     renewIntervalMs,
     authorityResolver,
+    migrateLegacy = true,
   } = {}) {
     if (typeof now !== "function") {
       throw routineError(
@@ -657,6 +949,7 @@ export class RoutineSchedulerBridge {
     this.routineStore = routineStore;
     this.schedulerStore = schedulerStore;
     this.now = now;
+    this.migrateLegacy = migrateLegacy !== false;
     this.runtime = new SchedulerRuntime({
       store: schedulerStore,
       adapters: [createRoutineSchedulerAdapter({ routineStore, runAgent })],
@@ -688,7 +981,32 @@ export class RoutineSchedulerBridge {
         "Routine GitHub scheduler requires a fetchEvents function",
       );
     }
-    const current = this.routineStore.get(routine.id);
+    const activeMigration =
+      this.migrateLegacy &&
+      this.schedulerStore.getActiveDomainMigrationBySource({
+        domain: "routine",
+        sourceScope: {
+          store: "routines",
+          directory: canonicalSchedulerSourcePath(this.routineStore.dir),
+        },
+        sourceId: routine.id,
+      });
+    if (
+      this.migrateLegacy &&
+      ((routine.schedulerMigration && activeMigration?.state !== "retired") ||
+        activeMigration?.state !== undefined ||
+        (routine.enabled === true &&
+          !routine.schedulerMigration &&
+          routine.trigger?.kind === "github"))
+    ) {
+      migrateRoutineSchedule({
+        routineStore: this.routineStore,
+        schedulerStore: this.schedulerStore,
+        routine,
+      });
+    }
+    const stored = this.routineStore.get(routine.id);
+    const current = routineEffectiveSchedulerView(stored);
     if (
       !current ||
       current.enabled !== true ||
@@ -762,6 +1080,31 @@ export class RoutineSchedulerBridge {
   async runDue({ signal } = {}) {
     const results = [];
     const observedOccurrences = new Set();
+    if (!signal?.aborted && this.migrateLegacy) {
+      const sourceScope = {
+        store: "routines",
+        directory: canonicalSchedulerSourcePath(this.routineStore.dir),
+      };
+      const candidates = this.routineStore.list().filter((routine) => {
+        const activeMigration =
+          this.schedulerStore.getActiveDomainMigrationBySource({
+            domain: "routine",
+            sourceScope,
+            sourceId: routine.id,
+          });
+        if (routine.schedulerMigration) {
+          return activeMigration?.state !== "retired";
+        }
+        return activeMigration?.state !== undefined;
+      });
+      for (const routine of candidates) {
+        migrateRoutineSchedule({
+          routineStore: this.routineStore,
+          schedulerStore: this.schedulerStore,
+          routine,
+        });
+      }
+    }
     const recovered = await this.runtime.runUntilIdle({
       limit: 10_000,
       signal,
@@ -783,7 +1126,21 @@ export class RoutineSchedulerBridge {
     if (signal?.aborted) return results;
 
     const now = normalizeEpochMs(Number(this.now()), "now");
-    const due = this.routineStore.due(now);
+    const due = [
+      ...this.routineStore.due(now),
+      ...this.routineStore.schedulerMigrated().filter((routine) => {
+        if (routine.trigger.kind === "cron") {
+          const from = routine.lastFiredAt || routine.createdAt || 0;
+          const next = nextCronTime(routine.trigger.cron, from);
+          return next !== null && next <= now;
+        }
+        return (
+          routine.trigger.kind === "once" &&
+          !routine.lastFiredAt &&
+          routine.trigger.at <= now
+        );
+      }),
+    ];
     for (const routine of due) {
       const occurrence = enqueueScheduledRoutine(this.schedulerStore, routine);
       if (observedOccurrences.has(occurrence.id)) continue;
@@ -797,6 +1154,23 @@ export class RoutineSchedulerBridge {
         result,
       });
     }
+    if (this.migrateLegacy) {
+      const candidates = this.routineStore
+        .list()
+        .filter(
+          (routine) =>
+            routine.enabled === true &&
+            !routine.schedulerMigration &&
+            ["cron", "once"].includes(routine.trigger?.kind),
+        );
+      for (const routine of candidates) {
+        migrateRoutineSchedule({
+          routineStore: this.routineStore,
+          schedulerStore: this.schedulerStore,
+          routine,
+        });
+      }
+    }
     return results;
   }
 }

package/src/lib/scheduler-kernel/runtime.js

@@ -409,11 +409,67 @@ export class SchedulerRuntime {
       occurrence,
       authority: occurrence.authority,
       decision,
+      adjudication:
+        typeof this.store.getOccurrenceAdjudication === "function"
+          ? this.store.getOccurrenceAdjudication(occurrence.id)
+          : null,
       signal: linked.controller.signal,
       renewLease: renew,
     };
     try {
-      const result = await adapter.execute(context);
+      let result;
+      if (context.adjudication?.status === "pending") {
+        const allowedAttempts =
+          context.adjudication.decision === "confirmed_applied"
+            ? [
+                context.adjudication.expectedAttempt + 1,
+                context.adjudication.expectedAttempt + 2,
+              ]
+            : [context.adjudication.expectedAttempt + 1];
+        if (
+          !allowedAttempts.includes(occurrence.attempt) ||
+          context.adjudication.expectedFence >= occurrence.fence
+        ) {
+          throw runtimeError(
+            "SCHEDULER_RUNTIME_ADJUDICATION_BINDING_MISMATCH",
+            `Scheduler adjudication is not bound to this claim: ${occurrence.id}`,
+          );
+        }
+        if (typeof adapter.adjudicate !== "function") {
+          throw runtimeError(
+            "SCHEDULER_RUNTIME_ADJUDICATION_UNSUPPORTED",
+            `Scheduler adapter cannot apply an outcome-unknown adjudication: ${job.kind}`,
+          );
+        }
+        const resolution = await adapter.adjudicate(context);
+        if (
+          context.adjudication.decision === "confirmed_applied" &&
+          resolution?.settled !== true
+        ) {
+          throw runtimeError(
+            "SCHEDULER_RUNTIME_ADJUDICATION_INVALID",
+            "confirmed_applied adjudication must settle without replay",
+          );
+        }
+        if (
+          context.adjudication.decision === "confirmed_not_applied" &&
+          resolution?.continue !== true
+        ) {
+          throw runtimeError(
+            "SCHEDULER_RUNTIME_ADJUDICATION_INVALID",
+            "confirmed_not_applied adjudication must explicitly authorize one execution",
+          );
+        }
+        result =
+          resolution?.settled === true
+            ? (resolution.result ?? {
+                adjudicationRequestId: context.adjudication.requestId,
+                decision: context.adjudication.decision,
+              })
+            : await adapter.execute(context);
+      } else {
+        result = await adapter.execute(context);
+      }
       if (leaseError) throw leaseError;
       if (linked.controller.signal.aborted) {
         const aborted = runtimeError(
@@ -429,6 +485,9 @@ export class SchedulerRuntime {
         fence: occurrence.fence,
         outcome: "succeeded",
         result: result ?? null,
+        ...(context.adjudication?.status === "pending"
+          ? { adjudicationRequestId: context.adjudication.requestId }
+          : {}),
       });
       return settledResult(settled);
     } catch (error) {
@@ -453,6 +512,9 @@ export class SchedulerRuntime {
         outcome: "failed",
         error: safeError(settlementError),
         retryable: policy.retryable,
+        ...(context.adjudication?.status === "pending"
+          ? { adjudicationRequestId: context.adjudication.requestId }
+          : {}),
         ...(policy.retryAt === undefined ? {} : { retryAt: policy.retryAt }),
       });
       return settledResult(settled);