chainlesschain 0.163.6 → 0.163.7

package/src/lib/scheduler-kernel/automation-adapter.js

@@ -1,4 +1,4 @@
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { nextCronTime } from "../agent-schedule-store.js";
 import {
   automationExecutionAuthorityDigest,
@@ -11,10 +11,19 @@ import {
   EXECUTION_STATUS,
   FLOW_STATUS,
   TRIGGER_TYPE,
+  automationEffectiveSchedulerFlow,
+  automationMigrationSourceDigest,
+  automationMigrationSourceSnapshot,
   executeFlow,
+  adjudicateAutomationExecution,
   getExecution,
   getFlow,
+  getAutomationSchedulerMigration,
   listFlows,
+  listAutomationSchedulerFlows,
+  prepareAutomationSchedulerMigration,
+  restoreAutomationSchedulerMigration,
+  retireAutomationSchedulerMigration,
 } from "../automation-engine.js";
 import {
   SchedulerKernelError,
@@ -24,15 +33,43 @@ import {
   normalizeJson,
 } from "./contract.js";
 import { SchedulerRuntime } from "./runtime.js";
+import {
+  schedulerJobDefinitionDigest,
+  schedulerMigrationSourceDigest,
+} from "./store.js";
 import {
   bindSchedulerAuthorityPolicy,
   createSchedulerAuthorityResolver,
 } from "./authority-resolver.js";
+import { canonicalSchedulerSourcePath } from "./source-locator-path.js";
 
 export const AUTOMATION_SCHEDULER_KIND = "automation";
 export const AUTOMATION_SCHEDULER_CAPABILITY = "automation.execute";
 export const AUTOMATION_SCHEDULER_CHANNEL = "scheduled";
 export const AUTOMATION_SCHEDULER_MAX_CATCHUP_STEPS = 527_040;
+const AUTOMATION_MEMORY_DATABASE_IDENTITIES = new WeakMap();
+
+export function automationDatabaseIdentity(db, pathOptions = undefined) {
+  if (typeof db?.databasePath === "string" && db.databasePath.length > 0) {
+    return canonicalSchedulerSourcePath(db.databasePath, pathOptions);
+  }
+  const main = db
+    .prepare("PRAGMA database_list")
+    .all()
+    .find((entry) => entry.name === "main");
+  const file = typeof main?.file === "string" ? main.file : "";
+  if (!file) {
+    // In-memory stores exist only for one process and are never safe to reopen
+    // from an operator command. Keep their scope isolated to this handle.
+    let identity = AUTOMATION_MEMORY_DATABASE_IDENTITIES.get(db);
+    if (!identity) {
+      identity = `memory:${randomUUID()}`;
+      AUTOMATION_MEMORY_DATABASE_IDENTITIES.set(db, identity);
+    }
+    return identity;
+  }
+  return canonicalSchedulerSourcePath(file, pathOptions);
+}
 
 function automationError(
   code,
@@ -231,6 +268,311 @@ export function syncAutomationSchedulerJob(
   }
 }
 
+export function migrateAutomationFlow({
+  db,
+  schedulerStore,
+  flow,
+  executionAuthority,
+}) {
+  if (db?.__isSqlJsCompat === true) {
+    throw automationError(
+      "AUTOMATION_SCHEDULER_MIGRATION_DURABILITY_UNAVAILABLE",
+      "Automation scheduler migration requires native SQLite durability",
+    );
+  }
+  const effective = automationEffectiveSchedulerFlow(db, flow);
+  const snapshot = automationFlowSnapshot(effective);
+  const database = automationDatabaseIdentity(db);
+  const sourceScope = { store: "automation-engine", database };
+  const sourceLocator = {
+    schemaVersion: 1,
+    type: "automation-database",
+    database,
+  };
+  const desired = buildAutomationSchedulerJob(effective, executionAuthority);
+  desired.authority = bindSchedulerAuthorityPolicy(
+    schedulerStore,
+    desired.authority,
+  );
+  const sourceSnapshot = automationMigrationSourceSnapshot(effective);
+  const sourceDigest = automationMigrationSourceDigest(effective);
+  const targetJobDigest = schedulerJobDefinitionDigest(desired);
+  const marker = getAutomationSchedulerMigration(db, snapshot.id);
+  let prepared;
+  if (marker) {
+    if (
+      marker.schemaVersion !== 1 ||
+      !["prepared", "retired"].includes(marker.state) ||
+      marker.sourceDigest !== sourceDigest ||
+      marker.targetJobId !== desired.id ||
+      (marker.state === "prepared" && marker.retirementToken !== null)
+    ) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_CONFLICT",
+        `Automation migration marker is invalid: ${snapshot.id}`,
+      );
+    }
+    prepared = schedulerStore.getDomainMigration(marker.migrationId);
+    if (
+      prepared.entries.length !== 1 ||
+      prepared.entries[0].domain !== "automation" ||
+      prepared.entries[0].sourceId !== snapshot.id ||
+      prepared.entries[0].sourceDigest !==
+        schedulerMigrationSourceDigest(sourceSnapshot) ||
+      prepared.entries[0].targetJobId !== desired.id ||
+      prepared.entries[0].targetJobDigest !== targetJobDigest ||
+      (marker.state === "retired" &&
+        prepared.entries[0].retirementToken !== marker.retirementToken)
+    ) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_CONFLICT",
+        `Automation migration marker does not match its journal: ${snapshot.id}`,
+      );
+    }
+    schedulerStore.bindDomainMigrationSourceLocator({
+      migrationId: prepared.id,
+      entryId: prepared.entries[0].entryId,
+      sourceLocator,
+      expectedSourceDigest: prepared.entries[0].sourceDigest,
+      expectedTargetJobId: desired.id,
+    });
+    prepared = schedulerStore.getDomainMigration(prepared.id);
+  } else {
+    const unboundLegacy = schedulerStore.getActiveDomainMigrationBySource({
+      domain: "automation",
+      sourceScope: { store: "automation-engine", database: "current" },
+      sourceId: snapshot.id,
+    });
+    if (unboundLegacy) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_SOURCE_UNBOUND",
+        `Legacy Automation migration has no source marker and cannot be bound safely: ${snapshot.id}`,
+      );
+    }
+    prepared = schedulerStore.prepareDomainMigration({
+      entries: [
+        {
+          domain: "automation",
+          sourceId: snapshot.id,
+          sourceScope,
+          sourceLocator,
+          source: sourceSnapshot,
+          targetJob: desired,
+          rollbackStrategy: "disable",
+        },
+      ],
+    });
+  }
+  if (
+    prepared.entries.length !== 1 ||
+    prepared.entries[0].domain !== "automation"
+  ) {
+    throw automationError(
+      "AUTOMATION_SCHEDULER_MIGRATION_DOMAIN_MISMATCH",
+      "Automation flow belongs to an invalid mixed-domain migration",
+    );
+  }
+  if (
+    prepared.entries[0].sourceDigest !==
+      schedulerMigrationSourceDigest(sourceSnapshot) ||
+    prepared.entries[0].targetJobId !== desired.id ||
+    prepared.entries[0].targetJobDigest !== targetJobDigest
+  ) {
+    throw automationError(
+      "AUTOMATION_SCHEDULER_MIGRATION_SOURCE_CHANGED",
+      `Automation flow changed after migration started: ${snapshot.id}`,
+    );
+  }
+  let migration = prepared;
+  if (["rolling_back", "rolled_back"].includes(migration.state)) {
+    throw automationError(
+      "AUTOMATION_SCHEDULER_MIGRATION_STATE_CONFLICT",
+      `Automation migration cannot resume from state ${migration.state}`,
+    );
+  }
+  if (migration.state === "retired") {
+    retireAutomationSchedulerMigration(db, snapshot.id, {
+      migrationId: migration.id,
+      sourceDigest,
+      targetJobId: desired.id,
+      retirementToken: migration.entries[0].retirementToken,
+    });
+    return migration;
+  }
+  if (migration.state === "prepared") {
+    prepareAutomationSchedulerMigration(db, snapshot.id, {
+      migrationId: migration.id,
+      sourceDigest,
+      targetJobId: desired.id,
+    });
+    migration = schedulerStore.applyDomainMigration(migration.id);
+  }
+  if (migration.state === "applied") {
+    migration = schedulerStore.verifyDomainMigration(migration.id, {
+      sources: [
+        { entryId: migration.entries[0].entryId, source: sourceSnapshot },
+      ],
+    });
+  }
+  const retiring =
+    migration.state === "retiring"
+      ? migration
+      : schedulerStore.beginDomainMigrationRetirement(migration.id);
+  const entry = retiring.entries[0];
+  const retired = retireAutomationSchedulerMigration(db, snapshot.id, {
+    migrationId: migration.id,
+    sourceDigest,
+    targetJobId: desired.id,
+    retirementToken: entry.retirementToken,
+  });
+  schedulerStore.confirmDomainMigrationEntryRetired({
+    migrationId: migration.id,
+    entryId: entry.entryId,
+    retirementToken: entry.retirementToken,
+    source: retired,
+  });
+  return schedulerStore.getDomainMigration(migration.id);
+}
+
+function assertAutomationRollbackBinding({
+  db,
+  migration,
+  migrationId,
+  database,
+}) {
+  if (
+    migration?.id !== migrationId ||
+    !Array.isArray(migration.entries) ||
+    migration.entries.length === 0 ||
+    migration.entries.some((entry) => entry.domain !== "automation")
+  ) {
+    throw automationError(
+      "AUTOMATION_SCHEDULER_MIGRATION_DOMAIN_MISMATCH",
+      "Automation rollback cannot operate on a mixed-domain migration",
+    );
+  }
+  for (const entry of migration.entries) {
+    const locator = entry.sourceLocator;
+    if (
+      locator?.schemaVersion !== 1 ||
+      locator.type !== "automation-database" ||
+      locator.database !== database
+    ) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_SOURCE_UNBOUND",
+        `Automation rollback source locator does not match its database: ${entry.sourceId}`,
+      );
+    }
+    const current = getFlow(db, entry.sourceId);
+    if (!current) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_SOURCE_CHANGED",
+        `Automation rollback source does not exist: ${entry.sourceId}`,
+      );
+    }
+    const marker = getAutomationSchedulerMigration(db, entry.sourceId);
+    const effective = automationEffectiveSchedulerFlow(db, current);
+    if (
+      schedulerMigrationSourceDigest(
+        automationMigrationSourceSnapshot(effective),
+      ) !== entry.sourceDigest
+    ) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_SOURCE_CHANGED",
+        `Automation rollback source changed: ${entry.sourceId}`,
+      );
+    }
+    if (!marker) {
+      if (
+        !["prepared", "rolling_back", "rolled_back"].includes(migration.state)
+      ) {
+        throw automationError(
+          "AUTOMATION_SCHEDULER_MIGRATION_CONFLICT",
+          `Automation rollback source marker is missing: ${entry.sourceId}`,
+        );
+      }
+      continue;
+    }
+    if (
+      marker.schemaVersion !== 1 ||
+      !["prepared", "retired"].includes(marker.state) ||
+      marker.migrationId !== migrationId ||
+      marker.targetJobId !== entry.targetJobId ||
+      marker.sourceDigest !== automationMigrationSourceDigest(effective) ||
+      (marker.state === "prepared" && marker.retirementToken !== null) ||
+      (marker.state === "retired" &&
+        marker.retirementToken !== entry.retirementToken)
+    ) {
+      throw automationError(
+        "AUTOMATION_SCHEDULER_MIGRATION_CONFLICT",
+        `Automation rollback source marker does not match its journal: ${entry.sourceId}`,
+      );
+    }
+  }
+  return migration;
+}
+
+export function rollbackAutomationMigration({
+  db,
+  schedulerStore,
+  migrationId,
+}) {
+  const database = automationDatabaseIdentity(db);
+  assertAutomationRollbackBinding({
+    db,
+    migration: schedulerStore.getDomainMigration(migrationId),
+    migrationId,
+    database,
+  });
+  const migration = assertAutomationRollbackBinding({
+    db,
+    migration: schedulerStore.beginDomainMigrationRollback(migrationId),
+    migrationId,
+    database,
+  });
+  if (migration.state === "rolled_back") return migration;
+  for (const entry of migration.entries) {
+    if (entry.state === "rolled_back") continue;
+    schedulerStore.restoreDomainMigrationEntrySource({
+      migrationId,
+      entryId: entry.entryId,
+      retirementToken: entry.retirementToken,
+      restoreSource: () => {
+        const current = getFlow(db, entry.sourceId);
+        const marker = getAutomationSchedulerMigration(db, entry.sourceId);
+        let restored;
+        if (!marker) {
+          const source = automationMigrationSourceSnapshot(current);
+          if (schedulerMigrationSourceDigest(source) !== entry.sourceDigest) {
+            throw automationError(
+              "AUTOMATION_SCHEDULER_MIGRATION_SOURCE_CHANGED",
+              `Automation flow changed after rollback source restoration: ${entry.sourceId}`,
+            );
+          }
+          restored = current;
+        } else {
+          const effective = automationEffectiveSchedulerFlow(db, current);
+          restored = restoreAutomationSchedulerMigration(db, entry.sourceId, {
+            migrationId,
+            sourceDigest: automationMigrationSourceDigest(effective),
+            targetJobId: entry.targetJobId,
+            // The scheduler journal allocates its retirement token before the
+            // Automation source transaction advances its marker from prepared
+            // to retired. A crash in that window therefore has a journal token
+            // but an intentionally null source-marker token. Keep the source
+            // marker authoritative for this phase; the restore primitive still
+            // verifies migration/source/target identity before deleting it.
+            retirementToken:
+              marker.state === "prepared" ? null : entry.retirementToken,
+          });
+        }
+        return automationMigrationSourceSnapshot(restored);
+      },
+    });
+  }
+  return schedulerStore.getDomainMigration(migrationId);
+}
+
 function latestEnqueuedScheduledFor(schedulerStore, jobId) {
   const event = schedulerStore
     .history({ jobId, limit: 100 })
@@ -399,6 +741,37 @@ export function createAutomationSchedulerAdapter({ db, now = Date.now } = {}) {
   }
   return {
     kind: AUTOMATION_SCHEDULER_KIND,
+    async adjudicate({ occurrence, adjudication }) {
+      const payload = occurrence?.payload;
+      const expected = automationFlowSnapshot(payload?.flow);
+      if (payload?.snapshotDigest !== automationFlowSnapshotDigest(expected)) {
+        throw automationError(
+          "AUTOMATION_SCHEDULER_ADJUDICATION_SNAPSHOT_INVALID",
+          `Automation adjudication snapshot is invalid: ${occurrence.id}`,
+        );
+      }
+      adjudicateAutomationExecution(
+        db,
+        automationSchedulerExecutionId(occurrence.id),
+        {
+          flowId: expected.id,
+          triggerType: TRIGGER_TYPE.SCHEDULE,
+          decision: adjudication.decision,
+          requestId: adjudication.requestId,
+        },
+      );
+      if (adjudication.decision === "confirmed_applied") {
+        return {
+          settled: true,
+          result: {
+            executionId: automationSchedulerExecutionId(occurrence.id),
+            status: "adjudicated-applied",
+            adjudicationRequestId: adjudication.requestId,
+          },
+        };
+      }
+      return { continue: true };
+    },
     async execute({ occurrence }) {
       const payload = occurrence?.payload;
       const expected = automationFlowSnapshot(payload?.flow);
@@ -426,13 +799,14 @@ export function createAutomationSchedulerAdapter({ db, now = Date.now } = {}) {
           `Automation flow disappeared before execution: ${expected.id}`,
         );
       }
-      if (current.status !== FLOW_STATUS.ACTIVE) {
+      const effectiveCurrent = automationEffectiveSchedulerFlow(db, current);
+      if (effectiveCurrent.status !== FLOW_STATUS.ACTIVE) {
         throw automationError(
           "AUTOMATION_SCHEDULER_FLOW_NOT_ACTIVE",
           `Automation flow is not active: ${expected.id}`,
         );
       }
-      if (automationFlowSnapshotDigest(current) !== expectedDigest) {
+      if (automationFlowSnapshotDigest(effectiveCurrent) !== expectedDigest) {
         throw automationError(
           "AUTOMATION_SCHEDULER_STALE_SNAPSHOT",
           `Automation flow changed after occurrence enqueue: ${expected.id}`,
@@ -441,7 +815,7 @@ export function createAutomationSchedulerAdapter({ db, now = Date.now } = {}) {
 
       reserveAutomationExecutionAuthority(
         db,
-        current,
+        effectiveCurrent,
         occurrence.id,
         payload.executionAuthority,
         payload.executionAuthorityDigest,
@@ -490,6 +864,7 @@ export class AutomationSchedulerBridge {
     leaseMs,
     renewIntervalMs,
     authorityResolver,
+    migrateLegacy = true,
   } = {}) {
     if (typeof now !== "function") {
       throw automationError(
@@ -500,6 +875,7 @@ export class AutomationSchedulerBridge {
     this.db = db;
     this.schedulerStore = schedulerStore;
     this.now = now;
+    this.migrateLegacy = migrateLegacy !== false;
     this.runtime = new SchedulerRuntime({
       store: schedulerStore,
       adapters: [createAutomationSchedulerAdapter({ db, now })],
@@ -545,10 +921,36 @@ export class AutomationSchedulerBridge {
     if (signal?.aborted) return results;
 
     const now = normalizeEpochMs(Number(this.now()), "now");
-    const flows = listFlows(this.db, {
-      status: FLOW_STATUS.ACTIVE,
-      limit: 10_000,
-    }).filter((flow) => typeof flow.schedule === "string" && flow.schedule);
+    if (this.migrateLegacy) {
+      const candidates = listFlows(this.db, {
+        status: FLOW_STATUS.ACTIVE,
+        limit: 10_000,
+      }).filter((flow) => typeof flow.schedule === "string" && flow.schedule);
+      for (const flow of candidates) {
+        try {
+          migrateAutomationFlow({
+            db: this.db,
+            schedulerStore: this.schedulerStore,
+            flow,
+            executionAuthority: automationExecutionAuthoritySnapshot(
+              this.db,
+              flow,
+            ),
+          });
+        } catch (error) {
+          if (!String(error?.code || "").startsWith("AUTOMATION_EXECUTION_")) {
+            throw error;
+          }
+        }
+      }
+    }
+    const flows = [
+      ...listFlows(this.db, {
+        status: FLOW_STATUS.ACTIVE,
+        limit: 10_000,
+      }),
+      ...listAutomationSchedulerFlows(this.db, { limit: 10_000 }),
+    ].filter((flow) => typeof flow.schedule === "string" && flow.schedule);
     for (const flow of flows) {
       let occurrence;
       try {

package/src/lib/scheduler-kernel/automation-event-adapter.js

@@ -11,6 +11,7 @@ import {
   EXECUTION_STATUS,
   FLOW_STATUS,
   TRIGGER_TYPE,
+  adjudicateAutomationExecution,
   executeFlow,
   getExecution,
   getFlow,
@@ -552,6 +553,41 @@ export function createAutomationEventAdapter({ db, now = Date.now } = {}) {
   }
   return {
     kind: AUTOMATION_EVENT_KIND,
+    async adjudicate({ occurrence, adjudication }) {
+      const payload = occurrence?.payload;
+      const expectedFlow = automationEventFlowSnapshot(payload?.flow);
+      const expectedTrigger = automationEventTriggerSnapshot(payload?.trigger);
+      if (
+        payload?.definitionDigest !==
+        automationEventDefinitionDigest(expectedFlow, expectedTrigger)
+      ) {
+        throw eventError(
+          "AUTOMATION_EVENT_ADJUDICATION_SNAPSHOT_INVALID",
+          `automation event adjudication snapshot is invalid: ${occurrence.id}`,
+        );
+      }
+      adjudicateAutomationExecution(
+        db,
+        automationEventExecutionId(occurrence.id),
+        {
+          flowId: expectedFlow.id,
+          triggerType: TRIGGER_TYPE.EVENT,
+          decision: adjudication.decision,
+          requestId: adjudication.requestId,
+        },
+      );
+      if (adjudication.decision === "confirmed_applied") {
+        return {
+          settled: true,
+          result: {
+            executionId: automationEventExecutionId(occurrence.id),
+            status: "adjudicated-applied",
+            adjudicationRequestId: adjudication.requestId,
+          },
+        };
+      }
+      return { continue: true };
+    },
     async execute({ occurrence }) {
       const payload = occurrence?.payload;
       const expectedFlow = automationEventFlowSnapshot(payload?.flow);