@objectstack/core 17.0.0-rc.1 → 17.0.0-rc.3

package/dist/index.js

@@ -1851,17 +1851,51 @@ var ObjectKernel = class {
     assertInitServiceRequirements(plugin, (name) => this.hasAnyService(name));
     this.currentlyInitializing = plugin.name;
     try {
-      const initPromise = plugin.init(this.context);
-      const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(() => {
-          reject(new Error(`Plugin ${plugin.name} init timeout after ${timeout}ms`));
-        }, timeout);
-      });
-      await Promise.race([initPromise, timeoutPromise]);
+      await this.raceStartupTimeout(
+        plugin.init(this.context),
+        timeout,
+        `Plugin ${plugin.name} init timeout after ${timeout}ms`
+      );
     } finally {
       this.currentlyInitializing = void 0;
     }
   }
+  /**
+   * Race a plugin lifecycle hook against its startup-timeout guard, and
+   * reclaim the guard the moment the race settles (#4813).
+   *
+   * The guard used to be armed and then abandoned: when the plugin won the
+   * race, its `setTimeout` stayed ref'd in the event loop for the full
+   * `startupTimeout`, so every process idled that long after its work was
+   * done. One `os migrate` finished in 3s and then sat for 120s
+   * (`ObjectQLPlugin.startupTimeout`), held open by 8 orphaned guards — one
+   * per init plus one per start.
+   *
+   * Clearing on settle rather than `unref()`-ing at arm time is deliberate.
+   * An unref'd guard also stops pinning the loop, but it stops being a guard
+   * as well: if the hook never settles and nothing else keeps the loop alive,
+   * Node exits before the timer can fire and the timeout is never reported.
+   * The guard has to stay ref'd exactly as long as the race is undecided,
+   * which is what `clearTimeout` in a `finally` expresses.
+   *
+   * `operation` is widened to `T | PromiseLike<T>` because the Plugin
+   * contract permits a synchronous hook (`init`/`start` return
+   * `void | Promise<void>`); such a hook wins the race immediately and the
+   * guard is reclaimed on the same turn.
+   */
+  async raceStartupTimeout(operation, timeout, message) {
+    let guard;
+    const timeoutPromise = new Promise((_, reject) => {
+      guard = setTimeout(() => {
+        reject(new Error(message));
+      }, timeout);
+    });
+    try {
+      return await Promise.race([operation, timeoutPromise]);
+    } finally {
+      clearTimeout(guard);
+    }
+  }
   /**
    * Whether a service is resolvable on this kernel right now — direct
    * registration or a loader-registered factory. Backs the init-service
@@ -1887,13 +1921,11 @@ var ObjectKernel = class {
     const startTime = Date.now();
     this.logger.debug(`Start: ${plugin.name}`, { plugin: plugin.name });
     try {
-      const startPromise = plugin.start(this.context);
-      const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(() => {
-          reject(new Error(`Plugin ${plugin.name} start timeout after ${timeout}ms`));
-        }, timeout);
-      });
-      await Promise.race([startPromise, timeoutPromise]);
+      await this.raceStartupTimeout(
+        plugin.start(this.context),
+        timeout,
+        `Plugin ${plugin.name} start timeout after ${timeout}ms`
+      );
       const duration = Date.now() - startTime;
       this.startedPlugins.add(plugin.name);
       this.pluginStartTimes.set(plugin.name, duration);
@@ -4955,6 +4987,395 @@ async function bulkWrite(rows, opts) {
   return results;
 }
 
+// src/utils/migration-journal.ts
+import { createHash as createHash2, randomUUID } from "crypto";
+import {
+  MIGRATION_JOURNAL_OBJECT
+} from "@objectstack/spec/system";
+var SYSTEM_CTX = { isSystem: true };
+var DEFAULT_CHUNK_SIZE = 200;
+function engineCanRollBack(engine) {
+  const e = engine;
+  if (typeof e?.transaction !== "function") return false;
+  const defaultDriverName = e.getDefaultDriverName?.();
+  const defaultDriver = defaultDriverName ? e.getDriverByName?.(defaultDriverName) : void 0;
+  return !defaultDriver || typeof defaultDriver.beginTransaction === "function";
+}
+var MigrationPlanRegistry = class {
+  constructor() {
+    this.plans = /* @__PURE__ */ new Map();
+  }
+  register(plan) {
+    this.plans.set(plan.id, plan);
+  }
+  get(planId) {
+    return this.plans.get(planId);
+  }
+  list() {
+    return [...this.plans.values()];
+  }
+};
+var MigrationJournalRefusal = class extends Error {
+  constructor(code, message) {
+    super(message);
+    this.name = "MigrationJournalRefusal";
+    this.code = code;
+  }
+};
+function planChunks(plan, rowCounts, chunkSize = plan.chunkSize ?? DEFAULT_CHUNK_SIZE) {
+  const size = Math.max(1, chunkSize);
+  const chunks = [];
+  plan.steps.forEach((step, stepIndex) => {
+    const total = rowCounts[stepIndex] ?? 0;
+    for (let offset = 0; offset < total; offset += size) {
+      chunks.push({
+        index: chunks.length,
+        stepIndex,
+        stepName: step.name,
+        offset,
+        length: Math.min(size, total - offset)
+      });
+    }
+  });
+  return chunks;
+}
+function hashMigrationPlan(plan, chunks) {
+  const shape = JSON.stringify({
+    id: plan.id,
+    steps: plan.steps.map((s) => s.name),
+    chunks: chunks.map((c) => [c.stepIndex, c.offset, c.length])
+  });
+  return createHash2("sha256").update(shape, "utf8").digest("hex").slice(0, 32);
+}
+async function appendEvent(engine, event, execContext) {
+  await engine.insert(
+    MIGRATION_JOURNAL_OBJECT,
+    { ...event, created_at: event.created_at ?? (/* @__PURE__ */ new Date()).toISOString() },
+    { context: execContext ?? { ...SYSTEM_CTX } }
+  );
+}
+async function readRunJournal(engine, runId) {
+  const rows = await engine.find(
+    MIGRATION_JOURNAL_OBJECT,
+    { where: { run_id: runId } },
+    { context: { ...SYSTEM_CTX } }
+  );
+  return [...rows ?? []].sort((a, b) => Number(a.seq) - Number(b.seq));
+}
+function chunkSetOf(events, kind) {
+  const out = /* @__PURE__ */ new Set();
+  for (const e of events) {
+    if (e.kind === kind && typeof e.chunk_index === "number") out.add(e.chunk_index);
+  }
+  return out;
+}
+async function findInterruptedRuns(engine) {
+  const started = await engine.find(
+    MIGRATION_JOURNAL_OBJECT,
+    { where: { kind: "run_started" } },
+    { context: { ...SYSTEM_CTX } }
+  );
+  const out = [];
+  for (const start of started ?? []) {
+    const events = await readRunJournal(engine, start.run_id);
+    if (events.some((e) => e.kind === "run_done")) continue;
+    const committed = chunkSetOf(events, "chunk_done");
+    const compensated = chunkSetOf(events, "compensated");
+    const outstanding = [...committed].filter((i) => !compensated.has(i));
+    if (events.some((e) => e.kind === "run_failed") && outstanding.length === 0) continue;
+    const unknown = [...chunkSetOf(events, "chunk_started")].filter((i) => !committed.has(i));
+    let planId = start.run_id;
+    try {
+      planId = start.detail ? JSON.parse(start.detail).planId ?? start.run_id : start.run_id;
+    } catch {
+    }
+    out.push({
+      runId: start.run_id,
+      planId,
+      planHash: start.plan_hash ?? "",
+      migrationId: start.migration_id,
+      startedAt: start.created_at,
+      committedChunks: [...committed].sort((a, b) => a - b),
+      unknownChunks: unknown.sort((a, b) => a - b),
+      compensatedChunks: [...compensated].sort((a, b) => a - b)
+    });
+  }
+  return out;
+}
+async function loadPlan(engine, plan, chunkSize) {
+  const rowsByStep = [];
+  for (const step of plan.steps) rowsByStep.push(await step.load(engine) ?? []);
+  const chunks = planChunks(plan, rowsByStep.map((r) => r.length), chunkSize ?? plan.chunkSize);
+  return { chunks, planHash: hashMigrationPlan(plan, chunks), rowsByStep };
+}
+async function runMigrationJournal(engine, plan, options = {}) {
+  const now = options.now ?? (() => (/* @__PURE__ */ new Date()).toISOString());
+  if (!engineCanRollBack(engine)) {
+    throw new MigrationJournalRefusal(
+      "NOT_IMPLEMENTED",
+      `Migration plan '${plan.id}' requires engine transaction support; this runtime cannot roll back. The journal's chunk_done markers would not mean "committed", so the run is refused rather than started.`
+    );
+  }
+  const { chunks, planHash, rowsByStep } = await loadPlan(engine, plan, options.chunkSize);
+  const resuming = Boolean(options.runId);
+  const runId = options.runId ?? randomUUID();
+  let events = [];
+  let seq = 0;
+  let committed = /* @__PURE__ */ new Set();
+  let compensated = /* @__PURE__ */ new Set();
+  const attemptsByChunk = /* @__PURE__ */ new Map();
+  if (resuming) {
+    events = await readRunJournal(engine, runId);
+    if (events.length === 0) {
+      throw new MigrationJournalRefusal("NO_SUCH_RUN", `No journal rows for run '${runId}'.`);
+    }
+    const start = events.find((e) => e.kind === "run_started");
+    if (start?.plan_hash && start.plan_hash !== planHash) {
+      throw new MigrationJournalRefusal(
+        "PLAN_CHANGED",
+        `Refusing to resume run '${runId}': plan hash ${planHash} does not match the journal's ${start.plan_hash}. The chunk boundaries recorded in the journal describe a different plan.`
+      );
+    }
+    if (events.some((e) => e.kind === "run_done")) {
+      return {
+        runId,
+        status: "completed",
+        chunksTotal: chunks.length,
+        chunksCommitted: chunkSetOf(events, "chunk_done").size,
+        chunksCompensated: chunkSetOf(events, "compensated").size,
+        planHash
+      };
+    }
+    seq = events.reduce((m, e) => Math.max(m, Number(e.seq) + 1), 0);
+    committed = chunkSetOf(events, "chunk_done");
+    compensated = chunkSetOf(events, "compensated");
+    for (const e of events) {
+      if (e.kind === "chunk_started" && typeof e.chunk_index === "number") {
+        attemptsByChunk.set(e.chunk_index, (attemptsByChunk.get(e.chunk_index) ?? 0) + 1);
+      }
+    }
+  }
+  for (const step of plan.steps) {
+    if (!step.preflight) continue;
+    try {
+      await step.preflight(engine);
+    } catch (err) {
+      throw new MigrationJournalRefusal(
+        "PREFLIGHT_FAILED",
+        `Migration plan '${plan.id}' refused: preflight for step '${step.name}' failed: ${errText(err)}`
+      );
+    }
+  }
+  if (plan.onCrash === "compensate") {
+    const missing = plan.steps.filter((s) => !s.compensate).map((s) => s.name);
+    if (missing.length > 0) {
+      throw new MigrationJournalRefusal(
+        "NOT_COMPENSABLE",
+        `Migration plan '${plan.id}' declares onCrash: 'compensate' but step(s) ${missing.join(", ")} declare no compensate().`
+      );
+    }
+  }
+  const rowsOf = (c) => rowsByStep[c.stepIndex].slice(c.offset, c.offset + c.length);
+  const next = () => seq++;
+  if (!resuming) {
+    await appendEvent(engine, {
+      run_id: runId,
+      seq: next(),
+      kind: "run_started",
+      plan_hash: planHash,
+      migration_id: plan.migrationId,
+      created_at: now(),
+      detail: JSON.stringify({
+        planId: plan.id,
+        onCrash: plan.onCrash ?? "resume",
+        chunks: chunks.map((c) => ({ i: c.index, step: c.stepName, offset: c.offset, length: c.length }))
+      })
+    });
+  }
+  if (resuming && plan.onCrash === "compensate") {
+    return await unwind(engine, plan, {
+      runId,
+      planHash,
+      chunks,
+      rowsOf,
+      next,
+      now,
+      committed,
+      compensated,
+      chunksTotal: chunks.length,
+      cause: new Error(`run '${runId}' rediscovered after interruption; plan policy is compensate`)
+    });
+  }
+  for (const chunk of chunks) {
+    if (committed.has(chunk.index)) continue;
+    const attempt = (attemptsByChunk.get(chunk.index) ?? 0) + 1;
+    attemptsByChunk.set(chunk.index, attempt);
+    const step = plan.steps[chunk.stepIndex];
+    const rows = rowsOf(chunk);
+    await appendEvent(engine, {
+      run_id: runId,
+      seq: next(),
+      kind: "chunk_started",
+      chunk_index: chunk.index,
+      attempt,
+      migration_id: plan.migrationId,
+      created_at: now()
+    });
+    try {
+      await engine.transaction(async (trxCtx) => {
+        await step.forward(rows, { runId, chunkIndex: chunk.index, attempt, context: trxCtx }, engine);
+        await appendEvent(
+          engine,
+          {
+            run_id: runId,
+            seq: next(),
+            kind: "chunk_done",
+            chunk_index: chunk.index,
+            attempt,
+            migration_id: plan.migrationId,
+            created_at: now()
+          },
+          trxCtx
+        );
+      }, { ...SYSTEM_CTX });
+      committed.add(chunk.index);
+    } catch (err) {
+      return await unwind(engine, plan, {
+        runId,
+        planHash,
+        chunks,
+        rowsOf,
+        next,
+        now,
+        committed,
+        compensated,
+        chunksTotal: chunks.length,
+        cause: err
+      });
+    }
+  }
+  await appendEvent(engine, {
+    run_id: runId,
+    seq: next(),
+    kind: "run_done",
+    migration_id: plan.migrationId,
+    created_at: now()
+  });
+  return {
+    runId,
+    status: "completed",
+    chunksTotal: chunks.length,
+    chunksCommitted: committed.size,
+    chunksCompensated: compensated.size,
+    planHash
+  };
+}
+async function unwind(engine, plan, a) {
+  const order = [...a.committed].sort((x, y) => y - x);
+  for (const index of order) {
+    if (a.compensated.has(index)) continue;
+    const chunk = a.chunks[index];
+    const step = plan.steps[chunk.stepIndex];
+    if (!step.compensate) {
+      await appendEvent(engine, {
+        run_id: a.runId,
+        seq: a.next(),
+        kind: "run_failed",
+        chunk_index: index,
+        migration_id: plan.migrationId,
+        created_at: a.now(),
+        detail: JSON.stringify({
+          phase: "compensate",
+          reason: "step declares no compensate()",
+          step: step.name,
+          cause: errText(a.cause)
+        })
+      });
+      return {
+        runId: a.runId,
+        status: "failed",
+        chunksTotal: a.chunksTotal,
+        chunksCommitted: a.committed.size,
+        chunksCompensated: a.compensated.size,
+        planHash: a.planHash,
+        error: a.cause
+      };
+    }
+    const attempt = 1;
+    try {
+      await engine.transaction(async (trxCtx) => {
+        await step.compensate(a.rowsOf(chunk), { runId: a.runId, chunkIndex: index, attempt, context: trxCtx }, engine);
+        await appendEvent(
+          engine,
+          {
+            run_id: a.runId,
+            seq: a.next(),
+            kind: "compensated",
+            chunk_index: index,
+            attempt,
+            migration_id: plan.migrationId,
+            created_at: a.now()
+          },
+          trxCtx
+        );
+      }, { ...SYSTEM_CTX });
+      a.compensated.add(index);
+    } catch (err) {
+      await appendEvent(engine, {
+        run_id: a.runId,
+        seq: a.next(),
+        kind: "run_failed",
+        chunk_index: index,
+        migration_id: plan.migrationId,
+        created_at: a.now(),
+        detail: JSON.stringify({
+          phase: "compensate",
+          step: step.name,
+          error: errText(err),
+          cause: errText(a.cause)
+        })
+      });
+      return {
+        runId: a.runId,
+        status: "failed",
+        chunksTotal: a.chunksTotal,
+        chunksCommitted: a.committed.size,
+        chunksCompensated: a.compensated.size,
+        planHash: a.planHash,
+        error: err
+      };
+    }
+  }
+  await appendEvent(engine, {
+    run_id: a.runId,
+    seq: a.next(),
+    kind: "run_failed",
+    migration_id: plan.migrationId,
+    created_at: a.now(),
+    detail: JSON.stringify({ phase: "forward", error: errText(a.cause), compensated: [...a.compensated].sort((x, y) => x - y) })
+  });
+  return {
+    runId: a.runId,
+    status: "compensated",
+    chunksTotal: a.chunksTotal,
+    chunksCommitted: a.committed.size,
+    chunksCompensated: a.compensated.size,
+    planHash: a.planHash,
+    error: a.cause
+  };
+}
+async function resumeMigrationJournal(engine, plan, runId, options = {}) {
+  return runMigrationJournal(engine, plan, { ...options, runId });
+}
+function errText(err) {
+  if (err instanceof Error) return err.message;
+  try {
+    return String(err);
+  } catch {
+    return "<unprintable error>";
+  }
+}
+
 // src/utils/filter-tokens.ts
 import {
   classifyFilterToken,
@@ -5404,7 +5825,7 @@ var PluginHealthMonitor = class {
 };
 
 // src/hot-reload.ts
-import { createHash as createHash2 } from "crypto";
+import { createHash as createHash3 } from "crypto";
 var generateUUID = () => {
   if (typeof crypto !== "undefined" && crypto.randomUUID) {
     return crypto.randomUUID();
@@ -5499,7 +5920,7 @@ var PluginStateManager = class {
    */
   calculateChecksum(state) {
     const stateStr = JSON.stringify(state);
-    return createHash2("sha256").update(stateStr).digest("hex");
+    return createHash3("sha256").update(stateStr).digest("hex");
   }
   /**
    * Shutdown state manager
@@ -6085,6 +6506,8 @@ export {
   DependencyResolver,
   HotReloadManager,
   LiteKernel,
+  MigrationJournalRefusal,
+  MigrationPlanRegistry,
   NamespaceResolver,
   ObjectKernel,
   ObjectKernelBase,
@@ -6127,14 +6550,17 @@ export {
   defaultIsTransientError,
   derivePosture,
   describeInitOrderFault,
+  engineCanRollBack,
   evaluateAuthGate,
   extractApiKey,
   filterTokenContextFrom,
+  findInterruptedRuns,
   generateApiKey,
   generateEd25519KeyPair,
   getEnv,
   getMemoryUsage,
   hashApiKey,
+  hashMigrationPlan,
   isAuthGateAllowlisted,
   isExpired,
   isGrantActive,
@@ -6143,8 +6569,10 @@ export {
   nextUtcCalendarDay,
   parseScopes,
   parseSignature,
+  planChunks,
   postureVisibleRows,
   readAuthoredTranslationLayer,
+  readRunJournal,
   resolveApiKeyPrincipal,
   resolveAuthzContext,
   resolveFilterToken,
@@ -6153,6 +6581,8 @@ export {
   resolveLocalizationContext,
   resolvePluginOrder,
   resolveUserAuthzGrants,
+  resumeMigrationJournal,
+  runMigrationJournal,
   safeExit,
   shouldDenyAnonymous,
   signPayload,