@lostgradient/weft 0.3.0 → 0.4.0

package/dist/core/engine/operations-wait-condition.js

@@ -0,0 +1,62 @@
+function evaluateConditionOutcome(internals, predicate, deadline) {
+  if (predicate())
+    return { done: !0, value: deadline === void 0 ? void 0 : !0 };
+  if (deadline !== void 0 && internals.options.getNow() >= deadline)
+    return { done: !0, value: !1 };
+  return { done: !1 };
+}
+export async function processWaitConditionOperation(internals, workflowId, operation, callbacks) {
+  const abortSignal = internals.abortController.signal, { predicate, deadline } = operation;
+  let settled = !1;
+  const complete = (value) => {
+    if (settled)
+      return;
+    settled = !0;
+    callbacks.completeOperation(workflowId, value);
+  };
+  if (abortSignal.aborted)
+    return;
+  try {
+    const initial = evaluateConditionOutcome(internals, predicate, deadline);
+    if (initial.done) {
+      complete(initial.value);
+      return;
+    }
+    if (deadline !== void 0)
+      await callbacks.scheduleConditionDeadline(workflowId, operation.step, deadline);
+    try {
+      let step = await runConditionWaitStep(internals, workflowId, predicate, deadline, callbacks);
+      while (step.status === "continue")
+        step = await runConditionWaitStep(internals, workflowId, predicate, deadline, callbacks);
+      if (step.status === "complete")
+        complete(step.value);
+    } finally {
+      releaseConditionWaiter(internals, workflowId);
+      if (deadline !== void 0)
+        await callbacks.cancelConditionDeadline(workflowId, operation.step);
+    }
+  } catch (error) {
+    callbacks.failOperation(workflowId, operation, error);
+  }
+}
+async function runConditionWaitStep(internals, workflowId, predicate, deadline, callbacks) {
+  const abortSignal = internals.abortController.signal;
+  if (abortSignal.aborted)
+    return { status: "stop" };
+  const { promise, resolve } = Promise.withResolvers();
+  internals.conditionWaiters.set(workflowId, resolve);
+  const outcome = abortSignal.aborted ? { done: !0, value: void 0, aborted: !0 } : { ...evaluateConditionOutcome(internals, predicate, deadline), aborted: !1 };
+  if (outcome.done) {
+    releaseConditionWaiter(internals, workflowId);
+    return outcome.aborted ? { status: "stop" } : { status: "complete", value: outcome.value };
+  }
+  await promise;
+  if (abortSignal.aborted)
+    return { status: "stop" };
+  if (!await callbacks.isWorkflowRunning(workflowId))
+    return { status: "stop" };
+  return { status: "continue" };
+}
+function releaseConditionWaiter(internals, workflowId) {
+  internals.conditionWaiters.delete(workflowId);
+}

package/dist/core/engine/ownership-options.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Resolution of the `ownership` posture options (`Engine.create({ ownership })`)
+ * into {@link ResolvedOptions}. Extracted from `construction.ts` to keep that
+ * file under the `max-lines` ceiling; mirrors the second-instance resolver's
+ * "only validate when the feature is enabled" discipline.
+ *
+ * @module core/engine/ownership-options
+ */
+import type { EngineConstructorOptions, ResolvedOptions } from './engine-internal-types.ts';
+/** Default lease time-to-live for `ownership: 'lease'`. */
+export declare const DEFAULT_LEASE_TTL_MS = 30000;
+/** Default lease renewal interval for `ownership: 'lease'`. */
+export declare const DEFAULT_LEASE_RENEW_INTERVAL_MS = 5000;
+/** Default boot-time lease acquisition wait window for `ownership: 'lease'`. */
+export declare const DEFAULT_LEASE_WAIT_TIMEOUT_MS = 60000;
+/**
+ * Resolve the ownership posture and lease tuning into their `ResolvedOptions`
+ * fields. Defaults to `'none'`. The lease durations are documented as "ignored
+ * when ownership is not 'lease'", so they are only normalized (and thus only
+ * able to throw on an invalid value) when lease ownership is actually enabled —
+ * an invalid duration must not make an off-by-default config fatal at boot.
+ */
+export declare function resolveOwnershipFields(options: EngineConstructorOptions | undefined): Pick<ResolvedOptions, 'ownershipMode' | 'leaseTtlMs' | 'leaseRenewIntervalMs' | 'leaseWaitTimeoutMs'>;

package/dist/core/engine/ownership-options.js

@@ -0,0 +1,26 @@
+import { normalizeRetentionDuration } from "./validation.js";
+export const DEFAULT_LEASE_TTL_MS = 30000, DEFAULT_LEASE_RENEW_INTERVAL_MS = 5000, DEFAULT_LEASE_WAIT_TIMEOUT_MS = 60000;
+export function resolveOwnershipFields(options) {
+  const ownershipMode = assertKnownOwnershipMode(options?.ownership ?? "none");
+  if (ownershipMode !== "lease")
+    return {
+      ownershipMode,
+      leaseTtlMs: DEFAULT_LEASE_TTL_MS,
+      leaseRenewIntervalMs: DEFAULT_LEASE_RENEW_INTERVAL_MS,
+      leaseWaitTimeoutMs: DEFAULT_LEASE_WAIT_TIMEOUT_MS
+    };
+  const leaseTtlMs = normalizeRetentionDuration(options?.leaseTtl, "options.leaseTtl") ?? DEFAULT_LEASE_TTL_MS, leaseRenewIntervalMs = normalizeRetentionDuration(options?.leaseRenewInterval, "options.leaseRenewInterval") ?? DEFAULT_LEASE_RENEW_INTERVAL_MS, leaseWaitTimeoutMs = normalizeRetentionDuration(options?.leaseWaitTimeout, "options.leaseWaitTimeout") ?? DEFAULT_LEASE_WAIT_TIMEOUT_MS;
+  assertLeaseTimingCoherent(leaseTtlMs, leaseRenewIntervalMs, leaseWaitTimeoutMs);
+  return { ownershipMode, leaseTtlMs, leaseRenewIntervalMs, leaseWaitTimeoutMs };
+}
+function assertKnownOwnershipMode(mode) {
+  if (mode !== "none" && mode !== "lease")
+    throw Error(`Unknown ownership posture "${String(mode)}". Expected 'none' or 'lease'.`);
+  return mode;
+}
+function assertLeaseTimingCoherent(leaseTtlMs, leaseRenewIntervalMs, leaseWaitTimeoutMs) {
+  if (leaseTtlMs <= 0 || leaseRenewIntervalMs <= 0 || leaseWaitTimeoutMs <= 0)
+    throw Error(`ownership: 'lease' requires positive leaseTtl, leaseRenewInterval, and leaseWaitTimeout (got leaseTtl=${leaseTtlMs}ms, leaseRenewInterval=${leaseRenewIntervalMs}ms, leaseWaitTimeout=${leaseWaitTimeoutMs}ms).`);
+  if (leaseRenewIntervalMs >= leaseTtlMs)
+    throw Error(`ownership: 'lease' requires leaseRenewInterval (${leaseRenewIntervalMs}ms) to be strictly less than leaseTtl (${leaseTtlMs}ms), so a renewal fires before the lease can lapse.`);
+}

package/dist/core/engine/payload-size-policy.d.ts

@@ -0,0 +1,3 @@
+import type { Engine } from './index.ts';
+/** Internal server bridge for the engine's normalized payload-size cap. */
+export declare function getEnginePayloadSizeMaxBytes(engine: Engine): number | null;

package/dist/core/engine/payload-size-policy.js

@@ -0,0 +1,4 @@
+import { getInternals } from "./internals.js";
+export function getEnginePayloadSizeMaxBytes(engine) {
+  return getInternals(engine).options.payloadSizePolicy.maxBytes;
+}

package/dist/core/engine/pending-updates.d.ts

@@ -14,5 +14,22 @@ export declare function invokeUpdateHandler(internals: EngineInternals, name: st
  */
 export declare function processPendingUpdatesAfterInlineAdvance(internals: EngineInternals, workflowId: string, callbacks: PendingUpdateCallbacks): Promise<void>;
 export declare function schedulePendingInlineUpdateDrain(internals: EngineInternals, workflowId: string, callbacks: PendingUpdateCallbacks): void;
+/**
+ * Drain a workflow's buffered coordinated updates and deliver each to its
+ * handler exactly once, even when several drains race.
+ *
+ * Several triggers can fire near-simultaneously: each `engine.update()` schedules
+ * a `setTimeout(0)` drain, and the post-advance path drains too. Their consume-
+ * deletes are durable (`storage.batch`) and lag the in-memory `getPendingUpdates`
+ * scan, so two overlapping drains both see the same pending set. Each update id
+ * is therefore CLAIMED synchronously in `deliveredPendingUpdateIds` before
+ * delivery (no `await` between the membership check and the add, so a concurrent
+ * drain cannot interleave): a racing drain that re-reads the same id skips it.
+ *
+ * After the drain, re-drive a parked `ctx.waitUntil` once. A coordinated update
+ * buffered before `ctx.onUpdate` was registered drains here (not on the inline
+ * `tryInlineUpdateHandler` path), and a handler may have mutated the workflow-
+ * local state a predicate reads, so the wait must be poked to re-evaluate.
+ */
 export declare function processPendingUpdatesForHandlers(internals: EngineInternals, workflowId: string, callbacks: PendingUpdateCallbacks): Promise<void>;
 export {};

package/dist/core/engine/pending-updates.js

@@ -1,5 +1,7 @@
 import { UpdateCompletedEvent, UpdateReceivedEvent } from "../events.js";
 import { UpdateValidationError } from "../updates.js";
+import { notifyConditionWaiters } from "./condition-waiters.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 import {
   extractStandardSchemaIssues,
   invokeUpdateHandler as invokeUpdateHandlerFromInternals
@@ -71,17 +73,37 @@ export async function processPendingUpdatesForHandlers(internals, workflowId, ca
   const pendingUpdates = await internals.updateCoordinator.getPendingUpdates(workflowId);
   if (pendingUpdates.length === 0)
     return;
-  for (const update of pendingUpdates) {
-    const handler = handlers.get(update.name);
-    if (!handler)
-      continue;
-    const validatorRejectionError = await runValidatorIfPresent(context, update);
-    if (validatorRejectionError !== null) {
-      await rejectPendingUpdate(internals, workflowId, update, validatorRejectionError, callbacks);
-      continue;
+  let handlerRan = !1;
+  try {
+    for (const update of pendingUpdates) {
+      const handler = handlers.get(update.name);
+      if (!handler)
+        continue;
+      if (!claimPendingUpdateForDelivery(internals, workflowId, update.updateId))
+        continue;
+      const validatorRejectionError = await runValidatorIfPresent(context, update);
+      if (validatorRejectionError !== null) {
+        await rejectPendingUpdate(internals, workflowId, update, validatorRejectionError, callbacks);
+        continue;
+      }
+      handlerRan = !0;
+      await deliverPendingUpdate(internals, workflowId, update, handler, callbacks);
     }
-    await deliverPendingUpdate(internals, workflowId, update, handler, callbacks);
+  } finally {
+    if (handlerRan)
+      notifyConditionWaiters(internals, workflowId);
+  }
+}
+function claimPendingUpdateForDelivery(internals, workflowId, updateId) {
+  let claimed = internals.deliveredPendingUpdateIds.get(workflowId);
+  if (claimed === void 0) {
+    claimed = new Set;
+    internals.deliveredPendingUpdateIds.set(workflowId, claimed);
   }
+  if (claimed.has(updateId))
+    return !1;
+  claimed.add(updateId);
+  return !0;
 }
 async function runValidatorIfPresent(context, update) {
   const validator = context.updateValidators?.get(update.name);
@@ -91,7 +113,7 @@ async function runValidatorIfPresent(context, update) {
 }
 async function rejectPendingUpdate(internals, workflowId, update, errorMessage, callbacks) {
   const responseOperations = internals.updateCoordinator.buildResponseOperations(update.updateId, workflowId, void 0, errorMessage, update.idempotencyKey);
-  await internals.storage.batch(responseOperations);
+  await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Update rejection for workflow "${workflowId}" lost its CAS race.`));
   callbacks.broadcast({ type: "update:completed", workflowId, updateId: update.updateId });
 }
 async function deliverPendingUpdate(internals, workflowId, update, handler, callbacks) {
@@ -103,7 +125,7 @@ async function deliverPendingUpdate(internals, workflowId, update, handler, call
     error = handlerError instanceof Error ? handlerError.message : String(handlerError);
   }
   const responseOperations = internals.updateCoordinator.buildResponseOperations(update.updateId, workflowId, result, error, update.idempotencyKey);
-  await internals.storage.batch(responseOperations);
+  await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Update response for workflow "${workflowId}" lost its CAS race.`));
   callbacks.dispatchEvent(new UpdateCompletedEvent(update.updateId, workflowId, update.name, result, error));
   callbacks.broadcast({ type: "update:completed", workflowId, updateId: update.updateId });
 }

package/dist/core/engine/queries.js

@@ -4,7 +4,7 @@ export async function query(internals, workflowId, name, input) {
   const inlineStrategy = internals.inlineStrategy;
   if (!inlineStrategy)
     throw Error("Workflow queries are not supported when using the worker execution strategy.");
-  const context = inlineStrategy.getContext(workflowId);
+  const context = inlineStrategy.getContext(workflowId) ?? inlineStrategy.getParkedContext(workflowId);
   if (!context)
     return;
   const queryHandler = context.queryHandlers.get(name);

package/dist/core/engine/registration.d.ts

@@ -1,6 +1,7 @@
 import type { EngineInternals } from './internals.ts';
 export type RegistrationCallbacks = {
     ensureRetentionSweepInterval: () => void;
+    dispatchEvent: (event: Event) => void;
 };
 export declare function register(internals: EngineInternals, definition: unknown, callbacks: RegistrationCallbacks): void;
 export declare function resolveWorkflowTypeTarget(internals: EngineInternals, target: string | Function, _callbacks: RegistrationCallbacks): string;

package/dist/core/engine/registration.js

@@ -1,4 +1,5 @@
 import { ActivityRegistry } from "../activity-registry.js";
+import { WorkflowDefinitionRegisteredEvent } from "../events.js";
 import {
   activity,
   validateDefinitionSchemaMetadata
@@ -31,16 +32,29 @@ function buildBaseRegistrationEntry(name, registration) {
     ...registration.outputSchema === void 0 ? {} : {
       outputSchema: validateDefinitionSchemaMetadata(registration.outputSchema, `registration("${name}").outputSchema`)
     },
-    ...normalizedRetention !== null && { retention: normalizedRetention }
+    ...normalizedRetention !== null && { retention: normalizedRetention },
+    ...registration.concurrency !== void 0 ? {
+      concurrency: normalizeWorkflowConcurrencyOptions(registration.concurrency, `registration("${name}").concurrency`)
+    } : {}
   };
 }
+function normalizeWorkflowConcurrencyOptions(concurrency, fieldName) {
+  if (!Number.isInteger(concurrency.max) || concurrency.max < 1)
+    throw RangeError(`${fieldName}.max must be a positive integer`);
+  if (concurrency.key !== void 0 && typeof concurrency.key !== "function")
+    throw TypeError(`${fieldName}.key must be a function when provided`);
+  return Object.freeze({
+    max: concurrency.max,
+    ...concurrency.key === void 0 ? {} : { key: concurrency.key }
+  });
+}
 function applyOptionalRegistrationFields(entry, registration) {
-  if (registration.migrate)
-    entry.migrate = registration.migrate;
   if (registration.searchAttributes && Object.keys(registration.searchAttributes).length > 0)
     entry.searchAttributes = registration.searchAttributes;
   if (registration.constraints && registration.constraints.length > 0)
     entry.constraints = registration.constraints;
+  if (registration.finalizer !== void 0)
+    entry.finalizer = registration.finalizer;
 }
 function buildRegistrationEntry(name, registration) {
   const entry = buildBaseRegistrationEntry(name, registration);
@@ -55,6 +69,7 @@ function commitWorkflowDefinition(internals, definition, callbacks) {
   internals.registrations.set(name, entry);
   callbacks.ensureRetentionSweepInterval();
   internals.workflowTypesByHandler.set(definition.handler, name);
+  callbacks.dispatchEvent(new WorkflowDefinitionRegisteredEvent(name));
 }
 function hasNonNullObjectField(value, key) {
   if (!(key in value))

package/dist/core/engine/schedule-jitter.d.ts

@@ -0,0 +1,5 @@
+import type { ScheduleState } from '../types.ts';
+type JitteredSchedule = Pick<ScheduleState, 'id' | 'jitterMs'>;
+export declare function computeScheduleJitterOffset(scheduleId: string, nominalFireAt: number, jitterMs: number | undefined): number;
+export declare function resolveEffectiveScheduleFireAt(schedule: JitteredSchedule, nominalFireAt: number): number;
+export {};

package/dist/core/engine/schedule-jitter.js

@@ -0,0 +1,10 @@
+import { hashString } from "../../runtime/portable.js";
+export function computeScheduleJitterOffset(scheduleId, nominalFireAt, jitterMs) {
+  if (jitterMs === void 0 || jitterMs <= 0)
+    return 0;
+  const hash = BigInt(`0x${hashString(`${scheduleId}\x00${nominalFireAt}`)}`);
+  return Number(hash % BigInt(jitterMs));
+}
+export function resolveEffectiveScheduleFireAt(schedule, nominalFireAt) {
+  return nominalFireAt + computeScheduleJitterOffset(schedule.id, nominalFireAt, schedule.jitterMs);
+}

package/dist/core/engine/schedule-run.d.ts

@@ -0,0 +1,17 @@
+import type { ScheduleState } from '../types.ts';
+import type { EngineInternals } from './internals.ts';
+import type { ScheduleCallbacks } from './schedules.ts';
+/**
+ * Launch one scheduled occurrence's workflow run and emit `schedule:fired`.
+ *
+ * This is the single chokepoint every occurrence launch flows through — initial
+ * cadence ticks, `cancel-running` replacements, and `queue`-drained runs — so it
+ * is the one place that reliably holds the launched `workflowId` across all
+ * overlap policies. The blocked policies (`skip`, and `queue` while a run is
+ * active) never reach here, so they correctly do not fire.
+ *
+ * `occurrence` is the scheduled grid timestamp the run was due, threaded from the
+ * timer loop. It is `undefined` for a `queue`-drained run, whose grid slot was
+ * tracked only as a count and whose original timestamp is therefore gone.
+ */
+export declare function startScheduledRun(internals: EngineInternals, state: ScheduleState, callbacks: Pick<ScheduleCallbacks, 'startWorkflow' | 'failWorkflow' | 'handleCleanupError'>, occurrence?: number): Promise<string>;

package/dist/core/engine/schedule-run.js

@@ -0,0 +1,38 @@
+import { KEYS } from "../../storage/interface.js";
+import { encode } from "../codec.js";
+import { ScheduleFiredEvent } from "../events.js";
+import { unavailableServicesError } from "./lifecycle/recovered-services.js";
+import { EMPTY_STORAGE_VALUE } from "./lifecycle/shared.js";
+export async function startScheduledRun(internals, state, callbacks, occurrence) {
+  const workflowId = crypto.randomUUID(), scheduleRunOperations = state.overlap === "allow" ? [] : [{ type: "put", key: KEYS.scheduleRun(workflowId), value: encode(state.id) }], resolution = await resolveScheduledRunServices(internals, workflowId, state);
+  if (resolution !== null) {
+    scheduleRunOperations.push({ type: "put", key: KEYS.workflowHasServices(workflowId), value: EMPTY_STORAGE_VALUE }, { type: "put", key: KEYS.terminalCleanupNeeded(workflowId), value: EMPTY_STORAGE_VALUE });
+    internals.workflowsNeedingTerminalCleanup.add(workflowId);
+    if (resolution.status === "available")
+      internals.workflowServices.set(workflowId, resolution.services);
+  }
+  await callbacks.startWorkflow(state.workflowType, state.input, { id: workflowId }, scheduleRunOperations);
+  internals.engine.dispatchEvent(new ScheduleFiredEvent(state.id, workflowId, internals.options.getNow(), occurrence));
+  if (resolution !== null && resolution.status === "unavailable") {
+    const unavailableError = unavailableServicesError(workflowId, resolution.reason);
+    try {
+      await callbacks.failWorkflow(workflowId, unavailableError);
+    } catch (error) {
+      callbacks.handleCleanupError("startScheduledRun", error, workflowId);
+    }
+  }
+  return workflowId;
+}
+async function resolveScheduledRunServices(internals, workflowId, state) {
+  const resolver = internals.options.resolveWorkflowServices;
+  if (internals.inlineStrategy === null || !resolver)
+    return null;
+  try {
+    return await resolver({ workflowId, workflowType: state.workflowType, input: state.input });
+  } catch (error) {
+    return {
+      status: "unavailable",
+      reason: error instanceof Error ? error.message : String(error)
+    };
+  }
+}

package/dist/core/engine/schedule-timer.js

@@ -1,6 +1,16 @@
+import { ScheduleMissedFireEvent } from "../events/schedule-events.js";
+import { resolveEffectiveScheduleFireAt } from "./schedule-jitter.js";
 import { collectDueScheduleOccurrences, getNextScheduleOccurrence } from "./schedule-occurrence.js";
 import { loadScheduleState, writeScheduleState } from "./storage-io.js";
-const SCHEDULE_LATE_GRACE_MILLISECONDS = 1000, MAX_SCHEDULE_BACKFILL_OCCURRENCES_PER_TICK = 256;
+const SCHEDULE_LATE_GRACE_MILLISECONDS = 1000, MAX_SCHEDULE_BACKFILL_OCCURRENCES_PER_TICK = 256, MAX_MISSED_FIRE_COUNT_PER_TICK = MAX_SCHEDULE_BACKFILL_OCCURRENCES_PER_TICK;
+
+class ScheduleStatePersistenceError extends Error {
+  scheduleId;
+  constructor(scheduleId, cause) {
+    super(`Failed to persist schedule "${scheduleId}" while processing its timer`, { cause });
+    this.scheduleId = scheduleId;
+  }
+}
 export async function handleScheduleTimer(internals, entry, callbacks) {
   const state = await loadScheduleState(internals, entry.workflowId);
   if (!isCurrentActiveScheduleTimer(state, entry))
@@ -10,28 +20,43 @@ export async function handleScheduleTimer(internals, entry, callbacks) {
     const now = internals.options.getNow(), work = planScheduleTimerWork(state, entry, now);
     if (!work)
       return;
-    nextState = await processScheduleTimerOccurrences(internals, state, work.occurrencesToProcess, now, callbacks, (processedState) => {
-      nextState = processedState;
-    });
-    await writeScheduleState(internals, {
-      ...nextState,
-      updatedAt: now,
-      nextFireAt: resolveNextScheduleFireAt(nextState, work, now)
+    if (work.occurrencesToProcess.length === 0) {
+      nextState = {
+        ...nextState,
+        updatedAt: now,
+        nextFireAt: resolveNextScheduleFireAt(nextState, work, now),
+        ...work.missedOccurrences && {
+          lastMissedFireAt: work.missedOccurrences.lastMissedFireAt,
+          missedFireCount: nextState.missedFireCount + work.missedOccurrences.count
+        }
+      };
+      await persistScheduleTimerState(internals, nextState);
+      if (work.missedOccurrences)
+        internals.engine.dispatchEvent(new ScheduleMissedFireEvent(nextState.id, work.missedOccurrences.count, work.missedOccurrences.windowStart, work.missedOccurrences.windowEnd));
+      return;
+    }
+    nextState = await processScheduleTimerOccurrences(internals, state, work, now, callbacks, {
+      onProcessedState: (processedState) => {
+        nextState = processedState;
+      }
     });
   } catch (error) {
+    if (error instanceof ScheduleStatePersistenceError)
+      throw error;
     await pauseScheduleAfterTimerFailure(internals, nextState, error);
   }
 }
 function isCurrentActiveScheduleTimer(state, entry) {
-  return state?.status === "active" && state.nextFireAt === entry.fireAt;
+  return state?.status === "active" && state.nextFireAt !== null && resolveEffectiveScheduleFireAt(state, state.nextFireAt) === entry.fireAt;
 }
 function planScheduleTimerWork(state, entry, now) {
   const dueOccurrences = collectDueScheduleOccurrences(state, state.nextFireAt, Math.max(now, entry.fireAt), state.backfill ? MAX_SCHEDULE_BACKFILL_OCCURRENCES_PER_TICK : 2);
   if (dueOccurrences.length === 0)
     return null;
-  if (!state.backfill && now - state.nextFireAt > SCHEDULE_LATE_GRACE_MILLISECONDS)
+  if (!state.backfill && now - entry.fireAt > SCHEDULE_LATE_GRACE_MILLISECONDS)
     return {
       dueOccurrences,
+      missedOccurrences: countMissedScheduleOccurrences(state, state.nextFireAt, now),
       occurrencesToProcess: [],
       skipMissedOccurrences: !0
     };
@@ -42,29 +67,68 @@ function planScheduleTimerWork(state, entry, now) {
     skipMissedOccurrences: !1
   };
 }
+function countMissedScheduleOccurrences(state, firstDueAt, throughTimestamp) {
+  let count = 0, occurrence = firstDueAt, lastMissedFireAt = firstDueAt;
+  while (occurrence <= throughTimestamp && count < MAX_MISSED_FIRE_COUNT_PER_TICK) {
+    count += 1;
+    lastMissedFireAt = occurrence;
+    const nextOccurrence = getNextScheduleOccurrence(state, occurrence);
+    if (nextOccurrence <= occurrence)
+      throw Error(`Schedule "${state.id}" produced a non-advancing occurrence`);
+    occurrence = nextOccurrence;
+  }
+  const cappedBeforeThroughTimestamp = occurrence <= throughTimestamp;
+  return {
+    count,
+    lastMissedFireAt,
+    windowStart: firstDueAt,
+    windowEnd: cappedBeforeThroughTimestamp ? lastMissedFireAt : throughTimestamp
+  };
+}
 function resolveNextScheduleFireAt(state, work, now) {
   if (work.skipMissedOccurrences)
-    return getNextScheduleOccurrence(state, now);
+    return getNextScheduleOccurrence(state, work.missedOccurrences?.lastMissedFireAt ?? now);
   const anchorOccurrence = work.occurrencesToProcess.at(-1) ?? work.dueOccurrences.at(-1);
   return getNextScheduleOccurrence(state, anchorOccurrence);
 }
-async function processScheduleTimerOccurrences(internals, state, occurrencesToProcess, now, callbacks, onProcessedState) {
+async function processScheduleTimerOccurrences(internals, state, work, now, callbacks, options) {
   let nextState = state;
-  for (const occurrence of occurrencesToProcess) {
+  for (const [index, occurrence] of work.occurrencesToProcess.entries()) {
+    const finalOccurrence = index === work.occurrencesToProcess.length - 1;
     nextState = {
-      ...await callbacks.applyScheduleOccurrence(nextState),
+      ...await callbacks.applyScheduleOccurrence(nextState, occurrence),
       lastFireAt: occurrence,
       updatedAt: now
     };
-    onProcessedState(nextState);
-    await writeScheduleState(internals, nextState, { includeTimer: !1 });
+    if (finalOccurrence)
+      nextState = {
+        ...nextState,
+        nextFireAt: resolveNextScheduleFireAt(nextState, work, now)
+      };
+    options.onProcessedState(nextState);
+    await persistScheduleTimerState(internals, nextState, { includeTimer: finalOccurrence });
     if (state.backfill && nextState.currentWorkflowId !== void 0) {
       nextState = await callbacks.settleBackfillScheduleState(nextState);
-      onProcessedState(nextState);
+      if (finalOccurrence) {
+        nextState = {
+          ...nextState,
+          updatedAt: now,
+          nextFireAt: resolveNextScheduleFireAt(nextState, work, now)
+        };
+        await persistScheduleTimerState(internals, nextState);
+      }
+      options.onProcessedState(nextState);
     }
   }
   return nextState;
 }
+async function persistScheduleTimerState(internals, state, options) {
+  try {
+    await writeScheduleState(internals, state, options);
+  } catch (error) {
+    throw new ScheduleStatePersistenceError(state.id, error);
+  }
+}
 async function pauseScheduleAfterTimerFailure(internals, state, error) {
   const errorNow = internals.options.getNow(), pausedState = {
     ...state,

package/dist/core/engine/schedules.d.ts

@@ -2,6 +2,7 @@ import type { BatchOperation } from '../../storage/interface.ts';
 import type { PaginatedResult, ScheduleFilter, ScheduleOptions, ScheduleSpec, ScheduleState, ScheduleSummary, WorkflowState } from '../types.ts';
 import type { EngineInternals } from './internals.ts';
 import { ScheduleHandle } from './schedule-handle.ts';
+export { startScheduledRun } from './schedule-run.ts';
 export { handleScheduleTimer } from './schedule-timer.ts';
 export type RefreshedScheduleState = {
     state: ScheduleState;
@@ -15,20 +16,22 @@ export type ScheduleCallbacks = {
     cancelWorkflow: (workflowId: string) => Promise<void>;
     getWorkflowResult: (workflowId: string) => Promise<unknown>;
     refreshScheduledWorkflowState: (state: ScheduleState) => Promise<RefreshedScheduleState>;
-    startScheduledRun: (state: ScheduleState) => Promise<string>;
-    applyScheduleOccurrence: (state: ScheduleState) => Promise<ScheduleState>;
+    startScheduledRun: (state: ScheduleState, occurrence?: number) => Promise<string>;
+    applyScheduleOccurrence: (state: ScheduleState, occurrence?: number) => Promise<ScheduleState>;
     settleBackfillScheduleState: (state: ScheduleState) => Promise<ScheduleState>;
     flushQueuedInlineWorkflowStartsDirectly: () => Promise<void>;
+    failWorkflow: (workflowId: string, error: Error) => Promise<void>;
+    handleCleanupError: (source: string, error: unknown, workflowId: string) => void;
 };
 export declare function schedule(internals: EngineInternals, type: string, input: unknown, spec: string | ScheduleSpec, options?: ScheduleOptions): Promise<ScheduleHandle>;
 export declare function listSchedules(internals: EngineInternals, filter?: ScheduleFilter): Promise<PaginatedResult<ScheduleSummary>>;
+export declare function recoverOrphanedScheduleTimers(internals: EngineInternals): Promise<void>;
 export declare function toScheduleSummary(state: ScheduleState): ScheduleSummary;
 export declare function pauseSchedule(internals: EngineInternals, scheduleId: string): Promise<void>;
 export declare function resumeSchedule(internals: EngineInternals, scheduleId: string): Promise<void>;
 export declare function cancelSchedule(internals: EngineInternals, scheduleId: string): Promise<void>;
 export declare function updateSchedule(internals: EngineInternals, scheduleId: string, newSpec: string | ScheduleSpec): Promise<void>;
 export declare function refreshScheduledWorkflowState(internals: EngineInternals, state: ScheduleState, callbacks: Pick<ScheduleCallbacks, 'loadWorkflowState'>): Promise<RefreshedScheduleState>;
-export declare function startScheduledRun(_internals: EngineInternals, state: ScheduleState, callbacks: Pick<ScheduleCallbacks, 'startWorkflow'>): Promise<string>;
-export declare function applyScheduleOccurrence(_internals: EngineInternals, state: ScheduleState, callbacks: ScheduleCallbacks): Promise<ScheduleState>;
+export declare function applyScheduleOccurrence(_internals: EngineInternals, state: ScheduleState, callbacks: ScheduleCallbacks, occurrence?: number): Promise<ScheduleState>;
 export declare function settleBackfillScheduleState(internals: EngineInternals, state: ScheduleState, callbacks: Pick<ScheduleCallbacks, 'flushQueuedInlineWorkflowStartsDirectly' | 'refreshScheduledWorkflowState'>): Promise<ScheduleState>;
 export declare function handleScheduledWorkflowTerminal(internals: EngineInternals, workflowId: string, callbacks: ScheduleCallbacks): Promise<void>;