@lostgradient/weft 0.2.1 → 0.4.0

package/dist/core/engine/activity-heartbeat-tracking.d.ts

@@ -0,0 +1,64 @@
+import type { ActivityContext } from '../types.ts';
+import type { EngineInternals } from './internals.ts';
+/**
+ * Per-step heartbeat tracking for {@link ActivityContext.lastHeartbeatDetails}.
+ *
+ * The store is keyed `workflowId -> step` (not `workflowId` alone) so a retry of
+ * an activity step reads the heartbeat ITS prior attempt recorded — and so a
+ * later step's first attempt never inherits an earlier step's heartbeat, and
+ * concurrent `ctx.all` activities never clobber one another. The step is stable
+ * across retry attempts (assigned once at `stepIndex++`). Held only in engine
+ * memory and cleared by workflowId (the outer key) on terminal cleanup and purge.
+ */
+/** Record the heartbeat the current attempt of a step sent. */
+export declare function recordLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number, details: unknown): void;
+/** Read the heartbeat a prior attempt of this step recorded, or `undefined`. */
+export declare function readLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number): unknown;
+/**
+ * Drop the heartbeat tracked for a single step once that step has completed
+ * successfully — after inline verify, and (on the idempotency path) after the
+ * reconciliation write, so a verify-rejection retry still reads its prior
+ * attempt's heartbeat. The plain path has no reconciliation write; both paths
+ * call this only at their successful return. Removes the inner-map entry and
+ * drops the outer workflow entry when its last step clears, so a long-lived
+ * workflow doesn't accumulate stale per-step heartbeats. Clearing late is
+ * harmless (bounded by terminal cleanup); clearing early would strip the
+ * resumable-batch heartbeat a retry depends on, so this is only ever called from
+ * the non-speculative success path.
+ */
+export declare function clearLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number): void;
+/**
+ * #493: in development mode, emit a COARSE warning when an inline activity RETRY
+ * (`attempt > 1`) starts with no `lastHeartbeatDetails` — the closest runtime
+ * proxy for the resumable-batch footgun. We cannot distinguish "the prior attempt
+ * never heartbeated" (benign) from "the heartbeat was discarded by a process
+ * restart" (the dangerous case): the distinguishing fact is the wiped in-memory
+ * state, gone by definition. So the message names BOTH possibilities and never
+ * claims a restart happened. The gates (development, inline, attempt, missing
+ * heartbeat) are read straight off the code below; two facts are NOT derivable
+ * from it:
+ *
+ * - It fires on the speculative path too. A speculative `attempt > 1` that later
+ *   rolls back still ran twice and still lost any resume payload, so the footgun
+ *   is real regardless — letting it warn is the deliberate, honest choice.
+ * - It pairs with {@link clearLastHeartbeatForStep} without interference: the
+ *   clear fires only on SUCCESS (a step that never retries) and this reads
+ *   `undefined` only on a RETRY, and a cleared step is a completed step that by
+ *   definition will not be retried.
+ *
+ * The inline gate keys on `!activityWorkerDispatcher` because dispatch is
+ * engine-wide today; the heartbeat cache is only ever written on the inline path,
+ * so whenever a dispatcher exists the cache is structurally empty and the gate is
+ * required to suppress a false warning on every worker retry. A future per-activity
+ * routing model would need this gate to key on the resolved per-dispatch mode
+ * instead.
+ */
+export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals, workflowId: string, step: number, attempt: number): void;
+/**
+ * Build the {@link ActivityContext} handed to an inline activity function. The
+ * signal comes from the per-workflow AbortController (so workflow cancellation
+ * aborts it); `lastHeartbeatDetails` and `heartbeat` are wired to the per-step
+ * tracking above; `completeAsync` is supplied by the caller (it owns the
+ * async-completion token machinery).
+ */
+export declare function buildActivityContext(internals: EngineInternals, workflowId: string, step: number, signal: AbortSignal, completeAsync: () => never): ActivityContext;

package/dist/core/engine/activity-heartbeat-tracking.js

@@ -0,0 +1,42 @@
+import { DevelopmentWarningEvent } from "../events.js";
+export function recordLastHeartbeatForStep(internals, workflowId, step, details) {
+  let byStep = internals.lastHeartbeatDetailsByStep.get(workflowId);
+  if (byStep === void 0) {
+    byStep = new Map;
+    internals.lastHeartbeatDetailsByStep.set(workflowId, byStep);
+  }
+  byStep.set(step, details);
+}
+export function readLastHeartbeatForStep(internals, workflowId, step) {
+  return internals.lastHeartbeatDetailsByStep.get(workflowId)?.get(step);
+}
+export function clearLastHeartbeatForStep(internals, workflowId, step) {
+  const byStep = internals.lastHeartbeatDetailsByStep.get(workflowId);
+  if (byStep === void 0)
+    return;
+  byStep.delete(step);
+  if (byStep.size === 0)
+    internals.lastHeartbeatDetailsByStep.delete(workflowId);
+}
+export function warnIfRetryMissingHeartbeat(internals, workflowId, step, attempt) {
+  if (!internals.options.development)
+    return;
+  if (internals.activityWorkerDispatcher)
+    return;
+  if (attempt <= 1)
+    return;
+  if (readLastHeartbeatForStep(internals, workflowId, step) !== void 0)
+    return;
+  internals.engine.dispatchEvent(new DevelopmentWarningEvent(workflowId, `Activity retry (attempt ${attempt}) at step ${step} has no lastHeartbeatDetails. Either the previous attempt never recorded heartbeat details (it never called heartbeat(), or called it with no details), or the engine process restarted and discarded the in-memory heartbeat (it is not durable). The resumable-batch pattern only resumes across in-process retries; design the activity to restart cleanly when lastHeartbeatDetails is undefined.`, [`step.${step}.lastHeartbeatDetails`]));
+}
+export function buildActivityContext(internals, workflowId, step, signal, completeAsync) {
+  return {
+    signal,
+    lastHeartbeatDetails: readLastHeartbeatForStep(internals, workflowId, step),
+    heartbeat: (details) => {
+      internals.heartbeatDetails.set(workflowId, details);
+      recordLastHeartbeatForStep(internals, workflowId, step, details);
+    },
+    completeAsync
+  };
+}

package/dist/core/engine/activity-per-attempt-timeout.d.ts

@@ -0,0 +1,41 @@
+import type { ContextOperationRequest } from '../context.ts';
+import type { EngineInternals } from './internals.ts';
+type ActivityOperation = Extract<ContextOperationRequest, {
+    type: 'activity';
+}>;
+/**
+ * Resolve the per-attempt `timeout` setup for an activity attempt (#494).
+ *
+ * The cap is INLINE-only: worker-mode per-attempt bounds are governed by
+ * `visibilityTimeout`, and racing the engine's await against a deadline while a
+ * remote worker keeps running would orphan its result and risk double-execution on
+ * the next visibility-timeout re-dispatch. When a cap is configured, a fresh
+ * per-attempt `AbortController` drives cooperative cancellation; its signal is
+ * composited with the workflow-wide signal so the activity's `ctx.signal` aborts on
+ * EITHER workflow cancellation OR a per-attempt timeout — without the timeout
+ * reaching back to cancel the whole workflow (which would also poison the next
+ * retry's signal).
+ */
+export declare function resolvePerAttemptTimeout(internals: EngineInternals, workflowId: string, operation: ActivityOperation): {
+    perAttemptTimeoutMs: number | undefined;
+    attemptAbortController: AbortController | undefined;
+    activitySignal: AbortSignal;
+};
+/**
+ * Bound a single inline activity attempt by its per-attempt `timeout` (#494).
+ *
+ * When `timeoutMs` is set and the attempt does not settle first, the returned
+ * promise rejects with {@link ActivityPerAttemptTimeoutError} and the per-attempt
+ * `AbortController` is aborted with that same error — so the activity's composite
+ * `ctx.signal` flips, letting a cooperating activity stop promptly. Weft cannot
+ * forcibly preempt the activity function: a non-cooperating activity keeps running
+ * in the background until it returns; only the workflow stops awaiting it. The
+ * abandoned invocation's eventual settlement is swallowed so it never surfaces as
+ * an unhandled rejection after the race has moved on. The timed-out rejection
+ * flows through the normal retry path, so a retry policy gets a fresh attempt with
+ * a fresh cap (and a fresh per-attempt controller).
+ *
+ * Returns the invocation unchanged (no timer armed) when no cap is configured.
+ */
+export declare function withPerAttemptTimeout(invocation: unknown, timeoutMs: number | undefined, activityName: string, attempt: number, attemptAbortController: AbortController | undefined): Promise<unknown>;
+export {};

package/dist/core/engine/activity-per-attempt-timeout.js

@@ -0,0 +1,36 @@
+import {
+  ActivityPerAttemptTimeoutError,
+  parsePerAttemptTimeoutMs
+} from "../context/activity-schedule-to-close.js";
+export function resolvePerAttemptTimeout(internals, workflowId, operation) {
+  const workflowAbortController = internals.inlineStrategy?.getAbortController(workflowId), perAttemptTimeoutMs = internals.activityWorkerDispatcher ? void 0 : parsePerAttemptTimeoutMs(operation.options?.timeout), attemptAbortController = perAttemptTimeoutMs === void 0 ? void 0 : new AbortController, activitySignal = composeActivitySignal(workflowAbortController?.signal, attemptAbortController?.signal);
+  return { perAttemptTimeoutMs, attemptAbortController, activitySignal };
+}
+function composeActivitySignal(workflowSignal, attemptSignal) {
+  const signals = [workflowSignal, attemptSignal].filter((signal) => signal !== void 0), [first] = signals;
+  if (first === void 0)
+    return new AbortController().signal;
+  if (signals.length === 1)
+    return first;
+  return AbortSignal.any(signals);
+}
+export async function withPerAttemptTimeout(invocation, timeoutMs, activityName, attempt, attemptAbortController) {
+  if (timeoutMs === void 0)
+    return invocation;
+  const activityPromise = Promise.resolve(invocation);
+  activityPromise.catch(() => {});
+  let timer;
+  const deadline = new Promise((_resolve, reject) => {
+    timer = setTimeout(() => {
+      const error = new ActivityPerAttemptTimeoutError(activityName, attempt, timeoutMs);
+      reject(error);
+      attemptAbortController?.abort(error);
+    }, timeoutMs);
+  });
+  try {
+    return await Promise.race([activityPromise, deadline]);
+  } finally {
+    if (timer !== void 0)
+      clearTimeout(timer);
+  }
+}

package/dist/core/engine/activity-reconciliation.d.ts

@@ -101,6 +101,7 @@ export declare function resolveStartedActivityReconciliationRecord(internals: En
     completedResult: unknown;
 }>;
 export declare function writeActivityReconciliationTransition(storage: Storage, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
+export declare function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals: EngineInternals, workflowId: string, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): void;
 export declare function normalizePreDispatchVerificationResult(result: ActivityVerificationResult): 'not-completed' | 'completed-result-unavailable' | 'indeterminate' | {
     result: unknown;
 };

package/dist/core/engine/activity-reconciliation.js

@@ -6,6 +6,7 @@ import {
 import { decode, encode } from "../codec.js";
 import { assertPayloadWithinLimit } from "../payload-size.js";
 import { WeftError } from "../weft-error.js";
+import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
 
 export class ActivityReconciliationCapabilityError extends WeftError {
   constructor() {
@@ -95,9 +96,19 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
   return { completedResult: normalized.result };
 }
 export async function writeActivityReconciliationTransition(storage, reference, expectedRecord, nextRecord) {
-  if (!await storageConditionalBatch(storage, [{ key: reference.key, expectedValue: encode(expectedRecord) }], [{ type: "put", key: reference.key, value: encode(nextRecord) }]))
+  const sideEffects = buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord);
+  if (!await storageConditionalBatch(storage, sideEffects.conditions, sideEffects.operations))
     throw new ActivityReconciliationConflictError("Activity reconciliation completion lost compare-and-set ownership.");
 }
+export function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals, workflowId, reference, expectedRecord, nextRecord) {
+  stageAtomicWorkflowCommitSideEffects(internals, workflowId, buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord));
+}
+function buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord) {
+  return {
+    conditions: [{ key: reference.key, expectedValue: encode(expectedRecord) }],
+    operations: [{ type: "put", key: reference.key, value: encode(nextRecord) }]
+  };
+}
 export function normalizePreDispatchVerificationResult(result) {
   if (result === "not-completed")
     return "not-completed";

package/dist/core/engine/activity-resolution.d.ts

@@ -0,0 +1,23 @@
+import type { ContextOperationRequest } from '../context.ts';
+import type { EngineInternals } from './internals.ts';
+import type { ActivityFunctionWithMetadata } from './operations-activity.ts';
+type ActivityOperation = Extract<ContextOperationRequest, {
+    type: 'activity';
+}>;
+/**
+ * Resolve the activity callable along with its attached metadata (compensation,
+ * verification). Returns `undefined` when no registry and no `operation.fn`
+ * resolve the name — the metadata path treats a miss as advisory rather than
+ * fatal.
+ */
+export declare function getActivityFunctionWithMetadata(internals: EngineInternals, workflowId: string, operation: ActivityOperation): ActivityFunctionWithMetadata | undefined;
+/**
+ * Resolve the activity function for a given operation. Uses the same
+ * per-workflow-first-then-global ordering as `getActivityFunctionWithMetadata`.
+ * For inline-mode callers that pass an `operation.fn` directly, the registries
+ * are still consulted first so a workflow's locally-scoped activity wins over
+ * the bare callable. Throws `ActivityResolutionError` when neither path
+ * resolves.
+ */
+export declare function resolveActivityFunction(internals: EngineInternals, workflowId: string, operation: ActivityOperation): (...arguments_: unknown[]) => unknown;
+export {};

package/dist/core/engine/activity-resolution.js

@@ -0,0 +1,37 @@
+import { ActivityResolutionError } from "./errors.js";
+function resolveActivityViaRegistries(internals, workflowId, activityName) {
+  const workflowType = internals.workflowTypeByWorkflowId.get(workflowId);
+  if (workflowType !== void 0) {
+    const perWorkflow = internals.activityRegistriesByWorkflow.get(workflowType);
+    if (perWorkflow !== void 0) {
+      const perWorkflowFn = perWorkflow.resolve(activityName);
+      if (perWorkflowFn)
+        return { fn: perWorkflowFn, workflowType };
+    }
+    const globalFn = internals.activityRegistry.resolve(activityName);
+    if (globalFn)
+      return { fn: globalFn, workflowType };
+    return;
+  }
+  const globalFn = internals.activityRegistry.resolve(activityName);
+  if (globalFn)
+    return { fn: globalFn, workflowType: "<unknown>" };
+  return;
+}
+export function getActivityFunctionWithMetadata(internals, workflowId, operation) {
+  const resolved = resolveActivityViaRegistries(internals, workflowId, operation.activityName);
+  if (resolved)
+    return resolved.fn;
+  if (typeof operation.fn === "function")
+    return operation.fn;
+  return;
+}
+export function resolveActivityFunction(internals, workflowId, operation) {
+  const resolved = resolveActivityViaRegistries(internals, workflowId, operation.activityName);
+  if (resolved)
+    return resolved.fn;
+  if (operation.fn)
+    return operation.fn;
+  const workflowType = internals.workflowTypeByWorkflowId.get(workflowId) ?? "<unknown>";
+  throw new ActivityResolutionError(workflowType, operation.activityName);
+}

package/dist/core/engine/aggregate.js

@@ -5,12 +5,16 @@ import {
   MAX_AGGREGATE_DISTINCT_KEYS,
   normalizeAggregateOptions
 } from "../aggregate-validation.js";
-import { decode } from "../codec.js";
 import { normalizeListFilter } from "../list-filter-validation.js";
 import { WeftError } from "../weft-error.js";
 import { normalizeWorkflowTags } from "../workflow-tags.js";
+import { CONSTRAINED_ID_CHUNK_SIZE } from "./candidate-read-batching.js";
 import { resolveListCandidateIds } from "./list-candidate-resolution.js";
-import { matchesListFilter } from "./state-utilities.js";
+import {
+  readSearchAttributesForFilter,
+  readSearchAttributesForStates
+} from "./search-attribute-records.js";
+import { decodeSearchAttributeRecord, matchesListFilter } from "./state-utilities.js";
 import { decodeWorkflowState } from "./validation.js";
 import { MAX_LIST_SCAN_ROWS, WorkflowListScanCapExceededError } from "./workflow-indexes.js";
 import { isTopLevelWorkflowStateKey } from "./workflow-state-stream.js";
@@ -27,19 +31,6 @@ const ATTRIBUTE_VALUE_TO_KEY = (value) => {
     return value.join(",");
   return String(value);
 };
-async function resolveDimensionKey(internals, state, groupBy) {
-  if (groupBy === "status")
-    return state.status;
-  if (groupBy === "type")
-    return state.type;
-  if (groupBy === "failureCategory")
-    return state.failureCategory ?? null;
-  const attributeBytes = await internals.storage.get(KEYS.attribute(state.id));
-  if (!attributeBytes)
-    return null;
-  const attributes = decode(attributeBytes);
-  return ATTRIBUTE_VALUE_TO_KEY(attributes[groupBy.attribute]);
-}
 
 export class UnknownAggregateAttributeError extends WeftError {
   attribute;
@@ -72,8 +63,8 @@ function validateAggregateInputs(internals, filter, options, executionOptions) {
     distinctKeyCap: executionOptions.distinctKeyCap ?? MAX_AGGREGATE_DISTINCT_KEYS
   };
 }
-async function accumulateAggregateState(internals, state, groupBy, distinctKeyCap, accumulator) {
-  const key = await resolveDimensionKey(internals, state, groupBy), current = accumulator.counts.get(key);
+function accumulateAggregateKey(key, distinctKeyCap, accumulator) {
+  const current = accumulator.counts.get(key);
   if (current === void 0) {
     if (accumulator.counts.size >= distinctKeyCap)
       throw new AggregateDistinctKeyCapExceededError(distinctKeyCap);
@@ -82,32 +73,77 @@ async function accumulateAggregateState(internals, state, groupBy, distinctKeyCa
     accumulator.counts.set(key, current + 1);
   accumulator.total += 1;
 }
+async function resolveDimensionKeys(internals, states, groupBy, searchAttributesByWorkflowId = new Map) {
+  if (groupBy === "status")
+    return states.map((state) => state.status);
+  if (groupBy === "type")
+    return states.map((state) => state.type);
+  if (groupBy === "failureCategory")
+    return states.map((state) => state.failureCategory ?? null);
+  return resolveAttributeDimensionKeys(internals, states, groupBy, searchAttributesByWorkflowId);
+}
+async function resolveAttributeDimensionKeys(internals, states, groupBy, searchAttributesByWorkflowId) {
+  const keys = [];
+  for (let start = 0;start < states.length; start += CONSTRAINED_ID_CHUNK_SIZE) {
+    const stateChunk = states.slice(start, start + CONSTRAINED_ID_CHUNK_SIZE), statesNeedingReads = stateChunk.filter((state) => !searchAttributesByWorkflowId.has(state.id)), attributeBytesByWorkflowId = new Map;
+    await Promise.all(statesNeedingReads.map(async (state) => {
+      attributeBytesByWorkflowId.set(state.id, await internals.storage.get(KEYS.attribute(state.id)));
+    }));
+    for (const state of stateChunk) {
+      const attributes = searchAttributesByWorkflowId.has(state.id) ? searchAttributesByWorkflowId.get(state.id) ?? null : decodeSearchAttributeRecord(attributeBytesByWorkflowId.get(state.id) ?? null);
+      if (attributes === null) {
+        keys.push(null);
+        continue;
+      }
+      keys.push(ATTRIBUTE_VALUE_TO_KEY(attributes[groupBy.attribute]));
+    }
+  }
+  return keys;
+}
+async function accumulateAggregateStates(internals, states, groupBy, distinctKeyCap, accumulator, searchAttributesByWorkflowId) {
+  const dimensionKeys = await resolveDimensionKeys(internals, states, groupBy, searchAttributesByWorkflowId);
+  for (const key of dimensionKeys)
+    accumulateAggregateKey(key, distinctKeyCap, accumulator);
+}
 async function accumulateFromConstrainedIds(internals, constrainedIds, normalizedFilter, normalizedTagFilters, groupBy, distinctKeyCap, accumulator) {
-  if (constrainedIds.size > MAX_LIST_SCAN_ROWS)
+  const orderedIds = [...constrainedIds];
+  if (orderedIds.length > MAX_LIST_SCAN_ROWS)
     throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
-  for (const workflowId of constrainedIds) {
-    const stateBytes = await internals.storage.get(KEYS.workflow(workflowId));
-    if (!stateBytes)
-      continue;
-    const state = decodeWorkflowState(stateBytes);
-    if (!matchesListFilter(state, normalizedFilter, constrainedIds, normalizedTagFilters))
-      continue;
-    await accumulateAggregateState(internals, state, groupBy, distinctKeyCap, accumulator);
+  for (let start = 0;start < orderedIds.length; start += CONSTRAINED_ID_CHUNK_SIZE) {
+    const idChunk = orderedIds.slice(start, start + CONSTRAINED_ID_CHUNK_SIZE), stateBytesChunk = await Promise.all(idChunk.map((workflowId) => internals.storage.get(KEYS.workflow(workflowId)))), searchAttributesByWorkflowId = await readSearchAttributesForStates(internals, stateBytesChunk, normalizedFilter), matchingStates = [];
+    for (const stateBytes of stateBytesChunk) {
+      if (!stateBytes)
+        continue;
+      const state = decodeWorkflowState(stateBytes);
+      if (!matchesListFilter(state, normalizedFilter, constrainedIds, normalizedTagFilters, searchAttributesByWorkflowId.get(state.id) ?? null))
+        continue;
+      matchingStates.push(state);
+    }
+    await accumulateAggregateStates(internals, matchingStates, groupBy, distinctKeyCap, accumulator, searchAttributesByWorkflowId);
   }
 }
 async function accumulateFromFullScan(internals, normalizedFilter, normalizedTagFilters, groupBy, distinctKeyCap, accumulator) {
-  let scanned = 0;
+  let scanned = 0, matchingStates = [], matchingSearchAttributesByWorkflowId = new Map;
   for await (const [storageKey, value] of internals.storage.scan("wf:")) {
     if (!isTopLevelWorkflowStateKey(storageKey))
       continue;
     scanned += 1;
     if (scanned > MAX_LIST_SCAN_ROWS)
       throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
-    const state = decodeWorkflowState(value);
-    if (!matchesListFilter(state, normalizedFilter, null, normalizedTagFilters))
+    const state = decodeWorkflowState(value), searchAttributes = await readSearchAttributesForFilter(internals, state.id, normalizedFilter);
+    if (!matchesListFilter(state, normalizedFilter, null, normalizedTagFilters, searchAttributes))
       continue;
-    await accumulateAggregateState(internals, state, groupBy, distinctKeyCap, accumulator);
+    matchingStates.push(state);
+    if (searchAttributes !== null)
+      matchingSearchAttributesByWorkflowId.set(state.id, searchAttributes);
+    if (matchingStates.length === CONSTRAINED_ID_CHUNK_SIZE) {
+      await accumulateAggregateStates(internals, matchingStates, groupBy, distinctKeyCap, accumulator, matchingSearchAttributesByWorkflowId);
+      matchingStates = [];
+      matchingSearchAttributesByWorkflowId = new Map;
+    }
   }
+  if (matchingStates.length > 0)
+    await accumulateAggregateStates(internals, matchingStates, groupBy, distinctKeyCap, accumulator, matchingSearchAttributesByWorkflowId);
 }
 function finalizeAggregateResult(accumulator, requestedLimit) {
   const sortedGroups = [...accumulator.counts.entries()].map(([groupKey, count]) => ({ key: groupKey, count })).toSorted((left, right) => {

package/dist/core/engine/async-activity-completion.d.ts

@@ -22,7 +22,10 @@
  * across replay. After an engine restart, `recoverAll()` replays the workflow
  * from its checkpoint; the deferred activity re-runs, re-defers, and produces
  * the *same* token. A callback that arrives after the crash therefore still
- * resolves the correct activity.
+ * resolves the correct activity. If that callback races `recoverAll()` and
+ * arrives after token recovery but before replay has adopted the workflow
+ * generator, the engine buffers the completion or failure outcome and drains it
+ * when replay reaches the same deterministic token.
  *
  * Internal-only. Imported from `src/core/engine/**` and the `ActivityContext`
  * construction path.
@@ -30,6 +33,12 @@
 import type { OperationOutcome } from '../types.ts';
 import { WeftError } from '../weft-error.ts';
 import type { EngineInternals } from './internals.ts';
+type AsyncActivityResolutionCallbacks = {
+    feedOperationResult: (workflowId: string, outcome: OperationOutcome, originalReason?: {
+        value: unknown;
+    }) => void;
+    finalizeTimeline: (workflowId: string, status: 'completed' | 'failed', output: unknown) => void;
+};
 /**
  * Per-workflow prefix for all async-activity storage keys. Used by cleanup and
  * purge paths that need to sweep every async-activity record for a workflow
@@ -80,6 +89,15 @@ export type PendingAsyncActivity = {
     readonly attempt: number;
     readonly createdAt: number;
 };
+export type PendingAsyncActivityResolution = {
+    readonly token: string;
+    readonly outcome: OperationOutcome;
+    readonly originalReason?: {
+        value: unknown;
+    };
+    readonly timelineStatus: 'completed' | 'failed';
+    readonly timelineOutput: unknown;
+};
 /**
  * Derive the durable, deterministic task token for an async activity.
  *
@@ -106,7 +124,7 @@ export declare function registerPendingAsyncActivity(internals: EngineInternals,
  * out-of-band completion resumes it. Keeps the operation-pipeline catch site a
  * one-liner.
  */
-export declare function parkDeferredAsyncActivity(internals: EngineInternals, deferral: AsyncActivityDeferral, details: Omit<PendingAsyncActivity, 'token' | 'createdAt'>): Promise<never>;
+export declare function parkDeferredAsyncActivity(internals: EngineInternals, deferral: AsyncActivityDeferral, details: Omit<PendingAsyncActivity, 'token' | 'createdAt'>, callbacks: AsyncActivityResolutionCallbacks): Promise<never>;
 /**
  * Reload pending async-activity records from storage into memory. Called by
  * `recoverAll()` so a token minted before a crash is resolvable again — even
@@ -117,7 +135,7 @@ export declare function recoverPendingAsyncActivities(internals: EngineInternals
  * Complete a deferred activity out-of-band with `result`, resuming the parked
  * workflow as though the activity had returned `result` inline.
  */
-export declare function completeAsyncActivity(internals: EngineInternals, token: string, result: unknown, feedOperationResult: (workflowId: string, outcome: OperationOutcome) => void, finalizeTimeline: (workflowId: string, status: 'completed' | 'failed', output: unknown) => void): Promise<void>;
+export declare function completeAsyncActivity(internals: EngineInternals, token: string, result: unknown, callbacks: AsyncActivityResolutionCallbacks): Promise<void>;
 /**
  * Drive a generator that may yield promises — the workflow interceptor's
  * `activity` hook returns such a generator. Forwards rejections into the
@@ -136,6 +154,5 @@ export declare function driveWorkflowInterceptorGenerator(generator: Generator<u
  * that threw — so the workflow's own try/catch and any configured retry policy
  * apply unchanged.
  */
-export declare function failAsyncActivity(internals: EngineInternals, token: string, error: unknown, feedOperationResult: (workflowId: string, outcome: OperationOutcome, originalReason?: {
-    value: unknown;
-}) => void, finalizeTimeline: (workflowId: string, status: 'completed' | 'failed', output: unknown) => void): Promise<void>;
+export declare function failAsyncActivity(internals: EngineInternals, token: string, error: unknown, callbacks: AsyncActivityResolutionCallbacks): Promise<void>;
+export {};

package/dist/core/engine/async-activity-completion.js

@@ -53,7 +53,12 @@ export async function registerPendingAsyncActivity(internals, pending) {
   if (!alreadyRegistered)
     internals.engine.dispatchEvent(new ActivityAsyncPendingEvent(pending.token, pending.operationId, pending.workflowId, pending.activityName, pending.attempt));
 }
-export async function parkDeferredAsyncActivity(internals, deferral, details) {
+export async function parkDeferredAsyncActivity(internals, deferral, details, callbacks) {
+  const queuedResolution = takePendingAsyncActivityResolution(internals, details.workflowId, deferral.token);
+  if (queuedResolution !== void 0) {
+    deliverPendingAsyncActivityResolution(details.workflowId, queuedResolution, callbacks);
+    return new Promise(() => {});
+  }
   await registerPendingAsyncActivity(internals, {
     token: deferral.token,
     createdAt: internals.options.getNow(),
@@ -90,14 +95,52 @@ async function consumePendingAsyncActivity(internals, token) {
   }
   return pending;
 }
-async function resolvePendingAsyncActivity(internals, token, outcome, feedOperationResult, finalizeTimeline) {
-  const pending = await consumePendingAsyncActivity(internals, token), timelineOutput = outcome.status === "completed" ? outcome.value : outcome.error;
-  finalizeTimeline(pending.workflowId, outcome.status, timelineOutput);
-  feedOperationResult(pending.workflowId, outcome);
+function shouldBufferPendingAsyncActivityResolution(internals, workflowId) {
+  return internals.inlineStrategy !== null && !internals.inlineStrategy.hasGenerator(workflowId);
+}
+function queuePendingAsyncActivityResolution(internals, workflowId, resolution) {
+  internals.pendingAsyncActivityResolutions ??= new Map;
+  const queued = internals.pendingAsyncActivityResolutions.get(workflowId) ?? [];
+  queued.push(resolution);
+  internals.pendingAsyncActivityResolutions.set(workflowId, queued);
+}
+function takePendingAsyncActivityResolution(internals, workflowId, token) {
+  internals.pendingAsyncActivityResolutions ??= new Map;
+  const queued = internals.pendingAsyncActivityResolutions.get(workflowId);
+  if (queued === void 0)
+    return;
+  const index = queued.findIndex((resolution) => resolution.token === token);
+  if (index === -1)
+    return;
+  const resolution = queued[index];
+  if (resolution === void 0)
+    return;
+  queued.splice(index, 1);
+  if (queued.length === 0)
+    internals.pendingAsyncActivityResolutions.delete(workflowId);
+  return resolution;
+}
+function deliverPendingAsyncActivityResolution(workflowId, resolution, callbacks) {
+  callbacks.finalizeTimeline(workflowId, resolution.timelineStatus, resolution.timelineOutput);
+  callbacks.feedOperationResult(workflowId, resolution.outcome, resolution.originalReason);
+}
+async function resolvePendingAsyncActivity(internals, token, outcome, callbacks, originalReason) {
+  const pending = await consumePendingAsyncActivity(internals, token), timelineOutput = outcome.status === "completed" ? outcome.value : outcome.error, resolution = {
+    token,
+    outcome,
+    timelineStatus: outcome.status,
+    timelineOutput,
+    ...originalReason !== void 0 ? { originalReason } : {}
+  };
+  if (shouldBufferPendingAsyncActivityResolution(internals, pending.workflowId)) {
+    queuePendingAsyncActivityResolution(internals, pending.workflowId, resolution);
+    return;
+  }
+  deliverPendingAsyncActivityResolution(pending.workflowId, resolution, callbacks);
 }
-export async function completeAsyncActivity(internals, token, result, feedOperationResult, finalizeTimeline) {
+export async function completeAsyncActivity(internals, token, result, callbacks) {
   assertPayloadWithinLimit(result, internals.options.payloadSizePolicy.maxBytes, "activity result");
-  await resolvePendingAsyncActivity(internals, token, { status: "completed", value: result }, feedOperationResult, finalizeTimeline);
+  await resolvePendingAsyncActivity(internals, token, { status: "completed", value: result }, callbacks);
 }
 export async function driveWorkflowInterceptorGenerator(generator) {
   let current = generator.next();
@@ -117,15 +160,13 @@ export async function driveWorkflowInterceptorGenerator(generator) {
   }
   return current.value;
 }
-export async function failAsyncActivity(internals, token, error, feedOperationResult, finalizeTimeline) {
+export async function failAsyncActivity(internals, token, error, callbacks) {
   const message = error instanceof Error ? error.message : String(error), errorName = error instanceof Error ? error.name : void 0;
   assertPayloadWithinLimit({ message, name: errorName }, internals.options.payloadSizePolicy.maxBytes, "activity result");
-  const pending = await consumePendingAsyncActivity(internals, token);
-  finalizeTimeline(pending.workflowId, "failed", message);
-  feedOperationResult(pending.workflowId, {
+  await resolvePendingAsyncActivity(internals, token, {
     status: "failed",
     error: message,
     ...error instanceof Error ? { errorName: error.name } : {},
     failureCategory: "application"
-  }, { value: error });
+  }, callbacks, { value: error });
 }

package/dist/core/engine/attributes-tags.js

@@ -9,7 +9,10 @@ import {
 import { assertWorkflowTagCount, coerceStartWorkflowTags } from "../start-workflow-validation.js";
 import { buildWorkflowTagIndexOperations, normalizeWorkflowTags } from "../workflow-tags.js";
 import { WorkflowNotFoundError } from "./errors.js";
-import { commitWorkflowStateOperations, runSerializedWorkflowStateWrite } from "./storage-io.js";
+import {
+  commitFencedWorkflowStateOperations,
+  runSerializedWorkflowStateWrite
+} from "./storage-io.js";
 import {
   decodeWorkflowState,
   isTerminalWorkflowStatus,
@@ -73,12 +76,14 @@ export async function updateWorkflowState(internals, workflowId, updates, option
       ...updates,
       updatedAt
     }, additionalOperations = options.buildAdditionalOperations?.(state, updatedAt) ?? [];
-    await commitWorkflowStateOperations(internals, state, [
+    await commitFencedWorkflowStateOperations(internals, state, [
       ...buildTerminalWorkflowIndexOperations(state, updated),
       { type: "put", key: KEYS.workflow(workflowId), value: encode(updated) },
       ...buildWorkflowVisibilityIndexTransition(workflowId, state, updated).batchOps,
       ...additionalOperations
-    ]);
+    ], {
+      includePendingAtomicSideEffects: isTerminalWorkflowStatus(updated.status)
+    });
     return {
       previousState: state,
       updatedAt
@@ -172,7 +177,7 @@ async function snapshotMatchingWorkflowIds(internals, filter) {
     if (remainingLimit !== void 0)
       remainingLimit -= 1;
     if (remainingLimit === 0)
-      break;
+      return workflowIds;
   }
   return workflowIds;
 }