@abloatai/humans 0.37.0

package/src/local/transactions/mutations/pendingDrain.ts

@@ -0,0 +1,160 @@
+import type { RuntimeContext } from '../../RuntimeContext.js';
+import type { QueuedMutation } from './commitPayload.js';
+import type { MutationStore } from './MutationStore.js';
+import type { OptimisticUpdateEntry } from './localMutation.js';
+import type { MutationCommitResult } from '@abloatai/transaction/wire/commit';
+import type { DurableCommitEnvelope } from '@abloatai/transaction/transactions/settlement/commitEnvelope';
+import { applyWriteOptions, TX_TYPE_TO_MUTATION_OP } from './commitPayload.js';
+
+export interface PendingDrainContext {
+  readonly runtime: RuntimeContext;
+  readonly config: { deltaConfirmationTimeout: number };
+  readonly store: MutationStore;
+  executionQueue: QueuedMutation[];
+  readonly optimisticUpdates: Map<string, OptimisticUpdateEntry>;
+  readonly assertDurableReplayOpen: () => void;
+  readonly processCommitLane: () => Promise<void>;
+  readonly takePendingDrainBatch: (pending: QueuedMutation[]) => QueuedMutation[];
+  readonly ensureCommitEnvelope: (batch: QueuedMutation[]) => string;
+  readonly ensureDerivedFields: (transaction: QueuedMutation) => void;
+  readonly sourceMutationIdsFor: (batch: readonly QueuedMutation[]) => string[];
+  readonly sealDurableCommit: (input: Parameters<typeof import('./commitTransport.js').sealDurableCommit>[1]) => Promise<DurableCommitEnvelope>;
+  readonly assertEnvelopeInsideReplayWindow: (envelope: DurableCommitEnvelope) => void;
+  readonly parseMutationCommitResult: (value: Awaited<ReturnType<import('../../interfaces/index.js').MutationExecutor['commit']>>) => MutationCommitResult;
+  readonly dispatchCommitBounded: (...args: Parameters<import('../../interfaces/index.js').MutationExecutor['commit']>) => ReturnType<import('../../interfaces/index.js').MutationExecutor['commit']>;
+  readonly persistDurableCommitAcceptance: (envelope: DurableCommitEnvelope, result: MutationCommitResult) => Promise<DurableCommitEnvelope>;
+  readonly removeDurableCommit: (idempotencyKey: string) => Promise<void>;
+  readonly scheduleReplicationLagTimeout: (transactionId: string, clientTxId?: string, correlationId?: string) => void;
+  readonly scheduleDeltaConfirmationTimeout: (transaction: QueuedMutation, timeoutMs: number) => void;
+  readonly enqueue: (transaction: QueuedMutation) => void;
+  readonly recentDeltaCorrelations: Map<string, number>;
+  readonly emit: (event: string, payload: object) => boolean;
+}
+
+export async function drainPendingSettlements(ctx: PendingDrainContext): Promise<void> {
+    ctx.assertDurableReplayOpen();
+    // Kick the commit lane too: atomic envelopes from `commits.create()` may
+    // have been left at the head of the lane while the connection was down.
+    // Fire-and-forget; processCommitLane serializes itself.
+    void ctx.processCommitLane();
+
+    // Collect pending transactions in created order
+    const pending = ctx.store.getByStatus('pending').sort((a, b) => a.createdAt - b.createdAt);
+    if (pending.length === 0) return;
+    const pendingIds = new Set(pending.map((tx) => tx.id));
+    // These rows may already be waiting behind the normal batch timer. The
+    // reconnect fast path takes ownership of them for this attempt so the same
+    // transaction cannot dispatch concurrently through both paths.
+    ctx.executionQueue = ctx.executionQueue.filter(
+      (tx) => !pendingIds.has(tx.id),
+    );
+
+    const remaining = [...pending];
+    while (remaining.length > 0) {
+      const batch = ctx.takePendingDrainBatch(remaining);
+      if (batch.length === 0) break;
+      const batchIds = new Set(batch.map((tx) => tx.id));
+      const nextRemaining = remaining.filter((tx) => !batchIds.has(tx.id));
+
+      try {
+        const idempotencyKey = ctx.ensureCommitEnvelope(batch);
+        const projectedOperations = batch.map((tx) => {
+          ctx.ensureDerivedFields(tx);
+          return applyWriteOptions(
+            {
+              type: TX_TYPE_TO_MUTATION_OP[tx.type],
+              model: tx.modelKey,
+              id: tx.modelId,
+              input: tx.type === 'create' || tx.type === 'update' ? tx.data || {} : undefined,
+              transactionId: tx.id,
+            },
+            tx,
+          );
+        });
+        const durableEnvelope = await ctx.sealDurableCommit({
+          idempotencyKey,
+          origin: 'model_batch',
+          operations: projectedOperations,
+          sourceMutationIds: ctx.sourceMutationIdsFor(batch),
+          createdAt: Math.min(...batch.map((transaction) => transaction.createdAt)),
+          sealedAt: batch[0]?.commitEnvelope?.sealedAt ?? Date.now(),
+          sequence: batch[0]?.commitEnvelope?.sequence,
+        });
+        ctx.assertEnvelopeInsideReplayWindow(durableEnvelope);
+        const result = ctx.parseMutationCommitResult(
+          await ctx.dispatchCommitBounded(durableEnvelope.operations, {
+            idempotencyKey,
+          }),
+        );
+        await ctx.persistDurableCommitAcceptance(durableEnvelope, result);
+        if (result.status === 'queued') {
+          // Reconnect flushes use the same accepted-vs-confirmed contract as
+          // the normal lane. A queued source receipt retains the envelope and
+          // waits for exact correlation; it is never promoted by the reconnect
+          // shortcut itself.
+          for (const tx of batch) {
+            tx.requiresCorrelatedDelta = true;
+            tx.syncIdNeededForCompletion = undefined;
+            tx.correlationId = result.correlationId;
+            const echoSyncId = result.correlationId
+              ? ctx.recentDeltaCorrelations.get(result.correlationId)
+              : undefined;
+            if (echoSyncId !== undefined) {
+              ctx.store.updateStatus(tx.id, 'completed');
+              ctx.emit('transaction:completed', tx);
+              ctx.emit(`transaction:completed:${tx.id}`, tx);
+              ctx.optimisticUpdates.delete(tx.id);
+              continue;
+            }
+            ctx.store.updateStatus(tx.id, 'awaiting_delta');
+            ctx.scheduleReplicationLagTimeout(
+              tx.id,
+              idempotencyKey,
+              result.correlationId,
+            );
+            ctx.scheduleDeltaConfirmationTimeout(
+              tx,
+              ctx.config.deltaConfirmationTimeout,
+            );
+          }
+          if (batch.every((tx) => tx.status === 'completed')) {
+            await ctx.removeDurableCommit(idempotencyKey);
+          }
+        } else {
+          await ctx.removeDurableCommit(idempotencyKey);
+          // Mark this request envelope as completed before moving to the next.
+          for (const tx of batch) {
+            ctx.store.updateStatus(tx.id, 'completed');
+            ctx.emit('transaction:completed', tx);
+            ctx.emit(`transaction:completed:${tx.id}`, tx);
+            ctx.optimisticUpdates.delete(tx.id);
+          }
+        }
+        ctx.runtime.logger.debug('txn:commit', 0, {
+          count: batch.length,
+          lastSyncId: result.lastSyncId,
+        });
+        remaining.splice(0, remaining.length, ...nextRemaining);
+      } catch (err) {
+        // If one request fails, hand it and every later request back to the
+        // normal lane. Their envelopes stay attached for safe retry.
+        const networkUnavailable = !ctx.runtime.onlineStatus.isOnline();
+        const isNetworkError =
+          err instanceof Error &&
+          (err.message.includes('Failed to fetch') ||
+            err.message.includes('Network request failed') ||
+            err.message.includes('NetworkError'));
+
+        if (!networkUnavailable || !isNetworkError) {
+          ctx.runtime.observability.breadcrumb('Batch flush fallback failed', 'sync.transaction', 'warning', {
+            error: err instanceof Error ? err.message : String(err),
+          });
+        }
+
+        for (const tx of [...batch, ...nextRemaining]) {
+          ctx.enqueue(tx);
+        }
+        return;
+      }
+    }
+}

package/src/local/transactions/mutations/processingScheduler.ts

@@ -0,0 +1,35 @@
+export interface ProcessingSchedulerContext {
+  readonly processScheduled: boolean;
+  readonly setProcessScheduled: (value: boolean) => void;
+  readonly processTimer: ReturnType<typeof setTimeout> | undefined;
+  readonly setProcessTimer: (timer: ReturnType<typeof setTimeout> | undefined) => void;
+  readonly executingCount: number;
+  readonly maxExecutingTransactions: number;
+  readonly batchDelay: number;
+  readonly processBatch: () => void;
+  readonly logger: { debug: (message: string, fields?: Record<string, number>) => void };
+}
+
+export function scheduleProcessing(ctx: ProcessingSchedulerContext, immediate = false): void {
+  if (ctx.processScheduled) return;
+  if (ctx.executingCount >= ctx.maxExecutingTransactions) {
+    ctx.logger.debug('[MutationQueue] Backpressure: delaying batch, too many executing', {
+      executingCount: ctx.executingCount, max: ctx.maxExecutingTransactions,
+    });
+    return;
+  }
+  ctx.setProcessScheduled(true);
+  const run = () => {
+    ctx.setProcessScheduled(false);
+    ctx.processBatch();
+  };
+  if (immediate || ctx.batchDelay <= 0) {
+    (typeof queueMicrotask === 'function' ? queueMicrotask : (callback: () => void) => Promise.resolve().then(callback))(run);
+    return;
+  }
+  const timer = setTimeout(() => {
+    ctx.setProcessTimer(undefined);
+    run();
+  }, Math.max(0, ctx.batchDelay));
+  ctx.setProcessTimer(timer);
+}

package/src/local/transactions/mutations/queueCoalescing.ts

@@ -0,0 +1,45 @@
+import type { QueuedMutation } from './commitPayload.js';
+import { mergeUpdateData } from './coalesceRules.js';
+import { hasCommitCoalescingBarrier } from './commitPayload.js';
+
+export interface QueueCoalescingContext {
+  readonly executionQueue: QueuedMutation[];
+  readonly inFlightByModel: Set<string>;
+  readonly pendingMergeByModel: Map<string, { data: QueuedMutation['data']; sourceMutationIds: string[] }>;
+  readonly ensureDerivedFields: (transaction: QueuedMutation) => void;
+  readonly scheduleProcessing: (immediate: boolean) => void;
+  readonly storeRemove: (transactionId: string) => void;
+}
+
+export function enqueueTransaction(ctx: QueueCoalescingContext, transaction: QueuedMutation): void {
+  ctx.ensureDerivedFields(transaction);
+  const modelKey = `${transaction.modelName}:${transaction.modelId}`;
+  if (transaction.type === 'update' && transaction.attempts === 0 && !transaction.commitEnvelope) {
+    const preserveWatermark = hasCommitCoalescingBarrier(transaction.writeOptions);
+    if (!preserveWatermark && ctx.inFlightByModel.has(modelKey)) {
+      const previous = ctx.pendingMergeByModel.get(modelKey);
+      const merged = mergeUpdateData(previous?.data ?? {}, transaction.data || {}, transaction.modelName);
+      ctx.pendingMergeByModel.set(modelKey, {
+        data: merged,
+        sourceMutationIds: [...new Set([...(previous?.sourceMutationIds ?? []), ...(transaction.sourceMutationIds ?? [])])],
+      });
+      ctx.storeRemove(transaction.id);
+      return;
+    }
+    const pendingInQueue = ctx.executionQueue.find((candidate) =>
+      candidate.id !== transaction.id && candidate.type === 'update' &&
+      candidate.modelId === transaction.modelId && candidate.modelName === transaction.modelName &&
+      !hasCommitCoalescingBarrier(candidate.writeOptions));
+    if (!preserveWatermark && pendingInQueue) {
+      pendingInQueue.data = mergeUpdateData(pendingInQueue.data || {}, transaction.data || {}, transaction.modelName);
+      pendingInQueue.sourceMutationIds = [...new Set([
+        ...(pendingInQueue.sourceMutationIds ?? []), ...(transaction.sourceMutationIds ?? []),
+      ])];
+      ctx.storeRemove(transaction.id);
+      return;
+    }
+  }
+  if (transaction.priority === 'high') ctx.executionQueue.unshift(transaction);
+  else ctx.executionQueue.push(transaction);
+  ctx.scheduleProcessing(transaction.priority === 'high');
+}

package/src/local/transactions/mutations/replayValidation.ts

@@ -0,0 +1,192 @@
+/**
+ * The validation boundary for replaying persisted transactions after a restart.
+ *
+ * Rows read back from the on-disk transaction store may have been written by an
+ * earlier run — possibly by an older version of this package, possibly
+ * corrupted. Rather than trust them, the schemas here validate exactly the
+ * fields the settlement queue and the local journal restore read during
+ * replay. A row that fails to parse is dropped and reported, never replayed as
+ * a malformed commit.
+ *
+ * The same store also holds two kinds of rows that are not replayable
+ * transactions — the legacy local mutation journal (`type: 'queue'`) and delta-await
+ * markers (`type: 'awaiting_delta'`), each owned by another part of the client.
+ * {@link isNonReplayablePersistedRow} recognizes them so they are skipped
+ * quietly rather than flagged as corruption.
+ */
+
+import { z } from 'zod';
+import type { QueuedMutation } from './commitPayload.js';
+import { computePriorityScore, normalizeModelKey } from './commitPayload.js';
+import { globalRuntime } from '../../context.js';
+import type { RuntimeContext } from '../../RuntimeContext.js';
+import {
+  commitEnvelopeMemberSchema,
+  commitOutboxScopeSchema,
+} from '@abloatai/transaction/transactions/settlement/commitEnvelope';
+import { onStaleModeSchema } from '@abloatai/transaction/coordination/schema';
+
+/** The subset of a write's options that is stored with each transaction or queued mutation. */
+const persistedWriteOptionsSchema = z
+  .object({
+    readAt: z.number().nullable().optional(),
+    onStale: onStaleModeSchema.nullable().optional(),
+    idempotencyKey: z.string().optional(),
+    label: z.string().optional(),
+    // Aligned with the `WriteOptions` type: a claimed write persisted locally
+    // must replay carrying its fencing token (Option B), or a queued write that
+    // survives a reload would lose its fence and could land as a stale blind
+    // write. Declared (not just loose-passthrough) so it is validated as a
+    // number on rehydration.
+    fenceToken: z.number().nullable().optional(),
+  })
+  .loose();
+
+/**
+ * The shape of a persisted transaction that can be replayed: the fields the
+ * transaction queue reads when it re-enqueues the row — its id, operation type,
+ * model addressing, payload, and identity context. Any remaining bookkeeping is
+ * filled in with defaults when the row is rehydrated by
+ * {@link deserializePersistedTransaction}.
+ */
+export const persistedTransactionSchema = z
+  .object({
+    id: z.string().min(1),
+    type: z.enum(['create', 'update', 'delete', 'archive', 'unarchive']),
+    modelName: z.string().min(1),
+    modelId: z.string().min(1),
+    modelKey: z.string().min(1).optional(),
+    data: z.record(z.string(), z.unknown()).optional(),
+    previousData: z.record(z.string(), z.unknown()).nullable().optional(),
+    context: z.object({
+      userId: z.string().min(1),
+      organizationId: z.string().min(1),
+      role: z.string().optional(),
+      teamIds: z.array(z.string()).optional(),
+    }),
+    createdAt: z.number().optional(),
+    batchId: z.string().optional(),
+    commitEnvelope: commitEnvelopeMemberSchema.optional(),
+    sourceMutationIds: z.array(z.string().min(1)).optional(),
+    writeOptions: persistedWriteOptionsSchema.optional(),
+    localOnly: z.boolean().optional(),
+  })
+  .loose();
+
+export type PersistedReplayableTransaction = z.infer<typeof persistedTransactionSchema>;
+
+/** The `type` values of stored rows that belong to other parts of the client and are not replayable transactions. */
+const NON_REPLAYABLE_TYPES = new Set([
+  'queue',
+  'awaiting_delta',
+  'pending_mutation',
+  'commit_envelope',
+  'http_commit_envelope',
+]);
+
+/**
+ * Reports whether a stored row is one of the non-transaction kinds, so callers
+ * skip it instead of treating it as a corrupt transaction.
+ */
+export function isNonReplayablePersistedRow(row: unknown): boolean {
+  return (
+    typeof row === 'object' &&
+    row !== null &&
+    typeof (row as { type?: unknown }).type === 'string' &&
+    NON_REPLAYABLE_TYPES.has((row as { type: string }).type)
+  );
+}
+
+/**
+ * Validates one stored row and rehydrates it into a {@link QueuedMutation} ready
+ * to replay, or returns `null` when the row fails validation. Bookkeeping
+ * fields the stored row lacks — status, attempts, priority, timestamp — are
+ * re-derived the same way a freshly staged transaction derives them.
+ */
+export function deserializePersistedTransaction(
+  row: unknown,
+  runtime: RuntimeContext = globalRuntime,
+): QueuedMutation | null {
+  const parsed = persistedTransactionSchema.safeParse(row);
+  if (!parsed.success) return null;
+  const tx = parsed.data;
+  return {
+    id: tx.id,
+    type: tx.type,
+    modelName: tx.modelName,
+    modelId: tx.modelId,
+    modelKey: tx.modelKey ?? normalizeModelKey(tx.modelName),
+    ...(tx.data !== undefined ? { data: tx.data } : {}),
+    ...(tx.previousData !== undefined ? { previousData: tx.previousData } : {}),
+    context: tx.context,
+    status: 'pending',
+    createdAt: tx.createdAt ?? Date.now(),
+    attempts: 0,
+    priority: 'normal',
+    priorityScore: computePriorityScore(tx.type, tx.modelName, runtime),
+    ...(tx.batchId !== undefined ? { batchId: tx.batchId } : {}),
+    ...(tx.commitEnvelope !== undefined
+      ? { commitEnvelope: tx.commitEnvelope }
+      : {}),
+    ...(tx.sourceMutationIds !== undefined
+      ? { sourceMutationIds: tx.sourceMutationIds }
+      : {}),
+    ...(tx.writeOptions !== undefined ? { writeOptions: tx.writeOptions } : {}),
+    ...(tx.localOnly !== undefined ? { localOnly: tx.localOnly } : {}),
+  };
+}
+
+/**
+ * The shape of one entry in the persisted legacy local mutation journal — an item of
+ * the `'queue'` row's `mutations` array, carrying the fields read when the
+ * queue is restored on reconnect.
+ */
+export const persistedMutationSchema = z
+  .object({
+    mutationId: z.string().min(1).optional(),
+    type: z.enum(['create', 'update', 'delete', 'archive']),
+    modelData: z.record(z.string(), z.unknown()),
+    modelName: z.string().min(1),
+    timestamp: z.string(),
+    capturedChanges: z.record(z.string(), z.unknown()).optional(),
+    writeOptions: persistedWriteOptionsSchema.optional(),
+  })
+  .loose();
+
+export type PersistedQueuedMutation = z.infer<typeof persistedMutationSchema>;
+
+export const PENDING_MUTATION_RECORD_PREFIX = 'pending-mutation:';
+
+/**
+ * Stay one hour inside the server's 24-hour idempotency retention window.
+ * A journaled write older than this can no longer be deduplicated by the
+ * server, so restore holds it for review instead of replaying it.
+ */
+export const PENDING_MUTATION_REPLAY_WINDOW_MS = 23 * 60 * 60 * 1000;
+
+const pendingMutationRecordBaseShape = {
+  id: z.string().startsWith(PENDING_MUTATION_RECORD_PREFIX),
+  type: z.literal('pending_mutation'),
+  mutation: persistedMutationSchema.extend({
+    mutationId: z.string().min(1),
+  }),
+  timestamp: z.number().int().nonnegative(),
+} as const;
+
+/** Scope-less records written by the first aggregate-journal release. */
+export const legacyPendingMutationRecordSchema = z.strictObject({
+  ...pendingMutationRecordBaseShape,
+  storageVersion: z.literal(1),
+});
+
+export const pendingMutationRecordSchema = z.strictObject({
+  ...pendingMutationRecordBaseShape,
+  storageVersion: z.literal(2),
+  scope: commitOutboxScopeSchema,
+});
+
+export type PendingMutationRecord = z.infer<typeof pendingMutationRecordSchema>;
+
+export function pendingMutationRecordId(mutationId: string): string {
+  return `${PENDING_MUTATION_RECORD_PREFIX}${mutationId}`;
+}

package/src/local/transactions/reconnectDrain.ts

@@ -0,0 +1,24 @@
+/**
+ * Human-side reconnect adapter.
+ *
+ * The settlement callback owns ordering and receipts. This adapter owns the
+ * local client's reconnect concurrency: repeated online/connect signals join
+ * the same drain instead of starting competing local recovery passes.
+ */
+export interface ReconnectDrain {
+  drain(run: () => Promise<void>): Promise<void>;
+}
+
+export function createReconnectDrain(): ReconnectDrain {
+  let inFlight: Promise<void> | null = null;
+
+  return {
+    drain(run) {
+      if (inFlight) return inFlight;
+      inFlight = run().finally(() => {
+        inFlight = null;
+      });
+      return inFlight;
+    },
+  };
+}