@abloatai/humans 0.37.0

package/src/local/transactions/mutations/deltaConfirmation.ts

@@ -0,0 +1,298 @@
+/**
+ * Tracks whether the confirming delta for a write has arrived. It holds the
+ * acknowledgement watermark (through the shared {@link LogPosition}), the
+ * per-transaction confirmation timeouts, and the retry-with-backoff and
+ * reconciliation policy for transactions in the `awaiting_delta` status. It
+ * reaches back to {@link MutationQueue} only through the small
+ * {@link DeltaConfirmationContext} interface, not the queue class itself, so it
+ * has no cyclic dependency and can be tested on its own.
+ */
+
+import { globalRuntime } from '../../context.js';
+import type { RuntimeContext } from '../../RuntimeContext.js';
+import type { LogPositionPort } from '../../logPosition.js';
+import type { QueuedMutation } from './commitPayload.js';
+
+/**
+ * The subset of {@link MutationQueue} that the confirmation tracker needs:
+ * store lookups and status changes, removing optimistic entries once a write
+ * confirms, the queue's event emitter (`transaction:completed`,
+ * `reconciliation:needed`, and so on), a connection check (so timeouts
+ * re-schedule instead of escalating while offline), and the shared client
+ * position (`noteAck` advances the acknowledgement cursor; diagnostics read the
+ * applied cursor).
+ */
+export interface DeltaConfirmationContext {
+  store: {
+    get(id: string): QueuedMutation | undefined;
+    getByStatus(status: QueuedMutation['status']): QueuedMutation[];
+    updateStatus(id: string, status: QueuedMutation['status']): void;
+  };
+  optimisticUpdates: { delete(id: string): boolean };
+  emit(event: string, payload?: unknown): void;
+  isConnected(): boolean;
+  position: LogPositionPort;
+  /** The owning client's runtime. Defaults to the module-global bridge. */
+  runtime?: RuntimeContext;
+}
+
+export class DeltaConfirmationTracker {
+  // Retry configuration for delta confirmation, using exponential backoff.
+  // Maximum retries before requesting a full reconciliation.
+  private static readonly DELTA_MAX_RETRIES = 5;
+  // Upper bound on the backoff timeout.
+  private static readonly DELTA_MAX_TIMEOUT_MS = 120_000;
+
+  // Pending confirmation timeouts for transactions awaiting their delta. On
+  // timeout the tracker retries with backoff rather than rolling back.
+  private deltaConfirmationTimeouts = new Map<string, ReturnType<typeof setTimeout>>();
+
+  // Track retry attempts per transaction for exponential backoff
+  private deltaConfirmationRetries = new Map<string, number>();
+
+  private readonly runtime: RuntimeContext;
+
+  constructor(private readonly ctx: DeltaConfirmationContext) {
+    this.runtime = ctx.runtime ?? globalRuntime;
+  }
+
+  /** Applied-cursor alias, kept so the read sites below stay legible. */
+  private get lastSeenSyncId(): number {
+    return this.ctx.position.applied;
+  }
+
+  private matchesSourceEcho(
+    tx: QueuedMutation,
+    correlationId: string | undefined,
+  ): boolean {
+    return (
+      correlationId !== undefined &&
+      tx.correlationId !== undefined &&
+      correlationId === tx.correlationId
+    );
+  }
+
+  noteAck(lastSyncId: number | undefined): void {
+    this.ctx.position.noteAck(lastSyncId);
+  }
+
+  /**
+   * Confirms every awaiting transaction whose sync-id threshold this delta
+   * meets or exceeds.
+   * @param syncId - The sync id of the received delta.
+   */
+  onDeltaReceived(syncId: number, correlationId?: string): void {
+    // The cursor advances where the delta is applied (the store calls
+    // position.advanceApplied / advancePersisted); this hook only resolves
+    // confirmation thresholds against the incoming id.
+
+    const awaitingTxs = this.ctx.store.getByStatus('awaiting_delta');
+    const executingTxs = this.ctx.store.getByStatus('executing');
+
+    // Debug: Show state when delta arrives
+    if (awaitingTxs.length > 0 || executingTxs.length > 0) {
+      this.runtime.logger.debug('tx:delta_received', {
+        syncId,
+        lastSeenSyncId: this.lastSeenSyncId,
+        awaitingCount: awaitingTxs.length,
+        executingCount: executingTxs.length,
+        awaitingThresholds: awaitingTxs.map((tx) => ({
+          txId: tx.id.slice(0, 8),
+          model: tx.modelName,
+          needed: tx.syncIdNeededForCompletion,
+          requiresCorrelatedDelta: tx.requiresCorrelatedDelta === true,
+          willConfirm:
+            tx.requiresCorrelatedDelta === true
+              ? this.matchesSourceEcho(tx, correlationId)
+              : tx.syncIdNeededForCompletion !== undefined &&
+                syncId >= tx.syncIdNeededForCompletion,
+        })),
+      });
+    }
+
+    // Fast path: no awaiting transactions
+    if (awaitingTxs.length === 0) return;
+
+    let confirmedCount = 0;
+
+    for (const tx of awaitingTxs) {
+      // Queued forward receipts deliberately have no watermark. They confirm
+      // only when the authoritative source echoes the receipt's opaque,
+      // authenticated-scope batch identity; the legacy anomaly path continues
+      // to use its sync-id threshold.
+      const confirmedByCorrelation =
+        tx.requiresCorrelatedDelta === true &&
+        this.matchesSourceEcho(tx, correlationId);
+      const confirmedByThreshold =
+        tx.requiresCorrelatedDelta !== true &&
+        tx.syncIdNeededForCompletion !== undefined &&
+        syncId >= tx.syncIdNeededForCompletion;
+      if (confirmedByCorrelation || confirmedByThreshold) {
+        this.cancelDeltaConfirmationTimeout(tx.id);
+        this.ctx.store.updateStatus(tx.id, 'completed');
+        this.ctx.emit('transaction:completed', tx);
+        this.ctx.emit(`transaction:completed:${tx.id}`, tx);
+        this.ctx.optimisticUpdates.delete(tx.id);
+        confirmedCount++;
+
+        this.runtime.logger.debug('tx:confirm_via_delta', {
+          txId: tx.id.slice(0, 8),
+          model: tx.modelName,
+          neededSyncId: tx.syncIdNeededForCompletion,
+          receivedSyncId: syncId,
+          confirmation: confirmedByCorrelation ? 'source_correlation' : 'sync_id',
+        });
+      }
+    }
+
+    // Log batch summary only if we confirmed something
+    if (confirmedCount > 0) {
+      // Leave a breadcrumb when transactions confirm.
+      this.runtime.observability.breadcrumb('Transactions confirmed via delta', 'sync.transaction', 'info', {
+        count: confirmedCount,
+        syncId,
+        remainingAwaiting: awaitingTxs.length - confirmedCount,
+      });
+    }
+  }
+
+  // Schedule the confirmation wait for a transaction. On timeout the tracker
+  // retries with exponential backoff and requests reconciliation to catch up on
+  // missed deltas, rather than rolling back, which would discard state the
+  // server has already confirmed. A rollback happens only on an explicit server
+  // rejection, never on a timeout.
+  scheduleDeltaConfirmationTimeout(tx: QueuedMutation, timeoutMs: number): void {
+    // Cancel any existing timeout for this transaction
+    this.cancelDeltaConfirmationTimeout(tx.id);
+
+    // Deliberately not an async callback: the body is fully synchronous, and
+    // `setTimeout(async …)` would turn any throw into an unhandled promise
+    // rejection instead of a catchable synchronous error.
+    const timeoutHandle = setTimeout(() => {
+      const currentTx = this.ctx.store.get(tx.id);
+      if (currentTx?.status !== 'awaiting_delta') {
+        this.deltaConfirmationRetries.delete(tx.id);
+        return; // Already confirmed or failed
+      }
+
+      // If disconnected, re-schedule with same timeout (no backoff while offline)
+      if (!this.ctx.isConnected()) {
+        // Self-healing: re-schedule the confirmation wait while offline, no
+        // consumer action needed → debug.
+        this.runtime.logger.debug('[MutationQueue] Timeout fired while disconnected - re-scheduling', {
+          txId: tx.id.slice(0, 8),
+          model: tx.modelName,
+        });
+        this.deltaConfirmationTimeouts.delete(tx.id);
+        this.scheduleDeltaConfirmationTimeout(tx, timeoutMs);
+        return;
+      }
+
+      const retryCount = this.deltaConfirmationRetries.get(tx.id) ?? 0;
+
+      this.runtime.observability.captureReconciliation({
+        reason: 'delta_timeout',
+        model: tx.modelName,
+        modelId: tx.modelId,
+        syncIdNeeded: currentTx.syncIdNeededForCompletion,
+        lastSeenSyncId: this.lastSeenSyncId,
+        retryCount,
+        connectionState: this.ctx.isConnected() ? 'connected' : 'disconnected',
+      });
+
+      if (retryCount < DeltaConfirmationTracker.DELTA_MAX_RETRIES) {
+        // Retry: request reconciliation and re-schedule with exponential
+        // backoff. The server has already committed the mutation; only the
+        // delta is outstanding.
+        this.deltaConfirmationRetries.set(tx.id, retryCount + 1);
+        this.deltaConfirmationTimeouts.delete(tx.id);
+
+        // Exponential backoff: 30s → 60s → 120s → 120s → 120s (capped)
+        const nextTimeout = Math.min(timeoutMs * 2, DeltaConfirmationTracker.DELTA_MAX_TIMEOUT_MS);
+
+        // Request reconciliation so the client can cycle the connection and
+        // catch up on missed deltas from the server.
+        this.ctx.emit('reconciliation:needed', {
+          reason: 'delta_confirmation_timeout',
+          txId: tx.id,
+          model: tx.modelName,
+          modelId: tx.modelId,
+          syncIdNeeded: currentTx.syncIdNeededForCompletion,
+          lastSeenSyncId: this.lastSeenSyncId,
+          retryCount: retryCount + 1,
+        });
+
+        // Self-healing retry with backoff — the server already committed; we're
+        // just waiting on the delta. No consumer action → debug.
+        this.runtime.logger.debug('[MutationQueue] Re-scheduling with backoff', {
+          txId: tx.id.slice(0, 8),
+          model: tx.modelName,
+          nextTimeoutMs: nextTimeout,
+          retry: retryCount + 1,
+        });
+
+        this.scheduleDeltaConfirmationTimeout(tx, nextTimeout);
+      } else {
+        // Retries exhausted: persist the awaiting state instead of rolling back.
+        // The commit succeeded on the server, so the data exists there. Saving
+        // the awaiting state lets it survive the page closing; on the next
+        // session, reconnecting and catching up on deltas will confirm it.
+        this.deltaConfirmationRetries.delete(tx.id);
+        this.deltaConfirmationTimeouts.delete(tx.id);
+
+        this.runtime.observability.captureDeltaRetryExhausted({
+          txId: tx.id,
+          model: tx.modelName,
+          modelId: tx.modelId,
+          retryCount: DeltaConfirmationTracker.DELTA_MAX_RETRIES,
+          syncIdNeeded: currentTx.syncIdNeededForCompletion,
+        });
+
+        // Emit the persist event; the client performs the write to local storage.
+        this.ctx.emit('transaction:persist_awaiting', {
+          txId: tx.id,
+          model: tx.modelName,
+          modelId: tx.modelId,
+          operationType: tx.type,
+          syncIdNeeded: currentTx.syncIdNeededForCompletion,
+        });
+
+        // Also request one final reconciliation cycle
+        this.ctx.emit('reconciliation:needed', {
+          reason: 'delta_retries_exhausted',
+          txId: tx.id,
+          model: tx.modelName,
+          modelId: tx.modelId,
+          syncIdNeeded: currentTx.syncIdNeededForCompletion,
+          lastSeenSyncId: this.lastSeenSyncId,
+          retryCount: DeltaConfirmationTracker.DELTA_MAX_RETRIES,
+        });
+      }
+    }, timeoutMs);
+
+    this.deltaConfirmationTimeouts.set(tx.id, timeoutHandle);
+  }
+
+  // Cancel a pending delta confirmation timeout and clean up retry tracking
+  private cancelDeltaConfirmationTimeout(id: string): void {
+    const timeoutHandle = this.deltaConfirmationTimeouts.get(id);
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+      this.deltaConfirmationTimeouts.delete(id);
+    }
+    this.deltaConfirmationRetries.delete(id);
+  }
+
+  /**
+   * Clears every armed confirmation timer, one per in-flight transaction.
+   * {@link MutationQueue.dispose} calls this; without it a disposed queue
+   * would keep the process alive and fire callbacks against a cleared store.
+   */
+  dispose(): void {
+    for (const timeoutHandle of this.deltaConfirmationTimeouts.values()) {
+      clearTimeout(timeoutHandle);
+    }
+    this.deltaConfirmationTimeouts.clear();
+    this.deltaConfirmationRetries.clear();
+  }
+}

package/src/local/transactions/mutations/durableCommitRestore.ts

@@ -0,0 +1,128 @@
+import { AbloIdempotencyError, AbloValidationError } from '@abloatai/transaction/errors';
+import type { RuntimeContext } from '../../RuntimeContext.js';
+import type { CommitTransaction } from './commitLane.js';
+import type { DurableCommitEnvelope, CommitOutboxScope } from '@abloatai/transaction/transactions/settlement/commitEnvelope';
+import type { DurableWriteStore } from './durableWriteStore.js';
+import { durableCommitEnvelopeSchema } from '@abloatai/transaction/transactions/settlement/commitEnvelope';
+
+export interface DurableCommitRestoreContext {
+  readonly config: { enablePersistence: boolean };
+  readonly commitOutbox: DurableWriteStore | null;
+  readonly commitOutboxScope: CommitOutboxScope | null;
+  readonly commitStore: Map<string, CommitTransaction>;
+  readonly commitLane: CommitTransaction[];
+  readonly runtime: RuntimeContext;
+  readonly processCommitLane: () => Promise<void>;
+  readonly durableReplayWindowMs: number;
+}
+
+export async function restoreDurableCommits(ctx: DurableCommitRestoreContext): Promise<Set<string>> {
+    if (!ctx.config.enablePersistence) return new Set();
+
+    const sourceMutationIds = new Set<string>();
+    try {
+      if (!ctx.commitOutbox) return sourceMutationIds;
+      const rows = await ctx.commitOutbox.list();
+      const envelopes: DurableCommitEnvelope[] = [];
+      for (const row of rows) {
+        if (
+          typeof row !== 'object' ||
+          row === null ||
+          (row as { type?: unknown }).type !== 'commit_envelope'
+        ) continue;
+        const parsed = durableCommitEnvelopeSchema.safeParse(row);
+        if (parsed.success) {
+          envelopes.push(parsed.data);
+        } else {
+          ctx.runtime.logger.warn('A saved local write is unreadable and was held for review.');
+          ctx.runtime.observability.captureMutationFailure({
+            context: 'restore-commit-envelope',
+            error: parsed.error,
+          });
+          throw new AbloValidationError(
+            'A saved commit envelope is unreadable; replay stopped before newer writes were sent.',
+            { code: 'write_options_invalid', cause: parsed.error },
+          );
+        }
+      }
+      envelopes.sort(
+        (a, b) =>
+          (a.sequence ?? a.sealedAt * 1_000) -
+            (b.sequence ?? b.sealedAt * 1_000) ||
+          a.id.localeCompare(b.id),
+      );
+
+      for (const envelope of envelopes) {
+        for (const mutationId of envelope.sourceMutationIds) {
+          sourceMutationIds.add(mutationId);
+        }
+        if (
+          envelope.acceptedAt === undefined &&
+          Date.now() - envelope.sealedAt >=
+          ctx.durableReplayWindowMs
+        ) {
+          ctx.runtime.logger.warn(
+            'A saved local write is too old to retry safely and was held for review.',
+          );
+          ctx.runtime.observability.captureMutationFailure({
+            context: 'quarantine-expired-commit-envelope',
+            error: `Envelope ${envelope.idempotencyKey} is too old to replay safely`,
+          });
+          throw new AbloIdempotencyError(
+            'A saved commit is older than the server idempotency window and cannot be replayed safely.',
+            { code: 'idempotency_conflict' },
+          );
+        }
+        if (
+          ctx.commitOutboxScope &&
+          (
+            !envelope.scope || // eslint-disable-line @typescript-eslint/prefer-optional-chain -- missing scope must quarantine
+            envelope.scope.organizationId !== ctx.commitOutboxScope.organizationId ||
+            envelope.scope.participantId !== ctx.commitOutboxScope.participantId ||
+            envelope.scope.namespace !== ctx.commitOutboxScope.namespace
+          )
+        ) {
+          ctx.runtime.logger.warn(
+            'A saved local write belongs to a different account or server and was held for review.',
+          );
+          continue;
+        }
+        if (ctx.commitStore.has(envelope.idempotencyKey)) continue;
+        const transaction: CommitTransaction = {
+          id: envelope.idempotencyKey,
+          kind: 'commit',
+          operations: envelope.operations.map((operation) => ({ ...operation })),
+          ...(envelope.commitOptions.reads
+            ? { reads: [...envelope.commitOptions.reads] }
+            : {}),
+          ...(envelope.commitOptions.track
+            ? { track: [...envelope.commitOptions.track] }
+            : {}),
+          status: 'pending',
+          createdAt: envelope.createdAt,
+          sealedAt: envelope.sealedAt,
+          sequence: envelope.sequence ?? envelope.sealedAt * 1_000,
+          attempts: 0,
+          ...(envelope.correlationId
+            ? { correlationId: envelope.correlationId }
+            : {}),
+          sourceMutationIds: [...envelope.sourceMutationIds],
+          durableEnvelope: envelope,
+        };
+        ctx.commitStore.set(transaction.id, transaction);
+        ctx.commitLane.push(transaction);
+      }
+
+      if (ctx.commitLane.length > 0) void ctx.processCommitLane();
+    } catch (error) {
+      ctx.runtime.logger.debug('[MutationQueue] Failed to restore durable writes', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+      ctx.runtime.observability.captureMutationFailure({
+        context: 'restore-commit-envelopes',
+        error: error instanceof Error ? error : String(error),
+      });
+      throw error;
+    }
+    return sourceMutationIds;
+}

package/src/local/transactions/mutations/durableWriteStore.ts

@@ -0,0 +1,21 @@
+/**
+ * The durable-write port moved to the settlement core (ADR 0016): it is a
+ * contract over commit envelopes and holds no local rows. Re-exported here so
+ * the existing `transactions/mutations/durableWriteStore` import path keeps
+ * resolving for the queue, the outbox, and the client options.
+ *
+ * The port and its config live in the core's `durableWrites` module (a behavior
+ * contract, not a persisted shape); the records that cross it are owned by
+ * `transactions/settlement/pendingWrite`.
+ */
+
+export {
+  durableWriteStoreSchema,
+  durableWritesConfigSchema,
+} from '@abloatai/transaction/durableWrites';
+export type {
+  DurableWriteStore,
+  DurableWritesConfig,
+} from '@abloatai/transaction/durableWrites';
+export { pendingWriteSchema } from '@abloatai/transaction/transactions/settlement/pendingWrite';
+export type { PendingWrite } from '@abloatai/transaction/transactions/settlement/pendingWrite';

package/src/local/transactions/mutations/executionSelection.ts

@@ -0,0 +1,42 @@
+import type { QueuedMutation } from './commitPayload.js';
+
+export function takeNextExecutionBatch(
+  executionQueue: QueuedMutation[],
+  maxBatchSize: number,
+): { batch: QueuedMutation[]; remaining: QueuedMutation[] } {
+  const retryGroups = new Map<string, Map<string, QueuedMutation>>();
+  for (const tx of executionQueue) {
+    const envelope = tx.commitEnvelope;
+    if (!envelope) continue;
+    const group = retryGroups.get(envelope.idempotencyKey) ?? new Map<string, QueuedMutation>();
+    group.set(tx.id, tx);
+    retryGroups.set(envelope.idempotencyKey, group);
+  }
+  for (const [idempotencyKey, byId] of retryGroups) {
+    const members = [...byId.values()];
+    const expectedCount = members[0]?.commitEnvelope?.operationCount;
+    if (expectedCount === undefined || members.length !== expectedCount) continue;
+    const remaining = executionQueue.filter((tx) => tx.commitEnvelope?.idempotencyKey !== idempotencyKey);
+    members.sort((a, b) => (a.commitEnvelope?.operationIndex ?? 0) - (b.commitEnvelope?.operationIndex ?? 0));
+    return { batch: members, remaining };
+  }
+  const fresh = executionQueue.filter((tx) => !tx.commitEnvelope);
+  const firstFresh = fresh[0];
+  if (!firstFresh) return { batch: [], remaining: executionQueue };
+  const explicitIndex = fresh.findIndex((tx) => typeof tx.writeOptions?.idempotencyKey === 'string');
+  const selected = explicitIndex === 0
+    ? [firstFresh]
+    : fresh.slice(0, Math.min(maxBatchSize, explicitIndex > 0 ? explicitIndex : fresh.length));
+  const selectedIds = new Set(selected.map((tx) => tx.id));
+  return { batch: selected, remaining: executionQueue.filter((tx) => !selectedIds.has(tx.id)) };
+}
+
+export function takePendingDrainBatch(pending: QueuedMutation[], maxBatchSize: number): QueuedMutation[] {
+  const first = pending[0];
+  if (!first) return [];
+  const envelope = first.commitEnvelope;
+  if (envelope) return pending.filter((tx) => tx.commitEnvelope?.idempotencyKey === envelope.idempotencyKey);
+  if (typeof first.writeOptions?.idempotencyKey === 'string') return [first];
+  const explicitIndex = pending.findIndex((tx) => typeof tx.writeOptions?.idempotencyKey === 'string');
+  return pending.slice(0, Math.min(maxBatchSize, explicitIndex > 0 ? explicitIndex : pending.length));
+}

package/src/local/transactions/mutations/failureHandling.ts

@@ -0,0 +1,154 @@
+import type { RuntimeContext } from '../../RuntimeContext.js';
+import type { MutationQueueConfig } from './MutationQueue.js';
+import type { QueuedMutation } from './commitPayload.js';
+import type { MutationStore } from './MutationStore.js';
+import { AbloError } from '@abloatai/transaction/errors';
+import { extractStatusCode } from './commitPayload.js';
+
+export interface FailureHandlingContext {
+  readonly runtime: RuntimeContext;
+  readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff'>;
+  readonly store: MutationStore;
+  readonly isPermanentError: (error: Error) => boolean;
+  readonly rollbackOptimistic: (transaction: QueuedMutation, reason: string, error?: Error) => Promise<void>;
+  readonly enqueue: (transaction: QueuedMutation) => void;
+  readonly getLastPermanentErrorSignature: () => string | undefined;
+  readonly setLastPermanentErrorSignature: (signature: string) => void;
+  readonly emit: (event: string, payload: object) => boolean;
+}
+
+export async function handleFailure(ctx: FailureHandlingContext, transaction: QueuedMutation, error: Error): Promise<void> {
+    transaction.attempts++;
+
+    // Check whether this is a permanent error that should not be retried.
+    if (ctx.isPermanentError(error)) {
+      // Logged at warn: a permanent error means the server rejected the write,
+      // so the developer should see the reason in the console. The typed
+      // AbloError fields (`type`, `code`, `httpStatus`) are included so the
+      // cause is visible — for example a foreign-key violation
+      // (AbloValidationError) versus expired authentication
+      // (AbloAuthenticationError).
+      try {
+        const abloErr = error instanceof AbloError ? error : undefined;
+        const details = {
+          txId: transaction.id.slice(0, 8),
+          type: transaction.type,
+          model: transaction.modelName,
+          modelId: transaction.modelId.slice(0, 12),
+          errorType: abloErr?.type ?? error?.name,
+          errorCode: abloErr?.code,
+          httpStatus: abloErr?.httpStatus,
+          requestId: abloErr?.requestId,
+          message: error?.message,
+          inputKeys: transaction.data ? Object.keys(transaction.data) : undefined,
+        };
+
+        // A `create` whose id already exists is the benign idempotency case:
+        // "this row is already there." It's the least alarming permanent
+        // error, so it doesn't warrant a `warn` — `info` keeps it visible
+        // without crying wolf. Everything else (FK violation, auth expiry,
+        // server 500) stays at `warn`.
+        const isBenignIdempotent =
+          transaction.type === 'create' &&
+          (abloErr?.code === 'unique_violation' ||
+            abloErr?.type === 'AbloIdempotencyError');
+
+        // Demote exact repeats (same write rejected for the same reason on
+        // each reconnect replay) to `debug` so the loop logs once.
+        const sig = `${details.type}:${details.model}:${details.modelId}:${details.errorCode ?? details.errorType}`;
+        const isRepeat = sig === ctx.getLastPermanentErrorSignature();
+        ctx.setLastPermanentErrorSignature(sig);
+
+        const logger = ctx.runtime.logger;
+
+        // Two registers from one call site, split by log level (the default
+        // logger is gated at `warn`, so `debug` stays hidden unless
+        // ABLO_LOG_LEVEL=debug is set to inspect the engine):
+        //   - the default-visible line speaks the application developer's
+        //     language: their verb (such as `update`), their model, the typed
+        //     error's own message, and the wire `code` for searching. It uses
+        //     no engine jargon and prints no JSON dump, which would alarm
+        //     without helping.
+        //   - the forensic `details` ride a companion `debug` line for anyone
+        //     debugging the engine internals.
+        const revertNote = ctx.config.enableOptimistic
+          ? ' The local change was reverted.'
+          : '';
+        const reason = abloErr?.message ? ` — ${abloErr.message}` : '';
+        const code = abloErr?.code ? ` (code: ${abloErr.code})` : '';
+        // An optimistic write resolves before the server answers, so a later
+        // rejection has no caller left to return to and this log is the only
+        // place it appears. That reads to an application developer as their own
+        // save silently failing — the write showed, then vanished — and sends
+        // them into their editor instead of here. Name the subscription that
+        // hands them the same typed error, so the application can say what
+        // happened rather than only the console.
+        const channelNote = ctx.config.enableOptimistic
+          ? ' To surface this in your app, subscribe with `ablo.onMutationFailure(…)`.'
+          : '';
+        const headline = `Your ${transaction.type} to "${transaction.modelName}" was not saved${reason}${code}.${revertNote}${channelNote}`;
+
+        if (isRepeat) {
+          // Same write rejected for the same reason on each reconnect replay —
+          // log the forensics once, stay quiet after.
+          logger.debug('write rejected again (same reason)', details);
+        } else if (isBenignIdempotent) {
+          // Already-exists on a `create` is expected on replay, not a problem.
+          logger.info(`Your ${transaction.type} to "${transaction.modelName}" was skipped — this row already exists.`);
+          logger.debug('idempotent skip — details', details);
+        } else {
+          logger.warn(headline);
+          logger.debug('write rejection — details', details);
+        }
+      } catch {}
+
+      // Mark as failed immediately and rollback
+      ctx.store.updateStatus(transaction.id, 'failed');
+
+      if (ctx.config.enableOptimistic) {
+        await ctx.rollbackOptimistic(transaction, 'permanent_error', error);
+      }
+
+      ctx.emit('transaction:failed', { transaction, error, permanent: true });
+      // The id-suffixed event is what `waitForConfirmation` (the
+      // `wait:'confirmed'` path) listens on — without it a permanently
+      // rejected write left the caller's promise hanging forever.
+      ctx.emit(`transaction:failed:${transaction.id}`, { error });
+      return;
+    }
+
+    if (transaction.attempts < ctx.config.maxRetries) {
+      // Exponential backoff with full jitter on every transient retry:
+      // `sleep = random(0, min(cap, base * 2^attempt))`. Throttling responses
+      // (429/503) use a longer base than other transient errors. The re-enqueue
+      // is scheduled rather than awaited, so one backing-off transaction cannot
+      // stall unrelated commits.
+      const { baseMs, capMs } = ctx.config.retryBackoff;
+      let base = baseMs;
+      try {
+        const status = extractStatusCode(error);
+        if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
+      } catch {}
+      const ceiling = Math.min(capMs, base * Math.pow(2, transaction.attempts - 1));
+      const delay = Math.floor(Math.random() * ceiling);
+
+      ctx.store.updateStatus(transaction.id, 'pending');
+      setTimeout(() => {
+        // The queue may have shut down or the tx may have been settled
+        // (e.g. delta-confirmed) while we backed off.
+        if (ctx.store.get(transaction.id)?.status !== 'pending') return;
+        ctx.enqueue(transaction);
+      }, delay);
+    } else {
+      // Mark as failed and rollback
+      ctx.store.updateStatus(transaction.id, 'failed');
+
+      if (ctx.config.enableOptimistic) {
+        await ctx.rollbackOptimistic(transaction, 'max_retries_exhausted', error);
+      }
+
+      ctx.emit('transaction:failed', { transaction, error });
+      // Settle `waitForConfirmation` waiters (see the permanent branch above).
+      ctx.emit(`transaction:failed:${transaction.id}`, { error });
+    }
+  }