@abloatai/humans 0.44.0 → 0.46.0

package/src/local/transactions/mutations/MutationQueue.ts

@@ -158,6 +158,15 @@ export interface MutationQueueConfig {
   maxBatchSize: number;
   batchDelay: number;
   maxRetries: number;
+  /**
+   * Minimum wall-clock window for retrying transient write failures with the
+   * same durable envelope and idempotency key. This absorbs managed-database
+   * promotion and brief regional network incidents without double-applying a
+   * write. Defaults to 120 seconds: the Aurora promotion drill recovered
+   * writes just beyond 60 seconds, so a one-minute boundary discarded exact
+   * envelopes at the instant the new writer became usable.
+   */
+  availabilityRetryWindowMs: number;
   conflictResolution: ConflictResolution;
   enablePersistence: boolean;
   enableOptimistic: boolean;
@@ -289,6 +298,7 @@ export class MutationQueue extends EventEmitter {
   private replicationLagTimeouts = new Map<string, ReturnType<typeof setTimeout>>();
   private replicationLagErrors = new Map<string, AbloConnectionError>();
   private commitProcessing = false;
+  private commitRetryTimer: ReturnType<typeof setTimeout> | null = null;
   private lastCommitSequence = 0;
   private durableReplayBlock: AbloIdempotencyError | null = null;
   /** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
@@ -312,7 +322,11 @@ export class MutationQueue extends EventEmitter {
   private get commitLaneContext(): CommitLaneContext {
     return {
       runtime: this.runtime,
-      config: { maxRetries: this.config.maxRetries },
+      config: {
+        maxRetries: this.config.maxRetries,
+        availabilityRetryWindowMs: this.config.availabilityRetryWindowMs,
+        retryBackoff: this.config.retryBackoff,
+      },
       commitLane: this.commitLane,
       commitNotifications: this.commitNotifications,
       commitMissingIds: this.commitMissingIds,
@@ -336,6 +350,13 @@ export class MutationQueue extends EventEmitter {
       noteAck: (syncId) => this.noteAck(syncId),
       isDefinitiveRejection: (error) => this.isDefinitiveRejection(error),
       isPermanentError: (error) => this.isPermanentError(error),
+      scheduleRetry: (delayMs) => {
+        if (this.commitRetryTimer !== null) clearTimeout(this.commitRetryTimer);
+        this.commitRetryTimer = setTimeout(() => {
+          this.commitRetryTimer = null;
+          void this.processCommitLane();
+        }, delayMs);
+      },
       emitCommitLifecycle: (event, payload) => this.emitCommitLifecycle(event, payload),
     };
   }
@@ -782,6 +803,7 @@ export class MutationQueue extends EventEmitter {
     maxBatchSize: 50, // send up to this many operations per commit
     batchDelay: 150, // milliseconds to wait for more operations before sending
     maxRetries: 3,
+    availabilityRetryWindowMs: 120_000,
     conflictResolution: {
       strategy: 'last-write-wins',
     },
@@ -2004,6 +2026,10 @@ export class MutationQueue extends EventEmitter {
       clearTimeout(this.commitOfflineGraceTimer);
       this.commitOfflineGraceTimer = null;
     }
+    if (this.commitRetryTimer !== null) {
+      clearTimeout(this.commitRetryTimer);
+      this.commitRetryTimer = null;
+    }
 
     // Clear store
     this.store.clear();

package/src/local/transactions/mutations/commitLane.ts

@@ -1,12 +1,12 @@
 import type { RuntimeContext } from '../../RuntimeContext.js';
 import type { ReadDependency, TrackDependency, OnStaleMode, StaleNotification } from '@abloatai/transaction/coordination/schema';
-import { AbloConnectionError } from '@abloatai/transaction/errors';
 import type { MutationCommitResult } from '@abloatai/transaction/wire/commit';
 import type {
   DurableCommitEnvelope,
   DurableCommitOperation,
 } from '@abloatai/transaction/transactions/settlement/commitEnvelope';
 import type { SealDurableCommitInput } from './commitTransport.js';
+import { transientRetryDelayMs } from './failureHandling.js';
 
 export interface CommitTransaction {
   id: string;
@@ -26,6 +26,7 @@ export interface CommitTransaction {
   createdAt: number;
   attempts: number;
   transientAttempts?: number;
+  firstTransientFailureAt?: number;
   lastSyncId?: number;
   correlationId?: string;
   error?: Error;
@@ -38,7 +39,11 @@ export interface CommitTransaction {
 
 export interface CommitLaneContext {
   readonly runtime: RuntimeContext;
-  readonly config: { maxRetries: number };
+  readonly config: {
+    maxRetries: number;
+    availabilityRetryWindowMs: number;
+    retryBackoff: { baseMs: number; capMs: number };
+  };
   readonly commitLane: CommitTransaction[];
   readonly commitNotifications: Map<string, StaleNotification[]>;
   readonly commitMissingIds: Map<string, string[]>;
@@ -56,6 +61,7 @@ export interface CommitLaneContext {
   readonly noteAck: (syncId: number | undefined) => void;
   readonly isDefinitiveRejection: (error: Error) => boolean;
   readonly isPermanentError: (error: Error) => boolean;
+  readonly scheduleRetry: (delayMs: number) => void;
   readonly emitCommitLifecycle: (event: string, payload: object) => void;
 }
 
@@ -165,14 +171,24 @@ export async function processCommitLane(ctx: CommitLaneContext): Promise<void> {
       } catch (cause) {
         const error = cause instanceof Error ? cause : new Error(String(cause));
         if (dispatchStarted && ctx.isDefinitiveRejection(error)) await ctx.removeDurableCommit(tx.id);
-        if (!(error instanceof AbloConnectionError)) tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
-        const exhausted = (tx.transientAttempts ?? 0) > ctx.config.maxRetries;
+        tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
+        tx.firstTransientFailureAt ??= Date.now();
+        const outsideAvailabilityWindow =
+          Date.now() - tx.firstTransientFailureAt >= ctx.config.availabilityRetryWindowMs;
+        const exhausted =
+          tx.transientAttempts > ctx.config.maxRetries && outsideAvailabilityWindow;
         if (!ctx.isPermanentError(error) && !exhausted) {
           tx.status = 'pending';
+          const delayMs = transientRetryDelayMs(
+            error,
+            tx.transientAttempts,
+            ctx.config.retryBackoff,
+          );
           ctx.runtime.logger.debug('[MutationQueue] commit lane transient', {
             txId: tx.id.slice(0, 12), attempts: tx.attempts,
-            transientAttempts: tx.transientAttempts ?? 0, message: error.message,
+            transientAttempts: tx.transientAttempts, delayMs, message: error.message,
           });
+          ctx.scheduleRetry(delayMs);
           break;
         }
         tx.status = 'failed';

package/src/local/transactions/mutations/commitPayload.ts

@@ -108,6 +108,8 @@ export interface QueuedMutation {
   status: 'pending' | 'executing' | 'awaiting_delta' | 'completed' | 'failed' | 'rolled_back';
   createdAt: number;
   attempts: number;
+  /** First transient dispatch failure in the current availability incident. */
+  firstTransientFailureAt?: number;
   priority: 'normal' | 'high';
   priorityScore: number; // foreign-key-aware priority, derived, used for sorting
   writeOptions?: WriteOptions;

package/src/local/transactions/mutations/failureHandling.ts

@@ -7,7 +7,10 @@ import { extractStatusCode } from './commitPayload.js';
 
 export interface FailureHandlingContext {
   readonly runtime: RuntimeContext;
-  readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff'>;
+  readonly config: Pick<
+    MutationQueueConfig,
+    'enableOptimistic' | 'maxRetries' | 'retryBackoff' | 'availabilityRetryWindowMs'
+  >;
   readonly store: MutationStore;
   readonly isPermanentError: (error: Error) => boolean;
   readonly rollbackOptimistic: (transaction: QueuedMutation, reason: string, error?: Error) => Promise<void>;
@@ -17,6 +20,21 @@ export interface FailureHandlingContext {
   readonly emit: (event: string, payload: object) => boolean;
 }
 
+export function transientRetryDelayMs(
+  error: Error,
+  attempt: number,
+  retryBackoff: MutationQueueConfig['retryBackoff'],
+): number {
+  const { baseMs, capMs } = retryBackoff;
+  let base = baseMs;
+  try {
+    const status = extractStatusCode(error);
+    if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
+  } catch {}
+  const ceiling = Math.min(capMs, base * Math.pow(2, Math.max(0, attempt - 1)));
+  return Math.floor(Math.random() * ceiling);
+}
+
 export async function handleFailure(ctx: FailureHandlingContext, transaction: QueuedMutation, error: Error): Promise<void> {
     transaction.attempts++;
 
@@ -120,20 +138,17 @@ export async function handleFailure(ctx: FailureHandlingContext, transaction: Qu
       return;
     }
 
-    if (transaction.attempts < ctx.config.maxRetries) {
+    transaction.firstTransientFailureAt ??= Date.now();
+    const insideAvailabilityWindow =
+      Date.now() - transaction.firstTransientFailureAt < ctx.config.availabilityRetryWindowMs;
+
+    if (transaction.attempts < ctx.config.maxRetries || insideAvailabilityWindow) {
       // Exponential backoff with full jitter on every transient retry:
       // `sleep = random(0, min(cap, base * 2^attempt))`. Throttling responses
       // (429/503) use a longer base than other transient errors. The re-enqueue
       // is scheduled rather than awaited, so one backing-off transaction cannot
       // stall unrelated commits.
-      const { baseMs, capMs } = ctx.config.retryBackoff;
-      let base = baseMs;
-      try {
-        const status = extractStatusCode(error);
-        if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
-      } catch {}
-      const ceiling = Math.min(capMs, base * Math.pow(2, transaction.attempts - 1));
-      const delay = Math.floor(Math.random() * ceiling);
+      const delay = transientRetryDelayMs(error, transaction.attempts, ctx.config.retryBackoff);
 
       ctx.store.updateStatus(transaction.id, 'pending');
       setTimeout(() => {