@abloatai/humans 0.45.0 → 0.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -83,6 +83,28 @@ export function createClaimStream(config, transport = null) {
83
83
  }
84
84
  }
85
85
  };
86
+ const observeForeignClaim = (heldBy, claim, participantKind, isAgent) => {
87
+ const description = claim.description ??
88
+ descriptionFromMeta(claim.meta) ??
89
+ 'editing';
90
+ const { meta, ...details } = subTarget(claim);
91
+ activeByClaimId.set(claim.claimId, {
92
+ object: 'claim',
93
+ id: claim.claimId,
94
+ status: 'active',
95
+ heldBy,
96
+ participantKind: participantKindFromWire(participantKind, isAgent),
97
+ target: {
98
+ ...streamTarget(claim),
99
+ ...details,
100
+ ...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
101
+ },
102
+ description,
103
+ ttlSeconds: Math.max(0, Math.floor((claim.expiresAt - Date.now()) / 1000)),
104
+ createdAt: claim.declaredAt,
105
+ expiresAt: claim.expiresAt,
106
+ });
107
+ };
86
108
  // ── Wire wiring ──────────────────────────────────────────────────
87
109
  let attached = null;
88
110
  const unsubs = [];
@@ -125,30 +147,7 @@ export function createClaimStream(config, transport = null) {
125
147
  // `settled()`. Absent status means active (wire back-compat).
126
148
  if (claim.status && claim.status !== 'active')
127
149
  continue;
128
- // Resolve the always-present public field, tolerating a frame that
129
- // carries the value in `meta` rather than as an explicit description.
130
- const description = claim.description ??
131
- descriptionFromMeta(claim.meta) ??
132
- 'editing';
133
- // The frame is parsed permissively, on purpose; `declaredMeta` is where
134
- // that wire value becomes the shape the program declared.
135
- const { meta, ...details } = subTarget(claim);
136
- activeByClaimId.set(claim.claimId, {
137
- object: 'claim',
138
- id: claim.claimId,
139
- status: 'active',
140
- heldBy: event.userId,
141
- participantKind: participantKindFromWire(event.participantKind, event.isAgent),
142
- target: {
143
- ...streamTarget(claim),
144
- ...details,
145
- ...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
146
- },
147
- description,
148
- ttlSeconds: Math.max(0, Math.floor((claim.expiresAt - Date.now()) / 1000)),
149
- createdAt: claim.declaredAt,
150
- expiresAt: claim.expiresAt,
151
- });
150
+ observeForeignClaim(event.userId, claim, event.participantKind, event.isAgent);
152
151
  mutated = true;
153
152
  }
154
153
  if (mutated)
@@ -168,6 +167,16 @@ export function createClaimStream(config, transport = null) {
168
167
  // a claim the server already rejected (would just spam both
169
168
  // sides with conflicts).
170
169
  ownClaims.delete(rejection.claimId);
170
+ // A holder on another server may have claimed before this client joined
171
+ // the row group, so its one-shot presence frame was missed. A conflict
172
+ // reply carries the authoritative holder summary; seed the same local
173
+ // state immediately instead of continuing to report the row as free.
174
+ if (rejection.reason === 'conflict' &&
175
+ rejection.heldBy &&
176
+ rejection.heldByClaim) {
177
+ observeForeignClaim(rejection.heldBy, rejection.heldByClaim, rejection.heldByKind);
178
+ notifyListeners();
179
+ }
171
180
  for (const l of rejectionListeners) {
172
181
  try {
173
182
  l(rejection);
@@ -45,6 +45,15 @@ export interface MutationQueueConfig {
45
45
  maxBatchSize: number;
46
46
  batchDelay: number;
47
47
  maxRetries: number;
48
+ /**
49
+ * Minimum wall-clock window for retrying transient write failures with the
50
+ * same durable envelope and idempotency key. This absorbs managed-database
51
+ * promotion and brief regional network incidents without double-applying a
52
+ * write. Defaults to 120 seconds: the Aurora promotion drill recovered
53
+ * writes just beyond 60 seconds, so a one-minute boundary discarded exact
54
+ * envelopes at the instant the new writer became usable.
55
+ */
56
+ availabilityRetryWindowMs: number;
48
57
  conflictResolution: ConflictResolution;
49
58
  enablePersistence: boolean;
50
59
  enableOptimistic: boolean;
@@ -127,6 +136,7 @@ export declare class MutationQueue extends EventEmitter {
127
136
  private replicationLagTimeouts;
128
137
  private replicationLagErrors;
129
138
  private commitProcessing;
139
+ private commitRetryTimer;
130
140
  private lastCommitSequence;
131
141
  private durableReplayBlock;
132
142
  /** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
@@ -418,6 +428,15 @@ export declare class MutationQueue extends EventEmitter {
418
428
  maxBatchSize: number;
419
429
  batchDelay: number;
420
430
  maxRetries: number;
431
+ /**
432
+ * Minimum wall-clock window for retrying transient write failures with the
433
+ * same durable envelope and idempotency key. This absorbs managed-database
434
+ * promotion and brief regional network incidents without double-applying a
435
+ * write. Defaults to 120 seconds: the Aurora promotion drill recovered
436
+ * writes just beyond 60 seconds, so a one-minute boundary discarded exact
437
+ * envelopes at the instant the new writer became usable.
438
+ */
439
+ availabilityRetryWindowMs: number;
421
440
  conflictResolution: ConflictResolution;
422
441
  enablePersistence: boolean;
423
442
  enableOptimistic: boolean;
@@ -104,6 +104,7 @@ export class MutationQueue extends EventEmitter {
104
104
  replicationLagTimeouts = new Map();
105
105
  replicationLagErrors = new Map();
106
106
  commitProcessing = false;
107
+ commitRetryTimer = null;
107
108
  lastCommitSequence = 0;
108
109
  durableReplayBlock = null;
109
110
  /** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
@@ -125,7 +126,11 @@ export class MutationQueue extends EventEmitter {
125
126
  get commitLaneContext() {
126
127
  return {
127
128
  runtime: this.runtime,
128
- config: { maxRetries: this.config.maxRetries },
129
+ config: {
130
+ maxRetries: this.config.maxRetries,
131
+ availabilityRetryWindowMs: this.config.availabilityRetryWindowMs,
132
+ retryBackoff: this.config.retryBackoff,
133
+ },
129
134
  commitLane: this.commitLane,
130
135
  commitNotifications: this.commitNotifications,
131
136
  commitMissingIds: this.commitMissingIds,
@@ -147,6 +152,14 @@ export class MutationQueue extends EventEmitter {
147
152
  noteAck: (syncId) => this.noteAck(syncId),
148
153
  isDefinitiveRejection: (error) => this.isDefinitiveRejection(error),
149
154
  isPermanentError: (error) => this.isPermanentError(error),
155
+ scheduleRetry: (delayMs) => {
156
+ if (this.commitRetryTimer !== null)
157
+ clearTimeout(this.commitRetryTimer);
158
+ this.commitRetryTimer = setTimeout(() => {
159
+ this.commitRetryTimer = null;
160
+ void this.processCommitLane();
161
+ }, delayMs);
162
+ },
150
163
  emitCommitLifecycle: (event, payload) => this.emitCommitLifecycle(event, payload),
151
164
  };
152
165
  }
@@ -516,6 +529,7 @@ export class MutationQueue extends EventEmitter {
516
529
  maxBatchSize: 50, // send up to this many operations per commit
517
530
  batchDelay: 150, // milliseconds to wait for more operations before sending
518
531
  maxRetries: 3,
532
+ availabilityRetryWindowMs: 120_000,
519
533
  conflictResolution: {
520
534
  strategy: 'last-write-wins',
521
535
  },
@@ -1506,6 +1520,10 @@ export class MutationQueue extends EventEmitter {
1506
1520
  clearTimeout(this.commitOfflineGraceTimer);
1507
1521
  this.commitOfflineGraceTimer = null;
1508
1522
  }
1523
+ if (this.commitRetryTimer !== null) {
1524
+ clearTimeout(this.commitRetryTimer);
1525
+ this.commitRetryTimer = null;
1526
+ }
1509
1527
  // Clear store
1510
1528
  this.store.clear();
1511
1529
  this.localMutationPort.updates.clear();
@@ -21,6 +21,7 @@ export interface CommitTransaction {
21
21
  createdAt: number;
22
22
  attempts: number;
23
23
  transientAttempts?: number;
24
+ firstTransientFailureAt?: number;
24
25
  lastSyncId?: number;
25
26
  correlationId?: string;
26
27
  error?: Error;
@@ -34,6 +35,11 @@ export interface CommitLaneContext {
34
35
  readonly runtime: RuntimeContext;
35
36
  readonly config: {
36
37
  maxRetries: number;
38
+ availabilityRetryWindowMs: number;
39
+ retryBackoff: {
40
+ baseMs: number;
41
+ capMs: number;
42
+ };
37
43
  };
38
44
  readonly commitLane: CommitTransaction[];
39
45
  readonly commitNotifications: Map<string, StaleNotification[]>;
@@ -52,6 +58,7 @@ export interface CommitLaneContext {
52
58
  readonly noteAck: (syncId: number | undefined) => void;
53
59
  readonly isDefinitiveRejection: (error: Error) => boolean;
54
60
  readonly isPermanentError: (error: Error) => boolean;
61
+ readonly scheduleRetry: (delayMs: number) => void;
55
62
  readonly emitCommitLifecycle: (event: string, payload: object) => void;
56
63
  }
57
64
  export interface CommitReceiptContext {
@@ -1,4 +1,4 @@
1
- import { AbloConnectionError } from '@abloatai/transaction/errors';
1
+ import { transientRetryDelayMs } from './failureHandling.js';
2
2
  export function waitForCommitReceipt(ctx, clientTxId) {
3
3
  const drainNotifications = () => {
4
4
  const notifications = ctx.commitNotifications.get(clientTxId);
@@ -112,15 +112,18 @@ export async function processCommitLane(ctx) {
112
112
  const error = cause instanceof Error ? cause : new Error(String(cause));
113
113
  if (dispatchStarted && ctx.isDefinitiveRejection(error))
114
114
  await ctx.removeDurableCommit(tx.id);
115
- if (!(error instanceof AbloConnectionError))
116
- tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
117
- const exhausted = (tx.transientAttempts ?? 0) > ctx.config.maxRetries;
115
+ tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
116
+ tx.firstTransientFailureAt ??= Date.now();
117
+ const outsideAvailabilityWindow = Date.now() - tx.firstTransientFailureAt >= ctx.config.availabilityRetryWindowMs;
118
+ const exhausted = tx.transientAttempts > ctx.config.maxRetries && outsideAvailabilityWindow;
118
119
  if (!ctx.isPermanentError(error) && !exhausted) {
119
120
  tx.status = 'pending';
121
+ const delayMs = transientRetryDelayMs(error, tx.transientAttempts, ctx.config.retryBackoff);
120
122
  ctx.runtime.logger.debug('[MutationQueue] commit lane transient', {
121
123
  txId: tx.id.slice(0, 12), attempts: tx.attempts,
122
- transientAttempts: tx.transientAttempts ?? 0, message: error.message,
124
+ transientAttempts: tx.transientAttempts, delayMs, message: error.message,
123
125
  });
126
+ ctx.scheduleRetry(delayMs);
124
127
  break;
125
128
  }
126
129
  tx.status = 'failed';
@@ -54,6 +54,8 @@ export interface QueuedMutation {
54
54
  status: 'pending' | 'executing' | 'awaiting_delta' | 'completed' | 'failed' | 'rolled_back';
55
55
  createdAt: number;
56
56
  attempts: number;
57
+ /** First transient dispatch failure in the current availability incident. */
58
+ firstTransientFailureAt?: number;
57
59
  priority: 'normal' | 'high';
58
60
  priorityScore: number;
59
61
  writeOptions?: WriteOptions;
@@ -4,7 +4,7 @@ import type { QueuedMutation } from './commitPayload.js';
4
4
  import type { MutationStore } from './MutationStore.js';
5
5
  export interface FailureHandlingContext {
6
6
  readonly runtime: RuntimeContext;
7
- readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff'>;
7
+ readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff' | 'availabilityRetryWindowMs'>;
8
8
  readonly store: MutationStore;
9
9
  readonly isPermanentError: (error: Error) => boolean;
10
10
  readonly rollbackOptimistic: (transaction: QueuedMutation, reason: string, error?: Error) => Promise<void>;
@@ -13,4 +13,5 @@ export interface FailureHandlingContext {
13
13
  readonly setLastPermanentErrorSignature: (signature: string) => void;
14
14
  readonly emit: (event: string, payload: object) => boolean;
15
15
  }
16
+ export declare function transientRetryDelayMs(error: Error, attempt: number, retryBackoff: MutationQueueConfig['retryBackoff']): number;
16
17
  export declare function handleFailure(ctx: FailureHandlingContext, transaction: QueuedMutation, error: Error): Promise<void>;
@@ -1,5 +1,17 @@
1
1
  import { AbloError } from '@abloatai/transaction/errors';
2
2
  import { extractStatusCode } from './commitPayload.js';
3
+ export function transientRetryDelayMs(error, attempt, retryBackoff) {
4
+ const { baseMs, capMs } = retryBackoff;
5
+ let base = baseMs;
6
+ try {
7
+ const status = extractStatusCode(error);
8
+ if (status === 429 || status === 503)
9
+ base = Math.max(baseMs, 1_000);
10
+ }
11
+ catch { }
12
+ const ceiling = Math.min(capMs, base * Math.pow(2, Math.max(0, attempt - 1)));
13
+ return Math.floor(Math.random() * ceiling);
14
+ }
3
15
  export async function handleFailure(ctx, transaction, error) {
4
16
  transaction.attempts++;
5
17
  // Check whether this is a permanent error that should not be retried.
@@ -95,22 +107,15 @@ export async function handleFailure(ctx, transaction, error) {
95
107
  ctx.emit(`transaction:failed:${transaction.id}`, { error });
96
108
  return;
97
109
  }
98
- if (transaction.attempts < ctx.config.maxRetries) {
110
+ transaction.firstTransientFailureAt ??= Date.now();
111
+ const insideAvailabilityWindow = Date.now() - transaction.firstTransientFailureAt < ctx.config.availabilityRetryWindowMs;
112
+ if (transaction.attempts < ctx.config.maxRetries || insideAvailabilityWindow) {
99
113
  // Exponential backoff with full jitter on every transient retry:
100
114
  // `sleep = random(0, min(cap, base * 2^attempt))`. Throttling responses
101
115
  // (429/503) use a longer base than other transient errors. The re-enqueue
102
116
  // is scheduled rather than awaited, so one backing-off transaction cannot
103
117
  // stall unrelated commits.
104
- const { baseMs, capMs } = ctx.config.retryBackoff;
105
- let base = baseMs;
106
- try {
107
- const status = extractStatusCode(error);
108
- if (status === 429 || status === 503)
109
- base = Math.max(baseMs, 1_000);
110
- }
111
- catch { }
112
- const ceiling = Math.min(capMs, base * Math.pow(2, transaction.attempts - 1));
113
- const delay = Math.floor(Math.random() * ceiling);
118
+ const delay = transientRetryDelayMs(error, transaction.attempts, ctx.config.retryBackoff);
114
119
  ctx.store.updateStatus(transaction.id, 'pending');
115
120
  setTimeout(() => {
116
121
  // The queue may have shut down or the tx may have been settled
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@abloatai/humans",
3
- "version": "0.45.0",
3
+ "version": "0.46.0",
4
4
  "description": "The optional human-facing local-state package for Ablo: presence, live queries, and React bindings.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -84,7 +84,7 @@
84
84
  "directory": "packages/humans"
85
85
  },
86
86
  "dependencies": {
87
- "@abloatai/transaction": "^0.45.0",
87
+ "@abloatai/transaction": "^0.46.0",
88
88
  "mobx": "^6.13.7",
89
89
  "uuid": "^11.1.0",
90
90
  "zod": "^4.4.3"
@@ -44,6 +44,7 @@ import {
44
44
  claimDescription,
45
45
  descriptionFromMeta,
46
46
  participantKindFromWire,
47
+ type WireClaimSummary,
47
48
  } from '@abloatai/transaction/coordination/schema';
48
49
  import {
49
50
  isTargetTuple,
@@ -198,6 +199,38 @@ export function createClaimStream(
198
199
  }
199
200
  };
200
201
 
202
+ const observeForeignClaim = (
203
+ heldBy: string,
204
+ claim: WireClaimSummary,
205
+ participantKind?: 'user' | 'agent' | 'system',
206
+ isAgent?: boolean,
207
+ ): void => {
208
+ const description =
209
+ claim.description ??
210
+ descriptionFromMeta(claim.meta) ??
211
+ 'editing';
212
+ const { meta, ...details } = subTarget(claim);
213
+ activeByClaimId.set(claim.claimId, {
214
+ object: 'claim',
215
+ id: claim.claimId,
216
+ status: 'active',
217
+ heldBy,
218
+ participantKind: participantKindFromWire(participantKind, isAgent),
219
+ target: {
220
+ ...streamTarget(claim),
221
+ ...details,
222
+ ...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
223
+ },
224
+ description,
225
+ ttlSeconds: Math.max(
226
+ 0,
227
+ Math.floor((claim.expiresAt - Date.now()) / 1000),
228
+ ),
229
+ createdAt: claim.declaredAt,
230
+ expiresAt: claim.expiresAt,
231
+ });
232
+ };
233
+
201
234
  // ── Wire wiring ──────────────────────────────────────────────────
202
235
  let attached: ClaimTransport | null = null;
203
236
  const unsubs: (() => void)[] = [];
@@ -241,37 +274,12 @@ export function createClaimStream(
241
274
  // drops it from `others`, which is what resolves a contender's
242
275
  // `settled()`. Absent status means active (wire back-compat).
243
276
  if (claim.status && claim.status !== 'active') continue;
244
- // Resolve the always-present public field, tolerating a frame that
245
- // carries the value in `meta` rather than as an explicit description.
246
- const description =
247
- claim.description ??
248
- descriptionFromMeta(claim.meta) ??
249
- 'editing';
250
- // The frame is parsed permissively, on purpose; `declaredMeta` is where
251
- // that wire value becomes the shape the program declared.
252
- const { meta, ...details } = subTarget(claim);
253
- activeByClaimId.set(claim.claimId, {
254
- object: 'claim',
255
- id: claim.claimId,
256
- status: 'active',
257
- heldBy: event.userId,
258
- participantKind: participantKindFromWire(
259
- event.participantKind,
260
- event.isAgent,
261
- ),
262
- target: {
263
- ...streamTarget(claim),
264
- ...details,
265
- ...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
266
- },
267
- description,
268
- ttlSeconds: Math.max(
269
- 0,
270
- Math.floor((claim.expiresAt - Date.now()) / 1000),
271
- ),
272
- createdAt: claim.declaredAt,
273
- expiresAt: claim.expiresAt,
274
- });
277
+ observeForeignClaim(
278
+ event.userId,
279
+ claim,
280
+ event.participantKind,
281
+ event.isAgent,
282
+ );
275
283
  mutated = true;
276
284
  }
277
285
  if (mutated) notifyListeners();
@@ -295,6 +303,22 @@ export function createClaimStream(
295
303
  // a claim the server already rejected (would just spam both
296
304
  // sides with conflicts).
297
305
  ownClaims.delete(rejection.claimId);
306
+ // A holder on another server may have claimed before this client joined
307
+ // the row group, so its one-shot presence frame was missed. A conflict
308
+ // reply carries the authoritative holder summary; seed the same local
309
+ // state immediately instead of continuing to report the row as free.
310
+ if (
311
+ rejection.reason === 'conflict' &&
312
+ rejection.heldBy &&
313
+ rejection.heldByClaim
314
+ ) {
315
+ observeForeignClaim(
316
+ rejection.heldBy,
317
+ rejection.heldByClaim,
318
+ rejection.heldByKind,
319
+ );
320
+ notifyListeners();
321
+ }
298
322
  for (const l of rejectionListeners) {
299
323
  try {
300
324
  l(rejection);
@@ -158,6 +158,15 @@ export interface MutationQueueConfig {
158
158
  maxBatchSize: number;
159
159
  batchDelay: number;
160
160
  maxRetries: number;
161
+ /**
162
+ * Minimum wall-clock window for retrying transient write failures with the
163
+ * same durable envelope and idempotency key. This absorbs managed-database
164
+ * promotion and brief regional network incidents without double-applying a
165
+ * write. Defaults to 120 seconds: the Aurora promotion drill recovered
166
+ * writes just beyond 60 seconds, so a one-minute boundary discarded exact
167
+ * envelopes at the instant the new writer became usable.
168
+ */
169
+ availabilityRetryWindowMs: number;
161
170
  conflictResolution: ConflictResolution;
162
171
  enablePersistence: boolean;
163
172
  enableOptimistic: boolean;
@@ -289,6 +298,7 @@ export class MutationQueue extends EventEmitter {
289
298
  private replicationLagTimeouts = new Map<string, ReturnType<typeof setTimeout>>();
290
299
  private replicationLagErrors = new Map<string, AbloConnectionError>();
291
300
  private commitProcessing = false;
301
+ private commitRetryTimer: ReturnType<typeof setTimeout> | null = null;
292
302
  private lastCommitSequence = 0;
293
303
  private durableReplayBlock: AbloIdempotencyError | null = null;
294
304
  /** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
@@ -312,7 +322,11 @@ export class MutationQueue extends EventEmitter {
312
322
  private get commitLaneContext(): CommitLaneContext {
313
323
  return {
314
324
  runtime: this.runtime,
315
- config: { maxRetries: this.config.maxRetries },
325
+ config: {
326
+ maxRetries: this.config.maxRetries,
327
+ availabilityRetryWindowMs: this.config.availabilityRetryWindowMs,
328
+ retryBackoff: this.config.retryBackoff,
329
+ },
316
330
  commitLane: this.commitLane,
317
331
  commitNotifications: this.commitNotifications,
318
332
  commitMissingIds: this.commitMissingIds,
@@ -336,6 +350,13 @@ export class MutationQueue extends EventEmitter {
336
350
  noteAck: (syncId) => this.noteAck(syncId),
337
351
  isDefinitiveRejection: (error) => this.isDefinitiveRejection(error),
338
352
  isPermanentError: (error) => this.isPermanentError(error),
353
+ scheduleRetry: (delayMs) => {
354
+ if (this.commitRetryTimer !== null) clearTimeout(this.commitRetryTimer);
355
+ this.commitRetryTimer = setTimeout(() => {
356
+ this.commitRetryTimer = null;
357
+ void this.processCommitLane();
358
+ }, delayMs);
359
+ },
339
360
  emitCommitLifecycle: (event, payload) => this.emitCommitLifecycle(event, payload),
340
361
  };
341
362
  }
@@ -782,6 +803,7 @@ export class MutationQueue extends EventEmitter {
782
803
  maxBatchSize: 50, // send up to this many operations per commit
783
804
  batchDelay: 150, // milliseconds to wait for more operations before sending
784
805
  maxRetries: 3,
806
+ availabilityRetryWindowMs: 120_000,
785
807
  conflictResolution: {
786
808
  strategy: 'last-write-wins',
787
809
  },
@@ -2004,6 +2026,10 @@ export class MutationQueue extends EventEmitter {
2004
2026
  clearTimeout(this.commitOfflineGraceTimer);
2005
2027
  this.commitOfflineGraceTimer = null;
2006
2028
  }
2029
+ if (this.commitRetryTimer !== null) {
2030
+ clearTimeout(this.commitRetryTimer);
2031
+ this.commitRetryTimer = null;
2032
+ }
2007
2033
 
2008
2034
  // Clear store
2009
2035
  this.store.clear();
@@ -1,12 +1,12 @@
1
1
  import type { RuntimeContext } from '../../RuntimeContext.js';
2
2
  import type { ReadDependency, TrackDependency, OnStaleMode, StaleNotification } from '@abloatai/transaction/coordination/schema';
3
- import { AbloConnectionError } from '@abloatai/transaction/errors';
4
3
  import type { MutationCommitResult } from '@abloatai/transaction/wire/commit';
5
4
  import type {
6
5
  DurableCommitEnvelope,
7
6
  DurableCommitOperation,
8
7
  } from '@abloatai/transaction/transactions/settlement/commitEnvelope';
9
8
  import type { SealDurableCommitInput } from './commitTransport.js';
9
+ import { transientRetryDelayMs } from './failureHandling.js';
10
10
 
11
11
  export interface CommitTransaction {
12
12
  id: string;
@@ -26,6 +26,7 @@ export interface CommitTransaction {
26
26
  createdAt: number;
27
27
  attempts: number;
28
28
  transientAttempts?: number;
29
+ firstTransientFailureAt?: number;
29
30
  lastSyncId?: number;
30
31
  correlationId?: string;
31
32
  error?: Error;
@@ -38,7 +39,11 @@ export interface CommitTransaction {
38
39
 
39
40
  export interface CommitLaneContext {
40
41
  readonly runtime: RuntimeContext;
41
- readonly config: { maxRetries: number };
42
+ readonly config: {
43
+ maxRetries: number;
44
+ availabilityRetryWindowMs: number;
45
+ retryBackoff: { baseMs: number; capMs: number };
46
+ };
42
47
  readonly commitLane: CommitTransaction[];
43
48
  readonly commitNotifications: Map<string, StaleNotification[]>;
44
49
  readonly commitMissingIds: Map<string, string[]>;
@@ -56,6 +61,7 @@ export interface CommitLaneContext {
56
61
  readonly noteAck: (syncId: number | undefined) => void;
57
62
  readonly isDefinitiveRejection: (error: Error) => boolean;
58
63
  readonly isPermanentError: (error: Error) => boolean;
64
+ readonly scheduleRetry: (delayMs: number) => void;
59
65
  readonly emitCommitLifecycle: (event: string, payload: object) => void;
60
66
  }
61
67
 
@@ -165,14 +171,24 @@ export async function processCommitLane(ctx: CommitLaneContext): Promise<void> {
165
171
  } catch (cause) {
166
172
  const error = cause instanceof Error ? cause : new Error(String(cause));
167
173
  if (dispatchStarted && ctx.isDefinitiveRejection(error)) await ctx.removeDurableCommit(tx.id);
168
- if (!(error instanceof AbloConnectionError)) tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
169
- const exhausted = (tx.transientAttempts ?? 0) > ctx.config.maxRetries;
174
+ tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
175
+ tx.firstTransientFailureAt ??= Date.now();
176
+ const outsideAvailabilityWindow =
177
+ Date.now() - tx.firstTransientFailureAt >= ctx.config.availabilityRetryWindowMs;
178
+ const exhausted =
179
+ tx.transientAttempts > ctx.config.maxRetries && outsideAvailabilityWindow;
170
180
  if (!ctx.isPermanentError(error) && !exhausted) {
171
181
  tx.status = 'pending';
182
+ const delayMs = transientRetryDelayMs(
183
+ error,
184
+ tx.transientAttempts,
185
+ ctx.config.retryBackoff,
186
+ );
172
187
  ctx.runtime.logger.debug('[MutationQueue] commit lane transient', {
173
188
  txId: tx.id.slice(0, 12), attempts: tx.attempts,
174
- transientAttempts: tx.transientAttempts ?? 0, message: error.message,
189
+ transientAttempts: tx.transientAttempts, delayMs, message: error.message,
175
190
  });
191
+ ctx.scheduleRetry(delayMs);
176
192
  break;
177
193
  }
178
194
  tx.status = 'failed';
@@ -108,6 +108,8 @@ export interface QueuedMutation {
108
108
  status: 'pending' | 'executing' | 'awaiting_delta' | 'completed' | 'failed' | 'rolled_back';
109
109
  createdAt: number;
110
110
  attempts: number;
111
+ /** First transient dispatch failure in the current availability incident. */
112
+ firstTransientFailureAt?: number;
111
113
  priority: 'normal' | 'high';
112
114
  priorityScore: number; // foreign-key-aware priority, derived, used for sorting
113
115
  writeOptions?: WriteOptions;
@@ -7,7 +7,10 @@ import { extractStatusCode } from './commitPayload.js';
7
7
 
8
8
  export interface FailureHandlingContext {
9
9
  readonly runtime: RuntimeContext;
10
- readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff'>;
10
+ readonly config: Pick<
11
+ MutationQueueConfig,
12
+ 'enableOptimistic' | 'maxRetries' | 'retryBackoff' | 'availabilityRetryWindowMs'
13
+ >;
11
14
  readonly store: MutationStore;
12
15
  readonly isPermanentError: (error: Error) => boolean;
13
16
  readonly rollbackOptimistic: (transaction: QueuedMutation, reason: string, error?: Error) => Promise<void>;
@@ -17,6 +20,21 @@ export interface FailureHandlingContext {
17
20
  readonly emit: (event: string, payload: object) => boolean;
18
21
  }
19
22
 
23
+ export function transientRetryDelayMs(
24
+ error: Error,
25
+ attempt: number,
26
+ retryBackoff: MutationQueueConfig['retryBackoff'],
27
+ ): number {
28
+ const { baseMs, capMs } = retryBackoff;
29
+ let base = baseMs;
30
+ try {
31
+ const status = extractStatusCode(error);
32
+ if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
33
+ } catch {}
34
+ const ceiling = Math.min(capMs, base * Math.pow(2, Math.max(0, attempt - 1)));
35
+ return Math.floor(Math.random() * ceiling);
36
+ }
37
+
20
38
  export async function handleFailure(ctx: FailureHandlingContext, transaction: QueuedMutation, error: Error): Promise<void> {
21
39
  transaction.attempts++;
22
40
 
@@ -120,20 +138,17 @@ export async function handleFailure(ctx: FailureHandlingContext, transaction: Qu
120
138
  return;
121
139
  }
122
140
 
123
- if (transaction.attempts < ctx.config.maxRetries) {
141
+ transaction.firstTransientFailureAt ??= Date.now();
142
+ const insideAvailabilityWindow =
143
+ Date.now() - transaction.firstTransientFailureAt < ctx.config.availabilityRetryWindowMs;
144
+
145
+ if (transaction.attempts < ctx.config.maxRetries || insideAvailabilityWindow) {
124
146
  // Exponential backoff with full jitter on every transient retry:
125
147
  // `sleep = random(0, min(cap, base * 2^attempt))`. Throttling responses
126
148
  // (429/503) use a longer base than other transient errors. The re-enqueue
127
149
  // is scheduled rather than awaited, so one backing-off transaction cannot
128
150
  // stall unrelated commits.
129
- const { baseMs, capMs } = ctx.config.retryBackoff;
130
- let base = baseMs;
131
- try {
132
- const status = extractStatusCode(error);
133
- if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
134
- } catch {}
135
- const ceiling = Math.min(capMs, base * Math.pow(2, transaction.attempts - 1));
136
- const delay = Math.floor(Math.random() * ceiling);
151
+ const delay = transientRetryDelayMs(error, transaction.attempts, ctx.config.retryBackoff);
137
152
 
138
153
  ctx.store.updateStatus(transaction.id, 'pending');
139
154
  setTimeout(() => {