@abloatai/humans 0.45.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/local/sync/createClaimStream.js +33 -24
- package/dist/local/transactions/mutations/MutationQueue.d.ts +19 -0
- package/dist/local/transactions/mutations/MutationQueue.js +19 -1
- package/dist/local/transactions/mutations/commitLane.d.ts +7 -0
- package/dist/local/transactions/mutations/commitLane.js +8 -5
- package/dist/local/transactions/mutations/commitPayload.d.ts +2 -0
- package/dist/local/transactions/mutations/failureHandling.d.ts +2 -1
- package/dist/local/transactions/mutations/failureHandling.js +16 -11
- package/package.json +2 -2
- package/src/local/sync/createClaimStream.ts +55 -31
- package/src/local/transactions/mutations/MutationQueue.ts +27 -1
- package/src/local/transactions/mutations/commitLane.ts +21 -5
- package/src/local/transactions/mutations/commitPayload.ts +2 -0
- package/src/local/transactions/mutations/failureHandling.ts +25 -10
|
@@ -83,6 +83,28 @@ export function createClaimStream(config, transport = null) {
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
};
|
|
86
|
+
const observeForeignClaim = (heldBy, claim, participantKind, isAgent) => {
|
|
87
|
+
const description = claim.description ??
|
|
88
|
+
descriptionFromMeta(claim.meta) ??
|
|
89
|
+
'editing';
|
|
90
|
+
const { meta, ...details } = subTarget(claim);
|
|
91
|
+
activeByClaimId.set(claim.claimId, {
|
|
92
|
+
object: 'claim',
|
|
93
|
+
id: claim.claimId,
|
|
94
|
+
status: 'active',
|
|
95
|
+
heldBy,
|
|
96
|
+
participantKind: participantKindFromWire(participantKind, isAgent),
|
|
97
|
+
target: {
|
|
98
|
+
...streamTarget(claim),
|
|
99
|
+
...details,
|
|
100
|
+
...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
|
|
101
|
+
},
|
|
102
|
+
description,
|
|
103
|
+
ttlSeconds: Math.max(0, Math.floor((claim.expiresAt - Date.now()) / 1000)),
|
|
104
|
+
createdAt: claim.declaredAt,
|
|
105
|
+
expiresAt: claim.expiresAt,
|
|
106
|
+
});
|
|
107
|
+
};
|
|
86
108
|
// ── Wire wiring ──────────────────────────────────────────────────
|
|
87
109
|
let attached = null;
|
|
88
110
|
const unsubs = [];
|
|
@@ -125,30 +147,7 @@ export function createClaimStream(config, transport = null) {
|
|
|
125
147
|
// `settled()`. Absent status means active (wire back-compat).
|
|
126
148
|
if (claim.status && claim.status !== 'active')
|
|
127
149
|
continue;
|
|
128
|
-
|
|
129
|
-
// carries the value in `meta` rather than as an explicit description.
|
|
130
|
-
const description = claim.description ??
|
|
131
|
-
descriptionFromMeta(claim.meta) ??
|
|
132
|
-
'editing';
|
|
133
|
-
// The frame is parsed permissively, on purpose; `declaredMeta` is where
|
|
134
|
-
// that wire value becomes the shape the program declared.
|
|
135
|
-
const { meta, ...details } = subTarget(claim);
|
|
136
|
-
activeByClaimId.set(claim.claimId, {
|
|
137
|
-
object: 'claim',
|
|
138
|
-
id: claim.claimId,
|
|
139
|
-
status: 'active',
|
|
140
|
-
heldBy: event.userId,
|
|
141
|
-
participantKind: participantKindFromWire(event.participantKind, event.isAgent),
|
|
142
|
-
target: {
|
|
143
|
-
...streamTarget(claim),
|
|
144
|
-
...details,
|
|
145
|
-
...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
|
|
146
|
-
},
|
|
147
|
-
description,
|
|
148
|
-
ttlSeconds: Math.max(0, Math.floor((claim.expiresAt - Date.now()) / 1000)),
|
|
149
|
-
createdAt: claim.declaredAt,
|
|
150
|
-
expiresAt: claim.expiresAt,
|
|
151
|
-
});
|
|
150
|
+
observeForeignClaim(event.userId, claim, event.participantKind, event.isAgent);
|
|
152
151
|
mutated = true;
|
|
153
152
|
}
|
|
154
153
|
if (mutated)
|
|
@@ -168,6 +167,16 @@ export function createClaimStream(config, transport = null) {
|
|
|
168
167
|
// a claim the server already rejected (would just spam both
|
|
169
168
|
// sides with conflicts).
|
|
170
169
|
ownClaims.delete(rejection.claimId);
|
|
170
|
+
// A holder on another server may have claimed before this client joined
|
|
171
|
+
// the row group, so its one-shot presence frame was missed. A conflict
|
|
172
|
+
// reply carries the authoritative holder summary; seed the same local
|
|
173
|
+
// state immediately instead of continuing to report the row as free.
|
|
174
|
+
if (rejection.reason === 'conflict' &&
|
|
175
|
+
rejection.heldBy &&
|
|
176
|
+
rejection.heldByClaim) {
|
|
177
|
+
observeForeignClaim(rejection.heldBy, rejection.heldByClaim, rejection.heldByKind);
|
|
178
|
+
notifyListeners();
|
|
179
|
+
}
|
|
171
180
|
for (const l of rejectionListeners) {
|
|
172
181
|
try {
|
|
173
182
|
l(rejection);
|
|
@@ -45,6 +45,15 @@ export interface MutationQueueConfig {
|
|
|
45
45
|
maxBatchSize: number;
|
|
46
46
|
batchDelay: number;
|
|
47
47
|
maxRetries: number;
|
|
48
|
+
/**
|
|
49
|
+
* Minimum wall-clock window for retrying transient write failures with the
|
|
50
|
+
* same durable envelope and idempotency key. This absorbs managed-database
|
|
51
|
+
* promotion and brief regional network incidents without double-applying a
|
|
52
|
+
* write. Defaults to 120 seconds: the Aurora promotion drill recovered
|
|
53
|
+
* writes just beyond 60 seconds, so a one-minute boundary discarded exact
|
|
54
|
+
* envelopes at the instant the new writer became usable.
|
|
55
|
+
*/
|
|
56
|
+
availabilityRetryWindowMs: number;
|
|
48
57
|
conflictResolution: ConflictResolution;
|
|
49
58
|
enablePersistence: boolean;
|
|
50
59
|
enableOptimistic: boolean;
|
|
@@ -127,6 +136,7 @@ export declare class MutationQueue extends EventEmitter {
|
|
|
127
136
|
private replicationLagTimeouts;
|
|
128
137
|
private replicationLagErrors;
|
|
129
138
|
private commitProcessing;
|
|
139
|
+
private commitRetryTimer;
|
|
130
140
|
private lastCommitSequence;
|
|
131
141
|
private durableReplayBlock;
|
|
132
142
|
/** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
|
|
@@ -418,6 +428,15 @@ export declare class MutationQueue extends EventEmitter {
|
|
|
418
428
|
maxBatchSize: number;
|
|
419
429
|
batchDelay: number;
|
|
420
430
|
maxRetries: number;
|
|
431
|
+
/**
|
|
432
|
+
* Minimum wall-clock window for retrying transient write failures with the
|
|
433
|
+
* same durable envelope and idempotency key. This absorbs managed-database
|
|
434
|
+
* promotion and brief regional network incidents without double-applying a
|
|
435
|
+
* write. Defaults to 120 seconds: the Aurora promotion drill recovered
|
|
436
|
+
* writes just beyond 60 seconds, so a one-minute boundary discarded exact
|
|
437
|
+
* envelopes at the instant the new writer became usable.
|
|
438
|
+
*/
|
|
439
|
+
availabilityRetryWindowMs: number;
|
|
421
440
|
conflictResolution: ConflictResolution;
|
|
422
441
|
enablePersistence: boolean;
|
|
423
442
|
enableOptimistic: boolean;
|
|
@@ -104,6 +104,7 @@ export class MutationQueue extends EventEmitter {
|
|
|
104
104
|
replicationLagTimeouts = new Map();
|
|
105
105
|
replicationLagErrors = new Map();
|
|
106
106
|
commitProcessing = false;
|
|
107
|
+
commitRetryTimer = null;
|
|
107
108
|
lastCommitSequence = 0;
|
|
108
109
|
durableReplayBlock = null;
|
|
109
110
|
/** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
|
|
@@ -125,7 +126,11 @@ export class MutationQueue extends EventEmitter {
|
|
|
125
126
|
get commitLaneContext() {
|
|
126
127
|
return {
|
|
127
128
|
runtime: this.runtime,
|
|
128
|
-
config: {
|
|
129
|
+
config: {
|
|
130
|
+
maxRetries: this.config.maxRetries,
|
|
131
|
+
availabilityRetryWindowMs: this.config.availabilityRetryWindowMs,
|
|
132
|
+
retryBackoff: this.config.retryBackoff,
|
|
133
|
+
},
|
|
129
134
|
commitLane: this.commitLane,
|
|
130
135
|
commitNotifications: this.commitNotifications,
|
|
131
136
|
commitMissingIds: this.commitMissingIds,
|
|
@@ -147,6 +152,14 @@ export class MutationQueue extends EventEmitter {
|
|
|
147
152
|
noteAck: (syncId) => this.noteAck(syncId),
|
|
148
153
|
isDefinitiveRejection: (error) => this.isDefinitiveRejection(error),
|
|
149
154
|
isPermanentError: (error) => this.isPermanentError(error),
|
|
155
|
+
scheduleRetry: (delayMs) => {
|
|
156
|
+
if (this.commitRetryTimer !== null)
|
|
157
|
+
clearTimeout(this.commitRetryTimer);
|
|
158
|
+
this.commitRetryTimer = setTimeout(() => {
|
|
159
|
+
this.commitRetryTimer = null;
|
|
160
|
+
void this.processCommitLane();
|
|
161
|
+
}, delayMs);
|
|
162
|
+
},
|
|
150
163
|
emitCommitLifecycle: (event, payload) => this.emitCommitLifecycle(event, payload),
|
|
151
164
|
};
|
|
152
165
|
}
|
|
@@ -516,6 +529,7 @@ export class MutationQueue extends EventEmitter {
|
|
|
516
529
|
maxBatchSize: 50, // send up to this many operations per commit
|
|
517
530
|
batchDelay: 150, // milliseconds to wait for more operations before sending
|
|
518
531
|
maxRetries: 3,
|
|
532
|
+
availabilityRetryWindowMs: 120_000,
|
|
519
533
|
conflictResolution: {
|
|
520
534
|
strategy: 'last-write-wins',
|
|
521
535
|
},
|
|
@@ -1506,6 +1520,10 @@ export class MutationQueue extends EventEmitter {
|
|
|
1506
1520
|
clearTimeout(this.commitOfflineGraceTimer);
|
|
1507
1521
|
this.commitOfflineGraceTimer = null;
|
|
1508
1522
|
}
|
|
1523
|
+
if (this.commitRetryTimer !== null) {
|
|
1524
|
+
clearTimeout(this.commitRetryTimer);
|
|
1525
|
+
this.commitRetryTimer = null;
|
|
1526
|
+
}
|
|
1509
1527
|
// Clear store
|
|
1510
1528
|
this.store.clear();
|
|
1511
1529
|
this.localMutationPort.updates.clear();
|
|
@@ -21,6 +21,7 @@ export interface CommitTransaction {
|
|
|
21
21
|
createdAt: number;
|
|
22
22
|
attempts: number;
|
|
23
23
|
transientAttempts?: number;
|
|
24
|
+
firstTransientFailureAt?: number;
|
|
24
25
|
lastSyncId?: number;
|
|
25
26
|
correlationId?: string;
|
|
26
27
|
error?: Error;
|
|
@@ -34,6 +35,11 @@ export interface CommitLaneContext {
|
|
|
34
35
|
readonly runtime: RuntimeContext;
|
|
35
36
|
readonly config: {
|
|
36
37
|
maxRetries: number;
|
|
38
|
+
availabilityRetryWindowMs: number;
|
|
39
|
+
retryBackoff: {
|
|
40
|
+
baseMs: number;
|
|
41
|
+
capMs: number;
|
|
42
|
+
};
|
|
37
43
|
};
|
|
38
44
|
readonly commitLane: CommitTransaction[];
|
|
39
45
|
readonly commitNotifications: Map<string, StaleNotification[]>;
|
|
@@ -52,6 +58,7 @@ export interface CommitLaneContext {
|
|
|
52
58
|
readonly noteAck: (syncId: number | undefined) => void;
|
|
53
59
|
readonly isDefinitiveRejection: (error: Error) => boolean;
|
|
54
60
|
readonly isPermanentError: (error: Error) => boolean;
|
|
61
|
+
readonly scheduleRetry: (delayMs: number) => void;
|
|
55
62
|
readonly emitCommitLifecycle: (event: string, payload: object) => void;
|
|
56
63
|
}
|
|
57
64
|
export interface CommitReceiptContext {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { transientRetryDelayMs } from './failureHandling.js';
|
|
2
2
|
export function waitForCommitReceipt(ctx, clientTxId) {
|
|
3
3
|
const drainNotifications = () => {
|
|
4
4
|
const notifications = ctx.commitNotifications.get(clientTxId);
|
|
@@ -112,15 +112,18 @@ export async function processCommitLane(ctx) {
|
|
|
112
112
|
const error = cause instanceof Error ? cause : new Error(String(cause));
|
|
113
113
|
if (dispatchStarted && ctx.isDefinitiveRejection(error))
|
|
114
114
|
await ctx.removeDurableCommit(tx.id);
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
const
|
|
115
|
+
tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
|
|
116
|
+
tx.firstTransientFailureAt ??= Date.now();
|
|
117
|
+
const outsideAvailabilityWindow = Date.now() - tx.firstTransientFailureAt >= ctx.config.availabilityRetryWindowMs;
|
|
118
|
+
const exhausted = tx.transientAttempts > ctx.config.maxRetries && outsideAvailabilityWindow;
|
|
118
119
|
if (!ctx.isPermanentError(error) && !exhausted) {
|
|
119
120
|
tx.status = 'pending';
|
|
121
|
+
const delayMs = transientRetryDelayMs(error, tx.transientAttempts, ctx.config.retryBackoff);
|
|
120
122
|
ctx.runtime.logger.debug('[MutationQueue] commit lane transient', {
|
|
121
123
|
txId: tx.id.slice(0, 12), attempts: tx.attempts,
|
|
122
|
-
transientAttempts: tx.transientAttempts
|
|
124
|
+
transientAttempts: tx.transientAttempts, delayMs, message: error.message,
|
|
123
125
|
});
|
|
126
|
+
ctx.scheduleRetry(delayMs);
|
|
124
127
|
break;
|
|
125
128
|
}
|
|
126
129
|
tx.status = 'failed';
|
|
@@ -54,6 +54,8 @@ export interface QueuedMutation {
|
|
|
54
54
|
status: 'pending' | 'executing' | 'awaiting_delta' | 'completed' | 'failed' | 'rolled_back';
|
|
55
55
|
createdAt: number;
|
|
56
56
|
attempts: number;
|
|
57
|
+
/** First transient dispatch failure in the current availability incident. */
|
|
58
|
+
firstTransientFailureAt?: number;
|
|
57
59
|
priority: 'normal' | 'high';
|
|
58
60
|
priorityScore: number;
|
|
59
61
|
writeOptions?: WriteOptions;
|
|
@@ -4,7 +4,7 @@ import type { QueuedMutation } from './commitPayload.js';
|
|
|
4
4
|
import type { MutationStore } from './MutationStore.js';
|
|
5
5
|
export interface FailureHandlingContext {
|
|
6
6
|
readonly runtime: RuntimeContext;
|
|
7
|
-
readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff'>;
|
|
7
|
+
readonly config: Pick<MutationQueueConfig, 'enableOptimistic' | 'maxRetries' | 'retryBackoff' | 'availabilityRetryWindowMs'>;
|
|
8
8
|
readonly store: MutationStore;
|
|
9
9
|
readonly isPermanentError: (error: Error) => boolean;
|
|
10
10
|
readonly rollbackOptimistic: (transaction: QueuedMutation, reason: string, error?: Error) => Promise<void>;
|
|
@@ -13,4 +13,5 @@ export interface FailureHandlingContext {
|
|
|
13
13
|
readonly setLastPermanentErrorSignature: (signature: string) => void;
|
|
14
14
|
readonly emit: (event: string, payload: object) => boolean;
|
|
15
15
|
}
|
|
16
|
+
export declare function transientRetryDelayMs(error: Error, attempt: number, retryBackoff: MutationQueueConfig['retryBackoff']): number;
|
|
16
17
|
export declare function handleFailure(ctx: FailureHandlingContext, transaction: QueuedMutation, error: Error): Promise<void>;
|
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
import { AbloError } from '@abloatai/transaction/errors';
|
|
2
2
|
import { extractStatusCode } from './commitPayload.js';
|
|
3
|
+
export function transientRetryDelayMs(error, attempt, retryBackoff) {
|
|
4
|
+
const { baseMs, capMs } = retryBackoff;
|
|
5
|
+
let base = baseMs;
|
|
6
|
+
try {
|
|
7
|
+
const status = extractStatusCode(error);
|
|
8
|
+
if (status === 429 || status === 503)
|
|
9
|
+
base = Math.max(baseMs, 1_000);
|
|
10
|
+
}
|
|
11
|
+
catch { }
|
|
12
|
+
const ceiling = Math.min(capMs, base * Math.pow(2, Math.max(0, attempt - 1)));
|
|
13
|
+
return Math.floor(Math.random() * ceiling);
|
|
14
|
+
}
|
|
3
15
|
export async function handleFailure(ctx, transaction, error) {
|
|
4
16
|
transaction.attempts++;
|
|
5
17
|
// Check whether this is a permanent error that should not be retried.
|
|
@@ -95,22 +107,15 @@ export async function handleFailure(ctx, transaction, error) {
|
|
|
95
107
|
ctx.emit(`transaction:failed:${transaction.id}`, { error });
|
|
96
108
|
return;
|
|
97
109
|
}
|
|
98
|
-
|
|
110
|
+
transaction.firstTransientFailureAt ??= Date.now();
|
|
111
|
+
const insideAvailabilityWindow = Date.now() - transaction.firstTransientFailureAt < ctx.config.availabilityRetryWindowMs;
|
|
112
|
+
if (transaction.attempts < ctx.config.maxRetries || insideAvailabilityWindow) {
|
|
99
113
|
// Exponential backoff with full jitter on every transient retry:
|
|
100
114
|
// `sleep = random(0, min(cap, base * 2^attempt))`. Throttling responses
|
|
101
115
|
// (429/503) use a longer base than other transient errors. The re-enqueue
|
|
102
116
|
// is scheduled rather than awaited, so one backing-off transaction cannot
|
|
103
117
|
// stall unrelated commits.
|
|
104
|
-
const
|
|
105
|
-
let base = baseMs;
|
|
106
|
-
try {
|
|
107
|
-
const status = extractStatusCode(error);
|
|
108
|
-
if (status === 429 || status === 503)
|
|
109
|
-
base = Math.max(baseMs, 1_000);
|
|
110
|
-
}
|
|
111
|
-
catch { }
|
|
112
|
-
const ceiling = Math.min(capMs, base * Math.pow(2, transaction.attempts - 1));
|
|
113
|
-
const delay = Math.floor(Math.random() * ceiling);
|
|
118
|
+
const delay = transientRetryDelayMs(error, transaction.attempts, ctx.config.retryBackoff);
|
|
114
119
|
ctx.store.updateStatus(transaction.id, 'pending');
|
|
115
120
|
setTimeout(() => {
|
|
116
121
|
// The queue may have shut down or the tx may have been settled
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@abloatai/humans",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.46.0",
|
|
4
4
|
"description": "The optional human-facing local-state package for Ablo: presence, live queries, and React bindings.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -84,7 +84,7 @@
|
|
|
84
84
|
"directory": "packages/humans"
|
|
85
85
|
},
|
|
86
86
|
"dependencies": {
|
|
87
|
-
"@abloatai/transaction": "^0.
|
|
87
|
+
"@abloatai/transaction": "^0.46.0",
|
|
88
88
|
"mobx": "^6.13.7",
|
|
89
89
|
"uuid": "^11.1.0",
|
|
90
90
|
"zod": "^4.4.3"
|
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
claimDescription,
|
|
45
45
|
descriptionFromMeta,
|
|
46
46
|
participantKindFromWire,
|
|
47
|
+
type WireClaimSummary,
|
|
47
48
|
} from '@abloatai/transaction/coordination/schema';
|
|
48
49
|
import {
|
|
49
50
|
isTargetTuple,
|
|
@@ -198,6 +199,38 @@ export function createClaimStream(
|
|
|
198
199
|
}
|
|
199
200
|
};
|
|
200
201
|
|
|
202
|
+
const observeForeignClaim = (
|
|
203
|
+
heldBy: string,
|
|
204
|
+
claim: WireClaimSummary,
|
|
205
|
+
participantKind?: 'user' | 'agent' | 'system',
|
|
206
|
+
isAgent?: boolean,
|
|
207
|
+
): void => {
|
|
208
|
+
const description =
|
|
209
|
+
claim.description ??
|
|
210
|
+
descriptionFromMeta(claim.meta) ??
|
|
211
|
+
'editing';
|
|
212
|
+
const { meta, ...details } = subTarget(claim);
|
|
213
|
+
activeByClaimId.set(claim.claimId, {
|
|
214
|
+
object: 'claim',
|
|
215
|
+
id: claim.claimId,
|
|
216
|
+
status: 'active',
|
|
217
|
+
heldBy,
|
|
218
|
+
participantKind: participantKindFromWire(participantKind, isAgent),
|
|
219
|
+
target: {
|
|
220
|
+
...streamTarget(claim),
|
|
221
|
+
...details,
|
|
222
|
+
...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
|
|
223
|
+
},
|
|
224
|
+
description,
|
|
225
|
+
ttlSeconds: Math.max(
|
|
226
|
+
0,
|
|
227
|
+
Math.floor((claim.expiresAt - Date.now()) / 1000),
|
|
228
|
+
),
|
|
229
|
+
createdAt: claim.declaredAt,
|
|
230
|
+
expiresAt: claim.expiresAt,
|
|
231
|
+
});
|
|
232
|
+
};
|
|
233
|
+
|
|
201
234
|
// ── Wire wiring ──────────────────────────────────────────────────
|
|
202
235
|
let attached: ClaimTransport | null = null;
|
|
203
236
|
const unsubs: (() => void)[] = [];
|
|
@@ -241,37 +274,12 @@ export function createClaimStream(
|
|
|
241
274
|
// drops it from `others`, which is what resolves a contender's
|
|
242
275
|
// `settled()`. Absent status means active (wire back-compat).
|
|
243
276
|
if (claim.status && claim.status !== 'active') continue;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
// The frame is parsed permissively, on purpose; `declaredMeta` is where
|
|
251
|
-
// that wire value becomes the shape the program declared.
|
|
252
|
-
const { meta, ...details } = subTarget(claim);
|
|
253
|
-
activeByClaimId.set(claim.claimId, {
|
|
254
|
-
object: 'claim',
|
|
255
|
-
id: claim.claimId,
|
|
256
|
-
status: 'active',
|
|
257
|
-
heldBy: event.userId,
|
|
258
|
-
participantKind: participantKindFromWire(
|
|
259
|
-
event.participantKind,
|
|
260
|
-
event.isAgent,
|
|
261
|
-
),
|
|
262
|
-
target: {
|
|
263
|
-
...streamTarget(claim),
|
|
264
|
-
...details,
|
|
265
|
-
...(meta !== undefined ? { meta: declaredMeta(meta) } : {}),
|
|
266
|
-
},
|
|
267
|
-
description,
|
|
268
|
-
ttlSeconds: Math.max(
|
|
269
|
-
0,
|
|
270
|
-
Math.floor((claim.expiresAt - Date.now()) / 1000),
|
|
271
|
-
),
|
|
272
|
-
createdAt: claim.declaredAt,
|
|
273
|
-
expiresAt: claim.expiresAt,
|
|
274
|
-
});
|
|
277
|
+
observeForeignClaim(
|
|
278
|
+
event.userId,
|
|
279
|
+
claim,
|
|
280
|
+
event.participantKind,
|
|
281
|
+
event.isAgent,
|
|
282
|
+
);
|
|
275
283
|
mutated = true;
|
|
276
284
|
}
|
|
277
285
|
if (mutated) notifyListeners();
|
|
@@ -295,6 +303,22 @@ export function createClaimStream(
|
|
|
295
303
|
// a claim the server already rejected (would just spam both
|
|
296
304
|
// sides with conflicts).
|
|
297
305
|
ownClaims.delete(rejection.claimId);
|
|
306
|
+
// A holder on another server may have claimed before this client joined
|
|
307
|
+
// the row group, so its one-shot presence frame was missed. A conflict
|
|
308
|
+
// reply carries the authoritative holder summary; seed the same local
|
|
309
|
+
// state immediately instead of continuing to report the row as free.
|
|
310
|
+
if (
|
|
311
|
+
rejection.reason === 'conflict' &&
|
|
312
|
+
rejection.heldBy &&
|
|
313
|
+
rejection.heldByClaim
|
|
314
|
+
) {
|
|
315
|
+
observeForeignClaim(
|
|
316
|
+
rejection.heldBy,
|
|
317
|
+
rejection.heldByClaim,
|
|
318
|
+
rejection.heldByKind,
|
|
319
|
+
);
|
|
320
|
+
notifyListeners();
|
|
321
|
+
}
|
|
298
322
|
for (const l of rejectionListeners) {
|
|
299
323
|
try {
|
|
300
324
|
l(rejection);
|
|
@@ -158,6 +158,15 @@ export interface MutationQueueConfig {
|
|
|
158
158
|
maxBatchSize: number;
|
|
159
159
|
batchDelay: number;
|
|
160
160
|
maxRetries: number;
|
|
161
|
+
/**
|
|
162
|
+
* Minimum wall-clock window for retrying transient write failures with the
|
|
163
|
+
* same durable envelope and idempotency key. This absorbs managed-database
|
|
164
|
+
* promotion and brief regional network incidents without double-applying a
|
|
165
|
+
* write. Defaults to 120 seconds: the Aurora promotion drill recovered
|
|
166
|
+
* writes just beyond 60 seconds, so a one-minute boundary discarded exact
|
|
167
|
+
* envelopes at the instant the new writer became usable.
|
|
168
|
+
*/
|
|
169
|
+
availabilityRetryWindowMs: number;
|
|
161
170
|
conflictResolution: ConflictResolution;
|
|
162
171
|
enablePersistence: boolean;
|
|
163
172
|
enableOptimistic: boolean;
|
|
@@ -289,6 +298,7 @@ export class MutationQueue extends EventEmitter {
|
|
|
289
298
|
private replicationLagTimeouts = new Map<string, ReturnType<typeof setTimeout>>();
|
|
290
299
|
private replicationLagErrors = new Map<string, AbloConnectionError>();
|
|
291
300
|
private commitProcessing = false;
|
|
301
|
+
private commitRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
|
292
302
|
private lastCommitSequence = 0;
|
|
293
303
|
private durableReplayBlock: AbloIdempotencyError | null = null;
|
|
294
304
|
/** Browser-backed strict outbox; absent for standalone/in-memory consumers. */
|
|
@@ -312,7 +322,11 @@ export class MutationQueue extends EventEmitter {
|
|
|
312
322
|
private get commitLaneContext(): CommitLaneContext {
|
|
313
323
|
return {
|
|
314
324
|
runtime: this.runtime,
|
|
315
|
-
config: {
|
|
325
|
+
config: {
|
|
326
|
+
maxRetries: this.config.maxRetries,
|
|
327
|
+
availabilityRetryWindowMs: this.config.availabilityRetryWindowMs,
|
|
328
|
+
retryBackoff: this.config.retryBackoff,
|
|
329
|
+
},
|
|
316
330
|
commitLane: this.commitLane,
|
|
317
331
|
commitNotifications: this.commitNotifications,
|
|
318
332
|
commitMissingIds: this.commitMissingIds,
|
|
@@ -336,6 +350,13 @@ export class MutationQueue extends EventEmitter {
|
|
|
336
350
|
noteAck: (syncId) => this.noteAck(syncId),
|
|
337
351
|
isDefinitiveRejection: (error) => this.isDefinitiveRejection(error),
|
|
338
352
|
isPermanentError: (error) => this.isPermanentError(error),
|
|
353
|
+
scheduleRetry: (delayMs) => {
|
|
354
|
+
if (this.commitRetryTimer !== null) clearTimeout(this.commitRetryTimer);
|
|
355
|
+
this.commitRetryTimer = setTimeout(() => {
|
|
356
|
+
this.commitRetryTimer = null;
|
|
357
|
+
void this.processCommitLane();
|
|
358
|
+
}, delayMs);
|
|
359
|
+
},
|
|
339
360
|
emitCommitLifecycle: (event, payload) => this.emitCommitLifecycle(event, payload),
|
|
340
361
|
};
|
|
341
362
|
}
|
|
@@ -782,6 +803,7 @@ export class MutationQueue extends EventEmitter {
|
|
|
782
803
|
maxBatchSize: 50, // send up to this many operations per commit
|
|
783
804
|
batchDelay: 150, // milliseconds to wait for more operations before sending
|
|
784
805
|
maxRetries: 3,
|
|
806
|
+
availabilityRetryWindowMs: 120_000,
|
|
785
807
|
conflictResolution: {
|
|
786
808
|
strategy: 'last-write-wins',
|
|
787
809
|
},
|
|
@@ -2004,6 +2026,10 @@ export class MutationQueue extends EventEmitter {
|
|
|
2004
2026
|
clearTimeout(this.commitOfflineGraceTimer);
|
|
2005
2027
|
this.commitOfflineGraceTimer = null;
|
|
2006
2028
|
}
|
|
2029
|
+
if (this.commitRetryTimer !== null) {
|
|
2030
|
+
clearTimeout(this.commitRetryTimer);
|
|
2031
|
+
this.commitRetryTimer = null;
|
|
2032
|
+
}
|
|
2007
2033
|
|
|
2008
2034
|
// Clear store
|
|
2009
2035
|
this.store.clear();
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { RuntimeContext } from '../../RuntimeContext.js';
|
|
2
2
|
import type { ReadDependency, TrackDependency, OnStaleMode, StaleNotification } from '@abloatai/transaction/coordination/schema';
|
|
3
|
-
import { AbloConnectionError } from '@abloatai/transaction/errors';
|
|
4
3
|
import type { MutationCommitResult } from '@abloatai/transaction/wire/commit';
|
|
5
4
|
import type {
|
|
6
5
|
DurableCommitEnvelope,
|
|
7
6
|
DurableCommitOperation,
|
|
8
7
|
} from '@abloatai/transaction/transactions/settlement/commitEnvelope';
|
|
9
8
|
import type { SealDurableCommitInput } from './commitTransport.js';
|
|
9
|
+
import { transientRetryDelayMs } from './failureHandling.js';
|
|
10
10
|
|
|
11
11
|
export interface CommitTransaction {
|
|
12
12
|
id: string;
|
|
@@ -26,6 +26,7 @@ export interface CommitTransaction {
|
|
|
26
26
|
createdAt: number;
|
|
27
27
|
attempts: number;
|
|
28
28
|
transientAttempts?: number;
|
|
29
|
+
firstTransientFailureAt?: number;
|
|
29
30
|
lastSyncId?: number;
|
|
30
31
|
correlationId?: string;
|
|
31
32
|
error?: Error;
|
|
@@ -38,7 +39,11 @@ export interface CommitTransaction {
|
|
|
38
39
|
|
|
39
40
|
export interface CommitLaneContext {
|
|
40
41
|
readonly runtime: RuntimeContext;
|
|
41
|
-
readonly config: {
|
|
42
|
+
readonly config: {
|
|
43
|
+
maxRetries: number;
|
|
44
|
+
availabilityRetryWindowMs: number;
|
|
45
|
+
retryBackoff: { baseMs: number; capMs: number };
|
|
46
|
+
};
|
|
42
47
|
readonly commitLane: CommitTransaction[];
|
|
43
48
|
readonly commitNotifications: Map<string, StaleNotification[]>;
|
|
44
49
|
readonly commitMissingIds: Map<string, string[]>;
|
|
@@ -56,6 +61,7 @@ export interface CommitLaneContext {
|
|
|
56
61
|
readonly noteAck: (syncId: number | undefined) => void;
|
|
57
62
|
readonly isDefinitiveRejection: (error: Error) => boolean;
|
|
58
63
|
readonly isPermanentError: (error: Error) => boolean;
|
|
64
|
+
readonly scheduleRetry: (delayMs: number) => void;
|
|
59
65
|
readonly emitCommitLifecycle: (event: string, payload: object) => void;
|
|
60
66
|
}
|
|
61
67
|
|
|
@@ -165,14 +171,24 @@ export async function processCommitLane(ctx: CommitLaneContext): Promise<void> {
|
|
|
165
171
|
} catch (cause) {
|
|
166
172
|
const error = cause instanceof Error ? cause : new Error(String(cause));
|
|
167
173
|
if (dispatchStarted && ctx.isDefinitiveRejection(error)) await ctx.removeDurableCommit(tx.id);
|
|
168
|
-
|
|
169
|
-
|
|
174
|
+
tx.transientAttempts = (tx.transientAttempts ?? 0) + 1;
|
|
175
|
+
tx.firstTransientFailureAt ??= Date.now();
|
|
176
|
+
const outsideAvailabilityWindow =
|
|
177
|
+
Date.now() - tx.firstTransientFailureAt >= ctx.config.availabilityRetryWindowMs;
|
|
178
|
+
const exhausted =
|
|
179
|
+
tx.transientAttempts > ctx.config.maxRetries && outsideAvailabilityWindow;
|
|
170
180
|
if (!ctx.isPermanentError(error) && !exhausted) {
|
|
171
181
|
tx.status = 'pending';
|
|
182
|
+
const delayMs = transientRetryDelayMs(
|
|
183
|
+
error,
|
|
184
|
+
tx.transientAttempts,
|
|
185
|
+
ctx.config.retryBackoff,
|
|
186
|
+
);
|
|
172
187
|
ctx.runtime.logger.debug('[MutationQueue] commit lane transient', {
|
|
173
188
|
txId: tx.id.slice(0, 12), attempts: tx.attempts,
|
|
174
|
-
transientAttempts: tx.transientAttempts
|
|
189
|
+
transientAttempts: tx.transientAttempts, delayMs, message: error.message,
|
|
175
190
|
});
|
|
191
|
+
ctx.scheduleRetry(delayMs);
|
|
176
192
|
break;
|
|
177
193
|
}
|
|
178
194
|
tx.status = 'failed';
|
|
@@ -108,6 +108,8 @@ export interface QueuedMutation {
|
|
|
108
108
|
status: 'pending' | 'executing' | 'awaiting_delta' | 'completed' | 'failed' | 'rolled_back';
|
|
109
109
|
createdAt: number;
|
|
110
110
|
attempts: number;
|
|
111
|
+
/** First transient dispatch failure in the current availability incident. */
|
|
112
|
+
firstTransientFailureAt?: number;
|
|
111
113
|
priority: 'normal' | 'high';
|
|
112
114
|
priorityScore: number; // foreign-key-aware priority, derived, used for sorting
|
|
113
115
|
writeOptions?: WriteOptions;
|
|
@@ -7,7 +7,10 @@ import { extractStatusCode } from './commitPayload.js';
|
|
|
7
7
|
|
|
8
8
|
export interface FailureHandlingContext {
|
|
9
9
|
readonly runtime: RuntimeContext;
|
|
10
|
-
readonly config: Pick<
|
|
10
|
+
readonly config: Pick<
|
|
11
|
+
MutationQueueConfig,
|
|
12
|
+
'enableOptimistic' | 'maxRetries' | 'retryBackoff' | 'availabilityRetryWindowMs'
|
|
13
|
+
>;
|
|
11
14
|
readonly store: MutationStore;
|
|
12
15
|
readonly isPermanentError: (error: Error) => boolean;
|
|
13
16
|
readonly rollbackOptimistic: (transaction: QueuedMutation, reason: string, error?: Error) => Promise<void>;
|
|
@@ -17,6 +20,21 @@ export interface FailureHandlingContext {
|
|
|
17
20
|
readonly emit: (event: string, payload: object) => boolean;
|
|
18
21
|
}
|
|
19
22
|
|
|
23
|
+
export function transientRetryDelayMs(
|
|
24
|
+
error: Error,
|
|
25
|
+
attempt: number,
|
|
26
|
+
retryBackoff: MutationQueueConfig['retryBackoff'],
|
|
27
|
+
): number {
|
|
28
|
+
const { baseMs, capMs } = retryBackoff;
|
|
29
|
+
let base = baseMs;
|
|
30
|
+
try {
|
|
31
|
+
const status = extractStatusCode(error);
|
|
32
|
+
if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
|
|
33
|
+
} catch {}
|
|
34
|
+
const ceiling = Math.min(capMs, base * Math.pow(2, Math.max(0, attempt - 1)));
|
|
35
|
+
return Math.floor(Math.random() * ceiling);
|
|
36
|
+
}
|
|
37
|
+
|
|
20
38
|
export async function handleFailure(ctx: FailureHandlingContext, transaction: QueuedMutation, error: Error): Promise<void> {
|
|
21
39
|
transaction.attempts++;
|
|
22
40
|
|
|
@@ -120,20 +138,17 @@ export async function handleFailure(ctx: FailureHandlingContext, transaction: Qu
|
|
|
120
138
|
return;
|
|
121
139
|
}
|
|
122
140
|
|
|
123
|
-
|
|
141
|
+
transaction.firstTransientFailureAt ??= Date.now();
|
|
142
|
+
const insideAvailabilityWindow =
|
|
143
|
+
Date.now() - transaction.firstTransientFailureAt < ctx.config.availabilityRetryWindowMs;
|
|
144
|
+
|
|
145
|
+
if (transaction.attempts < ctx.config.maxRetries || insideAvailabilityWindow) {
|
|
124
146
|
// Exponential backoff with full jitter on every transient retry:
|
|
125
147
|
// `sleep = random(0, min(cap, base * 2^attempt))`. Throttling responses
|
|
126
148
|
// (429/503) use a longer base than other transient errors. The re-enqueue
|
|
127
149
|
// is scheduled rather than awaited, so one backing-off transaction cannot
|
|
128
150
|
// stall unrelated commits.
|
|
129
|
-
const
|
|
130
|
-
let base = baseMs;
|
|
131
|
-
try {
|
|
132
|
-
const status = extractStatusCode(error);
|
|
133
|
-
if (status === 429 || status === 503) base = Math.max(baseMs, 1_000);
|
|
134
|
-
} catch {}
|
|
135
|
-
const ceiling = Math.min(capMs, base * Math.pow(2, transaction.attempts - 1));
|
|
136
|
-
const delay = Math.floor(Math.random() * ceiling);
|
|
151
|
+
const delay = transientRetryDelayMs(error, transaction.attempts, ctx.config.retryBackoff);
|
|
137
152
|
|
|
138
153
|
ctx.store.updateStatus(transaction.id, 'pending');
|
|
139
154
|
setTimeout(() => {
|