@abloatai/ablo 0.29.3 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +3 -2
- package/dist/BaseSyncedStore.js +3 -1
- package/dist/Database.js +29 -2
- package/dist/SyncClient.d.ts +5 -3
- package/dist/SyncClient.js +8 -4
- package/dist/ai-sdk/coordinatedTool.d.ts +0 -1
- package/dist/ai-sdk/coordinatedTool.js +0 -1
- package/dist/ai-sdk/coordinationContext.js +1 -5
- package/dist/cli.cjs +377 -84
- package/dist/client/Ablo.d.ts +1 -1
- package/dist/client/Ablo.js +24 -36
- package/dist/client/auth.d.ts +8 -18
- package/dist/client/auth.js +16 -96
- package/dist/client/createModelProxy.d.ts +16 -11
- package/dist/client/createModelProxy.js +162 -20
- package/dist/client/httpClient.d.ts +1 -1
- package/dist/client/httpTransport.js +263 -89
- package/dist/client/options.d.ts +0 -15
- package/dist/client/resourceTypes.d.ts +13 -26
- package/dist/client/schemaConfig.js +5 -0
- package/dist/client/writeOptionsSchema.d.ts +1 -0
- package/dist/client/writeOptionsSchema.js +11 -2
- package/dist/client/wsMutationExecutor.d.ts +3 -5
- package/dist/coordination/schema.d.ts +17 -13
- package/dist/coordination/schema.js +36 -10
- package/dist/errorCodes.d.ts +9 -4
- package/dist/errorCodes.js +10 -5
- package/dist/errors.js +4 -9
- package/dist/index.d.ts +4 -0
- package/dist/interfaces/index.d.ts +31 -19
- package/dist/schema/index.d.ts +1 -1
- package/dist/schema/index.js +4 -1
- package/dist/schema/schema.d.ts +13 -0
- package/dist/schema/select.js +7 -0
- package/dist/schema/syncDeltaRow.d.ts +31 -13
- package/dist/schema/syncDeltaRow.js +33 -15
- package/dist/schema/tenancy.d.ts +52 -16
- package/dist/schema/tenancy.js +86 -25
- package/dist/server/commit.d.ts +8 -24
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/readConfig.d.ts +10 -0
- package/dist/source/adapter.d.ts +26 -15
- package/dist/source/adapter.js +5 -3
- package/dist/source/adapters/drizzle.js +35 -9
- package/dist/source/adapters/kysely.d.ts +33 -61
- package/dist/source/adapters/kysely.js +170 -177
- package/dist/source/adapters/kyselyMutationCore.d.ts +76 -0
- package/dist/source/adapters/kyselyMutationCore.js +125 -0
- package/dist/source/adapters/memory.d.ts +1 -1
- package/dist/source/adapters/memory.js +24 -10
- package/dist/source/adapters/prisma.js +35 -8
- package/dist/source/conformance.d.ts +26 -28
- package/dist/source/conformance.js +142 -69
- package/dist/source/contract.d.ts +46 -4
- package/dist/source/contract.js +66 -5
- package/dist/source/factory.d.ts +3 -3
- package/dist/source/factory.js +24 -3
- package/dist/source/idempotency.d.ts +46 -0
- package/dist/source/idempotency.js +129 -0
- package/dist/source/index.d.ts +6 -4
- package/dist/source/index.js +5 -3
- package/dist/source/migrations.d.ts +9 -5
- package/dist/source/migrations.js +39 -6
- package/dist/source/types.d.ts +88 -14
- package/dist/source/types.js +16 -0
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +0 -1
- package/dist/sync/BootstrapFetcher.js +7 -0
- package/dist/sync/SyncWebSocket.d.ts +5 -27
- package/dist/sync/awaitClaimGrant.d.ts +6 -0
- package/dist/sync/awaitClaimGrant.js +12 -2
- package/dist/sync/commitFrames.d.ts +2 -4
- package/dist/sync/commitFrames.js +1 -0
- package/dist/sync/createClaimStream.d.ts +1 -1
- package/dist/sync/createClaimStream.js +13 -15
- package/dist/sync/deltaPipeline.d.ts +1 -1
- package/dist/sync/deltaPipeline.js +1 -1
- package/dist/sync/participants.d.ts +2 -3
- package/dist/sync/participants.js +2 -2
- package/dist/sync/wsFrameHandlers.js +35 -15
- package/dist/testing/mocks/MockMutationExecutor.d.ts +21 -0
- package/dist/testing/mocks/MockMutationExecutor.js +53 -1
- package/dist/transactions/TransactionQueue.d.ts +61 -9
- package/dist/transactions/TransactionQueue.js +502 -114
- package/dist/transactions/commitEnvelope.d.ts +4 -0
- package/dist/transactions/commitEnvelope.js +25 -0
- package/dist/transactions/commitPayload.d.ts +23 -5
- package/dist/transactions/commitPayload.js +3 -0
- package/dist/transactions/deltaConfirmation.d.ts +2 -1
- package/dist/transactions/deltaConfirmation.js +22 -4
- package/dist/transactions/durableWriteStore.d.ts +8 -1
- package/dist/transactions/httpCommitEnvelope.d.ts +3 -1
- package/dist/transactions/httpCommitEnvelope.js +30 -0
- package/dist/transactions/replayValidation.d.ts +4 -0
- package/dist/transactions/replayValidation.js +6 -0
- package/dist/types/streams.d.ts +29 -17
- package/dist/wire/commit.d.ts +671 -0
- package/dist/wire/commit.js +300 -0
- package/dist/wire/delta.d.ts +4 -0
- package/dist/wire/delta.js +9 -0
- package/dist/wire/frames.d.ts +17 -45
- package/dist/wire/index.d.ts +2 -0
- package/dist/wire/index.js +4 -0
- package/docs/agent-messaging.md +4 -5
- package/docs/agents.md +1 -2
- package/docs/api-keys.md +1 -2
- package/docs/client-behavior.md +4 -6
- package/docs/coordination.md +64 -9
- package/docs/data-sources.md +4 -7
- package/docs/integration-guide.md +6 -8
- package/docs/migration.md +0 -3
- package/docs/quickstart.md +7 -8
- package/docs/schema-contract.md +7 -8
- package/package.json +4 -4
- package/dist/client/registerDataSource.d.ts +0 -19
- package/dist/client/registerDataSource.js +0 -66
|
@@ -15,8 +15,9 @@ import { EventEmitter } from 'events';
|
|
|
15
15
|
import { v4 as uuid } from 'uuid';
|
|
16
16
|
import { Model } from '../Model.js';
|
|
17
17
|
import { getContext } from '../context.js';
|
|
18
|
-
import { AbloError, AbloConnectionError, AbloIdempotencyError, AbloValidationError, errorCodeSpec, } from '../errors.js';
|
|
18
|
+
import { AbloError, AbloConnectionError, AbloIdempotencyError, AbloNotFoundError, AbloValidationError, errorCodeSpec, } from '../errors.js';
|
|
19
19
|
import { SyncPosition } from '../sync/syncPosition.js';
|
|
20
|
+
import { mutationCommitResultSchema, } from '../wire/commit.js';
|
|
20
21
|
import { projectCommitPayload, computePriorityScore, normalizeModelKey,
|
|
21
22
|
// Includes stale guards as well as request identity/audit barriers.
|
|
22
23
|
hasCommitCoalescingBarrier, applyWriteOptions, asTransportError, extractStatusCode, TX_TYPE_TO_MUTATION_OP, } from './commitPayload.js';
|
|
@@ -69,6 +70,20 @@ export class TransactionQueue extends EventEmitter {
|
|
|
69
70
|
// coalescing with model-proxy transactions.
|
|
70
71
|
commitLane = [];
|
|
71
72
|
commitStore = new Map();
|
|
73
|
+
/**
|
|
74
|
+
* Small race buffer for authoritative echoes that arrive before the queued
|
|
75
|
+
* mutation receipt. The forward and WAL stream are independent channels, so
|
|
76
|
+
* either can win without changing the settlement result.
|
|
77
|
+
*/
|
|
78
|
+
recentDeltaCorrelations = new Map();
|
|
79
|
+
/**
|
|
80
|
+
* Client-facing confirmation deadlines for source-forwarded writes. These
|
|
81
|
+
* timers settle only the caller waiting for `confirmed`; the accepted write
|
|
82
|
+
* remains in `awaiting_delta`, with its durable envelope intact, until the
|
|
83
|
+
* authoritative WAL echo arrives.
|
|
84
|
+
*/
|
|
85
|
+
replicationLagTimeouts = new Map();
|
|
86
|
+
replicationLagErrors = new Map();
|
|
72
87
|
commitProcessing = false;
|
|
73
88
|
lastCommitSequence = 0;
|
|
74
89
|
durableReplayBlock = null;
|
|
@@ -97,8 +112,9 @@ export class TransactionQueue extends EventEmitter {
|
|
|
97
112
|
}
|
|
98
113
|
assertEnvelopeInsideReplayWindow(envelope) {
|
|
99
114
|
this.assertDurableReplayOpen();
|
|
100
|
-
if (
|
|
101
|
-
|
|
115
|
+
if (envelope.acceptedAt === undefined &&
|
|
116
|
+
Date.now() - envelope.sealedAt >=
|
|
117
|
+
TransactionQueue.DURABLE_REPLAY_WINDOW_MS) {
|
|
102
118
|
this.durableReplayBlock = new AbloIdempotencyError('A pending commit is older than the server idempotency window; newer writes are blocked until it is reviewed.', { code: 'idempotency_conflict' });
|
|
103
119
|
throw this.durableReplayBlock;
|
|
104
120
|
}
|
|
@@ -117,6 +133,51 @@ export class TransactionQueue extends EventEmitter {
|
|
|
117
133
|
entityKey(modelName, modelId) {
|
|
118
134
|
return entityKey(modelName, modelId);
|
|
119
135
|
}
|
|
136
|
+
/** Collision-safe receipt target identity across models sharing a row id. */
|
|
137
|
+
receiptTargetKey(modelName, modelId) {
|
|
138
|
+
return stableStringify([normalizeModelKey(modelName), modelId]);
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Relates stale notifications back to write targets without assuming the
|
|
142
|
+
* server's canonical model name uses the same spelling as the public schema
|
|
143
|
+
* key (`Task` versus `tasks`). Exact `(model,id)` wins; a globally unique id
|
|
144
|
+
* is the compatibility fallback. An ambiguous same-id cross-model mismatch
|
|
145
|
+
* is deliberately left unclassified, so it cannot falsely settle a queued
|
|
146
|
+
* write. A notification with no write-target id (or an explicit group) is a
|
|
147
|
+
* declared-read conflict and holds the whole batch.
|
|
148
|
+
*/
|
|
149
|
+
classifyReceiptNotifications(operations, notifications) {
|
|
150
|
+
const targets = operations.map((operation) => ({
|
|
151
|
+
id: operation.id,
|
|
152
|
+
key: this.receiptTargetKey(operation.model, operation.id),
|
|
153
|
+
}));
|
|
154
|
+
const heldTargets = new Set();
|
|
155
|
+
const notificationsByTarget = new Map();
|
|
156
|
+
let holdsEntireBatch = false;
|
|
157
|
+
for (const notification of notifications) {
|
|
158
|
+
const candidates = targets.filter((target) => target.id === notification.id);
|
|
159
|
+
const notificationKey = this.receiptTargetKey(notification.model, notification.id);
|
|
160
|
+
const exactTargets = candidates.filter((target) => target.key === notificationKey);
|
|
161
|
+
const candidateKeys = new Set((exactTargets.length > 0 ? exactTargets : candidates).map((target) => target.key));
|
|
162
|
+
if (notification.group || candidates.length === 0) {
|
|
163
|
+
holdsEntireBatch = true;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (candidateKeys.size !== 1) {
|
|
167
|
+
// Same id across multiple differently named models with no exact match:
|
|
168
|
+
// the id-only compatibility fallback is ambiguous. Await the echo.
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
const [targetKey] = candidateKeys;
|
|
172
|
+
if (!targetKey)
|
|
173
|
+
continue;
|
|
174
|
+
heldTargets.add(targetKey);
|
|
175
|
+
const targetNotifications = notificationsByTarget.get(targetKey) ?? [];
|
|
176
|
+
targetNotifications.push(notification);
|
|
177
|
+
notificationsByTarget.set(targetKey, targetNotifications);
|
|
178
|
+
}
|
|
179
|
+
return { holdsEntireBatch, heldTargets, notificationsByTarget };
|
|
180
|
+
}
|
|
120
181
|
resolveConfirmation(transaction) {
|
|
121
182
|
const resolver = this.confirmationResolvers.get(transaction.id);
|
|
122
183
|
if (!resolver)
|
|
@@ -207,6 +268,8 @@ export class TransactionQueue extends EventEmitter {
|
|
|
207
268
|
// the notification lands here from the commit acknowledgement and is drained
|
|
208
269
|
// by `waitForCommitReceipt`, so the receipt can carry it back to the caller.
|
|
209
270
|
commitNotifications = new Map();
|
|
271
|
+
/** Zero-row targets returned on a successful atomic commit receipt. */
|
|
272
|
+
commitMissingIds = new Map();
|
|
210
273
|
// Delta-confirmation tracking (ack watermark advance + the awaiting_delta
|
|
211
274
|
// timeout/retry maps) lives in `./deltaConfirmation.js`. Constructed in the
|
|
212
275
|
// constructor once `position` is bound.
|
|
@@ -362,7 +425,7 @@ export class TransactionQueue extends EventEmitter {
|
|
|
362
425
|
}
|
|
363
426
|
return envelope;
|
|
364
427
|
}
|
|
365
|
-
/** Best-effort cleanup after a definitive
|
|
428
|
+
/** Best-effort cleanup after a definitive rejection, confirmed ack, or echo. */
|
|
366
429
|
async removeDurableCommit(idempotencyKey) {
|
|
367
430
|
if (!this.config.enablePersistence || !this.commitOutbox)
|
|
368
431
|
return;
|
|
@@ -376,6 +439,115 @@ export class TransactionQueue extends EventEmitter {
|
|
|
376
439
|
});
|
|
377
440
|
}
|
|
378
441
|
}
|
|
442
|
+
/**
|
|
443
|
+
* Upgrade a sealed request with permanent connected-source acceptance before
|
|
444
|
+
* exposing the queued receipt. This is not completion: the envelope remains
|
|
445
|
+
* until the matching WAL correlation arrives. The upgrade only makes a
|
|
446
|
+
* crash/restart safe after the hosted 24-hour replay window has elapsed.
|
|
447
|
+
*/
|
|
448
|
+
async persistDurableCommitAcceptance(envelope, result) {
|
|
449
|
+
if (result.status !== 'queued')
|
|
450
|
+
return envelope;
|
|
451
|
+
const correlationId = result.correlationId;
|
|
452
|
+
if (!correlationId) {
|
|
453
|
+
throw new AbloConnectionError('The source accepted the commit without durable correlation evidence.', { code: 'commit_no_result' });
|
|
454
|
+
}
|
|
455
|
+
if (envelope.correlationId !== undefined &&
|
|
456
|
+
envelope.correlationId !== correlationId) {
|
|
457
|
+
throw new AbloIdempotencyError('The same commit replay returned a different source correlation.', { code: 'idempotency_conflict' });
|
|
458
|
+
}
|
|
459
|
+
if (envelope.acceptedAt !== undefined)
|
|
460
|
+
return envelope;
|
|
461
|
+
const accepted = durableCommitEnvelopeSchema.parse({
|
|
462
|
+
...envelope,
|
|
463
|
+
acceptedAt: Math.max(Date.now(), envelope.sealedAt),
|
|
464
|
+
correlationId,
|
|
465
|
+
});
|
|
466
|
+
if (this.config.enablePersistence && this.commitOutbox) {
|
|
467
|
+
try {
|
|
468
|
+
await this.commitOutbox.seal(accepted, []);
|
|
469
|
+
}
|
|
470
|
+
catch (cause) {
|
|
471
|
+
if (cause instanceof AbloError)
|
|
472
|
+
throw cause;
|
|
473
|
+
throw new AbloConnectionError('The source accepted the commit, but that acceptance could not be persisted locally.', { code: 'db_not_opened', cause });
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
return accepted;
|
|
477
|
+
}
|
|
478
|
+
/** Parse an untrusted/custom executor receipt without making ambiguity fatal. */
|
|
479
|
+
parseMutationCommitResult(value) {
|
|
480
|
+
const parsed = mutationCommitResultSchema.safeParse(value);
|
|
481
|
+
if (parsed.success)
|
|
482
|
+
return parsed.data;
|
|
483
|
+
throw new AbloConnectionError('The mutation transport returned an invalid commit receipt; its outcome remains pending and is safe to retry.', { code: 'commit_no_result', cause: parsed.error });
|
|
484
|
+
}
|
|
485
|
+
clearReplicationLagState(transactionId) {
|
|
486
|
+
const timeout = this.replicationLagTimeouts.get(transactionId);
|
|
487
|
+
if (timeout)
|
|
488
|
+
clearTimeout(timeout);
|
|
489
|
+
this.replicationLagTimeouts.delete(transactionId);
|
|
490
|
+
this.replicationLagErrors.delete(transactionId);
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Bounds the public `wait: 'confirmed'` promise without changing the
|
|
494
|
+
* accepted write's lifecycle. A lag timeout is not a rejection from the
|
|
495
|
+
* source database, so it must never emit `transaction:failed`, roll back
|
|
496
|
+
* optimistic state, or remove the durable replay envelope.
|
|
497
|
+
*/
|
|
498
|
+
scheduleReplicationLagTimeout(transactionId, clientTxId = transactionId, correlationId) {
|
|
499
|
+
const previous = this.replicationLagTimeouts.get(transactionId);
|
|
500
|
+
if (previous)
|
|
501
|
+
clearTimeout(previous);
|
|
502
|
+
this.replicationLagErrors.delete(transactionId);
|
|
503
|
+
const timeoutMs = this.config.deltaConfirmationTimeout;
|
|
504
|
+
const timeout = setTimeout(() => {
|
|
505
|
+
this.replicationLagTimeouts.delete(transactionId);
|
|
506
|
+
const modelTx = this.store.get(transactionId);
|
|
507
|
+
const commitTx = this.commitStore.get(transactionId);
|
|
508
|
+
if (modelTx?.status !== 'awaiting_delta' &&
|
|
509
|
+
commitTx?.status !== 'awaiting_delta')
|
|
510
|
+
return;
|
|
511
|
+
const error = new AbloConnectionError(`The source accepted commit ${clientTxId}, but its replication echo did not arrive within ${timeoutMs}ms.`, {
|
|
512
|
+
code: 'replication_lag_timeout',
|
|
513
|
+
httpStatus: 504,
|
|
514
|
+
details: {
|
|
515
|
+
clientTxId,
|
|
516
|
+
...(correlationId ? { correlationId } : {}),
|
|
517
|
+
timeoutMs,
|
|
518
|
+
accepted: true,
|
|
519
|
+
},
|
|
520
|
+
});
|
|
521
|
+
this.replicationLagErrors.set(transactionId, error);
|
|
522
|
+
// Model-proxy writes expose their waiter through the resolver table.
|
|
523
|
+
// Reject that promise without moving the transaction out of
|
|
524
|
+
// `awaiting_delta`; the eventual echo still completes it normally.
|
|
525
|
+
const resolver = this.confirmationResolvers.get(transactionId);
|
|
526
|
+
if (resolver) {
|
|
527
|
+
this.confirmationResolvers.delete(transactionId);
|
|
528
|
+
resolver.reject(error);
|
|
529
|
+
}
|
|
530
|
+
this.emitCommitLifecycle('transaction:confirmation_lagged', {
|
|
531
|
+
transactionId,
|
|
532
|
+
error,
|
|
533
|
+
});
|
|
534
|
+
this.emitCommitLifecycle(`transaction:confirmation_lagged:${transactionId}`, {
|
|
535
|
+
error,
|
|
536
|
+
});
|
|
537
|
+
if (commitTx) {
|
|
538
|
+
const firstOperation = commitTx.operations[0];
|
|
539
|
+
this.emitCommitLifecycle('reconciliation:needed', {
|
|
540
|
+
reason: 'replication_lag_timeout',
|
|
541
|
+
txId: transactionId,
|
|
542
|
+
model: firstOperation?.model ?? 'commit',
|
|
543
|
+
modelId: firstOperation?.id ?? transactionId,
|
|
544
|
+
lastSeenSyncId: this.lastSeenSyncId,
|
|
545
|
+
retryCount: 1,
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
}, timeoutMs);
|
|
549
|
+
this.replicationLagTimeouts.set(transactionId, timeout);
|
|
550
|
+
}
|
|
379
551
|
/**
|
|
380
552
|
* Takes either one complete retry envelope or a fresh batch. A retry waits
|
|
381
553
|
* until every original member has re-entered the queue, preventing an
|
|
@@ -473,6 +645,7 @@ export class TransactionQueue extends EventEmitter {
|
|
|
473
645
|
// Any successful write clears the permanent-error dedup, so a genuine
|
|
474
646
|
// recurrence after recovery warns again instead of staying demoted.
|
|
475
647
|
this.lastPermanentErrorSig = undefined;
|
|
648
|
+
this.clearReplicationLagState(tx.id);
|
|
476
649
|
const r = this.confirmationResolvers.get(tx.id);
|
|
477
650
|
if (r) {
|
|
478
651
|
this.confirmationResolvers.delete(tx.id);
|
|
@@ -727,16 +900,46 @@ export class TransactionQueue extends EventEmitter {
|
|
|
727
900
|
sequence: batch[0]?.commitEnvelope?.sequence,
|
|
728
901
|
});
|
|
729
902
|
this.assertEnvelopeInsideReplayWindow(durableEnvelope);
|
|
730
|
-
const result = await this.mutationExecutor.commit(durableEnvelope.operations, {
|
|
903
|
+
const result = this.parseMutationCommitResult(await this.mutationExecutor.commit(durableEnvelope.operations, {
|
|
731
904
|
idempotencyKey,
|
|
732
|
-
});
|
|
733
|
-
await this.
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
905
|
+
}));
|
|
906
|
+
await this.persistDurableCommitAcceptance(durableEnvelope, result);
|
|
907
|
+
if (result.status === 'queued') {
|
|
908
|
+
// Reconnect flushes use the same accepted-vs-confirmed contract as
|
|
909
|
+
// the normal lane. A queued source receipt retains the envelope and
|
|
910
|
+
// waits for exact correlation; it is never promoted by the reconnect
|
|
911
|
+
// shortcut itself.
|
|
912
|
+
for (const tx of batch) {
|
|
913
|
+
tx.requiresCorrelatedDelta = true;
|
|
914
|
+
tx.syncIdNeededForCompletion = undefined;
|
|
915
|
+
tx.correlationId = result.correlationId;
|
|
916
|
+
const echoSyncId = result.correlationId
|
|
917
|
+
? this.recentDeltaCorrelations.get(result.correlationId)
|
|
918
|
+
: undefined;
|
|
919
|
+
if (echoSyncId !== undefined) {
|
|
920
|
+
this.store.updateStatus(tx.id, 'completed');
|
|
921
|
+
this.emit('transaction:completed', tx);
|
|
922
|
+
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
923
|
+
this.optimisticUpdates.delete(tx.id);
|
|
924
|
+
continue;
|
|
925
|
+
}
|
|
926
|
+
this.store.updateStatus(tx.id, 'awaiting_delta');
|
|
927
|
+
this.scheduleReplicationLagTimeout(tx.id, idempotencyKey, result.correlationId);
|
|
928
|
+
this.scheduleDeltaConfirmationTimeout(tx, this.config.deltaConfirmationTimeout);
|
|
929
|
+
}
|
|
930
|
+
if (batch.every((tx) => tx.status === 'completed')) {
|
|
931
|
+
await this.removeDurableCommit(idempotencyKey);
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
else {
|
|
935
|
+
await this.removeDurableCommit(idempotencyKey);
|
|
936
|
+
// Mark this request envelope as completed before moving to the next.
|
|
937
|
+
for (const tx of batch) {
|
|
938
|
+
this.store.updateStatus(tx.id, 'completed');
|
|
939
|
+
this.emit('transaction:completed', tx);
|
|
940
|
+
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
941
|
+
this.optimisticUpdates.delete(tx.id);
|
|
942
|
+
}
|
|
740
943
|
}
|
|
741
944
|
getContext().logger.debug('txn:commit', 0, {
|
|
742
945
|
count: batch.length,
|
|
@@ -1222,106 +1425,140 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1222
1425
|
// the exact key assigned before the first transport attempt.
|
|
1223
1426
|
this.assertEnvelopeInsideReplayWindow(durableEnvelope);
|
|
1224
1427
|
dispatchStarted = true;
|
|
1225
|
-
const result = await this.mutationExecutor.commit(operations, {
|
|
1428
|
+
const result = this.parseMutationCommitResult(await this.mutationExecutor.commit(operations, {
|
|
1226
1429
|
idempotencyKey: commitIdempotencyKey,
|
|
1227
|
-
});
|
|
1228
|
-
await this.
|
|
1229
|
-
const lastSyncId = result
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
// marked `onStale: 'notify'` whose read premise had moved. Each
|
|
1233
|
-
// notified operation was held rather than written, so its
|
|
1234
|
-
// optimistic state must be rolled back here — no delta will ever
|
|
1235
|
-
// confirm it — and the signal is stamped so
|
|
1236
|
-
// {@link waitForCommitReceipt} can carry it onto the receipt.
|
|
1237
|
-
const notifications = result?.notifications;
|
|
1238
|
-
const heldIds = new Set((notifications ?? []).map((n) => n.id));
|
|
1430
|
+
}));
|
|
1431
|
+
await this.persistDurableCommitAcceptance(durableEnvelope, result);
|
|
1432
|
+
const lastSyncId = result.lastSyncId;
|
|
1433
|
+
const notifications = result.notifications;
|
|
1434
|
+
const { holdsEntireBatch, heldTargets, notificationsByTarget, } = this.classifyReceiptNotifications(batchOps.map(({ op }) => ({ model: op.model, id: op.id })), notifications ?? []);
|
|
1239
1435
|
for (const { tx } of batchOps) {
|
|
1240
|
-
const
|
|
1436
|
+
const txTarget = this.receiptTargetKey(tx.modelKey, tx.modelId);
|
|
1437
|
+
const txNotifs = holdsEntireBatch
|
|
1438
|
+
? notifications
|
|
1439
|
+
: notificationsByTarget.get(txTarget);
|
|
1241
1440
|
if (txNotifs && txNotifs.length > 0) {
|
|
1242
1441
|
this.commitNotifications.set(tx.id, txNotifs);
|
|
1243
1442
|
}
|
|
1244
1443
|
}
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
});
|
|
1252
|
-
}
|
|
1253
|
-
// Mark each transaction with the sync-id threshold that will
|
|
1254
|
-
// confirm it: any delta whose id is at least lastSyncId.
|
|
1255
|
-
for (const { tx } of batchOps) {
|
|
1256
|
-
// Held op ('notify'): the server withheld the write, so no delta
|
|
1257
|
-
// will confirm it. Roll back the optimistic update (server
|
|
1258
|
-
// wins) and complete the transaction now — the agent self-heals
|
|
1259
|
-
// from the notification rather than waiting out the delta
|
|
1260
|
-
// timeout. The receipt still resolves (commit succeeded).
|
|
1261
|
-
if (heldIds.has(tx.modelId)) {
|
|
1262
|
-
await this.rollbackOptimistic(tx, 'conflict_server_wins');
|
|
1263
|
-
this.store.updateStatus(tx.id, 'completed');
|
|
1264
|
-
this.emit('transaction:completed', tx);
|
|
1265
|
-
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
1266
|
-
this.optimisticUpdates.delete(tx.id);
|
|
1444
|
+
const missingIds = new Set(result.missingIds ?? []);
|
|
1445
|
+
const settlingBatchOps = [];
|
|
1446
|
+
for (const entry of batchOps) {
|
|
1447
|
+
const { tx } = entry;
|
|
1448
|
+
if (missingIds.has(tx.modelId)) {
|
|
1449
|
+
await this.handleFailure(tx, new AbloNotFoundError(`${tx.modelName}/${tx.modelId} was not found or is outside this credential's scope.`, [tx.modelId]));
|
|
1267
1450
|
continue;
|
|
1268
1451
|
}
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
// is already gone and the claim was achieved. Parking DELETEs in awaiting_delta
|
|
1273
|
-
// with threshold 0 causes 30s reconciliation delays.
|
|
1274
|
-
if (lastSyncId === 0 && tx.type === 'delete') {
|
|
1452
|
+
if (holdsEntireBatch ||
|
|
1453
|
+
heldTargets.has(this.receiptTargetKey(tx.modelKey, tx.modelId))) {
|
|
1454
|
+
await this.rollbackOptimistic(tx, 'conflict_server_wins');
|
|
1275
1455
|
this.store.updateStatus(tx.id, 'completed');
|
|
1276
1456
|
this.emit('transaction:completed', tx);
|
|
1277
1457
|
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
1278
1458
|
this.optimisticUpdates.delete(tx.id);
|
|
1279
|
-
getContext().logger.debug('tx:confirm_delete_zero_syncid', {
|
|
1280
|
-
txId: tx.id.slice(0, 8),
|
|
1281
|
-
model: tx.modelName,
|
|
1282
|
-
reason: 'delete_idempotent_no_delta',
|
|
1283
|
-
});
|
|
1284
1459
|
continue;
|
|
1285
1460
|
}
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
//
|
|
1295
|
-
//
|
|
1296
|
-
//
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
this.
|
|
1300
|
-
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
1301
|
-
this.optimisticUpdates.delete(tx.id);
|
|
1302
|
-
getContext().logger.debug('tx:confirm_ack', {
|
|
1303
|
-
txId: tx.id.slice(0, 8),
|
|
1304
|
-
model: tx.modelName,
|
|
1305
|
-
serverSyncId: lastSyncId,
|
|
1306
|
-
lastSeenSyncId: this.lastSeenSyncId,
|
|
1307
|
-
});
|
|
1461
|
+
settlingBatchOps.push(entry);
|
|
1462
|
+
}
|
|
1463
|
+
// A forwarded write has been accepted, but the source database
|
|
1464
|
+
// remains authoritative. It must not enter any of the confirmed
|
|
1465
|
+
// paths below: in particular, a queued DELETE with sync id 0 is
|
|
1466
|
+
// not the hosted-path idempotent-delete shortcut, and a later
|
|
1467
|
+
// unrelated delta cannot stand in for the source echo.
|
|
1468
|
+
if (result.status === 'queued') {
|
|
1469
|
+
// Keep the exact envelope crash-durable while any forwarded
|
|
1470
|
+
// member is awaiting its WAL echo. If coordination/missing-row
|
|
1471
|
+
// handling consumed the whole request, there is no echo left
|
|
1472
|
+
// to await and the durable intent is definitive.
|
|
1473
|
+
if (settlingBatchOps.length === 0) {
|
|
1474
|
+
await this.removeDurableCommit(commitIdempotencyKey);
|
|
1308
1475
|
}
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1476
|
+
for (const { tx } of settlingBatchOps) {
|
|
1477
|
+
tx.requiresCorrelatedDelta = true;
|
|
1478
|
+
tx.syncIdNeededForCompletion = undefined;
|
|
1479
|
+
tx.correlationId = result.correlationId;
|
|
1480
|
+
const echoSyncId = result.correlationId
|
|
1481
|
+
? this.recentDeltaCorrelations.get(result.correlationId)
|
|
1482
|
+
: undefined;
|
|
1483
|
+
if (echoSyncId !== undefined) {
|
|
1484
|
+
this.store.updateStatus(tx.id, 'completed');
|
|
1485
|
+
this.emit('transaction:completed', tx);
|
|
1486
|
+
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
1487
|
+
this.optimisticUpdates.delete(tx.id);
|
|
1488
|
+
continue;
|
|
1489
|
+
}
|
|
1315
1490
|
this.store.updateStatus(tx.id, 'awaiting_delta');
|
|
1491
|
+
this.scheduleReplicationLagTimeout(tx.id, commitIdempotencyKey, result.correlationId);
|
|
1316
1492
|
getContext().logger.debug('tx:awaiting_delta', {
|
|
1317
1493
|
txId: tx.id.slice(0, 8),
|
|
1318
1494
|
model: tx.modelName,
|
|
1319
|
-
|
|
1320
|
-
lastSeenSyncId: this.lastSeenSyncId,
|
|
1321
|
-
reason: 'zero_sync_id_anomaly',
|
|
1495
|
+
reason: 'queued_forward_waiting_for_correlated_echo',
|
|
1322
1496
|
});
|
|
1323
1497
|
this.scheduleDeltaConfirmationTimeout(tx, this.config.deltaConfirmationTimeout);
|
|
1324
1498
|
}
|
|
1499
|
+
if (settlingBatchOps.length > 0 &&
|
|
1500
|
+
settlingBatchOps.every(({ tx }) => tx.status === 'completed')) {
|
|
1501
|
+
await this.removeDurableCommit(commitIdempotencyKey);
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
else {
|
|
1505
|
+
await this.removeDurableCommit(commitIdempotencyKey);
|
|
1506
|
+
this.noteAck(lastSyncId);
|
|
1507
|
+
// A lastSyncId of 0 means the mutation succeeded but the server
|
|
1508
|
+
// emitted no sync delta; record that anomaly for observability.
|
|
1509
|
+
if (lastSyncId === 0) {
|
|
1510
|
+
getContext().observability.captureCommitZeroSyncId({
|
|
1511
|
+
operationCount: operations.length,
|
|
1512
|
+
operations: operations.map((op) => `${op.type}:${op.model}:${op.id?.slice(0, 8) ?? '?'}`),
|
|
1513
|
+
});
|
|
1514
|
+
}
|
|
1515
|
+
// Mark each remaining transaction with the sync-id threshold
|
|
1516
|
+
// that will confirm it: any delta whose id is at least
|
|
1517
|
+
// lastSyncId.
|
|
1518
|
+
for (const { tx } of settlingBatchOps) {
|
|
1519
|
+
tx.syncIdNeededForCompletion = lastSyncId;
|
|
1520
|
+
// Safety net: a confirmed zero-sync DELETE is idempotent.
|
|
1521
|
+
// This shortcut is intentionally unreachable for queued
|
|
1522
|
+
// forwards, which must wait for their correlated echo.
|
|
1523
|
+
if (lastSyncId === 0 && tx.type === 'delete') {
|
|
1524
|
+
this.store.updateStatus(tx.id, 'completed');
|
|
1525
|
+
this.emit('transaction:completed', tx);
|
|
1526
|
+
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
1527
|
+
this.optimisticUpdates.delete(tx.id);
|
|
1528
|
+
getContext().logger.debug('tx:confirm_delete_zero_syncid', {
|
|
1529
|
+
txId: tx.id.slice(0, 8),
|
|
1530
|
+
model: tx.modelName,
|
|
1531
|
+
reason: 'delete_idempotent_no_delta',
|
|
1532
|
+
});
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
// A real watermark is the established hosted-path
|
|
1536
|
+
// confirmation. A zero watermark on a non-delete stays on
|
|
1537
|
+
// the anomaly/reconciliation path.
|
|
1538
|
+
if (lastSyncId > 0) {
|
|
1539
|
+
this.store.updateStatus(tx.id, 'completed');
|
|
1540
|
+
this.emit('transaction:completed', tx);
|
|
1541
|
+
this.emit(`transaction:completed:${tx.id}`, tx);
|
|
1542
|
+
this.optimisticUpdates.delete(tx.id);
|
|
1543
|
+
getContext().logger.debug('tx:confirm_ack', {
|
|
1544
|
+
txId: tx.id.slice(0, 8),
|
|
1545
|
+
model: tx.modelName,
|
|
1546
|
+
serverSyncId: lastSyncId,
|
|
1547
|
+
lastSeenSyncId: this.lastSeenSyncId,
|
|
1548
|
+
});
|
|
1549
|
+
}
|
|
1550
|
+
else {
|
|
1551
|
+
this.store.updateStatus(tx.id, 'awaiting_delta');
|
|
1552
|
+
getContext().logger.debug('tx:awaiting_delta', {
|
|
1553
|
+
txId: tx.id.slice(0, 8),
|
|
1554
|
+
model: tx.modelName,
|
|
1555
|
+
neededSyncId: lastSyncId,
|
|
1556
|
+
lastSeenSyncId: this.lastSeenSyncId,
|
|
1557
|
+
reason: 'zero_sync_id_anomaly',
|
|
1558
|
+
});
|
|
1559
|
+
this.scheduleDeltaConfirmationTimeout(tx, this.config.deltaConfirmationTimeout);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1325
1562
|
}
|
|
1326
1563
|
}
|
|
1327
1564
|
catch (error) {
|
|
@@ -1453,14 +1690,74 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1453
1690
|
}
|
|
1454
1691
|
}, { batchSize: this.executionQueue.length + (batch?.length ?? 0) });
|
|
1455
1692
|
}
|
|
1693
|
+
rememberDeltaCorrelation(correlationId, syncId) {
|
|
1694
|
+
// Refresh insertion order when a replay repeats the same correlation id.
|
|
1695
|
+
this.recentDeltaCorrelations.delete(correlationId);
|
|
1696
|
+
this.recentDeltaCorrelations.set(correlationId, syncId);
|
|
1697
|
+
if (this.recentDeltaCorrelations.size <= 2_048)
|
|
1698
|
+
return;
|
|
1699
|
+
const oldest = this.recentDeltaCorrelations.keys().next().value;
|
|
1700
|
+
if (typeof oldest === 'string')
|
|
1701
|
+
this.recentDeltaCorrelations.delete(oldest);
|
|
1702
|
+
}
|
|
1703
|
+
queuedCommitEchoSyncId(tx) {
|
|
1704
|
+
return tx.correlationId
|
|
1705
|
+
? this.recentDeltaCorrelations.get(tx.correlationId)
|
|
1706
|
+
: undefined;
|
|
1707
|
+
}
|
|
1708
|
+
queuedCommitMatchesCorrelation(tx, correlationId) {
|
|
1709
|
+
return tx.correlationId !== undefined && tx.correlationId === correlationId;
|
|
1710
|
+
}
|
|
1711
|
+
completeQueuedCommit(tx, syncId) {
|
|
1712
|
+
if (tx.status !== 'awaiting_delta')
|
|
1713
|
+
return;
|
|
1714
|
+
this.clearReplicationLagState(tx.id);
|
|
1715
|
+
tx.lastSyncId = syncId;
|
|
1716
|
+
tx.status = 'completed';
|
|
1717
|
+
// The queued receipt was only acceptance. The correlated source echo is
|
|
1718
|
+
// the first definitive success and therefore the point where the durable
|
|
1719
|
+
// replay envelope may be removed.
|
|
1720
|
+
void this.removeDurableCommit(tx.id);
|
|
1721
|
+
this.emitCommitLifecycle('transaction:completed', tx);
|
|
1722
|
+
this.emitCommitLifecycle(`transaction:completed:${tx.id}`, tx);
|
|
1723
|
+
}
|
|
1456
1724
|
/**
|
|
1457
|
-
* Confirms
|
|
1458
|
-
*
|
|
1459
|
-
* {@link DeltaConfirmationTracker} (`./deltaConfirmation.js`).
|
|
1460
|
-
* @param syncId - The sync id of the received delta.
|
|
1725
|
+
* Confirms awaiting writes. Hosted/anomaly receipts keep their sync-id
|
|
1726
|
+
* threshold semantics; queued forwards require the exact server correlation.
|
|
1461
1727
|
*/
|
|
1462
|
-
onDeltaReceived(syncId) {
|
|
1463
|
-
|
|
1728
|
+
onDeltaReceived(syncId, _transactionId, correlationId) {
|
|
1729
|
+
if (correlationId)
|
|
1730
|
+
this.rememberDeltaCorrelation(correlationId, syncId);
|
|
1731
|
+
const correlatedModelTransactions = correlationId
|
|
1732
|
+
? this.store.getByStatus('awaiting_delta').filter((tx) => tx.requiresCorrelatedDelta === true &&
|
|
1733
|
+
tx.correlationId !== undefined &&
|
|
1734
|
+
tx.correlationId === correlationId)
|
|
1735
|
+
: [];
|
|
1736
|
+
this.deltaConfirmation.onDeltaReceived(syncId, correlationId);
|
|
1737
|
+
if (correlatedModelTransactions.length > 0) {
|
|
1738
|
+
const envelopeIds = new Set();
|
|
1739
|
+
for (const tx of correlatedModelTransactions) {
|
|
1740
|
+
this.clearReplicationLagState(tx.id);
|
|
1741
|
+
if (tx.commitEnvelope)
|
|
1742
|
+
envelopeIds.add(tx.commitEnvelope.idempotencyKey);
|
|
1743
|
+
}
|
|
1744
|
+
for (const envelopeId of envelopeIds) {
|
|
1745
|
+
const envelopeStillAwaiting = this.store
|
|
1746
|
+
.getByStatus('awaiting_delta')
|
|
1747
|
+
.some((tx) => tx.commitEnvelope?.idempotencyKey === envelopeId);
|
|
1748
|
+
if (!envelopeStillAwaiting)
|
|
1749
|
+
void this.removeDurableCommit(envelopeId);
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
if (!correlationId)
|
|
1753
|
+
return;
|
|
1754
|
+
for (const tx of this.commitStore.values()) {
|
|
1755
|
+
if (tx.status !== 'awaiting_delta')
|
|
1756
|
+
continue;
|
|
1757
|
+
if (this.queuedCommitMatchesCorrelation(tx, correlationId)) {
|
|
1758
|
+
this.completeQueuedCommit(tx, syncId);
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1464
1761
|
}
|
|
1465
1762
|
// Schedule the retry-and-reconciliation wait for a transaction's confirming
|
|
1466
1763
|
// delta; see {@link DeltaConfirmationTracker} in `./deltaConfirmation.js`.
|
|
@@ -1480,6 +1777,11 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1480
1777
|
resolve();
|
|
1481
1778
|
return;
|
|
1482
1779
|
}
|
|
1780
|
+
const lagError = this.replicationLagErrors.get(transactionId);
|
|
1781
|
+
if (lagError) {
|
|
1782
|
+
reject(lagError);
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1483
1785
|
const onCompleted = () => {
|
|
1484
1786
|
cleanup();
|
|
1485
1787
|
resolve();
|
|
@@ -1488,13 +1790,19 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1488
1790
|
cleanup();
|
|
1489
1791
|
reject(error);
|
|
1490
1792
|
};
|
|
1793
|
+
const onLagged = ({ error }) => {
|
|
1794
|
+
cleanup();
|
|
1795
|
+
reject(error);
|
|
1796
|
+
};
|
|
1491
1797
|
const cleanup = () => {
|
|
1492
1798
|
this.off(`transaction:completed:${transactionId}`, onCompleted);
|
|
1493
1799
|
this.off(`transaction:failed:${transactionId}`, onFailed);
|
|
1800
|
+
this.off(`transaction:confirmation_lagged:${transactionId}`, onLagged);
|
|
1494
1801
|
};
|
|
1495
1802
|
// Listen to existing events (timeout already handled by scheduleDeltaConfirmationTimeout)
|
|
1496
1803
|
this.on(`transaction:completed:${transactionId}`, onCompleted);
|
|
1497
1804
|
this.on(`transaction:failed:${transactionId}`, onFailed);
|
|
1805
|
+
this.on(`transaction:confirmation_lagged:${transactionId}`, onLagged);
|
|
1498
1806
|
});
|
|
1499
1807
|
}
|
|
1500
1808
|
// Reports whether a client mutation id is known to this queue, which helps
|
|
@@ -1507,8 +1815,10 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1507
1815
|
* `ablo.commits.create()` path. The caller supplies the operations; the queue
|
|
1508
1816
|
* only retries on reconnect and de-duplicates, and does not apply the change
|
|
1509
1817
|
* optimistically or reorder for foreign keys. A duplicate `clientTxId`
|
|
1510
|
-
* already in flight is ignored
|
|
1511
|
-
*
|
|
1818
|
+
* already in flight is ignored. After an accepted write exceeds its
|
|
1819
|
+
* replication-confirmation deadline, an explicit same-key retry reuses the
|
|
1820
|
+
* exact sealed envelope as an idempotent status probe; the server's
|
|
1821
|
+
* `mutation_log` still prevents a second logical write.
|
|
1512
1822
|
*/
|
|
1513
1823
|
async enqueueCommit(clientTxId, operations, options = {}) {
|
|
1514
1824
|
this.assertDurableReplayOpen();
|
|
@@ -1528,6 +1838,18 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1528
1838
|
if (existingIntent !== incomingIntent) {
|
|
1529
1839
|
throw new AbloIdempotencyError('Idempotency key reused with a different atomic commit request', { code: 'idempotency_conflict' });
|
|
1530
1840
|
}
|
|
1841
|
+
if (existing.status === 'awaiting_delta' &&
|
|
1842
|
+
this.replicationLagErrors.has(existing.id)) {
|
|
1843
|
+
// An explicit same-key retry after the confirmation deadline is an
|
|
1844
|
+
// idempotent status probe, not a second logical write. Reuse the exact
|
|
1845
|
+
// durable envelope and ask the server again: once the WAL echo has
|
|
1846
|
+
// materialized, mutation-log replay upgrades the old queued receipt to
|
|
1847
|
+
// confirmed. Passive awaiting remains untouched before the deadline,
|
|
1848
|
+
// so duplicate callers cannot cause eager re-dispatch.
|
|
1849
|
+
this.clearReplicationLagState(existing.id);
|
|
1850
|
+
existing.status = 'pending';
|
|
1851
|
+
this.commitLane.push(existing);
|
|
1852
|
+
}
|
|
1531
1853
|
if (existing.status === 'pending')
|
|
1532
1854
|
void this.processCommitLane();
|
|
1533
1855
|
return;
|
|
@@ -1627,23 +1949,48 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1627
1949
|
tx.durableEnvelope = durableEnvelope;
|
|
1628
1950
|
this.assertEnvelopeInsideReplayWindow(durableEnvelope);
|
|
1629
1951
|
dispatchStarted = true;
|
|
1630
|
-
const result = await this.mutationExecutor.commit(durableEnvelope.operations, {
|
|
1952
|
+
const result = this.parseMutationCommitResult(await this.mutationExecutor.commit(durableEnvelope.operations, {
|
|
1631
1953
|
idempotencyKey: tx.id,
|
|
1632
1954
|
causedByTaskId: durableEnvelope.commitOptions.causedByTaskId ?? undefined,
|
|
1633
1955
|
...(durableEnvelope.commitOptions.reads
|
|
1634
1956
|
? { reads: durableEnvelope.commitOptions.reads }
|
|
1635
1957
|
: {}),
|
|
1636
|
-
});
|
|
1637
|
-
await this.
|
|
1638
|
-
tx.lastSyncId = result
|
|
1639
|
-
|
|
1640
|
-
|
|
1958
|
+
}));
|
|
1959
|
+
tx.durableEnvelope = await this.persistDurableCommitAcceptance(durableEnvelope, result);
|
|
1960
|
+
tx.lastSyncId = result.lastSyncId;
|
|
1961
|
+
const notifications = result.notifications;
|
|
1962
|
+
if (notifications && notifications.length > 0) {
|
|
1963
|
+
this.commitNotifications.set(tx.id, notifications);
|
|
1964
|
+
}
|
|
1965
|
+
const missingIds = result.missingIds;
|
|
1966
|
+
if (missingIds && missingIds.length > 0) {
|
|
1967
|
+
this.commitMissingIds.set(tx.id, missingIds);
|
|
1968
|
+
}
|
|
1641
1969
|
this.commitLane.shift();
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1970
|
+
if (result.status === 'queued') {
|
|
1971
|
+
tx.correlationId = result.correlationId;
|
|
1972
|
+
tx.status = 'awaiting_delta';
|
|
1973
|
+
const echoSyncId = this.queuedCommitEchoSyncId(tx);
|
|
1974
|
+
if (echoSyncId !== undefined) {
|
|
1975
|
+
this.completeQueuedCommit(tx, echoSyncId);
|
|
1976
|
+
}
|
|
1977
|
+
else {
|
|
1978
|
+
this.scheduleReplicationLagTimeout(tx.id, tx.id, result.correlationId);
|
|
1979
|
+
getContext().logger.debug('[TransactionQueue] commit lane awaiting source echo', {
|
|
1980
|
+
txId: tx.id.slice(0, 12),
|
|
1981
|
+
});
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
else {
|
|
1985
|
+
await this.removeDurableCommit(tx.id);
|
|
1986
|
+
this.noteAck(tx.lastSyncId);
|
|
1987
|
+
tx.status = 'completed';
|
|
1988
|
+
// Guarded: a throwing observer here would land in the catch below,
|
|
1989
|
+
// whose permanent branch shifts the lane a second time and rejects a
|
|
1990
|
+
// commit the server has already durably applied.
|
|
1991
|
+
this.emitCommitLifecycle('transaction:completed', tx);
|
|
1992
|
+
this.emitCommitLifecycle(`transaction:completed:${tx.id}`, tx);
|
|
1993
|
+
}
|
|
1647
1994
|
}
|
|
1648
1995
|
catch (err) {
|
|
1649
1996
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
@@ -1697,30 +2044,57 @@ export class TransactionQueue extends EventEmitter {
|
|
|
1697
2044
|
this.commitNotifications.delete(clientTxId);
|
|
1698
2045
|
return n.length > 0 ? n : undefined;
|
|
1699
2046
|
};
|
|
2047
|
+
const drainMissingIds = () => {
|
|
2048
|
+
const ids = this.commitMissingIds.get(clientTxId);
|
|
2049
|
+
if (!ids)
|
|
2050
|
+
return undefined;
|
|
2051
|
+
this.commitMissingIds.delete(clientTxId);
|
|
2052
|
+
return ids.length > 0 ? ids : undefined;
|
|
2053
|
+
};
|
|
2054
|
+
const receipt = (lastSyncId) => {
|
|
2055
|
+
const notifications = drainNotifications();
|
|
2056
|
+
const missingIds = drainMissingIds();
|
|
2057
|
+
return {
|
|
2058
|
+
lastSyncId,
|
|
2059
|
+
notifications,
|
|
2060
|
+
...(missingIds ? { missingIds } : {}),
|
|
2061
|
+
};
|
|
2062
|
+
};
|
|
1700
2063
|
return new Promise((resolve, reject) => {
|
|
1701
2064
|
const existing = this.commitStore.get(clientTxId);
|
|
1702
2065
|
if (existing?.status === 'completed') {
|
|
1703
|
-
resolve(
|
|
2066
|
+
resolve(receipt(existing.lastSyncId ?? 0));
|
|
1704
2067
|
return;
|
|
1705
2068
|
}
|
|
1706
2069
|
if (existing?.status === 'failed' && existing.error) {
|
|
1707
2070
|
reject(existing.error);
|
|
1708
2071
|
return;
|
|
1709
2072
|
}
|
|
2073
|
+
const lagError = this.replicationLagErrors.get(clientTxId);
|
|
2074
|
+
if (lagError) {
|
|
2075
|
+
reject(lagError);
|
|
2076
|
+
return;
|
|
2077
|
+
}
|
|
1710
2078
|
const onCompleted = (tx) => {
|
|
1711
2079
|
cleanup();
|
|
1712
|
-
resolve(
|
|
2080
|
+
resolve(receipt(tx.lastSyncId ?? 0));
|
|
1713
2081
|
};
|
|
1714
2082
|
const onFailed = ({ error }) => {
|
|
1715
2083
|
cleanup();
|
|
1716
2084
|
reject(error);
|
|
1717
2085
|
};
|
|
2086
|
+
const onLagged = ({ error }) => {
|
|
2087
|
+
cleanup();
|
|
2088
|
+
reject(error);
|
|
2089
|
+
};
|
|
1718
2090
|
const cleanup = () => {
|
|
1719
2091
|
this.off(`transaction:completed:${clientTxId}`, onCompleted);
|
|
1720
2092
|
this.off(`transaction:failed:${clientTxId}`, onFailed);
|
|
2093
|
+
this.off(`transaction:confirmation_lagged:${clientTxId}`, onLagged);
|
|
1721
2094
|
};
|
|
1722
2095
|
this.on(`transaction:completed:${clientTxId}`, onCompleted);
|
|
1723
2096
|
this.on(`transaction:failed:${clientTxId}`, onFailed);
|
|
2097
|
+
this.on(`transaction:confirmation_lagged:${clientTxId}`, onLagged);
|
|
1724
2098
|
});
|
|
1725
2099
|
}
|
|
1726
2100
|
isReorderPayload(data) {
|
|
@@ -2044,8 +2418,9 @@ export class TransactionQueue extends EventEmitter {
|
|
|
2044
2418
|
for (const mutationId of envelope.sourceMutationIds) {
|
|
2045
2419
|
sourceMutationIds.add(mutationId);
|
|
2046
2420
|
}
|
|
2047
|
-
if (
|
|
2048
|
-
|
|
2421
|
+
if (envelope.acceptedAt === undefined &&
|
|
2422
|
+
Date.now() - envelope.sealedAt >=
|
|
2423
|
+
TransactionQueue.DURABLE_REPLAY_WINDOW_MS) {
|
|
2049
2424
|
getContext().logger.warn('A saved local write is too old to retry safely and was held for review.');
|
|
2050
2425
|
getContext().observability.captureTransactionFailure({
|
|
2051
2426
|
context: 'quarantine-expired-commit-envelope',
|
|
@@ -2076,6 +2451,9 @@ export class TransactionQueue extends EventEmitter {
|
|
|
2076
2451
|
sealedAt: envelope.sealedAt,
|
|
2077
2452
|
sequence: envelope.sequence ?? envelope.sealedAt * 1_000,
|
|
2078
2453
|
attempts: 0,
|
|
2454
|
+
...(envelope.correlationId
|
|
2455
|
+
? { correlationId: envelope.correlationId }
|
|
2456
|
+
: {}),
|
|
2079
2457
|
sourceMutationIds: [...envelope.sourceMutationIds],
|
|
2080
2458
|
durableEnvelope: envelope,
|
|
2081
2459
|
};
|
|
@@ -2320,6 +2698,11 @@ export class TransactionQueue extends EventEmitter {
|
|
|
2320
2698
|
// 30–120s each) — a disposed queue must not keep the process alive or
|
|
2321
2699
|
// fire confirmation callbacks against the cleared store below.
|
|
2322
2700
|
this.deltaConfirmation.dispose();
|
|
2701
|
+
for (const timeout of this.replicationLagTimeouts.values()) {
|
|
2702
|
+
clearTimeout(timeout);
|
|
2703
|
+
}
|
|
2704
|
+
this.replicationLagTimeouts.clear();
|
|
2705
|
+
this.replicationLagErrors.clear();
|
|
2323
2706
|
// Clear the offline-grace timer armed by setConnectionState('disconnected').
|
|
2324
2707
|
if (this.commitOfflineGraceTimer !== null) {
|
|
2325
2708
|
clearTimeout(this.commitOfflineGraceTimer);
|
|
@@ -2331,6 +2714,11 @@ export class TransactionQueue extends EventEmitter {
|
|
|
2331
2714
|
this.executionQueue = [];
|
|
2332
2715
|
this.createdTransactions = [];
|
|
2333
2716
|
this.deferredDeletesByCreate.clear();
|
|
2717
|
+
this.recentDeltaCorrelations.clear();
|
|
2718
|
+
this.commitLane = [];
|
|
2719
|
+
this.commitStore.clear();
|
|
2720
|
+
this.commitNotifications.clear();
|
|
2721
|
+
this.commitMissingIds.clear();
|
|
2334
2722
|
// Clear event listeners
|
|
2335
2723
|
this.removeAllListeners();
|
|
2336
2724
|
// Reset state
|