@abloatai/humans 0.64.5 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/local/BaseSyncedStore.d.ts +1 -1
- package/dist/local/BaseSyncedStore.js +3 -3
- package/dist/local/client/createModelOperations.js +23 -8
- package/dist/local/sync/BootstrapFetcher.js +32 -17
- package/dist/local/sync/SyncWebSocket.d.ts +10 -0
- package/dist/local/sync/SyncWebSocket.js +43 -0
- package/dist/local/sync/deltaPipeline.d.ts +3 -1
- package/dist/local/sync/deltaPipeline.js +6 -2
- package/dist/local/sync/socketEventWiring.d.ts +1 -1
- package/dist/local/sync/socketEventWiring.js +22 -2
- package/dist/local/transactions/mutations/batchProcessing.js +4 -0
- package/dist/local/transactions/mutations/commitLatency.d.ts +1 -1
- package/dist/local/transactions/mutations/commitLatency.js +10 -1
- package/package.json +2 -2
- package/src/local/BaseSyncedStore.ts +4 -4
- package/src/local/client/createModelOperations.ts +57 -14
- package/src/local/sync/BootstrapFetcher.ts +32 -17
- package/src/local/sync/SyncWebSocket.ts +39 -0
- package/src/local/sync/deltaPipeline.ts +7 -2
- package/src/local/sync/socketEventWiring.ts +24 -3
- package/src/local/transactions/mutations/batchProcessing.ts +4 -0
- package/src/local/transactions/mutations/commitLatency.ts +10 -2
|
@@ -676,7 +676,7 @@ export declare class BaseSyncedStore<TCollaboration extends EventMap<TCollaborat
|
|
|
676
676
|
* with {@link Database.processDeltaBatch} — the lower-level local write this
|
|
677
677
|
* eventually drives through `flushPendingDeltas`.
|
|
678
678
|
*/
|
|
679
|
-
protected applyDeltaFrame(deltas: SyncDelta[]): void
|
|
679
|
+
protected applyDeltaFrame(deltas: SyncDelta[]): Promise<void>;
|
|
680
680
|
/**
|
|
681
681
|
* Per-delta bookkeeping + enqueue. Returns `true` when the delta was
|
|
682
682
|
* pushed onto `pendingDeltas` (a regular batchable I/U/C/D delta that a
|
|
@@ -877,7 +877,7 @@ export class BaseSyncedStore {
|
|
|
877
877
|
getAllPoolIds: () => this.objectPool.getAllIds(),
|
|
878
878
|
get bootstrapDeltaQueue() { return store.bootstrapDeltaQueue; },
|
|
879
879
|
set bootstrapDeltaQueue(queue) { store.bootstrapDeltaQueue = queue; },
|
|
880
|
-
applyDeltaFrame: (deltas) => { this.applyDeltaFrame(deltas); },
|
|
880
|
+
applyDeltaFrame: (deltas) => { void this.applyDeltaFrame(deltas); },
|
|
881
881
|
};
|
|
882
882
|
}
|
|
883
883
|
/** Apply bootstrap data to the {@link InstanceCache}, removing entities that are no longer present (ghost removal). Pool writes are delegated to {@link SyncClient}. */
|
|
@@ -1100,7 +1100,7 @@ export class BaseSyncedStore {
|
|
|
1100
1100
|
onConnectionEvent: this.onConnectionEvent,
|
|
1101
1101
|
updateSyncStatus: (updates) => { this.updateSyncStatus(updates); },
|
|
1102
1102
|
processDeltaWithBatching: (delta) => { this.processDeltaWithBatching(delta); },
|
|
1103
|
-
applyDeltaFrame: (deltas) =>
|
|
1103
|
+
applyDeltaFrame: (deltas) => this.applyDeltaFrame(deltas),
|
|
1104
1104
|
handleBootstrapRequired: (hint) => { this.handleBootstrapRequired(hint); },
|
|
1105
1105
|
handleBootstrapData: (data) => { this.handleBootstrapData(data); },
|
|
1106
1106
|
performCredentialRefresh: () => this.performCredentialRefresh(),
|
|
@@ -1231,7 +1231,7 @@ export class BaseSyncedStore {
|
|
|
1231
1231
|
* eventually drives through `flushPendingDeltas`.
|
|
1232
1232
|
*/
|
|
1233
1233
|
applyDeltaFrame(deltas) {
|
|
1234
|
-
deltaPipeline.
|
|
1234
|
+
return deltaPipeline.persistDeltaFrame(this.deltaPipelineContext, deltas);
|
|
1235
1235
|
}
|
|
1236
1236
|
/**
|
|
1237
1237
|
* Per-delta bookkeeping + enqueue. Returns `true` when the delta was
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* lookup table from one of these per model.
|
|
12
12
|
*/
|
|
13
13
|
import { autorun } from 'mobx';
|
|
14
|
-
import { AbloClaimedError, AbloValidationError, toAbloError, } from '@abloatai/transaction/errors';
|
|
14
|
+
import { AbloClaimedError, AbloStaleContextError, AbloValidationError, toAbloError, } from '@abloatai/transaction/errors';
|
|
15
15
|
import { reconcileFunctionalUpdate, } from '@abloatai/transaction/client/resources/functionalUpdate';
|
|
16
16
|
import { claimDescription, } from '@abloatai/transaction/coordination/schema';
|
|
17
17
|
import { Model, modelAsRow } from '../Model.js';
|
|
@@ -31,7 +31,7 @@ import { ModelScope } from '@abloatai/transaction/types';
|
|
|
31
31
|
import { bindClaimLifetime, claimLifetimeOf, } from '@abloatai/transaction/claims/lifetime';
|
|
32
32
|
import { claimQueueView, resolveClaimContentionOptions, } from '@abloatai/transaction/client/resources/modelOperations';
|
|
33
33
|
import { modelEventInputSchema } from '@abloatai/transaction/collaboration';
|
|
34
|
-
import { capturePointRead, prepareReadSet, } from '@abloatai/transaction/internal/read-set';
|
|
34
|
+
import { capturePointRead, prepareReadSet, targetGuardForRow, } from '@abloatai/transaction/internal/read-set';
|
|
35
35
|
const ignoreSeparatelyObservedMutationFailure = () => undefined;
|
|
36
36
|
const ignoreBestEffortClaimReleaseFailure = () => undefined;
|
|
37
37
|
const ignoreBestEffortScopeFailure = () => undefined;
|
|
@@ -215,8 +215,9 @@ hydration, collaboration, readSetContext) {
|
|
|
215
215
|
typeof value.id === 'string' &&
|
|
216
216
|
typeof value.release === 'function';
|
|
217
217
|
const preparedMutation = (params) => {
|
|
218
|
-
const prepared = prepareReadSet(readSetContext, readSetClientIdentity,
|
|
218
|
+
const prepared = prepareReadSet(readSetContext, readSetClientIdentity, params.readAt, params.idempotencyKey, params.reads);
|
|
219
219
|
const rest = {
|
|
220
|
+
...(prepared.readAt !== undefined ? { readAt: prepared.readAt } : {}),
|
|
220
221
|
...(prepared.idempotencyKey !== undefined
|
|
221
222
|
? { idempotencyKey: prepared.idempotencyKey }
|
|
222
223
|
: params.idempotencyKey !== undefined
|
|
@@ -225,9 +226,7 @@ hydration, collaboration, readSetContext) {
|
|
|
225
226
|
...(params.label !== undefined ? { label: params.label } : {}),
|
|
226
227
|
...(prepared.reads !== undefined
|
|
227
228
|
? { reads: prepared.reads === null ? null : [...prepared.reads] }
|
|
228
|
-
:
|
|
229
|
-
? { reads: params.reads }
|
|
230
|
-
: {}),
|
|
229
|
+
: {}),
|
|
231
230
|
};
|
|
232
231
|
// The write-options schema — the runtime twin of the compile-time params.
|
|
233
232
|
// Catches plain-JavaScript callers at the call site with a typed error
|
|
@@ -235,6 +234,15 @@ hydration, collaboration, readSetContext) {
|
|
|
235
234
|
assertWriteOptions(rest, `${schemaKey} write`);
|
|
236
235
|
return rest;
|
|
237
236
|
};
|
|
237
|
+
const guardedMutation = (params) => {
|
|
238
|
+
if (params.ifUnchanged === undefined)
|
|
239
|
+
return params;
|
|
240
|
+
const { ifUnchanged, ...rest } = params;
|
|
241
|
+
return {
|
|
242
|
+
...rest,
|
|
243
|
+
...targetGuardForRow(readSetContext, readSetClientIdentity, wireModel, ifUnchanged),
|
|
244
|
+
};
|
|
245
|
+
};
|
|
238
246
|
const releaseClaim = async (claimId) => {
|
|
239
247
|
const held = removeActiveClaim(claimId);
|
|
240
248
|
if (!held)
|
|
@@ -969,7 +977,7 @@ hydration, collaboration, readSetContext) {
|
|
|
969
977
|
},
|
|
970
978
|
});
|
|
971
979
|
}
|
|
972
|
-
const params = arg;
|
|
980
|
+
const params = guardedMutation(arg);
|
|
973
981
|
// Named before anything reads it. Without this the row lookup below
|
|
974
982
|
// reports `Entity not found: Model/undefined`, which sends the reader
|
|
975
983
|
// looking for a missing row rather than at the unaddressed write.
|
|
@@ -991,6 +999,9 @@ hydration, collaboration, readSetContext) {
|
|
|
991
999
|
}
|
|
992
1000
|
const { id } = params;
|
|
993
1001
|
const model = ownRowOrThrow(id);
|
|
1002
|
+
if (!model && params.readAt !== undefined) {
|
|
1003
|
+
throw new AbloStaleContextError(`Update rejected: ${registeredModelName}/${id} changed since read. Re-read and retry.`, { code: 'stale_context', httpStatus: 409 });
|
|
1004
|
+
}
|
|
994
1005
|
if (!model)
|
|
995
1006
|
throw new AbloValidationError(`Entity not found: ${registeredModelName}/${id}`, { code: 'entity_not_found' });
|
|
996
1007
|
const opts = preparedMutation(params);
|
|
@@ -1025,7 +1036,8 @@ hydration, collaboration, readSetContext) {
|
|
|
1025
1036
|
}
|
|
1026
1037
|
return update;
|
|
1027
1038
|
})(),
|
|
1028
|
-
delete: guardWrite(async (
|
|
1039
|
+
delete: guardWrite(async (input) => {
|
|
1040
|
+
const params = guardedMutation(input);
|
|
1029
1041
|
// Before the idempotent "ensure absent" below can read this as a row that
|
|
1030
1042
|
// is simply not here. An unaddressed delete is a mistake, not an absence.
|
|
1031
1043
|
assertWriteTarget('delete', registeredModelName, params.id);
|
|
@@ -1055,6 +1067,9 @@ hydration, collaboration, readSetContext) {
|
|
|
1055
1067
|
// HTTP client and makes delete safe to retry or race (two actors deleting
|
|
1056
1068
|
// the same row).
|
|
1057
1069
|
if (!model) {
|
|
1070
|
+
if (params.readAt !== undefined) {
|
|
1071
|
+
throw new AbloStaleContextError(`Delete rejected: ${registeredModelName}/${id} changed since read. Re-read and retry.`, { code: 'stale_context', httpStatus: 409 });
|
|
1072
|
+
}
|
|
1058
1073
|
const handle = isClaimHandle(params.claim) ? params.claim : undefined;
|
|
1059
1074
|
await settleClaimsAfterWrite(id, handle);
|
|
1060
1075
|
return;
|
|
@@ -28,6 +28,19 @@ const PAGE_LIMIT = 5000;
|
|
|
28
28
|
const MAX_PAGES_PER_MODEL = 200;
|
|
29
29
|
/** How many model chunks a cold start fetches at once. */
|
|
30
30
|
const CHUNK_CONCURRENCY = 3;
|
|
31
|
+
/** Keep bootstrap URLs below common proxy limits when the subscribed group set grows. */
|
|
32
|
+
function bootstrapRequest(baseUrl, params) {
|
|
33
|
+
const url = `${baseUrl}/sync/bootstrap?${params.toString()}`;
|
|
34
|
+
if (url.length <= 8_000)
|
|
35
|
+
return { url, method: 'GET' };
|
|
36
|
+
const syncGroups = params.getAll('syncGroups');
|
|
37
|
+
params.delete('syncGroups');
|
|
38
|
+
return {
|
|
39
|
+
url: `${baseUrl}/sync/bootstrap?${params.toString()}`,
|
|
40
|
+
method: 'POST',
|
|
41
|
+
body: JSON.stringify({ syncGroups }),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
31
44
|
/**
|
|
32
45
|
* The reason handed to `abort()` when a request is stopped deliberately —
|
|
33
46
|
* superseded by a newer bootstrap, or abandoned because the bootstrap it
|
|
@@ -359,7 +372,7 @@ export class BootstrapFetcher {
|
|
|
359
372
|
if (this.options.instantModels && this.options.instantModels.length > 0) {
|
|
360
373
|
params.append('models', this.options.instantModels.join(','));
|
|
361
374
|
}
|
|
362
|
-
const
|
|
375
|
+
const request = bootstrapRequest(this.options.baseUrl, params);
|
|
363
376
|
// If offline, try the cached bootstrap. Skipped for a scoped override: the
|
|
364
377
|
// cache holds the full snapshot, which is not a valid answer to a subset
|
|
365
378
|
// request; a scoped hydrate just soft-fails offline and retries on re-enter.
|
|
@@ -384,7 +397,7 @@ export class BootstrapFetcher {
|
|
|
384
397
|
code: 'bootstrap_offline_no_cache',
|
|
385
398
|
});
|
|
386
399
|
}
|
|
387
|
-
this.runtime.logger.info('Fetching fresh bootstrap data', { url });
|
|
400
|
+
this.runtime.logger.info('Fetching fresh bootstrap data', { url: request.url });
|
|
388
401
|
const lane = syncGroupsOverride ? 'scoped' : 'bootstrap';
|
|
389
402
|
// Chunk a COLD start by model: each instant model is its own request, so
|
|
390
403
|
// one giant model can't make the whole snapshot undeliverable, and a
|
|
@@ -401,7 +414,7 @@ export class BootstrapFetcher {
|
|
|
401
414
|
try {
|
|
402
415
|
const data = chunked
|
|
403
416
|
? await this.fetchChunkedBootstrap(instantModels, this.options.syncGroups)
|
|
404
|
-
: await this.fetchWithRetries(
|
|
417
|
+
: await this.fetchWithRetries(request, lane);
|
|
405
418
|
this.runtime.logger.info('Bootstrap data fetched', {
|
|
406
419
|
type: data.type,
|
|
407
420
|
lastSyncId: data.lastSyncId,
|
|
@@ -448,12 +461,12 @@ export class BootstrapFetcher {
|
|
|
448
461
|
* (5xx, 429, timeouts, network blips) consume attempts. A cancellation is
|
|
449
462
|
* deliberate and therefore non-retryable — it leaves through the same gate.
|
|
450
463
|
*/
|
|
451
|
-
async fetchWithRetries(
|
|
464
|
+
async fetchWithRetries(request, lane) {
|
|
452
465
|
let lastError = null;
|
|
453
466
|
const capacityDeadline = Date.now() + this.options.fetchTimeout;
|
|
454
467
|
for (let attempt = 0; attempt < this.options.maxRetries;) {
|
|
455
468
|
try {
|
|
456
|
-
return await this.fetchOnce(
|
|
469
|
+
return await this.fetchOnce(request, lane);
|
|
457
470
|
}
|
|
458
471
|
catch (error) {
|
|
459
472
|
// SessionError should NOT be retried - the session is invalid and needs re-authentication
|
|
@@ -539,8 +552,8 @@ export class BootstrapFetcher {
|
|
|
539
552
|
params.append('limit', String(PAGE_LIMIT));
|
|
540
553
|
if (cursor !== undefined)
|
|
541
554
|
params.append('cursor', cursor);
|
|
542
|
-
const
|
|
543
|
-
const data = await this.fetchWithRetries(
|
|
555
|
+
const request = bootstrapRequest(this.options.baseUrl, params);
|
|
556
|
+
const data = await this.fetchWithRetries(request, 'bootstrap');
|
|
544
557
|
chunks.push(data);
|
|
545
558
|
if (data.nextCursor === undefined)
|
|
546
559
|
break;
|
|
@@ -576,7 +589,7 @@ export class BootstrapFetcher {
|
|
|
576
589
|
if (this.options.instantModels && this.options.instantModels.length > 0) {
|
|
577
590
|
params.append('models', this.options.instantModels.join(','));
|
|
578
591
|
}
|
|
579
|
-
const
|
|
592
|
+
const request = bootstrapRequest(this.options.baseUrl, params);
|
|
580
593
|
// Note: ETag caching is deliberately app-side, not SDK-side. The server
|
|
581
594
|
// still returns an ETag on responses, which is captured below and
|
|
582
595
|
// forwarded to callers via BootstrapFetchResult.etag — apps that want
|
|
@@ -587,15 +600,16 @@ export class BootstrapFetcher {
|
|
|
587
600
|
const controller = new AbortController();
|
|
588
601
|
this.activeControllers.set(controller, 'bootstrap');
|
|
589
602
|
try {
|
|
590
|
-
return await this.fetchWithETagUsing(
|
|
603
|
+
return await this.fetchWithETagUsing(request, headers, controller);
|
|
591
604
|
}
|
|
592
605
|
finally {
|
|
593
606
|
this.activeControllers.delete(controller);
|
|
594
607
|
}
|
|
595
608
|
}
|
|
596
|
-
async fetchWithETagUsing(
|
|
597
|
-
const res = await fetch(url, {
|
|
598
|
-
method:
|
|
609
|
+
async fetchWithETagUsing(request, headers, controller) {
|
|
610
|
+
const res = await fetch(request.url, {
|
|
611
|
+
method: request.method,
|
|
612
|
+
body: request.body,
|
|
599
613
|
headers,
|
|
600
614
|
signal: controller.signal,
|
|
601
615
|
});
|
|
@@ -726,17 +740,17 @@ export class BootstrapFetcher {
|
|
|
726
740
|
* registry) — chunk requests run through here concurrently and must
|
|
727
741
|
* not cancel each other.
|
|
728
742
|
*/
|
|
729
|
-
async fetchOnce(
|
|
743
|
+
async fetchOnce(request, lane) {
|
|
730
744
|
const controller = new AbortController();
|
|
731
745
|
this.activeControllers.set(controller, lane);
|
|
732
746
|
try {
|
|
733
|
-
return await this.fetchOnceWith(
|
|
747
|
+
return await this.fetchOnceWith(request, controller);
|
|
734
748
|
}
|
|
735
749
|
finally {
|
|
736
750
|
this.activeControllers.delete(controller);
|
|
737
751
|
}
|
|
738
752
|
}
|
|
739
|
-
async fetchOnceWith(
|
|
753
|
+
async fetchOnceWith(request, controller) {
|
|
740
754
|
const timeoutId = setTimeout(() => {
|
|
741
755
|
this.runtime.observability.breadcrumb('Bootstrap fetch timeout', 'sync.bootstrap', 'warning', {
|
|
742
756
|
timeoutMs: this.options.fetchTimeout,
|
|
@@ -745,8 +759,9 @@ export class BootstrapFetcher {
|
|
|
745
759
|
}, this.options.fetchTimeout);
|
|
746
760
|
let response;
|
|
747
761
|
try {
|
|
748
|
-
response = await fetch(url, {
|
|
749
|
-
method:
|
|
762
|
+
response = await fetch(request.url, {
|
|
763
|
+
method: request.method,
|
|
764
|
+
body: request.body,
|
|
750
765
|
headers: withAuthHeaders(this.options.getAuthToken, {
|
|
751
766
|
'Content-Type': 'application/json',
|
|
752
767
|
'Cache-Control': 'no-cache, no-store, must-revalidate',
|
|
@@ -62,6 +62,7 @@ export declare class SyncWebSocket<TCollaboration extends EventMap<TCollaboratio
|
|
|
62
62
|
* the state itself lives in {@link SyncCursor}.
|
|
63
63
|
*/
|
|
64
64
|
private readonly cursor;
|
|
65
|
+
private catchUp;
|
|
65
66
|
constructor(options: SyncWebSocketOptions);
|
|
66
67
|
/** The persisted resume position, sent on the upgrade URL. */
|
|
67
68
|
protected resumeCursor(): string;
|
|
@@ -121,6 +122,12 @@ export declare class SyncWebSocket<TCollaboration extends EventMap<TCollaboratio
|
|
|
121
122
|
* Public wrapper for sending ack from outside the class
|
|
122
123
|
*/
|
|
123
124
|
acknowledge(syncId: number): void;
|
|
125
|
+
/** Publish completion only after the store's chunk-persistence lane drains. */
|
|
126
|
+
completeCatchUp(payload: {
|
|
127
|
+
exchangeId: string;
|
|
128
|
+
currentSyncId: number;
|
|
129
|
+
chunks: number;
|
|
130
|
+
}): void;
|
|
124
131
|
/**
|
|
125
132
|
* Stop the periodic catchup interval
|
|
126
133
|
*/
|
|
@@ -160,6 +167,9 @@ export declare class SyncWebSocket<TCollaboration extends EventMap<TCollaboratio
|
|
|
160
167
|
* of the batch).
|
|
161
168
|
*/
|
|
162
169
|
protected handleSyncResponse(rawPayload: unknown): void;
|
|
170
|
+
protected handleCatchUpBegin(rawPayload: unknown): void;
|
|
171
|
+
protected handleCatchUpChunk(rawPayload: unknown): void;
|
|
172
|
+
protected handleCatchUpEnd(rawPayload: unknown): void;
|
|
163
173
|
/**
|
|
164
174
|
* Handle bootstrap response from server
|
|
165
175
|
*/
|
|
@@ -38,6 +38,7 @@ export class SyncWebSocket extends WsTransport {
|
|
|
38
38
|
* the state itself lives in {@link SyncCursor}.
|
|
39
39
|
*/
|
|
40
40
|
cursor;
|
|
41
|
+
catchUp = null;
|
|
41
42
|
constructor(options) {
|
|
42
43
|
super({
|
|
43
44
|
...options,
|
|
@@ -221,6 +222,10 @@ export class SyncWebSocket extends WsTransport {
|
|
|
221
222
|
acknowledge(syncId) {
|
|
222
223
|
this.sendAck(syncId);
|
|
223
224
|
}
|
|
225
|
+
/** Publish completion only after the store's chunk-persistence lane drains. */
|
|
226
|
+
completeCatchUp(payload) {
|
|
227
|
+
this.emit('catchup_complete', payload);
|
|
228
|
+
}
|
|
224
229
|
/**
|
|
225
230
|
* Stop the periodic catchup interval
|
|
226
231
|
*/
|
|
@@ -415,6 +420,44 @@ export class SyncWebSocket extends WsTransport {
|
|
|
415
420
|
this.cursor.syncCursor = payload.cursor;
|
|
416
421
|
}
|
|
417
422
|
}
|
|
423
|
+
handleCatchUpBegin(rawPayload) {
|
|
424
|
+
if (!isRecord(rawPayload))
|
|
425
|
+
return;
|
|
426
|
+
const { exchangeId, fromSyncId, currentSyncId } = rawPayload;
|
|
427
|
+
if (typeof exchangeId !== 'string' || !Number.isSafeInteger(fromSyncId) || !Number.isSafeInteger(currentSyncId))
|
|
428
|
+
return;
|
|
429
|
+
this.catchUp = { exchangeId, currentSyncId: currentSyncId, nextSequence: 0 };
|
|
430
|
+
this.emit('catchup_begin', { exchangeId, fromSyncId: fromSyncId, currentSyncId: currentSyncId });
|
|
431
|
+
}
|
|
432
|
+
handleCatchUpChunk(rawPayload) {
|
|
433
|
+
if (!isRecord(rawPayload) || !this.catchUp)
|
|
434
|
+
return;
|
|
435
|
+
const { exchangeId, sequence, position, deltas } = rawPayload;
|
|
436
|
+
if (exchangeId !== this.catchUp.exchangeId || sequence !== this.catchUp.nextSequence
|
|
437
|
+
|| !Number.isSafeInteger(position) || position > this.catchUp.currentSyncId
|
|
438
|
+
|| !Array.isArray(deltas))
|
|
439
|
+
return;
|
|
440
|
+
const normalized = [];
|
|
441
|
+
for (const raw of deltas) {
|
|
442
|
+
const delta = this.normalizeWireDelta(raw);
|
|
443
|
+
if (delta)
|
|
444
|
+
normalized.push(delta);
|
|
445
|
+
}
|
|
446
|
+
if (normalized.length !== deltas.length)
|
|
447
|
+
return;
|
|
448
|
+
this.catchUp.nextSequence++;
|
|
449
|
+
this.emit('catchup_chunk', { exchangeId, sequence, position, deltas: normalized });
|
|
450
|
+
}
|
|
451
|
+
handleCatchUpEnd(rawPayload) {
|
|
452
|
+
if (!isRecord(rawPayload) || !this.catchUp)
|
|
453
|
+
return;
|
|
454
|
+
const { exchangeId, currentSyncId, chunks } = rawPayload;
|
|
455
|
+
if (exchangeId !== this.catchUp.exchangeId || currentSyncId !== this.catchUp.currentSyncId
|
|
456
|
+
|| chunks !== this.catchUp.nextSequence)
|
|
457
|
+
return;
|
|
458
|
+
this.catchUp = null;
|
|
459
|
+
this.emit('catchup_end', { exchangeId, currentSyncId, chunks });
|
|
460
|
+
}
|
|
418
461
|
/**
|
|
419
462
|
* Handle bootstrap response from server
|
|
420
463
|
*/
|
|
@@ -118,8 +118,10 @@ export declare function enqueueDelta(ctx: DeltaPipelineContext, delta: SyncDelta
|
|
|
118
118
|
}): boolean;
|
|
119
119
|
/** Debounce a flush for live single-delta traffic. */
|
|
120
120
|
export declare function scheduleDeltaFlush(ctx: DeltaPipelineContext): void;
|
|
121
|
-
/** Apply an authoritative
|
|
121
|
+
/** Apply an authoritative legacy frame without exposing its persistence promise. */
|
|
122
122
|
export declare function applyDeltaFrame(ctx: DeltaPipelineContext, deltas: SyncDelta[]): void;
|
|
123
|
+
/** Persist one resumable chunk before its raw tail position may advance. */
|
|
124
|
+
export declare function persistDeltaFrame(ctx: DeltaPipelineContext, deltas: SyncDelta[]): Promise<void>;
|
|
123
125
|
/**
|
|
124
126
|
* Flushes the queued deltas: deduplicates them, applies custom-entity deltas
|
|
125
127
|
* straight to the pool, writes the rest to the local store and then the pool,
|
|
@@ -164,8 +164,12 @@ export function scheduleDeltaFlush(ctx) {
|
|
|
164
164
|
}, ctx.smartSyncOptions.batchingDelay);
|
|
165
165
|
}
|
|
166
166
|
}
|
|
167
|
-
/** Apply an authoritative
|
|
167
|
+
/** Apply an authoritative legacy frame without exposing its persistence promise. */
|
|
168
168
|
export function applyDeltaFrame(ctx, deltas) {
|
|
169
|
+
void persistDeltaFrame(ctx, deltas).catch(ctx.handleFlushError);
|
|
170
|
+
}
|
|
171
|
+
/** Persist one resumable chunk before its raw tail position may advance. */
|
|
172
|
+
export async function persistDeltaFrame(ctx, deltas) {
|
|
169
173
|
let enqueuedAny = false;
|
|
170
174
|
for (const delta of deltas) {
|
|
171
175
|
if (enqueueDelta(ctx, delta, { authoritative: true }))
|
|
@@ -177,7 +181,7 @@ export function applyDeltaFrame(ctx, deltas) {
|
|
|
177
181
|
clearTimeout(ctx.batchTimer);
|
|
178
182
|
ctx.batchTimer = null;
|
|
179
183
|
}
|
|
180
|
-
|
|
184
|
+
await ctx.flushPendingDeltas();
|
|
181
185
|
}
|
|
182
186
|
/**
|
|
183
187
|
* Flushes the queued deltas: deduplicates them, applies custom-entity deltas
|
|
@@ -20,7 +20,7 @@ export interface SocketEventHost<TCollaboration extends EventMap<TCollaboration>
|
|
|
20
20
|
onConnectionEvent?: (event: string) => void;
|
|
21
21
|
updateSyncStatus(updates: Partial<SyncStatus>): void;
|
|
22
22
|
processDeltaWithBatching(delta: SyncDelta): void;
|
|
23
|
-
applyDeltaFrame(deltas: SyncDelta[]): void
|
|
23
|
+
applyDeltaFrame(deltas: SyncDelta[]): Promise<void>;
|
|
24
24
|
handleBootstrapRequired(hint: BootstrapHint): void;
|
|
25
25
|
handleBootstrapData(data: BootstrapDataEvent): void;
|
|
26
26
|
performCredentialRefresh(): Promise<'refreshed' | 'session_error' | 'network_error'>;
|
|
@@ -35,7 +35,27 @@ export function wireSocketEvents(deps) {
|
|
|
35
35
|
// A catch-up/reconnect frame is already complete — apply it as ONE
|
|
36
36
|
// atomic flush so the gallery re-renders once, not once per 50-delta
|
|
37
37
|
// chunk. See `applyDeltaFrame`.
|
|
38
|
-
deps.applyDeltaFrame(deltas)
|
|
38
|
+
void deps.applyDeltaFrame(deltas).catch((error) => {
|
|
39
|
+
deps.updateSyncStatus({ state: 'error', error: error instanceof Error ? error : new Error(String(error)) });
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
let catchUpLane = Promise.resolve();
|
|
43
|
+
const onCatchUpChunk = deps.syncWebSocket.subscribe('catchup_chunk', (chunk) => {
|
|
44
|
+
catchUpLane = catchUpLane.then(async () => {
|
|
45
|
+
await deps.applyDeltaFrame(chunk.deltas);
|
|
46
|
+
// The raw position covers filtered rows too. Advance only after this
|
|
47
|
+
// chunk's visible rows are durable; replay is therefore idempotent.
|
|
48
|
+
deps.syncWebSocket.acknowledge(chunk.position);
|
|
49
|
+
});
|
|
50
|
+
void catchUpLane.catch((error) => {
|
|
51
|
+
deps.updateSyncStatus({ state: 'error', error: error instanceof Error ? error : new Error(String(error)) });
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
const onCatchUpEnd = deps.syncWebSocket.subscribe('catchup_end', (end) => {
|
|
55
|
+
void catchUpLane.then(() => {
|
|
56
|
+
deps.syncWebSocket.acknowledge(end.currentSyncId);
|
|
57
|
+
deps.syncWebSocket.completeCatchUp(end);
|
|
58
|
+
}).catch(() => undefined);
|
|
39
59
|
});
|
|
40
60
|
// Bootstrap events
|
|
41
61
|
const onBootstrapRequired = deps.syncWebSocket.subscribe('bootstrap_required', (hint) => { deps.handleBootstrapRequired(hint); });
|
|
@@ -122,5 +142,5 @@ export function wireSocketEvents(deps) {
|
|
|
122
142
|
deps.runtime.logger.debug('[BaseSyncedStore] WebSocket reconnection gave up', { attempts });
|
|
123
143
|
deps.updateSyncStatus({ state: 'reconnecting' });
|
|
124
144
|
});
|
|
125
|
-
deps.disposers.push(onConnected, onDisconnected, onReconnecting, onDelta, onDeltaBatch, onBootstrapRequired, onBootstrapData, onError, onSessionError, onHandshakeFailed, onReconnectFailed, () => { deps.areaOfInterest.dispose(); });
|
|
145
|
+
deps.disposers.push(onConnected, onDisconnected, onReconnecting, onDelta, onDeltaBatch, onCatchUpChunk, onCatchUpEnd, onBootstrapRequired, onBootstrapData, onError, onSessionError, onHandshakeFailed, onReconnectFailed, () => { deps.areaOfInterest.dispose(); });
|
|
126
146
|
}
|
|
@@ -73,6 +73,9 @@ export async function processBatch(ctx) {
|
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
76
|
+
for (const tx of batch) {
|
|
77
|
+
ctx.emitCommitLifecycle('model:sealing', { clientTxId: tx.id });
|
|
78
|
+
}
|
|
76
79
|
durableEnvelope = await ctx.sealDurableCommit({
|
|
77
80
|
idempotencyKey: commitIdempotencyKey,
|
|
78
81
|
origin: 'model_batch',
|
|
@@ -85,6 +88,7 @@ export async function processBatch(ctx) {
|
|
|
85
88
|
});
|
|
86
89
|
for (const transaction of batch) {
|
|
87
90
|
transaction.durableEnvelope = durableEnvelope;
|
|
91
|
+
ctx.emitCommitLifecycle('model:sealed', { clientTxId: transaction.id });
|
|
88
92
|
}
|
|
89
93
|
}
|
|
90
94
|
const operations = durableEnvelope.operations;
|
|
@@ -34,7 +34,7 @@ export interface CommitEventSource {
|
|
|
34
34
|
}
|
|
35
35
|
/** One completed commit, broken into its local and remote halves. */
|
|
36
36
|
export interface CommitLatencySample {
|
|
37
|
-
/**
|
|
37
|
+
/** Local transaction id; for atomic commits this is also the wire clientTxId. */
|
|
38
38
|
clientTxId: string;
|
|
39
39
|
/** Milliseconds sealing the durable envelope locally. */
|
|
40
40
|
sealMs: number;
|
|
@@ -64,6 +64,9 @@ export function observeCommitLatency(source, onSample) {
|
|
|
64
64
|
const id = readClientTxId(payload);
|
|
65
65
|
if (id === null)
|
|
66
66
|
return;
|
|
67
|
+
// Retain the first seal attempt across retries rather than hiding its wait.
|
|
68
|
+
if (pending.has(id))
|
|
69
|
+
return;
|
|
67
70
|
// Map preserves insertion order, so the first key is the stalest entry.
|
|
68
71
|
if (pending.size >= MAX_PENDING_COMMITS) {
|
|
69
72
|
const oldest = pending.keys().next();
|
|
@@ -79,7 +82,7 @@ export function observeCommitLatency(source, onSample) {
|
|
|
79
82
|
const timing = pending.get(id);
|
|
80
83
|
if (timing === undefined)
|
|
81
84
|
return;
|
|
82
|
-
timing.sealedAt
|
|
85
|
+
timing.sealedAt ??= nowMs();
|
|
83
86
|
};
|
|
84
87
|
const handleCompleted = (payload) => {
|
|
85
88
|
const id = readTransactionId(payload);
|
|
@@ -116,12 +119,18 @@ export function observeCommitLatency(source, onSample) {
|
|
|
116
119
|
};
|
|
117
120
|
source.on('commit:staging', handleStaging);
|
|
118
121
|
source.on('commit:created', handleCreated);
|
|
122
|
+
// Model batches share an envelope, but confirmation belongs to each local
|
|
123
|
+
// transaction. Restored envelopes have no observed seal and emit no sample.
|
|
124
|
+
source.on('model:sealing', handleStaging);
|
|
125
|
+
source.on('model:sealed', handleCreated);
|
|
119
126
|
source.on('commit:seal_failed', handleSealFailed);
|
|
120
127
|
source.on('transaction:completed', handleCompleted);
|
|
121
128
|
source.on('transaction:failed', handleFailed);
|
|
122
129
|
return () => {
|
|
123
130
|
source.off('commit:staging', handleStaging);
|
|
124
131
|
source.off('commit:created', handleCreated);
|
|
132
|
+
source.off('model:sealing', handleStaging);
|
|
133
|
+
source.off('model:sealed', handleCreated);
|
|
125
134
|
source.off('commit:seal_failed', handleSealFailed);
|
|
126
135
|
source.off('transaction:completed', handleCompleted);
|
|
127
136
|
source.off('transaction:failed', handleFailed);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@abloatai/humans",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.66.0",
|
|
4
4
|
"description": "The optional human-facing local-state package for Ablo: presence, live queries, and React bindings.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -85,7 +85,7 @@
|
|
|
85
85
|
"directory": "packages/humans"
|
|
86
86
|
},
|
|
87
87
|
"dependencies": {
|
|
88
|
-
"@abloatai/transaction": "0.
|
|
88
|
+
"@abloatai/transaction": "0.66.0",
|
|
89
89
|
"events": "^3.3.0",
|
|
90
90
|
"mobx": "^6.13.7",
|
|
91
91
|
"uuid": "^11.1.0",
|
|
@@ -1247,7 +1247,7 @@ export class BaseSyncedStore<
|
|
|
1247
1247
|
getAllPoolIds: () => this.objectPool.getAllIds(),
|
|
1248
1248
|
get bootstrapDeltaQueue() { return store.bootstrapDeltaQueue; },
|
|
1249
1249
|
set bootstrapDeltaQueue(queue) { store.bootstrapDeltaQueue = queue; },
|
|
1250
|
-
applyDeltaFrame: (deltas) => { this.applyDeltaFrame(deltas); },
|
|
1250
|
+
applyDeltaFrame: (deltas) => { void this.applyDeltaFrame(deltas); },
|
|
1251
1251
|
};
|
|
1252
1252
|
}
|
|
1253
1253
|
|
|
@@ -1495,7 +1495,7 @@ export class BaseSyncedStore<
|
|
|
1495
1495
|
onConnectionEvent: this.onConnectionEvent,
|
|
1496
1496
|
updateSyncStatus: (updates) => { this.updateSyncStatus(updates); },
|
|
1497
1497
|
processDeltaWithBatching: (delta) => { this.processDeltaWithBatching(delta); },
|
|
1498
|
-
applyDeltaFrame: (deltas) =>
|
|
1498
|
+
applyDeltaFrame: (deltas) => this.applyDeltaFrame(deltas),
|
|
1499
1499
|
handleBootstrapRequired: (hint) => { this.handleBootstrapRequired(hint); },
|
|
1500
1500
|
handleBootstrapData: (data) => { this.handleBootstrapData(data); },
|
|
1501
1501
|
performCredentialRefresh: () => this.performCredentialRefresh(),
|
|
@@ -1636,8 +1636,8 @@ export class BaseSyncedStore<
|
|
|
1636
1636
|
* with {@link Database.processDeltaBatch} — the lower-level local write this
|
|
1637
1637
|
* eventually drives through `flushPendingDeltas`.
|
|
1638
1638
|
*/
|
|
1639
|
-
protected applyDeltaFrame(deltas: SyncDelta[]): void {
|
|
1640
|
-
deltaPipeline.
|
|
1639
|
+
protected applyDeltaFrame(deltas: SyncDelta[]): Promise<void> {
|
|
1640
|
+
return deltaPipeline.persistDeltaFrame(this.deltaPipelineContext, deltas);
|
|
1641
1641
|
}
|
|
1642
1642
|
/**
|
|
1643
1643
|
* Per-delta bookkeeping + enqueue. Returns `true` when the delta was
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import { autorun } from 'mobx';
|
|
15
15
|
import {
|
|
16
16
|
AbloClaimedError,
|
|
17
|
+
AbloStaleContextError,
|
|
17
18
|
AbloValidationError,
|
|
18
19
|
toAbloError,
|
|
19
20
|
} from '@abloatai/transaction/errors';
|
|
@@ -49,7 +50,10 @@ import type {
|
|
|
49
50
|
CommitCreateOptions,
|
|
50
51
|
CommitReceipt,
|
|
51
52
|
} from '@abloatai/transaction/client/resources/httpResources';
|
|
52
|
-
import type {
|
|
53
|
+
import type {
|
|
54
|
+
HttpGuardedMutationParams,
|
|
55
|
+
HttpModelMutationParams,
|
|
56
|
+
} from '@abloatai/transaction/transport/http';
|
|
53
57
|
import {
|
|
54
58
|
collectModelList,
|
|
55
59
|
modelList,
|
|
@@ -154,6 +158,7 @@ import { modelEventInputSchema } from '@abloatai/transaction/collaboration';
|
|
|
154
158
|
import {
|
|
155
159
|
capturePointRead,
|
|
156
160
|
prepareReadSet,
|
|
161
|
+
targetGuardForRow,
|
|
157
162
|
type ReadSetContext,
|
|
158
163
|
} from '@abloatai/transaction/internal/read-set';
|
|
159
164
|
|
|
@@ -684,19 +689,22 @@ export function createModelOperations<T, C>(
|
|
|
684
689
|
typeof (value as { release?: unknown }).release === 'function';
|
|
685
690
|
|
|
686
691
|
const preparedMutation = (
|
|
687
|
-
params:
|
|
688
|
-
|
|
|
689
|
-
|
|
690
|
-
|
|
692
|
+
params: {
|
|
693
|
+
readonly idempotencyKey?: string | null;
|
|
694
|
+
readonly label?: string;
|
|
695
|
+
readonly readAt?: number;
|
|
696
|
+
readonly reads?: readonly unknown[] | null;
|
|
697
|
+
},
|
|
691
698
|
): MutationOptions => {
|
|
692
699
|
const prepared = prepareReadSet(
|
|
693
700
|
readSetContext,
|
|
694
701
|
readSetClientIdentity,
|
|
695
|
-
|
|
702
|
+
params.readAt,
|
|
696
703
|
params.idempotencyKey,
|
|
697
704
|
params.reads,
|
|
698
705
|
);
|
|
699
706
|
const rest: MutationOptions = {
|
|
707
|
+
...(prepared.readAt !== undefined ? { readAt: prepared.readAt } : {}),
|
|
700
708
|
...(prepared.idempotencyKey !== undefined
|
|
701
709
|
? { idempotencyKey: prepared.idempotencyKey }
|
|
702
710
|
: params.idempotencyKey !== undefined
|
|
@@ -705,9 +713,7 @@ export function createModelOperations<T, C>(
|
|
|
705
713
|
...(params.label !== undefined ? { label: params.label } : {}),
|
|
706
714
|
...(prepared.reads !== undefined
|
|
707
715
|
? { reads: prepared.reads === null ? null : [...prepared.reads] }
|
|
708
|
-
:
|
|
709
|
-
? { reads: params.reads }
|
|
710
|
-
: {}),
|
|
716
|
+
: {}),
|
|
711
717
|
};
|
|
712
718
|
// The write-options schema — the runtime twin of the compile-time params.
|
|
713
719
|
// Catches plain-JavaScript callers at the call site with a typed error
|
|
@@ -716,6 +722,22 @@ export function createModelOperations<T, C>(
|
|
|
716
722
|
return rest;
|
|
717
723
|
};
|
|
718
724
|
|
|
725
|
+
const guardedMutation = <P extends { readonly ifUnchanged?: unknown }>(
|
|
726
|
+
params: P,
|
|
727
|
+
): Omit<P, 'ifUnchanged'> & { readonly readAt?: number } => {
|
|
728
|
+
if (params.ifUnchanged === undefined) return params;
|
|
729
|
+
const { ifUnchanged, ...rest } = params;
|
|
730
|
+
return {
|
|
731
|
+
...rest,
|
|
732
|
+
...targetGuardForRow(
|
|
733
|
+
readSetContext,
|
|
734
|
+
readSetClientIdentity,
|
|
735
|
+
wireModel,
|
|
736
|
+
ifUnchanged,
|
|
737
|
+
),
|
|
738
|
+
};
|
|
739
|
+
};
|
|
740
|
+
|
|
719
741
|
const releaseClaim = async (claimId: string): Promise<void> => {
|
|
720
742
|
const held = removeActiveClaim(claimId);
|
|
721
743
|
if (!held) return;
|
|
@@ -1547,7 +1569,9 @@ export function createModelOperations<T, C>(
|
|
|
1547
1569
|
update: ((): ModelOperations<T, C>['update'] => {
|
|
1548
1570
|
const updateImpl = guardWrite(
|
|
1549
1571
|
async (
|
|
1550
|
-
arg:
|
|
1572
|
+
arg:
|
|
1573
|
+
| HttpGuardedMutationParams<ModelUpdateParams<T, C>, T>
|
|
1574
|
+
| string,
|
|
1551
1575
|
updater?: ModelUpdater<T>,
|
|
1552
1576
|
contention?: FunctionalUpdateOptions<ReadDependency | CapturedRow>,
|
|
1553
1577
|
): Promise<T | undefined> => {
|
|
@@ -1634,7 +1658,7 @@ export function createModelOperations<T, C>(
|
|
|
1634
1658
|
},
|
|
1635
1659
|
});
|
|
1636
1660
|
}
|
|
1637
|
-
const params = arg;
|
|
1661
|
+
const params = guardedMutation(arg);
|
|
1638
1662
|
// Named before anything reads it. Without this the row lookup below
|
|
1639
1663
|
// reports `Entity not found: Model/undefined`, which sends the reader
|
|
1640
1664
|
// looking for a missing row rather than at the unaddressed write.
|
|
@@ -1659,6 +1683,12 @@ export function createModelOperations<T, C>(
|
|
|
1659
1683
|
}
|
|
1660
1684
|
const { id } = params;
|
|
1661
1685
|
const model = ownRowOrThrow(id);
|
|
1686
|
+
if (!model && params.readAt !== undefined) {
|
|
1687
|
+
throw new AbloStaleContextError(
|
|
1688
|
+
`Update rejected: ${registeredModelName}/${id} changed since read. Re-read and retry.`,
|
|
1689
|
+
{ code: 'stale_context', httpStatus: 409 },
|
|
1690
|
+
);
|
|
1691
|
+
}
|
|
1662
1692
|
if (!model)
|
|
1663
1693
|
throw new AbloValidationError(
|
|
1664
1694
|
`Entity not found: ${registeredModelName}/${id}`,
|
|
@@ -1696,14 +1726,18 @@ export function createModelOperations<T, C>(
|
|
|
1696
1726
|
return updated;
|
|
1697
1727
|
},
|
|
1698
1728
|
);
|
|
1699
|
-
function update(
|
|
1729
|
+
function update(
|
|
1730
|
+
params: HttpGuardedMutationParams<ModelUpdateParams<T, C>, T>,
|
|
1731
|
+
): Promise<T>;
|
|
1700
1732
|
function update(
|
|
1701
1733
|
id: string,
|
|
1702
1734
|
updater: ModelUpdater<T>,
|
|
1703
1735
|
options?: FunctionalUpdateOptions<ReadDependency | CapturedRow>,
|
|
1704
1736
|
): Promise<T | undefined>;
|
|
1705
1737
|
function update(
|
|
1706
|
-
arg:
|
|
1738
|
+
arg:
|
|
1739
|
+
| HttpGuardedMutationParams<ModelUpdateParams<T, C>, T>
|
|
1740
|
+
| string,
|
|
1707
1741
|
updater?: ModelUpdater<T>,
|
|
1708
1742
|
contention?: FunctionalUpdateOptions<ReadDependency | CapturedRow>,
|
|
1709
1743
|
): Promise<T | undefined> {
|
|
@@ -1712,7 +1746,10 @@ export function createModelOperations<T, C>(
|
|
|
1712
1746
|
return update;
|
|
1713
1747
|
})(),
|
|
1714
1748
|
|
|
1715
|
-
delete: guardWrite(async (
|
|
1749
|
+
delete: guardWrite(async (
|
|
1750
|
+
input: HttpGuardedMutationParams<ModelDeleteParams<T, C>, T>,
|
|
1751
|
+
): Promise<void> => {
|
|
1752
|
+
const params = guardedMutation(input);
|
|
1716
1753
|
// Before the idempotent "ensure absent" below can read this as a row that
|
|
1717
1754
|
// is simply not here. An unaddressed delete is a mistake, not an absence.
|
|
1718
1755
|
assertWriteTarget('delete', registeredModelName, params.id);
|
|
@@ -1745,6 +1782,12 @@ export function createModelOperations<T, C>(
|
|
|
1745
1782
|
// HTTP client and makes delete safe to retry or race (two actors deleting
|
|
1746
1783
|
// the same row).
|
|
1747
1784
|
if (!model) {
|
|
1785
|
+
if (params.readAt !== undefined) {
|
|
1786
|
+
throw new AbloStaleContextError(
|
|
1787
|
+
`Delete rejected: ${registeredModelName}/${id} changed since read. Re-read and retry.`,
|
|
1788
|
+
{ code: 'stale_context', httpStatus: 409 },
|
|
1789
|
+
);
|
|
1790
|
+
}
|
|
1748
1791
|
const handle = isClaimHandle(params.claim) ? params.claim : undefined;
|
|
1749
1792
|
await settleClaimsAfterWrite(id, handle);
|
|
1750
1793
|
return;
|
|
@@ -134,6 +134,19 @@ const MAX_PAGES_PER_MODEL = 200;
|
|
|
134
134
|
/** How many model chunks a cold start fetches at once. */
|
|
135
135
|
const CHUNK_CONCURRENCY = 3;
|
|
136
136
|
|
|
137
|
+
/** Keep bootstrap URLs below common proxy limits when the subscribed group set grows. */
|
|
138
|
+
function bootstrapRequest(baseUrl: string, params: URLSearchParams): { url: string; method: 'GET' | 'POST'; body?: string } {
|
|
139
|
+
const url = `${baseUrl}/sync/bootstrap?${params.toString()}`;
|
|
140
|
+
if (url.length <= 8_000) return { url, method: 'GET' };
|
|
141
|
+
const syncGroups = params.getAll('syncGroups');
|
|
142
|
+
params.delete('syncGroups');
|
|
143
|
+
return {
|
|
144
|
+
url: `${baseUrl}/sync/bootstrap?${params.toString()}`,
|
|
145
|
+
method: 'POST',
|
|
146
|
+
body: JSON.stringify({ syncGroups }),
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
137
150
|
/**
|
|
138
151
|
* Which cancellation lane a request belongs to. Cancellation targets one lane
|
|
139
152
|
* at a time, so superseding a cold-start bootstrap cannot take down a scoped
|
|
@@ -526,7 +539,7 @@ export class BootstrapFetcher {
|
|
|
526
539
|
params.append('models', this.options.instantModels.join(','));
|
|
527
540
|
}
|
|
528
541
|
|
|
529
|
-
const
|
|
542
|
+
const request = bootstrapRequest(this.options.baseUrl, params);
|
|
530
543
|
|
|
531
544
|
// If offline, try the cached bootstrap. Skipped for a scoped override: the
|
|
532
545
|
// cache holds the full snapshot, which is not a valid answer to a subset
|
|
@@ -554,7 +567,7 @@ export class BootstrapFetcher {
|
|
|
554
567
|
});
|
|
555
568
|
}
|
|
556
569
|
|
|
557
|
-
this.runtime.logger.info('Fetching fresh bootstrap data', { url });
|
|
570
|
+
this.runtime.logger.info('Fetching fresh bootstrap data', { url: request.url });
|
|
558
571
|
|
|
559
572
|
const lane: CancelLane = syncGroupsOverride ? 'scoped' : 'bootstrap';
|
|
560
573
|
|
|
@@ -575,7 +588,7 @@ export class BootstrapFetcher {
|
|
|
575
588
|
try {
|
|
576
589
|
const data = chunked
|
|
577
590
|
? await this.fetchChunkedBootstrap(instantModels, this.options.syncGroups)
|
|
578
|
-
: await this.fetchWithRetries(
|
|
591
|
+
: await this.fetchWithRetries(request, lane);
|
|
579
592
|
|
|
580
593
|
this.runtime.logger.info('Bootstrap data fetched', {
|
|
581
594
|
type: data.type,
|
|
@@ -628,12 +641,12 @@ export class BootstrapFetcher {
|
|
|
628
641
|
* (5xx, 429, timeouts, network blips) consume attempts. A cancellation is
|
|
629
642
|
* deliberate and therefore non-retryable — it leaves through the same gate.
|
|
630
643
|
*/
|
|
631
|
-
private async fetchWithRetries(
|
|
644
|
+
private async fetchWithRetries(request: ReturnType<typeof bootstrapRequest>, lane: CancelLane): Promise<BootstrapData> {
|
|
632
645
|
let lastError: Error | null = null;
|
|
633
646
|
const capacityDeadline = Date.now() + this.options.fetchTimeout;
|
|
634
647
|
for (let attempt = 0; attempt < this.options.maxRetries;) {
|
|
635
648
|
try {
|
|
636
|
-
return await this.fetchOnce(
|
|
649
|
+
return await this.fetchOnce(request, lane);
|
|
637
650
|
} catch (error) {
|
|
638
651
|
// SessionError should NOT be retried - the session is invalid and needs re-authentication
|
|
639
652
|
if (AbloSessionError.isSessionError(error)) {
|
|
@@ -740,8 +753,8 @@ export class BootstrapFetcher {
|
|
|
740
753
|
params.append('models', model);
|
|
741
754
|
params.append('limit', String(PAGE_LIMIT));
|
|
742
755
|
if (cursor !== undefined) params.append('cursor', cursor);
|
|
743
|
-
const
|
|
744
|
-
const data = await this.fetchWithRetries(
|
|
756
|
+
const request = bootstrapRequest(this.options.baseUrl, params);
|
|
757
|
+
const data = await this.fetchWithRetries(request, 'bootstrap');
|
|
745
758
|
chunks.push(data);
|
|
746
759
|
if (data.nextCursor === undefined) break;
|
|
747
760
|
cursor = data.nextCursor;
|
|
@@ -781,7 +794,7 @@ export class BootstrapFetcher {
|
|
|
781
794
|
if (this.options.instantModels && this.options.instantModels.length > 0) {
|
|
782
795
|
params.append('models', this.options.instantModels.join(','));
|
|
783
796
|
}
|
|
784
|
-
const
|
|
797
|
+
const request = bootstrapRequest(this.options.baseUrl, params);
|
|
785
798
|
|
|
786
799
|
// Note: ETag caching is deliberately app-side, not SDK-side. The server
|
|
787
800
|
// still returns an ETag on responses, which is captured below and
|
|
@@ -799,19 +812,20 @@ export class BootstrapFetcher {
|
|
|
799
812
|
const controller = new AbortController();
|
|
800
813
|
this.activeControllers.set(controller, 'bootstrap');
|
|
801
814
|
try {
|
|
802
|
-
return await this.fetchWithETagUsing(
|
|
815
|
+
return await this.fetchWithETagUsing(request, headers, controller);
|
|
803
816
|
} finally {
|
|
804
817
|
this.activeControllers.delete(controller);
|
|
805
818
|
}
|
|
806
819
|
}
|
|
807
820
|
|
|
808
821
|
private async fetchWithETagUsing(
|
|
809
|
-
|
|
822
|
+
request: ReturnType<typeof bootstrapRequest>,
|
|
810
823
|
headers: Record<string, string>,
|
|
811
824
|
controller: AbortController,
|
|
812
825
|
): Promise<BootstrapFetchResult> {
|
|
813
|
-
const res = await fetch(url, {
|
|
814
|
-
method:
|
|
826
|
+
const res = await fetch(request.url, {
|
|
827
|
+
method: request.method,
|
|
828
|
+
body: request.body,
|
|
815
829
|
headers,
|
|
816
830
|
signal: controller.signal,
|
|
817
831
|
});
|
|
@@ -973,17 +987,17 @@ export class BootstrapFetcher {
|
|
|
973
987
|
* registry) — chunk requests run through here concurrently and must
|
|
974
988
|
* not cancel each other.
|
|
975
989
|
*/
|
|
976
|
-
private async fetchOnce(
|
|
990
|
+
private async fetchOnce(request: ReturnType<typeof bootstrapRequest>, lane: CancelLane): Promise<BootstrapData> {
|
|
977
991
|
const controller = new AbortController();
|
|
978
992
|
this.activeControllers.set(controller, lane);
|
|
979
993
|
try {
|
|
980
|
-
return await this.fetchOnceWith(
|
|
994
|
+
return await this.fetchOnceWith(request, controller);
|
|
981
995
|
} finally {
|
|
982
996
|
this.activeControllers.delete(controller);
|
|
983
997
|
}
|
|
984
998
|
}
|
|
985
999
|
|
|
986
|
-
private async fetchOnceWith(
|
|
1000
|
+
private async fetchOnceWith(request: ReturnType<typeof bootstrapRequest>, controller: AbortController): Promise<BootstrapData> {
|
|
987
1001
|
const timeoutId = setTimeout(() => {
|
|
988
1002
|
this.runtime.observability.breadcrumb('Bootstrap fetch timeout', 'sync.bootstrap', 'warning', {
|
|
989
1003
|
timeoutMs: this.options.fetchTimeout,
|
|
@@ -998,8 +1012,9 @@ export class BootstrapFetcher {
|
|
|
998
1012
|
|
|
999
1013
|
let response: Response;
|
|
1000
1014
|
try {
|
|
1001
|
-
response = await fetch(url, {
|
|
1002
|
-
method:
|
|
1015
|
+
response = await fetch(request.url, {
|
|
1016
|
+
method: request.method,
|
|
1017
|
+
body: request.body,
|
|
1003
1018
|
headers: withAuthHeaders(this.options.getAuthToken, {
|
|
1004
1019
|
'Content-Type': 'application/json',
|
|
1005
1020
|
'Cache-Control': 'no-cache, no-store, must-revalidate',
|
|
@@ -110,6 +110,7 @@ export class SyncWebSocket<
|
|
|
110
110
|
* the state itself lives in {@link SyncCursor}.
|
|
111
111
|
*/
|
|
112
112
|
private readonly cursor: SyncCursor;
|
|
113
|
+
private catchUp: { exchangeId: string; currentSyncId: number; nextSequence: number } | null = null;
|
|
113
114
|
|
|
114
115
|
constructor(options: SyncWebSocketOptions) {
|
|
115
116
|
super({
|
|
@@ -318,6 +319,11 @@ export class SyncWebSocket<
|
|
|
318
319
|
this.sendAck(syncId);
|
|
319
320
|
}
|
|
320
321
|
|
|
322
|
+
/** Publish completion only after the store's chunk-persistence lane drains. */
|
|
323
|
+
completeCatchUp(payload: { exchangeId: string; currentSyncId: number; chunks: number }): void {
|
|
324
|
+
this.emit('catchup_complete', payload);
|
|
325
|
+
}
|
|
326
|
+
|
|
321
327
|
/**
|
|
322
328
|
* Stop the periodic catchup interval
|
|
323
329
|
*/
|
|
@@ -548,6 +554,39 @@ export class SyncWebSocket<
|
|
|
548
554
|
}
|
|
549
555
|
}
|
|
550
556
|
|
|
557
|
+
protected override handleCatchUpBegin(rawPayload: unknown): void {
|
|
558
|
+
if (!isRecord(rawPayload)) return;
|
|
559
|
+
const { exchangeId, fromSyncId, currentSyncId } = rawPayload;
|
|
560
|
+
if (typeof exchangeId !== 'string' || !Number.isSafeInteger(fromSyncId) || !Number.isSafeInteger(currentSyncId)) return;
|
|
561
|
+
this.catchUp = { exchangeId, currentSyncId: currentSyncId as number, nextSequence: 0 };
|
|
562
|
+
this.emit('catchup_begin', { exchangeId, fromSyncId: fromSyncId as number, currentSyncId: currentSyncId as number });
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
protected override handleCatchUpChunk(rawPayload: unknown): void {
|
|
566
|
+
if (!isRecord(rawPayload) || !this.catchUp) return;
|
|
567
|
+
const { exchangeId, sequence, position, deltas } = rawPayload;
|
|
568
|
+
if (exchangeId !== this.catchUp.exchangeId || sequence !== this.catchUp.nextSequence
|
|
569
|
+
|| !Number.isSafeInteger(position) || (position as number) > this.catchUp.currentSyncId
|
|
570
|
+
|| !Array.isArray(deltas)) return;
|
|
571
|
+
const normalized: SyncDelta[] = [];
|
|
572
|
+
for (const raw of deltas) {
|
|
573
|
+
const delta = this.normalizeWireDelta(raw);
|
|
574
|
+
if (delta) normalized.push(delta);
|
|
575
|
+
}
|
|
576
|
+
if (normalized.length !== deltas.length) return;
|
|
577
|
+
this.catchUp.nextSequence++;
|
|
578
|
+
this.emit('catchup_chunk', { exchangeId, sequence, position, deltas: normalized });
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
protected override handleCatchUpEnd(rawPayload: unknown): void {
|
|
582
|
+
if (!isRecord(rawPayload) || !this.catchUp) return;
|
|
583
|
+
const { exchangeId, currentSyncId, chunks } = rawPayload;
|
|
584
|
+
if (exchangeId !== this.catchUp.exchangeId || currentSyncId !== this.catchUp.currentSyncId
|
|
585
|
+
|| chunks !== this.catchUp.nextSequence) return;
|
|
586
|
+
this.catchUp = null;
|
|
587
|
+
this.emit('catchup_end', { exchangeId, currentSyncId, chunks });
|
|
588
|
+
}
|
|
589
|
+
|
|
551
590
|
/**
|
|
552
591
|
* Handle bootstrap response from server
|
|
553
592
|
*/
|
|
@@ -282,8 +282,13 @@ export function scheduleDeltaFlush(ctx: DeltaPipelineContext): void {
|
|
|
282
282
|
}
|
|
283
283
|
}
|
|
284
284
|
|
|
285
|
-
/** Apply an authoritative
|
|
285
|
+
/** Apply an authoritative legacy frame without exposing its persistence promise. */
|
|
286
286
|
export function applyDeltaFrame(ctx: DeltaPipelineContext, deltas: SyncDelta[]): void {
|
|
287
|
+
void persistDeltaFrame(ctx, deltas).catch(ctx.handleFlushError);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** Persist one resumable chunk before its raw tail position may advance. */
|
|
291
|
+
export async function persistDeltaFrame(ctx: DeltaPipelineContext, deltas: SyncDelta[]): Promise<void> {
|
|
287
292
|
let enqueuedAny = false;
|
|
288
293
|
for (const delta of deltas) {
|
|
289
294
|
if (enqueueDelta(ctx, delta, { authoritative: true })) enqueuedAny = true;
|
|
@@ -293,7 +298,7 @@ export function applyDeltaFrame(ctx: DeltaPipelineContext, deltas: SyncDelta[]):
|
|
|
293
298
|
clearTimeout(ctx.batchTimer);
|
|
294
299
|
ctx.batchTimer = null;
|
|
295
300
|
}
|
|
296
|
-
|
|
301
|
+
await ctx.flushPendingDeltas();
|
|
297
302
|
}
|
|
298
303
|
|
|
299
304
|
/**
|
|
@@ -27,7 +27,7 @@ export interface SocketEventHost<TCollaboration extends EventMap<TCollaboration>
|
|
|
27
27
|
onConnectionEvent?: (event: string) => void;
|
|
28
28
|
updateSyncStatus(updates: Partial<SyncStatus>): void;
|
|
29
29
|
processDeltaWithBatching(delta: SyncDelta): void;
|
|
30
|
-
applyDeltaFrame(deltas: SyncDelta[]): void
|
|
30
|
+
applyDeltaFrame(deltas: SyncDelta[]): Promise<void>;
|
|
31
31
|
handleBootstrapRequired(hint: BootstrapHint): void;
|
|
32
32
|
handleBootstrapData(data: BootstrapDataEvent): void;
|
|
33
33
|
performCredentialRefresh(): Promise<'refreshed' | 'session_error' | 'network_error'>;
|
|
@@ -77,7 +77,28 @@ export function wireSocketEvents<TCollaboration extends EventMap<TCollaboration>
|
|
|
77
77
|
// A catch-up/reconnect frame is already complete — apply it as ONE
|
|
78
78
|
// atomic flush so the gallery re-renders once, not once per 50-delta
|
|
79
79
|
// chunk. See `applyDeltaFrame`.
|
|
80
|
-
deps.applyDeltaFrame(deltas)
|
|
80
|
+
void deps.applyDeltaFrame(deltas).catch((error: unknown) => {
|
|
81
|
+
deps.updateSyncStatus({ state: 'error', error: error instanceof Error ? error : new Error(String(error)) });
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
let catchUpLane = Promise.resolve();
|
|
86
|
+
const onCatchUpChunk = deps.syncWebSocket.subscribe('catchup_chunk', (chunk) => {
|
|
87
|
+
catchUpLane = catchUpLane.then(async () => {
|
|
88
|
+
await deps.applyDeltaFrame(chunk.deltas);
|
|
89
|
+
// The raw position covers filtered rows too. Advance only after this
|
|
90
|
+
// chunk's visible rows are durable; replay is therefore idempotent.
|
|
91
|
+
deps.syncWebSocket.acknowledge(chunk.position);
|
|
92
|
+
});
|
|
93
|
+
void catchUpLane.catch((error: unknown) => {
|
|
94
|
+
deps.updateSyncStatus({ state: 'error', error: error instanceof Error ? error : new Error(String(error)) });
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
const onCatchUpEnd = deps.syncWebSocket.subscribe('catchup_end', (end) => {
|
|
98
|
+
void catchUpLane.then(() => {
|
|
99
|
+
deps.syncWebSocket.acknowledge(end.currentSyncId);
|
|
100
|
+
deps.syncWebSocket.completeCatchUp(end);
|
|
101
|
+
}).catch(() => undefined);
|
|
81
102
|
});
|
|
82
103
|
|
|
83
104
|
// Bootstrap events
|
|
@@ -181,7 +202,7 @@ export function wireSocketEvents<TCollaboration extends EventMap<TCollaboration>
|
|
|
181
202
|
|
|
182
203
|
deps.disposers.push(
|
|
183
204
|
onConnected, onDisconnected, onReconnecting,
|
|
184
|
-
onDelta, onDeltaBatch, onBootstrapRequired,
|
|
205
|
+
onDelta, onDeltaBatch, onCatchUpChunk, onCatchUpEnd, onBootstrapRequired,
|
|
185
206
|
onBootstrapData,
|
|
186
207
|
onError, onSessionError, onHandshakeFailed, onReconnectFailed,
|
|
187
208
|
() => { deps.areaOfInterest.dispose(); },
|
|
@@ -144,6 +144,9 @@ export async function processBatch(ctx: BatchProcessingContext): Promise<void> {
|
|
|
144
144
|
throw new Error('Cannot replay a model batch with inconsistent durable envelopes');
|
|
145
145
|
}
|
|
146
146
|
} else {
|
|
147
|
+
for (const tx of batch) {
|
|
148
|
+
ctx.emitCommitLifecycle('model:sealing', { clientTxId: tx.id });
|
|
149
|
+
}
|
|
147
150
|
durableEnvelope = await ctx.sealDurableCommit({
|
|
148
151
|
idempotencyKey: commitIdempotencyKey,
|
|
149
152
|
origin: 'model_batch',
|
|
@@ -156,6 +159,7 @@ export async function processBatch(ctx: BatchProcessingContext): Promise<void> {
|
|
|
156
159
|
});
|
|
157
160
|
for (const transaction of batch) {
|
|
158
161
|
transaction.durableEnvelope = durableEnvelope;
|
|
162
|
+
ctx.emitCommitLifecycle('model:sealed', { clientTxId: transaction.id });
|
|
159
163
|
}
|
|
160
164
|
}
|
|
161
165
|
const operations = durableEnvelope.operations;
|
|
@@ -36,7 +36,7 @@ export interface CommitEventSource {
|
|
|
36
36
|
|
|
37
37
|
/** One completed commit, broken into its local and remote halves. */
|
|
38
38
|
export interface CommitLatencySample {
|
|
39
|
-
/**
|
|
39
|
+
/** Local transaction id; for atomic commits this is also the wire clientTxId. */
|
|
40
40
|
clientTxId: string;
|
|
41
41
|
/** Milliseconds sealing the durable envelope locally. */
|
|
42
42
|
sealMs: number;
|
|
@@ -98,6 +98,8 @@ export function observeCommitLatency(
|
|
|
98
98
|
const handleStaging = (payload: unknown): void => {
|
|
99
99
|
const id = readClientTxId(payload);
|
|
100
100
|
if (id === null) return;
|
|
101
|
+
// Retain the first seal attempt across retries rather than hiding its wait.
|
|
102
|
+
if (pending.has(id)) return;
|
|
101
103
|
// Map preserves insertion order, so the first key is the stalest entry.
|
|
102
104
|
if (pending.size >= MAX_PENDING_COMMITS) {
|
|
103
105
|
const oldest = pending.keys().next();
|
|
@@ -111,7 +113,7 @@ export function observeCommitLatency(
|
|
|
111
113
|
if (id === null) return;
|
|
112
114
|
const timing = pending.get(id);
|
|
113
115
|
if (timing === undefined) return;
|
|
114
|
-
timing.sealedAt
|
|
116
|
+
timing.sealedAt ??= nowMs();
|
|
115
117
|
};
|
|
116
118
|
|
|
117
119
|
const handleCompleted = (payload: unknown): void => {
|
|
@@ -149,6 +151,10 @@ export function observeCommitLatency(
|
|
|
149
151
|
|
|
150
152
|
source.on('commit:staging', handleStaging);
|
|
151
153
|
source.on('commit:created', handleCreated);
|
|
154
|
+
// Model batches share an envelope, but confirmation belongs to each local
|
|
155
|
+
// transaction. Restored envelopes have no observed seal and emit no sample.
|
|
156
|
+
source.on('model:sealing', handleStaging);
|
|
157
|
+
source.on('model:sealed', handleCreated);
|
|
152
158
|
source.on('commit:seal_failed', handleSealFailed);
|
|
153
159
|
source.on('transaction:completed', handleCompleted);
|
|
154
160
|
source.on('transaction:failed', handleFailed);
|
|
@@ -156,6 +162,8 @@ export function observeCommitLatency(
|
|
|
156
162
|
return () => {
|
|
157
163
|
source.off('commit:staging', handleStaging);
|
|
158
164
|
source.off('commit:created', handleCreated);
|
|
165
|
+
source.off('model:sealing', handleStaging);
|
|
166
|
+
source.off('model:sealed', handleCreated);
|
|
159
167
|
source.off('commit:seal_failed', handleSealFailed);
|
|
160
168
|
source.off('transaction:completed', handleCompleted);
|
|
161
169
|
source.off('transaction:failed', handleFailed);
|