@terreno/syncdb 56.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +419 -0
- package/dist/auth/betterAuthAdapter.d.ts +22 -0
- package/dist/auth/betterAuthAdapter.d.ts.map +1 -0
- package/dist/auth/betterAuthAdapter.js +181 -0
- package/dist/auth/betterAuthAdapter.js.map +1 -0
- package/dist/auth/betterAuthAdapter.test.d.ts +2 -0
- package/dist/auth/betterAuthAdapter.test.d.ts.map +1 -0
- package/dist/auth/betterAuthAdapter.test.js +350 -0
- package/dist/auth/betterAuthAdapter.test.js.map +1 -0
- package/dist/auth/types.d.ts +42 -0
- package/dist/auth/types.d.ts.map +1 -0
- package/dist/auth/types.js +9 -0
- package/dist/auth/types.js.map +1 -0
- package/dist/client.d.ts +246 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +1415 -0
- package/dist/client.js.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/client.test.js +2310 -0
- package/dist/client.test.js.map +1 -0
- package/dist/crypto/aesGcmCodec.d.ts +33 -0
- package/dist/crypto/aesGcmCodec.d.ts.map +1 -0
- package/dist/crypto/aesGcmCodec.js +80 -0
- package/dist/crypto/aesGcmCodec.js.map +1 -0
- package/dist/crypto/aesGcmCodec.test.d.ts +2 -0
- package/dist/crypto/aesGcmCodec.test.d.ts.map +1 -0
- package/dist/crypto/aesGcmCodec.test.js +83 -0
- package/dist/crypto/aesGcmCodec.test.js.map +1 -0
- package/dist/crypto/identityCodec.d.ts +7 -0
- package/dist/crypto/identityCodec.d.ts.map +1 -0
- package/dist/crypto/identityCodec.js +9 -0
- package/dist/crypto/identityCodec.js.map +1 -0
- package/dist/crypto/keyProviders.d.ts +34 -0
- package/dist/crypto/keyProviders.d.ts.map +1 -0
- package/dist/crypto/keyProviders.js +102 -0
- package/dist/crypto/keyProviders.js.map +1 -0
- package/dist/crypto/keyProviders.test.d.ts +2 -0
- package/dist/crypto/keyProviders.test.d.ts.map +1 -0
- package/dist/crypto/keyProviders.test.js +189 -0
- package/dist/crypto/keyProviders.test.js.map +1 -0
- package/dist/crypto/types.d.ts +24 -0
- package/dist/crypto/types.d.ts.map +1 -0
- package/dist/crypto/types.js +8 -0
- package/dist/crypto/types.js.map +1 -0
- package/dist/debug/debugChannel.d.ts +52 -0
- package/dist/debug/debugChannel.d.ts.map +1 -0
- package/dist/debug/debugChannel.js +73 -0
- package/dist/debug/debugChannel.js.map +1 -0
- package/dist/debug/debugChannel.test.d.ts +2 -0
- package/dist/debug/debugChannel.test.d.ts.map +1 -0
- package/dist/debug/debugChannel.test.js +105 -0
- package/dist/debug/debugChannel.test.js.map +1 -0
- package/dist/debug/debugLog.d.ts +111 -0
- package/dist/debug/debugLog.d.ts.map +1 -0
- package/dist/debug/debugLog.js +150 -0
- package/dist/debug/debugLog.js.map +1 -0
- package/dist/debug/debugLog.test.d.ts +2 -0
- package/dist/debug/debugLog.test.d.ts.map +1 -0
- package/dist/debug/debugLog.test.js +121 -0
- package/dist/debug/debugLog.test.js.map +1 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +51 -0
- package/dist/index.js.map +1 -0
- package/dist/isolated/defaultPersisterFactoryNative.isolated.d.ts +2 -0
- package/dist/isolated/defaultPersisterFactoryNative.isolated.d.ts.map +1 -0
- package/dist/isolated/defaultPersisterFactoryNative.isolated.js +38 -0
- package/dist/isolated/defaultPersisterFactoryNative.isolated.js.map +1 -0
- package/dist/mutations/conflicts.d.ts +31 -0
- package/dist/mutations/conflicts.d.ts.map +1 -0
- package/dist/mutations/conflicts.js +73 -0
- package/dist/mutations/conflicts.js.map +1 -0
- package/dist/mutations/outbox.d.ts +174 -0
- package/dist/mutations/outbox.d.ts.map +1 -0
- package/dist/mutations/outbox.js +385 -0
- package/dist/mutations/outbox.js.map +1 -0
- package/dist/mutations/outbox.test.d.ts +2 -0
- package/dist/mutations/outbox.test.d.ts.map +1 -0
- package/dist/mutations/outbox.test.js +643 -0
- package/dist/mutations/outbox.test.js.map +1 -0
- package/dist/mutations/resolveConflict.d.ts +32 -0
- package/dist/mutations/resolveConflict.d.ts.map +1 -0
- package/dist/mutations/resolveConflict.js +101 -0
- package/dist/mutations/resolveConflict.js.map +1 -0
- package/dist/mutations/resolveConflict.test.d.ts +2 -0
- package/dist/mutations/resolveConflict.test.d.ts.map +1 -0
- package/dist/mutations/resolveConflict.test.js +282 -0
- package/dist/mutations/resolveConflict.test.js.map +1 -0
- package/dist/persisters/defaultPersisterFactory.d.ts +9 -0
- package/dist/persisters/defaultPersisterFactory.d.ts.map +1 -0
- package/dist/persisters/defaultPersisterFactory.js +9 -0
- package/dist/persisters/defaultPersisterFactory.js.map +1 -0
- package/dist/persisters/defaultPersisterFactory.native.d.ts +10 -0
- package/dist/persisters/defaultPersisterFactory.native.d.ts.map +1 -0
- package/dist/persisters/defaultPersisterFactory.native.js +44 -0
- package/dist/persisters/defaultPersisterFactory.native.js.map +1 -0
- package/dist/persisters/defaultPersisterFactory.test.d.ts +2 -0
- package/dist/persisters/defaultPersisterFactory.test.d.ts.map +1 -0
- package/dist/persisters/defaultPersisterFactory.test.js +177 -0
- package/dist/persisters/defaultPersisterFactory.test.js.map +1 -0
- package/dist/persisters/defaultPersisterFactory.web.d.ts +17 -0
- package/dist/persisters/defaultPersisterFactory.web.d.ts.map +1 -0
- package/dist/persisters/defaultPersisterFactory.web.js +65 -0
- package/dist/persisters/defaultPersisterFactory.web.js.map +1 -0
- package/dist/persisters/encryptedIndexedDbPersister.d.ts +46 -0
- package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -0
- package/dist/persisters/encryptedIndexedDbPersister.js +171 -0
- package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -0
- package/dist/persisters/encryptedIndexedDbPersister.test.d.ts +2 -0
- package/dist/persisters/encryptedIndexedDbPersister.test.d.ts.map +1 -0
- package/dist/persisters/encryptedIndexedDbPersister.test.js +352 -0
- package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -0
- package/dist/persisters/memoryPersister.d.ts +19 -0
- package/dist/persisters/memoryPersister.d.ts.map +1 -0
- package/dist/persisters/memoryPersister.js +31 -0
- package/dist/persisters/memoryPersister.js.map +1 -0
- package/dist/persisters/memoryPersister.test.d.ts +2 -0
- package/dist/persisters/memoryPersister.test.d.ts.map +1 -0
- package/dist/persisters/memoryPersister.test.js +113 -0
- package/dist/persisters/memoryPersister.test.js.map +1 -0
- package/dist/persisters/types.d.ts +65 -0
- package/dist/persisters/types.d.ts.map +1 -0
- package/dist/persisters/types.js +2 -0
- package/dist/persisters/types.js.map +1 -0
- package/dist/react/hooks.d.ts +132 -0
- package/dist/react/hooks.d.ts.map +1 -0
- package/dist/react/hooks.js +269 -0
- package/dist/react/hooks.js.map +1 -0
- package/dist/react/hooks.test.d.ts +2 -0
- package/dist/react/hooks.test.d.ts.map +1 -0
- package/dist/react/hooks.test.js +645 -0
- package/dist/react/hooks.test.js.map +1 -0
- package/dist/react/index.d.ts +11 -0
- package/dist/react/index.d.ts.map +1 -0
- package/dist/react/index.js +11 -0
- package/dist/react/index.js.map +1 -0
- package/dist/react/provider.d.ts +11 -0
- package/dist/react/provider.d.ts.map +1 -0
- package/dist/react/provider.js +17 -0
- package/dist/react/provider.js.map +1 -0
- package/dist/storage/idb.d.ts +27 -0
- package/dist/storage/idb.d.ts.map +1 -0
- package/dist/storage/idb.js +72 -0
- package/dist/storage/idb.js.map +1 -0
- package/dist/storage/idb.test.d.ts +2 -0
- package/dist/storage/idb.test.d.ts.map +1 -0
- package/dist/storage/idb.test.js +100 -0
- package/dist/storage/idb.test.js.map +1 -0
- package/dist/storage/purgeUnknownStream.test.d.ts +2 -0
- package/dist/storage/purgeUnknownStream.test.d.ts.map +1 -0
- package/dist/storage/purgeUnknownStream.test.js +64 -0
- package/dist/storage/purgeUnknownStream.test.js.map +1 -0
- package/dist/storage/schema.d.ts +33 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +92 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/storage/store.d.ts +136 -0
- package/dist/storage/store.d.ts.map +1 -0
- package/dist/storage/store.js +275 -0
- package/dist/storage/store.js.map +1 -0
- package/dist/storage/store.test.d.ts +2 -0
- package/dist/storage/store.test.d.ts.map +1 -0
- package/dist/storage/store.test.js +357 -0
- package/dist/storage/store.test.js.map +1 -0
- package/dist/storage/types.d.ts +146 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +32 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/storage/wipe.d.ts +22 -0
- package/dist/storage/wipe.d.ts.map +1 -0
- package/dist/storage/wipe.js +43 -0
- package/dist/storage/wipe.js.map +1 -0
- package/dist/storage/wipe.test.d.ts +2 -0
- package/dist/storage/wipe.test.d.ts.map +1 -0
- package/dist/storage/wipe.test.js +95 -0
- package/dist/storage/wipe.test.js.map +1 -0
- package/dist/sync/bootstrap.d.ts +64 -0
- package/dist/sync/bootstrap.d.ts.map +1 -0
- package/dist/sync/bootstrap.js +160 -0
- package/dist/sync/bootstrap.js.map +1 -0
- package/dist/sync/bootstrap.test.d.ts +2 -0
- package/dist/sync/bootstrap.test.d.ts.map +1 -0
- package/dist/sync/bootstrap.test.js +447 -0
- package/dist/sync/bootstrap.test.js.map +1 -0
- package/dist/sync/cursor.d.ts +67 -0
- package/dist/sync/cursor.d.ts.map +1 -0
- package/dist/sync/cursor.js +74 -0
- package/dist/sync/cursor.js.map +1 -0
- package/dist/sync/cursor.test.d.ts +2 -0
- package/dist/sync/cursor.test.d.ts.map +1 -0
- package/dist/sync/cursor.test.js +73 -0
- package/dist/sync/cursor.test.js.map +1 -0
- package/dist/sync/deltaApplier.d.ts +30 -0
- package/dist/sync/deltaApplier.d.ts.map +1 -0
- package/dist/sync/deltaApplier.js +61 -0
- package/dist/sync/deltaApplier.js.map +1 -0
- package/dist/sync/deltaApplier.test.d.ts +2 -0
- package/dist/sync/deltaApplier.test.d.ts.map +1 -0
- package/dist/sync/deltaApplier.test.js +212 -0
- package/dist/sync/deltaApplier.test.js.map +1 -0
- package/dist/sync/entityRepair.d.ts +45 -0
- package/dist/sync/entityRepair.d.ts.map +1 -0
- package/dist/sync/entityRepair.js +91 -0
- package/dist/sync/entityRepair.js.map +1 -0
- package/dist/sync/entityRepair.test.d.ts +2 -0
- package/dist/sync/entityRepair.test.d.ts.map +1 -0
- package/dist/sync/entityRepair.test.js +146 -0
- package/dist/sync/entityRepair.test.js.map +1 -0
- package/dist/sync/httpChannel.d.ts +74 -0
- package/dist/sync/httpChannel.d.ts.map +1 -0
- package/dist/sync/httpChannel.js +123 -0
- package/dist/sync/httpChannel.js.map +1 -0
- package/dist/sync/httpChannel.test.d.ts +2 -0
- package/dist/sync/httpChannel.test.d.ts.map +1 -0
- package/dist/sync/httpChannel.test.js +215 -0
- package/dist/sync/httpChannel.test.js.map +1 -0
- package/dist/sync/replayCoordinator.d.ts +212 -0
- package/dist/sync/replayCoordinator.d.ts.map +1 -0
- package/dist/sync/replayCoordinator.js +1110 -0
- package/dist/sync/replayCoordinator.js.map +1 -0
- package/dist/sync/replayCoordinator.test.d.ts +2 -0
- package/dist/sync/replayCoordinator.test.d.ts.map +1 -0
- package/dist/sync/replayCoordinator.test.js +1496 -0
- package/dist/sync/replayCoordinator.test.js.map +1 -0
- package/dist/sync/socketTransport.d.ts +56 -0
- package/dist/sync/socketTransport.d.ts.map +1 -0
- package/dist/sync/socketTransport.js +292 -0
- package/dist/sync/socketTransport.js.map +1 -0
- package/dist/sync/socketTransport.test.d.ts +2 -0
- package/dist/sync/socketTransport.test.d.ts.map +1 -0
- package/dist/sync/socketTransport.test.js +346 -0
- package/dist/sync/socketTransport.test.js.map +1 -0
- package/dist/sync/transport.d.ts +80 -0
- package/dist/sync/transport.d.ts.map +1 -0
- package/dist/sync/transport.js +5 -0
- package/dist/sync/transport.js.map +1 -0
- package/dist/testing/fakeTransport.d.ts +55 -0
- package/dist/testing/fakeTransport.d.ts.map +1 -0
- package/dist/testing/fakeTransport.js +139 -0
- package/dist/testing/fakeTransport.js.map +1 -0
- package/dist/testing/fakeTransport.test.d.ts +2 -0
- package/dist/testing/fakeTransport.test.d.ts.map +1 -0
- package/dist/testing/fakeTransport.test.js +112 -0
- package/dist/testing/fakeTransport.test.js.map +1 -0
- package/dist/testing/index.d.ts +13 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +13 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/types.d.ts +271 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/dist/types.js.map +1 -0
- package/package.json +71 -0
- package/src/auth/betterAuthAdapter.test.ts +455 -0
- package/src/auth/betterAuthAdapter.ts +214 -0
- package/src/auth/types.ts +49 -0
- package/src/client.test.ts +2670 -0
- package/src/client.ts +1748 -0
- package/src/crypto/aesGcmCodec.test.ts +103 -0
- package/src/crypto/aesGcmCodec.ts +98 -0
- package/src/crypto/identityCodec.ts +10 -0
- package/src/crypto/keyProviders.test.ts +228 -0
- package/src/crypto/keyProviders.ts +153 -0
- package/src/crypto/types.ts +23 -0
- package/src/debug/debugChannel.test.ts +121 -0
- package/src/debug/debugChannel.ts +112 -0
- package/src/debug/debugLog.test.ts +139 -0
- package/src/debug/debugLog.ts +272 -0
- package/src/index.ts +157 -0
- package/src/isolated/defaultPersisterFactoryNative.isolated.ts +45 -0
- package/src/mutations/conflicts.ts +108 -0
- package/src/mutations/outbox.test.ts +754 -0
- package/src/mutations/outbox.ts +585 -0
- package/src/mutations/resolveConflict.test.ts +338 -0
- package/src/mutations/resolveConflict.ts +115 -0
- package/src/persisters/defaultPersisterFactory.native.ts +53 -0
- package/src/persisters/defaultPersisterFactory.test.ts +207 -0
- package/src/persisters/defaultPersisterFactory.ts +12 -0
- package/src/persisters/defaultPersisterFactory.web.ts +78 -0
- package/src/persisters/encryptedIndexedDbPersister.test.ts +381 -0
- package/src/persisters/encryptedIndexedDbPersister.ts +218 -0
- package/src/persisters/memoryPersister.test.ts +136 -0
- package/src/persisters/memoryPersister.ts +52 -0
- package/src/persisters/types.ts +71 -0
- package/src/react/hooks.test.tsx +761 -0
- package/src/react/hooks.ts +409 -0
- package/src/react/index.ts +11 -0
- package/src/react/provider.tsx +27 -0
- package/src/storage/idb.test.ts +116 -0
- package/src/storage/idb.ts +112 -0
- package/src/storage/purgeUnknownStream.test.ts +74 -0
- package/src/storage/schema.ts +103 -0
- package/src/storage/store.test.ts +424 -0
- package/src/storage/store.ts +461 -0
- package/src/storage/types.ts +159 -0
- package/src/storage/wipe.test.ts +111 -0
- package/src/storage/wipe.ts +58 -0
- package/src/sync/bootstrap.test.ts +486 -0
- package/src/sync/bootstrap.ts +217 -0
- package/src/sync/cursor.test.ts +84 -0
- package/src/sync/cursor.ts +113 -0
- package/src/sync/deltaApplier.test.ts +237 -0
- package/src/sync/deltaApplier.ts +84 -0
- package/src/sync/entityRepair.test.ts +161 -0
- package/src/sync/entityRepair.ts +124 -0
- package/src/sync/httpChannel.test.ts +265 -0
- package/src/sync/httpChannel.ts +220 -0
- package/src/sync/replayCoordinator.test.ts +1709 -0
- package/src/sync/replayCoordinator.ts +1370 -0
- package/src/sync/socketTransport.test.ts +413 -0
- package/src/sync/socketTransport.ts +379 -0
- package/src/sync/transport.ts +76 -0
- package/src/testing/fakeTransport.test.ts +125 -0
- package/src/testing/fakeTransport.ts +206 -0
- package/src/testing/index.ts +18 -0
- package/src/types.ts +284 -0
|
@@ -0,0 +1,1370 @@
|
|
|
1
|
+
import type {SyncDebugLog} from "../debug/debugLog";
|
|
2
|
+
import {listConflicts, writeConflict} from "../mutations/conflicts";
|
|
3
|
+
import type {Outbox} from "../mutations/outbox";
|
|
4
|
+
import type {SyncStore} from "../storage/store";
|
|
5
|
+
import type {
|
|
6
|
+
OutboxMutation,
|
|
7
|
+
SyncAck,
|
|
8
|
+
SyncMutateBatchRequest,
|
|
9
|
+
SyncMutateRequest,
|
|
10
|
+
SyncNack,
|
|
11
|
+
} from "../types";
|
|
12
|
+
import {AuthRequiredError} from "./httpChannel";
|
|
13
|
+
import {
|
|
14
|
+
DEFAULT_BATCH_SIZE,
|
|
15
|
+
type SendMutationBatchResult,
|
|
16
|
+
type SendMutationResult,
|
|
17
|
+
} from "./transport";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Parse a mutation's JSON-serialized `args` back into the create/update body for
|
|
21
|
+
* display in the debug event stream. Falls back to the raw string if it is not
|
|
22
|
+
* valid JSON. Only ever evaluated when the debug log is enabled — optional
|
|
23
|
+
* chaining (`debug?.record({...})`) short-circuits the argument object entirely
|
|
24
|
+
* when `debug` is undefined, so this stays free in production.
|
|
25
|
+
*/
|
|
26
|
+
const debugBody = (mutation: OutboxMutation): Record<string, unknown> | string => {
|
|
27
|
+
try {
|
|
28
|
+
return JSON.parse(mutation.args) as Record<string, unknown>;
|
|
29
|
+
} catch {
|
|
30
|
+
return mutation.args;
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/** Error-nack retries beyond this attempt count become terminal failures. */
|
|
35
|
+
export const MAX_ERROR_NACK_ATTEMPTS = 5;
|
|
36
|
+
|
|
37
|
+
/** Base delay for the error-nack exponential backoff (doubles per attempt). */
|
|
38
|
+
export const ERROR_NACK_BASE_BACKOFF_MS = 1_000;
|
|
39
|
+
|
|
40
|
+
/** Base delay for transport-failure backoff (unlimited retries, same cap/jitter shape). */
|
|
41
|
+
export const TRANSPORT_FAILURE_BASE_BACKOFF_MS = 1_000;
|
|
42
|
+
|
|
43
|
+
/** Cap applied to every jittered backoff (error-nack and transport-failure alike). */
|
|
44
|
+
export const MAX_BACKOFF_MS = 30_000;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* FIX 5: consecutive `unsupported` batch results required before the
|
|
48
|
+
* per-connection `batchUnsupported` latch engages — a single slow-but-
|
|
49
|
+
* supported batch (grace elapsed with no receipt is impossible once a
|
|
50
|
+
* receipt lands, but a genuinely borderline network hiccup could still
|
|
51
|
+
* produce one stray `unsupported`) must never permanently downgrade the
|
|
52
|
+
* session to single-sends.
|
|
53
|
+
*/
|
|
54
|
+
export const BATCH_UNSUPPORTED_LATCH_THRESHOLD = 2;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* FIX 5: while latched `batchUnsupported`, re-probe batch support on this
|
|
58
|
+
* interval even without a reconnect — a load balancer could route later
|
|
59
|
+
* sends to an upgraded instance, or the original instance could itself be
|
|
60
|
+
* upgraded, without the connection ever dropping.
|
|
61
|
+
*/
|
|
62
|
+
export const BATCH_UNSUPPORTED_REPROBE_INTERVAL_MS = 60_000;
|
|
63
|
+
|
|
64
|
+
export interface ReplayResult {
|
|
65
|
+
/** Set when replay stopped early because the server rejected our auth. */
|
|
66
|
+
paused?: "auth";
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface ReplayCoordinator {
|
|
70
|
+
/**
|
|
71
|
+
* Drain the user's queued mutations in one global FIFO pass ordered by
|
|
72
|
+
* `enqueueOrder` (INV-1) — never per-collection parallel. Resolves once the
|
|
73
|
+
* queue is fully empty or parked (auth pause, or every remaining mutation is
|
|
74
|
+
* backing off). A second call for the same user while one is running
|
|
75
|
+
* returns the in-flight promise.
|
|
76
|
+
*/
|
|
77
|
+
replay: (args: {userId: string}) => Promise<ReplayResult>;
|
|
78
|
+
/**
|
|
79
|
+
* Clear the single armed wake-up timer (if any) for a user. Called from
|
|
80
|
+
* `client.stop()` so no post-stop send can fire.
|
|
81
|
+
*/
|
|
82
|
+
dispose: (args?: {userId?: string}) => void;
|
|
83
|
+
/**
|
|
84
|
+
* FIX 3: clear ALL in-memory coordinator state — every armed wake-up timer
|
|
85
|
+
* (all users), retry/backoff bookkeeping (`retryAt`, `transportFailures`),
|
|
86
|
+
* `authPauseNotified`, the batch-capability latch (`batchUnsupported`), drain
|
|
87
|
+
* progress counters, and `validationBlockedEntities` — so a lifecycle
|
|
88
|
+
* boundary (client `dispose()`, or a different-user login's wipe path)
|
|
89
|
+
* never leaks state across users or across a stop/start cycle. Unlike
|
|
90
|
+
* `dispose()` (which only clears one user's timer), this is a full reset:
|
|
91
|
+
* call it from `dispose()` itself AND from the different-user wipe path.
|
|
92
|
+
*/
|
|
93
|
+
reset: () => void;
|
|
94
|
+
/**
|
|
95
|
+
* Re-probe batch support on the next drain (B3): called on transport
|
|
96
|
+
* reconnect, since a previous `batchUnsupported` determination may have been
|
|
97
|
+
* against an old server that has since been upgraded (or the reconnect landed
|
|
98
|
+
* on a different, batch-capable instance behind a load balancer).
|
|
99
|
+
*/
|
|
100
|
+
notifyReconnect: () => void;
|
|
101
|
+
/**
|
|
102
|
+
* Re-enable an entity's blocked queued successors (B4) after a terminal
|
|
103
|
+
* validation failure: subsequent drains stop skipping mutations for this
|
|
104
|
+
* entity. Does not touch unresolved conflicts — those clear via
|
|
105
|
+
* `resolveConflict`.
|
|
106
|
+
*/
|
|
107
|
+
retryFailed: (args: {entityId: string}) => void;
|
|
108
|
+
/** Entity ids currently blocked by an unresolved conflict or a skipped validation failure (B4). */
|
|
109
|
+
getBlockedEntities: (args: {userId: string}) => string[];
|
|
110
|
+
/**
|
|
111
|
+
* Current drain progress snapshot for a user (B5): `draining` is true only
|
|
112
|
+
* while a `replay()` call for this user is in flight. `sentThisDrain` /
|
|
113
|
+
* `totalThisDrain` reflect the most recent (or in-progress) drain.
|
|
114
|
+
*/
|
|
115
|
+
getDrainProgress: (args: {userId: string}) => {
|
|
116
|
+
draining: boolean;
|
|
117
|
+
sentThisDrain: number;
|
|
118
|
+
totalThisDrain: number;
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export interface CreateReplayCoordinatorArgs {
|
|
123
|
+
store: SyncStore;
|
|
124
|
+
outbox: Outbox;
|
|
125
|
+
sendMutation: (request: SyncMutateRequest) => Promise<SendMutationResult>;
|
|
126
|
+
/**
|
|
127
|
+
* Batch send function (B3). When present, the coordinator builds contiguous
|
|
128
|
+
* chunks of the global FIFO queue (≤ `batchSize`) and sends them here instead
|
|
129
|
+
* of one `sendMutation` per mutation, falling back to single sends when the
|
|
130
|
+
* batch transport reports `unsupported` or is absent.
|
|
131
|
+
*/
|
|
132
|
+
sendMutationBatch?: (request: SyncMutateBatchRequest) => Promise<SendMutationBatchResult>;
|
|
133
|
+
/** Max mutations per batch chunk (default {@link DEFAULT_BATCH_SIZE}). */
|
|
134
|
+
batchSize?: number;
|
|
135
|
+
/**
|
|
136
|
+
* When true (B4), a conflict halts the ENTIRE drain instead of just blocking
|
|
137
|
+
* its entity. Default false (per-entity blocking).
|
|
138
|
+
*/
|
|
139
|
+
haltQueueOnConflict?: boolean;
|
|
140
|
+
/** Millisecond clock, injectable for deterministic backoff tests. */
|
|
141
|
+
now?: () => number;
|
|
142
|
+
/** Random source in [0, 1), injectable for deterministic jitter tests. */
|
|
143
|
+
random?: () => number;
|
|
144
|
+
/** Optional debug log; when present, each mutation outcome is recorded. */
|
|
145
|
+
debug?: SyncDebugLog;
|
|
146
|
+
/**
|
|
147
|
+
* Called whenever a drain parks on an armed wake-up timer, so the host can
|
|
148
|
+
* schedule the callback (defaults to the global `setTimeout`/`clearTimeout`).
|
|
149
|
+
*/
|
|
150
|
+
setTimeoutFn?: (handler: () => void, ms: number) => unknown;
|
|
151
|
+
clearTimeoutFn?: (handle: unknown) => void;
|
|
152
|
+
/**
|
|
153
|
+
* Called after a drain pass that made forward progress (at least one ack)
|
|
154
|
+
* so the caller can prune acked rows (A5). Optional — tests that don't care
|
|
155
|
+
* about pruning may omit it.
|
|
156
|
+
*/
|
|
157
|
+
onDrainPass?: (args: {userId: string}) => void;
|
|
158
|
+
/**
|
|
159
|
+
* Called the first time a drain pass for a user enters the auth-paused
|
|
160
|
+
* state (A4). Not called again until the pause clears and re-triggers.
|
|
161
|
+
*/
|
|
162
|
+
onAuthPause?: (args: {userId: string}) => void;
|
|
163
|
+
/**
|
|
164
|
+
* B5: called after each mutation/chunk send attempt with drain progress —
|
|
165
|
+
* `sentThisDrain` (attempts made so far in this `replay()` call) and
|
|
166
|
+
* `totalThisDrain` (queue length observed when this `replay()` call began,
|
|
167
|
+
* a lower bound since more may enqueue mid-drain). Lets the host surface a
|
|
168
|
+
* progress indicator (e.g. "12 / 40 synced").
|
|
169
|
+
*/
|
|
170
|
+
onProgress?: (args: {userId: string; sentThisDrain: number; totalThisDrain: number}) => void;
|
|
171
|
+
/**
|
|
172
|
+
* Consulted before every drain attempt — including internally-scheduled
|
|
173
|
+
* ones (the armed wake-up timer and the mid-drain-enqueue recheck) that
|
|
174
|
+
* bypass the caller's own call site. Lets the host gate sending on state the
|
|
175
|
+
* coordinator does not itself track (simulated offline, auth pause at the
|
|
176
|
+
* client level). Defaults to always-allow.
|
|
177
|
+
*/
|
|
178
|
+
shouldDrain?: () => boolean;
|
|
179
|
+
/**
|
|
180
|
+
* Called when a mutation is terminal and the entity's `pendingMutationId` was
|
|
181
|
+
* released without a server ack seq — skipped server deltas may need repair.
|
|
182
|
+
*/
|
|
183
|
+
onEntityReleasedWithoutServerAck?: (mutation: OutboxMutation) => void;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const entityKey = (collection: string, entityId: string): string => `${collection}:${entityId}`;
|
|
187
|
+
|
|
188
|
+
/** FIX 3: userId-scoped key for `validationBlockedEntities` — see its declaration comment. */
|
|
189
|
+
const userEntityKey = (userId: string, collection: string, entityId: string): string =>
|
|
190
|
+
`${userId}|${entityKey(collection, entityId)}`;
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Bridges the durable outbox and the send channel, resolving each mutation's
|
|
194
|
+
* server outcome:
|
|
195
|
+
*
|
|
196
|
+
* - **ack** → markAcked, clear the entity's `pendingMutationId` (when it still
|
|
197
|
+
* belongs to this mutation) and stamp the acked seq;
|
|
198
|
+
* - **nack conflict** → markConflicted + record a `_conflicts` row; the
|
|
199
|
+
* entity's optimistic state stays in place until the user resolves. The
|
|
200
|
+
* ENTITY is blocked (B4): later drains skip its other queued mutations
|
|
201
|
+
* until the conflict resolves, unless `haltQueueOnConflict` is set, in
|
|
202
|
+
* which case the whole drain halts instead;
|
|
203
|
+
* - **nack unauthorized** / thrown `AuthRequiredError` → back to queued and
|
|
204
|
+
* pause replay for the user (`{paused: "auth"}`), consuming NO retry budget
|
|
205
|
+
* (INV-2); the next replay call (after re-auth) retries;
|
|
206
|
+
* - **nack validation** → markFailed (terminal) and release the entity's
|
|
207
|
+
* `pendingMutationId` so future deltas are not blocked forever. The ENTITY
|
|
208
|
+
* is blocked (B4) — its queued successors are skipped until `retryFailed`;
|
|
209
|
+
* - **nack error** → back to queued with jittered exponential backoff against
|
|
210
|
+
* the dedicated `errorNackCount` budget, terminal markFailed after
|
|
211
|
+
* {@link MAX_ERROR_NACK_ATTEMPTS} attempts. HALTS THE WHOLE DRAIN (B4);
|
|
212
|
+
* - **send rejection** (timeout / network / disconnect) → back to queued with
|
|
213
|
+
* unlimited jittered exponential backoff (never terminal, never burns the
|
|
214
|
+
* error-nack budget) and the drain parks until the backoff elapses. HALTS
|
|
215
|
+
* THE WHOLE DRAIN (B4);
|
|
216
|
+
* - **nack rate_limited** → treated EXACTLY like a transport failure (FIX 1):
|
|
217
|
+
* back to queued with the same unlimited jittered backoff (respecting the
|
|
218
|
+
* server's `retryAfterMs` as a floor when present), NEVER touching
|
|
219
|
+
* `errorNackCount` or `attemptCount`'s terminality — a rate limit is never
|
|
220
|
+
* allowed to burn the durable-data retry budget. HALTS THE WHOLE DRAIN (B4).
|
|
221
|
+
*
|
|
222
|
+
* The drain is a SINGLE global FIFO over `enqueueOrder` (INV-1) — no more
|
|
223
|
+
* per-collection parallelism. `replay()` keeps re-running drain passes until
|
|
224
|
+
* the queue is empty or parked; when it parks on a backoff, a single
|
|
225
|
+
* `setTimeout` wakes the next drain automatically so callers never need to
|
|
226
|
+
* poll.
|
|
227
|
+
*
|
|
228
|
+
* B3 layers a batched send on top: each pass sends the next contiguous chunk
|
|
229
|
+
* of the FIFO queue (≤ `batchSize`, at most one mutation per entity per
|
|
230
|
+
* chunk) through `sendMutationBatch` when available, walking the results in
|
|
231
|
+
* order with the same per-outcome handlers above, then applying the B4
|
|
232
|
+
* stop-the-line policy to whatever the batch response implies. Chunks with no
|
|
233
|
+
* eligible mutations (everything blocked/backing off) fall through exactly
|
|
234
|
+
* like the single-send path.
|
|
235
|
+
*/
|
|
236
|
+
export const createReplayCoordinator = ({
|
|
237
|
+
store,
|
|
238
|
+
outbox,
|
|
239
|
+
sendMutation,
|
|
240
|
+
sendMutationBatch,
|
|
241
|
+
batchSize = DEFAULT_BATCH_SIZE,
|
|
242
|
+
haltQueueOnConflict = false,
|
|
243
|
+
now = () => Date.now(),
|
|
244
|
+
random = () => Math.random(),
|
|
245
|
+
debug,
|
|
246
|
+
setTimeoutFn = (handler, ms) => setTimeout(handler, ms),
|
|
247
|
+
clearTimeoutFn = (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
248
|
+
onDrainPass,
|
|
249
|
+
onAuthPause,
|
|
250
|
+
onProgress,
|
|
251
|
+
onEntityReleasedWithoutServerAck,
|
|
252
|
+
shouldDrain = () => true,
|
|
253
|
+
}: CreateReplayCoordinatorArgs): ReplayCoordinator => {
|
|
254
|
+
const inFlightReplays = new Map<string, Promise<ReplayResult>>();
|
|
255
|
+
/**
|
|
256
|
+
* Set when a `replay()` call coalesces into an already-running drain whose
|
|
257
|
+
* final pass may have already observed the queue as empty (a narrow race:
|
|
258
|
+
* the in-flight promise's cleanup runs on a later microtask than the
|
|
259
|
+
* enqueue that prompted this call). Consumed once the in-flight promise
|
|
260
|
+
* settles to schedule exactly one immediate follow-up drain, so a mutation
|
|
261
|
+
* enqueued mid-drain is never stranded until an unrelated external trigger.
|
|
262
|
+
*/
|
|
263
|
+
const recheckRequested = new Set<string>();
|
|
264
|
+
/** mutationId → earliest epoch-ms the next attempt may run (error-nack backoff). */
|
|
265
|
+
const retryAt = new Map<string, number>();
|
|
266
|
+
/** mutationId → count of transport-failure attempts (in-memory only; unlimited budget). */
|
|
267
|
+
const transportFailures = new Map<string, number>();
|
|
268
|
+
/** userId → armed wake-up timer handle (never more than one per user). */
|
|
269
|
+
const wakeTimers = new Map<string, unknown>();
|
|
270
|
+
/** userId → whether onAuthPause has already fired for the current pause episode. */
|
|
271
|
+
const authPauseNotified = new Set<string>();
|
|
272
|
+
/**
|
|
273
|
+
* B4: entities blocked by a terminal validation failure whose queued
|
|
274
|
+
* successors must be skipped until `retryFailed({entityId})`. Keyed by
|
|
275
|
+
* `${userId}|${collection}:${entityId}` (FIX 3 — userId-scoped so a stale
|
|
276
|
+
* entry from one user can never falsely block a different user reusing the
|
|
277
|
+
* same deterministic entity id, e.g. a singleton settings doc). Unresolved-
|
|
278
|
+
* conflict blocking is derived live from the `_conflicts` table instead (it
|
|
279
|
+
* already tracks resolution).
|
|
280
|
+
*/
|
|
281
|
+
const validationBlockedEntities = new Set<string>();
|
|
282
|
+
/** userId → {sent, total} progress counters for the CURRENT replay() call (B5). */
|
|
283
|
+
const drainProgress = new Map<string, {sent: number; total: number}>();
|
|
284
|
+
|
|
285
|
+
const reportProgress = (userId: string, sentDelta: number): void => {
|
|
286
|
+
if (!onProgress) {
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
const progress = drainProgress.get(userId);
|
|
290
|
+
if (!progress) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
progress.sent += sentDelta;
|
|
294
|
+
onProgress({sentThisDrain: progress.sent, totalThisDrain: progress.total, userId});
|
|
295
|
+
};
|
|
296
|
+
/**
|
|
297
|
+
* B3/FIX 5: per-connection flag set once {@link BATCH_UNSUPPORTED_LATCH_THRESHOLD}
|
|
298
|
+
* CONSECUTIVE batch sends report `unsupported` (HTTP 404, or socket grace
|
|
299
|
+
* elapsing with no `sync:batchReceived` receipt) — a single slow-but-
|
|
300
|
+
* supported batch must never trip this. Cleared by `notifyReconnect()` so
|
|
301
|
+
* a reconnect re-probes, and additionally re-probed every
|
|
302
|
+
* {@link BATCH_UNSUPPORTED_REPROBE_INTERVAL_MS} while latched, even
|
|
303
|
+
* without a reconnect (a load balancer or in-place upgrade can flip
|
|
304
|
+
* support without the connection ever dropping).
|
|
305
|
+
*/
|
|
306
|
+
let batchUnsupported = false;
|
|
307
|
+
/** FIX 5: consecutive `unsupported` results observed since the last successful/real batch response. */
|
|
308
|
+
let consecutiveUnsupported = 0;
|
|
309
|
+
/** FIX 5: armed only while `batchUnsupported` is latched; re-probes on a timer. */
|
|
310
|
+
let reprobeTimer: unknown;
|
|
311
|
+
|
|
312
|
+
const clearReprobeTimer = (): void => {
|
|
313
|
+
if (reprobeTimer !== undefined) {
|
|
314
|
+
clearTimeoutFn(reprobeTimer);
|
|
315
|
+
reprobeTimer = undefined;
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
const armReprobeTimer = (): void => {
|
|
320
|
+
clearReprobeTimer();
|
|
321
|
+
reprobeTimer = setTimeoutFn(() => {
|
|
322
|
+
reprobeTimer = undefined;
|
|
323
|
+
// Re-probe: clear the latch and let the next drain attempt a real
|
|
324
|
+
// batch send again. A fresh two-strikes count applies if it fails
|
|
325
|
+
// again, exactly like the original latch logic.
|
|
326
|
+
batchUnsupported = false;
|
|
327
|
+
consecutiveUnsupported = 0;
|
|
328
|
+
}, BATCH_UNSUPPORTED_REPROBE_INTERVAL_MS);
|
|
329
|
+
};
|
|
330
|
+
|
|
331
|
+
const clearWakeTimer = (userId: string): void => {
|
|
332
|
+
const handle = wakeTimers.get(userId);
|
|
333
|
+
if (handle !== undefined) {
|
|
334
|
+
clearTimeoutFn(handle);
|
|
335
|
+
wakeTimers.delete(userId);
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
const armWakeTimer = (userId: string, delayMs: number): void => {
|
|
340
|
+
clearWakeTimer(userId);
|
|
341
|
+
const handle = setTimeoutFn(() => {
|
|
342
|
+
wakeTimers.delete(userId);
|
|
343
|
+
void replay({userId});
|
|
344
|
+
}, delayMs);
|
|
345
|
+
wakeTimers.set(userId, handle);
|
|
346
|
+
};
|
|
347
|
+
|
|
348
|
+
/** Full jitter: random(0, base * 2^(attempt-1)), capped at MAX_BACKOFF_MS. */
|
|
349
|
+
const jitteredBackoff = (attempt: number, baseMs: number): number => {
|
|
350
|
+
const cap = Math.min(MAX_BACKOFF_MS, baseMs * 2 ** (attempt - 1));
|
|
351
|
+
return Math.floor(random() * cap);
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
const buildRequest = (mutation: OutboxMutation): SyncMutateRequest => {
|
|
355
|
+
const request: SyncMutateRequest = {
|
|
356
|
+
collection: mutation.collection,
|
|
357
|
+
id: mutation.entityId,
|
|
358
|
+
mutationId: mutation.mutationId,
|
|
359
|
+
operation: mutation.operation,
|
|
360
|
+
};
|
|
361
|
+
// A2: send-time baseVersion refresh. The entity's CURRENT seq is used as
|
|
362
|
+
// the base when it is NEWER than the value captured at enqueue time — the
|
|
363
|
+
// stored value is the floor, never sent lower. A prior mutation in the
|
|
364
|
+
// same queue may have already acked (stamping the entity's seq via
|
|
365
|
+
// releaseEntity) since this mutation was enqueued; reading the live seq
|
|
366
|
+
// here chains bases correctly through a queue of edits to one entity
|
|
367
|
+
// without any coalescing.
|
|
368
|
+
if (mutation.operation !== "create") {
|
|
369
|
+
const entity = store.getEntity({collection: mutation.collection, id: mutation.entityId});
|
|
370
|
+
const liveSeq = entity?.seq;
|
|
371
|
+
const floor = mutation.baseVersion;
|
|
372
|
+
const base =
|
|
373
|
+
liveSeq !== undefined && (floor === undefined || liveSeq > floor) ? liveSeq : floor;
|
|
374
|
+
if (base !== undefined) {
|
|
375
|
+
request.baseVersion = base;
|
|
376
|
+
}
|
|
377
|
+
} else if (mutation.baseVersion !== undefined) {
|
|
378
|
+
request.baseVersion = mutation.baseVersion;
|
|
379
|
+
}
|
|
380
|
+
if (mutation.operation !== "delete") {
|
|
381
|
+
try {
|
|
382
|
+
request.data = JSON.parse(mutation.args) as Record<string, unknown>;
|
|
383
|
+
} catch {
|
|
384
|
+
request.data = {};
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
return request;
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
/** Clear the entity's pendingMutationId when it still points at this mutation. */
|
|
391
|
+
const releaseEntity = (mutation: OutboxMutation, seq?: number): void => {
|
|
392
|
+
const entity = store.getEntity({collection: mutation.collection, id: mutation.entityId});
|
|
393
|
+
if (!entity) {
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
const ownsPending = entity.pendingMutationId === mutation.mutationId;
|
|
397
|
+
if (!ownsPending && seq === undefined) {
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
store.upsertEntity({
|
|
401
|
+
collection: mutation.collection,
|
|
402
|
+
data: entity.data,
|
|
403
|
+
id: mutation.entityId,
|
|
404
|
+
pendingMutationId: ownsPending ? "" : undefined,
|
|
405
|
+
seq,
|
|
406
|
+
});
|
|
407
|
+
};
|
|
408
|
+
|
|
409
|
+
const handleAck = (mutation: OutboxMutation, ack: SyncAck): void => {
|
|
410
|
+
outbox.markAcked({mutationId: mutation.mutationId});
|
|
411
|
+
retryAt.delete(mutation.mutationId);
|
|
412
|
+
transportFailures.delete(mutation.mutationId);
|
|
413
|
+
releaseEntity(mutation, ack.seq);
|
|
414
|
+
debug?.record({
|
|
415
|
+
collection: mutation.collection,
|
|
416
|
+
detail: {data: debugBody(mutation)},
|
|
417
|
+
direction: "inbound",
|
|
418
|
+
entityId: mutation.entityId,
|
|
419
|
+
label: `ack ${mutation.collection}/${mutation.entityId} @${ack.seq}`,
|
|
420
|
+
mutationId: mutation.mutationId,
|
|
421
|
+
ok: true,
|
|
422
|
+
operation: mutation.operation,
|
|
423
|
+
seq: ack.seq,
|
|
424
|
+
type: "ack",
|
|
425
|
+
});
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
const handleConflict = (mutation: OutboxMutation, nack: SyncNack): void => {
|
|
429
|
+
outbox.markConflicted({mutationId: mutation.mutationId});
|
|
430
|
+
retryAt.delete(mutation.mutationId);
|
|
431
|
+
transportFailures.delete(mutation.mutationId);
|
|
432
|
+
const entity = store.getEntity({collection: mutation.collection, id: mutation.entityId});
|
|
433
|
+
writeConflict({
|
|
434
|
+
conflict: {
|
|
435
|
+
collection: mutation.collection,
|
|
436
|
+
dismissed: false,
|
|
437
|
+
entityId: mutation.entityId,
|
|
438
|
+
localData: JSON.stringify(entity?.data ?? null),
|
|
439
|
+
mutationId: mutation.mutationId,
|
|
440
|
+
serverData: JSON.stringify(nack.serverDoc ?? null),
|
|
441
|
+
// Task 9.12: preserved so `useServer` resolution knows to write a
|
|
442
|
+
// tombstone rather than the nack's empty server document.
|
|
443
|
+
serverDeleted: nack.serverDeleted === true,
|
|
444
|
+
serverSeq: nack.serverSeq ?? 0,
|
|
445
|
+
},
|
|
446
|
+
store,
|
|
447
|
+
});
|
|
448
|
+
debug?.record({
|
|
449
|
+
collection: mutation.collection,
|
|
450
|
+
detail: {
|
|
451
|
+
data: debugBody(mutation),
|
|
452
|
+
localData: entity?.data ?? null,
|
|
453
|
+
message: nack.message,
|
|
454
|
+
serverDoc: nack.serverDoc ?? null,
|
|
455
|
+
serverSeq: nack.serverSeq,
|
|
456
|
+
},
|
|
457
|
+
direction: "inbound",
|
|
458
|
+
entityId: mutation.entityId,
|
|
459
|
+
label: `conflict ${mutation.collection}/${mutation.entityId}`,
|
|
460
|
+
mutationId: mutation.mutationId,
|
|
461
|
+
ok: false,
|
|
462
|
+
operation: mutation.operation,
|
|
463
|
+
type: "conflict",
|
|
464
|
+
});
|
|
465
|
+
};
|
|
466
|
+
|
|
467
|
+
const handleTerminalFailure = (mutation: OutboxMutation, reason: string): void => {
|
|
468
|
+
outbox.markFailed({mutationId: mutation.mutationId});
|
|
469
|
+
retryAt.delete(mutation.mutationId);
|
|
470
|
+
transportFailures.delete(mutation.mutationId);
|
|
471
|
+
// Task 9.11a: the server rejected this mutation outright, so the optimistic
|
|
472
|
+
// row is backed by nothing. Absent a concurrent server write the entity's
|
|
473
|
+
// seq never moves, so snapshot reconcile skips it and the rejected data
|
|
474
|
+
// would persist locally forever. Mark needs-repair UNCONDITIONALLY (before
|
|
475
|
+
// the repair hook runs) so the entity converges to canonical server state —
|
|
476
|
+
// and so a later reconcile retries the repair if this one cannot reach the
|
|
477
|
+
// server. Ids the server declines to return are unmarked by the repair pass
|
|
478
|
+
// itself, so a rejected create never leaves a permanent mark behind.
|
|
479
|
+
const pendingEntity = store.getEntity({
|
|
480
|
+
collection: mutation.collection,
|
|
481
|
+
id: mutation.entityId,
|
|
482
|
+
});
|
|
483
|
+
store.markNeedsRepair({
|
|
484
|
+
collection: mutation.collection,
|
|
485
|
+
entityId: mutation.entityId,
|
|
486
|
+
missedSeq: pendingEntity?.seq ?? 0,
|
|
487
|
+
stream: pendingEntity?.stream ?? "",
|
|
488
|
+
});
|
|
489
|
+
// A failed mutation never replays; leaving pendingMutationId set would
|
|
490
|
+
// block server deltas for this entity forever.
|
|
491
|
+
releaseEntity(mutation);
|
|
492
|
+
onEntityReleasedWithoutServerAck?.(mutation);
|
|
493
|
+
debug?.record({
|
|
494
|
+
collection: mutation.collection,
|
|
495
|
+
detail: {data: debugBody(mutation), reason},
|
|
496
|
+
direction: "inbound",
|
|
497
|
+
entityId: mutation.entityId,
|
|
498
|
+
label: `failed ${mutation.collection}/${mutation.entityId} (${reason})`,
|
|
499
|
+
mutationId: mutation.mutationId,
|
|
500
|
+
ok: false,
|
|
501
|
+
operation: mutation.operation,
|
|
502
|
+
type: "failed",
|
|
503
|
+
});
|
|
504
|
+
};
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* FIX 4: an entity's validation block is garbage-collected once NO outbox
|
|
508
|
+
* row (queued OR still-unpruned failed) remains for it — the block exists
|
|
509
|
+
* only to protect the ordering of an already-queued successor (or to keep
|
|
510
|
+
* surfacing the failure until the user acts), not to quarantine the entity
|
|
511
|
+
* eternally. Checking "any row" rather than "queued only" matters: right
|
|
512
|
+
* after the validation failure itself, the failed row still exists (not
|
|
513
|
+
* yet pruned) with no successor queued yet — the block must survive that
|
|
514
|
+
* window so a successor enqueued moments later is still correctly blocked.
|
|
515
|
+
* Only once prune() removes the failed row AND nothing is queued for the
|
|
516
|
+
* entity does the block become stale and get dropped here. An entity WITH
|
|
517
|
+
* a queued successor, or an unpruned failed row, stays blocked until
|
|
518
|
+
* retryFailed/resolution — that remains intentional.
|
|
519
|
+
*/
|
|
520
|
+
const gcStaleValidationBlocks = (userId: string): void => {
|
|
521
|
+
for (const key of [...validationBlockedEntities]) {
|
|
522
|
+
if (!key.startsWith(`${userId}|`)) {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
const bare = key.slice(userId.length + 1);
|
|
526
|
+
const separatorIndex = bare.indexOf(":");
|
|
527
|
+
const collection = bare.slice(0, separatorIndex);
|
|
528
|
+
const entityId = bare.slice(separatorIndex + 1);
|
|
529
|
+
if (!outbox.hasAnyRowForEntity({collection, entityId, userId})) {
|
|
530
|
+
validationBlockedEntities.delete(key);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* FIX 2: recursively scan a parsed JSON value for any string that exactly
|
|
537
|
+
* equals one of `blockedEntityIds` — used to detect a mutation in ANOTHER
|
|
538
|
+
* collection that references a currently-blocked entity's id (e.g. create
|
|
539
|
+
* project P conflicts → create todo T {projectId: P} must not drain while
|
|
540
|
+
* P is blocked). Conservative false positives are acceptable (INV-1: too
|
|
541
|
+
* much blocking is safe, too little is not).
|
|
542
|
+
*/
|
|
543
|
+
const referencesBlockedId = (value: unknown, blockedEntityIds: ReadonlySet<string>): boolean => {
|
|
544
|
+
if (typeof value === "string") {
|
|
545
|
+
return blockedEntityIds.has(value);
|
|
546
|
+
}
|
|
547
|
+
if (Array.isArray(value)) {
|
|
548
|
+
return value.some((item) => referencesBlockedId(item, blockedEntityIds));
|
|
549
|
+
}
|
|
550
|
+
if (value && typeof value === "object") {
|
|
551
|
+
return Object.values(value as Record<string, unknown>).some((item) =>
|
|
552
|
+
referencesBlockedId(item, blockedEntityIds)
|
|
553
|
+
);
|
|
554
|
+
}
|
|
555
|
+
return false;
|
|
556
|
+
};
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* FIX 2: compute the full transitive block set for one drain pass, from
|
|
560
|
+
* current block state (never persisted — recomputed live each call so
|
|
561
|
+
* resolving a root via `resolveConflict`/`retryFailed` naturally unblocks
|
|
562
|
+
* its dependents on the NEXT drain). Starts from directly-blocked entities
|
|
563
|
+
* (unresolved conflict or skipped validation failure) and expands to any
|
|
564
|
+
* OTHER queued mutation belonging to the same user whose parsed `args`
|
|
565
|
+
* reference a blocked entity's id anywhere (recursively through nested
|
|
566
|
+
* objects/arrays) — repeated to a fixpoint so chains of references (P ← T
|
|
567
|
+
* ← further-mutation-referencing-T) are also blocked.
|
|
568
|
+
*/
|
|
569
|
+
const computeBlockedEntityKeys = (userId: string): Set<string> => {
|
|
570
|
+
const queued = outbox.listQueued({userId});
|
|
571
|
+
const blockedKeys = new Set<string>();
|
|
572
|
+
const blockedIds = new Set<string>();
|
|
573
|
+
// Seed directly from recorded block state FIRST — a root entity (e.g. a
|
|
574
|
+
// validation failure with no queued successor of its own yet, or an
|
|
575
|
+
// entity whose only mutation already left `queued` — conflicted or
|
|
576
|
+
// failed) still needs to be in the reference set so OTHER queued
|
|
577
|
+
// mutations that reference its id are caught, even though the root
|
|
578
|
+
// itself has nothing queued to represent it in `queued` below.
|
|
579
|
+
for (const key of validationBlockedEntities) {
|
|
580
|
+
if (!key.startsWith(`${userId}|`)) {
|
|
581
|
+
continue;
|
|
582
|
+
}
|
|
583
|
+
const bare = key.slice(userId.length + 1);
|
|
584
|
+
blockedKeys.add(bare);
|
|
585
|
+
const separatorIndex = bare.indexOf(":");
|
|
586
|
+
blockedIds.add(bare.slice(separatorIndex + 1));
|
|
587
|
+
}
|
|
588
|
+
// Same seeding for unresolved conflicts (the store is scoped one-user-
|
|
589
|
+
// at-a-time — wiped on user switch — so `listConflicts` needs no
|
|
590
|
+
// additional userId filter here).
|
|
591
|
+
for (const conflict of listConflicts({store})) {
|
|
592
|
+
blockedKeys.add(entityKey(conflict.collection, conflict.entityId));
|
|
593
|
+
blockedIds.add(conflict.entityId);
|
|
594
|
+
}
|
|
595
|
+
// Seeding above is exhaustive for direct blocks: a queued mutation is directly
|
|
596
|
+
// blocked only by a `validationBlockedEntities` entry or a recorded conflict for
|
|
597
|
+
// its own entity, and both sets are fully enumerated above. Re-walking `queued`
|
|
598
|
+
// to test each mutation individually would rescan the whole `_conflicts` table
|
|
599
|
+
// per mutation (O(queued × conflicts)) and could never add a key.
|
|
600
|
+
// Fixpoint expansion: a mutation whose args reference a currently-blocked
|
|
601
|
+
// id becomes blocked itself (its own entityId joins the reference set),
|
|
602
|
+
// which can in turn block a mutation referencing IT.
|
|
603
|
+
let changed = true;
|
|
604
|
+
while (changed) {
|
|
605
|
+
changed = false;
|
|
606
|
+
for (const mutation of queued) {
|
|
607
|
+
const key = entityKey(mutation.collection, mutation.entityId);
|
|
608
|
+
if (blockedKeys.has(key)) {
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
let parsedArgs: unknown;
|
|
612
|
+
try {
|
|
613
|
+
parsedArgs = JSON.parse(mutation.args);
|
|
614
|
+
} catch {
|
|
615
|
+
continue;
|
|
616
|
+
}
|
|
617
|
+
if (referencesBlockedId(parsedArgs, blockedIds)) {
|
|
618
|
+
blockedKeys.add(key);
|
|
619
|
+
blockedIds.add(mutation.entityId);
|
|
620
|
+
changed = true;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
return blockedKeys;
|
|
625
|
+
};
|
|
626
|
+
|
|
627
|
+
/** Outcome of one drain pass over the full FIFO queue. */
|
|
628
|
+
interface DrainPassResult {
|
|
629
|
+
authPaused: boolean;
|
|
630
|
+
/** Earliest backoff wake-up across every parked mutation this pass, if any. */
|
|
631
|
+
nextWakeAt?: number;
|
|
632
|
+
/** True when at least one mutation acked this pass (progress signal for pruning). */
|
|
633
|
+
progressed: boolean;
|
|
634
|
+
/**
|
|
635
|
+
* True when this pass ended for a reason that must stop the CURRENT
|
|
636
|
+
* `replay()` call's drain-until-empty loop even though the queue is
|
|
637
|
+
* neither empty nor backing off with a known wake time: either every
|
|
638
|
+
* remaining queued mutation is B4-blocked (unresolved conflict / skipped
|
|
639
|
+
* validation failure — nothing more can happen until `resolveConflict`
|
|
640
|
+
* or `retryFailed`), or a halting decision fired with no backoff of its
|
|
641
|
+
* own (a plain conflict nack under `haltQueueOnConflict`). Without this,
|
|
642
|
+
* the outer loop would immediately re-call `drainOnce` and either spin
|
|
643
|
+
* forever (blocked case) or silently defeat the halt by draining
|
|
644
|
+
* unrelated entities anyway (haltQueueOnConflict case).
|
|
645
|
+
*/
|
|
646
|
+
stopReplayCall?: boolean;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Process one already-resolved outcome for `mutation` (ack/nack), applying
|
|
651
|
+
* the B4 stop-the-line policy. Returns instructions for the caller (single
|
|
652
|
+
* or batched send loop) on whether to keep going.
|
|
653
|
+
*/
|
|
654
|
+
interface OutcomeDecision {
|
|
655
|
+
/** True when the drain must stop this pass immediately (no more sends). */
|
|
656
|
+
haltDrain: boolean;
|
|
657
|
+
authPaused: boolean;
|
|
658
|
+
nextWakeAt?: number;
|
|
659
|
+
progressed: boolean;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
const applyAck = (mutation: OutboxMutation, ack: SyncAck): OutcomeDecision => {
|
|
663
|
+
handleAck(mutation, ack);
|
|
664
|
+
return {authPaused: false, haltDrain: false, progressed: true};
|
|
665
|
+
};
|
|
666
|
+
|
|
667
|
+
const applyUnauthorized = (mutation: OutboxMutation): OutcomeDecision => {
|
|
668
|
+
outbox.markQueued({mutationId: mutation.mutationId});
|
|
669
|
+
debug?.record({
|
|
670
|
+
collection: mutation.collection,
|
|
671
|
+
detail: {code: "unauthorized", data: debugBody(mutation), paused: true},
|
|
672
|
+
direction: "inbound",
|
|
673
|
+
entityId: mutation.entityId,
|
|
674
|
+
label: `nack ${mutation.collection}/${mutation.entityId} (unauthorized)`,
|
|
675
|
+
mutationId: mutation.mutationId,
|
|
676
|
+
ok: false,
|
|
677
|
+
operation: mutation.operation,
|
|
678
|
+
type: "nack",
|
|
679
|
+
});
|
|
680
|
+
return {authPaused: true, haltDrain: true, progressed: false};
|
|
681
|
+
};
|
|
682
|
+
|
|
683
|
+
const applyConflict = (mutation: OutboxMutation, nack: SyncNack): OutcomeDecision => {
|
|
684
|
+
handleConflict(mutation, nack);
|
|
685
|
+
validationBlockedEntities.delete(
|
|
686
|
+
userEntityKey(mutation.userId, mutation.collection, mutation.entityId)
|
|
687
|
+
);
|
|
688
|
+
// B4: per-entity blocking is the default; haltQueueOnConflict escalates to
|
|
689
|
+
// a whole-drain halt for apps with cross-entity ordering dependencies.
|
|
690
|
+
return {authPaused: false, haltDrain: haltQueueOnConflict, progressed: false};
|
|
691
|
+
};
|
|
692
|
+
|
|
693
|
+
const applyValidation = (mutation: OutboxMutation): OutcomeDecision => {
|
|
694
|
+
handleTerminalFailure(mutation, "validation");
|
|
695
|
+
// B4: the entity's queued successors are skipped-and-surfaced until the
|
|
696
|
+
// user calls retryFailed({entityId}) or resolves the underlying issue —
|
|
697
|
+
// other entities keep draining.
|
|
698
|
+
validationBlockedEntities.add(
|
|
699
|
+
userEntityKey(mutation.userId, mutation.collection, mutation.entityId)
|
|
700
|
+
);
|
|
701
|
+
return {authPaused: false, haltDrain: false, progressed: false};
|
|
702
|
+
};
|
|
703
|
+
|
|
704
|
+
const applyErrorNack = (mutation: OutboxMutation): OutcomeDecision => {
|
|
705
|
+
const beforeCount = outbox.getMutation({mutationId: mutation.mutationId})?.errorNackCount ?? 0;
|
|
706
|
+
const attempts = beforeCount + 1;
|
|
707
|
+
if (attempts >= MAX_ERROR_NACK_ATTEMPTS) {
|
|
708
|
+
handleTerminalFailure(mutation, "error");
|
|
709
|
+
return {authPaused: false, haltDrain: false, progressed: false};
|
|
710
|
+
}
|
|
711
|
+
outbox.markQueuedAfterErrorNack({mutationId: mutation.mutationId});
|
|
712
|
+
const backoffMs = jitteredBackoff(attempts, ERROR_NACK_BASE_BACKOFF_MS);
|
|
713
|
+
const wakeAt = now() + backoffMs;
|
|
714
|
+
retryAt.set(mutation.mutationId, wakeAt);
|
|
715
|
+
debug?.record({
|
|
716
|
+
collection: mutation.collection,
|
|
717
|
+
detail: {attempt: attempts, backoffMs, data: debugBody(mutation), reason: "error"},
|
|
718
|
+
direction: "system",
|
|
719
|
+
entityId: mutation.entityId,
|
|
720
|
+
label: `retry ${mutation.collection}/${mutation.entityId} (error, ${backoffMs}ms)`,
|
|
721
|
+
mutationId: mutation.mutationId,
|
|
722
|
+
operation: mutation.operation,
|
|
723
|
+
type: "retry",
|
|
724
|
+
});
|
|
725
|
+
// B4: error nacks halt the WHOLE drain (transient — retrying without user
|
|
726
|
+
// action will fix it), unlike the per-entity conflict/validation policy.
|
|
727
|
+
return {authPaused: false, haltDrain: true, nextWakeAt: wakeAt, progressed: false};
|
|
728
|
+
};
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Shared backoff-and-requeue path for outcomes that must NEVER burn a
|
|
732
|
+
* retry budget: transport failures and rate-limit nacks alike. Both track
|
|
733
|
+
* attempts in the same in-memory `transportFailures` map (unlimited
|
|
734
|
+
* retries, jittered capped backoff) and HALT THE WHOLE DRAIN (B4) — the
|
|
735
|
+
* remaining chunk/mutation is safe to resend (INV-3). `floorMs` (when
|
|
736
|
+
* present — a server-supplied `retryAfterMs`) raises the computed backoff
|
|
737
|
+
* so the client never retries before the server's own window clears.
|
|
738
|
+
*/
|
|
739
|
+
const requeueWithUnlimitedBackoff = (
|
|
740
|
+
mutation: OutboxMutation,
|
|
741
|
+
{reason, floorMs}: {reason: string; floorMs?: number}
|
|
742
|
+
): OutcomeDecision => {
|
|
743
|
+
outbox.markQueued({mutationId: mutation.mutationId});
|
|
744
|
+
const attempt = (transportFailures.get(mutation.mutationId) ?? 0) + 1;
|
|
745
|
+
transportFailures.set(mutation.mutationId, attempt);
|
|
746
|
+
const backoffMs = Math.max(
|
|
747
|
+
jitteredBackoff(attempt, TRANSPORT_FAILURE_BASE_BACKOFF_MS),
|
|
748
|
+
floorMs ?? 0
|
|
749
|
+
);
|
|
750
|
+
const wakeAt = now() + backoffMs;
|
|
751
|
+
retryAt.set(mutation.mutationId, wakeAt);
|
|
752
|
+
debug?.record({
|
|
753
|
+
collection: mutation.collection,
|
|
754
|
+
detail: {attempt, backoffMs, data: debugBody(mutation), reason},
|
|
755
|
+
direction: "system",
|
|
756
|
+
entityId: mutation.entityId,
|
|
757
|
+
label: `retry ${mutation.collection}/${mutation.entityId} (${reason}, ${backoffMs}ms)`,
|
|
758
|
+
mutationId: mutation.mutationId,
|
|
759
|
+
operation: mutation.operation,
|
|
760
|
+
type: "retry",
|
|
761
|
+
});
|
|
762
|
+
return {authPaused: false, haltDrain: true, nextWakeAt: wakeAt, progressed: false};
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
/**
|
|
766
|
+
* `rate_limited` nacks (FIX 1): the server is asking the client to slow
|
|
767
|
+
* down, not reporting a durable-data problem — treat it EXACTLY like a
|
|
768
|
+
* transport failure. Requeue (never markFailed/terminal), never touch
|
|
769
|
+
* `errorNackCount`, unlimited retries with the same capped jittered
|
|
770
|
+
* backoff, respecting the server's `retryAfterMs` as a floor.
|
|
771
|
+
*/
|
|
772
|
+
const applyRateLimited = (mutation: OutboxMutation, nack: SyncNack): OutcomeDecision => {
|
|
773
|
+
debug?.record({
|
|
774
|
+
collection: mutation.collection,
|
|
775
|
+
detail: {
|
|
776
|
+
code: "rate_limited",
|
|
777
|
+
data: debugBody(mutation),
|
|
778
|
+
message: nack.message,
|
|
779
|
+
retryAfterMs: nack.retryAfterMs,
|
|
780
|
+
},
|
|
781
|
+
direction: "inbound",
|
|
782
|
+
entityId: mutation.entityId,
|
|
783
|
+
label: `nack ${mutation.collection}/${mutation.entityId} (rate_limited)`,
|
|
784
|
+
mutationId: mutation.mutationId,
|
|
785
|
+
ok: false,
|
|
786
|
+
operation: mutation.operation,
|
|
787
|
+
type: "nack",
|
|
788
|
+
});
|
|
789
|
+
return requeueWithUnlimitedBackoff(mutation, {
|
|
790
|
+
floorMs: nack.retryAfterMs,
|
|
791
|
+
reason: "rate_limited",
|
|
792
|
+
});
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
/** Apply a resolved single-mutation SendMutationResult via the B4 outcome table. */
|
|
796
|
+
const applyResult = (mutation: OutboxMutation, result: SendMutationResult): OutcomeDecision => {
|
|
797
|
+
if (result.type === "ack") {
|
|
798
|
+
return applyAck(mutation, result.ack);
|
|
799
|
+
}
|
|
800
|
+
const {nack} = result;
|
|
801
|
+
if (nack.code === "conflict") {
|
|
802
|
+
return applyConflict(mutation, nack);
|
|
803
|
+
}
|
|
804
|
+
if (nack.code === "unauthorized") {
|
|
805
|
+
return applyUnauthorized(mutation);
|
|
806
|
+
}
|
|
807
|
+
if (nack.code === "validation") {
|
|
808
|
+
return applyValidation(mutation);
|
|
809
|
+
}
|
|
810
|
+
if (nack.code === "rate_limited") {
|
|
811
|
+
return applyRateLimited(mutation, nack);
|
|
812
|
+
}
|
|
813
|
+
return applyErrorNack(mutation);
|
|
814
|
+
};
|
|
815
|
+
|
|
816
|
+
/** Apply a transport failure (thrown send / AuthRequiredError) via the B4 outcome table. */
|
|
817
|
+
const applyTransportError = (mutation: OutboxMutation, error: unknown): OutcomeDecision => {
|
|
818
|
+
if (error instanceof AuthRequiredError) {
|
|
819
|
+
// INV-2: never burn a retry budget on an auth failure. Requeue and
|
|
820
|
+
// pause the whole drain; the caller retries after re-auth.
|
|
821
|
+
outbox.markQueued({mutationId: mutation.mutationId});
|
|
822
|
+
debug?.record({
|
|
823
|
+
collection: mutation.collection,
|
|
824
|
+
detail: {code: "unauthorized", data: debugBody(mutation), paused: true, source: "http"},
|
|
825
|
+
direction: "inbound",
|
|
826
|
+
entityId: mutation.entityId,
|
|
827
|
+
label: `nack ${mutation.collection}/${mutation.entityId} (unauthorized)`,
|
|
828
|
+
mutationId: mutation.mutationId,
|
|
829
|
+
ok: false,
|
|
830
|
+
operation: mutation.operation,
|
|
831
|
+
type: "nack",
|
|
832
|
+
});
|
|
833
|
+
return {authPaused: true, haltDrain: true, progressed: false};
|
|
834
|
+
}
|
|
835
|
+
// Transport failure (timeout, network, disconnect): unlimited retries
|
|
836
|
+
// with jittered capped backoff, tracked separately from the error-nack
|
|
837
|
+
// budget. HALTS THE WHOLE DRAIN (B4) — the whole batch is safe to resend
|
|
838
|
+
// (INV-3).
|
|
839
|
+
return requeueWithUnlimitedBackoff(mutation, {reason: "transport"});
|
|
840
|
+
};
|
|
841
|
+
|
|
842
|
+
/**
|
|
843
|
+
* Single-mutation send path (used when no batch transport is configured, or
|
|
844
|
+
* batching is marked unsupported for this connection, or a chunk's only
|
|
845
|
+
* eligible mutation count is 1).
|
|
846
|
+
*/
|
|
847
|
+
const sendSingle = async (userId: string, mutation: OutboxMutation): Promise<OutcomeDecision> => {
|
|
848
|
+
outbox.markInFlight({mutationId: mutation.mutationId});
|
|
849
|
+
debug?.record({
|
|
850
|
+
collection: mutation.collection,
|
|
851
|
+
detail: {attempt: mutation.attemptCount + 1, data: debugBody(mutation)},
|
|
852
|
+
direction: "outbound",
|
|
853
|
+
entityId: mutation.entityId,
|
|
854
|
+
label: `send ${mutation.operation} ${mutation.collection}/${mutation.entityId}`,
|
|
855
|
+
mutationId: mutation.mutationId,
|
|
856
|
+
operation: mutation.operation,
|
|
857
|
+
type: "send",
|
|
858
|
+
});
|
|
859
|
+
let result: SendMutationResult;
|
|
860
|
+
try {
|
|
861
|
+
result = await sendMutation(buildRequest(mutation));
|
|
862
|
+
} catch (error) {
|
|
863
|
+
const decision = applyTransportError(mutation, error);
|
|
864
|
+
reportProgress(userId, 1);
|
|
865
|
+
return decision;
|
|
866
|
+
}
|
|
867
|
+
const decision = applyResult(mutation, result);
|
|
868
|
+
reportProgress(userId, 1);
|
|
869
|
+
return decision;
|
|
870
|
+
};
|
|
871
|
+
|
|
872
|
+
/**
|
|
873
|
+
* Build the next contiguous chunk of the FIFO queue eligible for a batch
|
|
874
|
+
* send: skips mutations whose entity is currently blocked (B4, including
|
|
875
|
+
* FIX 2's transitive cross-collection reference blocks against entities
|
|
876
|
+
* blocked BEFORE this pass — `blockedKeys` is computed once per drain pass
|
|
877
|
+
* by the caller) or backing off (retryAt in the future), stops BEFORE a
|
|
878
|
+
* backing-off mutation (INV-1 — it must remain the effective head once
|
|
879
|
+
* nothing ahead of it can send), and cuts the chunk short before a second
|
|
880
|
+
* mutation for an entity already included (B3.2) so per-entity chaining
|
|
881
|
+
* stays correct without guessing server-assigned seqs.
|
|
882
|
+
*
|
|
883
|
+
* FIX 2 also guards the SAME-BATCH race: a mutation whose args reference
|
|
884
|
+
* the entity id of another mutation already placed EARLIER in this same
|
|
885
|
+
* chunk is excluded too, even though that referenced entity isn't blocked
|
|
886
|
+
* yet (its own outcome — ack or conflict/validation nack — isn't known
|
|
887
|
+
* until this very batch resolves). Without this, P (create) and T (create,
|
|
888
|
+
* referencing P) enqueued together would land in the same chunk and could
|
|
889
|
+
* both be sent before P's conflict is even discovered.
|
|
890
|
+
*/
|
|
891
|
+
const buildChunk = (
|
|
892
|
+
queued: OutboxMutation[],
|
|
893
|
+
blockedKeys: ReadonlySet<string>
|
|
894
|
+
): {chunk: OutboxMutation[]; blockedWakeAt?: number} => {
|
|
895
|
+
const chunk: OutboxMutation[] = [];
|
|
896
|
+
const seenEntities = new Set<string>();
|
|
897
|
+
const chunkEntityIds = new Set<string>();
|
|
898
|
+
let blockedWakeAt: number | undefined;
|
|
899
|
+
for (const mutation of queued) {
|
|
900
|
+
if (chunk.length >= batchSize) {
|
|
901
|
+
break;
|
|
902
|
+
}
|
|
903
|
+
const nextAttemptAt = retryAt.get(mutation.mutationId);
|
|
904
|
+
if (nextAttemptAt !== undefined && now() < nextAttemptAt) {
|
|
905
|
+
// INV-1: a backing-off mutation blocks the entire drain from this
|
|
906
|
+
// point on — stop building the chunk here.
|
|
907
|
+
blockedWakeAt =
|
|
908
|
+
blockedWakeAt === undefined ? nextAttemptAt : Math.min(blockedWakeAt, nextAttemptAt);
|
|
909
|
+
break;
|
|
910
|
+
}
|
|
911
|
+
const key = entityKey(mutation.collection, mutation.entityId);
|
|
912
|
+
if (blockedKeys.has(key)) {
|
|
913
|
+
// B4/FIX 2: this entity (directly or transitively, via a reference
|
|
914
|
+
// to another blocked entity) is blocked; skip it (stays queued) but
|
|
915
|
+
// keep scanning — other entities may still be eligible for this chunk.
|
|
916
|
+
continue;
|
|
917
|
+
}
|
|
918
|
+
if (chunkEntityIds.size > 0) {
|
|
919
|
+
let parsedArgs: unknown;
|
|
920
|
+
try {
|
|
921
|
+
parsedArgs = JSON.parse(mutation.args);
|
|
922
|
+
} catch {
|
|
923
|
+
parsedArgs = undefined;
|
|
924
|
+
}
|
|
925
|
+
if (parsedArgs !== undefined && referencesBlockedId(parsedArgs, chunkEntityIds)) {
|
|
926
|
+
// FIX 2 same-batch guard: this mutation references an entity whose
|
|
927
|
+
// outcome in THIS chunk is not yet known — skip it (stays queued)
|
|
928
|
+
// rather than risk sending it alongside a root that may conflict.
|
|
929
|
+
continue;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
if (seenEntities.has(key)) {
|
|
933
|
+
// B3.2: at most one mutation per entity per chunk — cut here.
|
|
934
|
+
break;
|
|
935
|
+
}
|
|
936
|
+
seenEntities.add(key);
|
|
937
|
+
chunkEntityIds.add(mutation.entityId);
|
|
938
|
+
chunk.push(mutation);
|
|
939
|
+
}
|
|
940
|
+
return {blockedWakeAt, chunk};
|
|
941
|
+
};
|
|
942
|
+
|
|
943
|
+
/** Send one chunk as a batch; returns the per-mutation outcome decisions and any halt reached. */
|
|
944
|
+
const sendChunk = async (
|
|
945
|
+
userId: string,
|
|
946
|
+
chunk: OutboxMutation[]
|
|
947
|
+
): Promise<{
|
|
948
|
+
decisions: Map<string, OutcomeDecision>;
|
|
949
|
+
unsupported: boolean;
|
|
950
|
+
haltedAt?: number;
|
|
951
|
+
/** True when the server returned fewer results than mutations sent. */
|
|
952
|
+
shortResponse: boolean;
|
|
953
|
+
}> => {
|
|
954
|
+
const decisions = new Map<string, OutcomeDecision>();
|
|
955
|
+
for (const mutation of chunk) {
|
|
956
|
+
outbox.markInFlight({mutationId: mutation.mutationId});
|
|
957
|
+
debug?.record({
|
|
958
|
+
collection: mutation.collection,
|
|
959
|
+
detail: {attempt: mutation.attemptCount + 1, batch: true, data: debugBody(mutation)},
|
|
960
|
+
direction: "outbound",
|
|
961
|
+
entityId: mutation.entityId,
|
|
962
|
+
label: `send ${mutation.operation} ${mutation.collection}/${mutation.entityId}`,
|
|
963
|
+
mutationId: mutation.mutationId,
|
|
964
|
+
operation: mutation.operation,
|
|
965
|
+
type: "send",
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
const request: SyncMutateBatchRequest = {mutations: chunk.map(buildRequest)};
|
|
969
|
+
let response: SendMutationBatchResult;
|
|
970
|
+
try {
|
|
971
|
+
// biome-ignore lint/style/noNonNullAssertion: caller only invokes sendChunk when sendMutationBatch is defined and batching is not marked unsupported.
|
|
972
|
+
response = await sendMutationBatch!(request);
|
|
973
|
+
} catch (error) {
|
|
974
|
+
// Transport failure for the WHOLE chunk: every mutation in it goes back
|
|
975
|
+
// to queued via the same transport-error handling as a single send, and
|
|
976
|
+
// the drain halts (INV-3 — resending the whole chunk is safe).
|
|
977
|
+
let haltedAt: number | undefined;
|
|
978
|
+
for (const mutation of chunk) {
|
|
979
|
+
const decision = applyTransportError(mutation, error);
|
|
980
|
+
decisions.set(mutation.mutationId, decision);
|
|
981
|
+
if (decision.nextWakeAt !== undefined) {
|
|
982
|
+
haltedAt =
|
|
983
|
+
haltedAt === undefined ? decision.nextWakeAt : Math.min(haltedAt, decision.nextWakeAt);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
reportProgress(userId, chunk.length);
|
|
987
|
+
return {decisions, haltedAt, shortResponse: false, unsupported: false};
|
|
988
|
+
}
|
|
989
|
+
if (response.type === "unsupported") {
|
|
990
|
+
// The chunk was marked inFlight before sending (for diagnostics/debug
|
|
991
|
+
// symmetry with the single-send path) but never actually attempted —
|
|
992
|
+
// revert every mutation to queued, untouched budgets, so the fallback
|
|
993
|
+
// single-send pass picks them straight back up.
|
|
994
|
+
for (const mutation of chunk) {
|
|
995
|
+
outbox.markQueued({mutationId: mutation.mutationId});
|
|
996
|
+
}
|
|
997
|
+
return {decisions, shortResponse: false, unsupported: true};
|
|
998
|
+
}
|
|
999
|
+
// Walk results in order (INV-1), applying the same per-outcome handlers as
|
|
1000
|
+
// the single-send path, then the B4 stop-the-line policy. Per B2, the
|
|
1001
|
+
// server itself stops at the first non-ack and truncates `results`
|
|
1002
|
+
// accordingly — but defensively, once THIS pass decides to halt (auth
|
|
1003
|
+
// pause in particular — nothing after it may be treated as final), every
|
|
1004
|
+
// remaining mutation in the chunk is requeued untouched rather than left
|
|
1005
|
+
// stranded `inFlight`, regardless of what a non-conforming server sent.
|
|
1006
|
+
let halted = false;
|
|
1007
|
+
let haltedAt: number | undefined;
|
|
1008
|
+
let processed = 0;
|
|
1009
|
+
const shortResponse = response.results.length < chunk.length;
|
|
1010
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
1011
|
+
const mutation = chunk[i];
|
|
1012
|
+
if (halted) {
|
|
1013
|
+
// Only siblings still `inFlight` need reverting — the mutation that
|
|
1014
|
+
// triggered the halt may already be conflicted/failed/acked.
|
|
1015
|
+
if (outbox.getMutation({mutationId: mutation.mutationId})?.status === "inFlight") {
|
|
1016
|
+
outbox.markQueued({mutationId: mutation.mutationId});
|
|
1017
|
+
}
|
|
1018
|
+
continue;
|
|
1019
|
+
}
|
|
1020
|
+
const result = response.results[i];
|
|
1021
|
+
if (!result) {
|
|
1022
|
+
// Server halted mid-batch (results shorter than the request): every
|
|
1023
|
+
// mutation with no result goes back to queued, untouched budgets.
|
|
1024
|
+
if (outbox.getMutation({mutationId: mutation.mutationId})?.status === "inFlight") {
|
|
1025
|
+
outbox.markQueued({mutationId: mutation.mutationId});
|
|
1026
|
+
}
|
|
1027
|
+
continue;
|
|
1028
|
+
}
|
|
1029
|
+
processed += 1;
|
|
1030
|
+
const decision = applyResult(mutation, result);
|
|
1031
|
+
decisions.set(mutation.mutationId, decision);
|
|
1032
|
+
if (decision.haltDrain) {
|
|
1033
|
+
halted = true;
|
|
1034
|
+
if (decision.nextWakeAt !== undefined) {
|
|
1035
|
+
haltedAt = decision.nextWakeAt;
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
reportProgress(userId, processed);
|
|
1040
|
+
return {decisions, haltedAt, shortResponse, unsupported: false};
|
|
1041
|
+
};
|
|
1042
|
+
|
|
1043
|
+
const drainOnce = async (userId: string): Promise<DrainPassResult> => {
|
|
1044
|
+
let progressed = false;
|
|
1045
|
+
let nextWakeAt: number | undefined;
|
|
1046
|
+
|
|
1047
|
+
for (;;) {
|
|
1048
|
+
// FIX 4: drop any validation block whose entity has no queued
|
|
1049
|
+
// successor left (pruned/resolved-away) before computing this pass's
|
|
1050
|
+
// eligibility — an entity must not stay quarantined once nothing
|
|
1051
|
+
// queued depends on the ordering the block was protecting.
|
|
1052
|
+
gcStaleValidationBlocks(userId);
|
|
1053
|
+
const queued = outbox.listQueued({userId});
|
|
1054
|
+
if (queued.length === 0) {
|
|
1055
|
+
return {authPaused: false, nextWakeAt, progressed};
|
|
1056
|
+
}
|
|
1057
|
+
// FIX 2: transitive block set (direct blocks + cross-collection
|
|
1058
|
+
// references to a blocked entity's id), recomputed fresh every pass so
|
|
1059
|
+
// resolving a root naturally unblocks its dependents on the next drain.
|
|
1060
|
+
const blockedKeys = computeBlockedEntityKeys(userId);
|
|
1061
|
+
|
|
1062
|
+
const useBatch = Boolean(sendMutationBatch) && !batchUnsupported;
|
|
1063
|
+
if (!useBatch) {
|
|
1064
|
+
// Single-mutation fallback (no batch transport, or marked unsupported):
|
|
1065
|
+
// send exactly the FIFO head, honoring blocked/backing-off skips.
|
|
1066
|
+
let sentAny = false;
|
|
1067
|
+
for (const mutation of queued) {
|
|
1068
|
+
const nextAttemptAt = retryAt.get(mutation.mutationId);
|
|
1069
|
+
if (nextAttemptAt !== undefined && now() < nextAttemptAt) {
|
|
1070
|
+
nextWakeAt =
|
|
1071
|
+
nextWakeAt === undefined ? nextAttemptAt : Math.min(nextWakeAt, nextAttemptAt);
|
|
1072
|
+
return {authPaused: false, nextWakeAt, progressed};
|
|
1073
|
+
}
|
|
1074
|
+
if (blockedKeys.has(entityKey(mutation.collection, mutation.entityId))) {
|
|
1075
|
+
continue;
|
|
1076
|
+
}
|
|
1077
|
+
const decision = await sendSingle(userId, mutation);
|
|
1078
|
+
sentAny = true;
|
|
1079
|
+
progressed = progressed || decision.progressed;
|
|
1080
|
+
if (decision.authPaused) {
|
|
1081
|
+
return {authPaused: true, progressed};
|
|
1082
|
+
}
|
|
1083
|
+
if (decision.haltDrain) {
|
|
1084
|
+
if (decision.nextWakeAt !== undefined) {
|
|
1085
|
+
nextWakeAt =
|
|
1086
|
+
nextWakeAt === undefined
|
|
1087
|
+
? decision.nextWakeAt
|
|
1088
|
+
: Math.min(nextWakeAt, decision.nextWakeAt);
|
|
1089
|
+
}
|
|
1090
|
+
// A halt with no backoff of its own (e.g. a plain conflict nack
|
|
1091
|
+
// under haltQueueOnConflict) must still stop the CURRENT
|
|
1092
|
+
// replay() call — otherwise the outer loop would immediately
|
|
1093
|
+
// drain unrelated entities anyway, defeating the halt.
|
|
1094
|
+
return {
|
|
1095
|
+
authPaused: false,
|
|
1096
|
+
nextWakeAt,
|
|
1097
|
+
progressed,
|
|
1098
|
+
stopReplayCall: decision.nextWakeAt === undefined,
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
break; // re-list after each send so newly-acked bases (A2) apply.
|
|
1102
|
+
}
|
|
1103
|
+
if (!sentAny) {
|
|
1104
|
+
// Nothing eligible this pass (everything blocked/backing off). When
|
|
1105
|
+
// nothing is backing off either (nextWakeAt still undefined), every
|
|
1106
|
+
// remaining mutation is purely B4-blocked — signal that so the
|
|
1107
|
+
// outer drain-until-empty loop stops instead of spinning forever on
|
|
1108
|
+
// a queue that can never shrink without an external event.
|
|
1109
|
+
return {
|
|
1110
|
+
authPaused: false,
|
|
1111
|
+
nextWakeAt,
|
|
1112
|
+
progressed,
|
|
1113
|
+
stopReplayCall: nextWakeAt === undefined,
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
continue;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
const {chunk, blockedWakeAt} = buildChunk(queued, blockedKeys);
|
|
1120
|
+
if (chunk.length === 0) {
|
|
1121
|
+
if (blockedWakeAt !== undefined) {
|
|
1122
|
+
nextWakeAt =
|
|
1123
|
+
nextWakeAt === undefined ? blockedWakeAt : Math.min(nextWakeAt, blockedWakeAt);
|
|
1124
|
+
}
|
|
1125
|
+
return {
|
|
1126
|
+
authPaused: false,
|
|
1127
|
+
nextWakeAt,
|
|
1128
|
+
progressed,
|
|
1129
|
+
stopReplayCall: blockedWakeAt === undefined,
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
if (chunk.length === 1) {
|
|
1133
|
+
// A lone eligible mutation: sending it as a "batch of one" adds no
|
|
1134
|
+
// value and costs an extra round-trip on unsupported servers — reuse
|
|
1135
|
+
// the single-send path directly (still counts as this pass's send).
|
|
1136
|
+
const decision = await sendSingle(userId, chunk[0]);
|
|
1137
|
+
progressed = progressed || decision.progressed;
|
|
1138
|
+
if (decision.authPaused) {
|
|
1139
|
+
return {authPaused: true, progressed};
|
|
1140
|
+
}
|
|
1141
|
+
if (decision.haltDrain) {
|
|
1142
|
+
if (decision.nextWakeAt !== undefined) {
|
|
1143
|
+
nextWakeAt =
|
|
1144
|
+
nextWakeAt === undefined
|
|
1145
|
+
? decision.nextWakeAt
|
|
1146
|
+
: Math.min(nextWakeAt, decision.nextWakeAt);
|
|
1147
|
+
}
|
|
1148
|
+
return {
|
|
1149
|
+
authPaused: false,
|
|
1150
|
+
nextWakeAt,
|
|
1151
|
+
progressed,
|
|
1152
|
+
stopReplayCall: decision.nextWakeAt === undefined,
|
|
1153
|
+
};
|
|
1154
|
+
}
|
|
1155
|
+
continue;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
const {decisions, unsupported, haltedAt, shortResponse} = await sendChunk(userId, chunk);
|
|
1159
|
+
if (unsupported) {
|
|
1160
|
+
// FIX 5: only latch after BATCH_UNSUPPORTED_LATCH_THRESHOLD
|
|
1161
|
+
// consecutive unsupported results — a single slow-but-supported
|
|
1162
|
+
// batch (grace elapsed without a receipt landing in time, but the
|
|
1163
|
+
// server really does have a handler) must not permanently downgrade
|
|
1164
|
+
// the connection to single-sends.
|
|
1165
|
+
consecutiveUnsupported += 1;
|
|
1166
|
+
if (consecutiveUnsupported >= BATCH_UNSUPPORTED_LATCH_THRESHOLD) {
|
|
1167
|
+
batchUnsupported = true;
|
|
1168
|
+
armReprobeTimer();
|
|
1169
|
+
}
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
// A real (non-unsupported) batch response proves the server supports
|
|
1173
|
+
// batching — reset the consecutive-failure counter so a later stray
|
|
1174
|
+
// unsupported result starts counting from zero again.
|
|
1175
|
+
consecutiveUnsupported = 0;
|
|
1176
|
+
let authPaused = false;
|
|
1177
|
+
for (const decision of decisions.values()) {
|
|
1178
|
+
progressed = progressed || decision.progressed;
|
|
1179
|
+
if (decision.authPaused) {
|
|
1180
|
+
authPaused = true;
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
if (authPaused) {
|
|
1184
|
+
return {authPaused: true, progressed};
|
|
1185
|
+
}
|
|
1186
|
+
if (haltedAt !== undefined) {
|
|
1187
|
+
nextWakeAt = nextWakeAt === undefined ? haltedAt : Math.min(nextWakeAt, haltedAt);
|
|
1188
|
+
}
|
|
1189
|
+
// A halt (conflict-with-haltQueueOnConflict, error-nack backoff,
|
|
1190
|
+
// transport failure) stops the WHOLE drain this pass, mirroring the
|
|
1191
|
+
// single-send path — detect it by re-checking whether any decision
|
|
1192
|
+
// requested a halt. A short response (server halted mid-batch with no
|
|
1193
|
+
// failing nack for the boundary mutation — e.g. it hasn't finished
|
|
1194
|
+
// ledgering yet) halts this pass too (INV-1): the tail already went
|
|
1195
|
+
// back to queued untouched, and the next drain resumes from there.
|
|
1196
|
+
const anyHalt =
|
|
1197
|
+
shortResponse || [...decisions.values()].some((decision) => decision.haltDrain);
|
|
1198
|
+
if (anyHalt) {
|
|
1199
|
+
// As above: a halt with no backoff of its own (plain conflict under
|
|
1200
|
+
// haltQueueOnConflict, or a short response) must stop the CURRENT
|
|
1201
|
+
// replay() call, not just this pass.
|
|
1202
|
+
return {authPaused: false, nextWakeAt, progressed, stopReplayCall: haltedAt === undefined};
|
|
1203
|
+
}
|
|
1204
|
+
// Progress made and no halt: loop to build the next chunk immediately
|
|
1205
|
+
// (drain-until-empty within this pass too, so a 120-mutation queue
|
|
1206
|
+
// drains in ceil(120/batchSize) round-trips within ONE replay() call).
|
|
1207
|
+
}
|
|
1208
|
+
};
|
|
1209
|
+
|
|
1210
|
+
const replay = ({userId}: {userId: string}): Promise<ReplayResult> => {
|
|
1211
|
+
const existing = inFlightReplays.get(userId);
|
|
1212
|
+
if (existing) {
|
|
1213
|
+
// The running drain may have already taken its final "queue empty"
|
|
1214
|
+
// snapshot before this call's mutation was enqueued; request a
|
|
1215
|
+
// follow-up check once it settles rather than silently stranding the
|
|
1216
|
+
// new mutation until an unrelated external trigger runs.
|
|
1217
|
+
recheckRequested.add(userId);
|
|
1218
|
+
return existing;
|
|
1219
|
+
}
|
|
1220
|
+
if (!shouldDrain()) {
|
|
1221
|
+
// The host has gated sending off (simulated offline, client-level auth
|
|
1222
|
+
// pause, post-stop, etc). Internally-scheduled callers (the wake timer,
|
|
1223
|
+
// the recheck above) must respect this exactly like an external caller
|
|
1224
|
+
// would — never send, never arm a new timer, never touch retry state.
|
|
1225
|
+
return Promise.resolve({});
|
|
1226
|
+
}
|
|
1227
|
+
// A fresh replay call supersedes any pending timed wake-up — it is about
|
|
1228
|
+
// to drain right now.
|
|
1229
|
+
clearWakeTimer(userId);
|
|
1230
|
+
// B5: (re)start this call's progress counters. `total` is a lower bound —
|
|
1231
|
+
// snapshotted once at entry — since more may enqueue mid-drain; onProgress
|
|
1232
|
+
// still reports every attempt made.
|
|
1233
|
+
drainProgress.set(userId, {sent: 0, total: outbox.listQueued({userId}).length});
|
|
1234
|
+
const run = (async (): Promise<ReplayResult> => {
|
|
1235
|
+
let authPaused = false;
|
|
1236
|
+
let parkedAt: number | undefined;
|
|
1237
|
+
// Drain-until-empty: keep re-running passes while there is still work
|
|
1238
|
+
// and nothing has parked the queue. shouldDrain() is re-checked before
|
|
1239
|
+
// EVERY pass (not just once at entry): a pass can be deferred onto a
|
|
1240
|
+
// later microtask/timer tick by the time it actually runs, during which
|
|
1241
|
+
// the host may have gone offline or paused — an internally-scheduled
|
|
1242
|
+
// continuation must never race past that.
|
|
1243
|
+
for (;;) {
|
|
1244
|
+
if (!shouldDrain()) {
|
|
1245
|
+
break;
|
|
1246
|
+
}
|
|
1247
|
+
const pass = await drainOnce(userId);
|
|
1248
|
+
if (pass.progressed) {
|
|
1249
|
+
onDrainPass?.({userId});
|
|
1250
|
+
}
|
|
1251
|
+
if (pass.authPaused) {
|
|
1252
|
+
authPaused = true;
|
|
1253
|
+
break;
|
|
1254
|
+
}
|
|
1255
|
+
if (pass.nextWakeAt !== undefined) {
|
|
1256
|
+
parkedAt = pass.nextWakeAt;
|
|
1257
|
+
break;
|
|
1258
|
+
}
|
|
1259
|
+
if (pass.stopReplayCall) {
|
|
1260
|
+
// Either every remaining queued mutation is B4-blocked (nothing
|
|
1261
|
+
// more can happen until resolveConflict/retryFailed — looping
|
|
1262
|
+
// again would spin forever on a queue that never shrinks), or a
|
|
1263
|
+
// halting decision with no backoff of its own fired (a plain
|
|
1264
|
+
// conflict nack under haltQueueOnConflict) — looping again would
|
|
1265
|
+
// silently defeat that halt by draining unrelated entities anyway.
|
|
1266
|
+
break;
|
|
1267
|
+
}
|
|
1268
|
+
// Only stop looping once nothing is queued (drain-until-empty).
|
|
1269
|
+
if (outbox.listQueued({userId}).length === 0) {
|
|
1270
|
+
break;
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
if (authPaused) {
|
|
1274
|
+
if (!authPauseNotified.has(userId)) {
|
|
1275
|
+
authPauseNotified.add(userId);
|
|
1276
|
+
onAuthPause?.({userId});
|
|
1277
|
+
}
|
|
1278
|
+
return {paused: "auth"};
|
|
1279
|
+
}
|
|
1280
|
+
authPauseNotified.delete(userId);
|
|
1281
|
+
if (parkedAt !== undefined) {
|
|
1282
|
+
// Timed wake-up: arm a single setTimeout for the earliest backoff
|
|
1283
|
+
// across the parked queue so callers never need to poll.
|
|
1284
|
+
armWakeTimer(userId, Math.max(0, parkedAt - now()));
|
|
1285
|
+
}
|
|
1286
|
+
return {};
|
|
1287
|
+
})();
|
|
1288
|
+
const tracked = run.finally(() => {
|
|
1289
|
+
inFlightReplays.delete(userId);
|
|
1290
|
+
if (recheckRequested.delete(userId)) {
|
|
1291
|
+
void replay({userId});
|
|
1292
|
+
}
|
|
1293
|
+
});
|
|
1294
|
+
inFlightReplays.set(userId, tracked);
|
|
1295
|
+
return tracked;
|
|
1296
|
+
};
|
|
1297
|
+
|
|
1298
|
+
/** FIX 3: full lifecycle reset — see the `ReplayCoordinator.reset` doc comment. */
|
|
1299
|
+
const reset = (): void => {
|
|
1300
|
+
for (const userId of [...wakeTimers.keys()]) {
|
|
1301
|
+
clearWakeTimer(userId);
|
|
1302
|
+
}
|
|
1303
|
+
clearReprobeTimer();
|
|
1304
|
+
retryAt.clear();
|
|
1305
|
+
transportFailures.clear();
|
|
1306
|
+
authPauseNotified.clear();
|
|
1307
|
+
validationBlockedEntities.clear();
|
|
1308
|
+
drainProgress.clear();
|
|
1309
|
+
recheckRequested.clear();
|
|
1310
|
+
batchUnsupported = false;
|
|
1311
|
+
consecutiveUnsupported = 0;
|
|
1312
|
+
};
|
|
1313
|
+
|
|
1314
|
+
/**
|
|
1315
|
+
* FIX 3: `dispose()` is the client-stop/sign-out teardown hook, so a full
|
|
1316
|
+
* `reset()` is always safe here regardless of the (legacy) `userId` arg —
|
|
1317
|
+
* nothing else uses this coordinator instance once dispose has run for the
|
|
1318
|
+
* current lifecycle.
|
|
1319
|
+
*/
|
|
1320
|
+
const dispose = (_args?: {userId?: string}): void => {
|
|
1321
|
+
reset();
|
|
1322
|
+
};
|
|
1323
|
+
|
|
1324
|
+
const notifyReconnect = (): void => {
|
|
1325
|
+
clearReprobeTimer();
|
|
1326
|
+
batchUnsupported = false;
|
|
1327
|
+
consecutiveUnsupported = 0;
|
|
1328
|
+
};
|
|
1329
|
+
|
|
1330
|
+
const retryFailed = ({entityId}: {entityId: string}): void => {
|
|
1331
|
+
for (const key of [...validationBlockedEntities]) {
|
|
1332
|
+
if (key.endsWith(`:${entityId}`)) {
|
|
1333
|
+
validationBlockedEntities.delete(key);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
};
|
|
1337
|
+
|
|
1338
|
+
const getBlockedEntities = ({userId}: {userId: string}): string[] => {
|
|
1339
|
+
// FIX 4: drop stale blocks (no queued successor left) before reporting.
|
|
1340
|
+
gcStaleValidationBlocks(userId);
|
|
1341
|
+
// FIX 2/3: report the full transitive block set (direct + cross-collection
|
|
1342
|
+
// references), scoped to THIS user only — a different user's stale
|
|
1343
|
+
// in-memory entry (or a same-id different-user block) must never surface
|
|
1344
|
+
// here (FIX 3: getBlockedEntities({userId}) filters accordingly).
|
|
1345
|
+
return [...computeBlockedEntityKeys(userId)];
|
|
1346
|
+
};
|
|
1347
|
+
|
|
1348
|
+
const getDrainProgress = ({
|
|
1349
|
+
userId,
|
|
1350
|
+
}: {
|
|
1351
|
+
userId: string;
|
|
1352
|
+
}): {draining: boolean; sentThisDrain: number; totalThisDrain: number} => {
|
|
1353
|
+
const progress = drainProgress.get(userId);
|
|
1354
|
+
return {
|
|
1355
|
+
draining: inFlightReplays.has(userId),
|
|
1356
|
+
sentThisDrain: progress?.sent ?? 0,
|
|
1357
|
+
totalThisDrain: progress?.total ?? 0,
|
|
1358
|
+
};
|
|
1359
|
+
};
|
|
1360
|
+
|
|
1361
|
+
return {
|
|
1362
|
+
dispose,
|
|
1363
|
+
getBlockedEntities,
|
|
1364
|
+
getDrainProgress,
|
|
1365
|
+
notifyReconnect,
|
|
1366
|
+
replay,
|
|
1367
|
+
reset,
|
|
1368
|
+
retryFailed,
|
|
1369
|
+
};
|
|
1370
|
+
};
|