@abloatai/ablo 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -3
- package/CHANGELOG.md +20 -0
- package/README.md +3 -3
- package/dist/BaseSyncedStore.d.ts +72 -195
- package/dist/BaseSyncedStore.js +240 -625
- package/dist/Database.d.ts +8 -9
- package/dist/Database.js +9 -14
- package/dist/LazyReferenceCollection.js +5 -0
- package/dist/Model.js +6 -8
- package/dist/ModelRegistry.d.ts +25 -6
- package/dist/ModelRegistry.js +12 -6
- package/dist/NetworkMonitor.js +1 -1
- package/dist/ObjectPool.js +9 -3
- package/dist/SyncClient.d.ts +15 -5
- package/dist/SyncClient.js +152 -66
- package/dist/SyncEngineContext.d.ts +1 -3
- package/dist/SyncEngineContext.js +1 -2
- package/dist/agent/Agent.d.ts +21 -8
- package/dist/agent/Agent.js +26 -13
- package/dist/ai-sdk/coordination-context.js +9 -3
- package/dist/ai-sdk/wrap.d.ts +1 -1
- package/dist/ai-sdk/wrap.js +1 -1
- package/dist/auth/credentialPolicy.d.ts +10 -2
- package/dist/auth/credentialSource.d.ts +1 -13
- package/dist/auth/credentialSource.js +6 -13
- package/dist/auth/index.js +3 -3
- package/dist/cli.cjs +328 -262
- package/dist/client/Ablo.d.ts +35 -764
- package/dist/client/Ablo.js +64 -688
- package/dist/client/ApiClient.d.ts +14 -1
- package/dist/client/ApiClient.js +63 -9
- package/dist/client/auth.d.ts +8 -17
- package/dist/client/auth.js +42 -6
- package/dist/client/consoleLogger.d.ts +36 -0
- package/dist/client/consoleLogger.js +45 -0
- package/dist/client/createInternalComponents.js +1 -1
- package/dist/client/createModelProxy.js +10 -6
- package/dist/client/credentialEndpoint.d.ts +63 -0
- package/dist/client/credentialEndpoint.js +87 -0
- package/dist/client/hostedEndpoints.d.ts +24 -0
- package/dist/client/hostedEndpoints.js +24 -0
- package/dist/client/httpClient.d.ts +1 -1
- package/dist/client/identity.js +2 -1
- package/dist/client/modelRegistration.d.ts +14 -0
- package/dist/client/modelRegistration.js +321 -0
- package/dist/client/options.d.ts +396 -0
- package/dist/client/options.js +10 -0
- package/dist/client/resourceTypes.d.ts +344 -0
- package/dist/client/resourceTypes.js +13 -0
- package/dist/client/schemaConfig.d.ts +56 -0
- package/dist/client/schemaConfig.js +188 -0
- package/dist/client/sessionMint.d.ts +1 -1
- package/dist/client/validateAbloOptions.js +1 -1
- package/dist/client/wsMutationExecutor.d.ts +33 -0
- package/dist/client/wsMutationExecutor.js +77 -0
- package/dist/context.js +0 -3
- package/dist/coordination/index.d.ts +8 -1
- package/dist/coordination/index.js +24 -1
- package/dist/coordination/trace.d.ts +0 -1
- package/dist/coordination/trace.js +0 -1
- package/dist/core/DatabaseManager.d.ts +0 -1
- package/dist/core/DatabaseManager.js +26 -24
- package/dist/core/QueryProcessor.js +7 -8
- package/dist/core/QueryView.d.ts +14 -2
- package/dist/core/QueryView.js +4 -5
- package/dist/core/StoreManager.d.ts +2 -2
- package/dist/core/StoreManager.js +5 -1
- package/dist/core/index.d.ts +1 -1
- package/dist/core/openIDBWithTimeout.js +15 -11
- package/dist/core/query-utils.js +1 -1
- package/dist/core/storeContract.d.ts +143 -0
- package/dist/core/storeContract.js +16 -0
- package/dist/errorCodes.d.ts +11 -2
- package/dist/errorCodes.js +155 -137
- package/dist/errors.d.ts +18 -7
- package/dist/errors.js +14 -1
- package/dist/index.js +5 -0
- package/dist/interfaces/index.d.ts +6 -20
- package/dist/keys/index.js +18 -9
- package/dist/mutators/UndoManager.js +6 -1
- package/dist/mutators/defineMutators.d.ts +1 -3
- package/dist/policy/types.d.ts +1 -1
- package/dist/query/client.d.ts +11 -0
- package/dist/query/client.js +81 -52
- package/dist/react/AbloProvider.d.ts +6 -10
- package/dist/react/AbloProvider.js +22 -15
- package/dist/react/context.d.ts +2 -118
- package/dist/react/useAblo.js +1 -1
- package/dist/react/useErrorListener.js +1 -1
- package/dist/react/useMutationFailureListener.js +1 -1
- package/dist/react/useMutators.js +7 -1
- package/dist/react/useReactive.js +0 -1
- package/dist/react/useUndoScope.js +2 -2
- package/dist/schema/ddlLock.d.ts +39 -0
- package/dist/schema/ddlLock.js +51 -0
- package/dist/schema/field.js +1 -1
- package/dist/schema/index.d.ts +4 -3
- package/dist/schema/index.js +15 -5
- package/dist/schema/model.d.ts +3 -3
- package/dist/schema/model.js +2 -2
- package/dist/schema/openapi.js +5 -1
- package/dist/schema/queries.d.ts +1 -1
- package/dist/schema/queries.js +1 -0
- package/dist/schema/residency.d.ts +35 -0
- package/dist/schema/residency.js +27 -0
- package/dist/schema/roles.d.ts +25 -0
- package/dist/schema/roles.js +37 -0
- package/dist/schema/schema.d.ts +3 -3
- package/dist/schema/schema.js +11 -7
- package/dist/schema/serialize.d.ts +8 -8
- package/dist/schema/sync-delta-row.d.ts +2 -2
- package/dist/schema/sync-delta-row.js +2 -2
- package/dist/server/storage-mode.d.ts +8 -1
- package/dist/server/storage-mode.js +14 -4
- package/dist/source/adapter.d.ts +1 -1
- package/dist/source/adapters/drizzle.js +4 -3
- package/dist/source/adapters/kysely.js +4 -3
- package/dist/source/adapters/memory.js +1 -1
- package/dist/source/adapters/prisma.js +5 -4
- package/dist/source/conformance.js +10 -8
- package/dist/source/connector.js +4 -2
- package/dist/source/factory.d.ts +106 -0
- package/dist/source/factory.js +273 -0
- package/dist/source/index.d.ts +11 -461
- package/dist/source/index.js +14 -416
- package/dist/source/next.d.ts +1 -1
- package/dist/source/next.js +1 -1
- package/dist/source/pushQueue.d.ts +1 -1
- package/dist/source/pushQueue.js +6 -3
- package/dist/source/signing.d.ts +59 -0
- package/dist/source/signing.js +142 -0
- package/dist/source/types.d.ts +334 -0
- package/dist/source/types.js +44 -0
- package/dist/stores/ObjectStore.d.ts +1 -1
- package/dist/stores/ObjectStore.js +23 -23
- package/dist/stores/SyncActionStore.d.ts +1 -1
- package/dist/stores/SyncActionStore.js +70 -35
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -0
- package/dist/sync/AreaOfInterestManager.d.ts +2 -2
- package/dist/sync/AreaOfInterestManager.js +1 -1
- package/dist/sync/BootstrapHelper.d.ts +1 -3
- package/dist/sync/BootstrapHelper.js +29 -8
- package/dist/sync/ConnectionManager.d.ts +8 -0
- package/dist/sync/ConnectionManager.js +19 -4
- package/dist/sync/HydrationCoordinator.d.ts +12 -0
- package/dist/sync/HydrationCoordinator.js +25 -5
- package/dist/sync/NetworkProbe.d.ts +3 -2
- package/dist/sync/NetworkProbe.js +7 -6
- package/dist/sync/SyncWebSocket.d.ts +71 -120
- package/dist/sync/SyncWebSocket.js +324 -630
- package/dist/sync/awaitClaimGrant.js +27 -19
- package/dist/sync/bootstrapApply.d.ts +60 -0
- package/dist/sync/bootstrapApply.js +65 -0
- package/dist/sync/commitFrames.d.ts +43 -0
- package/dist/sync/commitFrames.js +94 -0
- package/dist/sync/createClaimStream.js +3 -2
- package/dist/sync/createSnapshot.d.ts +1 -3
- package/dist/sync/credentialLifecycle.d.ts +175 -0
- package/dist/sync/credentialLifecycle.js +329 -0
- package/dist/sync/deltaPipeline.d.ts +111 -0
- package/dist/sync/deltaPipeline.js +262 -0
- package/dist/sync/groupChange.d.ts +111 -0
- package/dist/sync/groupChange.js +246 -0
- package/dist/sync/heartbeat.d.ts +62 -0
- package/dist/sync/heartbeat.js +91 -0
- package/dist/sync/participants.d.ts +8 -8
- package/dist/sync/syncCursor.d.ts +44 -0
- package/dist/sync/syncCursor.js +59 -0
- package/dist/sync/syncPlan.d.ts +62 -0
- package/dist/sync/syncPlan.js +51 -0
- package/dist/sync/wsFrameHandlers.d.ts +117 -0
- package/dist/sync/wsFrameHandlers.js +378 -0
- package/dist/testing/fixtures/bootstrap.d.ts +9 -11
- package/dist/testing/fixtures/deltas.d.ts +1 -1
- package/dist/testing/fixtures/models.js +1 -0
- package/dist/testing/helpers/sync-engine-harness.d.ts +0 -3
- package/dist/testing/helpers/sync-engine-harness.js +2 -1
- package/dist/testing/index.d.ts +1 -1
- package/dist/testing/index.js +1 -1
- package/dist/testing/mocks/MockSyncContext.d.ts +1 -17
- package/dist/testing/mocks/MockSyncContext.js +1 -32
- package/dist/testing/mocks/MockSyncStore.d.ts +4 -4
- package/dist/testing/mocks/MockWebSocket.d.ts +2 -2
- package/dist/transactions/TransactionQueue.d.ts +20 -56
- package/dist/transactions/TransactionQueue.js +101 -487
- package/dist/transactions/TransactionStore.d.ts +18 -0
- package/dist/transactions/TransactionStore.js +51 -0
- package/dist/transactions/coalesceRules.d.ts +34 -0
- package/dist/transactions/coalesceRules.js +117 -0
- package/dist/transactions/commitPayload.d.ts +134 -0
- package/dist/transactions/commitPayload.js +152 -0
- package/dist/transactions/deltaConfirmation.d.ts +60 -0
- package/dist/transactions/deltaConfirmation.js +223 -0
- package/dist/transactions/optimistic.d.ts +24 -0
- package/dist/transactions/optimistic.js +45 -0
- package/dist/transactions/persistedReplay.d.ts +93 -0
- package/dist/transactions/persistedReplay.js +105 -0
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.js +0 -5
- package/dist/types/modelData.d.ts +12 -0
- package/dist/types/modelData.js +11 -0
- package/dist/types/participant.d.ts +21 -0
- package/dist/types/participant.js +12 -0
- package/dist/types/streams.d.ts +8 -14
- package/dist/utils/mobx-setup.d.ts +1 -1
- package/dist/utils/mobx-setup.js +2 -2
- package/dist/wire/errorEnvelope.d.ts +11 -2
- package/dist/wire/errorEnvelope.js +14 -2
- package/dist/wire/frames.d.ts +90 -0
- package/dist/wire/frames.js +55 -1
- package/dist/wire/index.d.ts +4 -1
- package/dist/wire/index.js +20 -1
- package/dist/wire/protocol.d.ts +45 -0
- package/dist/wire/protocol.js +45 -0
- package/dist/wire/protocolVersion.d.ts +56 -0
- package/dist/wire/protocolVersion.js +63 -0
- package/docs/api-keys.md +4 -3
- package/docs/examples/existing-python-backend.md +3 -3
- package/docs/identity.md +4 -4
- package/docs/integration-guide.md +1 -1
- package/docs/react.md +1 -1
- package/docs/sessions.md +5 -7
- package/package.json +15 -13
- package/dist/client/index.d.ts +0 -36
- package/dist/client/index.js +0 -33
- package/dist/config/index.d.ts +0 -10
- package/dist/config/index.js +0 -12
- package/dist/interfaces/headless.d.ts +0 -95
- package/dist/interfaces/headless.js +0 -41
- package/dist/query/index.d.ts +0 -6
- package/dist/query/index.js +0 -5
- package/dist/realtime/index.d.ts +0 -10
- package/dist/realtime/index.js +0 -9
- package/dist/schema/plane.d.ts +0 -23
- package/dist/schema/plane.js +0 -19
- package/dist/server/next.d.ts +0 -51
- package/dist/server/next.js +0 -47
- package/dist/sync/OfflineFlush.d.ts +0 -9
- package/dist/sync/OfflineFlush.js +0 -22
- package/dist/sync/OfflineTransactionStore.d.ts +0 -37
- package/dist/sync/OfflineTransactionStore.js +0 -263
- package/dist/transactions/index.d.ts +0 -16
- package/dist/transactions/index.js +0 -7
|
@@ -9,11 +9,33 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { EventEmitter } from 'events';
|
|
11
11
|
import { getContext } from '../context.js';
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
import { AbloConnectionError, AbloError, SyncSessionError, toAbloError, } from '../errors.js';
|
|
13
|
+
import { clientSyncDeltaSchema } from '../schema/sync-delta-wire.js';
|
|
14
|
+
// Commit-path frame builders (pure) — extracted leaf; the host re-exports
|
|
15
|
+
// `CommitAck` below so importers keep this module as their path.
|
|
16
|
+
import { buildCommitFrame } from './commitFrames.js';
|
|
17
|
+
// Inbound frame dispatch table + the minimal session slice it operates on.
|
|
18
|
+
import { dispatchWsFrame, isRecord, isWsInboundFrame, } from './wsFrameHandlers.js';
|
|
19
|
+
// Sync-position state (lastSyncId watermark, version vector, server cursor).
|
|
20
|
+
import { SyncCursor } from './syncCursor.js';
|
|
21
|
+
import { PROTOCOL_VERSION, WS_CLOSE_PROTOCOL_VERSION } from '../wire/protocolVersion.js';
|
|
22
|
+
// Application-level heartbeat timers (see heartbeat.ts for the rationale).
|
|
23
|
+
import { HeartbeatController } from './heartbeat.js';
|
|
16
24
|
import { WS_BEARER_SUBPROTOCOL_PREFIX, WS_SYNC_SUBPROTOCOL, } from '../auth/credentialSource.js';
|
|
25
|
+
// SyncObservability replaced by getContext().observability
|
|
26
|
+
/**
|
|
27
|
+
* Periodic catch-up poll cadence — while connected, request any deltas
|
|
28
|
+
* whose fire-and-forget pub/sub broadcast was lost in transit. Deliberately
|
|
29
|
+
* LOCAL (not `wire/protocol.ts`): an SDK eventual-consistency knob, not a
|
|
30
|
+
* cross-boundary contract the server derives anything from. That it equals
|
|
31
|
+
* the 30s ping today is coincidence, not an invariant.
|
|
32
|
+
*/
|
|
33
|
+
const CATCHUP_POLL_INTERVAL_MS = 30_000;
|
|
34
|
+
/**
|
|
35
|
+
* Ceiling for the exponential reconnect backoff (`reconnectDelay * 2^n`,
|
|
36
|
+
* ±15% jitter). Local for the same reason as the catch-up poll.
|
|
37
|
+
*/
|
|
38
|
+
const MAX_RECONNECT_DELAY_MS = 30_000;
|
|
17
39
|
// ---------------------------------------------------------------------------
|
|
18
40
|
// Ablo-specific collaboration events moved to apps/web/src/lib/sync/collaboration-events.ts
|
|
19
41
|
// Consumers pass their own event types as TCollaboration generic parameter.
|
|
@@ -47,20 +69,20 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
47
69
|
/** Periodic catchup interval — polls for missed deltas every 30s while connected */
|
|
48
70
|
catchupInterval = null;
|
|
49
71
|
/**
|
|
50
|
-
* Application-level heartbeat
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* silently broke (laptop sleep, NAT timeout, mobile handoff). We send an
|
|
55
|
-
* application-level `{ type: 'ping' }` every 30s and force-close the
|
|
56
|
-
* socket if no inbound traffic arrives within 10s. ANY inbound message
|
|
57
|
-
* counts as proof-of-life — the explicit `pong` is just a guarantee that
|
|
58
|
-
* something will arrive even on an idle stream.
|
|
72
|
+
* Application-level heartbeat — ping every 30s, force-close on a 10s
|
|
73
|
+
* silent watchdog. The full zombie-socket rationale lives with the
|
|
74
|
+
* timers in `sync/heartbeat.ts`; the transport closures below are the
|
|
75
|
+
* only socket access the controller gets.
|
|
59
76
|
*/
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
77
|
+
heartbeat = new HeartbeatController({
|
|
78
|
+
isSocketOpen: () => this.ws?.readyState === WebSocket.OPEN,
|
|
79
|
+
// Optional-chained rather than asserted: the controller only calls
|
|
80
|
+
// this synchronously after `isSocketOpen()`, so `ws` is present.
|
|
81
|
+
sendPing: () => {
|
|
82
|
+
this.ws?.send(JSON.stringify({ type: 'ping' }));
|
|
83
|
+
},
|
|
84
|
+
forceClose: (reason) => { this.forceClose(reason); },
|
|
85
|
+
});
|
|
64
86
|
isConnecting = false;
|
|
65
87
|
isManualClose = false;
|
|
66
88
|
/** When true, a session error has been detected (from any path — WS close or HTTP bootstrap).
|
|
@@ -88,11 +110,23 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
88
110
|
lastCloseReason = null;
|
|
89
111
|
lastForceCloseReason = null;
|
|
90
112
|
sessionErrorAt = null;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Sync-position state (lastSyncId watermark, version vector, server
|
|
115
|
+
* cursor). The advance discipline stays documented at `sendAck` /
|
|
116
|
+
* `handleDelta`; the state itself lives in `sync/syncCursor.ts`.
|
|
117
|
+
*/
|
|
118
|
+
cursor;
|
|
94
119
|
/** Registered collaboration event keys (colon format) for dispatch in onmessage */
|
|
95
120
|
collaborationEventTypes;
|
|
121
|
+
/**
|
|
122
|
+
* Minimal session adapter handed to the frame dispatch table
|
|
123
|
+
* (`sync/wsFrameHandlers.ts`). Exposes ONLY the members the handlers
|
|
124
|
+
* touch; the closure members read live state so host-side reassignment
|
|
125
|
+
* (e.g. the `pendingSubscriptions` reset on close) can't strand the
|
|
126
|
+
* handlers on a stale reference. Built in the constructor, after the
|
|
127
|
+
* state it captures exists.
|
|
128
|
+
*/
|
|
129
|
+
frameSession;
|
|
96
130
|
/**
|
|
97
131
|
* In-flight `commit` mutation requests keyed by clientTxId. Resolved when
|
|
98
132
|
* a matching `mutation_result` frame arrives from the server, or rejected on
|
|
@@ -125,21 +159,10 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
125
159
|
this.options = {
|
|
126
160
|
url: wsUrl,
|
|
127
161
|
reconnectDelay: 1000,
|
|
128
|
-
maxReconnectDelay:
|
|
162
|
+
maxReconnectDelay: MAX_RECONNECT_DELAY_MS,
|
|
129
163
|
collaborationEvents: ['sheet:selection', 'slide:selection', 'slide:cursor'],
|
|
130
164
|
syncGroups: [],
|
|
131
165
|
lastSyncId: 0,
|
|
132
|
-
versions: {
|
|
133
|
-
tasks: 0,
|
|
134
|
-
projects: 0,
|
|
135
|
-
users: 0,
|
|
136
|
-
events: 0,
|
|
137
|
-
inboxitems: 0,
|
|
138
|
-
teams: 0,
|
|
139
|
-
assignments: 0,
|
|
140
|
-
comments: 0,
|
|
141
|
-
threads: 0,
|
|
142
|
-
},
|
|
143
166
|
capabilities: {
|
|
144
167
|
partialBootstrap: true,
|
|
145
168
|
compressedDeltas: true,
|
|
@@ -148,10 +171,25 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
148
171
|
},
|
|
149
172
|
...options,
|
|
150
173
|
};
|
|
151
|
-
this.
|
|
152
|
-
this.versionVector = { ...this.options.versions };
|
|
153
|
-
this.syncCursor = null;
|
|
174
|
+
this.cursor = new SyncCursor(this.options.lastSyncId);
|
|
154
175
|
this.collaborationEventTypes = new Set(options.collaborationEvents ?? ['sheet:selection', 'slide:selection', 'slide:cursor']);
|
|
176
|
+
// Session slice for the inbound frame dispatch table — see the field
|
|
177
|
+
// doc on `frameSession` for why members that the host reassigns are
|
|
178
|
+
// exposed through closures instead of captured references.
|
|
179
|
+
this.frameSession = {
|
|
180
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
181
|
+
pendingMutations: this.pendingMutations,
|
|
182
|
+
pendingClaims: this.pendingClaims,
|
|
183
|
+
shiftPendingSubscription: () => this.pendingSubscriptions.shift(),
|
|
184
|
+
options: this.options,
|
|
185
|
+
collaborationEventTypes: this.collaborationEventTypes,
|
|
186
|
+
handleDelta: (delta) => { this.handleDelta(delta); },
|
|
187
|
+
handleSyncResponse: (payload) => { this.handleSyncResponse(payload); },
|
|
188
|
+
handleBootstrapResponse: (payload) => { this.handleBootstrapResponse(payload); },
|
|
189
|
+
handlePresenceUpdate: (message) => {
|
|
190
|
+
this.handlePresenceUpdate(message);
|
|
191
|
+
},
|
|
192
|
+
};
|
|
155
193
|
}
|
|
156
194
|
/**
|
|
157
195
|
* Mark that a session error has been detected (e.g. 401 from HTTP bootstrap).
|
|
@@ -161,6 +199,17 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
161
199
|
this._sessionErrorDetected = true;
|
|
162
200
|
this.sessionErrorAt = Date.now();
|
|
163
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Clear the session-error latch so `connect()` / `scheduleReconnect()`
|
|
204
|
+
* work again. Called by the store's access-credential recovery path when
|
|
205
|
+
* the close was a re-mintable `ek_`/`rk_` expiry (`4001 credential_expired`),
|
|
206
|
+
* not a login loss — see `isAccessCredentialExpiryCloseReason`. Genuine
|
|
207
|
+
* session losses never clear the latch; re-auth builds a fresh client.
|
|
208
|
+
*/
|
|
209
|
+
clearSessionError() {
|
|
210
|
+
this._sessionErrorDetected = false;
|
|
211
|
+
this.sessionErrorAt = null;
|
|
212
|
+
}
|
|
164
213
|
/**
|
|
165
214
|
* Connect to the sync engine WebSocket
|
|
166
215
|
*/
|
|
@@ -169,8 +218,15 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
169
218
|
getContext().logger.debug('WebSocket connect suppressed — session error detected');
|
|
170
219
|
return;
|
|
171
220
|
}
|
|
172
|
-
|
|
173
|
-
|
|
221
|
+
// CLOSING counts as busy: the socket's close teardown is still in
|
|
222
|
+
// flight and its `onclose` (which runs `scheduleReconnect`) hasn't
|
|
223
|
+
// fired yet. Overwriting `this.ws` mid-teardown is what produced the
|
|
224
|
+
// orphaned-socket race — see the stale-socket guards in
|
|
225
|
+
// `setupEventHandlers`.
|
|
226
|
+
if (this.ws?.readyState === WebSocket.OPEN ||
|
|
227
|
+
this.ws?.readyState === WebSocket.CLOSING ||
|
|
228
|
+
this.isConnecting) {
|
|
229
|
+
getContext().logger.debug('WebSocket already connected, connecting, or closing');
|
|
174
230
|
return;
|
|
175
231
|
}
|
|
176
232
|
// Note: onlineStatus is advisory — we'll try to connect and let the WebSocket
|
|
@@ -187,9 +243,9 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
187
243
|
// server is bearer-only (`apiKeyProvider`) and resolves identity from the
|
|
188
244
|
// verified token — userId/organizationId are NEVER read from URL params.
|
|
189
245
|
const params = new URLSearchParams({
|
|
190
|
-
// Intentionally omit lastSyncId,
|
|
246
|
+
// Intentionally omit lastSyncId, capabilities from URL; these are sent in sync_request
|
|
191
247
|
// and ack messages to avoid stale baselines on reconnect.
|
|
192
|
-
cursor: this.syncCursor || '',
|
|
248
|
+
cursor: this.cursor.syncCursor || '',
|
|
193
249
|
});
|
|
194
250
|
// Participant kind — defaults to `user` for backward compatibility
|
|
195
251
|
// with web sessions. Agent runtimes pass `'agent'` so the server's
|
|
@@ -234,11 +290,22 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
234
290
|
* Setup WebSocket event handlers
|
|
235
291
|
*/
|
|
236
292
|
setupEventHandlers() {
|
|
237
|
-
|
|
293
|
+
// Capture the socket THIS call wires. Every handler below guards on
|
|
294
|
+
// `this.ws === socket` (onclose additionally tolerates a nulled host —
|
|
295
|
+
// see there) so a handler firing late, after `connect()` replaced the
|
|
296
|
+
// socket, can never clobber the NEW connection's shared state. Without
|
|
297
|
+
// this, an old socket's `onclose` ran `this.ws = null;
|
|
298
|
+
// stopCatchupInterval(); stopHeartbeat()` unconditionally — a reconnect
|
|
299
|
+
// during close teardown orphaned the fresh socket (zombie receiving
|
|
300
|
+
// deltas with no timers and broken send paths).
|
|
301
|
+
const socket = this.ws;
|
|
302
|
+
if (!socket)
|
|
238
303
|
return;
|
|
239
|
-
|
|
304
|
+
socket.onopen = () => {
|
|
305
|
+
if (this.ws !== socket)
|
|
306
|
+
return; // stale socket — a newer connect() owns the state
|
|
240
307
|
getContext().observability.breadcrumb('WebSocket connected', 'sync.websocket', 'info', {
|
|
241
|
-
lastSyncId: this.lastSyncId,
|
|
308
|
+
lastSyncId: this.cursor.lastSyncId,
|
|
242
309
|
reconnectAttempts: this.reconnectAttempts,
|
|
243
310
|
});
|
|
244
311
|
this.isConnecting = false;
|
|
@@ -249,376 +316,71 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
249
316
|
// Send presence update with timezone (server sets presence to "online" on connect,
|
|
250
317
|
// this improves localTime accuracy by providing the user's actual timezone)
|
|
251
318
|
this.sendPresenceUpdate('online');
|
|
252
|
-
//
|
|
253
|
-
//
|
|
254
|
-
(
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
getContext().observability.captureOfflineFlushFailure({
|
|
263
|
-
error: e instanceof Error ? e.message : String(e),
|
|
264
|
-
});
|
|
265
|
-
}
|
|
266
|
-
})();
|
|
267
|
-
// Immediately request incremental sync based on our stored cursor/versions
|
|
319
|
+
// Immediately request incremental sync based on our stored cursor.
|
|
320
|
+
// `requestIncrementalSync` is async — a bare call inside try/catch is a
|
|
321
|
+
// rejection hole (the catch never sees it); route failures through
|
|
322
|
+
// `.catch` so they land in the same breadcrumb instead of an
|
|
323
|
+
// unhandled rejection.
|
|
324
|
+
const reportSyncRequestFailure = (e) => {
|
|
325
|
+
getContext().observability.breadcrumb('Failed to request incremental sync on open', 'sync.websocket', 'warning', {
|
|
326
|
+
error: e instanceof Error ? e.message : String(e),
|
|
327
|
+
});
|
|
328
|
+
};
|
|
268
329
|
try {
|
|
269
|
-
if (this.lastSyncId && this.lastSyncId > 0) {
|
|
330
|
+
if (this.cursor.lastSyncId && this.cursor.lastSyncId > 0) {
|
|
270
331
|
// Let server know where we left off before requesting deltas
|
|
271
|
-
this.sendAck(this.lastSyncId);
|
|
332
|
+
this.sendAck(this.cursor.lastSyncId);
|
|
272
333
|
}
|
|
273
|
-
this.requestIncrementalSync();
|
|
274
334
|
}
|
|
275
335
|
catch (e) {
|
|
276
|
-
|
|
277
|
-
error: e instanceof Error ? e.message : String(e),
|
|
278
|
-
});
|
|
336
|
+
reportSyncRequestFailure(e);
|
|
279
337
|
}
|
|
280
|
-
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
338
|
+
this.requestIncrementalSync().catch(reportSyncRequestFailure);
|
|
339
|
+
// Start periodic catchup — polls for missed deltas every
|
|
340
|
+
// CATCHUP_POLL_INTERVAL_MS while connected. Real-time WebSocket
|
|
341
|
+
// delivery is best-effort (fire-and-forget Redis pub/sub). This
|
|
342
|
+
// interval guarantees eventual consistency by fetching any deltas
|
|
343
|
+
// that were committed to the DB but whose broadcast was lost in
|
|
344
|
+
// transit.
|
|
284
345
|
this.stopCatchupInterval();
|
|
285
346
|
this.catchupInterval = setInterval(() => {
|
|
286
347
|
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
287
|
-
|
|
348
|
+
// A rejected sync request must never surface as an unhandled
|
|
349
|
+
// rejection from a background interval — log and let the next
|
|
350
|
+
// poll retry.
|
|
351
|
+
this.requestIncrementalSync().catch((e) => {
|
|
352
|
+
getContext().observability.breadcrumb('Periodic catchup sync request failed', 'sync.websocket', 'warning', {
|
|
353
|
+
error: e instanceof Error ? e.message : String(e),
|
|
354
|
+
});
|
|
355
|
+
});
|
|
288
356
|
}
|
|
289
|
-
},
|
|
290
|
-
// Start application-level heartbeat — see
|
|
291
|
-
this.
|
|
357
|
+
}, CATCHUP_POLL_INTERVAL_MS);
|
|
358
|
+
// Start application-level heartbeat — see sync/heartbeat.ts for rationale.
|
|
359
|
+
this.heartbeat.start();
|
|
292
360
|
};
|
|
293
|
-
|
|
361
|
+
socket.onmessage = (event) => {
|
|
362
|
+
if (this.ws !== socket)
|
|
363
|
+
return; // stale socket — drop, don't feed shared state
|
|
294
364
|
try {
|
|
365
|
+
// Untrusted wire input: parse to `unknown`, then narrow through
|
|
366
|
+
// the frame-envelope guard before dispatch. Payload-level
|
|
367
|
+
// validation (deltas etc.) happens per-frame downstream.
|
|
295
368
|
const message = JSON.parse(event.data);
|
|
296
369
|
// ANY inbound frame proves the socket is alive — clear the
|
|
297
370
|
// heartbeat-timeout timer so we don't false-trip force-close
|
|
298
371
|
// during normal traffic.
|
|
299
|
-
this.clearHeartbeatTimeout();
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
372
|
+
this.heartbeat.clearHeartbeatTimeout();
|
|
373
|
+
if (!isWsInboundFrame(message)) {
|
|
374
|
+
getContext().logger.debug('[SyncWebSocket] dropped malformed wire frame', {
|
|
375
|
+
received: Array.isArray(message) ? 'array' : typeof message,
|
|
376
|
+
});
|
|
304
377
|
return;
|
|
305
378
|
}
|
|
306
|
-
//
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
case 'bootstrap_response':
|
|
312
|
-
this.handleBootstrapResponse(message.payload);
|
|
313
|
-
break;
|
|
314
|
-
case 'presence_update':
|
|
315
|
-
this.handlePresenceUpdate(message);
|
|
316
|
-
break;
|
|
317
|
-
case 'mutation_result': {
|
|
318
|
-
// Ack for a prior `commit` we sent. Canonical shape is
|
|
319
|
-
// `MutationResultMessage` in `@abloatai/ablo/wire`. This stays a
|
|
320
|
-
// DEFENSIVE parse (not a typed cast) because the payload is
|
|
321
|
-
// untrusted wire data that may be malformed or from an older server.
|
|
322
|
-
const p = message.payload ?? message;
|
|
323
|
-
const { clientTxId, success, lastSyncId, error } = p ?? {};
|
|
324
|
-
// Defensive: validate notifications against the canonical schema —
|
|
325
|
-
// untrusted wire data from a possibly-older/newer server.
|
|
326
|
-
const notifications = this.parseNotifications(p?.notifications);
|
|
327
|
-
const pending = typeof clientTxId === 'string'
|
|
328
|
-
? this.pendingMutations.get(clientTxId)
|
|
329
|
-
: undefined;
|
|
330
|
-
if (!pending)
|
|
331
|
-
break;
|
|
332
|
-
clearTimeout(pending.timeout);
|
|
333
|
-
this.pendingMutations.delete(clientTxId);
|
|
334
|
-
if (success) {
|
|
335
|
-
// Coerce defensively — bigint columns serialize as strings
|
|
336
|
-
// from older servers (see normalizeWireDelta).
|
|
337
|
-
const ackedSyncId = Number(lastSyncId);
|
|
338
|
-
// Notify-instead-of-abort: a guarded write's premise moved. Emit
|
|
339
|
-
// the advisory signal so an agent loop can self-heal, AND resolve
|
|
340
|
-
// the receipt with it (the commit still succeeded).
|
|
341
|
-
if (notifications && notifications.length > 0) {
|
|
342
|
-
const txId = typeof clientTxId === 'string' ? clientTxId : '';
|
|
343
|
-
const event = {
|
|
344
|
-
clientTxId: txId,
|
|
345
|
-
rows: notifications.map((n) => ({
|
|
346
|
-
model: n.model,
|
|
347
|
-
id: n.id,
|
|
348
|
-
fields: n.conflictingFields,
|
|
349
|
-
writtenBy: n.writtenBy?.kind,
|
|
350
|
-
})),
|
|
351
|
-
};
|
|
352
|
-
const message = formatConflict(event);
|
|
353
|
-
const ctx = getContext();
|
|
354
|
-
ctx.logger.warn(message);
|
|
355
|
-
ctx.observability.breadcrumb(message, 'sync.coordination', 'warning');
|
|
356
|
-
ctx.observability.captureConflict(event);
|
|
357
|
-
this.emit('conflict:notified', {
|
|
358
|
-
clientTxId: txId,
|
|
359
|
-
notifications,
|
|
360
|
-
});
|
|
361
|
-
}
|
|
362
|
-
pending.resolve({
|
|
363
|
-
lastSyncId: Number.isFinite(ackedSyncId) ? ackedSyncId : 0,
|
|
364
|
-
...(notifications && notifications.length > 0
|
|
365
|
-
? { notifications }
|
|
366
|
-
: {}),
|
|
367
|
-
});
|
|
368
|
-
}
|
|
369
|
-
else {
|
|
370
|
-
// Capture the FULL server error so the user can see what
|
|
371
|
-
// actually rejected the mutation. Without this, every
|
|
372
|
-
// rejection becomes the generic "mutation failed on
|
|
373
|
-
// server" — useless when debugging chart batches that
|
|
374
|
-
// tank 40+ ops at once. We stringify object errors so
|
|
375
|
-
// structured server payloads (e.g., Zod issues, schema
|
|
376
|
-
// violations) survive the trip through `new Error(...)`.
|
|
377
|
-
let errorMessage;
|
|
378
|
-
let errorCode;
|
|
379
|
-
let requiredCapability;
|
|
380
|
-
if (typeof error === 'string') {
|
|
381
|
-
errorMessage = error;
|
|
382
|
-
}
|
|
383
|
-
else if (error != null && typeof error === 'object') {
|
|
384
|
-
const obj = error;
|
|
385
|
-
if (typeof obj.code === 'string')
|
|
386
|
-
errorCode = obj.code;
|
|
387
|
-
if (typeof obj.message === 'string') {
|
|
388
|
-
errorMessage = obj.message;
|
|
389
|
-
}
|
|
390
|
-
else {
|
|
391
|
-
try {
|
|
392
|
-
errorMessage = JSON.stringify(error);
|
|
393
|
-
}
|
|
394
|
-
catch {
|
|
395
|
-
errorMessage = String(error);
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
if (obj.requiredCapability != null &&
|
|
399
|
-
typeof obj.requiredCapability === 'object' &&
|
|
400
|
-
typeof obj.requiredCapability.scope === 'string') {
|
|
401
|
-
requiredCapability = obj.requiredCapability;
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
else {
|
|
405
|
-
errorMessage = 'mutation failed on server';
|
|
406
|
-
}
|
|
407
|
-
// Coordination collision: a stale-context rejection (the write's
|
|
408
|
-
// readAt premise moved underneath) or a foreign-claim conflict is
|
|
409
|
-
// exactly the collision ClaimLog exists to surface. The notify
|
|
410
|
-
// path (success + notifications) emits captureConflict above; a
|
|
411
|
-
// HARD rejection must too — otherwise observability.collisions()
|
|
412
|
-
// silently misses every rejected write. The conflicted rows ride
|
|
413
|
-
// along on the typed error's `conflicts` detail (see
|
|
414
|
-
// AbloStaleContextError.toJSON / errorEnvelope).
|
|
415
|
-
if (errorCode === 'stale_context' ||
|
|
416
|
-
errorCode === 'claim_conflict' ||
|
|
417
|
-
errorCode === 'entity_claimed' ||
|
|
418
|
-
errorCode?.startsWith('policy:') === true) {
|
|
419
|
-
const rawConflicts = error != null &&
|
|
420
|
-
typeof error === 'object' &&
|
|
421
|
-
Array.isArray(error.conflicts)
|
|
422
|
-
? error
|
|
423
|
-
.conflicts
|
|
424
|
-
: [];
|
|
425
|
-
const conflictEvent = {
|
|
426
|
-
clientTxId: typeof clientTxId === 'string' ? clientTxId : '',
|
|
427
|
-
rows: rawConflicts.map((r) => ({
|
|
428
|
-
model: typeof r.model === 'string' ? r.model : 'unknown',
|
|
429
|
-
id: typeof r.id === 'string' ? r.id : 'unknown',
|
|
430
|
-
fields: [],
|
|
431
|
-
})),
|
|
432
|
-
};
|
|
433
|
-
const ctx = getContext();
|
|
434
|
-
ctx.observability.breadcrumb(formatConflict(conflictEvent), 'sync.coordination', 'warning');
|
|
435
|
-
ctx.observability.captureConflict(conflictEvent);
|
|
436
|
-
}
|
|
437
|
-
// Build the proper typed AbloError from the wire code via the
|
|
438
|
-
// shared factory — the same code→class mapping the HTTP commit
|
|
439
|
-
// path uses (`translateHttpError`). This keeps rejected commits
|
|
440
|
-
// inside the typed hierarchy (capability denials →
|
|
441
|
-
// CapabilityError with `.requiredCapability`; foreign-claim
|
|
442
|
-
// conflicts → AbloClaimedError; everything else → the subclass
|
|
443
|
-
// its registry `httpStatus` implies) instead of a hand-rolled
|
|
444
|
-
// `new Error`, so callers can `instanceof`/`e.type` it and
|
|
445
|
-
// downstream retry logic can read the contract's retryability.
|
|
446
|
-
pending.reject(errorFromWire(errorMessage, {
|
|
447
|
-
code: errorCode,
|
|
448
|
-
requiredCapability,
|
|
449
|
-
}));
|
|
450
|
-
}
|
|
451
|
-
break;
|
|
452
|
-
}
|
|
453
|
-
case 'claim_ack': {
|
|
454
|
-
// Ack for a prior `claim` we sent. Wire format mirrors
|
|
455
|
-
// apps/sync-server/src/hub/types.ts ClaimAckMessage:
|
|
456
|
-
// { type: 'claim_ack',
|
|
457
|
-
// payload: { claimId, success, syncGroups?,
|
|
458
|
-
// ttlSeconds?, error? } }
|
|
459
|
-
const p = message.payload ?? {};
|
|
460
|
-
const { claimId, success, syncGroups, ttlSeconds, error } = p;
|
|
461
|
-
const pending = typeof claimId === 'string'
|
|
462
|
-
? this.pendingClaims.get(claimId)
|
|
463
|
-
: undefined;
|
|
464
|
-
if (!pending)
|
|
465
|
-
break;
|
|
466
|
-
clearTimeout(pending.timeout);
|
|
467
|
-
this.pendingClaims.delete(claimId);
|
|
468
|
-
if (success) {
|
|
469
|
-
pending.resolve({
|
|
470
|
-
syncGroups: Array.isArray(syncGroups) ? syncGroups : [],
|
|
471
|
-
ttlSeconds: typeof ttlSeconds === 'number' ? ttlSeconds : undefined,
|
|
472
|
-
});
|
|
473
|
-
}
|
|
474
|
-
else {
|
|
475
|
-
const code = error?.code && typeof error.code === 'string'
|
|
476
|
-
? error.code
|
|
477
|
-
: 'claim_rejected';
|
|
478
|
-
const msg = error?.message && typeof error.message === 'string'
|
|
479
|
-
? error.message
|
|
480
|
-
: 'claim rejected by server';
|
|
481
|
-
// Capability denials get the typed CapabilityError so
|
|
482
|
-
// callers can read `.requiredCapability` and attenuate-
|
|
483
|
-
// and-retry the claim with a narrower token.
|
|
484
|
-
if (code === 'capability_scope_denied' ||
|
|
485
|
-
code === 'capability_invalid') {
|
|
486
|
-
const rc = error
|
|
487
|
-
?.requiredCapability;
|
|
488
|
-
const requiredCapability = rc != null &&
|
|
489
|
-
typeof rc === 'object' &&
|
|
490
|
-
typeof rc.scope === 'string'
|
|
491
|
-
? rc
|
|
492
|
-
: undefined;
|
|
493
|
-
pending.reject(new CapabilityError(code, msg, requiredCapability));
|
|
494
|
-
}
|
|
495
|
-
else {
|
|
496
|
-
// Route through the shared factory so a failed claim_ack is a
|
|
497
|
-
// typed AbloError (registry code → right subclass), symmetric
|
|
498
|
-
// with the commit `mutation_result` path — never a bare Error.
|
|
499
|
-
pending.reject(errorFromWire(msg, { code }));
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
break;
|
|
503
|
-
}
|
|
504
|
-
case 'subscription_ack': {
|
|
505
|
-
// Ack for a prior `update_subscription`. The wire carries no
|
|
506
|
-
// correlation id, so FIFO-match against the oldest pending
|
|
507
|
-
// request — the server applies and acks subscription updates
|
|
508
|
-
// in receive order. Validated through the canonical zod schema
|
|
509
|
-
// (mirrors how the Hub validates inbound frames).
|
|
510
|
-
const pending = this.pendingSubscriptions.shift();
|
|
511
|
-
if (!pending)
|
|
512
|
-
break;
|
|
513
|
-
clearTimeout(pending.timeout);
|
|
514
|
-
const parsed = subscriptionAckPayloadSchema.safeParse(message.payload);
|
|
515
|
-
if (!parsed.success) {
|
|
516
|
-
// Unreadable ack — resolve the pending request as a failure
|
|
517
|
-
// rather than hang it until timeout.
|
|
518
|
-
pending.reject(errorFromWire('malformed subscription_ack from server', {
|
|
519
|
-
code: 'malformed_subscription',
|
|
520
|
-
}));
|
|
521
|
-
break;
|
|
522
|
-
}
|
|
523
|
-
const ack = parsed.data;
|
|
524
|
-
if (ack.success) {
|
|
525
|
-
// Keep the reconnect URL aligned with current interest: a
|
|
526
|
-
// reconnect re-subscribes from `this.options.syncGroups`.
|
|
527
|
-
this.options.syncGroups = ack.syncGroups;
|
|
528
|
-
pending.resolve({ syncGroups: ack.syncGroups });
|
|
529
|
-
}
|
|
530
|
-
else {
|
|
531
|
-
pending.reject(errorFromWire(ack.error?.message ?? 'update_subscription rejected by server', { code: ack.error?.code ?? 'malformed_subscription' }));
|
|
532
|
-
}
|
|
533
|
-
break;
|
|
534
|
-
}
|
|
535
|
-
case 'claim_expired': {
|
|
536
|
-
// Server-initiated expiry notification. Emit as a typed
|
|
537
|
-
// event so consumers can react (re-claim with a fresh
|
|
538
|
-
// capability, or accept the drop). The claim is already
|
|
539
|
-
// inactive server-side by the time this arrives.
|
|
540
|
-
const p = message.payload ?? {};
|
|
541
|
-
if (typeof p.claimId === 'string') {
|
|
542
|
-
this.recordClaim('expired', p);
|
|
543
|
-
this.emit('claim_expired', { claimId: p.claimId });
|
|
544
|
-
}
|
|
545
|
-
break;
|
|
546
|
-
}
|
|
547
|
-
case 'claim_rejected': {
|
|
548
|
-
// Server denied an `claim_begin` because the target is
|
|
549
|
-
// already claimed by another participant. Forward the
|
|
550
|
-
// payload as-is — the ClaimStream consumer interprets
|
|
551
|
-
// the conflict shape (peerId, target, etc.).
|
|
552
|
-
this.recordClaim('rejected', message.payload ?? {});
|
|
553
|
-
this.emit('claim_rejected', message.payload ?? {});
|
|
554
|
-
break;
|
|
555
|
-
}
|
|
556
|
-
case 'claim_acquired': {
|
|
557
|
-
// Opt-in fair queue: the target was free, so the lease is ours
|
|
558
|
-
// immediately (no waiting). Payload carries { claimId, target }.
|
|
559
|
-
this.recordClaim('acquired', message.payload ?? {});
|
|
560
|
-
this.emit('claim_acquired', message.payload ?? {});
|
|
561
|
-
break;
|
|
562
|
-
}
|
|
563
|
-
case 'claim_queue': {
|
|
564
|
-
// Per-entity wait-queue snapshot for reactive `queue(id)`. Not a
|
|
565
|
-
// single claim's state change, so it isn't logged — the per-claim
|
|
566
|
-
// `queued`/`granted` events already tell that story.
|
|
567
|
-
this.emit('claim_queue', message.payload ?? {});
|
|
568
|
-
break;
|
|
569
|
-
}
|
|
570
|
-
case 'claim_queued': {
|
|
571
|
-
// Opt-in fair queue: our claim is waiting in line. Payload
|
|
572
|
-
// carries { claimId, target, position }.
|
|
573
|
-
this.recordClaim('queued', message.payload ?? {});
|
|
574
|
-
this.emit('claim_queued', message.payload ?? {});
|
|
575
|
-
break;
|
|
576
|
-
}
|
|
577
|
-
case 'claim_granted': {
|
|
578
|
-
// Our queued claim reached the head — the lease is now ours.
|
|
579
|
-
this.recordClaim('granted', message.payload ?? {});
|
|
580
|
-
this.emit('claim_granted', message.payload ?? {});
|
|
581
|
-
break;
|
|
582
|
-
}
|
|
583
|
-
case 'claim_lost': {
|
|
584
|
-
// A held/granted claim was taken from us (TTL lapse, revoke).
|
|
585
|
-
this.recordClaim('lost', message.payload ?? {});
|
|
586
|
-
this.emit('claim_lost', message.payload ?? {});
|
|
587
|
-
break;
|
|
588
|
-
}
|
|
589
|
-
case 'delta': {
|
|
590
|
-
const p = message.payload;
|
|
591
|
-
if (p?.actionType || p?.modelName) {
|
|
592
|
-
this.handleDelta(p);
|
|
593
|
-
}
|
|
594
|
-
else if (Array.isArray(p?.deltas)) {
|
|
595
|
-
for (const d of p.deltas) {
|
|
596
|
-
if (d?.actionType || d?.modelName)
|
|
597
|
-
this.handleDelta(d);
|
|
598
|
-
}
|
|
599
|
-
if (p?.newVersions) {
|
|
600
|
-
Object.assign(this.versionVector, p.newVersions);
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
break;
|
|
604
|
-
}
|
|
605
|
-
case undefined: // Legacy support: bare delta
|
|
606
|
-
if (message.actionType || message.modelName) {
|
|
607
|
-
this.handleDelta(message);
|
|
608
|
-
}
|
|
609
|
-
break;
|
|
610
|
-
default: {
|
|
611
|
-
// Collaboration events use underscore wire format (e.g., 'sheet_selection')
|
|
612
|
-
// Convert to colon format for the event map (e.g., 'sheet:selection')
|
|
613
|
-
const eventKey = message.type?.replace(/_/g, ':');
|
|
614
|
-
if (eventKey && this.collaborationEventTypes.has(eventKey)) {
|
|
615
|
-
this.emit(eventKey, message.payload);
|
|
616
|
-
}
|
|
617
|
-
else {
|
|
618
|
-
getContext().logger.debug('Received unknown message type', { message });
|
|
619
|
-
}
|
|
620
|
-
}
|
|
621
|
-
}
|
|
379
|
+
// Frame-type → handler dispatch (sync/wsFrameHandlers.ts). The
|
|
380
|
+
// session adapter exposes only the members the handlers touch;
|
|
381
|
+
// keepalives, the legacy bare-delta form, and collaboration
|
|
382
|
+
// events are all routed there too.
|
|
383
|
+
dispatchWsFrame(this.frameSession, message);
|
|
622
384
|
}
|
|
623
385
|
catch (error) {
|
|
624
386
|
getContext().observability.captureWebSocketError({
|
|
@@ -627,7 +389,9 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
627
389
|
});
|
|
628
390
|
}
|
|
629
391
|
};
|
|
630
|
-
|
|
392
|
+
socket.onerror = (_event) => {
|
|
393
|
+
if (this.ws !== socket)
|
|
394
|
+
return; // stale socket — its errors are no longer ours
|
|
631
395
|
// WebSocket errors are DOM Events, not Error objects
|
|
632
396
|
// Check if we're offline first
|
|
633
397
|
if (!getContext().onlineStatus.isOnline()) {
|
|
@@ -646,7 +410,17 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
646
410
|
}
|
|
647
411
|
this.emit('error', error);
|
|
648
412
|
};
|
|
649
|
-
|
|
413
|
+
socket.onclose = (event) => {
|
|
414
|
+
// Stale-socket close: a NEWER socket already owns the connection
|
|
415
|
+
// state — don't null it, stop its timers, or schedule a duplicate
|
|
416
|
+
// reconnect (the orphaning race this guard exists for). The one
|
|
417
|
+
// deliberate asymmetry vs the other handlers: `this.ws === null`
|
|
418
|
+
// (manual `disconnect()` nulls the field before the close event
|
|
419
|
+
// lands) still runs the full body, so in-flight work is rejected
|
|
420
|
+
// promptly and 'disconnected' reaches consumers — the pre-guard
|
|
421
|
+
// behavior manual close always had.
|
|
422
|
+
if (this.ws !== null && this.ws !== socket)
|
|
423
|
+
return;
|
|
650
424
|
const everOpened = this._everOpened;
|
|
651
425
|
this.lastCloseAt = Date.now();
|
|
652
426
|
this.lastCloseCode = event.code;
|
|
@@ -663,7 +437,7 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
663
437
|
this.isConnecting = false;
|
|
664
438
|
this.ws = null;
|
|
665
439
|
this.stopCatchupInterval();
|
|
666
|
-
this.
|
|
440
|
+
this.heartbeat.stop();
|
|
667
441
|
// Cancel in-flight mutations — the socket that was carrying them is
|
|
668
442
|
// gone, and the server-side state may or may not have accepted each
|
|
669
443
|
// one. Rejecting promptly is better than hanging the caller forever;
|
|
@@ -706,6 +480,20 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
706
480
|
}
|
|
707
481
|
this.pendingSubscriptions = [];
|
|
708
482
|
}
|
|
483
|
+
// Protocol-version rejection (4010): TERMINAL. Reconnecting cannot heal
|
|
484
|
+
// a version mismatch — only upgrading the SDK (or rolling the server
|
|
485
|
+
// forward) can — so a blind retry here would loop forever against the
|
|
486
|
+
// same typed close. Surface it and stop.
|
|
487
|
+
if (event.code === WS_CLOSE_PROTOCOL_VERSION) {
|
|
488
|
+
getContext().observability.captureWebSocketError({
|
|
489
|
+
context: 'protocol-version-close',
|
|
490
|
+
code: event.code,
|
|
491
|
+
reason: event.reason,
|
|
492
|
+
});
|
|
493
|
+
this.emit('protocol_mismatch', event);
|
|
494
|
+
this.emit('disconnected', event);
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
709
497
|
// Check for session-related close codes
|
|
710
498
|
// 1008 = Policy Violation (often auth)
|
|
711
499
|
// 4001 = Unauthorized (custom)
|
|
@@ -723,7 +511,11 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
723
511
|
reason: event.reason,
|
|
724
512
|
});
|
|
725
513
|
this.emit('session_error', new SyncSessionError(event.reason || 'Session expired', event.code));
|
|
726
|
-
// Don't reconnect
|
|
514
|
+
// Don't reconnect from HERE. For a genuine session loss the user
|
|
515
|
+
// must re-authenticate; for an expired ACCESS credential
|
|
516
|
+
// (`credential_expired`) the store's session_error handler re-mints,
|
|
517
|
+
// clears the latch, and drives the reconnect — see
|
|
518
|
+
// BaseSyncedStore.setupWebSocketSync.
|
|
727
519
|
this.emit('disconnected', event);
|
|
728
520
|
return;
|
|
729
521
|
}
|
|
@@ -756,39 +548,77 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
756
548
|
};
|
|
757
549
|
}
|
|
758
550
|
/**
|
|
759
|
-
*
|
|
760
|
-
* (`
|
|
761
|
-
*
|
|
762
|
-
*
|
|
763
|
-
*
|
|
764
|
-
*
|
|
765
|
-
*
|
|
766
|
-
*
|
|
767
|
-
*
|
|
768
|
-
*
|
|
551
|
+
* Validate + normalize a wire delta at the receive boundary — the ONE
|
|
552
|
+
* seam every inbound delta (`delta` frame, batch element, `sync_response`
|
|
553
|
+
* replay, legacy bare frame) passes through before it is emitted,
|
|
554
|
+
* persisted to IDB, or allowed to advance any watermark.
|
|
555
|
+
*
|
|
556
|
+
* Normalization (older/deployed servers stay compatible):
|
|
557
|
+
* - `id`: the contract says `number`, but deployed servers have sent the
|
|
558
|
+
* raw Postgres BIGINT serialization — a STRING — and every downstream
|
|
559
|
+
* watermark gate (`typeof syncId === 'number'` in
|
|
560
|
+
* `Database.processDeltaBatch`, the metadata-cursor update, numeric
|
|
561
|
+
* `>=` thresholds in TransactionQueue) silently breaks on strings:
|
|
562
|
+
* acks are withheld, the resume cursor never advances, and every
|
|
563
|
+
* reconnect replays from 0. Coerce ONCE here.
|
|
564
|
+
* - `transactionId` / `createdBy`: the SERVER projection sends these as
|
|
565
|
+
* nullable (and `createdBy` as a nested ParticipantRef); the client
|
|
566
|
+
* contract types them as optional strings and never reads them.
|
|
567
|
+
* Normalize to absent instead of rejecting every real server delta.
|
|
568
|
+
*
|
|
569
|
+
* Validation: `clientSyncDeltaSchema.safeParse` — the canonical Zod wire
|
|
570
|
+
* contract. A frame that fails is DROPPED (returns `null`) with a
|
|
571
|
+
* debug-level log + observability breadcrumb; it is never applied. One
|
|
572
|
+
* parse per delta — callers must not re-parse.
|
|
769
573
|
*/
|
|
770
|
-
normalizeWireDelta(
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
574
|
+
normalizeWireDelta(raw) {
|
|
575
|
+
let candidate = raw;
|
|
576
|
+
if (isRecord(raw)) {
|
|
577
|
+
const normalized = { ...raw };
|
|
578
|
+
if (typeof normalized.id !== 'number') {
|
|
579
|
+
const coerced = Number(normalized.id);
|
|
580
|
+
normalized.id = Number.isFinite(coerced) ? coerced : 0;
|
|
581
|
+
}
|
|
582
|
+
if (normalized.transactionId === null)
|
|
583
|
+
delete normalized.transactionId;
|
|
584
|
+
if (normalized.createdBy !== undefined && typeof normalized.createdBy !== 'string') {
|
|
585
|
+
delete normalized.createdBy;
|
|
586
|
+
}
|
|
587
|
+
candidate = normalized;
|
|
588
|
+
}
|
|
589
|
+
const parsed = clientSyncDeltaSchema.safeParse(candidate);
|
|
590
|
+
if (!parsed.success) {
|
|
591
|
+
const issue = parsed.error.issues[0];
|
|
592
|
+
const detail = {
|
|
593
|
+
issue: issue ? `${issue.path.join('.')}: ${issue.message}` : 'not an object',
|
|
594
|
+
modelName: isRecord(raw) && typeof raw.modelName === 'string' ? raw.modelName : undefined,
|
|
595
|
+
actionType: isRecord(raw) && typeof raw.actionType === 'string' ? raw.actionType : undefined,
|
|
596
|
+
};
|
|
597
|
+
getContext().logger.debug('[SyncWebSocket] dropped malformed wire delta', detail);
|
|
598
|
+
getContext().observability.breadcrumb('Dropped malformed wire delta', 'sync.websocket', 'warning', detail);
|
|
599
|
+
return null;
|
|
600
|
+
}
|
|
601
|
+
return parsed.data;
|
|
775
602
|
}
|
|
776
603
|
/**
|
|
777
|
-
* Handle incoming sync delta
|
|
604
|
+
* Handle incoming sync delta (untrusted wire input — validated and
|
|
605
|
+
* normalized by {@link normalizeWireDelta}; malformed deltas are dropped).
|
|
778
606
|
*/
|
|
779
607
|
handleDelta(rawDelta) {
|
|
780
608
|
const delta = this.normalizeWireDelta(rawDelta);
|
|
609
|
+
if (!delta)
|
|
610
|
+
return;
|
|
781
611
|
getContext().logger.debug('Received delta', {
|
|
782
612
|
action: delta.actionType,
|
|
783
613
|
model: delta.modelName,
|
|
784
614
|
id: delta.modelId,
|
|
785
615
|
syncId: delta.id,
|
|
786
616
|
});
|
|
787
|
-
// DO NOT advance `this.lastSyncId` on receipt. The runtime cursor
|
|
617
|
+
// DO NOT advance `this.cursor.lastSyncId` on receipt. The runtime cursor
|
|
788
618
|
// must stay consistent with what's persisted in IDB — otherwise the
|
|
789
619
|
// next `requestIncrementalSync()` (and the connect-time handshake)
|
|
790
620
|
// sends an optimistic cursor and the server skips deltas that never
|
|
791
|
-
// landed in IDB. `this.lastSyncId` is advanced only in `sendAck()`,
|
|
621
|
+
// landed in IDB. `this.cursor.lastSyncId` is advanced only in `sendAck()`,
|
|
792
622
|
// which is gated on `BaseSyncedStore.flushPendingDeltas`'s
|
|
793
623
|
// `persistedSyncId` watermark. See Replicache's "lastMutationID
|
|
794
624
|
// read in the same transaction as the client view" rule.
|
|
@@ -801,7 +631,7 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
801
631
|
/**
|
|
802
632
|
* Send acknowledgment for received delta with version vector.
|
|
803
633
|
*
|
|
804
|
-
* This is the SOLE forward-mover of `this.lastSyncId` for live
|
|
634
|
+
* This is the SOLE forward-mover of `this.cursor.lastSyncId` for live
|
|
805
635
|
* deltas. Called by `BaseSyncedStore.flushPendingDeltas` with the
|
|
806
636
|
* `persistedSyncId` watermark — i.e. only after the deltas have
|
|
807
637
|
* actually committed to IDB. Keeping the cursor advance here (rather
|
|
@@ -814,9 +644,7 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
814
644
|
// these are what `requestIncrementalSync` and the connect handshake
|
|
815
645
|
// will send next, and what `getLastSyncId()` reports for clean-
|
|
816
646
|
// shutdown persistence.
|
|
817
|
-
|
|
818
|
-
this.lastSyncId = syncId;
|
|
819
|
-
}
|
|
647
|
+
this.cursor.ackAdvance(syncId);
|
|
820
648
|
if (this.ws?.readyState !== WebSocket.OPEN)
|
|
821
649
|
return;
|
|
822
650
|
this.send({
|
|
@@ -862,35 +690,6 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
862
690
|
}
|
|
863
691
|
}
|
|
864
692
|
}
|
|
865
|
-
/**
|
|
866
|
-
* Project the SDK's `MutationOperation[]` onto the canonical wire
|
|
867
|
-
* `CommitMessage`. This is the single serialize boundary between the SDK op
|
|
868
|
-
* type (loose `type: string`, plus an SDK-internal `options` the server never
|
|
869
|
-
* reads) and the strict wire contract. The per-field map gives compile-time
|
|
870
|
-
* drift detection (a `CommitOperation` shape change breaks here) and the lone
|
|
871
|
-
* `as` narrows the validated op `type` to the wire union — the only
|
|
872
|
-
* loosening, localized to this boundary.
|
|
873
|
-
*/
|
|
874
|
-
buildCommitFrame(operations, clientTxId, causedByTaskId, reads) {
|
|
875
|
-
const payload = {
|
|
876
|
-
operations: operations.map((op) => ({
|
|
877
|
-
type: op.type,
|
|
878
|
-
model: op.model,
|
|
879
|
-
id: op.id,
|
|
880
|
-
input: op.input,
|
|
881
|
-
transactionId: op.transactionId,
|
|
882
|
-
readAt: op.readAt,
|
|
883
|
-
onStale: op.onStale,
|
|
884
|
-
})),
|
|
885
|
-
clientTxId,
|
|
886
|
-
};
|
|
887
|
-
if (causedByTaskId)
|
|
888
|
-
payload.causedByTaskId = causedByTaskId;
|
|
889
|
-
// Batch-level read-set (STORM layer): rows/groups the batch was premised on.
|
|
890
|
-
if (reads && reads.length > 0)
|
|
891
|
-
payload.reads = [...reads];
|
|
892
|
-
return { type: 'commit', payload };
|
|
893
|
-
}
|
|
894
693
|
/**
|
|
895
694
|
* Send a `commit` mutation request over the existing WebSocket and
|
|
896
695
|
* resolve when the server's `mutation_result` frame comes back with
|
|
@@ -910,62 +709,6 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
910
709
|
* NOT auto-retry here — the caller's TransactionQueue owns retry +
|
|
911
710
|
* offline replay semantics and the SDK shouldn't duplicate that logic.
|
|
912
711
|
*/
|
|
913
|
-
/**
|
|
914
|
-
* Defensively validate the optional `notifications` array off a commit ack.
|
|
915
|
-
* Untrusted wire data — a malformed entry is dropped rather than throwing,
|
|
916
|
-
* so a bad notification never sinks an otherwise-successful commit.
|
|
917
|
-
*/
|
|
918
|
-
parseNotifications(raw) {
|
|
919
|
-
if (!Array.isArray(raw) || raw.length === 0)
|
|
920
|
-
return undefined;
|
|
921
|
-
const out = [];
|
|
922
|
-
for (const entry of raw) {
|
|
923
|
-
const parsed = staleNotificationSchema.safeParse(entry);
|
|
924
|
-
if (parsed.success)
|
|
925
|
-
out.push(parsed.data);
|
|
926
|
-
}
|
|
927
|
-
return out.length > 0 ? out : undefined;
|
|
928
|
-
}
|
|
929
|
-
/**
|
|
930
|
-
* Single instrumentation point for claim events. Every `claim_*` frame routes
|
|
931
|
-
* through here so a developer debugging a collision gets one consistent trace
|
|
932
|
-
* — a console line AND a structured capture — without each dispatch case
|
|
933
|
-
* re-deriving the row/holder shape. The wire payload is loosely typed
|
|
934
|
-
* (`Record<string, unknown>`), so this is the one place that narrows it into
|
|
935
|
-
* a {@link ClaimEvent}.
|
|
936
|
-
*/
|
|
937
|
-
recordClaim(phase, payload) {
|
|
938
|
-
const str = (v) => typeof v === 'string' ? v : undefined;
|
|
939
|
-
// Targets arrive flat ({ entityType, entityId }) or nested under `target`.
|
|
940
|
-
const target = payload.target && typeof payload.target === 'object'
|
|
941
|
-
? payload.target
|
|
942
|
-
: payload;
|
|
943
|
-
const kind = wireParticipantKindSchema.safeParse(payload.participantKind);
|
|
944
|
-
const event = {
|
|
945
|
-
phase,
|
|
946
|
-
claimId: str(payload.claimId),
|
|
947
|
-
model: str(target.entityType) ?? str(target.model),
|
|
948
|
-
id: str(target.entityId) ?? str(target.id),
|
|
949
|
-
field: str(target.field),
|
|
950
|
-
actor: str(payload.actor) ?? str(payload.heldBy),
|
|
951
|
-
participantKind: kind.success ? kind.data : undefined,
|
|
952
|
-
position: typeof payload.position === 'number' ? payload.position : undefined,
|
|
953
|
-
reason: str(payload.policyReason) ?? str(payload.reason),
|
|
954
|
-
};
|
|
955
|
-
const message = formatClaim(event);
|
|
956
|
-
// A rejection or lost lease is the collision a developer is actively
|
|
957
|
-
// debugging → warn (shows at the default log level). The routine events
|
|
958
|
-
// (acquired/queued/granted/expired) are debug-only so they never drown the
|
|
959
|
-
// console until you opt in with `new Ablo({ debug: true })`.
|
|
960
|
-
const isCollision = phase === 'rejected' || phase === 'lost';
|
|
961
|
-
const ctx = getContext();
|
|
962
|
-
if (isCollision)
|
|
963
|
-
ctx.logger.warn(message);
|
|
964
|
-
else
|
|
965
|
-
ctx.logger.debug(message);
|
|
966
|
-
ctx.observability.breadcrumb(message, 'sync.coordination', isCollision ? 'warning' : 'info');
|
|
967
|
-
ctx.observability.captureClaim(event);
|
|
968
|
-
}
|
|
969
712
|
sendCommit(operations, clientTxId, timeoutMs = 15_000, causedByTaskId, reads) {
|
|
970
713
|
if (this.ws?.readyState !== WebSocket.OPEN) {
|
|
971
714
|
return Promise.reject(this.notConnectedError('commit'));
|
|
@@ -981,7 +724,7 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
981
724
|
// an open turn — keeps the wire shape stable for sessions
|
|
982
725
|
// that don't use turns. Servers that don't know the field
|
|
983
726
|
// ignore it; newer servers stamp it onto every delta.
|
|
984
|
-
const frame =
|
|
727
|
+
const frame = buildCommitFrame(operations, clientTxId, causedByTaskId, reads);
|
|
985
728
|
this.ws.send(JSON.stringify(frame));
|
|
986
729
|
}
|
|
987
730
|
catch (error) {
|
|
@@ -1003,7 +746,7 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1003
746
|
if (this.ws?.readyState !== WebSocket.OPEN) {
|
|
1004
747
|
throw this.notConnectedError('commit');
|
|
1005
748
|
}
|
|
1006
|
-
const frame =
|
|
749
|
+
const frame = buildCommitFrame(operations, clientTxId, causedByTaskId, reads);
|
|
1007
750
|
this.ws.send(JSON.stringify(frame));
|
|
1008
751
|
}
|
|
1009
752
|
/**
|
|
@@ -1281,7 +1024,7 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1281
1024
|
disconnect() {
|
|
1282
1025
|
this.isManualClose = true;
|
|
1283
1026
|
this.stopCatchupInterval();
|
|
1284
|
-
this.
|
|
1027
|
+
this.heartbeat.stop();
|
|
1285
1028
|
if (this.reconnectTimer) {
|
|
1286
1029
|
clearTimeout(this.reconnectTimer);
|
|
1287
1030
|
this.reconnectTimer = null;
|
|
@@ -1291,67 +1034,6 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1291
1034
|
this.ws = null;
|
|
1292
1035
|
}
|
|
1293
1036
|
}
|
|
1294
|
-
/**
|
|
1295
|
-
* Application-level heartbeat. Every `HEARTBEAT_INTERVAL_MS` while
|
|
1296
|
-
* `OPEN`, send `{ type: 'ping' }` and arm a `HEARTBEAT_TIMEOUT_MS`
|
|
1297
|
-
* watchdog. Any inbound frame (handled in `onmessage`) clears the
|
|
1298
|
-
* watchdog. If the watchdog fires, we treat the connection as
|
|
1299
|
-
* zombie and force-close it — `onclose` then triggers the existing
|
|
1300
|
-
* reconnect path.
|
|
1301
|
-
*
|
|
1302
|
-
* Why both sides need this:
|
|
1303
|
-
* - The server sends RFC 6455 protocol pings via `ws.ping()` every
|
|
1304
|
-
* 30s. Browsers auto-respond with a pong but DO NOT expose either
|
|
1305
|
-
* frame to JavaScript, so the client is blind to its own keepalive.
|
|
1306
|
-
* - On a half-open TCP (laptop wake, NAT timeout, mobile handoff)
|
|
1307
|
-
* the browser may keep `readyState === OPEN` for minutes before
|
|
1308
|
-
* the OS surfaces the broken connection. App-level traffic is
|
|
1309
|
-
* the only signal we can observe.
|
|
1310
|
-
*/
|
|
1311
|
-
startHeartbeat() {
|
|
1312
|
-
this.stopHeartbeat();
|
|
1313
|
-
this.heartbeatTimer = setInterval(() => {
|
|
1314
|
-
if (this.ws?.readyState !== WebSocket.OPEN)
|
|
1315
|
-
return;
|
|
1316
|
-
// Send the ping. If `send` throws, the socket is already dead —
|
|
1317
|
-
// force-close so onclose triggers the reconnect cycle.
|
|
1318
|
-
try {
|
|
1319
|
-
this.ws.send(JSON.stringify({ type: 'ping' }));
|
|
1320
|
-
}
|
|
1321
|
-
catch (err) {
|
|
1322
|
-
getContext().observability.captureWebSocketError({
|
|
1323
|
-
context: 'heartbeat-send-failed',
|
|
1324
|
-
error: err instanceof Error ? err.message : String(err),
|
|
1325
|
-
});
|
|
1326
|
-
this.forceClose('heartbeat-send-failed');
|
|
1327
|
-
return;
|
|
1328
|
-
}
|
|
1329
|
-
// Arm the timeout. ANY inbound message clears it (see onmessage).
|
|
1330
|
-
// We don't require an explicit `pong` — a delta or any other frame
|
|
1331
|
-
// is equally good proof-of-life.
|
|
1332
|
-
if (this.heartbeatTimeoutTimer)
|
|
1333
|
-
clearTimeout(this.heartbeatTimeoutTimer);
|
|
1334
|
-
this.heartbeatTimeoutTimer = setTimeout(() => {
|
|
1335
|
-
getContext().observability.captureWebSocketError({
|
|
1336
|
-
context: 'heartbeat-timeout',
|
|
1337
|
-
});
|
|
1338
|
-
this.forceClose('heartbeat-timeout');
|
|
1339
|
-
}, SyncWebSocket.HEARTBEAT_TIMEOUT_MS);
|
|
1340
|
-
}, SyncWebSocket.HEARTBEAT_INTERVAL_MS);
|
|
1341
|
-
}
|
|
1342
|
-
stopHeartbeat() {
|
|
1343
|
-
if (this.heartbeatTimer) {
|
|
1344
|
-
clearInterval(this.heartbeatTimer);
|
|
1345
|
-
this.heartbeatTimer = null;
|
|
1346
|
-
}
|
|
1347
|
-
this.clearHeartbeatTimeout();
|
|
1348
|
-
}
|
|
1349
|
-
clearHeartbeatTimeout() {
|
|
1350
|
-
if (this.heartbeatTimeoutTimer) {
|
|
1351
|
-
clearTimeout(this.heartbeatTimeoutTimer);
|
|
1352
|
-
this.heartbeatTimeoutTimer = null;
|
|
1353
|
-
}
|
|
1354
|
-
}
|
|
1355
1037
|
/**
|
|
1356
1038
|
* Force-close the socket from the client side using a private 4xxx
|
|
1357
1039
|
* code. Callers expect `onclose` to fire; that handler runs the
|
|
@@ -1416,11 +1098,19 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1416
1098
|
*/
|
|
1417
1099
|
notConnectedError(action) {
|
|
1418
1100
|
const d = this.getConnectionDiagnostics();
|
|
1419
|
-
|
|
1101
|
+
// Session-latched is NOT a transient transport hiccup: reconnection is
|
|
1102
|
+
// suppressed until re-auth (or the store's credential re-mint clears the
|
|
1103
|
+
// latch), so retrying can never succeed. Reject with the PERMANENT
|
|
1104
|
+
// session type — `isPermanentError` surfaces it to the caller as
|
|
1105
|
+
// "re-authenticate" instead of parking the write for a reconnect that
|
|
1106
|
+
// will never happen (the old `ws_not_ready` retry-forever hazard).
|
|
1420
1107
|
if (d.sessionErrorDetected) {
|
|
1421
|
-
|
|
1108
|
+
return Object.assign(new SyncSessionError(`SyncWebSocket not connected — cannot send ${action}: session expired` +
|
|
1109
|
+
(d.lastCloseReason ? ` (${d.lastCloseReason})` : '') +
|
|
1110
|
+
'; re-authenticate'), { diagnostics: d });
|
|
1422
1111
|
}
|
|
1423
|
-
|
|
1112
|
+
let detail;
|
|
1113
|
+
if (d.isManualClose) {
|
|
1424
1114
|
detail = 'manual_close';
|
|
1425
1115
|
}
|
|
1426
1116
|
else if (d.isConnecting) {
|
|
@@ -1455,47 +1145,31 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1455
1145
|
getSyncGroups() {
|
|
1456
1146
|
return this.options.syncGroups;
|
|
1457
1147
|
}
|
|
1148
|
+
// Cursor accessors — thin delegates; the state + semantics live in
|
|
1149
|
+
// sync/syncCursor.ts (SyncCursor).
|
|
1458
1150
|
/**
|
|
1459
1151
|
* Update last sync ID (for persistence)
|
|
1460
1152
|
*/
|
|
1461
1153
|
setLastSyncId(syncId) {
|
|
1462
|
-
this.
|
|
1463
|
-
}
|
|
1464
|
-
/**
|
|
1465
|
-
* Get current version vector
|
|
1466
|
-
*/
|
|
1467
|
-
getVersionVector() {
|
|
1468
|
-
return { ...this.versionVector };
|
|
1469
|
-
}
|
|
1470
|
-
/**
|
|
1471
|
-
* Update version vector for specific entity type
|
|
1472
|
-
*/
|
|
1473
|
-
updateVersionVector(entityType, version) {
|
|
1474
|
-
this.versionVector[entityType] = Math.max(this.versionVector[entityType] || 0, version);
|
|
1475
|
-
}
|
|
1476
|
-
/**
|
|
1477
|
-
* Set version vector (for initialization)
|
|
1478
|
-
*/
|
|
1479
|
-
setVersionVector(versions) {
|
|
1480
|
-
this.versionVector = { ...versions };
|
|
1154
|
+
this.cursor.setLastSyncId(syncId);
|
|
1481
1155
|
}
|
|
1482
1156
|
/**
|
|
1483
1157
|
* Update sync cursor (for incremental sync)
|
|
1484
1158
|
*/
|
|
1485
1159
|
setSyncCursor(cursor) {
|
|
1486
|
-
this.
|
|
1160
|
+
this.cursor.setSyncCursor(cursor);
|
|
1487
1161
|
}
|
|
1488
1162
|
/**
|
|
1489
1163
|
* Get current sync cursor
|
|
1490
1164
|
*/
|
|
1491
1165
|
getSyncCursor() {
|
|
1492
|
-
return this.
|
|
1166
|
+
return this.cursor.getSyncCursor();
|
|
1493
1167
|
}
|
|
1494
1168
|
/**
|
|
1495
1169
|
* Get the highest syncId seen this session (for persistence on clean shutdown)
|
|
1496
1170
|
*/
|
|
1497
1171
|
getLastSyncId() {
|
|
1498
|
-
return this.
|
|
1172
|
+
return this.cursor.getLastSyncId();
|
|
1499
1173
|
}
|
|
1500
1174
|
/**
|
|
1501
1175
|
* Linear-style incremental sync request
|
|
@@ -1516,9 +1190,12 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1516
1190
|
this.send({
|
|
1517
1191
|
type: 'sync_request',
|
|
1518
1192
|
payload: {
|
|
1519
|
-
cursor: this.syncCursor, // Always send lastSyncId to ensure server uses client's current position
|
|
1520
|
-
lastSyncId: this.lastSyncId,
|
|
1193
|
+
cursor: this.cursor.syncCursor, // Always send lastSyncId to ensure server uses client's current position
|
|
1194
|
+
lastSyncId: this.cursor.lastSyncId,
|
|
1521
1195
|
capabilities: capsArr,
|
|
1196
|
+
// Protocol handshake: the server rejects an out-of-range version with
|
|
1197
|
+
// WS close 4010 before serving any deltas (wire/protocolVersion.ts).
|
|
1198
|
+
protocolVersion: PROTOCOL_VERSION,
|
|
1522
1199
|
},
|
|
1523
1200
|
});
|
|
1524
1201
|
}
|
|
@@ -1541,9 +1218,16 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1541
1218
|
});
|
|
1542
1219
|
}
|
|
1543
1220
|
/**
|
|
1544
|
-
* Handle sync response from server
|
|
1221
|
+
* Handle sync response from server. Untrusted wire input — the envelope
|
|
1222
|
+
* fields are narrowed defensively and every delta is validated through
|
|
1223
|
+
* {@link normalizeWireDelta} (exactly once each; malformed ones drop out
|
|
1224
|
+
* of the batch).
|
|
1545
1225
|
*/
|
|
1546
|
-
handleSyncResponse(
|
|
1226
|
+
handleSyncResponse(rawPayload) {
|
|
1227
|
+
const payload = isRecord(rawPayload) ? rawPayload : {};
|
|
1228
|
+
const rawDeltas = Array.isArray(payload.deltas)
|
|
1229
|
+
? payload.deltas
|
|
1230
|
+
: null;
|
|
1547
1231
|
// Cursor reconciliation — Linear-style handshake. The server stamps
|
|
1548
1232
|
// its authoritative `currentSyncId` on every sync_response. If our
|
|
1549
1233
|
// local cursor is AHEAD of the server, our local view has somehow
|
|
@@ -1563,24 +1247,30 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1563
1247
|
// empty-delta responses eliminates this benign false positive while
|
|
1564
1248
|
// still catching the real corruption case (server head < local AND
|
|
1565
1249
|
// server has nothing new to send).
|
|
1566
|
-
const hasDeltas =
|
|
1250
|
+
const hasDeltas = rawDeltas !== null && rawDeltas.length > 0;
|
|
1567
1251
|
if (!hasDeltas && typeof payload.currentSyncId === 'number') {
|
|
1568
1252
|
const serverHead = payload.currentSyncId;
|
|
1569
|
-
if (serverHead < this.lastSyncId) {
|
|
1253
|
+
if (serverHead < this.cursor.lastSyncId) {
|
|
1570
1254
|
getContext().logger.debug('[SyncWebSocket] local cursor ahead of server head — resetting and resyncing', {
|
|
1571
|
-
local: this.lastSyncId,
|
|
1255
|
+
local: this.cursor.lastSyncId,
|
|
1572
1256
|
server: serverHead,
|
|
1573
|
-
drift: this.lastSyncId - serverHead,
|
|
1257
|
+
drift: this.cursor.lastSyncId - serverHead,
|
|
1574
1258
|
});
|
|
1575
|
-
getContext().observability.breadcrumb('Local sync cursor diverged from server — reset', 'sync.websocket', 'warning', { local: this.lastSyncId, server: serverHead });
|
|
1576
|
-
this.lastSyncId = serverHead;
|
|
1259
|
+
getContext().observability.breadcrumb('Local sync cursor diverged from server — reset', 'sync.websocket', 'warning', { local: this.cursor.lastSyncId, server: serverHead });
|
|
1260
|
+
this.cursor.lastSyncId = serverHead;
|
|
1577
1261
|
// Fire a follow-up incremental sync to re-deliver anything we
|
|
1578
1262
|
// were missing. Fire-and-forget — the next response will go
|
|
1579
1263
|
// through this same path. The infinite-loop concern is bounded
|
|
1580
|
-
// by the `serverHead < this.lastSyncId` strict-less check: once
|
|
1264
|
+
// by the `serverHead < this.cursor.lastSyncId` strict-less check: once
|
|
1581
1265
|
// we've reset to `serverHead`, the next response with the same
|
|
1582
|
-
// (or higher) `currentSyncId` won't re-enter this branch.
|
|
1583
|
-
|
|
1266
|
+
// (or higher) `currentSyncId` won't re-enter this branch. A `.catch`
|
|
1267
|
+
// (not `void`) so a failed send logs instead of surfacing as an
|
|
1268
|
+
// unhandled rejection.
|
|
1269
|
+
this.requestIncrementalSync().catch((e) => {
|
|
1270
|
+
getContext().observability.breadcrumb('Post-cursor-reset sync request failed', 'sync.websocket', 'warning', {
|
|
1271
|
+
error: e instanceof Error ? e.message : String(e),
|
|
1272
|
+
});
|
|
1273
|
+
});
|
|
1584
1274
|
}
|
|
1585
1275
|
}
|
|
1586
1276
|
if (payload.requiresBootstrap) {
|
|
@@ -1588,49 +1278,53 @@ export class SyncWebSocket extends EventEmitter {
|
|
|
1588
1278
|
return;
|
|
1589
1279
|
}
|
|
1590
1280
|
// Process incremental deltas
|
|
1591
|
-
if (
|
|
1281
|
+
if (rawDeltas) {
|
|
1592
1282
|
// Process all deltas from sync response - store handles idempotency.
|
|
1593
|
-
// Same receive-boundary normalization as handleDelta —
|
|
1594
|
-
// replays from older servers carry string ids too
|
|
1595
|
-
|
|
1283
|
+
// Same receive-boundary validation + normalization as handleDelta —
|
|
1284
|
+
// catch-up replays from older servers carry string ids too, and
|
|
1285
|
+
// malformed deltas drop out of the batch instead of being applied.
|
|
1286
|
+
const newDeltas = [];
|
|
1287
|
+
for (const d of rawDeltas) {
|
|
1288
|
+
const delta = this.normalizeWireDelta(d);
|
|
1289
|
+
if (delta)
|
|
1290
|
+
newDeltas.push(delta);
|
|
1291
|
+
}
|
|
1596
1292
|
if (newDeltas.length > 0) {
|
|
1597
|
-
// DO NOT pre-advance `this.lastSyncId` here. Same reasoning as
|
|
1293
|
+
// DO NOT pre-advance `this.cursor.lastSyncId` here. Same reasoning as
|
|
1598
1294
|
// `handleDelta`: the runtime cursor must stay consistent with
|
|
1599
1295
|
// IDB. The delta_batch event routes through
|
|
1600
1296
|
// `BaseSyncedStore.processDeltaWithBatching` →
|
|
1601
1297
|
// `flushPendingDeltas`, which calls `acknowledge()` with the
|
|
1602
1298
|
// honest `persistedSyncId` once IDB commits. That ack is what
|
|
1603
|
-
// moves `this.lastSyncId` forward.
|
|
1299
|
+
// moves `this.cursor.lastSyncId` forward.
|
|
1604
1300
|
// Emit ALL deltas as a single batch event
|
|
1605
1301
|
this.emit('delta_batch', newDeltas);
|
|
1606
1302
|
}
|
|
1607
1303
|
}
|
|
1608
|
-
// Update
|
|
1609
|
-
|
|
1610
|
-
|
|
1304
|
+
// Update cursor. (`newVersions` from pre-cutover servers is ignored —
|
|
1305
|
+
// the version vector was removed in W4a; `sync_id` is the causality token.)
|
|
1306
|
+
if (typeof payload.newCursor === 'string' && payload.newCursor) {
|
|
1307
|
+
this.cursor.syncCursor = payload.newCursor;
|
|
1611
1308
|
}
|
|
1612
|
-
else if (payload.cursor) {
|
|
1613
|
-
this.syncCursor = payload.cursor;
|
|
1614
|
-
}
|
|
1615
|
-
if (payload.newVersions) {
|
|
1616
|
-
Object.assign(this.versionVector, payload.newVersions);
|
|
1309
|
+
else if (typeof payload.cursor === 'string' && payload.cursor) {
|
|
1310
|
+
this.cursor.syncCursor = payload.cursor;
|
|
1617
1311
|
}
|
|
1618
1312
|
}
|
|
1619
1313
|
/**
|
|
1620
1314
|
* Handle bootstrap response from server
|
|
1621
1315
|
*/
|
|
1622
1316
|
handleBootstrapResponse(payload) {
|
|
1623
|
-
// Emit bootstrap data for processing
|
|
1317
|
+
// Emit bootstrap data for processing. (A `version` field from
|
|
1318
|
+
// pre-cutover servers is ignored — version vector removed in W4a.)
|
|
1319
|
+
// Field-wise typeof guards mirror the cursor handling above: the frame
|
|
1320
|
+
// is server-produced, so coercion only bites on malformed frames.
|
|
1321
|
+
const p = (payload && typeof payload === 'object' ? payload : {});
|
|
1624
1322
|
this.emit('bootstrap_data', {
|
|
1625
|
-
entityType:
|
|
1626
|
-
data:
|
|
1627
|
-
isComplete:
|
|
1628
|
-
cursor:
|
|
1323
|
+
entityType: typeof p.entityType === 'string' ? p.entityType : '',
|
|
1324
|
+
data: p.data,
|
|
1325
|
+
isComplete: p.isComplete === true,
|
|
1326
|
+
cursor: typeof p.cursor === 'string' ? p.cursor : undefined,
|
|
1629
1327
|
});
|
|
1630
|
-
// Update version vector if provided
|
|
1631
|
-
if (payload.version && payload.entityType) {
|
|
1632
|
-
this.updateVersionVector(payload.entityType.toLowerCase(), payload.version);
|
|
1633
|
-
}
|
|
1634
1328
|
}
|
|
1635
1329
|
/**
|
|
1636
1330
|
* Handle presence update from server. The wire frame's payload is
|