@spooky-sync/core 0.0.1-canary.24 → 0.0.1-canary.240
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +58 -0
- package/coverage/lcov-report/base.css +224 -0
- package/coverage/lcov-report/block-navigation.js +87 -0
- package/coverage/lcov-report/favicon.png +0 -0
- package/coverage/lcov-report/index.html +476 -0
- package/coverage/lcov-report/prettify.css +1 -0
- package/coverage/lcov-report/prettify.js +2 -0
- package/coverage/lcov-report/sort-arrow-sprite.png +0 -0
- package/coverage/lcov-report/sorter.js +210 -0
- package/coverage/lcov-report/src/boot/auth-flip.saga.ts.html +271 -0
- package/coverage/lcov-report/src/boot/boot.saga.ts.html +469 -0
- package/coverage/lcov-report/src/boot/bucket-switch.saga.ts.html +430 -0
- package/coverage/lcov-report/src/boot/index.html +161 -0
- package/coverage/lcov-report/src/boot/preload.saga.ts.html +178 -0
- package/coverage/lcov-report/src/client/bucket-handle.ts.html +766 -0
- package/coverage/lcov-report/src/client/index.html +161 -0
- package/coverage/lcov-report/src/client/router.ts.html +367 -0
- package/coverage/lcov-report/src/client/runtime.ts.html +940 -0
- package/coverage/lcov-report/src/client/sp00ky-client.ts.html +1531 -0
- package/coverage/lcov-report/src/events/index.html +116 -0
- package/coverage/lcov-report/src/events/index.ts.html +877 -0
- package/coverage/lcov-report/src/index.html +146 -0
- package/coverage/lcov-report/src/index.ts.html +202 -0
- package/coverage/lcov-report/src/kernel/constants.ts.html +247 -0
- package/coverage/lcov-report/src/kernel/effects.ts.html +562 -0
- package/coverage/lcov-report/src/kernel/events.ts.html +250 -0
- package/coverage/lcov-report/src/kernel/index.html +176 -0
- package/coverage/lcov-report/src/kernel/interpreter.ts.html +514 -0
- package/coverage/lcov-report/src/kernel/saga.ts.html +304 -0
- package/coverage/lcov-report/src/modules/app-release/index.html +116 -0
- package/coverage/lcov-report/src/modules/app-release/index.ts.html +712 -0
- package/coverage/lcov-report/src/modules/auth/events/index.html +116 -0
- package/coverage/lcov-report/src/modules/auth/events/index.ts.html +142 -0
- package/coverage/lcov-report/src/modules/auth/index.html +116 -0
- package/coverage/lcov-report/src/modules/auth/index.ts.html +1225 -0
- package/coverage/lcov-report/src/modules/crdt/crdt-field.ts.html +967 -0
- package/coverage/lcov-report/src/modules/crdt/index.html +146 -0
- package/coverage/lcov-report/src/modules/crdt/index.ts.html +1474 -0
- package/coverage/lcov-report/src/modules/crdt/loro-loader.ts.html +160 -0
- package/coverage/lcov-report/src/modules/devtools/flags.ts.html +1132 -0
- package/coverage/lcov-report/src/modules/devtools/index.html +161 -0
- package/coverage/lcov-report/src/modules/devtools/index.ts.html +2413 -0
- package/coverage/lcov-report/src/modules/devtools/storage-info.ts.html +589 -0
- package/coverage/lcov-report/src/modules/devtools/versions.ts.html +415 -0
- package/coverage/lcov-report/src/modules/feature-flag/index.html +116 -0
- package/coverage/lcov-report/src/modules/feature-flag/index.ts.html +1033 -0
- package/coverage/lcov-report/src/modules/index.html +116 -0
- package/coverage/lcov-report/src/modules/ref-tables.ts.html +349 -0
- package/coverage/lcov-report/src/mutation/index.html +191 -0
- package/coverage/lcov-report/src/mutation/jobs.ts.html +208 -0
- package/coverage/lcov-report/src/mutation/mutation-id.ts.html +190 -0
- package/coverage/lcov-report/src/mutation/push.saga.ts.html +730 -0
- package/coverage/lcov-report/src/mutation/rows.ts.html +1216 -0
- package/coverage/lcov-report/src/mutation/tray.saga.ts.html +229 -0
- package/coverage/lcov-report/src/mutation/write.saga.ts.html +583 -0
- package/coverage/lcov-report/src/otel/index.html +116 -0
- package/coverage/lcov-report/src/otel/index.ts.html +466 -0
- package/coverage/lcov-report/src/query/env.ts.html +253 -0
- package/coverage/lcov-report/src/query/fetch.saga.ts.html +412 -0
- package/coverage/lcov-report/src/query/hash.ts.html +139 -0
- package/coverage/lcov-report/src/query/index.html +281 -0
- package/coverage/lcov-report/src/query/lifecycle.saga.ts.html +388 -0
- package/coverage/lcov-report/src/query/materialize.saga.ts.html +310 -0
- package/coverage/lcov-report/src/query/materialize.ts.html +217 -0
- package/coverage/lcov-report/src/query/membership.saga.ts.html +709 -0
- package/coverage/lcov-report/src/query/membership.ts.html +559 -0
- package/coverage/lcov-report/src/query/register.saga.ts.html +655 -0
- package/coverage/lcov-report/src/query/render-set.ts.html +265 -0
- package/coverage/lcov-report/src/query/sql.ts.html +322 -0
- package/coverage/lcov-report/src/query/window-query.ts.html +670 -0
- package/coverage/lcov-report/src/services/blobs/blob-cache.ts.html +1894 -0
- package/coverage/lcov-report/src/services/blobs/blob-manifest.ts.html +766 -0
- package/coverage/lcov-report/src/services/blobs/blob-store.ts.html +1162 -0
- package/coverage/lcov-report/src/services/blobs/index.html +161 -0
- package/coverage/lcov-report/src/services/blobs/index.ts.html +295 -0
- package/coverage/lcov-report/src/services/database/cache-engine.ts.html +664 -0
- package/coverage/lcov-report/src/services/database/connection-supervisor.ts.html +1330 -0
- package/coverage/lcov-report/src/services/database/database.ts.html +658 -0
- package/coverage/lcov-report/src/services/database/engine-factory.ts.html +184 -0
- package/coverage/lcov-report/src/services/database/errors.ts.html +187 -0
- package/coverage/lcov-report/src/services/database/events/index.html +116 -0
- package/coverage/lcov-report/src/services/database/events/index.ts.html +184 -0
- package/coverage/lcov-report/src/services/database/index.html +386 -0
- package/coverage/lcov-report/src/services/database/index.ts.html +121 -0
- package/coverage/lcov-report/src/services/database/local-migrator.ts.html +703 -0
- package/coverage/lcov-report/src/services/database/local.ts.html +1633 -0
- package/coverage/lcov-report/src/services/database/plan-render.ts.html +409 -0
- package/coverage/lcov-report/src/services/database/relation-resolver.ts.html +682 -0
- package/coverage/lcov-report/src/services/database/remote.ts.html +1036 -0
- package/coverage/lcov-report/src/services/database/sqlite-cache-engine.ts.html +4132 -0
- package/coverage/lcov-report/src/services/database/sqlite-lock-verify.ts.html +220 -0
- package/coverage/lcov-report/src/services/database/sqlite-open.ts.html +577 -0
- package/coverage/lcov-report/src/services/database/sqlite-plan-sql.ts.html +499 -0
- package/coverage/lcov-report/src/services/database/sqlite-select.ts.html +478 -0
- package/coverage/lcov-report/src/services/database/sqlite-transport.ts.html +757 -0
- package/coverage/lcov-report/src/services/database/surql-translate.ts.html +1333 -0
- package/coverage/lcov-report/src/services/database/surreal-cache-engine.ts.html +568 -0
- package/coverage/lcov-report/src/services/logger/index.html +116 -0
- package/coverage/lcov-report/src/services/logger/index.ts.html +154 -0
- package/coverage/lcov-report/src/services/persistence/index.html +146 -0
- package/coverage/lcov-report/src/services/persistence/localstorage.ts.html +163 -0
- package/coverage/lcov-report/src/services/persistence/resilient.ts.html +208 -0
- package/coverage/lcov-report/src/services/persistence/surrealdb.ts.html +271 -0
- package/coverage/lcov-report/src/services/stream-processor/index.html +146 -0
- package/coverage/lcov-report/src/services/stream-processor/index.ts.html +3313 -0
- package/coverage/lcov-report/src/services/stream-processor/permissions.ts.html +244 -0
- package/coverage/lcov-report/src/services/stream-processor/wasm-types.ts.html +346 -0
- package/coverage/lcov-report/src/services/tabs/broker-client.ts.html +934 -0
- package/coverage/lcov-report/src/services/tabs/coordinator.ts.html +1984 -0
- package/coverage/lcov-report/src/services/tabs/index.html +191 -0
- package/coverage/lcov-report/src/services/tabs/leader-locks.ts.html +310 -0
- package/coverage/lcov-report/src/services/tabs/protocol.ts.html +871 -0
- package/coverage/lcov-report/src/services/tabs/support.ts.html +193 -0
- package/coverage/lcov-report/src/services/tabs/tabs-broker-worker.ts.html +2005 -0
- package/coverage/lcov-report/src/sp00ky.ts.html +100 -0
- package/coverage/lcov-report/src/state/client-state.ts.html +637 -0
- package/coverage/lcov-report/src/state/index.html +161 -0
- package/coverage/lcov-report/src/state/lifecycle.ts.html +343 -0
- package/coverage/lcov-report/src/state/reducers.ts.html +1471 -0
- package/coverage/lcov-report/src/state/selectors.ts.html +637 -0
- package/coverage/lcov-report/src/sync/connection.saga.ts.html +343 -0
- package/coverage/lcov-report/src/sync/index.html +176 -0
- package/coverage/lcov-report/src/sync/live.saga.ts.html +232 -0
- package/coverage/lcov-report/src/sync/policy.ts.html +1606 -0
- package/coverage/lcov-report/src/sync/poll.saga.ts.html +193 -0
- package/coverage/lcov-report/src/sync/tabs.saga.ts.html +361 -0
- package/coverage/lcov-report/src/types.ts.html +2419 -0
- package/coverage/lcov-report/src/utils/blurhash.ts.html +355 -0
- package/coverage/lcov-report/src/utils/error-classification.ts.html +244 -0
- package/coverage/lcov-report/src/utils/index.html +191 -0
- package/coverage/lcov-report/src/utils/index.ts.html +820 -0
- package/coverage/lcov-report/src/utils/parser.ts.html +313 -0
- package/coverage/lcov-report/src/utils/semver.ts.html +175 -0
- package/coverage/lcov-report/src/utils/surql.ts.html +667 -0
- package/coverage/lcov.info +19079 -0
- package/dist/index.d.ts +2946 -140
- package/dist/index.js +13424 -2356
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +322 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +472 -0
- package/dist/types.d.ts +769 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +36 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/auth.local-first.test.ts +141 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +138 -25
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +163 -43
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.activity.test.ts +109 -0
- package/src/modules/data/data.authority.test.ts +122 -0
- package/src/modules/data/data.heartbeat-wake.test.ts +115 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +623 -0
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.pending-ids.test.ts +199 -0
- package/src/modules/data/data.rebind.test.ts +170 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +235 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +2031 -150
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +451 -46
- package/src/modules/devtools/notify-throttle.test.ts +154 -0
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +170 -86
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +228 -0
- package/src/modules/sync/queue/queue-down.ts +108 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +237 -0
- package/src/modules/sync/scheduler.ts +215 -13
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +167 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.list-ref-meta.test.ts +193 -0
- package/src/modules/sync/sync.live-removal.test.ts +289 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.rematerialize-after-fetch.test.ts +93 -0
- package/src/modules/sync/sync.settled-poll.test.ts +58 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +2350 -133
- package/src/modules/sync/utils.test.ts +322 -2
- package/src/modules/sync/utils.ts +286 -16
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +193 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +140 -0
- package/src/services/database/database.ts +130 -62
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +484 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +467 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +222 -15
- package/src/services/database/sqlite-cache-engine.test.ts +616 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1358 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +138 -0
- package/src/services/database/sqlite-projection.test.ts +99 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +161 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +796 -84
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +59 -3
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +327 -0
- package/src/services/tabs/coordinator.test.ts +365 -0
- package/src/services/tabs/coordinator.ts +633 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +258 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +640 -0
- package/src/sp00ky.auth-order.test.ts +112 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.local-first.test.ts +60 -0
- package/src/sp00ky.ts +1854 -0
- package/src/types.ts +548 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +54 -0
- package/src/utils/error-classification.ts +25 -0
- package/src/utils/index.ts +79 -13
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +32 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1,111 +1,1653 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import type {
|
|
2
|
+
ConnectionSupervisor,
|
|
3
|
+
LocalStore,
|
|
4
|
+
RemoteDatabaseService,
|
|
5
|
+
} from '../../services/database/index';
|
|
6
|
+
import type {
|
|
7
|
+
ConnectionState,
|
|
8
|
+
RecordVersionArray,
|
|
9
|
+
RecordVersionDiff,
|
|
10
|
+
SyncHealth,
|
|
11
|
+
SyncHealthStatus,
|
|
12
|
+
ServerViewMeta,
|
|
13
|
+
MembershipOutcome,
|
|
14
|
+
} from '../../types';
|
|
3
15
|
import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
|
|
4
|
-
import { Logger } from '../../services/logger/index';
|
|
5
|
-
import { DownEvent,
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
16
|
+
import type { Logger } from '../../services/logger/index';
|
|
17
|
+
import type { DownEvent, UpEvent } from './queue/index';
|
|
18
|
+
import { DownQueue, UpQueue } from './queue/index';
|
|
19
|
+
import type { RecordId, Uuid } from 'surrealdb';
|
|
20
|
+
import {
|
|
21
|
+
applyRecordVersionDiff,
|
|
22
|
+
ArraySyncer,
|
|
23
|
+
buildListRefBatchSelect,
|
|
24
|
+
buildListRefSelect,
|
|
25
|
+
buildQueryRowCountBatchSelect,
|
|
26
|
+
buildQueryRowCountSelect,
|
|
27
|
+
buildSubqueryListRefSelect,
|
|
28
|
+
createDiffFromDbOp,
|
|
29
|
+
diffRecordVersionArray,
|
|
30
|
+
listRefPollDelayMs,
|
|
31
|
+
planListRefPollChunks,
|
|
32
|
+
recordVersionArraysEqual,
|
|
33
|
+
resolveListRefPollInterval,
|
|
34
|
+
} from './utils';
|
|
8
35
|
import { SyncEngine } from './engine';
|
|
36
|
+
|
|
37
|
+
/** One query's server-side `_00_list_ref` state as read by the poll. */
|
|
38
|
+
interface ListRefSnapshot {
|
|
39
|
+
primary: RecordVersionArray;
|
|
40
|
+
subquery: RecordVersionArray;
|
|
41
|
+
/** `_00_query.rowCount`; `null` when the row was not readable. */
|
|
42
|
+
rowCount: number | null;
|
|
43
|
+
/** The `_00_query` row's presence, `rowCount` and publish `state`. */
|
|
44
|
+
meta: ServerViewMeta;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The `{ rowCount, state }` object the row-count selects return per row. */
|
|
48
|
+
interface QueryMetaRow {
|
|
49
|
+
id?: RecordId<string>;
|
|
50
|
+
rowCount?: number | null;
|
|
51
|
+
state?: string | null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Fold a row-count select result into {@link ServerViewMeta}. */
|
|
55
|
+
function metaFromRow(row: QueryMetaRow | null | undefined): ServerViewMeta {
|
|
56
|
+
if (!row || typeof row !== 'object') return { present: false, rowCount: null, state: null };
|
|
57
|
+
const state = row.state === 'materializing' || row.state === 'ready' ? row.state : null;
|
|
58
|
+
return {
|
|
59
|
+
present: true,
|
|
60
|
+
rowCount: typeof row.rowCount === 'number' ? row.rowCount : null,
|
|
61
|
+
state,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Re-read ladder for a registration that answered while the SSP was still
|
|
67
|
+
* publishing (`state = 'materializing'`): the flusher window is ~100ms, so the
|
|
68
|
+
* first rung usually lands it; the rest cover a slow write without waiting
|
|
69
|
+
* for a backed-off poll tick.
|
|
70
|
+
*/
|
|
71
|
+
const MATERIALIZING_REFETCH_LADDER_MS: readonly number[] = [150, 400, 1000];
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* How long LIVE `_00_list_ref` DELETEs for one query are held before they are
|
|
75
|
+
* verified upstream as a batch. A view teardown (sweep, reset, republish)
|
|
76
|
+
* arrives as one DELETE per edge within a few ms; a single peer delete costs
|
|
77
|
+
* this plus one round trip.
|
|
78
|
+
*/
|
|
79
|
+
const LIVE_REMOVAL_COALESCE_MS = 100;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Collapse duplicate `(id, version)` pairs to one per id, keeping the highest
|
|
83
|
+
* version. `_00_list_ref` has no unique index on `(in, out)`, and a full
|
|
84
|
+
* republish over edges that were never cleaned up (a scheduler restart
|
|
85
|
+
* orphans them) used to leave two rows per record; the client must render
|
|
86
|
+
* each record once regardless.
|
|
87
|
+
*/
|
|
88
|
+
function dedupeRecordVersions(pairs: RecordVersionArray): RecordVersionArray {
|
|
89
|
+
if (pairs.length < 2) return pairs;
|
|
90
|
+
const best = new Map<string, number>();
|
|
91
|
+
for (const [id, version] of pairs) {
|
|
92
|
+
const prev = best.get(id);
|
|
93
|
+
if (prev === undefined || version > prev) best.set(id, version);
|
|
94
|
+
}
|
|
95
|
+
if (best.size === pairs.length) return pairs;
|
|
96
|
+
return Array.from(best.entries());
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
interface ListRefEdgeRow {
|
|
100
|
+
in: RecordId<string>;
|
|
101
|
+
out: RecordId<string>;
|
|
102
|
+
version: number;
|
|
103
|
+
parent?: unknown;
|
|
104
|
+
}
|
|
9
105
|
import { SyncScheduler } from './scheduler';
|
|
10
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
11
|
-
import { CacheModule } from '../cache/index';
|
|
12
|
-
import { DataModule } from '../data/index';
|
|
13
|
-
import {
|
|
106
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
107
|
+
import type { CacheModule } from '../cache/index';
|
|
108
|
+
import type { DataModule } from '../data/index';
|
|
109
|
+
import {
|
|
110
|
+
classifySyncError,
|
|
111
|
+
encodeRecordId,
|
|
112
|
+
extractIdPart,
|
|
113
|
+
extractTablePart,
|
|
114
|
+
surql,
|
|
115
|
+
withTimeout,
|
|
116
|
+
} from '../../utils/index';
|
|
117
|
+
import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
|
|
118
|
+
import { mutationOwnerTabId } from '../data/mutation-id';
|
|
119
|
+
import type { LeaderSyncHub, SyncForwarder } from '../../services/tabs/coordinator';
|
|
120
|
+
import type { IngestTuple } from '../../services/tabs/protocol';
|
|
121
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
14
122
|
|
|
15
123
|
/**
|
|
16
|
-
*
|
|
124
|
+
* Tunables for `Sp00kySync` construction.
|
|
125
|
+
*/
|
|
126
|
+
export interface Sp00kySyncOptions {
|
|
127
|
+
/**
|
|
128
|
+
* Cadence (ms) for the `_00_list_ref` poll fallback that catches
|
|
129
|
+
* cross-session UPDATEs the LIVE-permission gap drops. Non-positive
|
|
130
|
+
* values fall back to the default; see
|
|
131
|
+
* {@link resolveListRefPollInterval}.
|
|
132
|
+
*/
|
|
133
|
+
refSyncIntervalMs?: number;
|
|
134
|
+
/**
|
|
135
|
+
* Enable realtime sync for unauthenticated clients against the shared
|
|
136
|
+
* `_00_list_ref_anon` table. See {@link Sp00kyConfig.enableAnonymousLiveQueries}.
|
|
137
|
+
* Defaults to `false`.
|
|
138
|
+
*/
|
|
139
|
+
anonymousLiveQueries?: boolean;
|
|
140
|
+
/**
|
|
141
|
+
* Consecutive failed sync rounds before sync health flips to `degraded`.
|
|
142
|
+
* `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
|
|
143
|
+
* Defaults to `3`.
|
|
144
|
+
*/
|
|
145
|
+
degradeAfterConsecutiveFailures?: number;
|
|
146
|
+
/**
|
|
147
|
+
* Max time a single mutation push may take before it is treated as a network
|
|
148
|
+
* failure and retried. Guards against an RPC that never settles wedging the
|
|
149
|
+
* up-queue for the session. Defaults to 30000; `0` disables the timeout.
|
|
150
|
+
*/
|
|
151
|
+
pushTimeoutMs?: number;
|
|
152
|
+
/**
|
|
153
|
+
* Max time a single down event (`register`/`sync`/`cleanup`) may take before
|
|
154
|
+
* it is treated as a network failure and retried. The mirror of
|
|
155
|
+
* {@link pushTimeoutMs} for the read side, which had no such guard: a
|
|
156
|
+
* `fn::query::register` that never settled held its slot in the down drain,
|
|
157
|
+
* and every later registration behind it, for the rest of the session.
|
|
158
|
+
* Defaults to 30000; `0` disables the timeout.
|
|
159
|
+
*/
|
|
160
|
+
downTimeoutMs?: number;
|
|
161
|
+
/**
|
|
162
|
+
* Transport supervisor. Sync reads its state to report `connection` in
|
|
163
|
+
* {@link SyncHealth} so a UI can show "reconnecting…" the instant the socket
|
|
164
|
+
* drops, without waiting for the degrade threshold. Optional: omitted in
|
|
165
|
+
* tests, where `connection` then reports `connected`.
|
|
166
|
+
*/
|
|
167
|
+
connectionSupervisor?: ConnectionSupervisor;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The main synchronization engine for Sp00ky.
|
|
17
172
|
* Handles the bidirectional synchronization between the local database and the remote backend.
|
|
18
173
|
* Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
|
|
19
174
|
* @template S The schema structure type.
|
|
20
175
|
*/
|
|
21
|
-
export class
|
|
22
|
-
private clientId: string = '';
|
|
176
|
+
export class Sp00kySync<S extends SchemaStructure> {
|
|
23
177
|
private upQueue: UpQueue;
|
|
24
178
|
private downQueue: DownQueue;
|
|
25
179
|
private isInit: boolean = false;
|
|
26
180
|
private logger: Logger;
|
|
27
181
|
private syncEngine: SyncEngine;
|
|
182
|
+
/** Engine-level events (e.g. `SYNC_REMOTE_DATA_INGESTED`). Distinct
|
|
183
|
+
* from `this.events`, which carries Sp00kySync-level events like
|
|
184
|
+
* `SYNC_QUERY_UPDATED` and `SYNC_MUTATION_ROLLED_BACK`. */
|
|
185
|
+
public get engineEvents() {
|
|
186
|
+
return this.syncEngine.events;
|
|
187
|
+
}
|
|
28
188
|
private scheduler: SyncScheduler;
|
|
189
|
+
/**
|
|
190
|
+
* Set by any event that means the socket we registered on is gone, so the
|
|
191
|
+
* next `connected` knows it must re-subscribe rather than treat itself as the
|
|
192
|
+
* initial connect. See {@link subscribeToReconnect}.
|
|
193
|
+
*/
|
|
194
|
+
private needsResubscribe: boolean = false;
|
|
195
|
+
/** When the last reconnect-driven full refetch ran, for burst coalescing. */
|
|
196
|
+
private lastReconnectRefetchAt = 0;
|
|
197
|
+
/**
|
|
198
|
+
* Minimum gap between reconnect-driven full refetches. Long enough to absorb
|
|
199
|
+
* a flapping socket (the SDK reconnect ladder starts at 1s), short enough
|
|
200
|
+
* that a genuine drop minutes later still refetches.
|
|
201
|
+
*/
|
|
202
|
+
private static readonly RECONNECT_REFETCH_COOLDOWN_MS = 10_000;
|
|
203
|
+
/** Poll interval while waiting for a reconnected session to re-authenticate. */
|
|
204
|
+
private static readonly AUTH_READY_RETRY_MS = 500;
|
|
205
|
+
/** Attempts before giving up on re-auth and skipping the refetch entirely. */
|
|
206
|
+
private static readonly AUTH_READY_MAX_ATTEMPTS = 10;
|
|
29
207
|
public events = createSyncEventSystem();
|
|
30
208
|
|
|
31
|
-
|
|
32
|
-
|
|
209
|
+
// Auth identity that drives per-user `_00_list_ref_user_<id>` routing
|
|
210
|
+
// in `RefMode.Dedicated`. Updated by `setCurrentUserId` from the auth
|
|
211
|
+
// subscription in `Sp00kyClient`; null when unauthenticated.
|
|
212
|
+
private currentUserId: string | null = null;
|
|
213
|
+
|
|
214
|
+
// ---- shared-tabs role state ----
|
|
215
|
+
// Followers keep their own remote WS (registration, per-query sync, poll)
|
|
216
|
+
// but must never drain the shared outbox or hold a second list_ref LIVE.
|
|
217
|
+
// The leader relays its LIVE events and routes rollbacks by mutation owner.
|
|
218
|
+
private tabRole: 'solo' | 'leader' | 'follower' = 'solo';
|
|
219
|
+
private tabId: string | null = null;
|
|
220
|
+
private hub: LeaderSyncHub | null = null;
|
|
221
|
+
private forwarder: SyncForwarder | null = null;
|
|
222
|
+
|
|
223
|
+
private refMode: RefMode = DEFAULT_REF_MODE;
|
|
224
|
+
|
|
225
|
+
// When true, an unauthenticated client still runs the `_00_list_ref` poll
|
|
226
|
+
// and LIVE subscription, routed to the shared `_00_list_ref_anon` table, so
|
|
227
|
+
// a logged-out page gets realtime `useQuery` updates. Off by default.
|
|
228
|
+
private readonly anonLiveEnabled: boolean;
|
|
229
|
+
|
|
230
|
+
// Bookkeeping for the LIVE subscription on `_00_list_ref[_user_*]`.
|
|
231
|
+
// SurrealDB binds the permission context at LIVE-registration time and
|
|
232
|
+
// the table name in dedicated mode depends on the authenticated user,
|
|
233
|
+
// so we have to re-register whenever auth state flips.
|
|
234
|
+
private currentLiveQueryUuid: Uuid | null = null;
|
|
235
|
+
private liveQueryUnsubscribe: (() => void) | null = null;
|
|
236
|
+
|
|
237
|
+
// Periodic re-poll of `_00_list_ref` as a safety net for missed LIVE
|
|
238
|
+
// notifications. SurrealDB v3 occasionally drops LIVE deliveries
|
|
239
|
+
// across sessions even when the row matches the permission rule;
|
|
240
|
+
// this catches those without requiring users to reload. The
|
|
241
|
+
// interval is configurable via the constructor; see
|
|
242
|
+
// `resolveListRefPollInterval` for fallback semantics.
|
|
243
|
+
//
|
|
244
|
+
// Self-rescheduling rather than setInterval so each tick can pick
|
|
245
|
+
// its own delay via `nextPollDelayMs` — slows the poll down when
|
|
246
|
+
// LIVE is delivering events and speeds it back up when LIVE quiets.
|
|
247
|
+
private listRefPollTimer: ReturnType<typeof setTimeout> | null = null;
|
|
248
|
+
private listRefPollRunning: boolean = false;
|
|
249
|
+
// The currently-executing poll tick, if any. `stopListRefPoll` only stops
|
|
250
|
+
// future ticks; a bucket switch must also AWAIT the in-flight one so its
|
|
251
|
+
// local writes land in the store it started against.
|
|
252
|
+
private listRefPollInFlight: Promise<void> | null = null;
|
|
253
|
+
public readonly refSyncIntervalMs: number;
|
|
254
|
+
|
|
255
|
+
// Consecutive poll cycles that observed NO list_ref change. Drives the
|
|
256
|
+
// adaptive backoff in `startListRefPoll` via `listRefPollDelayMs`: an idle
|
|
257
|
+
// page coasts from the fast base cadence toward the 5s cap, and any activity
|
|
258
|
+
// (a poll-detected change or a LIVE event) resets it to 0 so the poll snaps
|
|
259
|
+
// back to responsive. Replaces the old LIVE-liveness backoff, which kept the
|
|
260
|
+
// poll pinned at 500ms forever whenever LIVE wasn't firing (the common case
|
|
261
|
+
// on a quiet page, thanks to the cross-session LIVE-permission gap).
|
|
262
|
+
private listRefIdleStreak: number = 0;
|
|
263
|
+
|
|
264
|
+
// `queryHash` -> wall-clock ms of its last poll refresh. Drives the per-cycle
|
|
265
|
+
// plan (`planListRefPollChunks`): oldest first, large views less often.
|
|
266
|
+
private listRefPolledAt: Map<string, number> = new Map();
|
|
267
|
+
|
|
268
|
+
// `${queryHash}:${recordId}` -> consecutive rounds the id has been "still
|
|
269
|
+
// remote" (left the query's list_ref but still exists upstream). Used to
|
|
270
|
+
// distinguish a PERSISTENT view-membership disagreement (the `job:` churn,
|
|
271
|
+
// converged once it crosses the threshold) from a record that's merely
|
|
272
|
+
// mid-deletion (still-remote for ~one round, then gone) — which must NOT be
|
|
273
|
+
// converged, or it gets stranded in this window before its delete is observed.
|
|
274
|
+
private stillRemoteStreaks: Map<string, number> = new Map();
|
|
275
|
+
|
|
276
|
+
// Wall-clock timestamp (ms) of the most recent LIVE event delivered
|
|
277
|
+
// through `handleRemoteListRefChange`. Kept as a diagnostic / liveness
|
|
278
|
+
// signal; the poll cadence is now driven by `listRefIdleStreak`.
|
|
279
|
+
private lastLiveEventAt: number | null = null;
|
|
280
|
+
|
|
281
|
+
// Number of times the initial `_00_list_ref[_user_*]` LIVE subscription
|
|
282
|
+
// had to retry on `setCurrentUserId`. Stays at 0 when the SSP has
|
|
283
|
+
// pre-emptively created the user's dedicated tables; otherwise
|
|
284
|
+
// increments on each retry attempt until LIVE succeeds or attempts
|
|
285
|
+
// are exhausted. Surfaced as a diagnostic so the e2e suite can prove
|
|
286
|
+
// the pre-emptive table-creation path is keeping the first sign-in
|
|
287
|
+
// off the lazy-creation race.
|
|
288
|
+
private _liveRetryCount: number = 0;
|
|
289
|
+
public get liveRetryCount(): number {
|
|
290
|
+
return this._liveRetryCount;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
get isSyncing() {
|
|
294
|
+
return this.scheduler.isSyncing;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
get pendingMutationCount(): number {
|
|
298
|
+
return this.upQueue.size;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
subscribeToPendingMutations(cb: (count: number) => void): () => void {
|
|
302
|
+
const id1 = this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, (event) =>
|
|
303
|
+
cb(event.payload.queueSize)
|
|
304
|
+
);
|
|
305
|
+
const id2 = this.upQueue.events.subscribe(SyncQueueEventTypes.MutationDequeued, (event) =>
|
|
306
|
+
cb(event.payload.queueSize)
|
|
307
|
+
);
|
|
308
|
+
return () => {
|
|
309
|
+
this.upQueue.events.unsubscribe(id1);
|
|
310
|
+
this.upQueue.events.unsubscribe(id2);
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// ---- Sync health -------------------------------------------------------
|
|
315
|
+
// `0` disables degraded reporting (config `syncHealth: false`). Resolved
|
|
316
|
+
// from config in Sp00kyClient and passed through the constructor options.
|
|
317
|
+
private readonly degradeAfterFailures: number;
|
|
318
|
+
/** Per-push RPC deadline; see {@link withPushTimeout}. */
|
|
319
|
+
private readonly pushTimeoutMs: number;
|
|
320
|
+
private readonly downTimeoutMs: number;
|
|
321
|
+
private consecutiveSyncFailures = 0;
|
|
322
|
+
private syncHealthStatus: SyncHealthStatus = 'healthy';
|
|
323
|
+
private lastSyncErrorKind: 'network' | 'application' | undefined;
|
|
324
|
+
private lastSyncErrorMessage: string | undefined;
|
|
325
|
+
// Latched `true` on the first successful sync round; never reset. Lets a UI
|
|
326
|
+
// tell a cold-start "connecting" phase (never reached the server) apart from
|
|
327
|
+
// a real lost connection after a working session.
|
|
328
|
+
private hasSyncedOnce = false;
|
|
329
|
+
|
|
330
|
+
// Self-heal: while degraded, re-drive sync on an exponential backoff so the
|
|
331
|
+
// app recovers on its own — even when the socket never actually dropped (in
|
|
332
|
+
// that case no `connected` event fires, so this re-registration is the ONLY
|
|
333
|
+
// thing that re-probes the server). Started on the degrade transition,
|
|
334
|
+
// cleared on recovery. Capped cadence so a long outage doesn't busy-loop.
|
|
335
|
+
private selfHealTimer: ReturnType<typeof setTimeout> | null = null;
|
|
336
|
+
private selfHealAttempts = 0;
|
|
337
|
+
private static readonly SELF_HEAL_BASE_MS = 2_000;
|
|
338
|
+
private static readonly SELF_HEAL_MAX_MS = 30_000;
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Transport supervisor, when one was supplied. Sync only reads state from it;
|
|
342
|
+
* it never drives reconnects itself.
|
|
343
|
+
*/
|
|
344
|
+
private readonly connectionSupervisor?: ConnectionSupervisor;
|
|
345
|
+
/**
|
|
346
|
+
* Mirror of the supervisor's state. Defaults to `connected` so a client
|
|
347
|
+
* constructed without a supervisor (tests, embedders) reports the same health
|
|
348
|
+
* shape it always has rather than a permanent false "disconnected".
|
|
349
|
+
*/
|
|
350
|
+
private connectionState: ConnectionState = 'connected';
|
|
351
|
+
|
|
352
|
+
/** Current sync-health snapshot. */
|
|
353
|
+
get syncHealth(): SyncHealth {
|
|
354
|
+
return {
|
|
355
|
+
status: this.syncHealthStatus,
|
|
356
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
357
|
+
kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
|
|
358
|
+
error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
|
|
359
|
+
everConnected: this.hasSyncedOnce,
|
|
360
|
+
connection: this.connectionState,
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Observe sync health. The callback fires immediately with the current
|
|
366
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
367
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
368
|
+
*/
|
|
369
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
|
|
370
|
+
cb(this.syncHealth);
|
|
371
|
+
const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) =>
|
|
372
|
+
cb(event.payload)
|
|
373
|
+
);
|
|
374
|
+
return () => this.events.unsubscribe(id);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
private emitSyncHealth(): void {
|
|
378
|
+
this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Mirror the supervisor's transport state into {@link SyncHealth} and emit on
|
|
383
|
+
* every change, so a UI can react to a dropped socket immediately instead of
|
|
384
|
+
* waiting for `degradeAfterFailures` failed rounds. `status` is untouched:
|
|
385
|
+
* a brief reconnect is not a degradation.
|
|
386
|
+
*
|
|
387
|
+
* No explicit unsubscribe: the supervisor is owned by the same client and
|
|
388
|
+
* drops all subscribers in its own `dispose()`, which `Sp00kyClient.close()`
|
|
389
|
+
* calls first.
|
|
390
|
+
*/
|
|
391
|
+
private subscribeToConnectionState(): void {
|
|
392
|
+
if (!this.connectionSupervisor) return;
|
|
393
|
+
this.connectionSupervisor.subscribe((state) => {
|
|
394
|
+
if (this.connectionState === state) return;
|
|
395
|
+
this.connectionState = state;
|
|
396
|
+
this.emitSyncHealth();
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
402
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
403
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
404
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
405
|
+
* is 0).
|
|
406
|
+
*/
|
|
407
|
+
private recordSyncOutcome(ok: boolean, error?: unknown): void {
|
|
408
|
+
if (this.degradeAfterFailures <= 0) return;
|
|
409
|
+
if (ok) {
|
|
410
|
+
// Latch first-ever success so a UI can drop the connecting phase. Set
|
|
411
|
+
// before the early return so a clean cold start (0 prior failures) counts.
|
|
412
|
+
this.hasSyncedOnce = true;
|
|
413
|
+
if (this.consecutiveSyncFailures === 0) return;
|
|
414
|
+
this.consecutiveSyncFailures = 0;
|
|
415
|
+
if (this.syncHealthStatus !== 'healthy') {
|
|
416
|
+
this.syncHealthStatus = 'healthy';
|
|
417
|
+
this.lastSyncErrorKind = undefined;
|
|
418
|
+
this.lastSyncErrorMessage = undefined;
|
|
419
|
+
this.stopSelfHeal();
|
|
420
|
+
this.logger.info(
|
|
421
|
+
{ Category: 'sp00ky-client::Sp00kySync::syncHealth' },
|
|
422
|
+
'Sync recovered; health back to healthy'
|
|
423
|
+
);
|
|
424
|
+
this.emitSyncHealth();
|
|
425
|
+
}
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
this.consecutiveSyncFailures++;
|
|
429
|
+
this.lastSyncErrorKind = classifySyncError(error);
|
|
430
|
+
this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
|
|
431
|
+
if (
|
|
432
|
+
this.syncHealthStatus !== 'degraded' &&
|
|
433
|
+
this.consecutiveSyncFailures >= this.degradeAfterFailures
|
|
434
|
+
) {
|
|
435
|
+
this.syncHealthStatus = 'degraded';
|
|
436
|
+
this.logger.warn(
|
|
437
|
+
{
|
|
438
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
439
|
+
kind: this.lastSyncErrorKind,
|
|
440
|
+
error,
|
|
441
|
+
Category: 'sp00ky-client::Sp00kySync::syncHealth',
|
|
442
|
+
},
|
|
443
|
+
'Sync degraded after sustained failures'
|
|
444
|
+
);
|
|
445
|
+
this.emitSyncHealth();
|
|
446
|
+
this.startSelfHeal();
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Begin self-heal retries (no-op if already running). Started on the
|
|
452
|
+
* healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
|
|
453
|
+
*/
|
|
454
|
+
private startSelfHeal(): void {
|
|
455
|
+
if (this.selfHealTimer !== null) return;
|
|
456
|
+
this.selfHealAttempts = 0;
|
|
457
|
+
this.scheduleSelfHeal();
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
private scheduleSelfHeal(): void {
|
|
461
|
+
const delay = Math.min(
|
|
462
|
+
Sp00kySync.SELF_HEAL_MAX_MS,
|
|
463
|
+
Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts
|
|
464
|
+
);
|
|
465
|
+
this.selfHealTimer = setTimeout(async () => {
|
|
466
|
+
this.selfHealTimer = null;
|
|
467
|
+
if (this.syncHealthStatus !== 'degraded') return;
|
|
468
|
+
this.selfHealAttempts++;
|
|
469
|
+
this.logger.debug(
|
|
470
|
+
{
|
|
471
|
+
attempt: this.selfHealAttempts,
|
|
472
|
+
delayMs: delay,
|
|
473
|
+
Category: 'sp00ky-client::Sp00kySync::selfHeal',
|
|
474
|
+
},
|
|
475
|
+
'Self-heal: re-driving sync while degraded'
|
|
476
|
+
);
|
|
477
|
+
try {
|
|
478
|
+
// Retry whatever is still queued first; the failing op (register or
|
|
479
|
+
// mutation) was re-queued by the queue, so this re-probes the server
|
|
480
|
+
// and reports the outcome through the scheduler → recordSyncOutcome.
|
|
481
|
+
if (this.upQueue.size > 0) {
|
|
482
|
+
await this.scheduler.syncUp();
|
|
483
|
+
} else if (this.downQueue.size > 0) {
|
|
484
|
+
await this.scheduler.syncDown();
|
|
485
|
+
} else {
|
|
486
|
+
// Nothing queued (e.g. the failing op was rolled back + dropped):
|
|
487
|
+
// re-register active queries — mirroring the reconnect handler — so
|
|
488
|
+
// there's a concrete op whose success flips health. If there are no
|
|
489
|
+
// active queries either, probe connectivity directly.
|
|
490
|
+
// Same identity gate as the reconnect path: self-heal fires while
|
|
491
|
+
// sync is degraded, which is exactly when a session is most likely
|
|
492
|
+
// to have lost its `$auth`, and a register issued there stamps the
|
|
493
|
+
// view with an empty identity permanently. See
|
|
494
|
+
// `remoteAuthEstablished`.
|
|
495
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
496
|
+
const canRegister = hashes.length > 0 && (await this.remoteAuthEstablished());
|
|
497
|
+
if (canRegister) {
|
|
498
|
+
for (const hash of hashes) {
|
|
499
|
+
this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
|
|
500
|
+
}
|
|
501
|
+
await this.scheduler.syncDown();
|
|
502
|
+
} else {
|
|
503
|
+
await this.remote.query('RETURN true');
|
|
504
|
+
this.recordSyncOutcome(true);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
} catch (err) {
|
|
508
|
+
// Only the direct connectivity probe can throw here (syncUp/syncDown
|
|
509
|
+
// swallow + self-report); treat a probe failure as another failed round.
|
|
510
|
+
this.recordSyncOutcome(false, err);
|
|
511
|
+
}
|
|
512
|
+
// Keep retrying until recovery. recordSyncOutcome(true) calls stopSelfHeal
|
|
513
|
+
// (clearing any pending timer), so only continue while still degraded.
|
|
514
|
+
if (this.syncHealthStatus === 'degraded') this.scheduleSelfHeal();
|
|
515
|
+
}, delay);
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
private stopSelfHeal(): void {
|
|
519
|
+
if (this.selfHealTimer !== null) {
|
|
520
|
+
clearTimeout(this.selfHealTimer);
|
|
521
|
+
this.selfHealTimer = null;
|
|
522
|
+
}
|
|
523
|
+
this.selfHealAttempts = 0;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* Release a deregistered query's remote view immediately instead of leaving
|
|
528
|
+
* it to the TTL sweep. Off by default; see the reasoning in
|
|
529
|
+
* {@link cleanupQuery}. Kept as a field rather than deleted so the eager path
|
|
530
|
+
* can be re-enabled in a test once the subquery-body repair path exists.
|
|
531
|
+
*/
|
|
532
|
+
private readonly releaseQueriesEagerly = false;
|
|
533
|
+
|
|
534
|
+
constructor(
|
|
535
|
+
private local: LocalStore,
|
|
536
|
+
private remote: RemoteDatabaseService,
|
|
537
|
+
private cache: CacheModule,
|
|
538
|
+
private dataModule: DataModule<S>,
|
|
539
|
+
private schema: S,
|
|
540
|
+
logger: Logger,
|
|
541
|
+
options?: Sp00kySyncOptions
|
|
542
|
+
) {
|
|
543
|
+
this.logger = logger.child({ service: 'Sp00kySync' });
|
|
544
|
+
this.upQueue = new UpQueue(this.local, this.logger, (dropped) =>
|
|
545
|
+
this.onMutationDropped(dropped)
|
|
546
|
+
);
|
|
547
|
+
this.downQueue = new DownQueue(this.local, this.logger);
|
|
548
|
+
this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
|
|
549
|
+
this.scheduler = new SyncScheduler(
|
|
550
|
+
this.upQueue,
|
|
551
|
+
this.downQueue,
|
|
552
|
+
this.processUpEvent.bind(this),
|
|
553
|
+
this.processDownEvent.bind(this),
|
|
554
|
+
this.logger,
|
|
555
|
+
this.handleRollback.bind(this),
|
|
556
|
+
this.recordSyncOutcome.bind(this),
|
|
557
|
+
this.handleMutationSettled.bind(this)
|
|
558
|
+
);
|
|
559
|
+
this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
|
|
560
|
+
this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
|
|
561
|
+
this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
|
|
562
|
+
this.pushTimeoutMs = Math.max(0, options?.pushTimeoutMs ?? 30_000);
|
|
563
|
+
this.downTimeoutMs = Math.max(0, options?.downTimeoutMs ?? 30_000);
|
|
564
|
+
this.connectionSupervisor = options?.connectionSupervisor;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Initializes the synchronization system.
|
|
569
|
+
* Starts the scheduler and initiates the initial sync cycles.
|
|
570
|
+
* @throws Error if already initialized.
|
|
571
|
+
*/
|
|
572
|
+
public async init() {
|
|
573
|
+
if (this.isInit) throw new Error('Sp00kySync is already initialized');
|
|
574
|
+
this.isInit = true;
|
|
575
|
+
await this.scheduler.init({ loadOutbox: this.tabRole !== 'follower' });
|
|
576
|
+
// Boot is local-first now, so init() routinely runs BEFORE the socket is
|
|
577
|
+
// up. Treat that as "the socket we registered on is gone": otherwise the
|
|
578
|
+
// first `connected` takes the initial-connect branch, returns early, and
|
|
579
|
+
// never re-enqueues `register` for queries that registered while offline.
|
|
580
|
+
// They would still heal via the down-queue backoff, but slowly and only
|
|
581
|
+
// because every query happens to enqueue its own register.
|
|
582
|
+
if (this.remote.getStatus() !== 'connected') this.needsResubscribe = true;
|
|
583
|
+
this.subscribeToReconnect();
|
|
584
|
+
this.subscribeToConnectionState();
|
|
585
|
+
void this.scheduler.syncUp();
|
|
586
|
+
void this.scheduler.syncDown();
|
|
587
|
+
// No initial LIVE subscription — wait for `setCurrentUserId` to fire
|
|
588
|
+
// from the auth subscription. In dedicated mode the table name
|
|
589
|
+
// depends on the authenticated user, and an unauthenticated
|
|
590
|
+
// subscription wouldn't match any of the per-user tables anyway.
|
|
591
|
+
//
|
|
592
|
+
// Exception: when anonymous live queries are enabled, start realtime now
|
|
593
|
+
// against the shared `_00_list_ref_anon` table so a logged-out client
|
|
594
|
+
// syncs immediately. `setCurrentUserId` re-points LIVE to the per-user
|
|
595
|
+
// table on sign-in. Guard on `currentUserId` because the auth callback can
|
|
596
|
+
// fire (and authenticate) before `init()` runs — don't clobber that back
|
|
597
|
+
// to the anon table. `setCurrentUserId(null)` is a no-op on first load
|
|
598
|
+
// (it's already null), so this is the only place anon realtime starts.
|
|
599
|
+
if (this.anonLiveEnabled && !this.currentUserId) {
|
|
600
|
+
this.startListRefPoll();
|
|
601
|
+
this.restartRefLiveQuery().catch((err) => {
|
|
602
|
+
this.logger.debug(
|
|
603
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::init' },
|
|
604
|
+
'Anonymous ref LIVE start failed; relying on periodic poll fallback'
|
|
605
|
+
);
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
// ---- shared-tabs roles ------------------------------------------------------
|
|
611
|
+
|
|
612
|
+
/** Set BEFORE init(): shapes what init boots (a follower loads no outbox and
|
|
613
|
+
* never starts LIVE; its own registration/poll paths stay untouched). */
|
|
614
|
+
public setTabContext(role: 'solo' | 'leader' | 'follower', tabId: string | null): void {
|
|
615
|
+
this.tabRole = role;
|
|
616
|
+
this.tabId = tabId;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/** In-flight {@link resumeLeaderDuties}, so a second call joins the first
|
|
620
|
+
* instead of double-draining the outbox. */
|
|
621
|
+
private leaderDutiesInFlight: Promise<void> | null = null;
|
|
622
|
+
|
|
623
|
+
/** Resolves once the in-browser circuit has been primed from the local
|
|
624
|
+
* store. Every sync diff waits on it: diffing against an empty circuit
|
|
625
|
+
* classifies the whole working set as missing and re-downloads it. */
|
|
626
|
+
private primeGate: () => Promise<void> = () => Promise.resolve();
|
|
627
|
+
/** The prime we last waited on. `whenPrimed` hands out one promise per
|
|
628
|
+
* prime, so a new identity means a new prime (boot, bucket switch) ran. */
|
|
629
|
+
private settledPrime: Promise<void> | null = null;
|
|
630
|
+
|
|
631
|
+
setPrimeGate(gate: () => Promise<void>): void {
|
|
632
|
+
this.primeGate = gate;
|
|
633
|
+
this.settledPrime = null;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Leader WIRING only, and deliberately synchronous.
|
|
638
|
+
*
|
|
639
|
+
* The coordinator publishes the leader role and tells the broker
|
|
640
|
+
* `leader-ready` the moment the store is adopted, and the broker can mint a
|
|
641
|
+
* follower's ports on the very next tick. So the follower-message handler
|
|
642
|
+
* has to be live before this returns, or a mutation forwarded in that window
|
|
643
|
+
* is dropped. Everything that can block (outbox reload, LIVE restart) moved
|
|
644
|
+
* to {@link resumeLeaderDuties}: a promotion that waits on the network holds
|
|
645
|
+
* `leader-ready` back, and a broker whose leader never reports ready serves
|
|
646
|
+
* no follower ports and re-elects no one, which wedges the whole namespace.
|
|
647
|
+
*/
|
|
648
|
+
public promoteToLeader(hub: LeaderSyncHub): void {
|
|
649
|
+
this.tabRole = 'leader';
|
|
650
|
+
this.hub = hub;
|
|
651
|
+
this.forwarder = null;
|
|
652
|
+
this.leaderDutiesInFlight = null;
|
|
653
|
+
hub.onFollowerMessage = (tabId, msg) => {
|
|
654
|
+
switch (msg.type) {
|
|
655
|
+
case 'sync-hello':
|
|
656
|
+
break;
|
|
657
|
+
case 'mutation-enqueued':
|
|
658
|
+
// A write is activity: snap the poll back to its base cadence so the
|
|
659
|
+
// membership for it lands fast even if LIVE drops the event.
|
|
660
|
+
this.listRefIdleStreak = 0;
|
|
661
|
+
void this.enqueueForwardedMutation(msg.mutationId);
|
|
662
|
+
break;
|
|
663
|
+
case 'ingest':
|
|
664
|
+
// A follower's optimistic write. The row is already in the shared
|
|
665
|
+
// store; feed this tab's circuit and fan it out to every OTHER
|
|
666
|
+
// follower, so the write shows up everywhere in one hop instead of
|
|
667
|
+
// after the server round-trip (which also depends on LIVE delivery).
|
|
668
|
+
this.applyRelayedIngest(msg.tuples);
|
|
669
|
+
hub.relayIngest(msg.tuples, tabId);
|
|
670
|
+
this.listRefIdleStreak = 0;
|
|
671
|
+
break;
|
|
672
|
+
case 'request-poll':
|
|
673
|
+
this.listRefIdleStreak = 0;
|
|
674
|
+
break;
|
|
675
|
+
}
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** Leader duties: drain the shared outbox, own the single list_ref LIVE,
|
|
680
|
+
* relay LIVE events and rollbacks to followers. Idempotent for a boot-time
|
|
681
|
+
* leader; a runtime promotion (failover) reloads the outbox, which now
|
|
682
|
+
* holds EVERY tab's rows, and restarts LIVE under this session. Runs in the
|
|
683
|
+
* background off the promotion path, so however long it takes (or if it
|
|
684
|
+
* never finishes) the tab is already a working leader. */
|
|
685
|
+
public resumeLeaderDuties(): Promise<void> {
|
|
686
|
+
if (this.leaderDutiesInFlight) return this.leaderDutiesInFlight;
|
|
687
|
+
const run = (async () => {
|
|
688
|
+
if (!this.isInit || this.tabRole !== 'leader') return;
|
|
689
|
+
await this.upQueue.loadFromDatabase();
|
|
690
|
+
void this.scheduler.syncUp();
|
|
691
|
+
if (this.currentUserId || this.anonLiveEnabled) {
|
|
692
|
+
this.startListRefPoll();
|
|
693
|
+
await this.restartRefLiveQuery().catch((err) => {
|
|
694
|
+
this.logger.warn(
|
|
695
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::resumeLeaderDuties' },
|
|
696
|
+
'LIVE restart failed on promotion; poll fallback covers it'
|
|
697
|
+
);
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
})();
|
|
701
|
+
this.leaderDutiesInFlight = run;
|
|
702
|
+
return run.finally(() => {
|
|
703
|
+
if (this.leaderDutiesInFlight === run) this.leaderDutiesInFlight = null;
|
|
704
|
+
});
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/** Follower duties: no outbox drain, no LIVE. Mutations forward to the
|
|
708
|
+
* leader; everything else (registration, per-query sync, poll) runs
|
|
709
|
+
* against this tab's own remote session as usual. */
|
|
710
|
+
public demoteToFollower(forwarder: SyncForwarder): void {
|
|
711
|
+
this.tabRole = 'follower';
|
|
712
|
+
this.hub = null;
|
|
713
|
+
this.forwarder = forwarder;
|
|
714
|
+
// A still-running resumeLeaderDuties self-cancels on its `tabRole` guard;
|
|
715
|
+
// drop the handle so a later re-promotion starts a fresh drain.
|
|
716
|
+
this.leaderDutiesInFlight = null;
|
|
717
|
+
void this.killRefLiveQuery();
|
|
718
|
+
forwarder.onLeaderMessage = (msg) => {
|
|
719
|
+
switch (msg.type) {
|
|
720
|
+
case 'ingest-relay':
|
|
721
|
+
this.applyRelayedIngest(msg.tuples);
|
|
722
|
+
break;
|
|
723
|
+
case 'mutation-settled':
|
|
724
|
+
// The leader pushed a write and deleted its outbox row from the
|
|
725
|
+
// shared store. Without this the row would leave this tab's render
|
|
726
|
+
// set (it is in neither membership nor pending writes) until the
|
|
727
|
+
// relayed `_00_list_ref` event lands: the blink the leader itself
|
|
728
|
+
// is already protected from by `handleMutationSettled`.
|
|
729
|
+
this.dataModule.noteWriteSettled(msg.recordId, msg.eventType);
|
|
730
|
+
break;
|
|
731
|
+
case 'list-ref-change':
|
|
732
|
+
void this.applyRelayedListRefChange(msg).catch((err) => {
|
|
733
|
+
this.logger.error(
|
|
734
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::relay' },
|
|
735
|
+
'Relayed list_ref change failed'
|
|
736
|
+
);
|
|
737
|
+
});
|
|
738
|
+
break;
|
|
739
|
+
case 'mutation-rolled-back':
|
|
740
|
+
this.events.emit(SyncEventTypes.MutationRolledBack, {
|
|
741
|
+
eventType: msg.eventType,
|
|
742
|
+
recordId: msg.recordId,
|
|
743
|
+
error: msg.error,
|
|
744
|
+
});
|
|
745
|
+
break;
|
|
746
|
+
default:
|
|
747
|
+
// db-ready is consumed by the coordinator's attach handshake before
|
|
748
|
+
// the sync handler is installed.
|
|
749
|
+
break;
|
|
750
|
+
}
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
/**
|
|
755
|
+
* A pending mutation was discarded because it can never be sent.
|
|
756
|
+
*
|
|
757
|
+
* This is a lost write, so it must not stay invisible. Every failure in this
|
|
758
|
+
* chain used to be a `logger.error` an app running `logLevel: 'fatal'` never
|
|
759
|
+
* shows, which is how an outbox could sit undrained for hours with the UI
|
|
760
|
+
* reporting nothing. Surfaces as a rollback event (the mutation will never
|
|
761
|
+
* apply, which is what a subscriber needs to know) and degrades sync health.
|
|
762
|
+
*/
|
|
763
|
+
private onMutationDropped(dropped: {
|
|
764
|
+
mutationId: string;
|
|
765
|
+
recordId?: string;
|
|
766
|
+
mutationType?: string;
|
|
767
|
+
reason: string;
|
|
768
|
+
}): void {
|
|
769
|
+
this.logger.error(
|
|
770
|
+
{ ...dropped, Category: 'sp00ky-client::Sp00kySync::onMutationDropped' },
|
|
771
|
+
'Dropped a pending mutation that can never be sent'
|
|
772
|
+
);
|
|
773
|
+
this.recordSyncOutcome(false, new Error(`dropped mutation: ${dropped.reason}`));
|
|
774
|
+
this.events.emit(SyncEventTypes.MutationRolledBack, {
|
|
775
|
+
eventType: (dropped.mutationType as 'create' | 'update' | 'delete') ?? 'update',
|
|
776
|
+
recordId: dropped.recordId ?? dropped.mutationId,
|
|
777
|
+
error: `dropped: ${dropped.reason}`,
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/** A forwarded outbox row from a follower: load + drain it. Idempotent. */
|
|
782
|
+
public async enqueueForwardedMutation(mutationId: string): Promise<void> {
|
|
783
|
+
if (this.tabRole !== 'leader') return;
|
|
784
|
+
await this.upQueue.enqueueFromDatabase(mutationId);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/**
|
|
788
|
+
* Tuples another tab already committed to the shared store: feed them to
|
|
789
|
+
* THIS tab's circuit (no local write). A DELETE additionally forces a
|
|
790
|
+
* re-materialize of the table's queries, exactly as the writing tab does
|
|
791
|
+
* for itself, because the SSP may not emit a view update for it.
|
|
792
|
+
*/
|
|
793
|
+
private applyRelayedIngest(tuples: IngestTuple[]): void {
|
|
794
|
+
this.cache.applyRelayedIngest(tuples);
|
|
795
|
+
const deletedTables = new Set<string>();
|
|
796
|
+
for (const t of tuples) if (t.op === 'DELETE') deletedTables.add(t.table);
|
|
797
|
+
for (const table of deletedTables) {
|
|
798
|
+
void this.dataModule.notifyTableQueries(table).catch((err) => {
|
|
799
|
+
this.logger.warn(
|
|
800
|
+
{ err, table, Category: 'sp00ky-client::Sp00kySync::applyRelayedIngest' },
|
|
801
|
+
'Re-materialize after relayed delete failed'
|
|
802
|
+
);
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
|
|
808
|
+
* and run the exact same handling the LIVE subscription would have. */
|
|
809
|
+
private async applyRelayedListRefChange(msg: {
|
|
810
|
+
action: 'CREATE' | 'UPDATE' | 'DELETE';
|
|
811
|
+
queryId: string;
|
|
812
|
+
recordId: string;
|
|
813
|
+
version: number;
|
|
814
|
+
parent: boolean;
|
|
815
|
+
}): Promise<void> {
|
|
816
|
+
const queryId = parseRecordIdString(msg.queryId);
|
|
817
|
+
// Foreign query (another tab's session-salted hash): not ours, ignore.
|
|
818
|
+
if (!this.dataModule.getQueryById(queryId)) return;
|
|
819
|
+
const recordId = parseRecordIdString(msg.recordId);
|
|
820
|
+
if (msg.parent) {
|
|
821
|
+
await this.handleRemoteSubqueryChange(msg.action, queryId, recordId, msg.version);
|
|
822
|
+
} else {
|
|
823
|
+
await this.handleRemoteListRefChange(msg.action, queryId, recordId, msg.version);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/** One immediate poll cycle (failover convergence). */
|
|
828
|
+
public async forcePollRound(): Promise<void> {
|
|
829
|
+
this.listRefIdleStreak = 0;
|
|
830
|
+
await this.pollListRefForActiveQueries().catch(() => false);
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
/**
|
|
834
|
+
* Quiesce all sync activity ahead of a local-bucket switch. After this
|
|
835
|
+
* resolves, nothing in the sync module writes to the local store: the poll
|
|
836
|
+
* loop is stopped AND its in-flight tick awaited, LIVE is killed, debounce
|
|
837
|
+
* timers are cancelled (their outbox rows are already persisted), and the
|
|
838
|
+
* scheduler has drained its in-flight queue item — including that item's
|
|
839
|
+
* outbox-row delete, which must land in the OLD bucket. Queued down-events
|
|
840
|
+
* are dropped (they reference old-bucket query rows; the post-switch rebind
|
|
841
|
+
* re-enqueues registrations). The old user's un-pushed outbox is deliberately
|
|
842
|
+
* NOT drained: the remote session already belongs to the next user.
|
|
843
|
+
*/
|
|
844
|
+
public async prepareBucketSwitch(): Promise<void> {
|
|
845
|
+
this.stopSelfHeal();
|
|
846
|
+
this.stopListRefPoll();
|
|
847
|
+
this.clearListRefRefetches();
|
|
848
|
+
this.clearLiveRemovals();
|
|
849
|
+
if (this.listRefPollInFlight) await this.listRefPollInFlight;
|
|
850
|
+
await this.killRefLiveQuery();
|
|
851
|
+
this.upQueue.clearDebounceTimers();
|
|
852
|
+
await this.scheduler.pause();
|
|
853
|
+
this.downQueue.clear();
|
|
854
|
+
this.stillRemoteStreaks.clear();
|
|
855
|
+
this.logger.info(
|
|
856
|
+
{ Category: 'sp00ky-client::Sp00kySync::prepareBucketSwitch' },
|
|
857
|
+
'Sync quiesced for bucket switch'
|
|
858
|
+
);
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/**
|
|
862
|
+
* Resume syncing against the freshly-opened bucket: reload the mutation
|
|
863
|
+
* outbox from ITS `_00_pending_mutations` (the new user's own un-pushed
|
|
864
|
+
* offline work) and restart the scheduler. LIVE + the list_ref poll restart
|
|
865
|
+
* via the `setCurrentUserId` call that follows in the auth listener.
|
|
866
|
+
*/
|
|
867
|
+
public async completeBucketSwitch(): Promise<void> {
|
|
868
|
+
await this.upQueue.loadFromDatabase();
|
|
869
|
+
this.scheduler.resume();
|
|
870
|
+
this.logger.info(
|
|
871
|
+
{ Category: 'sp00ky-client::Sp00kySync::completeBucketSwitch' },
|
|
872
|
+
'Sync resumed after bucket switch'
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Push the authenticated user's record id from the parent client's
|
|
878
|
+
* auth subscription. Tears down the existing `_00_list_ref` LIVE (if
|
|
879
|
+
* any) and re-registers it under the new user's dedicated table so
|
|
880
|
+
* SurrealDB binds the permission rule under the post-flip auth
|
|
881
|
+
* context. Pass `null` on sign-out.
|
|
882
|
+
*
|
|
883
|
+
* The dedicated `_00_list_ref_user_<id>` table is created lazily by
|
|
884
|
+
* the SSP when the first query registration arrives, which may be
|
|
885
|
+
* concurrent with this call. We retry the LIVE registration with a
|
|
886
|
+
* short backoff so a "table not found" race resolves without
|
|
887
|
+
* surfacing as a permanent auth-loading hang.
|
|
888
|
+
*/
|
|
889
|
+
public async setCurrentUserId(userId: string | null): Promise<void> {
|
|
890
|
+
if (this.currentUserId === userId) return;
|
|
891
|
+
this.currentUserId = userId;
|
|
892
|
+
if (!userId) {
|
|
893
|
+
if (this.anonLiveEnabled) {
|
|
894
|
+
// Signed out but anonymous realtime is on: keep the poll running and
|
|
895
|
+
// re-point LIVE from the (now stale) per-user table to the shared
|
|
896
|
+
// `_00_list_ref_anon`. `startListRefPoll` is idempotent; the poll
|
|
897
|
+
// re-resolves `listRefTable()` each tick so it follows automatically.
|
|
898
|
+
this.startListRefPoll();
|
|
899
|
+
await this.restartRefLiveQuery().catch((err) => {
|
|
900
|
+
this.logger.debug(
|
|
901
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
|
|
902
|
+
'Anonymous ref LIVE restart failed; relying on periodic poll fallback'
|
|
903
|
+
);
|
|
904
|
+
});
|
|
905
|
+
return;
|
|
906
|
+
}
|
|
907
|
+
await this.killRefLiveQuery();
|
|
908
|
+
this.stopListRefPoll();
|
|
909
|
+
return;
|
|
910
|
+
}
|
|
911
|
+
// Start periodic polling FIRST so we have a deterministic fallback
|
|
912
|
+
// even when LIVE registration fails or SurrealDB drops a delivery.
|
|
913
|
+
this.startListRefPoll();
|
|
914
|
+
// Try to start LIVE with backoff for low-latency delivery on the
|
|
915
|
+
// happy path; the poll handles the rest.
|
|
916
|
+
const attemptDelays = [0, 250, 500, 1000, 2000];
|
|
917
|
+
for (let i = 0; i < attemptDelays.length; i++) {
|
|
918
|
+
if (attemptDelays[i] > 0) {
|
|
919
|
+
this._liveRetryCount++;
|
|
920
|
+
await new Promise((r) => setTimeout(r, attemptDelays[i]));
|
|
921
|
+
}
|
|
922
|
+
try {
|
|
923
|
+
await this.restartRefLiveQuery();
|
|
924
|
+
return;
|
|
925
|
+
} catch (err) {
|
|
926
|
+
this.logger.debug(
|
|
927
|
+
{ err, attempt: i + 1, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
|
|
928
|
+
'Ref LIVE start failed; relying on periodic poll fallback'
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
private startListRefPoll(): void {
|
|
935
|
+
if (this.listRefPollRunning) return;
|
|
936
|
+
this.listRefPollRunning = true;
|
|
937
|
+
this.logger.debug(
|
|
938
|
+
{
|
|
939
|
+
intervalMs: this.refSyncIntervalMs,
|
|
940
|
+
Category: 'sp00ky-client::Sp00kySync::startListRefPoll',
|
|
941
|
+
},
|
|
942
|
+
'list_ref poll loop started'
|
|
943
|
+
);
|
|
944
|
+
const schedule = (delayMs: number) => {
|
|
945
|
+
this.listRefPollTimer = setTimeout(async () => {
|
|
946
|
+
if (!this.listRefPollRunning) return;
|
|
947
|
+
let changed = false;
|
|
948
|
+
const startedAt = Date.now();
|
|
949
|
+
const tick = (async () => {
|
|
950
|
+
changed = await this.pollListRefForActiveQueries();
|
|
951
|
+
})();
|
|
952
|
+
this.listRefPollInFlight = tick.catch(() => {});
|
|
953
|
+
try {
|
|
954
|
+
await tick;
|
|
955
|
+
} finally {
|
|
956
|
+
this.listRefPollInFlight = null;
|
|
957
|
+
if (!this.listRefPollRunning) return;
|
|
958
|
+
// Reset the idle streak on any observed change so the poll snaps
|
|
959
|
+
// back to the fast base cadence; otherwise grow it so a quiet page
|
|
960
|
+
// backs off toward the cap. (`handleRemoteListRefChange` also resets
|
|
961
|
+
// it when a LIVE event lands.) A settled write still waiting for its
|
|
962
|
+
// membership also pins the fast cadence: the row is on borrowed time
|
|
963
|
+
// (the settled-write grace), so the edge must be found on the first
|
|
964
|
+
// poll after it lands, not after the backoff has grown to the cap.
|
|
965
|
+
const awaitingMembership = this.dataModule.hasSettledWritesPending();
|
|
966
|
+
this.listRefIdleStreak = changed || awaitingMembership ? 0 : this.listRefIdleStreak + 1;
|
|
967
|
+
// Never let the poll take more than half the wall clock: a cycle that
|
|
968
|
+
// took longer than the cadence (many queries, slow link) waits at
|
|
969
|
+
// least its own duration before the next one, so it cannot occupy
|
|
970
|
+
// the remote back-to-back the way the old per-query poll did.
|
|
971
|
+
const cycleMs = Date.now() - startedAt;
|
|
972
|
+
const next = Math.max(
|
|
973
|
+
listRefPollDelayMs({
|
|
974
|
+
idleStreak: this.listRefIdleStreak,
|
|
975
|
+
baseIntervalMs: this.refSyncIntervalMs,
|
|
976
|
+
}),
|
|
977
|
+
cycleMs
|
|
978
|
+
);
|
|
979
|
+
schedule(next);
|
|
980
|
+
}
|
|
981
|
+
}, delayMs);
|
|
982
|
+
};
|
|
983
|
+
schedule(this.refSyncIntervalMs);
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
private stopListRefPoll(): void {
|
|
987
|
+
this.listRefPollRunning = false;
|
|
988
|
+
if (this.listRefPollTimer !== null) {
|
|
989
|
+
clearTimeout(this.listRefPollTimer);
|
|
990
|
+
this.listRefPollTimer = null;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
/**
|
|
995
|
+
* One poll cycle: refetch `_00_list_ref` for the active queries that are due,
|
|
996
|
+
* one round trip per chunk (see `planListRefPollChunks`). Returns whether ANY
|
|
997
|
+
* query's remoteArray actually changed — the scheduler uses this to drive the
|
|
998
|
+
* adaptive idle backoff.
|
|
999
|
+
*
|
|
1000
|
+
* Also the ONLY health signal that runs while the page is idle. Sync health is
|
|
1001
|
+
* otherwise activity-driven (mutations/registrations via the scheduler,
|
|
1002
|
+
* reconnect re-registration, self-heal), so on a quiet page a stale `degraded`
|
|
1003
|
+
* would linger until the next mutation and a genuine idle drop would be
|
|
1004
|
+
* invisible. We fold the cycle's aggregate reachability into `recordSyncOutcome`
|
|
1005
|
+
* so idle health self-recovers (and self-degrades) with no user action. A clean
|
|
1006
|
+
* cycle is idempotent when already healthy (`recordSyncOutcome` early-returns at
|
|
1007
|
+
* `consecutiveSyncFailures === 0`), so a healthy idle page pays nothing.
|
|
1008
|
+
*/
|
|
1009
|
+
private async pollListRefForActiveQueries(): Promise<boolean> {
|
|
1010
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
1011
|
+
if (hashes.length === 0) {
|
|
1012
|
+
// No active queries to piggyback on, but health still needs a heartbeat —
|
|
1013
|
+
// probe connectivity directly so an idle page with no live queries doesn't
|
|
1014
|
+
// go blind. Cheap, and gated by the same adaptive backoff (≤5s idle cap).
|
|
1015
|
+
try {
|
|
1016
|
+
await this.remote.query('RETURN true');
|
|
1017
|
+
this.recordSyncOutcome(true);
|
|
1018
|
+
} catch (err) {
|
|
1019
|
+
this.recordSyncOutcome(false, err);
|
|
1020
|
+
}
|
|
1021
|
+
return false;
|
|
1022
|
+
}
|
|
1023
|
+
const now = Date.now();
|
|
1024
|
+
// Forget queries that went away so the map cannot grow with the session.
|
|
1025
|
+
const active = new Set(hashes);
|
|
1026
|
+
for (const hash of this.listRefPolledAt.keys()) {
|
|
1027
|
+
if (!active.has(hash)) this.listRefPolledAt.delete(hash);
|
|
1028
|
+
}
|
|
1029
|
+
const chunks = planListRefPollChunks(
|
|
1030
|
+
hashes.map((hash) => ({
|
|
1031
|
+
hash,
|
|
1032
|
+
rows: this.dataModule.getQueryByHash(hash)?.config.remoteArray?.length ?? 0,
|
|
1033
|
+
lastPolledAt: this.listRefPolledAt.get(hash) ?? 0,
|
|
1034
|
+
})),
|
|
1035
|
+
{ now }
|
|
1036
|
+
);
|
|
1037
|
+
let anyChanged = false;
|
|
1038
|
+
// `reached` = the server answered at least once this cycle (a success, or an
|
|
1039
|
+
// *application* error, which still proves reachability). `firstNetworkErr`
|
|
1040
|
+
// holds the first network-classified failure. A cycle that only produced
|
|
1041
|
+
// network errors reports the outcome as a down round; a mixed/app cycle counts
|
|
1042
|
+
// as reached; an all-application cycle reports nothing (that's a query-shape
|
|
1043
|
+
// fault owned by the registration path, not a reachability signal).
|
|
1044
|
+
let reached = false;
|
|
1045
|
+
let firstNetworkErr: unknown;
|
|
1046
|
+
for (const chunk of chunks) {
|
|
1047
|
+
let snapshots: Map<string, ListRefSnapshot>;
|
|
1048
|
+
try {
|
|
1049
|
+
snapshots = await this.fetchListRefSnapshots(chunk);
|
|
1050
|
+
reached = true;
|
|
1051
|
+
} catch (err) {
|
|
1052
|
+
if (classifySyncError(err) === 'network') {
|
|
1053
|
+
if (firstNetworkErr === undefined) firstNetworkErr = err;
|
|
1054
|
+
} else {
|
|
1055
|
+
reached = true;
|
|
1056
|
+
}
|
|
1057
|
+
this.logger.debug(
|
|
1058
|
+
{
|
|
1059
|
+
err: (err as Error)?.message ?? err,
|
|
1060
|
+
hashes: chunk,
|
|
1061
|
+
Category: 'sp00ky-client::Sp00kySync::pollListRefForActiveQueries',
|
|
1062
|
+
},
|
|
1063
|
+
'list_ref poll round trip failed'
|
|
1064
|
+
);
|
|
1065
|
+
continue;
|
|
1066
|
+
}
|
|
1067
|
+
for (const hash of chunk) {
|
|
1068
|
+
const snapshot = snapshots.get(hash);
|
|
1069
|
+
if (!snapshot) continue;
|
|
1070
|
+
this.listRefPolledAt.set(hash, now);
|
|
1071
|
+
try {
|
|
1072
|
+
if ((await this.applyListRefSnapshot(hash, snapshot)).changed) anyChanged = true;
|
|
1073
|
+
} catch (err) {
|
|
1074
|
+
this.logger.debug(
|
|
1075
|
+
{
|
|
1076
|
+
err: (err as Error)?.message ?? err,
|
|
1077
|
+
hash,
|
|
1078
|
+
Category: 'sp00ky-client::Sp00kySync::pollListRefForActiveQueries',
|
|
1079
|
+
},
|
|
1080
|
+
'Per-query list_ref poll apply failed'
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
// Call the private outcome recorder directly rather than routing through the
|
|
1086
|
+
// scheduler — the scheduler only reports on rounds that drained ≥1 queue item
|
|
1087
|
+
// (`processedAny`), and this isn't a queue round.
|
|
1088
|
+
if (reached) {
|
|
1089
|
+
this.recordSyncOutcome(true);
|
|
1090
|
+
} else if (firstNetworkErr !== undefined) {
|
|
1091
|
+
this.recordSyncOutcome(false, firstNetworkErr);
|
|
1092
|
+
}
|
|
1093
|
+
return anyChanged;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* The server's current `_00_list_ref` state for a chunk of queries in ONE
|
|
1098
|
+
* round trip: primary edges, subquery-child edges and `_00_query.rowCount`.
|
|
1099
|
+
* Every hash whose query still exists gets an entry (empty arrays and
|
|
1100
|
+
* `rowCount: null` when nothing was readable), so a caller can tell "no
|
|
1101
|
+
* edges" from "not asked". A hash is left out only when the select did not
|
|
1102
|
+
* come back as an array at all — the old per-query code skipped the apply in
|
|
1103
|
+
* that case too, since a non-array is not a membership set.
|
|
1104
|
+
*
|
|
1105
|
+
* Guarded against a batch that silently matches nothing: if the select
|
|
1106
|
+
* returned no edge at all while the client holds edges for a query the
|
|
1107
|
+
* server still reports rows for, the batch is not trusted and the chunk is
|
|
1108
|
+
* re-read one query at a time with the single-query select. An empty set
|
|
1109
|
+
* taken at face value would be recorded as the removal of every row.
|
|
1110
|
+
*/
|
|
1111
|
+
private async fetchListRefSnapshots(hashes: string[]): Promise<Map<string, ListRefSnapshot>> {
|
|
1112
|
+
const listRefTbl = this.listRefTable();
|
|
1113
|
+
const ids: RecordId<string>[] = [];
|
|
1114
|
+
const hashById = new Map<string, string>();
|
|
1115
|
+
const out = new Map<string, ListRefSnapshot>();
|
|
1116
|
+
for (const hash of hashes) {
|
|
1117
|
+
const queryState = this.dataModule.getQueryByHash(hash);
|
|
1118
|
+
if (!queryState) continue;
|
|
1119
|
+
ids.push(queryState.config.id);
|
|
1120
|
+
hashById.set(encodeRecordId(queryState.config.id), hash);
|
|
1121
|
+
}
|
|
1122
|
+
if (ids.length === 0) return out;
|
|
1123
|
+
if (ids.length === 1) {
|
|
1124
|
+
const hash = hashById.get(encodeRecordId(ids[0]))!;
|
|
1125
|
+
const single = await this.fetchListRefSnapshot(ids[0]);
|
|
1126
|
+
if (single) out.set(hash, single);
|
|
1127
|
+
return out;
|
|
1128
|
+
}
|
|
1129
|
+
const [edges, counts] = await this.remote.query<
|
|
1130
|
+
[ListRefEdgeRow[] | null, (QueryMetaRow | null)[] | null]
|
|
1131
|
+
>(`${buildListRefBatchSelect(listRefTbl)};\n${buildQueryRowCountBatchSelect()}`, { ins: ids });
|
|
1132
|
+
if (!Array.isArray(edges)) return out;
|
|
1133
|
+
for (const hash of hashById.values()) {
|
|
1134
|
+
out.set(hash, {
|
|
1135
|
+
primary: [],
|
|
1136
|
+
subquery: [],
|
|
1137
|
+
rowCount: null,
|
|
1138
|
+
meta: { present: false, rowCount: null, state: null },
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
for (const row of edges) {
|
|
1142
|
+
const hash = hashById.get(encodeRecordId(row.in));
|
|
1143
|
+
if (!hash) continue;
|
|
1144
|
+
const snapshot = out.get(hash)!;
|
|
1145
|
+
const pair: [string, number] = [encodeRecordId(row.out), row.version];
|
|
1146
|
+
if (row.parent == null) snapshot.primary.push(pair);
|
|
1147
|
+
else snapshot.subquery.push(pair);
|
|
1148
|
+
}
|
|
1149
|
+
for (const snapshot of out.values()) {
|
|
1150
|
+
snapshot.primary = dedupeRecordVersions(snapshot.primary);
|
|
1151
|
+
snapshot.subquery = dedupeRecordVersions(snapshot.subquery);
|
|
1152
|
+
}
|
|
1153
|
+
for (const count of Array.isArray(counts) ? counts : []) {
|
|
1154
|
+
if (!count || !count.id) continue;
|
|
1155
|
+
const hash = hashById.get(encodeRecordId(count.id));
|
|
1156
|
+
if (!hash) continue;
|
|
1157
|
+
const snapshot = out.get(hash)!;
|
|
1158
|
+
snapshot.meta = metaFromRow(count);
|
|
1159
|
+
snapshot.rowCount = snapshot.meta.rowCount;
|
|
1160
|
+
}
|
|
1161
|
+
// Two shapes of a batch that must not be believed at face value: no edge
|
|
1162
|
+
// at all while the server still reports rows for a query we hold (a
|
|
1163
|
+
// silently empty batch), and a query we hold whose `_00_query` row did
|
|
1164
|
+
// not come back (a lost view, or a batch that missed it). Both are
|
|
1165
|
+
// re-read one query at a time before anything is applied, because an
|
|
1166
|
+
// empty set taken at face value is the removal of every row, and a
|
|
1167
|
+
// missing row taken at face value would re-register a healthy view.
|
|
1168
|
+
const suspect = Array.from(out.entries()).some(([hash, snapshot]) => {
|
|
1169
|
+
const held = this.dataModule.getQueryByHash(hash)?.config.remoteArray?.length ?? 0;
|
|
1170
|
+
if (held === 0) return false;
|
|
1171
|
+
if (!snapshot.meta.present) return true;
|
|
1172
|
+
return edges.length === 0 && (snapshot.rowCount ?? 0) > 0;
|
|
1173
|
+
});
|
|
1174
|
+
if (suspect) {
|
|
1175
|
+
this.logger.warn(
|
|
1176
|
+
{ hashes, Category: 'sp00ky-client::Sp00kySync::fetchListRefSnapshots' },
|
|
1177
|
+
'Batched list_ref select is not trusted for queries that hold rows (no edges, or a missing _00_query row); re-reading one query at a time'
|
|
1178
|
+
);
|
|
1179
|
+
for (const hash of Array.from(out.keys())) {
|
|
1180
|
+
const queryState = this.dataModule.getQueryByHash(hash);
|
|
1181
|
+
if (!queryState) {
|
|
1182
|
+
out.delete(hash);
|
|
1183
|
+
continue;
|
|
1184
|
+
}
|
|
1185
|
+
const single = await this.fetchListRefSnapshot(queryState.config.id);
|
|
1186
|
+
if (single) out.set(hash, single);
|
|
1187
|
+
else out.delete(hash);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
return out;
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
/**
|
|
1194
|
+
* Single-query form of {@link fetchListRefSnapshots}: the selects the
|
|
1195
|
+
* registration path has always used, in one statement batch. `null` when
|
|
1196
|
+
* the primary select did not come back as an array.
|
|
1197
|
+
*/
|
|
1198
|
+
private async fetchListRefSnapshot(id: RecordId<string>): Promise<ListRefSnapshot | null> {
|
|
1199
|
+
const listRefTbl = this.listRefTable();
|
|
1200
|
+
const [items, metaRow, children] = await this.remote.query<
|
|
1201
|
+
[
|
|
1202
|
+
{ out: RecordId<string>; version: number }[] | null,
|
|
1203
|
+
QueryMetaRow | null,
|
|
1204
|
+
{ out: RecordId<string>; version: number }[] | null,
|
|
1205
|
+
]
|
|
1206
|
+
>(
|
|
1207
|
+
`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()};\n${buildSubqueryListRefSelect(listRefTbl)}`,
|
|
1208
|
+
{ in: id }
|
|
1209
|
+
);
|
|
1210
|
+
if (!Array.isArray(items)) return null;
|
|
1211
|
+
const toPairs = (rows: { out: RecordId<string>; version: number }[] | null): RecordVersionArray =>
|
|
1212
|
+
dedupeRecordVersions(
|
|
1213
|
+
Array.isArray(rows) ? rows.map((item) => [encodeRecordId(item.out), item.version]) : []
|
|
1214
|
+
);
|
|
1215
|
+
const meta = metaFromRow(metaRow);
|
|
1216
|
+
return {
|
|
1217
|
+
primary: toPairs(items),
|
|
1218
|
+
subquery: toPairs(children),
|
|
1219
|
+
rowCount: meta.rowCount,
|
|
1220
|
+
meta,
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/**
|
|
1225
|
+
* Land one poll snapshot for `queryHash`: diff the server's edges against the
|
|
1226
|
+
* cached `remoteArray`, sync any added/updated rows through the SyncEngine,
|
|
1227
|
+
* persist the new remoteArray and converge the subquery children. This is
|
|
1228
|
+
* the same shape `createRemoteQuery` does for its initial fetch and what
|
|
1229
|
+
* `handleRemoteListRefChange` does per-LIVE-event — we reuse it on a timer
|
|
1230
|
+
* as a fallback for missed LIVE notifications. Returns whether membership
|
|
1231
|
+
* changed.
|
|
1232
|
+
*/
|
|
1233
|
+
private async applyListRefSnapshot(
|
|
1234
|
+
queryHash: string,
|
|
1235
|
+
snapshot: ListRefSnapshot
|
|
1236
|
+
): Promise<{ changed: boolean; outcome: MembershipOutcome | 'unchanged' }> {
|
|
1237
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
1238
|
+
if (!queryState) return { changed: false, outcome: 'unchanged' };
|
|
1239
|
+
const fresh = snapshot.primary;
|
|
1240
|
+
// Capture which ids LEFT the query's window (present in the cached
|
|
1241
|
+
// remoteArray, absent from `fresh`) BEFORE we overwrite remoteArray — these
|
|
1242
|
+
// are cross-window deletes (or rows that scrolled out). They drive the
|
|
1243
|
+
// forced re-render below. Only meaningful once the set is actually applied.
|
|
1244
|
+
const prevRemote = queryState.config.remoteArray ?? [];
|
|
1245
|
+
// Idempotent poll: only persist the remoteArray when it actually changed.
|
|
1246
|
+
// The poll runs continuously as a LIVE fallback, so on a quiet page `fresh`
|
|
1247
|
+
// equals the cached array every tick — re-writing it (an `UPDATE _00_query`
|
|
1248
|
+
// each cycle, per active query) was pure churn and the bulk of the idle
|
|
1249
|
+
// traffic. `recordVersionArraysEqual` is order-insensitive because the
|
|
1250
|
+
// list_ref SELECT has no `ORDER BY`.
|
|
1251
|
+
const differs = !recordVersionArraysEqual(fresh, queryState.config.remoteArray);
|
|
1252
|
+
let outcome: MembershipOutcome | 'unchanged' = 'unchanged';
|
|
1253
|
+
if (differs) {
|
|
1254
|
+
// Update the cached remoteArray so the next diff/sync sees the new state.
|
|
1255
|
+
// `syncQuery` (below) then writes through `cache.saveBatch`, which UPSERTs
|
|
1256
|
+
// the local DB row and ingests it into the in-browser SSP — the SSP's
|
|
1257
|
+
// stream updates run `processStreamUpdate`, which re-queries the local DB
|
|
1258
|
+
// and notifies subscribers. We skip an explicit `notifyQuerySynced`
|
|
1259
|
+
// because that path races the stream-update path (can notify with stale
|
|
1260
|
+
// records).
|
|
1261
|
+
//
|
|
1262
|
+
// The DataModule arbitrates an EMPTY set against the `_00_query` row it
|
|
1263
|
+
// was read with (`snapshot.meta`): a missing row is a lost view (kept +
|
|
1264
|
+
// re-registered), a `materializing` row is a publish in flight, and only
|
|
1265
|
+
// a `ready` row with `rowCount 0` empties the query.
|
|
1266
|
+
outcome = await this.dataModule.updateQueryRemoteArray(queryHash, fresh, {
|
|
1267
|
+
meta: snapshot.meta,
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
const applied = differs && outcome === 'applied';
|
|
1271
|
+
const freshIds = new Set(fresh.map(([id]) => id));
|
|
1272
|
+
const removedIds = applied
|
|
1273
|
+
? prevRemote.filter(([id]) => !freshIds.has(id)).map(([id]) => id)
|
|
1274
|
+
: [];
|
|
1275
|
+
// Run `syncQuery` every tick regardless: it's a no-op when localArray has
|
|
1276
|
+
// caught up to remoteArray (`if (!diff) return`, issues no query), but it
|
|
1277
|
+
// covers the rare case where remoteArray is stable yet localArray is behind
|
|
1278
|
+
// (a prior record fetch failed) — so a missed row still gets retried.
|
|
1279
|
+
// For REMOVALS it runs the ids through `handleRemovedRecords`, which deletes
|
|
1280
|
+
// confirmed-gone records from the local DB. A rejected snapshot left
|
|
1281
|
+
// `remoteArray` alone, so this cannot act on it.
|
|
1282
|
+
try {
|
|
1283
|
+
await this.syncQuery(queryHash);
|
|
1284
|
+
} catch (err) {
|
|
1285
|
+
this.logger.info(
|
|
1286
|
+
{
|
|
1287
|
+
err: (err as Error)?.message ?? err,
|
|
1288
|
+
queryHash,
|
|
1289
|
+
Category: 'sp00ky-client::Sp00kySync::applyListRefSnapshot',
|
|
1290
|
+
},
|
|
1291
|
+
'syncQuery failed during poll'
|
|
1292
|
+
);
|
|
1293
|
+
}
|
|
1294
|
+
// Membership moved but a row may have needed no fetch (this client wrote
|
|
1295
|
+
// it, so the engine already holds it at the published version): nothing
|
|
1296
|
+
// then re-materializes the query. Ask for one; it is a no-op behind a
|
|
1297
|
+
// stream update that a fetch above already queued.
|
|
1298
|
+
if (applied) this.dataModule.scheduleRematerialize(queryHash);
|
|
1299
|
+
// Cross-session fallback for `.related()` child rows: the LIVE-permission
|
|
1300
|
+
// gap can drop child-edge notifications, so converge their bodies on the
|
|
1301
|
+
// poll too (idempotent — no-op when nothing changed). Not for a snapshot
|
|
1302
|
+
// that was rejected: its child set is as untrustworthy as its primary set.
|
|
1303
|
+
if (outcome === 'applied' || outcome === 'unchanged') {
|
|
1304
|
+
await this.applySubqueryChildren(queryHash, snapshot.subquery).catch((err) => {
|
|
1305
|
+
this.logger.info(
|
|
1306
|
+
{
|
|
1307
|
+
err: (err as Error)?.message ?? err,
|
|
1308
|
+
queryHash,
|
|
1309
|
+
Category: 'sp00ky-client::Sp00kySync::applyListRefSnapshot',
|
|
1310
|
+
},
|
|
1311
|
+
'Subquery child sync failed during poll'
|
|
1312
|
+
);
|
|
1313
|
+
});
|
|
1314
|
+
}
|
|
1315
|
+
// A REMOVAL needs no record fetch, so unlike the added-row path it doesn't
|
|
1316
|
+
// get a re-render from the SSP stream on this code path reliably (and the
|
|
1317
|
+
// non-windowed window-0 query re-queries the local DB rather than the id-set).
|
|
1318
|
+
// Force a re-materialize + notify so the deleted row drops from the list in
|
|
1319
|
+
// this (second) window — the reliable, LIVE-independent cross-window path.
|
|
1320
|
+
if (removedIds.length > 0) {
|
|
1321
|
+
try {
|
|
1322
|
+
await this.dataModule.notifyQuerySynced(queryHash);
|
|
1323
|
+
} catch (err) {
|
|
1324
|
+
this.logger.info(
|
|
1325
|
+
{
|
|
1326
|
+
err: (err as Error)?.message ?? err,
|
|
1327
|
+
queryHash,
|
|
1328
|
+
Category: 'sp00ky-client::Sp00kySync::applyListRefSnapshot',
|
|
1329
|
+
},
|
|
1330
|
+
'notifyQuerySynced failed during poll-removal re-render'
|
|
1331
|
+
);
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
return { changed: applied, outcome };
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
/**
|
|
1338
|
+
* Re-read ONE query's server state now (edges + `_00_query` row) and land
|
|
1339
|
+
* it through {@link applyListRefSnapshot}. The targeted form of the poll,
|
|
1340
|
+
* for the moments that cannot wait for the next tick: a registration that
|
|
1341
|
+
* answered `materializing`, and a LIVE removal that has to be checked
|
|
1342
|
+
* against the row before it may empty a painted list.
|
|
1343
|
+
*/
|
|
1344
|
+
private async refetchListRefForQuery(
|
|
1345
|
+
queryHash: string
|
|
1346
|
+
): Promise<{ changed: boolean; outcome: MembershipOutcome | 'unchanged' } | null> {
|
|
1347
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
1348
|
+
if (!queryState) return null;
|
|
1349
|
+
const snapshot = await this.fetchListRefSnapshot(queryState.config.id);
|
|
1350
|
+
if (!snapshot) return null;
|
|
1351
|
+
return this.applyListRefSnapshot(queryHash, snapshot);
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
/** Per-query pending targeted re-reads: one timer and the ladder still to walk. */
|
|
1355
|
+
private listRefRefetches = new Map<string, { timer: ReturnType<typeof setTimeout>; ladder: number[] }>();
|
|
1356
|
+
|
|
1357
|
+
/**
|
|
1358
|
+
* Schedule a targeted re-read of `queryHash` after `delays[0]` ms. If the
|
|
1359
|
+
* read still says the server is publishing (`materializing`), the next rung
|
|
1360
|
+
* of the ladder is armed; any other outcome ends it. Re-scheduling a hash
|
|
1361
|
+
* that already has a timer replaces it (latest wins), so bursts coalesce.
|
|
1362
|
+
*/
|
|
1363
|
+
private scheduleListRefRefetch(queryHash: string, delays: number | readonly number[]): void {
|
|
1364
|
+
const ladder = typeof delays === 'number' ? [delays] : [...delays];
|
|
1365
|
+
if (ladder.length === 0) return;
|
|
1366
|
+
this.clearListRefRefetch(queryHash);
|
|
1367
|
+
const [first, ...rest] = ladder;
|
|
1368
|
+
const timer = setTimeout(() => {
|
|
1369
|
+
this.listRefRefetches.delete(queryHash);
|
|
1370
|
+
void this.refetchListRefForQuery(queryHash)
|
|
1371
|
+
.then((result) => {
|
|
1372
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
1373
|
+
const stillMaterializing =
|
|
1374
|
+
result?.outcome === 'ignored' && queryState?.serverState === 'materializing';
|
|
1375
|
+
if (stillMaterializing && rest.length > 0) {
|
|
1376
|
+
this.scheduleListRefRefetch(queryHash, rest);
|
|
1377
|
+
}
|
|
1378
|
+
})
|
|
1379
|
+
.catch((err) => {
|
|
1380
|
+
this.logger.info(
|
|
1381
|
+
{
|
|
1382
|
+
err: (err as Error)?.message ?? err,
|
|
1383
|
+
queryHash,
|
|
1384
|
+
Category: 'sp00ky-client::Sp00kySync::scheduleListRefRefetch',
|
|
1385
|
+
},
|
|
1386
|
+
'Targeted list_ref re-read failed; the poll will retry'
|
|
1387
|
+
);
|
|
1388
|
+
});
|
|
1389
|
+
}, first);
|
|
1390
|
+
this.listRefRefetches.set(queryHash, { timer, ladder: rest });
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
private clearListRefRefetch(queryHash: string): void {
|
|
1394
|
+
const pending = this.listRefRefetches.get(queryHash);
|
|
1395
|
+
if (pending) {
|
|
1396
|
+
clearTimeout(pending.timer);
|
|
1397
|
+
this.listRefRefetches.delete(queryHash);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
private clearListRefRefetches(): void {
|
|
1402
|
+
for (const { timer } of this.listRefRefetches.values()) clearTimeout(timer);
|
|
1403
|
+
this.listRefRefetches.clear();
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
/**
|
|
1407
|
+
* Resolve the current `_00_list_ref` table name for the active auth
|
|
1408
|
+
* context. Public so the `createRemoteQuery` initial-fetch path can
|
|
1409
|
+
* read from the right per-user table.
|
|
1410
|
+
*
|
|
1411
|
+
* Reads the user id from `DataModule` rather than the local mirror,
|
|
1412
|
+
* because `DataModule.setCurrentUserId` runs synchronously from the
|
|
1413
|
+
* auth callback (before any `await`), whereas `sync.setCurrentUserId`
|
|
1414
|
+
* is async — the userQuery's initial fetch can fire between those
|
|
1415
|
+
* two points and we need the correct table name immediately.
|
|
1416
|
+
*/
|
|
1417
|
+
public listRefTable(): string {
|
|
1418
|
+
const userId = this.dataModule.getCurrentUserId();
|
|
1419
|
+
// Unauthenticated with the flag on → the shared `_00_list_ref_anon` table.
|
|
1420
|
+
if (userId == null && this.anonLiveEnabled) {
|
|
1421
|
+
return listRefTableFor(this.refMode, ANON_USER_ID);
|
|
1422
|
+
}
|
|
1423
|
+
return listRefTableFor(this.refMode, userId);
|
|
33
1424
|
}
|
|
34
1425
|
|
|
35
|
-
|
|
36
|
-
|
|
1426
|
+
private async killRefLiveQuery(): Promise<void> {
|
|
1427
|
+
if (this.liveQueryUnsubscribe) {
|
|
1428
|
+
try {
|
|
1429
|
+
this.liveQueryUnsubscribe();
|
|
1430
|
+
} catch {
|
|
1431
|
+
/* ignore */
|
|
1432
|
+
}
|
|
1433
|
+
this.liveQueryUnsubscribe = null;
|
|
1434
|
+
}
|
|
1435
|
+
if (this.currentLiveQueryUuid !== null) {
|
|
1436
|
+
// A LIVE subscription is scoped to its WebSocket session, so after a
|
|
1437
|
+
// reconnect the server-side one is already gone and there is nothing to
|
|
1438
|
+
// KILL. Sending it anyway either fails (no connection) or races the fresh
|
|
1439
|
+
// socket's readiness while holding up the restart behind it. Local
|
|
1440
|
+
// bookkeeping is cleared either way.
|
|
1441
|
+
if (this.remote.getStatus() === 'connected') {
|
|
1442
|
+
try {
|
|
1443
|
+
await this.remote.query('KILL $u', { u: this.currentLiveQueryUuid });
|
|
1444
|
+
} catch (err) {
|
|
1445
|
+
this.logger.debug(
|
|
1446
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::killRefLiveQuery' },
|
|
1447
|
+
'Prior LIVE KILL failed; continuing'
|
|
1448
|
+
);
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
this.currentLiveQueryUuid = null;
|
|
1452
|
+
}
|
|
37
1453
|
}
|
|
38
1454
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
(event) => cb(event.payload.queueSize)
|
|
43
|
-
);
|
|
44
|
-
const id2 = this.upQueue.events.subscribe(
|
|
45
|
-
SyncQueueEventTypes.MutationDequeued,
|
|
46
|
-
(event) => cb(event.payload.queueSize)
|
|
47
|
-
);
|
|
48
|
-
return () => {
|
|
49
|
-
this.upQueue.events.unsubscribe(id1);
|
|
50
|
-
this.upQueue.events.unsubscribe(id2);
|
|
51
|
-
};
|
|
1455
|
+
private async restartRefLiveQuery(): Promise<void> {
|
|
1456
|
+
await this.killRefLiveQuery();
|
|
1457
|
+
await this.startRefLiveQueries();
|
|
52
1458
|
}
|
|
53
1459
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
this.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
1460
|
+
/**
|
|
1461
|
+
* Drop local LIVE bookkeeping without issuing a `KILL`.
|
|
1462
|
+
*
|
|
1463
|
+
* Called when the socket dies. The server-side subscription is scoped to that
|
|
1464
|
+
* WebSocket session and died with it, so there is nothing left to kill — and
|
|
1465
|
+
* by the time the reconnect handler runs, the client reports `connected`
|
|
1466
|
+
* again, which would otherwise send a `KILL` for a stale uuid on the *new*
|
|
1467
|
+
* session and hold up the restart queued behind it.
|
|
1468
|
+
*/
|
|
1469
|
+
private invalidateRefLiveQuery(): void {
|
|
1470
|
+
if (this.liveQueryUnsubscribe) {
|
|
1471
|
+
try {
|
|
1472
|
+
this.liveQueryUnsubscribe();
|
|
1473
|
+
} catch {
|
|
1474
|
+
/* ignore */
|
|
1475
|
+
}
|
|
1476
|
+
this.liveQueryUnsubscribe = null;
|
|
1477
|
+
}
|
|
1478
|
+
this.currentLiveQueryUuid = null;
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
// Only the connect that follows a prior drop counts as a reconnect; the
|
|
1482
|
+
// initial connect after init() must not trigger a refetch storm.
|
|
1483
|
+
//
|
|
1484
|
+
// Both drop events have to be watched. The SDK publishes `disconnected` ONLY
|
|
1485
|
+
// when it has given up entirely (attempts exhausted, or the engine
|
|
1486
|
+
// terminated); an ordinary recovered drop goes `error` -> `reconnecting` ->
|
|
1487
|
+
// `connected` and never touches `disconnected`. Listening for `disconnected`
|
|
1488
|
+
// alone therefore misses every successful reconnect — the exact case this
|
|
1489
|
+
// handler exists for — and leaves the dead server-side LIVE in place with the
|
|
1490
|
+
// poll as the only sync path.
|
|
1491
|
+
private subscribeToReconnect() {
|
|
1492
|
+
const client = this.remote.getClient();
|
|
1493
|
+
client.subscribe('disconnected', () => {
|
|
1494
|
+
this.needsResubscribe = true;
|
|
1495
|
+
this.invalidateRefLiveQuery();
|
|
1496
|
+
this.logger.info(
|
|
1497
|
+
{ Category: 'sp00ky-client::Sp00kySync::onDisconnect' },
|
|
1498
|
+
'Remote disconnected'
|
|
1499
|
+
);
|
|
1500
|
+
});
|
|
1501
|
+
client.subscribe('reconnecting', () => {
|
|
1502
|
+
this.needsResubscribe = true;
|
|
1503
|
+
this.invalidateRefLiveQuery();
|
|
1504
|
+
this.logger.info(
|
|
1505
|
+
{ Category: 'sp00ky-client::Sp00kySync::onReconnecting' },
|
|
1506
|
+
'Remote socket dropped; awaiting reconnect'
|
|
1507
|
+
);
|
|
1508
|
+
});
|
|
1509
|
+
client.subscribe('connected', () => {
|
|
1510
|
+
if (!this.needsResubscribe) return;
|
|
1511
|
+
this.needsResubscribe = false;
|
|
1512
|
+
// A flapping socket produces reconnecting -> connected repeatedly, and
|
|
1513
|
+
// each cycle used to re-register EVERY active query (a busy app has
|
|
1514
|
+
// dozens). That is the "everything reloads about a second after a blip"
|
|
1515
|
+
// symptom: the SDK's retryDelay is 1s, so the refetch lands right after
|
|
1516
|
+
// the drop the user never saw. Collapse bursts into one refetch.
|
|
1517
|
+
const sinceLast = Date.now() - this.lastReconnectRefetchAt;
|
|
1518
|
+
if (sinceLast < Sp00kySync.RECONNECT_REFETCH_COOLDOWN_MS) {
|
|
1519
|
+
this.logger.debug(
|
|
1520
|
+
{ sinceLast, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
1521
|
+
'Reconnected again within the cooldown; skipping duplicate refetch'
|
|
1522
|
+
);
|
|
1523
|
+
return;
|
|
1524
|
+
}
|
|
1525
|
+
this.lastReconnectRefetchAt = Date.now();
|
|
1526
|
+
void this.refetchAfterReconnect();
|
|
1527
|
+
});
|
|
74
1528
|
}
|
|
75
1529
|
|
|
76
1530
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
1531
|
+
* Whether the REMOTE SESSION currently carries `$auth.id`.
|
|
1532
|
+
*
|
|
1533
|
+
* Not the same question as `currentUserId`, and that gap is the whole point:
|
|
1534
|
+
* `currentUserId` is this client's own record of who signed in and survives a
|
|
1535
|
+
* socket drop untouched, while `$auth` lives on the WebSocket session and has
|
|
1536
|
+
* to be re-applied after every reconnect. Registering in the window between
|
|
1537
|
+
* the two is silently destructive, because `fn::query::register` sends
|
|
1538
|
+
* `<string>($auth.id OR '')` and the SSP stores that value write-once: the
|
|
1539
|
+
* view's edges then route to the global `_00_list_ref` stamped `auth_id = ''`,
|
|
1540
|
+
* which that table's own permission rule (`auth_id = $auth.id`) makes
|
|
1541
|
+
* unreadable to the very user who registered it.
|
|
1542
|
+
*
|
|
1543
|
+
* Signed-out clients answer `true`: `''` is the honest identity there, not a
|
|
1544
|
+
* race.
|
|
81
1545
|
*/
|
|
82
|
-
|
|
83
|
-
if (this.
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
1546
|
+
private async remoteAuthEstablished(): Promise<boolean> {
|
|
1547
|
+
if (!this.currentUserId) return true;
|
|
1548
|
+
try {
|
|
1549
|
+
const result = await this.remote.query<[string]>("RETURN <string>($auth.id OR '')");
|
|
1550
|
+
const authId = Array.isArray(result) ? result[0] : undefined;
|
|
1551
|
+
return typeof authId === 'string' && authId.length > 0;
|
|
1552
|
+
} catch (err) {
|
|
1553
|
+
this.logger.debug(
|
|
1554
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::remoteAuthEstablished' },
|
|
1555
|
+
'Auth probe failed; treating the session as not yet authenticated'
|
|
1556
|
+
);
|
|
1557
|
+
return false;
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* Wait for the reconnected session to carry an identity again, then
|
|
1563
|
+
* re-register every active query and re-bind LIVE.
|
|
1564
|
+
*
|
|
1565
|
+
* Giving up without registering is deliberately better than registering
|
|
1566
|
+
* anyway: a registration made with no `$auth.id` produces a view its own
|
|
1567
|
+
* owner cannot read, and it is write-once, so it stays that way. Skipping
|
|
1568
|
+
* leaves the query unregistered and visibly loading, which the next
|
|
1569
|
+
* reconnect or heartbeat retries.
|
|
1570
|
+
*/
|
|
1571
|
+
private async refetchAfterReconnect(): Promise<void> {
|
|
1572
|
+
for (let attempt = 1; attempt <= Sp00kySync.AUTH_READY_MAX_ATTEMPTS; attempt++) {
|
|
1573
|
+
if (await this.remoteAuthEstablished()) break;
|
|
1574
|
+
if (attempt === Sp00kySync.AUTH_READY_MAX_ATTEMPTS) {
|
|
1575
|
+
this.logger.warn(
|
|
1576
|
+
{
|
|
1577
|
+
attempts: attempt,
|
|
1578
|
+
Category: 'sp00ky-client::Sp00kySync::onReconnect',
|
|
1579
|
+
},
|
|
1580
|
+
'Reconnected but the session still carries no $auth.id; not re-registering (a registration now would stamp every view with an empty identity)'
|
|
1581
|
+
);
|
|
1582
|
+
return;
|
|
1583
|
+
}
|
|
1584
|
+
await new Promise((resolve) => setTimeout(resolve, Sp00kySync.AUTH_READY_RETRY_MS));
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
1588
|
+
this.logger.info(
|
|
1589
|
+
{ queries: hashes.length, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
1590
|
+
'Remote reconnected, refetching active queries'
|
|
1591
|
+
);
|
|
1592
|
+
for (const hash of hashes) {
|
|
1593
|
+
this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
|
|
1594
|
+
}
|
|
1595
|
+
// The WS reconnect leaves the server-side LIVE subscription dead — the
|
|
1596
|
+
// re-enqueued `register` events only re-fetch initial state, they don't
|
|
1597
|
+
// re-subscribe. Without this, LIVE never recovers after a reconnect and
|
|
1598
|
+
// the poll silently becomes the sole sync path (and never backs off).
|
|
1599
|
+
// Authenticated → per-user table; signed-out with anon live enabled →
|
|
1600
|
+
// the shared `_00_list_ref_anon`. Otherwise there's no table to re-bind.
|
|
1601
|
+
if (this.currentUserId || this.anonLiveEnabled) {
|
|
1602
|
+
this.restartRefLiveQuery().catch((err) => {
|
|
1603
|
+
this.logger.debug(
|
|
1604
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
1605
|
+
'LIVE restart after reconnect failed; relying on poll fallback'
|
|
1606
|
+
);
|
|
1607
|
+
});
|
|
1608
|
+
}
|
|
91
1609
|
}
|
|
92
1610
|
|
|
93
1611
|
private async startRefLiveQueries() {
|
|
1612
|
+
// Shared-tabs follower: exactly one LIVE per user exists, on the leader;
|
|
1613
|
+
// its events reach this tab through the relay.
|
|
1614
|
+
if (this.tabRole === 'follower') return;
|
|
1615
|
+
const tableName = this.listRefTable();
|
|
94
1616
|
this.logger.debug(
|
|
95
|
-
{
|
|
1617
|
+
{ tableName, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
96
1618
|
'Starting ref live queries'
|
|
97
1619
|
);
|
|
98
1620
|
|
|
99
|
-
const [queryUuid] = await this.remote.query<[Uuid]>(
|
|
100
|
-
|
|
101
|
-
);
|
|
1621
|
+
const [queryUuid] = await this.remote.query<[Uuid]>(`LIVE SELECT * FROM ${tableName}`);
|
|
1622
|
+
this.currentLiveQueryUuid = queryUuid;
|
|
102
1623
|
|
|
103
|
-
|
|
1624
|
+
const live = await this.remote.getClient().liveOf(queryUuid);
|
|
1625
|
+
this.liveQueryUnsubscribe = live.subscribe((message) => {
|
|
104
1626
|
this.logger.debug(
|
|
105
|
-
{ message, Category: '
|
|
1627
|
+
{ message, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
106
1628
|
'Live update received'
|
|
107
1629
|
);
|
|
108
1630
|
if (message.action === 'KILLED') return;
|
|
1631
|
+
// Subquery child edges (rows with `parent` set) are NOT primary window
|
|
1632
|
+
// rows — the client's `RecordVersionArray` only tracks primary rows, so
|
|
1633
|
+
// routing them through `handleRemoteListRefChange` would surface them as
|
|
1634
|
+
// spurious "added" diffs and pollute the window. Instead route them to
|
|
1635
|
+
// the dedicated child-body sync so `.related()` data stays realtime
|
|
1636
|
+
// cross-session (the LIVE-permission gap otherwise leaves it to the poll).
|
|
1637
|
+
if ((message.value as { parent?: unknown }).parent != null) {
|
|
1638
|
+
this.handleRemoteSubqueryChange(
|
|
1639
|
+
message.action,
|
|
1640
|
+
message.value.in as RecordId<string>,
|
|
1641
|
+
message.value.out as RecordId<string>,
|
|
1642
|
+
message.value.version as number
|
|
1643
|
+
).catch((err) => {
|
|
1644
|
+
this.logger.error(
|
|
1645
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
1646
|
+
'Error handling remote subquery change'
|
|
1647
|
+
);
|
|
1648
|
+
});
|
|
1649
|
+
return;
|
|
1650
|
+
}
|
|
109
1651
|
this.handleRemoteListRefChange(
|
|
110
1652
|
message.action,
|
|
111
1653
|
message.value.in as RecordId<string>,
|
|
@@ -113,7 +1655,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
113
1655
|
message.value.version as number
|
|
114
1656
|
).catch((err) => {
|
|
115
1657
|
this.logger.error(
|
|
116
|
-
{ err, Category: '
|
|
1658
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
117
1659
|
'Error handling remote list ref change'
|
|
118
1660
|
);
|
|
119
1661
|
});
|
|
@@ -126,16 +1668,47 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
126
1668
|
recordId: RecordId,
|
|
127
1669
|
version: number
|
|
128
1670
|
) {
|
|
1671
|
+
// Any LIVE delivery is evidence of activity — a CREATE/UPDATE/DELETE on a
|
|
1672
|
+
// query's window, or a notification for an unknown local query. Reset the
|
|
1673
|
+
// poll's idle streak so it snaps back to the fast base cadence (the page
|
|
1674
|
+
// is clearly not idle), and record the timestamp as a liveness diagnostic.
|
|
1675
|
+
this.lastLiveEventAt = Date.now();
|
|
1676
|
+
this.listRefIdleStreak = 0;
|
|
1677
|
+
|
|
1678
|
+
// Shared-tabs leader: the list_ref table is USER-scoped, so this LIVE also
|
|
1679
|
+
// carries events for FOLLOWER tabs' queries (their own session-salted
|
|
1680
|
+
// hashes). Relay every primary event; each follower resolves the queryId
|
|
1681
|
+
// against its own DataModule and ignores foreign ones. Then continue with
|
|
1682
|
+
// this tab's own handling below.
|
|
1683
|
+
this.hub?.broadcast({
|
|
1684
|
+
type: 'list-ref-change',
|
|
1685
|
+
action,
|
|
1686
|
+
queryId: encodeRecordId(queryId),
|
|
1687
|
+
recordId: encodeRecordId(recordId),
|
|
1688
|
+
version,
|
|
1689
|
+
parent: false,
|
|
1690
|
+
});
|
|
1691
|
+
|
|
1692
|
+
// NOTE: DELETE is handled like CREATE/UPDATE below. When another window (or
|
|
1693
|
+
// this one) deletes a record, the server's SSP removes it from `_00_list_ref`
|
|
1694
|
+
// and the LIVE subscription delivers a DELETE here — `createDiffFromDbOp`
|
|
1695
|
+
// turns it into a `removed: [recordId]` diff so the row drops from the window
|
|
1696
|
+
// in realtime. (It was previously ignored, so other windows only caught up on
|
|
1697
|
+
// reload / the slow poll.)
|
|
129
1698
|
const existing = this.dataModule.getQueryById(queryId);
|
|
130
1699
|
|
|
131
1700
|
if (!existing) {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
1701
|
+
// With a hub attached, an unknown query is the NORMAL case (it belongs
|
|
1702
|
+
// to a follower tab); without one it still warrants the warning.
|
|
1703
|
+
if (!this.hub) {
|
|
1704
|
+
this.logger.warn(
|
|
1705
|
+
{
|
|
1706
|
+
queryId: queryId.toString(),
|
|
1707
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
1708
|
+
},
|
|
1709
|
+
'Received remote update for unknown local query'
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
139
1712
|
return;
|
|
140
1713
|
}
|
|
141
1714
|
|
|
@@ -148,12 +1721,184 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
148
1721
|
recordId,
|
|
149
1722
|
version,
|
|
150
1723
|
localArray,
|
|
151
|
-
Category: '
|
|
1724
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
152
1725
|
},
|
|
153
1726
|
'Live update is being processed'
|
|
154
1727
|
);
|
|
155
1728
|
const diff = createDiffFromDbOp(action, recordId, version, localArray);
|
|
156
|
-
|
|
1729
|
+
// `config.id` is `_00_query:<hash>`, so its id-part IS the query hash
|
|
1730
|
+
// (a SHA-256 over query content + sessionId) — the key DataModule uses.
|
|
1731
|
+
const hash = extractIdPart(existing.config.id);
|
|
1732
|
+
|
|
1733
|
+
// Apply the event to `remoteArray` — the authoritative membership rows are
|
|
1734
|
+
// now rendered FROM. Only registration and the poll used to write it, so a
|
|
1735
|
+
// LIVE removal left the departed id in the list (in memory and persisted)
|
|
1736
|
+
// until the next poll tick, which is up to 5s of showing a deleted row. This
|
|
1737
|
+
// also persists the durable `_00_window` mirror, so the removal survives a
|
|
1738
|
+
// reload with no network.
|
|
1739
|
+
//
|
|
1740
|
+
// Derived from the raw action, NOT from `diff`: `createDiffFromDbOp` is
|
|
1741
|
+
// empty when the circuit already holds the row at this version, which is
|
|
1742
|
+
// every tab that ingested the write optimistically (its own, or one
|
|
1743
|
+
// relayed from another tab). The fetch is rightly skipped then, but the
|
|
1744
|
+
// membership still has to be recorded, or the row lives on the
|
|
1745
|
+
// settled-write grace alone until the poll catches it.
|
|
1746
|
+
// A DELETE is not applied on arrival. Every teardown of a view - the TTL
|
|
1747
|
+
// sweep, an SSP reset, a scheduler wipe, a full republish - reaches the
|
|
1748
|
+
// client as one DELETE per edge, indistinguishable from a peer deleting
|
|
1749
|
+
// rows, and obeying them blanked painted lists n-1 rows at a time. Removals
|
|
1750
|
+
// are coalesced briefly and VERIFIED upstream before membership moves: an
|
|
1751
|
+
// id that no longer exists is dropped (and its body deleted); anything
|
|
1752
|
+
// else is settled by re-reading the server's whole set for the query, where
|
|
1753
|
+
// a missing `_00_query` row means "view lost, keep everything".
|
|
1754
|
+
if (action === 'DELETE') {
|
|
1755
|
+
this.queueLiveRemoval(hash, recordId);
|
|
1756
|
+
return;
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
if (existing.config.membershipKnown) {
|
|
1760
|
+
const membershipDiff: RecordVersionDiff = {
|
|
1761
|
+
added: [{ id: recordId, version }],
|
|
1762
|
+
updated: [],
|
|
1763
|
+
removed: [],
|
|
1764
|
+
};
|
|
1765
|
+
const next = applyRecordVersionDiff(existing.config.remoteArray ?? [], membershipDiff);
|
|
1766
|
+
if (!recordVersionArraysEqual(next, existing.config.remoteArray ?? [])) {
|
|
1767
|
+
await this.dataModule.updateQueryRemoteArray(hash, next);
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
await this.runSyncForQuery(hash, diff);
|
|
1772
|
+
|
|
1773
|
+
if (diff.added.length === 0 && diff.updated.length === 0) {
|
|
1774
|
+
// Empty diff: the circuit already holds the row at this version (this
|
|
1775
|
+
// tab wrote it), so no fetch and no stream update - but the membership
|
|
1776
|
+
// recorded above is new, and the subscribers have to hear about it.
|
|
1777
|
+
this.dataModule.scheduleRematerialize(hash);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
/** LIVE removals awaiting verification, per query, with their coalescing timer. */
|
|
1782
|
+
private pendingLiveRemovals = new Map<string, { ids: Map<string, RecordId>; timer: ReturnType<typeof setTimeout> }>();
|
|
1783
|
+
|
|
1784
|
+
/**
|
|
1785
|
+
* Hold a LIVE `_00_list_ref` DELETE for {@link LIVE_REMOVAL_COALESCE_MS},
|
|
1786
|
+
* then verify the batch (see {@link flushLiveRemovals}). A sweep or a
|
|
1787
|
+
* republish delivers a view's whole set as one burst, so one verification
|
|
1788
|
+
* round covers it.
|
|
1789
|
+
*/
|
|
1790
|
+
private queueLiveRemoval(hash: string, recordId: RecordId): void {
|
|
1791
|
+
const key = encodeRecordId(recordId);
|
|
1792
|
+
const pending = this.pendingLiveRemovals.get(hash);
|
|
1793
|
+
if (pending) {
|
|
1794
|
+
pending.ids.set(key, recordId);
|
|
1795
|
+
return;
|
|
1796
|
+
}
|
|
1797
|
+
const ids = new Map<string, RecordId>([[key, recordId]]);
|
|
1798
|
+
const timer = setTimeout(() => {
|
|
1799
|
+
this.pendingLiveRemovals.delete(hash);
|
|
1800
|
+
void this.flushLiveRemovals(hash, Array.from(ids.values())).catch((err) => {
|
|
1801
|
+
this.logger.info(
|
|
1802
|
+
{
|
|
1803
|
+
err: (err as Error)?.message ?? err,
|
|
1804
|
+
hash,
|
|
1805
|
+
Category: 'sp00ky-client::Sp00kySync::flushLiveRemovals',
|
|
1806
|
+
},
|
|
1807
|
+
'LIVE removal verification failed; the poll will converge'
|
|
1808
|
+
);
|
|
1809
|
+
});
|
|
1810
|
+
}, LIVE_REMOVAL_COALESCE_MS);
|
|
1811
|
+
this.pendingLiveRemovals.set(hash, { ids, timer });
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* Verify a coalesced batch of LIVE removals against the upstream records.
|
|
1816
|
+
* Ids that no longer exist upstream are dropped from membership (the body
|
|
1817
|
+
* delete already happened inside `syncRecords`) and the query re-renders.
|
|
1818
|
+
* Anything still present upstream - or a verification that could not run -
|
|
1819
|
+
* is settled by a targeted re-read of the server's set for this query,
|
|
1820
|
+
* where the `_00_query` row decides: gone = view lost (keep rows,
|
|
1821
|
+
* re-register), `materializing` = wait, `ready` = adopt the smaller set.
|
|
1822
|
+
*/
|
|
1823
|
+
private async flushLiveRemovals(hash: string, ids: RecordId[]): Promise<void> {
|
|
1824
|
+
const existing = this.dataModule.getQueryByHash(hash);
|
|
1825
|
+
if (!existing || ids.length === 0) return;
|
|
1826
|
+
const { confirmedRemovedIds, stillRemoteIds } = await this.runSyncForQuery(hash, {
|
|
1827
|
+
added: [],
|
|
1828
|
+
updated: [],
|
|
1829
|
+
removed: ids,
|
|
1830
|
+
});
|
|
1831
|
+
if (confirmedRemovedIds.length > 0 && existing.config.membershipKnown) {
|
|
1832
|
+
const gone = new Set(confirmedRemovedIds);
|
|
1833
|
+
const held = existing.config.remoteArray ?? [];
|
|
1834
|
+
const next = held.filter(([id]) => !gone.has(id));
|
|
1835
|
+
if (next.length !== held.length) {
|
|
1836
|
+
await this.dataModule.updateQueryRemoteArray(hash, next, { verifiedRemoval: true });
|
|
1837
|
+
}
|
|
1838
|
+
// A removal-only diff sets `fetching` false in `runSyncForQuery`, so it
|
|
1839
|
+
// gets no `flushPendingStreamUpdate`/`endFetching` re-render - and a
|
|
1840
|
+
// removal needs no record fetch to trigger one either. Force it.
|
|
1841
|
+
await this.dataModule.notifyQuerySynced(hash);
|
|
1842
|
+
}
|
|
1843
|
+
const unverified = confirmedRemovedIds.length + stillRemoteIds.length < ids.length;
|
|
1844
|
+
if (stillRemoteIds.length > 0 || unverified) {
|
|
1845
|
+
this.scheduleListRefRefetch(hash, 0);
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
private clearLiveRemovals(): void {
|
|
1850
|
+
for (const { timer } of this.pendingLiveRemovals.values()) clearTimeout(timer);
|
|
1851
|
+
this.pendingLiveRemovals.clear();
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
/**
|
|
1855
|
+
* Handle a LIVE change to a SUBQUERY child edge (a `_00_list_ref` row with
|
|
1856
|
+
* `parent` set) for a `.related()` query. Unlike primary rows, child rows
|
|
1857
|
+
* must NOT touch the query's `localArray`/`remoteArray`/`rowCount`; we only
|
|
1858
|
+
* keep the child BODY fresh in the local cache so the in-browser SSP's
|
|
1859
|
+
* subquery-table dependency re-materializes the parent view.
|
|
1860
|
+
*
|
|
1861
|
+
* CREATE/UPDATE fetch+upsert the child body. DELETE is intentionally a
|
|
1862
|
+
* no-op: a child leaving this query's set must not delete a body another
|
|
1863
|
+
* query may still show (see `syncSubqueryChildren` deletion-safety note);
|
|
1864
|
+
* a genuine record delete propagates via the normal delete path.
|
|
1865
|
+
*/
|
|
1866
|
+
private async handleRemoteSubqueryChange(
|
|
1867
|
+
action: 'CREATE' | 'UPDATE' | 'DELETE',
|
|
1868
|
+
queryId: RecordId,
|
|
1869
|
+
childId: RecordId,
|
|
1870
|
+
version: number
|
|
1871
|
+
) {
|
|
1872
|
+
this.lastLiveEventAt = Date.now();
|
|
1873
|
+
this.listRefIdleStreak = 0;
|
|
1874
|
+
|
|
1875
|
+
if (action === 'DELETE') return;
|
|
1876
|
+
|
|
1877
|
+
// Relay child-edge events too (see handleRemoteListRefChange).
|
|
1878
|
+
this.hub?.broadcast({
|
|
1879
|
+
type: 'list-ref-change',
|
|
1880
|
+
action,
|
|
1881
|
+
queryId: encodeRecordId(queryId),
|
|
1882
|
+
recordId: encodeRecordId(childId),
|
|
1883
|
+
version,
|
|
1884
|
+
parent: true,
|
|
1885
|
+
});
|
|
1886
|
+
|
|
1887
|
+
const existing = this.dataModule.getQueryById(queryId);
|
|
1888
|
+
if (!existing) return;
|
|
1889
|
+
|
|
1890
|
+
const item = { id: childId, version };
|
|
1891
|
+
await this.syncEngine.syncRecords(
|
|
1892
|
+
action === 'CREATE'
|
|
1893
|
+
? { added: [item], updated: [], removed: [] }
|
|
1894
|
+
: { added: [], updated: [item], removed: [] }
|
|
1895
|
+
);
|
|
1896
|
+
|
|
1897
|
+
// Keep the in-memory child array in step so the poll's idempotent diff
|
|
1898
|
+
// doesn't re-fetch this body on the next tick.
|
|
1899
|
+
const key = encodeRecordId(childId);
|
|
1900
|
+
const prev = existing.config.subqueryRemoteArray ?? [];
|
|
1901
|
+
existing.config.subqueryRemoteArray = [...prev.filter(([id]) => id !== key), [key, version]];
|
|
157
1902
|
}
|
|
158
1903
|
|
|
159
1904
|
/**
|
|
@@ -164,49 +1909,103 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
164
1909
|
this.scheduler.enqueueDownEvent(event);
|
|
165
1910
|
}
|
|
166
1911
|
|
|
1912
|
+
/**
|
|
1913
|
+
* Bound a mutation push so it always settles.
|
|
1914
|
+
*
|
|
1915
|
+
* `SyncScheduler.syncUp` early-returns while `isSyncingUp` is true, and that
|
|
1916
|
+
* flag only clears in the `finally` of the drain loop. A push whose RPC never
|
|
1917
|
+
* settles (socket dropped mid-flight, response lost) therefore wedges the
|
|
1918
|
+
* up-queue for the rest of the session: no retry, no error, no further
|
|
1919
|
+
* mutation ever sent. A timeout turns that into an ordinary network failure,
|
|
1920
|
+
* which `UpQueue.next` re-queues for the next trigger. The message deliberately
|
|
1921
|
+
* contains "timed out" so `classifySyncError` treats it as `network` and
|
|
1922
|
+
* retries rather than rolling the mutation back.
|
|
1923
|
+
*/
|
|
1924
|
+
private withPushTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
|
|
1925
|
+
return withTimeout(
|
|
1926
|
+
promise,
|
|
1927
|
+
this.pushTimeoutMs,
|
|
1928
|
+
`Mutation push timed out after ${this.pushTimeoutMs}ms (${label})`
|
|
1929
|
+
);
|
|
1930
|
+
}
|
|
1931
|
+
|
|
167
1932
|
private async processUpEvent(event: UpEvent) {
|
|
168
1933
|
this.logger.debug(
|
|
169
|
-
{ event, Category: '
|
|
1934
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
|
|
170
1935
|
'Processing up event'
|
|
171
1936
|
);
|
|
172
1937
|
switch (event.type) {
|
|
173
|
-
case 'create':
|
|
1938
|
+
case 'create': {
|
|
174
1939
|
const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
|
|
175
1940
|
const prefixedParams = Object.fromEntries(
|
|
176
1941
|
dataKeys.map(({ key, variable }) => [variable, event.data[key]])
|
|
177
1942
|
);
|
|
178
1943
|
const query = surql.seal(surql.createSet('id', dataKeys));
|
|
179
|
-
await this.
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
1944
|
+
await this.withPushTimeout(
|
|
1945
|
+
this.remote.query(query, {
|
|
1946
|
+
id: event.record_id,
|
|
1947
|
+
...prefixedParams,
|
|
1948
|
+
}),
|
|
1949
|
+
'create'
|
|
1950
|
+
);
|
|
183
1951
|
break;
|
|
1952
|
+
}
|
|
184
1953
|
case 'update':
|
|
185
|
-
await this.
|
|
186
|
-
id
|
|
187
|
-
|
|
188
|
-
|
|
1954
|
+
await this.withPushTimeout(
|
|
1955
|
+
this.remote.query(`UPDATE $id MERGE $data`, {
|
|
1956
|
+
id: event.record_id,
|
|
1957
|
+
data: event.data,
|
|
1958
|
+
}),
|
|
1959
|
+
'update'
|
|
1960
|
+
);
|
|
189
1961
|
break;
|
|
190
1962
|
case 'delete':
|
|
191
|
-
await this.
|
|
192
|
-
id
|
|
193
|
-
|
|
1963
|
+
await this.withPushTimeout(
|
|
1964
|
+
this.remote.query(`DELETE $id`, {
|
|
1965
|
+
id: event.record_id,
|
|
1966
|
+
}),
|
|
1967
|
+
'delete'
|
|
1968
|
+
);
|
|
194
1969
|
break;
|
|
195
1970
|
default:
|
|
196
1971
|
this.logger.error(
|
|
197
|
-
{ event, Category: '
|
|
1972
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
|
|
198
1973
|
'processUpEvent unknown event type'
|
|
199
1974
|
);
|
|
200
1975
|
return;
|
|
201
1976
|
}
|
|
202
1977
|
}
|
|
203
1978
|
|
|
1979
|
+
/**
|
|
1980
|
+
* A mutation the server accepted, reported once its outbox row is gone.
|
|
1981
|
+
*
|
|
1982
|
+
* Keeps the written row in the render set until its membership arrives.
|
|
1983
|
+
* Without this the row is briefly in neither term of
|
|
1984
|
+
* `(membership ∪ pendingWrites) − pendingDeletes` — the outbox delete is
|
|
1985
|
+
* tied to the push, while membership waits on the SSP ingesting the row,
|
|
1986
|
+
* materializing the view, writing the `_00_list_ref` edge and this client
|
|
1987
|
+
* reading it back. The writer therefore watched its own comment appear,
|
|
1988
|
+
* vanish, and return, while every other client showed it throughout.
|
|
1989
|
+
*/
|
|
1990
|
+
private handleMutationSettled(event: UpEvent): void {
|
|
1991
|
+
const recordId = encodeRecordId(event.record_id);
|
|
1992
|
+
this.dataModule.noteWriteSettled(recordId, event.type);
|
|
1993
|
+
// Shared-tabs: the outbox row just left the SHARED store, so every
|
|
1994
|
+
// follower rendering the row as a pending write has the same gap. All of
|
|
1995
|
+
// them, not just the owner: any tab whose query matched the row was
|
|
1996
|
+
// showing it through `pendingWrites`.
|
|
1997
|
+
this.hub?.broadcast({
|
|
1998
|
+
type: 'mutation-settled',
|
|
1999
|
+
mutationId: encodeRecordId(event.mutation_id),
|
|
2000
|
+
recordId,
|
|
2001
|
+
eventType: event.type,
|
|
2002
|
+
});
|
|
2003
|
+
}
|
|
2004
|
+
|
|
204
2005
|
private async handleRollback(event: UpEvent, error: Error): Promise<void> {
|
|
205
2006
|
const recordId = encodeRecordId(event.record_id);
|
|
206
2007
|
const tableName =
|
|
207
|
-
event.type === 'create' && event.tableName
|
|
208
|
-
? event.tableName
|
|
209
|
-
: extractTablePart(recordId);
|
|
2008
|
+
event.type === 'create' && event.tableName ? event.tableName : extractTablePart(recordId);
|
|
210
2009
|
|
|
211
2010
|
this.logger.warn(
|
|
212
2011
|
{
|
|
@@ -214,7 +2013,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
214
2013
|
recordId,
|
|
215
2014
|
tableName,
|
|
216
2015
|
error: error.message,
|
|
217
|
-
Category: '
|
|
2016
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
218
2017
|
},
|
|
219
2018
|
'Rolling back failed mutation'
|
|
220
2019
|
);
|
|
@@ -230,7 +2029,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
230
2029
|
this.logger.warn(
|
|
231
2030
|
{
|
|
232
2031
|
recordId,
|
|
233
|
-
Category: '
|
|
2032
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
234
2033
|
},
|
|
235
2034
|
'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
|
|
236
2035
|
);
|
|
@@ -240,7 +2039,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
240
2039
|
this.logger.warn(
|
|
241
2040
|
{
|
|
242
2041
|
recordId,
|
|
243
|
-
Category: '
|
|
2042
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
244
2043
|
},
|
|
245
2044
|
'Delete rollback not implemented. Down-sync will reconcile.'
|
|
246
2045
|
);
|
|
@@ -252,13 +2051,46 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
252
2051
|
recordId,
|
|
253
2052
|
error: error.message,
|
|
254
2053
|
});
|
|
2054
|
+
|
|
2055
|
+
// Shared-tabs: the store rollback above already propagated to every tab
|
|
2056
|
+
// via the ingest relay; additionally deliver the EVENT to the tab that
|
|
2057
|
+
// owns the mutation so its UI (toasts, subscribeToRollbacks) fires there.
|
|
2058
|
+
const mutationId = encodeRecordId(event.mutation_id);
|
|
2059
|
+
const owner = mutationOwnerTabId(mutationId);
|
|
2060
|
+
if (this.hub && owner && this.tabId && owner !== this.tabId) {
|
|
2061
|
+
this.hub.sendTo(owner, {
|
|
2062
|
+
type: 'mutation-rolled-back',
|
|
2063
|
+
mutationId,
|
|
2064
|
+
recordId,
|
|
2065
|
+
eventType: event.type,
|
|
2066
|
+
error: error.message,
|
|
2067
|
+
});
|
|
2068
|
+
}
|
|
255
2069
|
}
|
|
256
2070
|
|
|
257
2071
|
private async processDownEvent(event: DownEvent) {
|
|
258
2072
|
this.logger.debug(
|
|
259
|
-
{ event, Category: '
|
|
2073
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processDownEvent' },
|
|
260
2074
|
'Processing down event'
|
|
261
2075
|
);
|
|
2076
|
+
// Bounded for the same reason a push is (see withPushTimeout): an RPC that
|
|
2077
|
+
// never settles would otherwise hold its slot in the concurrent down drain
|
|
2078
|
+
// — and, before that drain existed, the WHOLE queue — for the rest of the
|
|
2079
|
+
// session, with no retry, no error, and every dependent `useQuery` stuck
|
|
2080
|
+
// loading. "timed out" in the message keeps `classifySyncError` treating it
|
|
2081
|
+
// as a network failure, so `DownQueue.run` re-heads it for the next pass.
|
|
2082
|
+
return this.withDownTimeout(this.runDownEvent(event), `${event.type} ${event.payload.hash}`);
|
|
2083
|
+
}
|
|
2084
|
+
|
|
2085
|
+
private withDownTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
|
|
2086
|
+
return withTimeout(
|
|
2087
|
+
promise,
|
|
2088
|
+
this.downTimeoutMs,
|
|
2089
|
+
`Down event timed out after ${this.downTimeoutMs}ms (${label})`
|
|
2090
|
+
);
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
private async runDownEvent(event: DownEvent): Promise<void> {
|
|
262
2094
|
switch (event.type) {
|
|
263
2095
|
case 'register':
|
|
264
2096
|
return this.registerQuery(event.payload.hash);
|
|
@@ -280,7 +2112,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
280
2112
|
const queryState = this.dataModule.getQueryByHash(hash);
|
|
281
2113
|
if (!queryState) {
|
|
282
2114
|
this.logger.warn(
|
|
283
|
-
{ hash, Category: '
|
|
2115
|
+
{ hash, Category: 'sp00ky-client::Sp00kySync::syncQuery' },
|
|
284
2116
|
'Query not found'
|
|
285
2117
|
);
|
|
286
2118
|
return;
|
|
@@ -294,7 +2126,141 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
294
2126
|
if (!diff) {
|
|
295
2127
|
return;
|
|
296
2128
|
}
|
|
297
|
-
|
|
2129
|
+
await this.runSyncForQuery(hash, diff);
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
/**
|
|
2133
|
+
* Run a sync for a single query while reflecting its fetch status. Marks the
|
|
2134
|
+
* query `fetching` for the duration when the diff actually pulls records
|
|
2135
|
+
* (added/updated), then resets to `idle` in a `finally` so a failed sync
|
|
2136
|
+
* never leaves a query stuck `fetching`. Part A's notification coalescing
|
|
2137
|
+
* means the single resulting UI update lands after this completes.
|
|
2138
|
+
*/
|
|
2139
|
+
private async runSyncForQuery(
|
|
2140
|
+
hash: string,
|
|
2141
|
+
diff: RecordVersionDiff
|
|
2142
|
+
): Promise<{ confirmedRemovedIds: string[]; stillRemoteIds: string[] }> {
|
|
2143
|
+
const nothing = { confirmedRemovedIds: [] as string[], stillRemoteIds: [] as string[] };
|
|
2144
|
+
// The diff was computed against whatever the circuit held at call time; if
|
|
2145
|
+
// the boot prime is still filling it, wait and recompute from the primed
|
|
2146
|
+
// `localArray` so the delta is real rather than "everything".
|
|
2147
|
+
const prime = this.primeGate();
|
|
2148
|
+
if (prime !== this.settledPrime) {
|
|
2149
|
+
await prime;
|
|
2150
|
+
this.settledPrime = prime;
|
|
2151
|
+
const fresh = this.dataModule.getQueryByHash(hash);
|
|
2152
|
+
if (!fresh) return nothing;
|
|
2153
|
+
const recomputed = new ArraySyncer(fresh.config.localArray, fresh.config.remoteArray).nextSet();
|
|
2154
|
+
if (!recomputed) return nothing;
|
|
2155
|
+
diff = recomputed;
|
|
2156
|
+
}
|
|
2157
|
+
// Don't let sync re-add a record the user just deleted locally. The remote
|
|
2158
|
+
// delete is queued in the outbox, so until it's processed the server's
|
|
2159
|
+
// `_00_list_ref` still lists the record — the diff then classifies it as
|
|
2160
|
+
// `added` (present remotely, absent locally) and `syncRecords` re-fetches +
|
|
2161
|
+
// re-inserts it, so a deleted database reappears a few seconds later. Drop
|
|
2162
|
+
// any id with a pending local DELETE from the re-add paths. Once the remote
|
|
2163
|
+
// delete lands, the pending row clears and the server drops it from
|
|
2164
|
+
// `_00_list_ref`, so this guard naturally stops applying.
|
|
2165
|
+
if (diff.added.length > 0 || diff.updated.length > 0) {
|
|
2166
|
+
const pendingDeletes = await this.getPendingDeleteIds();
|
|
2167
|
+
if (pendingDeletes.size > 0) {
|
|
2168
|
+
diff = {
|
|
2169
|
+
added: diff.added.filter((r) => !pendingDeletes.has(encodeRecordId(r.id))),
|
|
2170
|
+
updated: diff.updated.filter((r) => !pendingDeletes.has(encodeRecordId(r.id))),
|
|
2171
|
+
removed: diff.removed,
|
|
2172
|
+
};
|
|
2173
|
+
}
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
const fetching = diff.added.length + diff.updated.length > 0;
|
|
2177
|
+
if (fetching) {
|
|
2178
|
+
this.dataModule.beginFetching(hash);
|
|
2179
|
+
}
|
|
2180
|
+
let summary = nothing;
|
|
2181
|
+
try {
|
|
2182
|
+
const { remoteFetchMs, stillRemoteIds, confirmedRemovedIds } =
|
|
2183
|
+
await this.syncEngine.syncRecords(diff);
|
|
2184
|
+
summary = { confirmedRemovedIds, stillRemoteIds };
|
|
2185
|
+
if (fetching) {
|
|
2186
|
+
this.dataModule.recordRemoteFetch(hash, remoteFetchMs);
|
|
2187
|
+
}
|
|
2188
|
+
// Converge localArray to the authoritative remoteArray for ids that left
|
|
2189
|
+
// the server's list_ref but still exist — a view-membership change, not a
|
|
2190
|
+
// delete — so the poll's diff stops re-flagging them every tick (the `job:`
|
|
2191
|
+
// churn). CRUCIAL: only converge after the id has been still-remote for
|
|
2192
|
+
// several CONSECUTIVE rounds. A record that's merely mid-deletion is
|
|
2193
|
+
// still-remote for ~one round (its delete hasn't committed when our
|
|
2194
|
+
// existence check races it) and is gone the next round → it never reaches
|
|
2195
|
+
// the threshold, so it's deleted normally instead of being stranded here.
|
|
2196
|
+
if (stillRemoteIds.length > 0) {
|
|
2197
|
+
const CONVERGE_AFTER = 3;
|
|
2198
|
+
const toConverge: string[] = [];
|
|
2199
|
+
for (const id of stillRemoteIds) {
|
|
2200
|
+
const key = `${hash}:${id}`;
|
|
2201
|
+
const n = (this.stillRemoteStreaks.get(key) ?? 0) + 1;
|
|
2202
|
+
if (n >= CONVERGE_AFTER) {
|
|
2203
|
+
this.stillRemoteStreaks.delete(key);
|
|
2204
|
+
toConverge.push(id);
|
|
2205
|
+
} else {
|
|
2206
|
+
this.stillRemoteStreaks.set(key, n);
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
if (toConverge.length > 0) {
|
|
2210
|
+
const qs = this.dataModule.getQueryByHash(hash);
|
|
2211
|
+
const local = qs?.config.localArray;
|
|
2212
|
+
if (local && local.length > 0) {
|
|
2213
|
+
const drop = new Set(toConverge);
|
|
2214
|
+
const next = local.filter(([id]) => !drop.has(id));
|
|
2215
|
+
if (next.length !== local.length) {
|
|
2216
|
+
await this.dataModule.updateQueryLocalArray(hash, next);
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
} finally {
|
|
2222
|
+
if (fetching) {
|
|
2223
|
+
// Land the coalesced result BEFORE flipping to idle: the final stream
|
|
2224
|
+
// update sits on a debounce timer, and an `idle` that races ahead of it
|
|
2225
|
+
// would let consumers treat a partially-filled window as authoritative.
|
|
2226
|
+
//
|
|
2227
|
+
// A fetch that changed a row already in membership can come with no
|
|
2228
|
+
// stream update at all: the body lands in the local store, but the
|
|
2229
|
+
// circuit sees the same id-set and stays silent (measured: a peer's
|
|
2230
|
+
// UPDATE to a `conversation` row reached the store at `_00_rv` 8 while
|
|
2231
|
+
// the query kept rendering version 2 indefinitely). Nothing else
|
|
2232
|
+
// re-reads the query then, so ask for a synthetic re-materialize and
|
|
2233
|
+
// land it here too. It is one local select, and a no-op when a real
|
|
2234
|
+
// update was pending, which the first flush already processed.
|
|
2235
|
+
try {
|
|
2236
|
+
const landed = await this.dataModule.flushPendingStreamUpdate(hash);
|
|
2237
|
+
if (!landed) {
|
|
2238
|
+
this.dataModule.scheduleRematerialize(hash);
|
|
2239
|
+
await this.dataModule.flushPendingStreamUpdate(hash);
|
|
2240
|
+
}
|
|
2241
|
+
} catch (err) {
|
|
2242
|
+
this.logger.warn(
|
|
2243
|
+
{ err, hash, Category: 'sp00ky-client::Sp00kySync::runSyncForQuery' },
|
|
2244
|
+
'Failed to flush pending stream update before idle'
|
|
2245
|
+
);
|
|
2246
|
+
}
|
|
2247
|
+
this.dataModule.endFetching(hash);
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
return summary;
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
/**
|
|
2254
|
+
* Record ids with a pending local DELETE in the outbox (`_00_pending_mutations`).
|
|
2255
|
+
* Sync must not re-fetch/re-insert these — the remote delete is async, so the
|
|
2256
|
+
* server's `_00_list_ref` still lists them until it's processed, and the diff
|
|
2257
|
+
* would otherwise resurrect a just-deleted record.
|
|
2258
|
+
*/
|
|
2259
|
+
private async getPendingDeleteIds(): Promise<Set<string>> {
|
|
2260
|
+
// Single implementation, shared with the render path: `materializeRecords`
|
|
2261
|
+
// subtracts the same set so a row whose DELETE is still in the outbox is
|
|
2262
|
+
// neither re-fetched here nor rendered there.
|
|
2263
|
+
return (await this.dataModule.getPendingRecordIds()).deletes;
|
|
298
2264
|
}
|
|
299
2265
|
|
|
300
2266
|
/**
|
|
@@ -302,26 +2268,101 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
302
2268
|
* @param mutations Array of UpEvents (create/update/delete) to enqueue.
|
|
303
2269
|
*/
|
|
304
2270
|
public async enqueueMutation(mutations: UpEvent[]) {
|
|
2271
|
+
// Follower: the outbox rows are already committed in the SHARED store (the
|
|
2272
|
+
// mutation tx went through the leader's worker); only the leader drains,
|
|
2273
|
+
// so hand over the ids instead of queueing locally. A notify lost in a
|
|
2274
|
+
// failover window is covered by the new leader's loadFromDatabase.
|
|
2275
|
+
if (this.tabRole === 'follower') {
|
|
2276
|
+
for (const m of mutations) {
|
|
2277
|
+
this.forwarder?.mutationEnqueued(encodeRecordId(m.mutation_id));
|
|
2278
|
+
}
|
|
2279
|
+
return;
|
|
2280
|
+
}
|
|
305
2281
|
this.scheduler.enqueueMutation(mutations);
|
|
306
2282
|
}
|
|
307
2283
|
|
|
2284
|
+
/**
|
|
2285
|
+
* Best-effort release of THIS tab's views as the page goes away.
|
|
2286
|
+
*
|
|
2287
|
+
* Without it a closed tab's views stay materialized on the SSP for a full
|
|
2288
|
+
* TTL (10 minutes by default), because nothing else tears them down:
|
|
2289
|
+
* `releaseQueriesEagerly` is off, so the TTL sweep is the only reclaim path.
|
|
2290
|
+
* On a busy tenant that is a generation of live views per reload, each still
|
|
2291
|
+
* being stepped by every ingest, for ten minutes after the last human left.
|
|
2292
|
+
*
|
|
2293
|
+
* This is NOT `releaseQueriesEagerly` re-enabled. That released views
|
|
2294
|
+
* mid-session on a viewport change and tore the window out from under a live
|
|
2295
|
+
* page (see `cleanupQuery`); this runs only when the page itself is going
|
|
2296
|
+
* away and nothing is left to render.
|
|
2297
|
+
*
|
|
2298
|
+
* Safe against other tabs by construction: `fn::query::unsubscribe` drops
|
|
2299
|
+
* only this session from `subscribers` and deletes the row solely when it
|
|
2300
|
+
* was the last one, so a second tab of the same user keeps its view.
|
|
2301
|
+
*
|
|
2302
|
+
* One statement, not one per id: an unload handler gets a few milliseconds,
|
|
2303
|
+
* and N round trips would not survive it. Fire-and-forget — if the frame
|
|
2304
|
+
* does not make it out, the TTL sweep still reclaims exactly as before, so
|
|
2305
|
+
* the worst case is today's behaviour.
|
|
2306
|
+
*/
|
|
2307
|
+
releaseViewsOnUnload(): void {
|
|
2308
|
+
let ids: unknown[];
|
|
2309
|
+
try {
|
|
2310
|
+
ids = this.dataModule
|
|
2311
|
+
.getActiveQueryHashes()
|
|
2312
|
+
.map((hash) => this.dataModule.getQueryByHash(hash)?.config.id)
|
|
2313
|
+
.filter((id): id is NonNullable<typeof id> => id != null);
|
|
2314
|
+
} catch {
|
|
2315
|
+
return;
|
|
2316
|
+
}
|
|
2317
|
+
if (ids.length === 0) return;
|
|
2318
|
+
|
|
2319
|
+
// Over HTTP with `keepalive`, NOT the live socket. A WebSocket send during
|
|
2320
|
+
// `pagehide` is not guaranteed to flush — the browser may tear the socket
|
|
2321
|
+
// down first and the frame is lost. Measured on staging: releasing over the
|
|
2322
|
+
// socket reached the server zero times; `fetch`/`keepalive` is the only
|
|
2323
|
+
// primitive the platform promises to finish after the page is gone.
|
|
2324
|
+
//
|
|
2325
|
+
// Ids are inlined rather than bound because this is a bare statement, not
|
|
2326
|
+
// an RPC call with a params channel. They are `_00_query:<sha256>` record
|
|
2327
|
+
// ids the client itself derived, so the only interpolation is a hex digest.
|
|
2328
|
+
const list = ids
|
|
2329
|
+
.map((id) => String(id))
|
|
2330
|
+
.filter((id) => /^_00_query:[0-9a-f]{64}$/.test(id))
|
|
2331
|
+
.join(', ');
|
|
2332
|
+
if (!list) return;
|
|
2333
|
+
|
|
2334
|
+
this.remote.beaconSql(
|
|
2335
|
+
`FOR $id IN [${list}] { LET $_released = fn::query::unsubscribe($id); };`
|
|
2336
|
+
);
|
|
2337
|
+
}
|
|
2338
|
+
|
|
308
2339
|
private async registerQuery(queryHash: string) {
|
|
2340
|
+
// Hold `fetching` across the WHOLE registration (remote view creation +
|
|
2341
|
+
// initial sync + post-sync notify). A query is born `fetching` in
|
|
2342
|
+
// createNewQuery; this refcounted cycle is what resolves it to `idle` — so
|
|
2343
|
+
// consumers (e.g. useQuery's `isSettled`) never see an idle query whose
|
|
2344
|
+
// window is still empty/partially materialized.
|
|
2345
|
+
this.dataModule.beginFetching(queryHash);
|
|
309
2346
|
try {
|
|
310
2347
|
this.logger.debug(
|
|
311
|
-
{ queryHash, Category: '
|
|
2348
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
|
|
312
2349
|
'Register Query state'
|
|
313
2350
|
);
|
|
314
2351
|
await this.createRemoteQuery(queryHash);
|
|
315
2352
|
await this.syncQuery(queryHash);
|
|
316
|
-
//
|
|
317
|
-
// where no stream updates fire but the UI needs to
|
|
2353
|
+
// Land any still-debounced stream result, then always notify — handles
|
|
2354
|
+
// empty result sets where no stream updates fire but the UI needs to
|
|
2355
|
+
// stop loading.
|
|
2356
|
+
await this.dataModule.flushPendingStreamUpdate(queryHash);
|
|
318
2357
|
await this.dataModule.notifyQuerySynced(queryHash);
|
|
319
2358
|
} catch (e) {
|
|
320
2359
|
this.logger.error(
|
|
321
|
-
{ err: e, Category: '
|
|
2360
|
+
{ err: e, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
|
|
322
2361
|
'registerQuery error'
|
|
323
2362
|
);
|
|
324
2363
|
throw e;
|
|
2364
|
+
} finally {
|
|
2365
|
+
this.dataModule.endFetching(queryHash);
|
|
325
2366
|
}
|
|
326
2367
|
}
|
|
327
2368
|
|
|
@@ -330,15 +2371,15 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
330
2371
|
|
|
331
2372
|
if (!queryState) {
|
|
332
2373
|
this.logger.warn(
|
|
333
|
-
{ queryHash, Category: '
|
|
2374
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::createRemoteQuery' },
|
|
334
2375
|
'Query to register not found'
|
|
335
2376
|
);
|
|
336
2377
|
throw new Error('Query to register not found');
|
|
337
2378
|
}
|
|
338
|
-
// Delegate to remote function which handles DBSP registration & persistence
|
|
2379
|
+
// Delegate to remote function which handles DBSP registration & persistence.
|
|
2380
|
+
// clientId is set server-side from session::id() — see fn::query::register.
|
|
339
2381
|
await this.remote.query('fn::query::register($config)', {
|
|
340
2382
|
config: {
|
|
341
|
-
clientId: this.clientId,
|
|
342
2383
|
id: queryState.config.id,
|
|
343
2384
|
surql: queryState.config.surql,
|
|
344
2385
|
params: queryState.config.params,
|
|
@@ -346,64 +2387,240 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
346
2387
|
},
|
|
347
2388
|
});
|
|
348
2389
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
);
|
|
2390
|
+
// Initial materialized-view fetch — pull from the same per-user
|
|
2391
|
+
// `_00_list_ref_user_<id>` (or global `_00_list_ref` in single
|
|
2392
|
+
// mode) that the LIVE subscription listens on, so the two stay in
|
|
2393
|
+
// sync. `parent IS NONE` excludes subquery entries; the
|
|
2394
|
+
// `localArray` cache only tracks primary records.
|
|
2395
|
+
const listRefTbl = this.listRefTable();
|
|
2396
|
+
// `rowCount` rides along: it is written by the SSP in the same statement
|
|
2397
|
+
// that registers the view, BEFORE the edges are flushed, so it is the only
|
|
2398
|
+
// way to tell "this query is empty" from "its edges have not landed yet".
|
|
2399
|
+
const [items, metaRow] = await this.remote.query<
|
|
2400
|
+
[{ out: RecordId<string>; version: number }[], QueryMetaRow | null]
|
|
2401
|
+
>(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()}`, {
|
|
2402
|
+
in: queryState.config.id,
|
|
2403
|
+
});
|
|
2404
|
+
const meta = metaFromRow(metaRow);
|
|
355
2405
|
|
|
356
2406
|
this.logger.trace(
|
|
357
2407
|
{
|
|
358
2408
|
queryId: encodeRecordId(queryState.config.id),
|
|
359
2409
|
items,
|
|
360
|
-
Category: '
|
|
2410
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
361
2411
|
},
|
|
362
2412
|
'Got query record version array from remote'
|
|
363
2413
|
);
|
|
364
2414
|
|
|
365
|
-
const array: RecordVersionArray =
|
|
2415
|
+
const array: RecordVersionArray = dedupeRecordVersions(
|
|
2416
|
+
items.map((item) => [encodeRecordId(item.out), item.version])
|
|
2417
|
+
);
|
|
366
2418
|
|
|
367
2419
|
this.logger.debug(
|
|
368
2420
|
{
|
|
369
2421
|
queryId: encodeRecordId(queryState.config.id),
|
|
370
2422
|
array,
|
|
371
|
-
Category: '
|
|
2423
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
372
2424
|
},
|
|
373
2425
|
'createdRemoteQuery'
|
|
374
2426
|
);
|
|
375
2427
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
2428
|
+
const outcome = await this.dataModule.updateQueryRemoteArray(queryHash, array, { meta });
|
|
2429
|
+
// A registration that answered before its edges landed: the SSP wrote
|
|
2430
|
+
// `materializing` and handed the publish to its flusher (100ms window).
|
|
2431
|
+
// Do not leave the fresh query waiting for a backed-off poll tick; re-read
|
|
2432
|
+
// it a few times on a short ladder. LIVE CREATEs are ignored for a query
|
|
2433
|
+
// whose membership is still unknown, so this is the only fast path.
|
|
2434
|
+
if (outcome === 'ignored' && meta.present && meta.state === 'materializing') {
|
|
2435
|
+
this.scheduleListRefRefetch(queryHash, MATERIALIZING_REFETCH_LADDER_MS);
|
|
2436
|
+
}
|
|
2437
|
+
|
|
2438
|
+
// Pull the bodies of any `.related()` subquery children into the local
|
|
2439
|
+
// cache. The primary fetch above (`parent IS NONE`) tracks only window
|
|
2440
|
+
// rows, so without this a cold-reload re-materialization of the
|
|
2441
|
+
// correlated surql finds no child rows and related fields come back
|
|
2442
|
+
// empty. Best-effort: never fail registration over it.
|
|
2443
|
+
await this.syncSubqueryChildren(queryHash).catch((err) => {
|
|
2444
|
+
this.logger.info(
|
|
2445
|
+
{
|
|
2446
|
+
err: (err as Error)?.message ?? err,
|
|
2447
|
+
queryHash,
|
|
2448
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
2449
|
+
},
|
|
2450
|
+
'Subquery child sync failed during registration; poll will retry'
|
|
2451
|
+
);
|
|
2452
|
+
});
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
/**
|
|
2456
|
+
* Sync the BODIES of a `.related()` query's subquery child rows into the
|
|
2457
|
+
* local cache, separately from the primary window array. The SSP writes
|
|
2458
|
+
* each matched child as a `_00_list_ref` edge tagged `parent`/`parent_rel`;
|
|
2459
|
+
* `buildSubqueryListRefSelect` pulls those `out`+`version` pairs (any
|
|
2460
|
+
* nesting depth). We diff against the in-memory `subqueryRemoteArray` and
|
|
2461
|
+
* fetch added/updated bodies through the SyncEngine — which `saveBatch`s
|
|
2462
|
+
* them into the local DB AND the in-browser SSP, whose subquery-table
|
|
2463
|
+
* dependency then re-materializes the parent view (no explicit notify).
|
|
2464
|
+
*
|
|
2465
|
+
* Deletion safety: we pass `removed: []` deliberately. A child body can be
|
|
2466
|
+
* shared by other queries; letting `handleRemovedRecords` delete one that
|
|
2467
|
+
* merely left THIS query's child set would clobber data another query still
|
|
2468
|
+
* shows. Genuine record deletes flow through the normal delete path; a
|
|
2469
|
+
* lingering orphan body is invisible (the correlated WHERE stops matching).
|
|
2470
|
+
*
|
|
2471
|
+
* Kept off `runSyncForQuery` on purpose so child fetches never flip the
|
|
2472
|
+
* query to `fetching` or skew its DevTools timings.
|
|
2473
|
+
*/
|
|
2474
|
+
private async syncSubqueryChildren(queryHash: string): Promise<void> {
|
|
2475
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
2476
|
+
if (!queryState) return;
|
|
2477
|
+
|
|
2478
|
+
const listRefTbl = this.listRefTable();
|
|
2479
|
+
const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
|
|
2480
|
+
buildSubqueryListRefSelect(listRefTbl),
|
|
2481
|
+
{ in: queryState.config.id }
|
|
2482
|
+
);
|
|
2483
|
+
if (!Array.isArray(items)) return;
|
|
2484
|
+
|
|
2485
|
+
const fresh: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
|
|
2486
|
+
await this.applySubqueryChildren(queryHash, fresh);
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
/**
|
|
2490
|
+
* Converge a `.related()` query's child bodies onto `fresh` (the server's
|
|
2491
|
+
* current child edges). Shared by the registration path above and the poll,
|
|
2492
|
+
* which reads the child edges in its batched round trip. See
|
|
2493
|
+
* {@link syncSubqueryChildren} for the deletion-safety rules.
|
|
2494
|
+
*/
|
|
2495
|
+
private async applySubqueryChildren(queryHash: string, fresh: RecordVersionArray): Promise<void> {
|
|
2496
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
2497
|
+
if (!queryState) return;
|
|
2498
|
+
const prev = queryState.config.subqueryRemoteArray ?? [];
|
|
2499
|
+
if (recordVersionArraysEqual(fresh, prev)) return; // idempotent: nothing new
|
|
2500
|
+
|
|
2501
|
+
const diff = diffRecordVersionArray(prev, fresh);
|
|
2502
|
+
if (diff.added.length > 0 || diff.updated.length > 0) {
|
|
2503
|
+
await this.syncEngine.syncRecords({
|
|
2504
|
+
added: diff.added,
|
|
2505
|
+
updated: diff.updated,
|
|
2506
|
+
removed: [], // never delete child bodies here — see method doc
|
|
2507
|
+
});
|
|
379
2508
|
}
|
|
2509
|
+
// In-memory only — child rows must never enter the persisted primary array.
|
|
2510
|
+
queryState.config.subqueryRemoteArray = fresh;
|
|
380
2511
|
}
|
|
381
2512
|
|
|
382
|
-
|
|
2513
|
+
public async heartbeatQuery(queryHash: string) {
|
|
383
2514
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
384
2515
|
if (!queryState) {
|
|
385
2516
|
this.logger.warn(
|
|
386
|
-
{ queryHash, Category: '
|
|
2517
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::heartbeatQuery' },
|
|
387
2518
|
'Query to register not found'
|
|
388
2519
|
);
|
|
389
2520
|
throw new Error('Query to register not found');
|
|
390
2521
|
}
|
|
391
|
-
|
|
2522
|
+
// `fn::query::heartbeat` is an `UPDATE $id SET ...`. On a record that no
|
|
2523
|
+
// longer exists that matches nothing and returns an empty array — it does
|
|
2524
|
+
// NOT recreate the row. So an unchecked heartbeat is indistinguishable from
|
|
2525
|
+
// a successful one, and a client whose row was reclaimed keeps beating
|
|
2526
|
+
// against nothing forever: no membership, no edges, no re-registration.
|
|
2527
|
+
// The page renders as if the data were deleted ("Game not found").
|
|
2528
|
+
//
|
|
2529
|
+
// A live query's row is reclaimed more easily than it looks. The sweep
|
|
2530
|
+
// expires on `lastActiveAt + ttl`, and this heartbeat runs on a timer that
|
|
2531
|
+
// browsers throttle hard in background tabs — so a second window left idle
|
|
2532
|
+
// past its TTL is the ordinary way to get here, not an edge case. Until
|
|
2533
|
+
// canary.194 the sweep could not actually remove the in-memory view (it
|
|
2534
|
+
// looked it up under the other of the two query-id spellings), which masked
|
|
2535
|
+
// this: the view survived its own row. Now reclamation is real, so the
|
|
2536
|
+
// client has to notice and rebuild.
|
|
2537
|
+
const result = await this.remote.query('fn::query::heartbeat($id)', {
|
|
392
2538
|
id: queryState.config.id,
|
|
393
2539
|
});
|
|
2540
|
+
const updated = Array.isArray(result) ? result[0] : undefined;
|
|
2541
|
+
const rowGone = Array.isArray(updated) && updated.length === 0;
|
|
2542
|
+
if (!rowGone) return;
|
|
2543
|
+
|
|
2544
|
+
this.logger.warn(
|
|
2545
|
+
{
|
|
2546
|
+
queryHash,
|
|
2547
|
+
id: String(queryState.config.id),
|
|
2548
|
+
Category: 'sp00ky-client::Sp00kySync::heartbeatQuery',
|
|
2549
|
+
},
|
|
2550
|
+
'Query row was reclaimed while still in use; re-registering'
|
|
2551
|
+
);
|
|
2552
|
+
// Re-register rather than recreate the row here: the row alone is useless
|
|
2553
|
+
// without the SSP view behind it, and only registration rebuilds the view,
|
|
2554
|
+
// republishes `_00_list_ref` and writes `rowCount`.
|
|
2555
|
+
this.enqueueDownEvent({ type: 'register', payload: { hash: queryHash } });
|
|
394
2556
|
}
|
|
395
2557
|
|
|
2558
|
+
// Eager teardown of a deregistered query's remote `_00_query` view (opt-in,
|
|
2559
|
+
// e.g. a viewport-windowed list cancelling an off-screen window). Query ids
|
|
2560
|
+
// are a deterministic hash of (surql+params), so a release racing a
|
|
2561
|
+
// scroll-back re-register (same id) could nuke a freshly-recreated view —
|
|
2562
|
+
// hence two guards: abort if a subscriber reappeared BEFORE the release;
|
|
2563
|
+
// re-register if one reappears DURING the release's network await. Tolerant
|
|
2564
|
+
// of a missing/already-gone query (no throw).
|
|
2565
|
+
//
|
|
2566
|
+
// Releases via `fn::query::unsubscribe` rather than deleting the row
|
|
2567
|
+
// outright. A `_00_query` row can be shared by several sessions of the same
|
|
2568
|
+
// user, so a bare `DELETE` would tear the view — and every `_00_list_ref`
|
|
2569
|
+
// edge hanging off it — out from under other live tabs. The function drops
|
|
2570
|
+
// only this session from `subscribers` and deletes the row when it was the
|
|
2571
|
+
// last one. (The old `DELETE $id` was harmless in practice only because the
|
|
2572
|
+
// table granted no delete permission and it silently affected zero rows.)
|
|
396
2573
|
private async cleanupQuery(queryHash: string) {
|
|
397
2574
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
398
|
-
if (!queryState)
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
2575
|
+
if (!queryState) return; // already torn down / never registered
|
|
2576
|
+
|
|
2577
|
+
// Re-subscribed before the queued cleanup ran → keep everything as-is.
|
|
2578
|
+
if (this.dataModule.hasSubscribers(queryHash)) return;
|
|
2579
|
+
|
|
2580
|
+
// EAGER REMOTE RELEASE IS DISABLED. Deliberate, and not a leak: the TTL
|
|
2581
|
+
// sweep reclaims the row and its edges on `lastActiveAt + ttl`, which is the
|
|
2582
|
+
// ONLY reclamation that has ever actually run in production.
|
|
2583
|
+
//
|
|
2584
|
+
// Until canary.190 `_00_query` granted no delete permission, so the bare
|
|
2585
|
+
// `DELETE $id` this used to issue affected zero rows. .190 granted delete
|
|
2586
|
+
// and .191 wired `fn::query::unsubscribe`, which made teardown real for the
|
|
2587
|
+
// first time -- and the guards above are best-effort by construction
|
|
2588
|
+
// (`hasSubscribers` can be momentarily false during a rebind or a windowed
|
|
2589
|
+
// list re-flow). Every misfire that had been silently inert for months
|
|
2590
|
+
// became a live delete of the row AND every `_00_list_ref` edge on it.
|
|
2591
|
+
//
|
|
2592
|
+
// That matches a report of chat suddenly rendering raw record ids instead
|
|
2593
|
+
// of users, with the message list re-flowing underneath. Server state was
|
|
2594
|
+
// measured intact at the time (`rowCount` equalled the actual edge count on
|
|
2595
|
+
// every row), so the damage is on the client side of a teardown, not in the
|
|
2596
|
+
// materialization.
|
|
2597
|
+
//
|
|
2598
|
+
// Re-enable only together with a repair path that can re-fetch subquery
|
|
2599
|
+
// child bodies whose `subqueryRemoteArray` entry claims they are already
|
|
2600
|
+
// synced -- otherwise a torn-down-and-recreated view never restores the
|
|
2601
|
+
// related records it dropped, because the idempotence check skips them.
|
|
2602
|
+
if (this.releaseQueriesEagerly) {
|
|
2603
|
+
await this.remote.query('fn::query::unsubscribe($id)', {
|
|
2604
|
+
id: queryState.config.id,
|
|
2605
|
+
});
|
|
2606
|
+
|
|
2607
|
+
// Re-subscribed while we awaited the release → re-register. Covers both
|
|
2608
|
+
// outcomes: if we were the last subscriber the remote view is gone and
|
|
2609
|
+
// this recreates it, and if it survived for other sessions this re-adds
|
|
2610
|
+
// us to `subscribers` so our heartbeats keep counting.
|
|
2611
|
+
if (this.dataModule.hasSubscribers(queryHash)) {
|
|
2612
|
+
this.enqueueDownEvent({ type: 'register', payload: { hash: queryHash } });
|
|
2613
|
+
return;
|
|
2614
|
+
}
|
|
404
2615
|
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
2616
|
+
|
|
2617
|
+
// No subscribers throughout → safe to free the local view + state.
|
|
2618
|
+
this.clearListRefRefetch(queryHash);
|
|
2619
|
+
const pendingRemoval = this.pendingLiveRemovals.get(queryHash);
|
|
2620
|
+
if (pendingRemoval) {
|
|
2621
|
+
clearTimeout(pendingRemoval.timer);
|
|
2622
|
+
this.pendingLiveRemovals.delete(queryHash);
|
|
2623
|
+
}
|
|
2624
|
+
this.dataModule.finalizeDeregister(queryHash);
|
|
408
2625
|
}
|
|
409
2626
|
}
|