@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1406 -364
- package/dist/index.js +7754 -1390
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-open.js +276 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +420 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +434 -0
- package/dist/types.d.ts +874 -0
- package/package.json +20 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +16 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +112 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +361 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1151 -129
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +325 -37
- package/src/modules/devtools/notify-throttle.test.ts +149 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +143 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +137 -0
- package/src/modules/sync/queue/queue-up.ts +198 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +77 -9
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1333 -92
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +160 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +478 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +464 -0
- package/src/services/database/sqlite-cache-engine.ts +1154 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +106 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +113 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +219 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +141 -0
- package/src/services/logger/index.ts +6 -109
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +444 -52
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +23 -2
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +278 -0
- package/src/services/tabs/coordinator.test.ts +192 -0
- package/src/services/tabs/coordinator.ts +567 -0
- package/src/services/tabs/fake-ports.fixture.ts +70 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +239 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +581 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.ts +1225 -0
- package/src/types.ts +342 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -1,28 +1,35 @@
|
|
|
1
1
|
import { RecordId, Duration } from 'surrealdb';
|
|
2
|
-
import {
|
|
2
|
+
import type {
|
|
3
3
|
SchemaStructure,
|
|
4
4
|
TableNames,
|
|
5
5
|
BackendNames,
|
|
6
6
|
BackendRoutes,
|
|
7
7
|
RoutePayload,
|
|
8
|
+
QueryPlan,
|
|
8
9
|
} from '@spooky-sync/query-builder';
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
10
|
+
import type { LocalStore } from '../../services/database/index';
|
|
11
|
+
import { StaleEpochError } from '../../services/database/index';
|
|
12
|
+
import type { CacheModule, RecordWithId, CacheRecord } from '../cache/index';
|
|
13
|
+
import type { Logger } from '../../services/logger/index';
|
|
14
|
+
import type { StreamUpdate } from '../../services/stream-processor/index';
|
|
15
|
+
import type {
|
|
15
16
|
QueryConfig,
|
|
16
17
|
QueryHash,
|
|
17
18
|
QueryState,
|
|
19
|
+
QueryStatus,
|
|
20
|
+
QueryStatusCallback,
|
|
18
21
|
QueryTimeToLive,
|
|
19
22
|
QueryUpdateCallback,
|
|
20
23
|
MutationCallback,
|
|
21
24
|
RecordVersionArray,
|
|
22
25
|
QueryConfigRecord,
|
|
23
26
|
UpdateOptions,
|
|
27
|
+
QueryTimings,
|
|
28
|
+
PhaseStat,
|
|
29
|
+
RegistrationTimings,
|
|
24
30
|
RunOptions,
|
|
25
31
|
} from '../../types';
|
|
32
|
+
import { MATERIALIZATION_SAMPLE_WINDOW } from '../../types';
|
|
26
33
|
import {
|
|
27
34
|
parseRecordIdString,
|
|
28
35
|
extractIdPart,
|
|
@@ -31,11 +38,30 @@ import {
|
|
|
31
38
|
withRetry,
|
|
32
39
|
surql,
|
|
33
40
|
parseParams,
|
|
41
|
+
cleanRecord,
|
|
34
42
|
extractTablePart,
|
|
35
43
|
generateId,
|
|
36
44
|
} from '../../utils/index';
|
|
37
|
-
import { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
|
|
38
|
-
import { PushEventOptions } from '../../events/index';
|
|
45
|
+
import type { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
|
|
46
|
+
import type { PushEventOptions } from '../../events/index';
|
|
47
|
+
import { buildWindowMaterialization, buildWindowMaterializationPlan } from './window-query';
|
|
48
|
+
import { mintMutationId } from './mutation-id';
|
|
49
|
+
|
|
50
|
+
/** Push a timing sample (ms) into a rolling window, capped at the sample window. */
|
|
51
|
+
function pushSample(samples: number[], ms: number): void {
|
|
52
|
+
samples.push(ms);
|
|
53
|
+
if (samples.length > MATERIALIZATION_SAMPLE_WINDOW) samples.shift();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Build a {lastMs,p50,p90,p99,count} summary from a rolling sample window. */
|
|
57
|
+
function phaseStatOf(samples: number[], lastMs: number | null): PhaseStat {
|
|
58
|
+
if (samples.length === 0) {
|
|
59
|
+
return { lastMs, p50: null, p90: null, p99: null, count: 0 };
|
|
60
|
+
}
|
|
61
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
62
|
+
const pick = (q: number) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!;
|
|
63
|
+
return { lastMs, p50: pick(0.5), p90: pick(0.9), p99: pick(0.99), count: samples.length };
|
|
64
|
+
}
|
|
39
65
|
|
|
40
66
|
/**
|
|
41
67
|
* DataModule - Unified query and mutation management
|
|
@@ -44,25 +70,117 @@ import { PushEventOptions } from '../../events/index';
|
|
|
44
70
|
* Uses CacheModule for all storage operations.
|
|
45
71
|
*/
|
|
46
72
|
export class DataModule<S extends SchemaStructure> {
|
|
73
|
+
/** Tab identity baked into mutation ids (shared-tabs rollback routing);
|
|
74
|
+
* undefined in solo mode, where mutation-id falls back to a session id. */
|
|
75
|
+
private tabId: string | undefined;
|
|
47
76
|
private activeQueries: Map<QueryHash, QueryState> = new Map();
|
|
48
77
|
private pendingQueries: Map<QueryHash, Promise<QueryHash>> = new Map();
|
|
49
78
|
private subscriptions: Map<QueryHash, Set<QueryUpdateCallback>> = new Map();
|
|
79
|
+
private statusSubscriptions: Map<QueryHash, Set<QueryStatusCallback>> = new Map();
|
|
50
80
|
private mutationCallbacks: Set<MutationCallback> = new Set();
|
|
51
81
|
private debounceTimers: Map<QueryHash, NodeJS.Timeout> = new Map();
|
|
82
|
+
// The update each debounce timer would process on its trailing edge. Kept in a
|
|
83
|
+
// map (not just the timer closure) so `flushPendingStreamUpdate` can process
|
|
84
|
+
// it early — the sync engine flushes before flipping a query to `idle`, so
|
|
85
|
+
// subscribers never observe idle status with stale (partial-window) rows.
|
|
86
|
+
private pendingStreamUpdates: Map<QueryHash, StreamUpdate> = new Map();
|
|
87
|
+
// Refcount of in-flight fetch cycles per query (registration + concurrent
|
|
88
|
+
// poll/LIVE sync rounds can overlap). Status flips to `fetching` on 0→1 and
|
|
89
|
+
// back to `idle` only on 1→0, so an inner cycle finishing can't emit a
|
|
90
|
+
// premature idle mid-registration.
|
|
91
|
+
private fetchDepth: Map<QueryHash, number> = new Map();
|
|
52
92
|
private logger: Logger;
|
|
93
|
+
/**
|
|
94
|
+
* Optional observer notified whenever a query's fetch status changes.
|
|
95
|
+
* Wired by Sp00kyClient to push status changes into DevTools. Kept as a
|
|
96
|
+
* settable field (rather than a constructor arg) because DevTools is
|
|
97
|
+
* constructed after DataModule.
|
|
98
|
+
*/
|
|
99
|
+
public onQueryStatusChange?: (hash: QueryHash, status: QueryStatus) => void;
|
|
100
|
+
/**
|
|
101
|
+
* Optional observer invoked when a still-subscribed query's TTL heartbeat
|
|
102
|
+
* fires (~90% of the TTL). Wired by Sp00kyClient to
|
|
103
|
+
* `Sp00kySync.heartbeatQuery`, which refreshes the remote `_00_query`
|
|
104
|
+
* row's `lastActiveAt` so an actively-watched query never expires. Settable
|
|
105
|
+
* field (not a constructor arg) because the sync engine is wired after
|
|
106
|
+
* DataModule is constructed — mirrors `onQueryStatusChange`.
|
|
107
|
+
*/
|
|
108
|
+
public onHeartbeat?: (hash: QueryHash) => void;
|
|
109
|
+
/**
|
|
110
|
+
* Optional hook fired by {@link deregisterQuery} when an opt-in query (e.g. a
|
|
111
|
+
* viewport-windowed list cancelling an off-screen window) loses its last
|
|
112
|
+
* subscriber. Wired by Sp00kyClient to enqueue a `cleanup` down-event, which
|
|
113
|
+
* tears the remote `_00_query` view down (releasing its `_00_list_ref` edges)
|
|
114
|
+
* instead of leaving it for the TTL sweep. The local view + state are freed in
|
|
115
|
+
* {@link finalizeDeregister} only after that remote delete, so a fast
|
|
116
|
+
* re-subscribe (scroll back) can abort/heal the teardown — see `cleanupQuery`.
|
|
117
|
+
*/
|
|
118
|
+
public onDeregister?: (hash: QueryHash) => void;
|
|
119
|
+
// Salt for query-id hashing. Set from SurrealDB's session::id() so two
|
|
120
|
+
// browser sessions registering the same logical query (same surql + params)
|
|
121
|
+
// don't collide on the same `_00_query` row — each session gets its own.
|
|
122
|
+
// Empty string until init(sessionId) is called.
|
|
123
|
+
private sessionId: string = '';
|
|
124
|
+
// Authenticated user record id (e.g. `"user:abc"`). Updated by
|
|
125
|
+
// `setCurrentUserId` from the auth subscription. null when
|
|
126
|
+
// unauthenticated. Consulted by `Sp00kySync.listRefTable()` so the
|
|
127
|
+
// poll and LIVE subscription target the same per-user
|
|
128
|
+
// `_00_list_ref_user_<id>` table the sync engine writes to.
|
|
129
|
+
private currentUserId: string | null = null;
|
|
53
130
|
|
|
54
131
|
constructor(
|
|
55
132
|
private cache: CacheModule,
|
|
56
|
-
private local:
|
|
133
|
+
private local: LocalStore,
|
|
57
134
|
private schema: S,
|
|
58
135
|
logger: Logger,
|
|
59
|
-
|
|
136
|
+
// Client-side SSP aggregation throttle: coalesces the in-browser
|
|
137
|
+
// StreamProcessor's per-record stream updates per query before notifying
|
|
138
|
+
// readers, so a burst of synced records repaints the UI once per window
|
|
139
|
+
// rather than row-by-row. 50ms keeps it responsive while still batching the
|
|
140
|
+
// initial-sync stream. Override via `SyncedDbConfig.streamDebounceTime`.
|
|
141
|
+
private streamDebounceTime: number = 50
|
|
60
142
|
) {
|
|
61
143
|
this.logger = logger.child({ service: 'DataModule' });
|
|
62
144
|
}
|
|
63
145
|
|
|
64
|
-
async init(): Promise<void> {
|
|
65
|
-
this.
|
|
146
|
+
async init(sessionId: string): Promise<void> {
|
|
147
|
+
this.sessionId = sessionId;
|
|
148
|
+
this.logger.info(
|
|
149
|
+
{ sessionId, Category: 'sp00ky-client::DataModule::init' },
|
|
150
|
+
'DataModule initialized'
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Update the session salt used in query-id hashing. Call this when the
|
|
156
|
+
* SurrealDB session changes (sign-in, sign-out, reconnect). Subsequently
|
|
157
|
+
* registered queries will get fresh, session-scoped IDs.
|
|
158
|
+
*/
|
|
159
|
+
setSessionId(sessionId: string): void {
|
|
160
|
+
this.sessionId = sessionId;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Shared-tabs: bake this tab's identity into mutation ids so a rollback of
|
|
164
|
+
* a follower's mutation routes back to the tab that made it. */
|
|
165
|
+
setTabId(tabId: string): void {
|
|
166
|
+
this.tabId = tabId;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Update the authenticated user record id. Pass `null` on sign-out.
|
|
171
|
+
* Read by `Sp00kySync.listRefTable()` so the LIVE subscription and
|
|
172
|
+
* the poll route to the same per-user `_00_list_ref_user_<id>` the
|
|
173
|
+
* SSP writes to.
|
|
174
|
+
*/
|
|
175
|
+
setCurrentUserId(userId: string | null): void {
|
|
176
|
+
this.currentUserId = userId;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Read-only view of the authenticated user id used for per-user
|
|
180
|
+
* `_00_list_ref` routing. Other modules consult this so they pick the
|
|
181
|
+
* same table name DataModule does. */
|
|
182
|
+
getCurrentUserId(): string | null {
|
|
183
|
+
return this.currentUserId;
|
|
66
184
|
}
|
|
67
185
|
|
|
68
186
|
// ==================== QUERY MANAGEMENT ====================
|
|
@@ -74,19 +192,22 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
74
192
|
tableName: T,
|
|
75
193
|
surqlString: string,
|
|
76
194
|
params: Record<string, any>,
|
|
77
|
-
ttl: QueryTimeToLive
|
|
195
|
+
ttl: QueryTimeToLive,
|
|
196
|
+
plan?: QueryPlan
|
|
78
197
|
): Promise<QueryHash> {
|
|
79
198
|
const hash = await this.calculateHash({ surql: surqlString, params });
|
|
80
199
|
this.logger.debug(
|
|
81
|
-
{ hash, Category: '
|
|
200
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
82
201
|
'Query Initialization: started'
|
|
83
202
|
);
|
|
84
203
|
|
|
85
|
-
|
|
204
|
+
// `_00_query` stays the single shared registration table in both
|
|
205
|
+
// ref-modes; the per-user split happens only on `_00_list_ref`.
|
|
206
|
+
const recordId = new RecordId('_00_query', hash);
|
|
86
207
|
|
|
87
208
|
if (this.activeQueries.has(hash)) {
|
|
88
209
|
this.logger.debug(
|
|
89
|
-
{ hash, Category: '
|
|
210
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
90
211
|
'Query Initialization: exists, returning'
|
|
91
212
|
);
|
|
92
213
|
return hash;
|
|
@@ -95,7 +216,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
95
216
|
// Another call is already creating this query — wait for it
|
|
96
217
|
if (this.pendingQueries.has(hash)) {
|
|
97
218
|
this.logger.debug(
|
|
98
|
-
{ hash, Category: '
|
|
219
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
99
220
|
'Query Initialization: pending, waiting for existing creation'
|
|
100
221
|
);
|
|
101
222
|
await this.pendingQueries.get(hash);
|
|
@@ -103,12 +224,20 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
103
224
|
}
|
|
104
225
|
|
|
105
226
|
this.logger.debug(
|
|
106
|
-
{ hash, Category: '
|
|
227
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
107
228
|
'Query Initialization: not found, creating new query'
|
|
108
229
|
);
|
|
109
230
|
|
|
110
231
|
// Create the query and track the pending promise
|
|
111
|
-
const promise = this.createAndRegisterQuery<T>(
|
|
232
|
+
const promise = this.createAndRegisterQuery<T>(
|
|
233
|
+
hash,
|
|
234
|
+
recordId,
|
|
235
|
+
surqlString,
|
|
236
|
+
params,
|
|
237
|
+
ttl,
|
|
238
|
+
tableName,
|
|
239
|
+
plan
|
|
240
|
+
);
|
|
112
241
|
this.pendingQueries.set(hash, promise);
|
|
113
242
|
try {
|
|
114
243
|
await promise;
|
|
@@ -147,11 +276,95 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
147
276
|
subs.delete(callback);
|
|
148
277
|
if (subs.size === 0) {
|
|
149
278
|
this.subscriptions.delete(queryHash);
|
|
279
|
+
// NOTE: intentionally do NOT tear down the query / free its in-browser
|
|
280
|
+
// SSP view here. The subscriber-gated heartbeat (startTTLHeartbeat)
|
|
281
|
+
// already self-stops once subscribers hit 0, so an abandoned query
|
|
282
|
+
// stops being kept alive and the server's TTL sweep removes it. Freeing
|
|
283
|
+
// the local view on every last-unsubscribe caused re-registration churn
|
|
284
|
+
// on navigation (open A → leave → open A again re-registers), which is
|
|
285
|
+
// a flakiness risk for no real benefit — keep the local view resident.
|
|
150
286
|
}
|
|
151
287
|
}
|
|
152
288
|
};
|
|
153
289
|
}
|
|
154
290
|
|
|
291
|
+
/**
|
|
292
|
+
* Subscribe to a query's fetch-status changes (idle/fetching).
|
|
293
|
+
* With `{ immediate: true }` the callback fires synchronously with the
|
|
294
|
+
* current status (defaults to `idle` if the query isn't registered yet).
|
|
295
|
+
*/
|
|
296
|
+
subscribeStatus(
|
|
297
|
+
queryHash: string,
|
|
298
|
+
callback: QueryStatusCallback,
|
|
299
|
+
options: { immediate?: boolean } = {}
|
|
300
|
+
): () => void {
|
|
301
|
+
if (!this.statusSubscriptions.has(queryHash)) {
|
|
302
|
+
this.statusSubscriptions.set(queryHash, new Set());
|
|
303
|
+
}
|
|
304
|
+
this.statusSubscriptions.get(queryHash)?.add(callback);
|
|
305
|
+
|
|
306
|
+
if (options.immediate) {
|
|
307
|
+
callback(this.activeQueries.get(queryHash)?.status ?? 'idle');
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
return () => {
|
|
311
|
+
const subs = this.statusSubscriptions.get(queryHash);
|
|
312
|
+
if (subs) {
|
|
313
|
+
subs.delete(callback);
|
|
314
|
+
if (subs.size === 0) {
|
|
315
|
+
this.statusSubscriptions.delete(queryHash);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Set a query's fetch status and notify status observers (DevTools +
|
|
323
|
+
* `subscribeStatus` listeners). No-op when the status is unchanged or the
|
|
324
|
+
* query is unknown.
|
|
325
|
+
*/
|
|
326
|
+
setQueryStatus(queryHash: string, status: QueryStatus): void {
|
|
327
|
+
const queryState = this.activeQueries.get(queryHash);
|
|
328
|
+
if (!queryState || queryState.status === status) {
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
queryState.status = status;
|
|
332
|
+
|
|
333
|
+
this.onQueryStatusChange?.(queryHash, status);
|
|
334
|
+
|
|
335
|
+
const subs = this.statusSubscriptions.get(queryHash);
|
|
336
|
+
if (subs) {
|
|
337
|
+
for (const callback of subs) {
|
|
338
|
+
callback(status);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Enter a fetch cycle for a query. Refcounted: registration and concurrent
|
|
345
|
+
* poll/LIVE sync rounds can overlap on the same hash, and only the OUTERMOST
|
|
346
|
+
* cycle may flip the status — 0→1 emits `fetching`, and `endFetching`'s 1→0
|
|
347
|
+
* emits `idle`. Always pair with `endFetching` in a `finally`.
|
|
348
|
+
*/
|
|
349
|
+
beginFetching(queryHash: string): void {
|
|
350
|
+
const depth = this.fetchDepth.get(queryHash) ?? 0;
|
|
351
|
+
this.fetchDepth.set(queryHash, depth + 1);
|
|
352
|
+
if (depth === 0) {
|
|
353
|
+
this.setQueryStatus(queryHash, 'fetching');
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/** Leave a fetch cycle started with {@link beginFetching}; emits `idle` on the last exit. */
|
|
358
|
+
endFetching(queryHash: string): void {
|
|
359
|
+
const depth = this.fetchDepth.get(queryHash) ?? 0;
|
|
360
|
+
if (depth <= 1) {
|
|
361
|
+
this.fetchDepth.delete(queryHash);
|
|
362
|
+
this.setQueryStatus(queryHash, 'idle');
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
this.fetchDepth.set(queryHash, depth - 1);
|
|
366
|
+
}
|
|
367
|
+
|
|
155
368
|
/**
|
|
156
369
|
* Subscribe to mutations (for sync)
|
|
157
370
|
*/
|
|
@@ -168,67 +381,208 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
168
381
|
async onStreamUpdate(update: StreamUpdate): Promise<void> {
|
|
169
382
|
const { queryHash, op } = update;
|
|
170
383
|
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
384
|
+
// DELETE propagates immediately — a removed row should disappear without
|
|
385
|
+
// waiting on a debounce.
|
|
386
|
+
//
|
|
387
|
+
// CREATE and UPDATE are coalesced per query on a trailing timer. A list's
|
|
388
|
+
// rows stream in from sync as many small `_00_list_ref` diffs, each its own
|
|
389
|
+
// `cache.saveBatch` → one stream update per chunk. `ingestMany` only
|
|
390
|
+
// coalesces records ingested synchronously together, so chunks spread over
|
|
391
|
+
// time each re-materialize and re-notify — a 50-row window can fire 30+
|
|
392
|
+
// updates as it fills. Each StreamUpdate carries the full materialized
|
|
393
|
+
// `localArray`, so the latest one already reflects every prior chunk: keep
|
|
394
|
+
// only it and fire once on the trailing edge, settling the query in a couple
|
|
395
|
+
// of notifications instead of one per chunk.
|
|
396
|
+
if (op === 'DELETE') {
|
|
397
|
+
const existing = this.debounceTimers.get(queryHash);
|
|
398
|
+
if (existing) {
|
|
399
|
+
clearTimeout(existing);
|
|
400
|
+
this.debounceTimers.delete(queryHash);
|
|
177
401
|
}
|
|
402
|
+
// The DELETE update carries the full latest localArray, so the coalesced
|
|
403
|
+
// CREATE/UPDATE it supersedes is already reflected — drop it.
|
|
404
|
+
this.pendingStreamUpdates.delete(queryHash);
|
|
405
|
+
await this.processStreamUpdate(update);
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
178
408
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
409
|
+
// Clear existing timer if any
|
|
410
|
+
if (this.debounceTimers.has(queryHash)) {
|
|
411
|
+
// oxlint-disable-next-line no-non-null-assertion -- guarded by .has() check above
|
|
412
|
+
clearTimeout(this.debounceTimers.get(queryHash)!);
|
|
413
|
+
}
|
|
184
414
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
415
|
+
// Set new timer
|
|
416
|
+
this.pendingStreamUpdates.set(queryHash, update);
|
|
417
|
+
const timer = setTimeout(async () => {
|
|
418
|
+
this.debounceTimers.delete(queryHash);
|
|
419
|
+
this.pendingStreamUpdates.delete(queryHash);
|
|
188
420
|
await this.processStreamUpdate(update);
|
|
421
|
+
}, this.streamDebounceTime);
|
|
422
|
+
|
|
423
|
+
this.debounceTimers.set(queryHash, timer);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Process a query's pending (debounced) stream update NOW instead of on the
|
|
428
|
+
* trailing edge. Called by the sync engine before it flips a query back to
|
|
429
|
+
* `idle`, so the status change never races ahead of the rows it fetched.
|
|
430
|
+
* No-op when nothing is pending. The pending entry is removed before the
|
|
431
|
+
* await so a concurrently-firing timer can't process it twice.
|
|
432
|
+
*/
|
|
433
|
+
async flushPendingStreamUpdate(queryHash: string): Promise<void> {
|
|
434
|
+
const timer = this.debounceTimers.get(queryHash);
|
|
435
|
+
if (timer) {
|
|
436
|
+
clearTimeout(timer);
|
|
437
|
+
this.debounceTimers.delete(queryHash);
|
|
189
438
|
}
|
|
439
|
+
const pending = this.pendingStreamUpdates.get(queryHash);
|
|
440
|
+
if (!pending) return;
|
|
441
|
+
this.pendingStreamUpdates.delete(queryHash);
|
|
442
|
+
await this.processStreamUpdate(pending);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// Materialize a query's result rows from the local DB. For a windowed query
|
|
446
|
+
// (`LIMIT n START m`, m>0) the original surql is NOT re-run — re-applying
|
|
447
|
+
// `START m` against the shared local store would skip the window's own rows
|
|
448
|
+
// (sparse windowing) and return nothing. Instead we select the window's
|
|
449
|
+
// record id-set directly, preferring the server's `list_ref` (`remoteArray`,
|
|
450
|
+
// authoritative) over the in-browser SSP's view (`sspArray`), and re-apply
|
|
451
|
+
// the original ORDER BY for stable order.
|
|
452
|
+
private async materializeRecords(
|
|
453
|
+
queryState: QueryState,
|
|
454
|
+
sspArray?: Array<[string, number]>
|
|
455
|
+
): Promise<Record<string, any>[]> {
|
|
456
|
+
const t0 = performance.now();
|
|
457
|
+
const plan = queryState.config.plan;
|
|
458
|
+
const windowMat = buildWindowMaterialization(queryState.config.surql);
|
|
459
|
+
let records: Record<string, any>[];
|
|
460
|
+
if (windowMat) {
|
|
461
|
+
const win =
|
|
462
|
+
(queryState.config.remoteArray?.length && queryState.config.remoteArray) ||
|
|
463
|
+
(sspArray?.length && sspArray) ||
|
|
464
|
+
queryState.config.localArray ||
|
|
465
|
+
[];
|
|
466
|
+
const winIds = win.map(([id]) => parseRecordIdString(id));
|
|
467
|
+
if (plan) {
|
|
468
|
+
// Engine-neutral window materialization: select exactly the id-set,
|
|
469
|
+
// keeping ORDER BY + relations. Works on any local engine.
|
|
470
|
+
const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
|
|
471
|
+
records = await this.local.select(winPlan, queryState.config.params);
|
|
472
|
+
} else {
|
|
473
|
+
const [rows] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
|
|
474
|
+
...queryState.config.params,
|
|
475
|
+
__win: winIds,
|
|
476
|
+
});
|
|
477
|
+
records = rows || [];
|
|
478
|
+
}
|
|
479
|
+
} else if (plan) {
|
|
480
|
+
records = await this.local.select(plan, queryState.config.params);
|
|
481
|
+
} else {
|
|
482
|
+
const [rows] = await this.local.query<[Record<string, any>[]]>(
|
|
483
|
+
queryState.config.surql,
|
|
484
|
+
queryState.config.params
|
|
485
|
+
);
|
|
486
|
+
records = rows || [];
|
|
487
|
+
}
|
|
488
|
+
// Local SurrealDB record-fetch time → DevTools "localFetch" phase.
|
|
489
|
+
this.recordPhase(queryState, 'localFetch', performance.now() - t0);
|
|
490
|
+
return records;
|
|
190
491
|
}
|
|
191
492
|
|
|
192
493
|
private async processStreamUpdate(update: StreamUpdate): Promise<void> {
|
|
193
|
-
const { queryHash, localArray } = update;
|
|
494
|
+
const { queryHash, localArray, materializationTimeMs } = update;
|
|
194
495
|
const queryState = this.activeQueries.get(queryHash);
|
|
195
496
|
if (!queryState) {
|
|
196
497
|
this.logger.warn(
|
|
197
|
-
{ queryHash, Category: '
|
|
498
|
+
{ queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
198
499
|
'Received update for unknown query. Skipping...'
|
|
199
500
|
);
|
|
200
501
|
return;
|
|
201
502
|
}
|
|
202
503
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
)
|
|
504
|
+
// Update the rolling materialization-sample window before the work that
|
|
505
|
+
// could throw, so the percentiles still move when the downstream local
|
|
506
|
+
// query fails (the materialization step itself ran).
|
|
507
|
+
if (typeof materializationTimeMs === 'number') {
|
|
508
|
+
queryState.materializationSamples.push(materializationTimeMs);
|
|
509
|
+
if (queryState.materializationSamples.length > MATERIALIZATION_SAMPLE_WINDOW) {
|
|
510
|
+
queryState.materializationSamples.shift();
|
|
511
|
+
}
|
|
512
|
+
queryState.lastIngestLatencyMs = materializationTimeMs;
|
|
513
|
+
}
|
|
514
|
+
// Record the SSP internal sub-phase timings (from the WASM binding) so
|
|
515
|
+
// DevTools can attribute ingest cost to store-apply vs circuit-step vs transform.
|
|
516
|
+
if (typeof update.storeApplyMs === 'number')
|
|
517
|
+
this.recordPhase(queryState, 'sspStoreApply', update.storeApplyMs);
|
|
518
|
+
if (typeof update.circuitStepMs === 'number')
|
|
519
|
+
this.recordPhase(queryState, 'sspCircuitStep', update.circuitStepMs);
|
|
520
|
+
if (typeof update.transformMs === 'number')
|
|
521
|
+
this.recordPhase(queryState, 'sspTransform', update.transformMs);
|
|
522
|
+
const percentiles = this.computeMaterializationPercentiles(queryState.materializationSamples);
|
|
523
|
+
|
|
524
|
+
// Fence against bucket switches: this update's `localArray` came from the
|
|
525
|
+
// pre-switch SSP circuit; applying it after a switch would show (and
|
|
526
|
+
// persist) the previous user's ids in the new bucket.
|
|
527
|
+
const epoch = this.local.epoch;
|
|
209
528
|
|
|
210
|
-
|
|
211
|
-
|
|
529
|
+
try {
|
|
530
|
+
// Materialize the query's rows. For a windowed (offset) query, re-running
|
|
531
|
+
// the original surql would re-apply `START n` against the shared local DB
|
|
532
|
+
// and skip the window's rows entirely; instead select the SSP's
|
|
533
|
+
// materialized window id-set (`localArray`) directly, re-applying the
|
|
534
|
+
// original ORDER BY for stable display order. Non-offset queries keep the
|
|
535
|
+
// normal re-query path.
|
|
536
|
+
const newRecords = await this.materializeRecords(queryState, localArray);
|
|
537
|
+
if (epoch !== this.local.epoch) return;
|
|
212
538
|
queryState.config.localArray = localArray;
|
|
213
|
-
await this.local.query(surql.seal(surql.updateSet('id', ['localArray'])), {
|
|
214
|
-
id: queryState.config.id,
|
|
215
|
-
localArray,
|
|
216
|
-
});
|
|
217
539
|
|
|
218
|
-
// Skip notification if records haven't changed
|
|
219
540
|
const prevJson = JSON.stringify(queryState.records);
|
|
220
541
|
const newJson = JSON.stringify(newRecords);
|
|
221
542
|
queryState.records = newRecords;
|
|
222
|
-
|
|
543
|
+
const recordsChanged = prevJson !== newJson;
|
|
544
|
+
|
|
545
|
+
// updateCount counts user-visible updates (matches the prior semantic),
|
|
546
|
+
// while the materialization sample/percentiles already moved above for
|
|
547
|
+
// every observed engine step.
|
|
548
|
+
if (recordsChanged) {
|
|
549
|
+
queryState.updateCount++;
|
|
550
|
+
queryState.lastUpdatedAt = Date.now();
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
await this.local.query(
|
|
554
|
+
surql.seal(
|
|
555
|
+
surql.updateSet('id', [
|
|
556
|
+
'localArray',
|
|
557
|
+
'rowCount',
|
|
558
|
+
'updateCount',
|
|
559
|
+
'lastIngestLatency',
|
|
560
|
+
'materializationP55',
|
|
561
|
+
'materializationP90',
|
|
562
|
+
'materializationP99',
|
|
563
|
+
])
|
|
564
|
+
),
|
|
565
|
+
{
|
|
566
|
+
id: queryState.config.id,
|
|
567
|
+
localArray,
|
|
568
|
+
rowCount: localArray.length,
|
|
569
|
+
updateCount: queryState.updateCount,
|
|
570
|
+
lastIngestLatency: queryState.lastIngestLatencyMs,
|
|
571
|
+
materializationP55: percentiles.p55,
|
|
572
|
+
materializationP90: percentiles.p90,
|
|
573
|
+
materializationP99: percentiles.p99,
|
|
574
|
+
},
|
|
575
|
+
{ epoch }
|
|
576
|
+
);
|
|
577
|
+
|
|
578
|
+
if (!recordsChanged) {
|
|
223
579
|
this.logger.debug(
|
|
224
|
-
{ queryHash, Category: '
|
|
580
|
+
{ queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
225
581
|
'Query records unchanged, skipping notification'
|
|
226
582
|
);
|
|
227
583
|
return;
|
|
228
584
|
}
|
|
229
585
|
|
|
230
|
-
queryState.updateCount++;
|
|
231
|
-
|
|
232
586
|
// Notify subscribers
|
|
233
587
|
const subscribers = this.subscriptions.get(queryHash);
|
|
234
588
|
if (subscribers) {
|
|
@@ -240,17 +594,108 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
240
594
|
this.logger.debug(
|
|
241
595
|
{
|
|
242
596
|
queryHash,
|
|
243
|
-
recordCount:
|
|
244
|
-
Category: '
|
|
597
|
+
recordCount: newRecords?.length,
|
|
598
|
+
Category: 'sp00ky-client::DataModule::onStreamUpdate',
|
|
245
599
|
},
|
|
246
600
|
'Query updated from stream'
|
|
247
601
|
);
|
|
248
602
|
} catch (err) {
|
|
603
|
+
if (err instanceof StaleEpochError) {
|
|
604
|
+
this.logger.debug(
|
|
605
|
+
{ queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
606
|
+
'Dropped stream update from before a bucket switch'
|
|
607
|
+
);
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
queryState.errorCount++;
|
|
249
611
|
this.logger.error(
|
|
250
|
-
{ err, queryHash, Category: '
|
|
612
|
+
{ err, queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
251
613
|
'Failed to fetch records for stream update'
|
|
252
614
|
);
|
|
615
|
+
// Best-effort persist of the bumped errorCount; swallow secondary
|
|
616
|
+
// failures to avoid masking the original error in logs.
|
|
617
|
+
try {
|
|
618
|
+
await this.local.query(surql.seal(surql.updateSet('id', ['errorCount'])), {
|
|
619
|
+
id: queryState.config.id,
|
|
620
|
+
errorCount: queryState.errorCount,
|
|
621
|
+
});
|
|
622
|
+
} catch (persistErr) {
|
|
623
|
+
this.logger.warn(
|
|
624
|
+
{
|
|
625
|
+
err: persistErr,
|
|
626
|
+
queryHash,
|
|
627
|
+
Category: 'sp00ky-client::DataModule::onStreamUpdate',
|
|
628
|
+
},
|
|
629
|
+
'Failed to persist incremented errorCount'
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* Compute p55/p90/p99 from a rolling window of materialization samples.
|
|
637
|
+
* Returns nulls for any percentile that has no samples yet so SurrealDB
|
|
638
|
+
* `option<float>` columns stay NONE rather than 0 before the first ingest.
|
|
639
|
+
*/
|
|
640
|
+
private computeMaterializationPercentiles(samples: number[]): {
|
|
641
|
+
p55: number | null;
|
|
642
|
+
p90: number | null;
|
|
643
|
+
p99: number | null;
|
|
644
|
+
} {
|
|
645
|
+
if (samples.length === 0) {
|
|
646
|
+
return { p55: null, p90: null, p99: null };
|
|
253
647
|
}
|
|
648
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
649
|
+
const pick = (q: number) => {
|
|
650
|
+
const idx = Math.min(sorted.length - 1, Math.floor(q * sorted.length));
|
|
651
|
+
return sorted[idx]!;
|
|
652
|
+
};
|
|
653
|
+
return { p55: pick(0.55), p90: pick(0.9), p99: pick(0.99) };
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** Record a per-phase timing sample (ms) on a query's rolling window. */
|
|
657
|
+
private recordPhase(qs: QueryState, phase: string, ms: number): void {
|
|
658
|
+
if (!Number.isFinite(ms)) return;
|
|
659
|
+
const arr = qs.phaseSamples[phase] ?? (qs.phaseSamples[phase] = []);
|
|
660
|
+
pushSample(arr, ms);
|
|
661
|
+
qs.phaseLast[phase] = ms;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/** Record the remote record-fetch time (ms) for a query. Called by the sync engine. */
|
|
665
|
+
recordRemoteFetch(hash: string, ms: number): void {
|
|
666
|
+
const qs = this.activeQueries.get(hash);
|
|
667
|
+
if (qs) this.recordPhase(qs, 'remoteFetch', ms);
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* Record the frontend reconcile time (ms) for a query. Called from `useQuery`
|
|
672
|
+
* via `Sp00kyClient.reportFrontendTiming` after it applies an update to its store.
|
|
673
|
+
*/
|
|
674
|
+
recordFrontendTiming(hash: string, ms: number): void {
|
|
675
|
+
const qs = this.activeQueries.get(hash);
|
|
676
|
+
if (qs) this.recordPhase(qs, 'frontend', ms);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* Build the per-query processing-time breakdown surfaced to the DevTools panel
|
|
681
|
+
* and the MCP. `ssp` is the WASM-ingest wall time (from `materializationSamples`);
|
|
682
|
+
* the rest come from the per-phase rolling windows + one-shot registration timings.
|
|
683
|
+
*/
|
|
684
|
+
phaseTimings(q: QueryState): QueryTimings {
|
|
685
|
+
const stat = (phase: string) =>
|
|
686
|
+
phaseStatOf(q.phaseSamples[phase] ?? [], q.phaseLast[phase] ?? null);
|
|
687
|
+
return {
|
|
688
|
+
ssp: phaseStatOf(q.materializationSamples, q.lastIngestLatencyMs),
|
|
689
|
+
sspStoreApply: stat('sspStoreApply'),
|
|
690
|
+
sspCircuitStep: stat('sspCircuitStep'),
|
|
691
|
+
sspTransform: stat('sspTransform'),
|
|
692
|
+
localFetch: stat('localFetch'),
|
|
693
|
+
remoteFetch: stat('remoteFetch'),
|
|
694
|
+
frontend: stat('frontend'),
|
|
695
|
+
registration: q.registrationTimings,
|
|
696
|
+
updateCount: q.updateCount,
|
|
697
|
+
errorCount: q.errorCount,
|
|
698
|
+
};
|
|
254
699
|
}
|
|
255
700
|
|
|
256
701
|
/**
|
|
@@ -260,6 +705,263 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
260
705
|
return this.activeQueries.get(hash);
|
|
261
706
|
}
|
|
262
707
|
|
|
708
|
+
/**
|
|
709
|
+
* Cold-query guard for instant-hydrate: true when the query exists, hasn't been
|
|
710
|
+
* hydrated, and has NOT yet fetched its server result (`remoteArray` empty).
|
|
711
|
+
* We gate on `remoteArray`, not local `records`: a windowed query is often
|
|
712
|
+
* partially pre-seeded from the circuit (e.g. the dashboard's 5-row preview),
|
|
713
|
+
* but it still hasn't loaded its own full window from the server — so it should
|
|
714
|
+
* still hydrate. A warm re-subscribe (remoteArray already populated) is skipped.
|
|
715
|
+
*/
|
|
716
|
+
isCold(hash: string): boolean {
|
|
717
|
+
const qs = this.activeQueries.get(hash);
|
|
718
|
+
return !!qs && !qs.hydrated && (qs.config.remoteArray?.length ?? 0) === 0;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* Walk a hydrated record's fields and append any EMBEDDED child records to
|
|
723
|
+
* `batch` (recursing for nested related fields). An embedded child is a
|
|
724
|
+
* value that is itself a record — a non-null object whose `id` is a
|
|
725
|
+
* `RecordId` — or an array of such records (one-to-many vs one-to-one). A
|
|
726
|
+
* bare `RecordId` (a foreign-key reference) or any other value is skipped,
|
|
727
|
+
* so this never mistakes a FK column for an embedded body. Children are
|
|
728
|
+
* keyed by their own `record.id.table`, versioned by `_00_rv`, and cleaned
|
|
729
|
+
* to their table's real columns (which strips the alias/related fields).
|
|
730
|
+
* `seen` dedupes within the batch.
|
|
731
|
+
*/
|
|
732
|
+
private collectEmbeddedChildren(
|
|
733
|
+
record: Record<string, any>,
|
|
734
|
+
batch: CacheRecord[],
|
|
735
|
+
seen: Set<string>
|
|
736
|
+
): void {
|
|
737
|
+
const isEmbeddedRecord = (v: unknown): v is RecordWithId =>
|
|
738
|
+
!!v &&
|
|
739
|
+
typeof v === 'object' &&
|
|
740
|
+
!(v instanceof RecordId) &&
|
|
741
|
+
(v as { id?: unknown }).id instanceof RecordId;
|
|
742
|
+
|
|
743
|
+
for (const value of Object.values(record)) {
|
|
744
|
+
const children = Array.isArray(value)
|
|
745
|
+
? value.filter(isEmbeddedRecord)
|
|
746
|
+
: isEmbeddedRecord(value)
|
|
747
|
+
? [value]
|
|
748
|
+
: [];
|
|
749
|
+
for (const child of children) {
|
|
750
|
+
const key = encodeRecordId(child.id);
|
|
751
|
+
if (seen.has(key)) continue;
|
|
752
|
+
seen.add(key);
|
|
753
|
+
// Recurse FIRST so nested grandchildren are captured before `cleanRecord`
|
|
754
|
+
// strips this child's alias fields.
|
|
755
|
+
this.collectEmbeddedChildren(child, batch, seen);
|
|
756
|
+
const table = child.id.table.toString();
|
|
757
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
758
|
+
batch.push({
|
|
759
|
+
table,
|
|
760
|
+
op: 'CREATE',
|
|
761
|
+
// Flatten the child too: a nested comment carries its own embedded
|
|
762
|
+
// `author` object which the schemafull `record<user>` field would
|
|
763
|
+
// reject. (Its author is captured as a separate row by the recursion
|
|
764
|
+
// above.)
|
|
765
|
+
record: this.flattenRelationsForStorage(child, tableSchema),
|
|
766
|
+
version: (child._00_rv as number) || 1,
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* Prepare a subquery-bearing row (preload / hydration) for the schemafull
|
|
774
|
+
* local store: replace an embedded FORWARD-relation object (`author = { id, … }`)
|
|
775
|
+
* with its RecordId so a `record<…>` field coerces, and DROP reverse-subquery
|
|
776
|
+
* ARRAYS (`comments = [ … ]`) since their rows are cached separately as their
|
|
777
|
+
* own bodies. A flat record — as the live `SELECT * FROM $ids` sync returns,
|
|
778
|
+
* with relations already RecordIds — passes through unchanged.
|
|
779
|
+
*/
|
|
780
|
+
private flattenRelationsForStorage(
|
|
781
|
+
record: Record<string, any>,
|
|
782
|
+
tableSchema?: { columns: Record<string, any> }
|
|
783
|
+
): RecordWithId {
|
|
784
|
+
const isEmbeddedRecord = (v: unknown): boolean =>
|
|
785
|
+
!!v &&
|
|
786
|
+
typeof v === 'object' &&
|
|
787
|
+
!(v instanceof RecordId) &&
|
|
788
|
+
(v as { id?: unknown }).id instanceof RecordId;
|
|
789
|
+
const flat: Record<string, any> = {};
|
|
790
|
+
for (const [k, v] of Object.entries(record)) {
|
|
791
|
+
if (isEmbeddedRecord(v)) {
|
|
792
|
+
flat[k] = (v as { id: unknown }).id;
|
|
793
|
+
} else if (Array.isArray(v) && v.some(isEmbeddedRecord)) {
|
|
794
|
+
continue;
|
|
795
|
+
} else {
|
|
796
|
+
flat[k] = v;
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
return (tableSchema ? cleanRecord(tableSchema.columns, flat) : flat) as RecordWithId;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/**
|
|
803
|
+
* Instant-hydrate: ingest rows fetched one-shot from the remote (the query's own
|
|
804
|
+
* surql run directly) so the query DISPLAYS immediately, while the full realtime
|
|
805
|
+
* registration proceeds in the background. Ingests with versions (`_00_rv`) so the
|
|
806
|
+
* later `syncRecords` dedup skips re-pulling unchanged bodies, and seeds
|
|
807
|
+
* `remoteArray` so windowed queries materialize the correct window (no sparse
|
|
808
|
+
* local-circuit issue). Runs at most once per query (the `hydrated` flag).
|
|
809
|
+
*/
|
|
810
|
+
async applyHydration(hash: string, rows: RecordWithId[]): Promise<void> {
|
|
811
|
+
const queryState = this.activeQueries.get(hash);
|
|
812
|
+
if (!queryState) return;
|
|
813
|
+
queryState.hydrated = true; // run-once, even when the remote returns nothing
|
|
814
|
+
if (rows.length === 0) return;
|
|
815
|
+
|
|
816
|
+
const epoch = this.local.epoch;
|
|
817
|
+
const tableName = queryState.config.tableName;
|
|
818
|
+
await this.buildAndSaveCacheBatch(tableName, rows);
|
|
819
|
+
// Bucket switched while we persisted: these rows were fetched under the
|
|
820
|
+
// previous auth context — don't let them prime the new bucket's query
|
|
821
|
+
// state; the rebind's re-registration refills it. (saveBatch's own epoch
|
|
822
|
+
// fence usually catches this, but the switch can land between it and here.)
|
|
823
|
+
if (epoch !== this.local.epoch) return;
|
|
824
|
+
|
|
825
|
+
// Prime remoteArray from the hydrated id+version pairs: `materializeRecords`
|
|
826
|
+
// prefers it for windowed queries (correct window) and it feeds the version
|
|
827
|
+
// dedup. Registration later overwrites it with the authoritative `_00_list_ref`.
|
|
828
|
+
queryState.config.remoteArray = rows.map(
|
|
829
|
+
(r) => [encodeRecordId(r.id), (r._00_rv as number) || 1] as [string, number]
|
|
830
|
+
);
|
|
831
|
+
|
|
832
|
+
queryState.records = await this.materializeRecords(queryState);
|
|
833
|
+
const subscribers = this.subscriptions.get(hash);
|
|
834
|
+
if (subscribers) {
|
|
835
|
+
for (const cb of subscribers) cb(queryState.records);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/**
|
|
840
|
+
* Build the cache batch for a set of one-shot rows and persist it to the
|
|
841
|
+
* local DB + in-browser SSP. Maps each row to a `CREATE` op on its own table
|
|
842
|
+
* and extracts EMBEDDED related children (any nesting depth) as their own
|
|
843
|
+
* records — a `.related()` query returns its children embedded, and a later
|
|
844
|
+
* correlated re-materialization needs them present as standalone rows.
|
|
845
|
+
* Shared by `applyHydration` (live registration) and `persistSnapshot`
|
|
846
|
+
* (preload).
|
|
847
|
+
*/
|
|
848
|
+
private async buildAndSaveCacheBatch(tableName: string, rows: RecordWithId[]): Promise<void> {
|
|
849
|
+
const tableSchema = this.schema.tables.find((t) => t.name === tableName);
|
|
850
|
+
const batch: CacheRecord[] = rows.map((record) => ({
|
|
851
|
+
table: tableName,
|
|
852
|
+
op: 'CREATE' as const,
|
|
853
|
+
// Flatten embedded relations first: a preload/hydration row can carry a
|
|
854
|
+
// forward relation as a full nested OBJECT (`author = { id, … }`) and a
|
|
855
|
+
// reverse subquery as an array of objects (`comments = [ … ]`). The
|
|
856
|
+
// schemafull local field is `record<user>`, which rejects an object
|
|
857
|
+
// (`Couldn't coerce … found { id: …, username: … }`) and would throw the
|
|
858
|
+
// WHOLE batch. Store the parent with `author` as its RecordId and drop the
|
|
859
|
+
// subquery arrays — the children are cached as their own rows below.
|
|
860
|
+
record: this.flattenRelationsForStorage(record, tableSchema),
|
|
861
|
+
version: (record._00_rv as number) || 1,
|
|
862
|
+
}));
|
|
863
|
+
|
|
864
|
+
const seen = new Set<string>(rows.map((r) => encodeRecordId(r.id)));
|
|
865
|
+
for (const record of rows) {
|
|
866
|
+
this.collectEmbeddedChildren(record, batch, seen);
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
await this.cache.saveBatch(batch);
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
/**
|
|
873
|
+
* Preload/prewarm: persist one-shot rows (and their embedded related children)
|
|
874
|
+
* into the local cache WITHOUT registering a query — no `activeQueries` entry,
|
|
875
|
+
* no `_00_query` view, no TTL heartbeat. The rows live in the local DB as
|
|
876
|
+
* ordinary bodies (never GC'd on their own) so a later `useQuery` seeds its
|
|
877
|
+
* first paint from them instantly, then registers a live view to freshen.
|
|
878
|
+
*/
|
|
879
|
+
async persistSnapshot(tableName: string, rows: RecordWithId[]): Promise<void> {
|
|
880
|
+
if (rows.length === 0) return;
|
|
881
|
+
await this.buildAndSaveCacheBatch(tableName, rows);
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/**
|
|
885
|
+
* Read the durable preload freshness marker for a query hash, or null if this
|
|
886
|
+
* query was never preloaded in the current bucket. Co-located with the cached
|
|
887
|
+
* rows (per-bucket `_00_preload` table) so a bucket switch that clears the
|
|
888
|
+
* data also clears the marker — a stale marker can't claim "warm" when the
|
|
889
|
+
* rows are gone. Any read error is treated as cold.
|
|
890
|
+
*/
|
|
891
|
+
async getPreloadMarker(hash: string): Promise<{ fetchedAt: number; rowCount: number } | null> {
|
|
892
|
+
try {
|
|
893
|
+
// Pass a real RecordId: a bare string id hits the SurrealDB engine's
|
|
894
|
+
// `FROM ONLY $__id` as a plain string, which "selects" the string itself
|
|
895
|
+
// (a truthy non-row) instead of the record — misread as a warm marker.
|
|
896
|
+
const row = await this.local.getById('_00_preload', new RecordId('_00_preload', hash));
|
|
897
|
+
if (!row || typeof row !== 'object') return null;
|
|
898
|
+
return {
|
|
899
|
+
fetchedAt: Number((row as any).fetchedAt) || 0,
|
|
900
|
+
rowCount: Number((row as any).rowCount) || 0,
|
|
901
|
+
};
|
|
902
|
+
} catch {
|
|
903
|
+
return null;
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/** Stamp the preload freshness marker after a successful snapshot fetch. */
|
|
908
|
+
async writePreloadMarker(hash: string, rowCount: number): Promise<void> {
|
|
909
|
+
// RecordId, not a bare string: the SurrealDB engine binds the id verbatim,
|
|
910
|
+
// and `UPSERT <string>` is an InternalError — the marker silently never
|
|
911
|
+
// landed (the write is awaited inside preload's best-effort catch).
|
|
912
|
+
await this.local.upsert(
|
|
913
|
+
'_00_preload',
|
|
914
|
+
new RecordId('_00_preload', hash),
|
|
915
|
+
{ fetchedAt: Date.now(), rowCount },
|
|
916
|
+
'replace'
|
|
917
|
+
);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/** True while ≥1 live subscriber is watching this query (refcount guard). */
|
|
921
|
+
hasSubscribers(hash: string): boolean {
|
|
922
|
+
return (this.subscriptions.get(hash)?.size ?? 0) > 0;
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/**
|
|
926
|
+
* Opt-in eager teardown for a query whose LAST subscriber just left — used by
|
|
927
|
+
* viewport-windowed lists to cancel off-screen windows instead of leaving
|
|
928
|
+
* their remote views to expire on the TTL sweep. No-op while any subscriber
|
|
929
|
+
* remains (refcount). Only enqueues the remote cleanup here; the local WASM
|
|
930
|
+
* view + in-memory state are freed in {@link finalizeDeregister} after the
|
|
931
|
+
* remote delete completes, so a re-subscribe in between aborts/heals it.
|
|
932
|
+
*
|
|
933
|
+
* NOTE: most queries should NOT use this — the default keep-alive on
|
|
934
|
+
* unsubscribe avoids re-registration churn on navigation.
|
|
935
|
+
*/
|
|
936
|
+
deregisterQuery(hash: string): void {
|
|
937
|
+
if (this.hasSubscribers(hash)) return;
|
|
938
|
+
if (!this.activeQueries.has(hash)) return;
|
|
939
|
+
this.onDeregister?.(hash);
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* Final local teardown after the remote `_00_query` row was deleted: free the
|
|
944
|
+
* WASM view, heartbeat timer, debounce timer, and in-memory state. Caller
|
|
945
|
+
* (`cleanupQuery`) guarantees no subscriber remains.
|
|
946
|
+
*/
|
|
947
|
+
finalizeDeregister(hash: string): void {
|
|
948
|
+
const qs = this.activeQueries.get(hash);
|
|
949
|
+
if (qs?.ttlTimer) {
|
|
950
|
+
clearTimeout(qs.ttlTimer);
|
|
951
|
+
qs.ttlTimer = null;
|
|
952
|
+
}
|
|
953
|
+
const debounce = this.debounceTimers.get(hash);
|
|
954
|
+
if (debounce) {
|
|
955
|
+
clearTimeout(debounce);
|
|
956
|
+
this.debounceTimers.delete(hash);
|
|
957
|
+
}
|
|
958
|
+
this.pendingStreamUpdates.delete(hash);
|
|
959
|
+
this.fetchDepth.delete(hash);
|
|
960
|
+
this.cache.unregisterQuery(hash);
|
|
961
|
+
this.activeQueries.delete(hash);
|
|
962
|
+
this.subscriptions.delete(hash);
|
|
963
|
+
}
|
|
964
|
+
|
|
263
965
|
/**
|
|
264
966
|
* Get query state by id (for sync and devtools)
|
|
265
967
|
*/
|
|
@@ -274,36 +976,163 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
274
976
|
return Array.from(this.activeQueries.values());
|
|
275
977
|
}
|
|
276
978
|
|
|
979
|
+
getActiveQueryHashes(): QueryHash[] {
|
|
980
|
+
return Array.from(this.activeQueries.keys());
|
|
981
|
+
}
|
|
982
|
+
|
|
277
983
|
async updateQueryLocalArray(id: string, localArray: RecordVersionArray): Promise<void> {
|
|
278
984
|
const queryState = this.activeQueries.get(id);
|
|
279
985
|
if (!queryState) {
|
|
280
986
|
this.logger.warn(
|
|
281
|
-
{ id, Category: '
|
|
987
|
+
{ id, Category: 'sp00ky-client::DataModule::updateQueryLocalArray' },
|
|
282
988
|
'Query to update local array not found'
|
|
283
989
|
);
|
|
284
990
|
return;
|
|
285
991
|
}
|
|
992
|
+
const epoch = this.local.epoch;
|
|
286
993
|
queryState.config.localArray = localArray;
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
994
|
+
try {
|
|
995
|
+
await this.local.query(
|
|
996
|
+
surql.seal(surql.updateSet('id', ['localArray'])),
|
|
997
|
+
{
|
|
998
|
+
id: queryState.config.id,
|
|
999
|
+
localArray,
|
|
1000
|
+
},
|
|
1001
|
+
{ epoch }
|
|
1002
|
+
);
|
|
1003
|
+
} catch (err) {
|
|
1004
|
+
if (err instanceof StaleEpochError) return;
|
|
1005
|
+
throw err;
|
|
1006
|
+
}
|
|
291
1007
|
}
|
|
292
1008
|
|
|
293
1009
|
async updateQueryRemoteArray(hash: string, remoteArray: RecordVersionArray): Promise<void> {
|
|
294
1010
|
const queryState = this.getQueryByHash(hash);
|
|
295
1011
|
if (!queryState) {
|
|
296
1012
|
this.logger.warn(
|
|
297
|
-
{ hash, Category: '
|
|
1013
|
+
{ hash, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
|
|
298
1014
|
'Query to update remote array not found'
|
|
299
1015
|
);
|
|
300
1016
|
return;
|
|
301
1017
|
}
|
|
1018
|
+
const epoch = this.local.epoch;
|
|
302
1019
|
queryState.config.remoteArray = remoteArray;
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
1020
|
+
try {
|
|
1021
|
+
await this.local.query(
|
|
1022
|
+
surql.seal(surql.updateSet('id', ['remoteArray'])),
|
|
1023
|
+
{
|
|
1024
|
+
id: queryState.config.id,
|
|
1025
|
+
remoteArray,
|
|
1026
|
+
},
|
|
1027
|
+
{ epoch }
|
|
1028
|
+
);
|
|
1029
|
+
} catch (err) {
|
|
1030
|
+
if (err instanceof StaleEpochError) return;
|
|
1031
|
+
throw err;
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/**
|
|
1036
|
+
* Cancel every armed timer ahead of a local-bucket switch: stream-update
|
|
1037
|
+
* debounce timers (their pending updates carry the OLD bucket's id-sets) and
|
|
1038
|
+
* per-query TTL heartbeats (they'd refresh the previous user's remote
|
|
1039
|
+
* `_00_query` rows under the new session). The rebind re-arms heartbeats.
|
|
1040
|
+
*/
|
|
1041
|
+
quiesce(): void {
|
|
1042
|
+
for (const timer of this.debounceTimers.values()) {
|
|
1043
|
+
clearTimeout(timer);
|
|
1044
|
+
}
|
|
1045
|
+
this.debounceTimers.clear();
|
|
1046
|
+
this.pendingStreamUpdates.clear();
|
|
1047
|
+
this.fetchDepth.clear();
|
|
1048
|
+
for (const queryState of this.activeQueries.values()) {
|
|
1049
|
+
if (queryState.ttlTimer) {
|
|
1050
|
+
clearTimeout(queryState.ttlTimer);
|
|
1051
|
+
queryState.ttlTimer = null;
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* Re-home every active query in a freshly-opened bucket, KEEPING its hash —
|
|
1058
|
+
* `useQuery` subscriptions are keyed by hash and don't re-register on auth
|
|
1059
|
+
* changes, so the hooks must stay attached. Per query:
|
|
1060
|
+
* 1. reset the sync arrays + hydration flag and drop the previous user's
|
|
1061
|
+
* records, notifying subscribers with the new-bucket materialization
|
|
1062
|
+
* (usually empty) so their rows leave the UI immediately;
|
|
1063
|
+
* 2. recreate the `_00_query` row in the new bucket;
|
|
1064
|
+
* 3. re-register the SSP view on the (fresh, post-reset) processor — this
|
|
1065
|
+
* also rebinds the view to the NEW `$auth` context;
|
|
1066
|
+
* 4. restart the TTL heartbeat.
|
|
1067
|
+
* Returns the hashes so the caller can enqueue remote re-registration, which
|
|
1068
|
+
* refills records from the server via the normal register→sync→notify path.
|
|
1069
|
+
*/
|
|
1070
|
+
async rebindAfterBucketSwitch(): Promise<QueryHash[]> {
|
|
1071
|
+
const hashes: QueryHash[] = [];
|
|
1072
|
+
for (const [hash, queryState] of this.activeQueries.entries()) {
|
|
1073
|
+
const config = queryState.config;
|
|
1074
|
+
config.localArray = [];
|
|
1075
|
+
config.remoteArray = [];
|
|
1076
|
+
config.subqueryRemoteArray = undefined;
|
|
1077
|
+
queryState.hydrated = false;
|
|
1078
|
+
queryState.syncNotified = false;
|
|
1079
|
+
queryState.records = [];
|
|
1080
|
+
// Via setQueryStatus (not a bare assignment) so status observers see the
|
|
1081
|
+
// flip back to a loading state.
|
|
1082
|
+
this.setQueryStatus(hash, 'fetching');
|
|
1083
|
+
|
|
1084
|
+
try {
|
|
1085
|
+
await withRetry(this.logger, () =>
|
|
1086
|
+
this.local.query<[QueryConfigRecord]>(surql.seal(surql.create('id', 'data')), {
|
|
1087
|
+
id: config.id,
|
|
1088
|
+
data: {
|
|
1089
|
+
surql: config.surql,
|
|
1090
|
+
params: config.params,
|
|
1091
|
+
localArray: [],
|
|
1092
|
+
remoteArray: [],
|
|
1093
|
+
lastActiveAt: new Date(),
|
|
1094
|
+
createdAt: new Date(),
|
|
1095
|
+
ttl: config.ttl,
|
|
1096
|
+
tableName: config.tableName,
|
|
1097
|
+
updateCount: queryState.updateCount,
|
|
1098
|
+
rowCount: 0,
|
|
1099
|
+
errorCount: queryState.errorCount,
|
|
1100
|
+
},
|
|
1101
|
+
})
|
|
1102
|
+
);
|
|
1103
|
+
|
|
1104
|
+
const { localArray } = this.cache.registerQuery({
|
|
1105
|
+
queryHash: hash,
|
|
1106
|
+
surql: config.surql,
|
|
1107
|
+
params: config.params,
|
|
1108
|
+
ttl: new Duration(config.ttl),
|
|
1109
|
+
lastActiveAt: new Date(),
|
|
1110
|
+
});
|
|
1111
|
+
config.localArray = localArray;
|
|
1112
|
+
await this.local.query(surql.seal(surql.updateSet('id', ['localArray', 'rowCount'])), {
|
|
1113
|
+
id: config.id,
|
|
1114
|
+
localArray,
|
|
1115
|
+
rowCount: localArray.length,
|
|
1116
|
+
});
|
|
1117
|
+
} catch (err) {
|
|
1118
|
+
this.logger.error(
|
|
1119
|
+
{ err, hash, Category: 'sp00ky-client::DataModule::rebindAfterBucketSwitch' },
|
|
1120
|
+
'Failed to rebind query after bucket switch; remote re-registration will retry'
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
// Notify AFTER the SSP re-registration so a subscriber that re-reads
|
|
1125
|
+
// synchronously sees consistent (empty) state.
|
|
1126
|
+
const subscribers = this.subscriptions.get(hash);
|
|
1127
|
+
if (subscribers) {
|
|
1128
|
+
for (const callback of subscribers) {
|
|
1129
|
+
callback(queryState.records);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
this.startTTLHeartbeat(queryState, hash);
|
|
1133
|
+
hashes.push(hash);
|
|
1134
|
+
}
|
|
1135
|
+
return hashes;
|
|
307
1136
|
}
|
|
308
1137
|
|
|
309
1138
|
/**
|
|
@@ -313,20 +1142,29 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
313
1142
|
async notifyQuerySynced(queryHash: string): Promise<void> {
|
|
314
1143
|
const queryState = this.activeQueries.get(queryHash);
|
|
315
1144
|
if (!queryState) return;
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
);
|
|
322
|
-
|
|
1145
|
+
const epoch = this.local.epoch;
|
|
1146
|
+
|
|
1147
|
+
// Re-query local DB for latest data (windowed queries materialize from the
|
|
1148
|
+
// list_ref window so they resolve even if the in-browser SSP never emits —
|
|
1149
|
+
// it can't compute a high offset whose preceding rows aren't resident).
|
|
1150
|
+
const newRecords = await this.materializeRecords(queryState);
|
|
1151
|
+
// Bucket switched while we materialized: these rows mix old-bucket reads
|
|
1152
|
+
// with new-bucket state — drop them; the rebind/re-registration re-emits.
|
|
1153
|
+
if (epoch !== this.local.epoch) return;
|
|
323
1154
|
const changed = JSON.stringify(queryState.records) !== JSON.stringify(newRecords);
|
|
324
1155
|
queryState.records = newRecords;
|
|
325
1156
|
|
|
326
|
-
// Notify if data changed OR if this
|
|
327
|
-
// The latter handles "query truly has no
|
|
328
|
-
|
|
1157
|
+
// Notify if data changed OR if this registration lifetime hasn't emitted a
|
|
1158
|
+
// post-sync notification yet. The latter handles "query truly has no
|
|
1159
|
+
// results" so the UI can stop loading — gated on the in-memory
|
|
1160
|
+
// `syncNotified` flag rather than `updateCount === 0`, because updateCount
|
|
1161
|
+
// is PERSISTED across deregister/re-register: a re-registered empty window
|
|
1162
|
+
// (updateCount > 0, records unchanged) would otherwise never emit and its
|
|
1163
|
+
// subscribers would show a loading state forever.
|
|
1164
|
+
if (changed || !queryState.syncNotified) {
|
|
1165
|
+
queryState.syncNotified = true;
|
|
329
1166
|
queryState.updateCount++;
|
|
1167
|
+
queryState.lastUpdatedAt = Date.now();
|
|
330
1168
|
const subscribers = this.subscriptions.get(queryHash);
|
|
331
1169
|
if (subscribers) {
|
|
332
1170
|
for (const callback of subscribers) {
|
|
@@ -344,6 +1182,24 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
344
1182
|
data: RoutePayload<S, B, R>,
|
|
345
1183
|
options?: RunOptions
|
|
346
1184
|
): Promise<void> {
|
|
1185
|
+
const { tableName, record } = this.buildJobRecord(backend, path, data, options);
|
|
1186
|
+
const recordId = `${tableName}:${generateId()}`;
|
|
1187
|
+
await this.create(recordId, record);
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
/**
|
|
1191
|
+
* Build the outbox job record + resolve its table for a backend route.
|
|
1192
|
+
*
|
|
1193
|
+
* Every job is a single execution. Recurring work is declared server-side
|
|
1194
|
+
* (`schedules:` in sp00ky.yml) and the scheduler creates a fresh row per cycle,
|
|
1195
|
+
* so nothing here needs to know about schedules.
|
|
1196
|
+
*/
|
|
1197
|
+
private buildJobRecord<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(
|
|
1198
|
+
backend: B,
|
|
1199
|
+
path: R,
|
|
1200
|
+
data: RoutePayload<S, B, R>,
|
|
1201
|
+
options?: RunOptions
|
|
1202
|
+
): { tableName: string; record: Record<string, unknown> } {
|
|
347
1203
|
const route = this.schema.backends?.[backend]?.routes?.[path];
|
|
348
1204
|
if (!route) {
|
|
349
1205
|
throw new Error(`Route ${backend}.${path} not found`);
|
|
@@ -366,16 +1222,28 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
366
1222
|
const record: Record<string, unknown> = {
|
|
367
1223
|
path,
|
|
368
1224
|
payload: JSON.stringify(payload),
|
|
1225
|
+
// Set explicitly: the schema's DEFAULT ALWAYS "pending" is server-side
|
|
1226
|
+
// only. An optimistic local create without it surfaces with status
|
|
1227
|
+
// undefined, so in-flight indicators keyed on pending/processing stay
|
|
1228
|
+
// off until the first server echo (seconds on a delayed job).
|
|
1229
|
+
status: 'pending',
|
|
369
1230
|
max_retries: options?.max_retries ?? 3,
|
|
370
1231
|
retry_strategy: options?.retry_strategy ?? 'linear',
|
|
371
1232
|
};
|
|
372
1233
|
|
|
1234
|
+
if (options?.timeout != null) {
|
|
1235
|
+
record.timeout = options.timeout;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
if (options?.delay != null) {
|
|
1239
|
+
record.delay = options.delay;
|
|
1240
|
+
}
|
|
1241
|
+
|
|
373
1242
|
if (options?.assignedTo) {
|
|
374
1243
|
record.assigned_to = options.assignedTo;
|
|
375
1244
|
}
|
|
376
1245
|
|
|
377
|
-
|
|
378
|
-
await this.create(recordId, record);
|
|
1246
|
+
return { tableName, record };
|
|
379
1247
|
}
|
|
380
1248
|
|
|
381
1249
|
// ==================== MUTATION MANAGEMENT ====================
|
|
@@ -392,7 +1260,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
392
1260
|
|
|
393
1261
|
const rid = parseRecordIdString(id);
|
|
394
1262
|
const params = parseParams(tableSchema.columns, data);
|
|
395
|
-
const mutationId = parseRecordIdString(
|
|
1263
|
+
const mutationId = parseRecordIdString(mintMutationId(this.tabId));
|
|
396
1264
|
|
|
397
1265
|
const dataKeys = Object.keys(params).map((key) => ({ key, variable: `data_${key}` }));
|
|
398
1266
|
const prefixedParams = Object.fromEntries(
|
|
@@ -410,6 +1278,12 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
410
1278
|
this.local.execute(query, {
|
|
411
1279
|
id: rid,
|
|
412
1280
|
mid: mutationId,
|
|
1281
|
+
// The record itself is written with per-key `data_<field>` vars
|
|
1282
|
+
// (`createSet`), but the OUTBOX row needs the whole payload in one
|
|
1283
|
+
// `data` field so the mutation can be replayed later. Without this the
|
|
1284
|
+
// row is written with `data` unbound: the create becomes unsendable and
|
|
1285
|
+
// blocks the queue behind it.
|
|
1286
|
+
data: params,
|
|
413
1287
|
...prefixedParams,
|
|
414
1288
|
})
|
|
415
1289
|
);
|
|
@@ -441,7 +1315,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
441
1315
|
callback([mutationEvent]);
|
|
442
1316
|
}
|
|
443
1317
|
|
|
444
|
-
this.logger.debug({ id, Category: '
|
|
1318
|
+
this.logger.debug({ id, Category: 'sp00ky-client::DataModule::create' }, 'Record created');
|
|
445
1319
|
|
|
446
1320
|
return target;
|
|
447
1321
|
}
|
|
@@ -463,7 +1337,10 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
463
1337
|
|
|
464
1338
|
const rid = parseRecordIdString(id);
|
|
465
1339
|
const params = parseParams(tableSchema.columns, data);
|
|
466
|
-
const mutationId = parseRecordIdString(
|
|
1340
|
+
const mutationId = parseRecordIdString(mintMutationId(this.tabId));
|
|
1341
|
+
|
|
1342
|
+
// Note: CRDT state is pushed directly to the _00_crdt table by CrdtField.pushToRemote(),
|
|
1343
|
+
// NOT through the record update pipeline. This keeps the record data clean.
|
|
467
1344
|
|
|
468
1345
|
// Capture current record state before mutation for rollback support
|
|
469
1346
|
const [beforeRecord] = await withRetry(this.logger, () =>
|
|
@@ -472,7 +1349,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
472
1349
|
|
|
473
1350
|
const query = surql.seal<{ target: T }>(
|
|
474
1351
|
surql.tx([
|
|
475
|
-
surql.updateSet('id', [{ statement: '
|
|
1352
|
+
surql.updateSet('id', [{ statement: '_00_rv += 1' }]),
|
|
476
1353
|
surql.let('updated', surql.updateMerge('id', 'data')),
|
|
477
1354
|
surql.createMutation('update', 'mid', 'id', 'data'),
|
|
478
1355
|
surql.returnObject([{ key: 'target', variable: 'updated' }]),
|
|
@@ -496,8 +1373,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
496
1373
|
updatedFields[key] = (target as Record<string, any>)[key];
|
|
497
1374
|
}
|
|
498
1375
|
}
|
|
499
|
-
if ('
|
|
500
|
-
updatedFields.
|
|
1376
|
+
if ('_00_rv' in (target as Record<string, any>)) {
|
|
1377
|
+
updatedFields._00_rv = (target as Record<string, any>)._00_rv;
|
|
501
1378
|
}
|
|
502
1379
|
this.replaceRecordInQueries(updatedFields);
|
|
503
1380
|
|
|
@@ -509,7 +1386,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
509
1386
|
table: table,
|
|
510
1387
|
op: 'UPDATE',
|
|
511
1388
|
record: parsedRecord,
|
|
512
|
-
version: target.
|
|
1389
|
+
version: target._00_rv as number,
|
|
513
1390
|
},
|
|
514
1391
|
true
|
|
515
1392
|
);
|
|
@@ -531,7 +1408,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
531
1408
|
callback([mutationEvent]);
|
|
532
1409
|
}
|
|
533
1410
|
|
|
534
|
-
this.logger.debug({ id, Category: '
|
|
1411
|
+
this.logger.debug({ id, Category: 'sp00ky-client::DataModule::update' }, 'Record updated');
|
|
535
1412
|
|
|
536
1413
|
return target;
|
|
537
1414
|
}
|
|
@@ -547,14 +1424,53 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
547
1424
|
}
|
|
548
1425
|
|
|
549
1426
|
const rid = parseRecordIdString(id);
|
|
550
|
-
const mutationId = parseRecordIdString(
|
|
1427
|
+
const mutationId = parseRecordIdString(mintMutationId(this.tabId));
|
|
1428
|
+
|
|
1429
|
+
// Fetch the record before deleting so DBSP can match it against query predicates
|
|
1430
|
+
const [beforeRecords] = await this.local.query<[Record<string, any>[]]>(
|
|
1431
|
+
'SELECT * FROM ONLY $id',
|
|
1432
|
+
{ id: rid }
|
|
1433
|
+
);
|
|
1434
|
+
const beforeRecord = beforeRecords ?? {};
|
|
551
1435
|
|
|
552
1436
|
const query = surql.seal<void>(
|
|
553
1437
|
surql.tx([surql.delete('id'), surql.createMutation('delete', 'mid', 'id')])
|
|
554
1438
|
);
|
|
555
1439
|
|
|
556
1440
|
await withRetry(this.logger, () => this.local.execute(query, { id: rid, mid: mutationId }));
|
|
557
|
-
|
|
1441
|
+
|
|
1442
|
+
// The local DELETE has now committed. Everything below must reflect that in
|
|
1443
|
+
// active live queries — so the deleted row disappears optimistically without
|
|
1444
|
+
// a reload — even if the optimistic SSP-view ingest below fails. Previously a
|
|
1445
|
+
// throw from `cache.delete` (the WASM ingest) aborted `delete()` after the
|
|
1446
|
+
// commit, so the manual notify loop never ran and the row lingered on screen
|
|
1447
|
+
// until reload. Ingesting the delete into the in-browser SSP view is
|
|
1448
|
+
// best-effort: the manual re-materialize reads the local DB (which already
|
|
1449
|
+
// excludes the row), so the result is correct regardless.
|
|
1450
|
+
try {
|
|
1451
|
+
await this.cache.delete(table, id, true, beforeRecord);
|
|
1452
|
+
} catch (err) {
|
|
1453
|
+
this.logger.error(
|
|
1454
|
+
{ err, id, Category: 'sp00ky-client::DataModule::delete' },
|
|
1455
|
+
'SSP delete-ingest failed; relying on query re-materialize to reflect the delete'
|
|
1456
|
+
);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
// DBSP may not emit view updates for DELETE ops — manually notify all queries
|
|
1460
|
+
// that reference this table. Each is isolated so one failing re-materialize
|
|
1461
|
+
// can't stop the others (or the sync emit below) from running.
|
|
1462
|
+
for (const [queryHash, queryState] of this.activeQueries) {
|
|
1463
|
+
if (queryState.config.tableName === tableName) {
|
|
1464
|
+
try {
|
|
1465
|
+
await this.notifyQuerySynced(queryHash);
|
|
1466
|
+
} catch (err) {
|
|
1467
|
+
this.logger.error(
|
|
1468
|
+
{ err, queryHash, Category: 'sp00ky-client::DataModule::delete' },
|
|
1469
|
+
'notifyQuerySynced failed after delete'
|
|
1470
|
+
);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
558
1474
|
|
|
559
1475
|
// Emit mutation event
|
|
560
1476
|
const mutationEvent: DeleteEvent = {
|
|
@@ -567,7 +1483,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
567
1483
|
callback([mutationEvent]);
|
|
568
1484
|
}
|
|
569
1485
|
|
|
570
|
-
this.logger.debug({ id, Category: '
|
|
1486
|
+
this.logger.debug({ id, Category: 'sp00ky-client::DataModule::delete' }, 'Record deleted');
|
|
571
1487
|
}
|
|
572
1488
|
|
|
573
1489
|
// ==================== ROLLBACK METHODS ====================
|
|
@@ -579,19 +1495,17 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
579
1495
|
const id = encodeRecordId(recordId);
|
|
580
1496
|
|
|
581
1497
|
try {
|
|
582
|
-
await withRetry(this.logger, () =>
|
|
583
|
-
this.local.query('DELETE $id', { id: recordId })
|
|
584
|
-
);
|
|
1498
|
+
await withRetry(this.logger, () => this.local.query('DELETE $id', { id: recordId }));
|
|
585
1499
|
await this.cache.delete(tableName, id, true);
|
|
586
1500
|
this.removeRecordFromQueries(recordId);
|
|
587
1501
|
|
|
588
1502
|
this.logger.info(
|
|
589
|
-
{ id, tableName, Category: '
|
|
1503
|
+
{ id, tableName, Category: 'sp00ky-client::DataModule::rollbackCreate' },
|
|
590
1504
|
'Rolled back optimistic create'
|
|
591
1505
|
);
|
|
592
1506
|
} catch (err) {
|
|
593
1507
|
this.logger.error(
|
|
594
|
-
{ err, id, tableName, Category: '
|
|
1508
|
+
{ err, id, tableName, Category: 'sp00ky-client::DataModule::rollbackCreate' },
|
|
595
1509
|
'Failed to rollback create'
|
|
596
1510
|
);
|
|
597
1511
|
}
|
|
@@ -626,7 +1540,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
626
1540
|
table: tableName,
|
|
627
1541
|
op: 'UPDATE',
|
|
628
1542
|
record: parsedRecord,
|
|
629
|
-
version: (beforeRecord.
|
|
1543
|
+
version: (beforeRecord._00_rv as number) || 1,
|
|
630
1544
|
},
|
|
631
1545
|
true
|
|
632
1546
|
);
|
|
@@ -635,12 +1549,12 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
635
1549
|
await this.replaceRecordInQueries(beforeRecord);
|
|
636
1550
|
|
|
637
1551
|
this.logger.info(
|
|
638
|
-
{ id, tableName, Category: '
|
|
1552
|
+
{ id, tableName, Category: 'sp00ky-client::DataModule::rollbackUpdate' },
|
|
639
1553
|
'Rolled back optimistic update'
|
|
640
1554
|
);
|
|
641
1555
|
} catch (err) {
|
|
642
1556
|
this.logger.error(
|
|
643
|
-
{ err, id, tableName, Category: '
|
|
1557
|
+
{ err, id, tableName, Category: 'sp00ky-client::DataModule::rollbackUpdate' },
|
|
644
1558
|
'Failed to rollback update'
|
|
645
1559
|
);
|
|
646
1560
|
}
|
|
@@ -678,7 +1592,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
678
1592
|
surqlString: string,
|
|
679
1593
|
params: Record<string, any>,
|
|
680
1594
|
ttl: QueryTimeToLive,
|
|
681
|
-
tableName: T
|
|
1595
|
+
tableName: T,
|
|
1596
|
+
plan?: QueryPlan
|
|
682
1597
|
): Promise<QueryHash> {
|
|
683
1598
|
const queryState = await this.createNewQuery<T>({
|
|
684
1599
|
recordId,
|
|
@@ -686,31 +1601,76 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
686
1601
|
params,
|
|
687
1602
|
ttl,
|
|
688
1603
|
tableName,
|
|
1604
|
+
plan,
|
|
689
1605
|
});
|
|
690
1606
|
|
|
691
|
-
const
|
|
1607
|
+
const t0 = performance.now();
|
|
1608
|
+
const { localArray, registrationTimings } = this.cache.registerQuery({
|
|
692
1609
|
queryHash: hash,
|
|
693
1610
|
surql: surqlString,
|
|
694
1611
|
params,
|
|
695
1612
|
ttl: new Duration(ttl),
|
|
696
1613
|
lastActiveAt: new Date(),
|
|
697
1614
|
});
|
|
1615
|
+
const registrationTime = performance.now() - t0;
|
|
1616
|
+
|
|
1617
|
+
// Record the one-shot SSP registration timings (parse/plan/snapshot from the
|
|
1618
|
+
// WASM binding) + the register_view wall time for DevTools.
|
|
1619
|
+
queryState.registrationTimings = {
|
|
1620
|
+
parseMs: registrationTimings?.parseMs ?? null,
|
|
1621
|
+
planMs: registrationTimings?.planMs ?? null,
|
|
1622
|
+
snapshotMs: registrationTimings?.snapshotMs ?? null,
|
|
1623
|
+
wallMs: registrationTime,
|
|
1624
|
+
};
|
|
698
1625
|
|
|
699
1626
|
await withRetry(this.logger, () =>
|
|
700
|
-
this.local.query(
|
|
701
|
-
id
|
|
702
|
-
|
|
703
|
-
|
|
1627
|
+
this.local.query(
|
|
1628
|
+
surql.seal(surql.updateSet('id', ['localArray', 'registrationTime', 'rowCount'])),
|
|
1629
|
+
{
|
|
1630
|
+
id: recordId,
|
|
1631
|
+
localArray,
|
|
1632
|
+
registrationTime,
|
|
1633
|
+
rowCount: localArray.length,
|
|
1634
|
+
}
|
|
1635
|
+
)
|
|
704
1636
|
);
|
|
705
1637
|
|
|
1638
|
+
// Windowed (`START n`) queries skipped the raw initial load in
|
|
1639
|
+
// createNewQuery (O(offset) + wrong rows for sparse windows). Seed the
|
|
1640
|
+
// initial rows now from the SSP's window id-set (`localArray`) via the same
|
|
1641
|
+
// window-materialization path the stream updates use — O(window), and the
|
|
1642
|
+
// ids are already the correct window — so the first paint isn't empty while
|
|
1643
|
+
// the remote `_00_list_ref` syncs in.
|
|
1644
|
+
const windowMat = buildWindowMaterialization(surqlString);
|
|
1645
|
+
if (windowMat && localArray.length > 0) {
|
|
1646
|
+
try {
|
|
1647
|
+
const winIds = localArray.map(([id]) => parseRecordIdString(id));
|
|
1648
|
+
if (plan) {
|
|
1649
|
+
const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
|
|
1650
|
+
queryState.records = await this.local.select(winPlan, params);
|
|
1651
|
+
} else {
|
|
1652
|
+
const [seeded] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
|
|
1653
|
+
...params,
|
|
1654
|
+
__win: winIds,
|
|
1655
|
+
});
|
|
1656
|
+
queryState.records = seeded || [];
|
|
1657
|
+
}
|
|
1658
|
+
} catch (err) {
|
|
1659
|
+
this.logger.warn(
|
|
1660
|
+
{ err, hash, Category: 'sp00ky-client::DataModule::createAndRegisterQuery' },
|
|
1661
|
+
'Failed to seed windowed initial records from localArray'
|
|
1662
|
+
);
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
|
|
706
1666
|
this.activeQueries.set(hash, queryState);
|
|
707
|
-
this.startTTLHeartbeat(queryState);
|
|
1667
|
+
this.startTTLHeartbeat(queryState, hash);
|
|
708
1668
|
this.logger.debug(
|
|
709
1669
|
{
|
|
710
1670
|
hash,
|
|
711
1671
|
tableName,
|
|
712
1672
|
recordCount: queryState.records.length,
|
|
713
|
-
Category: '
|
|
1673
|
+
Category: 'sp00ky-client::DataModule::query',
|
|
714
1674
|
},
|
|
715
1675
|
'Query registered'
|
|
716
1676
|
);
|
|
@@ -724,14 +1684,25 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
724
1684
|
params,
|
|
725
1685
|
ttl,
|
|
726
1686
|
tableName,
|
|
1687
|
+
plan,
|
|
727
1688
|
}: {
|
|
728
1689
|
recordId: RecordId;
|
|
729
1690
|
surql: string;
|
|
730
1691
|
params: Record<string, any>;
|
|
731
1692
|
ttl: QueryTimeToLive;
|
|
732
1693
|
tableName: T;
|
|
1694
|
+
plan?: QueryPlan;
|
|
733
1695
|
}): Promise<QueryState> {
|
|
734
|
-
|
|
1696
|
+
// `_00_*` meta tables (feature flags, app releases) are framework-owned:
|
|
1697
|
+
// they exist in the client db schema by construction but are never part of
|
|
1698
|
+
// the app's generated `schema.tables`, so give them an empty column map
|
|
1699
|
+
// instead of the not-found error (which silently broke every meta-table
|
|
1700
|
+
// live query, e.g. feature flags never updating on this path).
|
|
1701
|
+
const tableSchema =
|
|
1702
|
+
this.schema.tables.find((t) => t.name === tableName) ??
|
|
1703
|
+
(String(tableName).startsWith('_00_')
|
|
1704
|
+
? ({ name: tableName, columns: {} } as any)
|
|
1705
|
+
: undefined);
|
|
735
1706
|
if (!tableSchema) {
|
|
736
1707
|
throw new Error(`Table ${tableName} not found`);
|
|
737
1708
|
}
|
|
@@ -752,8 +1723,12 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
752
1723
|
localArray: [],
|
|
753
1724
|
remoteArray: [],
|
|
754
1725
|
lastActiveAt: new Date(),
|
|
1726
|
+
createdAt: new Date(),
|
|
755
1727
|
ttl,
|
|
756
1728
|
tableName,
|
|
1729
|
+
updateCount: 0,
|
|
1730
|
+
rowCount: 0,
|
|
1731
|
+
errorCount: 0,
|
|
757
1732
|
},
|
|
758
1733
|
})
|
|
759
1734
|
);
|
|
@@ -763,62 +1738,109 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
763
1738
|
const config: QueryConfig = {
|
|
764
1739
|
...configRecord,
|
|
765
1740
|
id: recordId,
|
|
1741
|
+
// In-memory only — carries the engine-neutral plan so non-SurrealQL local
|
|
1742
|
+
// engines materialize via `select(plan)` instead of parsing `surql`.
|
|
1743
|
+
plan,
|
|
766
1744
|
params: parseParams(tableSchema.columns, configRecord.params),
|
|
767
1745
|
};
|
|
768
1746
|
|
|
769
1747
|
let records: Record<string, any>[] = [];
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
1748
|
+
// Windowed (`START n`) queries: do NOT seed from the raw surql here. Running
|
|
1749
|
+
// `… LIMIT n START m` against the shared local store is O(m) — it sorts and
|
|
1750
|
+
// skips m rows on every window open — AND returns the wrong rows for sparse
|
|
1751
|
+
// windows (the reason `buildWindowMaterialization` exists). Those windows are
|
|
1752
|
+
// seeded from the SSP `localArray` in `createAndRegisterQuery` instead.
|
|
1753
|
+
if (buildWindowMaterialization(surqlString) === null) {
|
|
1754
|
+
try {
|
|
1755
|
+
// Prefer the engine-neutral plan (required for non-SurrealQL engines);
|
|
1756
|
+
// fall back to running the raw surql on the SurrealDB engine.
|
|
1757
|
+
const result = plan
|
|
1758
|
+
? await this.local.select(plan, params)
|
|
1759
|
+
: (await this.local.query<[Record<string, any>[]]>(surqlString, params))[0];
|
|
1760
|
+
records = result || [];
|
|
1761
|
+
} catch (err) {
|
|
1762
|
+
this.logger.warn(
|
|
1763
|
+
{ err, Category: 'sp00ky-client::DataModule::createNewQuery' },
|
|
1764
|
+
'Failed to load initial cached records'
|
|
1765
|
+
);
|
|
1766
|
+
}
|
|
778
1767
|
}
|
|
779
1768
|
|
|
1769
|
+
// Persisted counters survive a restart even though the rolling
|
|
1770
|
+
// sample window is rebuilt from scratch in memory.
|
|
1771
|
+
const persistedUpdateCount =
|
|
1772
|
+
typeof (configRecord as any)?.updateCount === 'number'
|
|
1773
|
+
? (configRecord as any).updateCount
|
|
1774
|
+
: 0;
|
|
1775
|
+
const persistedErrorCount =
|
|
1776
|
+
typeof (configRecord as any)?.errorCount === 'number' ? (configRecord as any).errorCount : 0;
|
|
1777
|
+
|
|
780
1778
|
return {
|
|
781
1779
|
config,
|
|
782
1780
|
records,
|
|
783
1781
|
ttlTimer: null,
|
|
784
1782
|
ttlDurationMs: parseDuration(ttl),
|
|
785
|
-
updateCount:
|
|
1783
|
+
updateCount: persistedUpdateCount,
|
|
1784
|
+
lastUpdatedAt: null,
|
|
1785
|
+
materializationSamples: [],
|
|
1786
|
+
lastIngestLatencyMs: null,
|
|
1787
|
+
errorCount: persistedErrorCount,
|
|
1788
|
+
// Born `fetching`, not `idle`: every cold registration is followed by a
|
|
1789
|
+
// `register` down-event whose lifecycle (Sp00kySync.registerQuery) resolves
|
|
1790
|
+
// the status to `idle` once the initial sync completed. Starting idle left
|
|
1791
|
+
// a gap where a fresh windowed query looked settled while still empty.
|
|
1792
|
+
status: 'fetching',
|
|
1793
|
+
phaseSamples: {},
|
|
1794
|
+
phaseLast: {},
|
|
1795
|
+
registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
|
|
786
1796
|
};
|
|
787
1797
|
}
|
|
788
1798
|
|
|
789
1799
|
private async calculateHash(data: any): Promise<string> {
|
|
790
|
-
|
|
1800
|
+
// sessionId is part of the hash so the same logical query from two
|
|
1801
|
+
// sessions (e.g. two browser tabs of the same user) lands on different
|
|
1802
|
+
// `_00_query` rows and doesn't fight over a shared one.
|
|
1803
|
+
const content = JSON.stringify({ ...data, sessionId: this.sessionId });
|
|
791
1804
|
const msgBuffer = new TextEncoder().encode(content);
|
|
792
1805
|
const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);
|
|
793
1806
|
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
794
1807
|
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
|
|
795
1808
|
}
|
|
796
1809
|
|
|
797
|
-
private startTTLHeartbeat(queryState: QueryState): void {
|
|
1810
|
+
private startTTLHeartbeat(queryState: QueryState, hash: QueryHash): void {
|
|
798
1811
|
if (queryState.ttlTimer) return;
|
|
799
1812
|
|
|
800
1813
|
const heartbeatTime = Math.floor(queryState.ttlDurationMs * 0.9);
|
|
801
1814
|
|
|
802
1815
|
queryState.ttlTimer = setTimeout(() => {
|
|
803
|
-
|
|
1816
|
+
queryState.ttlTimer = null;
|
|
1817
|
+
// Only keep the remote query alive while something is actually watching
|
|
1818
|
+
// it. With the server now sweeping ALL expired views (not just in-circuit
|
|
1819
|
+
// ones), an un-refreshed query WOULD be swept after its TTL — so a live
|
|
1820
|
+
// subscriber must heartbeat. An abandoned query (no subscribers) is left
|
|
1821
|
+
// to expire and get swept; its local view was already torn down at the
|
|
1822
|
+
// last unsubscribe, so we just stop the timer here.
|
|
1823
|
+
const subscriberCount = this.subscriptions.get(hash)?.size ?? 0;
|
|
1824
|
+
if (subscriberCount === 0) {
|
|
1825
|
+
this.logger.debug(
|
|
1826
|
+
{ hash, Category: 'sp00ky-client::DataModule::startTTLHeartbeat' },
|
|
1827
|
+
'TTL heartbeat: no subscribers, stopping'
|
|
1828
|
+
);
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
this.onHeartbeat?.(hash);
|
|
804
1832
|
this.logger.debug(
|
|
805
1833
|
{
|
|
1834
|
+
hash,
|
|
806
1835
|
id: encodeRecordId(queryState.config.id),
|
|
807
|
-
Category: '
|
|
1836
|
+
Category: 'sp00ky-client::DataModule::startTTLHeartbeat',
|
|
808
1837
|
},
|
|
809
|
-
'TTL heartbeat'
|
|
1838
|
+
'TTL heartbeat sent'
|
|
810
1839
|
);
|
|
811
|
-
this.startTTLHeartbeat(queryState);
|
|
1840
|
+
this.startTTLHeartbeat(queryState, hash);
|
|
812
1841
|
}, heartbeatTime);
|
|
813
1842
|
}
|
|
814
1843
|
|
|
815
|
-
private stopTTLHeartbeat(queryState: QueryState): void {
|
|
816
|
-
if (queryState.ttlTimer) {
|
|
817
|
-
clearTimeout(queryState.ttlTimer);
|
|
818
|
-
queryState.ttlTimer = null;
|
|
819
|
-
}
|
|
820
|
-
}
|
|
821
|
-
|
|
822
1844
|
private async replaceRecordInQueries(record: Record<string, any>): Promise<void> {
|
|
823
1845
|
for (const [queryHash, queryState] of this.activeQueries.entries()) {
|
|
824
1846
|
const index = queryState.records.findIndex((r) => r.id === record.id);
|
|
@@ -851,7 +1873,7 @@ export function parseUpdateOptions(
|
|
|
851
1873
|
const delay = options.debounced !== true ? (options.debounced?.delay ?? 200) : 200;
|
|
852
1874
|
const keyType = options.debounced !== true ? (options.debounced?.key ?? id) : id;
|
|
853
1875
|
const key =
|
|
854
|
-
keyType === 'recordId_x_fields' ? `${id}::${Object.keys(data).
|
|
1876
|
+
keyType === 'recordId_x_fields' ? `${id}::${Object.keys(data).toSorted().join('#')}` : id;
|
|
855
1877
|
|
|
856
1878
|
pushEventOptions = {
|
|
857
1879
|
debounced: {
|