@spooky-sync/core 0.0.1-canary.65 → 0.0.1-canary.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +105 -17
- package/dist/index.js +754 -133
- package/dist/otel/index.d.ts +1 -1
- package/dist/types.d.ts +29 -3
- package/package.json +2 -2
- package/src/modules/cache/index.ts +6 -1
- package/src/modules/crdt/crdt-field.ts +115 -23
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +207 -68
- package/src/modules/data/index.ts +170 -17
- package/src/modules/ref-tables.test.ts +56 -0
- package/src/modules/ref-tables.ts +57 -0
- package/src/modules/sync/engine.ts +47 -11
- package/src/modules/sync/sync.ts +304 -13
- package/src/modules/sync/utils.test.ts +157 -0
- package/src/modules/sync/utils.ts +79 -0
- package/src/services/stream-processor/index.ts +25 -0
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +112 -31
- package/src/types.ts +29 -2
- package/src/utils/surql.ts +9 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { listRefTableFor, sanitizeUserId } from './ref-tables';
|
|
4
|
+
|
|
5
|
+
describe('listRefTableFor', () => {
|
|
6
|
+
it('returns global table in single mode regardless of user', () => {
|
|
7
|
+
expect(listRefTableFor('single', 'user:abc')).toBe('_00_list_ref');
|
|
8
|
+
expect(listRefTableFor('single', null)).toBe('_00_list_ref');
|
|
9
|
+
expect(listRefTableFor('single', undefined)).toBe('_00_list_ref');
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
it('returns per-user table in dedicated mode with valid user id', () => {
|
|
13
|
+
expect(listRefTableFor('dedicated', 'user:abc')).toBe(
|
|
14
|
+
'_00_list_ref_user_abc'
|
|
15
|
+
);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it('accepts a RecordId object in dedicated mode', () => {
|
|
19
|
+
const rid = new RecordId('user', 'def');
|
|
20
|
+
expect(listRefTableFor('dedicated', rid)).toBe('_00_list_ref_user_def');
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it('falls back to global in dedicated mode when user id is missing', () => {
|
|
24
|
+
expect(listRefTableFor('dedicated', null)).toBe('_00_list_ref');
|
|
25
|
+
expect(listRefTableFor('dedicated', undefined)).toBe('_00_list_ref');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('falls back to global in dedicated mode when user id has invalid chars', () => {
|
|
29
|
+
// SurrealDB table identifiers only accept alphanumerics + underscore.
|
|
30
|
+
expect(listRefTableFor('dedicated', 'user:abc-with-dash')).toBe(
|
|
31
|
+
'_00_list_ref'
|
|
32
|
+
);
|
|
33
|
+
expect(listRefTableFor('dedicated', 'user:abc.dot')).toBe('_00_list_ref');
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe('sanitizeUserId', () => {
|
|
38
|
+
it('strips the user: prefix', () => {
|
|
39
|
+
expect(sanitizeUserId('user:abc123')).toBe('abc123');
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('accepts plain ids without the user: prefix', () => {
|
|
43
|
+
expect(sanitizeUserId('abc123')).toBe('abc123');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('accepts RecordId objects', () => {
|
|
47
|
+
expect(sanitizeUserId(new RecordId('user', 'xyz'))).toBe('xyz');
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('returns null for invalid id shapes', () => {
|
|
51
|
+
expect(sanitizeUserId(null)).toBeNull();
|
|
52
|
+
expect(sanitizeUserId(undefined)).toBeNull();
|
|
53
|
+
expect(sanitizeUserId('user:')).toBeNull();
|
|
54
|
+
expect(sanitizeUserId('user:has-dash')).toBeNull();
|
|
55
|
+
});
|
|
56
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Client-side mirror of `packages/ssp-protocol/src/lib.rs`'s
|
|
2
|
+
// `query_table_for` / `list_ref_table_for`. Same naming convention so
|
|
3
|
+
// the LIVE subscription, the initial-fetch read, and the SSP's writes
|
|
4
|
+
// all land on the same table.
|
|
5
|
+
//
|
|
6
|
+
// The mode is currently hardcoded to `dedicated` because that's the
|
|
7
|
+
// only mode the e2e suite exercises and threading the value through
|
|
8
|
+
// codegen wasn't necessary to land the cross-session fix. If single
|
|
9
|
+
// mode ever needs to be exposed from the TS client too, the SSP server
|
|
10
|
+
// already reads it from `SPKY_SSP_REF_MODE`; add a matching codegen
|
|
11
|
+
// export then.
|
|
12
|
+
|
|
13
|
+
export type RefMode = 'single' | 'dedicated';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Default ref-storage mode for this client build. Mirrors the SSP's
|
|
17
|
+
* default (`RefMode::Dedicated`) so cross-session sync works out of the
|
|
18
|
+
* box.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_REF_MODE: RefMode = 'dedicated';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Sanitize a user record id (e.g. `"user:abc"`) into the segment that
|
|
24
|
+
* goes into a dedicated table name (e.g. `"abc"`). Returns `null` if
|
|
25
|
+
* the id is missing the `user:` prefix or contains characters that
|
|
26
|
+
* aren't valid in a SurrealDB table identifier — the server-side
|
|
27
|
+
* `ssp_protocol::sanitize_user_id` uses the same predicate.
|
|
28
|
+
*
|
|
29
|
+
* Accepts both string ids (`"user:abc"`) and SurrealDB `RecordId`
|
|
30
|
+
* objects (which only stringify cleanly via `.toString()`), since
|
|
31
|
+
* `AuthService` passes the record-id object as-is to its subscribers.
|
|
32
|
+
*/
|
|
33
|
+
export function sanitizeUserId(userId: unknown): string | null {
|
|
34
|
+
if (userId === null || userId === undefined) return null;
|
|
35
|
+
const asString =
|
|
36
|
+
typeof userId === 'string'
|
|
37
|
+
? userId
|
|
38
|
+
: typeof (userId as { toString?: unknown }).toString === 'function'
|
|
39
|
+
? (userId as { toString: () => string }).toString()
|
|
40
|
+
: null;
|
|
41
|
+
if (!asString) return null;
|
|
42
|
+
const raw = asString.startsWith('user:') ? asString.slice('user:'.length) : asString;
|
|
43
|
+
if (raw.length === 0) return null;
|
|
44
|
+
if (!/^[A-Za-z0-9_]+$/.test(raw)) return null;
|
|
45
|
+
return raw;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Returns the `_00_list_ref` table name for `(mode, userId)`. Falls
|
|
50
|
+
* back to the global `_00_list_ref` when sanitization fails or in
|
|
51
|
+
* single mode.
|
|
52
|
+
*/
|
|
53
|
+
export function listRefTableFor(mode: RefMode, userId: unknown): string {
|
|
54
|
+
if (mode === 'single') return '_00_list_ref';
|
|
55
|
+
const uid = sanitizeUserId(userId);
|
|
56
|
+
return uid ? `_00_list_ref_user_${uid}` : '_00_list_ref';
|
|
57
|
+
}
|
|
@@ -129,6 +129,17 @@ export class SyncEngine {
|
|
|
129
129
|
|
|
130
130
|
/**
|
|
131
131
|
* Handle records that exist locally but not in remote array.
|
|
132
|
+
*
|
|
133
|
+
* "Removed" here is a derived signal: the SSP's `_00_list_ref` array no
|
|
134
|
+
* longer references a record that exists locally. That can mean the row
|
|
135
|
+
* was genuinely deleted upstream — but it can also be a benign race
|
|
136
|
+
* (e.g. a record we just created hasn't propagated into the SSP's
|
|
137
|
+
* incantation list yet). Before deleting locally we verify against
|
|
138
|
+
* upstream SurrealDB: if the row still exists there, skip the delete.
|
|
139
|
+
*
|
|
140
|
+
* On verification failure we skip deletion too. Losing a stale local
|
|
141
|
+
* row to a later sync round is recoverable; deleting a fresh row that
|
|
142
|
+
* upstream still has is not.
|
|
132
143
|
*/
|
|
133
144
|
private async handleRemovedRecords(removed: RecordId[]): Promise<void> {
|
|
134
145
|
this.logger.debug(
|
|
@@ -139,19 +150,44 @@ export class SyncEngine {
|
|
|
139
150
|
'Checking removed records'
|
|
140
151
|
);
|
|
141
152
|
|
|
142
|
-
|
|
153
|
+
// Group by table so we can issue `SELECT id FROM type::table($t)
|
|
154
|
+
// WHERE id IN $ids` per table. The earlier shape `SELECT id FROM
|
|
155
|
+
// $ids` returns Internal/0 in SurrealDB 3.0 when `$ids` is bound as
|
|
156
|
+
// an array of RecordIds; this form works because the array shows up
|
|
157
|
+
// only in the WHERE clause, not as the FROM target.
|
|
158
|
+
const byTable = new Map<string, RecordId[]>();
|
|
159
|
+
for (const r of removed) {
|
|
160
|
+
const list = byTable.get(r.table.name) ?? [];
|
|
161
|
+
list.push(r);
|
|
162
|
+
byTable.set(r.table.name, list);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
let existingRemoteIds: Set<string>;
|
|
143
166
|
try {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
167
|
+
existingRemoteIds = new Set();
|
|
168
|
+
for (const [table, ids] of byTable) {
|
|
169
|
+
const [existing] = await this.remote.query<[{ id: RecordId }[]]>(
|
|
170
|
+
'SELECT id FROM type::table($table) WHERE id IN $ids',
|
|
171
|
+
{ table, ids }
|
|
172
|
+
);
|
|
173
|
+
for (const row of existing) {
|
|
174
|
+
existingRemoteIds.add(encodeRecordId(row.id));
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
} catch (err) {
|
|
178
|
+
// Verification failed. Skip deletion entirely — the next sync
|
|
179
|
+
// round re-derives the diff and we get another shot. The
|
|
180
|
+
// alternative (delete on uncertainty) destroys freshly-created
|
|
181
|
+
// rows when the SSP hasn't yet refreshed `_00_list_ref`.
|
|
182
|
+
this.logger.warn(
|
|
183
|
+
{
|
|
184
|
+
err,
|
|
185
|
+
removed: removed.map((r) => r.toString()),
|
|
186
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
187
|
+
},
|
|
188
|
+
'Remote existence check failed, skipping deletion to avoid clobbering fresh data'
|
|
154
189
|
);
|
|
190
|
+
return;
|
|
155
191
|
}
|
|
156
192
|
|
|
157
193
|
for (const recordId of removed) {
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -5,13 +5,33 @@ import type { Logger } from '../../services/logger/index';
|
|
|
5
5
|
import type { DownEvent, UpEvent} from './queue/index';
|
|
6
6
|
import { DownQueue, UpQueue } from './queue/index';
|
|
7
7
|
import type { RecordId, Uuid } from 'surrealdb';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
ArraySyncer,
|
|
10
|
+
buildListRefSelect,
|
|
11
|
+
createDiffFromDbOp,
|
|
12
|
+
nextPollDelayMs,
|
|
13
|
+
resolveListRefPollInterval,
|
|
14
|
+
} from './utils';
|
|
9
15
|
import { SyncEngine } from './engine';
|
|
10
16
|
import { SyncScheduler } from './scheduler';
|
|
11
17
|
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
12
18
|
import type { CacheModule } from '../cache/index';
|
|
13
19
|
import type { DataModule } from '../data/index';
|
|
14
20
|
import { encodeRecordId, extractTablePart, surql } from '../../utils/index';
|
|
21
|
+
import { DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Tunables for `Sp00kySync` construction.
|
|
25
|
+
*/
|
|
26
|
+
export interface Sp00kySyncOptions {
|
|
27
|
+
/**
|
|
28
|
+
* Cadence (ms) for the `_00_list_ref` poll fallback that catches
|
|
29
|
+
* cross-session UPDATEs the LIVE-permission gap drops. Non-positive
|
|
30
|
+
* values fall back to the default; see
|
|
31
|
+
* {@link resolveListRefPollInterval}.
|
|
32
|
+
*/
|
|
33
|
+
refSyncIntervalMs?: number;
|
|
34
|
+
}
|
|
15
35
|
|
|
16
36
|
/**
|
|
17
37
|
* The main synchronization engine for Sp00ky.
|
|
@@ -20,15 +40,66 @@ import { encodeRecordId, extractTablePart, surql } from '../../utils/index';
|
|
|
20
40
|
* @template S The schema structure type.
|
|
21
41
|
*/
|
|
22
42
|
export class Sp00kySync<S extends SchemaStructure> {
|
|
23
|
-
private clientId: string = '';
|
|
24
43
|
private upQueue: UpQueue;
|
|
25
44
|
private downQueue: DownQueue;
|
|
26
45
|
private isInit: boolean = false;
|
|
27
46
|
private logger: Logger;
|
|
28
47
|
private syncEngine: SyncEngine;
|
|
48
|
+
/** Engine-level events (e.g. `SYNC_REMOTE_DATA_INGESTED`). Distinct
|
|
49
|
+
* from `this.events`, which carries Sp00kySync-level events like
|
|
50
|
+
* `SYNC_QUERY_UPDATED` and `SYNC_MUTATION_ROLLED_BACK`. */
|
|
51
|
+
public get engineEvents() {
|
|
52
|
+
return this.syncEngine.events;
|
|
53
|
+
}
|
|
29
54
|
private scheduler: SyncScheduler;
|
|
55
|
+
private wasDisconnected: boolean = false;
|
|
30
56
|
public events = createSyncEventSystem();
|
|
31
57
|
|
|
58
|
+
// Auth identity that drives per-user `_00_list_ref_user_<id>` routing
|
|
59
|
+
// in `RefMode.Dedicated`. Updated by `setCurrentUserId` from the auth
|
|
60
|
+
// subscription in `Sp00kyClient`; null when unauthenticated.
|
|
61
|
+
private currentUserId: string | null = null;
|
|
62
|
+
|
|
63
|
+
private refMode: RefMode = DEFAULT_REF_MODE;
|
|
64
|
+
|
|
65
|
+
// Bookkeeping for the LIVE subscription on `_00_list_ref[_user_*]`.
|
|
66
|
+
// SurrealDB binds the permission context at LIVE-registration time and
|
|
67
|
+
// the table name in dedicated mode depends on the authenticated user,
|
|
68
|
+
// so we have to re-register whenever auth state flips.
|
|
69
|
+
private currentLiveQueryUuid: Uuid | null = null;
|
|
70
|
+
private liveQueryUnsubscribe: (() => void) | null = null;
|
|
71
|
+
|
|
72
|
+
// Periodic re-poll of `_00_list_ref` as a safety net for missed LIVE
|
|
73
|
+
// notifications. SurrealDB v3 occasionally drops LIVE deliveries
|
|
74
|
+
// across sessions even when the row matches the permission rule;
|
|
75
|
+
// this catches those without requiring users to reload. The
|
|
76
|
+
// interval is configurable via the constructor; see
|
|
77
|
+
// `resolveListRefPollInterval` for fallback semantics.
|
|
78
|
+
//
|
|
79
|
+
// Self-rescheduling rather than setInterval so each tick can pick
|
|
80
|
+
// its own delay via `nextPollDelayMs` — slows the poll down when
|
|
81
|
+
// LIVE is delivering events and speeds it back up when LIVE quiets.
|
|
82
|
+
private listRefPollTimer: ReturnType<typeof setTimeout> | null = null;
|
|
83
|
+
private listRefPollRunning: boolean = false;
|
|
84
|
+
public readonly refSyncIntervalMs: number;
|
|
85
|
+
|
|
86
|
+
// Wall-clock timestamp (ms) of the most recent LIVE event delivered
|
|
87
|
+
// through `handleRemoteListRefChange`. Read by `nextPollDelayMs` to
|
|
88
|
+
// back the poll off when LIVE is healthy.
|
|
89
|
+
private lastLiveEventAt: number | null = null;
|
|
90
|
+
|
|
91
|
+
// Number of times the initial `_00_list_ref[_user_*]` LIVE subscription
|
|
92
|
+
// had to retry on `setCurrentUserId`. Stays at 0 when the SSP has
|
|
93
|
+
// pre-emptively created the user's dedicated tables; otherwise
|
|
94
|
+
// increments on each retry attempt until LIVE succeeds or attempts
|
|
95
|
+
// are exhausted. Surfaced as a diagnostic so the e2e suite can prove
|
|
96
|
+
// the pre-emptive table-creation path is keeping the first sign-in
|
|
97
|
+
// off the lazy-creation race.
|
|
98
|
+
private _liveRetryCount: number = 0;
|
|
99
|
+
public get liveRetryCount(): number {
|
|
100
|
+
return this._liveRetryCount;
|
|
101
|
+
}
|
|
102
|
+
|
|
32
103
|
get isSyncing() {
|
|
33
104
|
return this.scheduler.isSyncing;
|
|
34
105
|
}
|
|
@@ -58,7 +129,8 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
58
129
|
private cache: CacheModule,
|
|
59
130
|
private dataModule: DataModule<S>,
|
|
60
131
|
private schema: S,
|
|
61
|
-
logger: Logger
|
|
132
|
+
logger: Logger,
|
|
133
|
+
options?: Sp00kySyncOptions
|
|
62
134
|
) {
|
|
63
135
|
this.logger = logger.child({ service: 'Sp00kySync' });
|
|
64
136
|
this.upQueue = new UpQueue(this.local, this.logger);
|
|
@@ -72,35 +144,242 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
72
144
|
this.logger,
|
|
73
145
|
this.handleRollback.bind(this)
|
|
74
146
|
);
|
|
147
|
+
this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
|
|
75
148
|
}
|
|
76
149
|
|
|
77
150
|
/**
|
|
78
151
|
* Initializes the synchronization system.
|
|
79
152
|
* Starts the scheduler and initiates the initial sync cycles.
|
|
80
|
-
* @param clientId The unique identifier for this client instance.
|
|
81
153
|
* @throws Error if already initialized.
|
|
82
154
|
*/
|
|
83
|
-
public async init(
|
|
155
|
+
public async init() {
|
|
84
156
|
if (this.isInit) throw new Error('Sp00kySync is already initialized');
|
|
85
|
-
this.clientId = clientId;
|
|
86
157
|
this.isInit = true;
|
|
87
158
|
await this.scheduler.init();
|
|
159
|
+
this.subscribeToReconnect();
|
|
88
160
|
void this.scheduler.syncUp();
|
|
89
161
|
void this.scheduler.syncDown();
|
|
90
|
-
|
|
162
|
+
// No initial LIVE subscription — wait for `setCurrentUserId` to fire
|
|
163
|
+
// from the auth subscription. In dedicated mode the table name
|
|
164
|
+
// depends on the authenticated user, and an unauthenticated
|
|
165
|
+
// subscription wouldn't match any of the per-user tables anyway.
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Push the authenticated user's record id from the parent client's
|
|
170
|
+
* auth subscription. Tears down the existing `_00_list_ref` LIVE (if
|
|
171
|
+
* any) and re-registers it under the new user's dedicated table so
|
|
172
|
+
* SurrealDB binds the permission rule under the post-flip auth
|
|
173
|
+
* context. Pass `null` on sign-out.
|
|
174
|
+
*
|
|
175
|
+
* The dedicated `_00_list_ref_user_<id>` table is created lazily by
|
|
176
|
+
* the SSP when the first query registration arrives, which may be
|
|
177
|
+
* concurrent with this call. We retry the LIVE registration with a
|
|
178
|
+
* short backoff so a "table not found" race resolves without
|
|
179
|
+
* surfacing as a permanent auth-loading hang.
|
|
180
|
+
*/
|
|
181
|
+
public async setCurrentUserId(userId: string | null): Promise<void> {
|
|
182
|
+
if (this.currentUserId === userId) return;
|
|
183
|
+
this.currentUserId = userId;
|
|
184
|
+
if (!userId) {
|
|
185
|
+
await this.killRefLiveQuery();
|
|
186
|
+
this.stopListRefPoll();
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
// Start periodic polling FIRST so we have a deterministic fallback
|
|
190
|
+
// even when LIVE registration fails or SurrealDB drops a delivery.
|
|
191
|
+
this.startListRefPoll();
|
|
192
|
+
// Try to start LIVE with backoff for low-latency delivery on the
|
|
193
|
+
// happy path; the poll handles the rest.
|
|
194
|
+
const attemptDelays = [0, 250, 500, 1000, 2000];
|
|
195
|
+
for (let i = 0; i < attemptDelays.length; i++) {
|
|
196
|
+
if (attemptDelays[i] > 0) {
|
|
197
|
+
this._liveRetryCount++;
|
|
198
|
+
await new Promise((r) => setTimeout(r, attemptDelays[i]));
|
|
199
|
+
}
|
|
200
|
+
try {
|
|
201
|
+
await this.restartRefLiveQuery();
|
|
202
|
+
return;
|
|
203
|
+
} catch (err) {
|
|
204
|
+
this.logger.debug(
|
|
205
|
+
{ err, attempt: i + 1, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
|
|
206
|
+
'Ref LIVE start failed; relying on periodic poll fallback'
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private startListRefPoll(): void {
|
|
213
|
+
if (this.listRefPollRunning) return;
|
|
214
|
+
this.listRefPollRunning = true;
|
|
215
|
+
this.logger.debug(
|
|
216
|
+
{
|
|
217
|
+
intervalMs: this.refSyncIntervalMs,
|
|
218
|
+
Category: 'sp00ky-client::Sp00kySync::startListRefPoll',
|
|
219
|
+
},
|
|
220
|
+
'list_ref poll loop started'
|
|
221
|
+
);
|
|
222
|
+
const schedule = (delayMs: number) => {
|
|
223
|
+
this.listRefPollTimer = setTimeout(async () => {
|
|
224
|
+
if (!this.listRefPollRunning) return;
|
|
225
|
+
try {
|
|
226
|
+
await this.pollListRefForActiveQueries();
|
|
227
|
+
} finally {
|
|
228
|
+
if (!this.listRefPollRunning) return;
|
|
229
|
+
const next = nextPollDelayMs({
|
|
230
|
+
now: Date.now(),
|
|
231
|
+
lastLiveEventAt: this.lastLiveEventAt,
|
|
232
|
+
baseIntervalMs: this.refSyncIntervalMs,
|
|
233
|
+
});
|
|
234
|
+
schedule(next);
|
|
235
|
+
}
|
|
236
|
+
}, delayMs);
|
|
237
|
+
};
|
|
238
|
+
schedule(this.refSyncIntervalMs);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private stopListRefPoll(): void {
|
|
242
|
+
this.listRefPollRunning = false;
|
|
243
|
+
if (this.listRefPollTimer !== null) {
|
|
244
|
+
clearTimeout(this.listRefPollTimer);
|
|
245
|
+
this.listRefPollTimer = null;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private async pollListRefForActiveQueries(): Promise<void> {
|
|
250
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
251
|
+
if (hashes.length === 0) return;
|
|
252
|
+
for (const hash of hashes) {
|
|
253
|
+
try {
|
|
254
|
+
await this.refetchListRefForQuery(hash);
|
|
255
|
+
} catch (err) {
|
|
256
|
+
this.logger.debug(
|
|
257
|
+
{ err: (err as Error)?.message ?? err, hash, Category: 'sp00ky-client::Sp00kySync::pollListRefForActiveQueries' },
|
|
258
|
+
'Per-query list_ref poll failed'
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Pull the upstream list_ref entries for `queryHash`, diff them
|
|
266
|
+
* against the local `remoteArray` cache, sync any added/updated rows
|
|
267
|
+
* through the SyncEngine, then persist the new remoteArray. This is
|
|
268
|
+
* the same shape `createRemoteQuery` does for its initial fetch and
|
|
269
|
+
* what `handleRemoteListRefChange` does per-LIVE-event — we reuse
|
|
270
|
+
* it on a timer as a fallback for missed LIVE notifications.
|
|
271
|
+
*/
|
|
272
|
+
private async refetchListRefForQuery(queryHash: string): Promise<void> {
|
|
273
|
+
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
274
|
+
if (!queryState) return;
|
|
275
|
+
const listRefTbl = this.listRefTable();
|
|
276
|
+
const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
|
|
277
|
+
buildListRefSelect(listRefTbl),
|
|
278
|
+
{ in: queryState.config.id }
|
|
279
|
+
);
|
|
280
|
+
if (!Array.isArray(items)) return;
|
|
281
|
+
const fresh: RecordVersionArray = items.map((item) => [
|
|
282
|
+
encodeRecordId(item.out),
|
|
283
|
+
item.version,
|
|
284
|
+
]);
|
|
285
|
+
// Update the cached remoteArray and run `syncQuery` inline so the
|
|
286
|
+
// sync engine fetches any added/updated rows immediately (instead
|
|
287
|
+
// of waiting for the next scheduler pass). `syncQuery` writes
|
|
288
|
+
// through `cache.saveBatch`, which both UPSERTs the local DB row
|
|
289
|
+
// and ingests it into the in-browser SSP — the SSP's stream
|
|
290
|
+
// updates then run `processStreamUpdate`, which re-queries the
|
|
291
|
+
// local DB and notifies subscribers. We skip an explicit
|
|
292
|
+
// `notifyQuerySynced` because that path runs concurrently with
|
|
293
|
+
// the stream-update path and can race (e.g. notify with stale
|
|
294
|
+
// records).
|
|
295
|
+
await this.dataModule.updateQueryRemoteArray(queryHash, fresh);
|
|
296
|
+
try {
|
|
297
|
+
await this.syncQuery(queryHash);
|
|
298
|
+
} catch (err) {
|
|
299
|
+
this.logger.info(
|
|
300
|
+
{ err: (err as Error)?.message ?? err, queryHash, Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery' },
|
|
301
|
+
'syncQuery failed during poll'
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Resolve the current `_00_list_ref` table name for the active auth
|
|
308
|
+
* context. Public so the `createRemoteQuery` initial-fetch path can
|
|
309
|
+
* read from the right per-user table.
|
|
310
|
+
*
|
|
311
|
+
* Reads the user id from `DataModule` rather than the local mirror,
|
|
312
|
+
* because `DataModule.setCurrentUserId` runs synchronously from the
|
|
313
|
+
* auth callback (before any `await`), whereas `sync.setCurrentUserId`
|
|
314
|
+
* is async — the userQuery's initial fetch can fire between those
|
|
315
|
+
* two points and we need the correct table name immediately.
|
|
316
|
+
*/
|
|
317
|
+
public listRefTable(): string {
|
|
318
|
+
return listRefTableFor(this.refMode, this.dataModule.getCurrentUserId());
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private async killRefLiveQuery(): Promise<void> {
|
|
322
|
+
if (this.liveQueryUnsubscribe) {
|
|
323
|
+
try { this.liveQueryUnsubscribe(); } catch { /* ignore */ }
|
|
324
|
+
this.liveQueryUnsubscribe = null;
|
|
325
|
+
}
|
|
326
|
+
if (this.currentLiveQueryUuid !== null) {
|
|
327
|
+
try {
|
|
328
|
+
await this.remote.query('KILL $u', { u: this.currentLiveQueryUuid });
|
|
329
|
+
} catch (err) {
|
|
330
|
+
this.logger.debug(
|
|
331
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::killRefLiveQuery' },
|
|
332
|
+
'Prior LIVE KILL failed; continuing'
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
this.currentLiveQueryUuid = null;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
private async restartRefLiveQuery(): Promise<void> {
|
|
340
|
+
await this.killRefLiveQuery();
|
|
341
|
+
await this.startRefLiveQueries();
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Only the connect that follows a prior disconnect counts as a
|
|
345
|
+
// reconnect; the initial connect after init() must not trigger a
|
|
346
|
+
// refetch storm.
|
|
347
|
+
private subscribeToReconnect() {
|
|
348
|
+
const client = this.remote.getClient();
|
|
349
|
+
client.subscribe('disconnected', () => {
|
|
350
|
+
this.wasDisconnected = true;
|
|
351
|
+
this.logger.info(
|
|
352
|
+
{ Category: 'sp00ky-client::Sp00kySync::onDisconnect' },
|
|
353
|
+
'Remote disconnected'
|
|
354
|
+
);
|
|
355
|
+
});
|
|
356
|
+
client.subscribe('connected', () => {
|
|
357
|
+
if (!this.wasDisconnected) return;
|
|
358
|
+
this.wasDisconnected = false;
|
|
359
|
+
this.logger.info(
|
|
360
|
+
{ Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
361
|
+
'Remote reconnected, refetching active queries'
|
|
362
|
+
);
|
|
363
|
+
for (const hash of this.dataModule.getActiveQueryHashes()) {
|
|
364
|
+
this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
|
|
365
|
+
}
|
|
366
|
+
});
|
|
91
367
|
}
|
|
92
368
|
|
|
93
369
|
private async startRefLiveQueries() {
|
|
370
|
+
const tableName = this.listRefTable();
|
|
94
371
|
this.logger.debug(
|
|
95
|
-
{
|
|
372
|
+
{ tableName, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
96
373
|
'Starting ref live queries'
|
|
97
374
|
);
|
|
98
375
|
|
|
99
376
|
const [queryUuid] = await this.remote.query<[Uuid]>(
|
|
100
|
-
|
|
377
|
+
`LIVE SELECT * FROM ${tableName}`
|
|
101
378
|
);
|
|
379
|
+
this.currentLiveQueryUuid = queryUuid;
|
|
102
380
|
|
|
103
|
-
|
|
381
|
+
const live = await this.remote.getClient().liveOf(queryUuid);
|
|
382
|
+
this.liveQueryUnsubscribe = live.subscribe((message) => {
|
|
104
383
|
this.logger.debug(
|
|
105
384
|
{ message, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
106
385
|
'Live update received'
|
|
@@ -126,6 +405,12 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
126
405
|
recordId: RecordId,
|
|
127
406
|
version: number
|
|
128
407
|
) {
|
|
408
|
+
// Any LIVE delivery is evidence that the feed is healthy — even
|
|
409
|
+
// a DELETE we'll ignore below or a notification for an unknown
|
|
410
|
+
// local query counts. `nextPollDelayMs` reads this to slow the
|
|
411
|
+
// poll fallback while LIVE is doing its job.
|
|
412
|
+
this.lastLiveEventAt = Date.now();
|
|
413
|
+
|
|
129
414
|
if (action === 'DELETE') {
|
|
130
415
|
this.logger.debug(
|
|
131
416
|
{
|
|
@@ -348,10 +633,10 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
348
633
|
);
|
|
349
634
|
throw new Error('Query to register not found');
|
|
350
635
|
}
|
|
351
|
-
// Delegate to remote function which handles DBSP registration & persistence
|
|
636
|
+
// Delegate to remote function which handles DBSP registration & persistence.
|
|
637
|
+
// clientId is set server-side from session::id() — see fn::query::register.
|
|
352
638
|
await this.remote.query('fn::query::register($config)', {
|
|
353
639
|
config: {
|
|
354
|
-
clientId: this.clientId,
|
|
355
640
|
id: queryState.config.id,
|
|
356
641
|
surql: queryState.config.surql,
|
|
357
642
|
params: queryState.config.params,
|
|
@@ -359,8 +644,14 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
359
644
|
},
|
|
360
645
|
});
|
|
361
646
|
|
|
647
|
+
// Initial materialized-view fetch — pull from the same per-user
|
|
648
|
+
// `_00_list_ref_user_<id>` (or global `_00_list_ref` in single
|
|
649
|
+
// mode) that the LIVE subscription listens on, so the two stay in
|
|
650
|
+
// sync. `parent IS NONE` excludes subquery entries; the
|
|
651
|
+
// `localArray` cache only tracks primary records.
|
|
652
|
+
const listRefTbl = this.listRefTable();
|
|
362
653
|
const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
|
|
363
|
-
|
|
654
|
+
buildListRefSelect(listRefTbl),
|
|
364
655
|
{
|
|
365
656
|
in: queryState.config.id,
|
|
366
657
|
}
|