@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2514 -58
- package/dist/index.js +12561 -2449
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +303 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +472 -0
- package/dist/types.d.ts +751 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +36 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/auth.local-first.test.ts +101 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +127 -24
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +163 -43
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +523 -0
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.pending-ids.test.ts +199 -0
- package/src/modules/data/data.rebind.test.ts +170 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1815 -151
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +450 -46
- package/src/modules/devtools/notify-throttle.test.ts +154 -0
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +164 -82
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +180 -0
- package/src/modules/sync/queue/queue-down.ts +80 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +237 -0
- package/src/modules/sync/scheduler.ts +215 -13
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +175 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +1726 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +193 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +41 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +484 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +616 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1358 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +138 -0
- package/src/services/database/sqlite-projection.test.ts +99 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +161 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +796 -84
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +59 -3
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +327 -0
- package/src/services/tabs/coordinator.test.ts +365 -0
- package/src/services/tabs/coordinator.ts +633 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +258 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +640 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.local-first.test.ts +60 -0
- package/src/sp00ky.ts +1693 -0
- package/src/types.ts +528 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +79 -13
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +32 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { LocalStore } from '../../services/database/index';
|
|
2
|
+
import { StaleEpochError } from '../../services/database/index';
|
|
3
|
+
import type {
|
|
3
4
|
StreamProcessorService,
|
|
4
5
|
StreamUpdate,
|
|
5
6
|
StreamUpdateReceiver,
|
|
6
7
|
} from '../../services/stream-processor/index';
|
|
7
|
-
import { Logger } from '../../services/logger/index';
|
|
8
|
+
import type { Logger } from '../../services/logger/index';
|
|
8
9
|
import { parseRecordIdString, encodeRecordId, surql } from '../../utils/index';
|
|
9
|
-
import { CacheRecord, QueryConfig } from './types';
|
|
10
|
-
import { RecordVersionArray } from '../../types';
|
|
10
|
+
import type { CacheRecord, QueryConfig } from './types';
|
|
11
|
+
import type { RecordVersionArray } from '../../types';
|
|
11
12
|
|
|
12
13
|
export * from './types';
|
|
13
14
|
|
|
@@ -17,13 +18,29 @@ export * from './types';
|
|
|
17
18
|
* Single responsibility: Handle all local storage operations and DBSP ingestion.
|
|
18
19
|
* This module acts as the bridge between data operations and persistence.
|
|
19
20
|
*/
|
|
21
|
+
/** One ingested change, in exactly the shape `ingestMany` consumes. Shared
|
|
22
|
+
* with the tabs protocol so a leader can relay its ingests to followers. */
|
|
23
|
+
export interface CacheIngestTuple {
|
|
24
|
+
table: string;
|
|
25
|
+
op: 'CREATE' | 'UPDATE' | 'DELETE';
|
|
26
|
+
id: string;
|
|
27
|
+
record: Record<string, unknown>;
|
|
28
|
+
}
|
|
29
|
+
|
|
20
30
|
export class CacheModule implements StreamUpdateReceiver {
|
|
21
31
|
private logger: Logger;
|
|
22
32
|
private streamUpdateCallback: (update: StreamUpdate) => void;
|
|
23
33
|
private versionLookups: Record<string, number> = {};
|
|
34
|
+
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
35
|
+
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
36
|
+
* follower only needs the circuit feed). A follower relays its own
|
|
37
|
+
* mutations to the leader the same way, see {@link setIngestRelay}. */
|
|
38
|
+
private ingestRelay: ((tuples: CacheIngestTuple[]) => void) | null = null;
|
|
39
|
+
/** See {@link setIngestRelay}. */
|
|
40
|
+
private relayLocalWritesOnly = false;
|
|
24
41
|
|
|
25
42
|
constructor(
|
|
26
|
-
private local:
|
|
43
|
+
private local: LocalStore,
|
|
27
44
|
private streamProcessor: StreamProcessorService,
|
|
28
45
|
streamUpdateCallback: (update: StreamUpdate) => void,
|
|
29
46
|
logger: Logger
|
|
@@ -43,17 +60,69 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
43
60
|
{
|
|
44
61
|
queryHash: update.queryHash,
|
|
45
62
|
arrayLength: update.localArray?.length,
|
|
46
|
-
Category: '
|
|
63
|
+
Category: 'sp00ky-client::CacheModule::onStreamUpdate',
|
|
47
64
|
},
|
|
48
65
|
'Stream update received'
|
|
49
66
|
);
|
|
50
67
|
this.streamUpdateCallback(update);
|
|
51
68
|
}
|
|
52
69
|
|
|
70
|
+
/**
|
|
71
|
+
* Fan every committed ingest out to the other tabs. The leader relays
|
|
72
|
+
* everything (its sync fetches are the only copy the followers get). A
|
|
73
|
+
* follower relays with `localWritesOnly`: just the mutation path, which is
|
|
74
|
+
* the only thing it knows that the leader does not. Its sync-fetched
|
|
75
|
+
* batches are the leader's data coming back and must not be re-broadcast,
|
|
76
|
+
* or every follower registration would fan its whole working set to
|
|
77
|
+
* every tab.
|
|
78
|
+
*/
|
|
79
|
+
setIngestRelay(
|
|
80
|
+
cb: ((tuples: CacheIngestTuple[]) => void) | null,
|
|
81
|
+
opts: { localWritesOnly?: boolean } = {}
|
|
82
|
+
): void {
|
|
83
|
+
this.ingestRelay = cb;
|
|
84
|
+
this.relayLocalWritesOnly = opts.localWritesOnly === true;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
|
|
89
|
+
* The rows are already in the shared store (the leader wrote them), so no
|
|
90
|
+
* local write happens here; the normal chain then runs: SSP -> stream update
|
|
91
|
+
* -> DataModule debounce -> materializeRecords (re-reads via the port
|
|
92
|
+
* transport) -> this tab's subscriptions fire with this tab's hashes.
|
|
93
|
+
*/
|
|
94
|
+
applyRelayedIngest(tuples: CacheIngestTuple[]): void {
|
|
95
|
+
for (const t of tuples) {
|
|
96
|
+
const rv = (t.record as { _00_rv?: number } | undefined)?._00_rv;
|
|
97
|
+
if (t.op === 'DELETE') delete this.versionLookups[t.id];
|
|
98
|
+
else if (typeof rv === 'number') this.versionLookups[t.id] = rv;
|
|
99
|
+
}
|
|
100
|
+
this.streamProcessor.ingestMany(tuples as Parameters<StreamProcessorService['ingestMany']>[0]);
|
|
101
|
+
}
|
|
102
|
+
|
|
53
103
|
public lookup(recordId: string): number {
|
|
54
104
|
return this.versionLookups[recordId] ?? 0;
|
|
55
105
|
}
|
|
56
106
|
|
|
107
|
+
/**
|
|
108
|
+
* Seed the version memo from rows the circuit was primed with out of the
|
|
109
|
+
* local store, so the first post-reload sync diff does not re-download
|
|
110
|
+
* bodies the browser already has. Only rows the prime actually put into the
|
|
111
|
+
* circuit belong here: a memo entry with no circuit row would make the diff
|
|
112
|
+
* flag the id forever while nothing ever fetches it.
|
|
113
|
+
*/
|
|
114
|
+
public primeVersions(entries: [string, number][]): void {
|
|
115
|
+
for (const [id, rv] of entries) {
|
|
116
|
+
if (rv > (this.versionLookups[id] ?? 0)) this.versionLookups[id] = rv;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Drop the version cache on a bucket switch — a stale version would make
|
|
121
|
+
* the sync diff skip fetching a body the new bucket legitimately needs. */
|
|
122
|
+
public clearVersionLookups(): void {
|
|
123
|
+
this.versionLookups = {};
|
|
124
|
+
}
|
|
125
|
+
|
|
57
126
|
/**
|
|
58
127
|
* Save a single record to local DB and ingest into DBSP
|
|
59
128
|
* Used by mutations (create/update)
|
|
@@ -70,10 +139,17 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
70
139
|
async saveBatch(records: CacheRecord[], skipDbInsert: boolean = false): Promise<void> {
|
|
71
140
|
if (records.length === 0) return;
|
|
72
141
|
|
|
142
|
+
// Fence against bucket switches: this batch's records were derived from
|
|
143
|
+
// reads against the CURRENT store/user. If a switch lands while we await
|
|
144
|
+
// the (gated) local write, the write throws StaleEpochError and the whole
|
|
145
|
+
// batch — including the SSP ingest — is dropped: the new bucket re-syncs
|
|
146
|
+
// its own data from the server.
|
|
147
|
+
const epoch = this.local.epoch;
|
|
148
|
+
|
|
73
149
|
this.logger.debug(
|
|
74
150
|
{
|
|
75
151
|
count: records.length,
|
|
76
|
-
Category: '
|
|
152
|
+
Category: 'sp00ky-client::CacheModule::saveBatch',
|
|
77
153
|
},
|
|
78
154
|
'Saving record batch'
|
|
79
155
|
);
|
|
@@ -85,7 +161,7 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
85
161
|
...record,
|
|
86
162
|
record: {
|
|
87
163
|
...record.record,
|
|
88
|
-
|
|
164
|
+
_00_rv: record.version,
|
|
89
165
|
},
|
|
90
166
|
};
|
|
91
167
|
});
|
|
@@ -94,40 +170,69 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
94
170
|
const query = surql.seal<void>(
|
|
95
171
|
surql.tx(
|
|
96
172
|
populatedRecords.map((_, i) => {
|
|
97
|
-
|
|
173
|
+
// MERGE, not REPLACE: the remote payload omits local-only
|
|
174
|
+
// fields (`_00_crdt`, `_00_cursor`) injected by the CLI's
|
|
175
|
+
// local schema, so REPLACE would wipe the persisted CRDT
|
|
176
|
+
// snapshot on every sync-down round-trip and break offline
|
|
177
|
+
// reload of formatted text.
|
|
178
|
+
return surql.upsertMerge(`id${i}`, `content${i}`);
|
|
98
179
|
})
|
|
99
180
|
)
|
|
100
181
|
);
|
|
101
182
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
}
|
|
111
|
-
{}
|
|
112
|
-
|
|
183
|
+
// Filled in place, NOT with a spread-per-iteration reduce: spreading the
|
|
184
|
+
// accumulator copies every key written so far on each record, which is
|
|
185
|
+
// O(n^2) in the batch size. On a cold start that batches thousands of
|
|
186
|
+
// rows (a game library, a player-name registry) it was the single
|
|
187
|
+
// biggest main-thread cost of the whole boot - ~36% of samples, seconds
|
|
188
|
+
// of blocking - for a loop that does no real work.
|
|
189
|
+
const params: Record<string, any> = {};
|
|
190
|
+
for (let i = 0; i < populatedRecords.length; i++) {
|
|
191
|
+
const { id, ...content } = populatedRecords[i].record;
|
|
192
|
+
params[`id${i}`] = id;
|
|
193
|
+
params[`content${i}`] = content;
|
|
194
|
+
}
|
|
113
195
|
|
|
114
|
-
await this.local.execute(query, params);
|
|
196
|
+
await this.local.execute(query, params, { epoch });
|
|
115
197
|
}
|
|
116
198
|
|
|
117
|
-
//
|
|
118
|
-
|
|
199
|
+
// Late fence for the skipDbInsert path (no gated write above to trip on).
|
|
200
|
+
if (this.local.epoch !== epoch) throw new StaleEpochError();
|
|
201
|
+
|
|
202
|
+
// 2. Bulk ingest into DBSP (use populatedRecords which has _00_rv set).
|
|
203
|
+
// ingestMany coalesces the per-record stream updates into a single
|
|
204
|
+
// notification per affected query — the UI then updates once, after the
|
|
205
|
+
// whole batch is ingested, instead of row-by-row.
|
|
206
|
+
const versionOf = new Map<string, number>();
|
|
207
|
+
const bulk = populatedRecords.map((record) => {
|
|
119
208
|
const recordId = encodeRecordId(record.record.id);
|
|
120
|
-
|
|
121
|
-
|
|
209
|
+
versionOf.set(recordId, record.version);
|
|
210
|
+
return { table: record.table, op: record.op, id: recordId, record: record.record };
|
|
211
|
+
});
|
|
212
|
+
// Memo and relay only what the circuit actually took: a chunk that
|
|
213
|
+
// failed mid-batch is neither "known" here nor fanned out to followers.
|
|
214
|
+
const ingested = this.streamProcessor.ingestMany(bulk);
|
|
215
|
+
for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
|
|
216
|
+
// `skipDbInsert` is exactly the mutation path (the tx already wrote the
|
|
217
|
+
// row); sync-fetched batches pass false.
|
|
218
|
+
if (ingested.length > 0 && (!this.relayLocalWritesOnly || skipDbInsert)) {
|
|
219
|
+
this.ingestRelay?.(ingested as CacheIngestTuple[]);
|
|
122
220
|
}
|
|
123
221
|
|
|
124
222
|
this.logger.debug(
|
|
125
|
-
{ count: records.length, Category: '
|
|
223
|
+
{ count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
126
224
|
'Batch saved successfully'
|
|
127
225
|
);
|
|
128
226
|
} catch (err) {
|
|
227
|
+
if (err instanceof StaleEpochError) {
|
|
228
|
+
this.logger.debug(
|
|
229
|
+
{ count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
230
|
+
'Dropped batch from before a bucket switch'
|
|
231
|
+
);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
129
234
|
this.logger.error(
|
|
130
|
-
{ err, count: records.length, Category: '
|
|
235
|
+
{ err, count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
131
236
|
'Failed to save batch'
|
|
132
237
|
);
|
|
133
238
|
throw err;
|
|
@@ -137,29 +242,41 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
137
242
|
/**
|
|
138
243
|
* Delete a record from local DB and ingest deletion into DBSP
|
|
139
244
|
*/
|
|
140
|
-
async delete(table: string, id: string, skipDbDelete: boolean = false): Promise<void> {
|
|
245
|
+
async delete(table: string, id: string, skipDbDelete: boolean = false, recordData: Record<string, any> = {}): Promise<void> {
|
|
141
246
|
this.logger.debug(
|
|
142
|
-
{ table, id, Category: '
|
|
247
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
143
248
|
'Deleting record'
|
|
144
249
|
);
|
|
145
250
|
|
|
251
|
+
const epoch = this.local.epoch;
|
|
146
252
|
try {
|
|
147
253
|
// 1. Delete from local database
|
|
148
254
|
if (!skipDbDelete) {
|
|
149
|
-
await this.local.query('DELETE $id', { id: parseRecordIdString(id) });
|
|
255
|
+
await this.local.query('DELETE $id', { id: parseRecordIdString(id) }, { epoch });
|
|
150
256
|
}
|
|
257
|
+
if (this.local.epoch !== epoch) throw new StaleEpochError();
|
|
151
258
|
|
|
152
|
-
// 2. Ingest deletion into DBSP
|
|
259
|
+
// 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
|
|
153
260
|
delete this.versionLookups[id];
|
|
154
|
-
|
|
261
|
+
this.streamProcessor.ingestMany([{ table, op: 'DELETE', id, record: recordData }]);
|
|
262
|
+
if (!this.relayLocalWritesOnly || skipDbDelete) {
|
|
263
|
+
this.ingestRelay?.([{ table, op: 'DELETE', id, record: recordData }]);
|
|
264
|
+
}
|
|
155
265
|
|
|
156
266
|
this.logger.debug(
|
|
157
|
-
{ table, id, Category: '
|
|
267
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
158
268
|
'Record deleted successfully'
|
|
159
269
|
);
|
|
160
270
|
} catch (err) {
|
|
271
|
+
if (err instanceof StaleEpochError) {
|
|
272
|
+
this.logger.debug(
|
|
273
|
+
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
274
|
+
'Dropped delete from before a bucket switch'
|
|
275
|
+
);
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
161
278
|
this.logger.error(
|
|
162
|
-
{ err, table, id, Category: '
|
|
279
|
+
{ err, table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
163
280
|
'Failed to delete record'
|
|
164
281
|
);
|
|
165
282
|
throw err;
|
|
@@ -170,12 +287,15 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
170
287
|
* Register a query with DBSP to create a materialized view
|
|
171
288
|
* Returns the initial result array
|
|
172
289
|
*/
|
|
173
|
-
registerQuery(config: QueryConfig): {
|
|
290
|
+
registerQuery(config: QueryConfig): {
|
|
291
|
+
localArray: RecordVersionArray;
|
|
292
|
+
registrationTimings?: { parseMs: number; planMs: number; snapshotMs: number };
|
|
293
|
+
} {
|
|
174
294
|
this.logger.debug(
|
|
175
295
|
{
|
|
176
296
|
queryHash: config.queryHash,
|
|
177
297
|
surql: config.surql,
|
|
178
|
-
Category: '
|
|
298
|
+
Category: 'sp00ky-client::CacheModule::registerQuery',
|
|
179
299
|
},
|
|
180
300
|
'Registering query'
|
|
181
301
|
);
|
|
@@ -202,15 +322,15 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
202
322
|
{
|
|
203
323
|
queryHash: config.queryHash,
|
|
204
324
|
arrayLength: update.localArray?.length,
|
|
205
|
-
Category: '
|
|
325
|
+
Category: 'sp00ky-client::CacheModule::registerQuery',
|
|
206
326
|
},
|
|
207
327
|
'Query registered successfully'
|
|
208
328
|
);
|
|
209
329
|
|
|
210
|
-
return { localArray: update.localArray };
|
|
330
|
+
return { localArray: update.localArray, registrationTimings: update.registration };
|
|
211
331
|
} catch (err) {
|
|
212
332
|
this.logger.error(
|
|
213
|
-
{ err, queryHash: config.queryHash, Category: '
|
|
333
|
+
{ err, queryHash: config.queryHash, Category: 'sp00ky-client::CacheModule::registerQuery' },
|
|
214
334
|
'Failed to register query'
|
|
215
335
|
);
|
|
216
336
|
throw err;
|
|
@@ -222,18 +342,18 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
222
342
|
*/
|
|
223
343
|
unregisterQuery(queryHash: string): void {
|
|
224
344
|
this.logger.debug(
|
|
225
|
-
{ queryHash, Category: '
|
|
345
|
+
{ queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
226
346
|
'Unregistering query'
|
|
227
347
|
);
|
|
228
348
|
try {
|
|
229
349
|
this.streamProcessor.unregisterQueryPlan(queryHash);
|
|
230
350
|
this.logger.debug(
|
|
231
|
-
{ queryHash, Category: '
|
|
351
|
+
{ queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
232
352
|
'Query unregistered successfully'
|
|
233
353
|
);
|
|
234
354
|
} catch (err) {
|
|
235
355
|
this.logger.error(
|
|
236
|
-
{ err, queryHash, Category: '
|
|
356
|
+
{ err, queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
|
|
237
357
|
'Failed to unregister query'
|
|
238
358
|
);
|
|
239
359
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RecordId, Duration } from 'surrealdb';
|
|
2
|
-
import { QueryTimeToLive
|
|
1
|
+
import type { RecordId, Duration } from 'surrealdb';
|
|
2
|
+
import type { QueryTimeToLive } from '../../types';
|
|
3
3
|
|
|
4
4
|
export type RecordWithId = Record<string, any> & { id: RecordId<string> };
|
|
5
5
|
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import type { LoroDoc } from 'loro-crdt';
|
|
2
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
3
|
+
import type { Logger } from '../../services/logger/index';
|
|
4
|
+
import { parseRecordIdString } from '../../utils/index';
|
|
5
|
+
|
|
6
|
+
// ==================== CURSOR UTILITIES ====================
|
|
7
|
+
|
|
8
|
+
export const CURSOR_COLORS = [
|
|
9
|
+
'#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
|
|
10
|
+
'#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
|
|
11
|
+
];
|
|
12
|
+
|
|
13
|
+
export function cursorColorFromName(name: string): string {
|
|
14
|
+
let hash = 0;
|
|
15
|
+
for (let i = 0; i < name.length; i++) {
|
|
16
|
+
hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
|
|
17
|
+
}
|
|
18
|
+
return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// ==================== CRDT FIELD ====================
|
|
22
|
+
|
|
23
|
+
export class CrdtField {
|
|
24
|
+
private doc: LoroDoc;
|
|
25
|
+
private pushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
26
|
+
private local: LocalStore | null = null;
|
|
27
|
+
private remote: RemoteDatabaseService | null = null;
|
|
28
|
+
private recordId: string | null = null;
|
|
29
|
+
private sessionId: string = '';
|
|
30
|
+
private unsubscribe: (() => void) | null = null;
|
|
31
|
+
private lastPushTime = 0;
|
|
32
|
+
private lastCursorPushTime = 0;
|
|
33
|
+
private loadedFromCrdt = false;
|
|
34
|
+
private pushRetryCount = 0;
|
|
35
|
+
private logger: Logger | null;
|
|
36
|
+
private cursorsEnabled: boolean;
|
|
37
|
+
/** Remote-push debounce. Local writes happen immediately on every Loro
|
|
38
|
+
* update; the remote UPSERT is coalesced over this window. Configured
|
|
39
|
+
* via `Sp00kyConfig.crdtDebounceMs`, default 500. */
|
|
40
|
+
private remoteDebounceMs: number = 500;
|
|
41
|
+
|
|
42
|
+
private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
|
|
43
|
+
private pendingCursorUpdate: Uint8Array | null = null;
|
|
44
|
+
|
|
45
|
+
/** Callback set by the editor to receive remote cursor updates.
|
|
46
|
+
* Any cursor data that arrived before this callback was set will be replayed. */
|
|
47
|
+
set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
|
|
48
|
+
this._onCursorUpdate = cb;
|
|
49
|
+
if (cb && this.pendingCursorUpdate) {
|
|
50
|
+
try { cb(this.pendingCursorUpdate); } catch (e) {
|
|
51
|
+
this.logger?.warn(
|
|
52
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
|
|
53
|
+
'Failed to replay pending cursor update'
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
this.pendingCursorUpdate = null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
get onCursorUpdate() { return this._onCursorUpdate; }
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* @param LoroDocClass the `LoroDoc` constructor, injected by the caller after
|
|
64
|
+
* awaiting {@link loadLoro} — keeps `loro-crdt` out of this module's static
|
|
65
|
+
* import graph so it only ships to apps that use CRDT fields.
|
|
66
|
+
*/
|
|
67
|
+
constructor(
|
|
68
|
+
private fieldName: string,
|
|
69
|
+
cursorsEnabled: boolean,
|
|
70
|
+
LoroDocClass: typeof LoroDoc,
|
|
71
|
+
initialState?: Uint8Array,
|
|
72
|
+
logger?: Logger | null,
|
|
73
|
+
) {
|
|
74
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
`CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
this.logger = logger ?? null;
|
|
80
|
+
this.cursorsEnabled = cursorsEnabled;
|
|
81
|
+
this.doc = new LoroDocClass();
|
|
82
|
+
if (initialState && initialState.length > 0) {
|
|
83
|
+
// Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
|
|
84
|
+
// value left over from a pre-`bytes` migration) so the editor still
|
|
85
|
+
// mounts. Without this guard the rejection bubbles through
|
|
86
|
+
// `useCrdtField` → permanent fallback `<p>`, with no cursor.
|
|
87
|
+
try {
|
|
88
|
+
this.doc.import(initialState);
|
|
89
|
+
this.loadedFromCrdt = true;
|
|
90
|
+
} catch (e) {
|
|
91
|
+
this.logger?.warn(
|
|
92
|
+
{
|
|
93
|
+
error: e,
|
|
94
|
+
fieldName,
|
|
95
|
+
Category: 'sp00ky-client::CrdtField::constructor',
|
|
96
|
+
},
|
|
97
|
+
'Initial CRDT state is not a valid LoroDoc snapshot — starting empty and will seed from fallback text'
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
getDoc(): LoroDoc { return this.doc; }
|
|
104
|
+
|
|
105
|
+
/** Whether the LoroDoc was loaded from saved CRDT state */
|
|
106
|
+
hasContent(): boolean {
|
|
107
|
+
return this.loadedFromCrdt;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
startSync(
|
|
111
|
+
local: LocalStore,
|
|
112
|
+
remote: RemoteDatabaseService,
|
|
113
|
+
recordId: string,
|
|
114
|
+
sessionId: string,
|
|
115
|
+
debounceMs: number,
|
|
116
|
+
): void {
|
|
117
|
+
this.local = local;
|
|
118
|
+
this.remote = remote;
|
|
119
|
+
this.recordId = recordId;
|
|
120
|
+
this.sessionId = sessionId;
|
|
121
|
+
this.remoteDebounceMs = debounceMs;
|
|
122
|
+
// Every local Loro update writes the snapshot to the local cache
|
|
123
|
+
// *immediately* (so reload/offline see the latest text), then
|
|
124
|
+
// schedules a debounced push to remote. The local UPSERT is cheap —
|
|
125
|
+
// it's an in-memory SurrealKV write — but errors are swallowed so a
|
|
126
|
+
// bad write never blocks user input.
|
|
127
|
+
this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
|
|
128
|
+
void this.persistLocal();
|
|
129
|
+
this.scheduleRemotePush();
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Stop syncing this field. Flushes one final remote push by default so the
|
|
135
|
+
* last keystrokes aren't lost. Pass `{ flush: false }` on a bucket switch —
|
|
136
|
+
* the remote session already belongs to the NEXT user, and pushing this
|
|
137
|
+
* (previous user's) snapshot under it would clobber the record remotely.
|
|
138
|
+
*/
|
|
139
|
+
stopSync(options: { flush?: boolean } = {}): void {
|
|
140
|
+
const flush = options.flush ?? true;
|
|
141
|
+
if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
|
|
142
|
+
if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
|
|
143
|
+
if (flush && this.remote && this.recordId) { void this.pushToRemote(); }
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
importRemote(state: Uint8Array): void {
|
|
147
|
+
// Echo suppression: skip imports within `remoteDebounceMs + 200` of
|
|
148
|
+
// our own push. The +200 guards against round-trip jitter where our
|
|
149
|
+
// own write echoes back from the LIVE feed before the debounce
|
|
150
|
+
// window closes.
|
|
151
|
+
if (Date.now() - this.lastPushTime < this.remoteDebounceMs + 200) return;
|
|
152
|
+
try {
|
|
153
|
+
this.doc.import(state);
|
|
154
|
+
// Persist the merged snapshot locally so the next reload/offline
|
|
155
|
+
// open sees the freshest converged state without waiting for the
|
|
156
|
+
// LIVE feed.
|
|
157
|
+
void this.persistLocal();
|
|
158
|
+
} catch (e) {
|
|
159
|
+
this.logger?.warn(
|
|
160
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
|
|
161
|
+
'Failed to import remote CRDT state'
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
exportSnapshot(): Uint8Array {
|
|
167
|
+
return this.doc.export({ mode: 'snapshot' });
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Push this session's cursor blob into the parent row at
|
|
171
|
+
* `<field>.cursors[$sid]`. No-op when cursors aren't enabled on this
|
|
172
|
+
* field — the editor still calls this method optimistically, but
|
|
173
|
+
* without `@cursor` on the schema there's nowhere to store the blob.
|
|
174
|
+
* The UPDATE itself fires the parent table's LIVE feed, so other
|
|
175
|
+
* browsers receive the cursor change without a separate `_00_rv` bump. */
|
|
176
|
+
async pushCursorState(encoded: Uint8Array): Promise<void> {
|
|
177
|
+
if (!this.remote || !this.recordId) return;
|
|
178
|
+
if (!this.cursorsEnabled) return;
|
|
179
|
+
this.lastCursorPushTime = Date.now();
|
|
180
|
+
try {
|
|
181
|
+
const state = encodeBase64(encoded);
|
|
182
|
+
await this.remote.query(
|
|
183
|
+
`UPDATE $id SET ${this.fieldName}.cursors[$sid] = $state RETURN NONE;`,
|
|
184
|
+
{
|
|
185
|
+
id: parseRecordIdString(this.recordId),
|
|
186
|
+
sid: this.sessionId,
|
|
187
|
+
state,
|
|
188
|
+
}
|
|
189
|
+
);
|
|
190
|
+
} catch (e) {
|
|
191
|
+
this.logger?.warn(
|
|
192
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
|
|
193
|
+
'Failed to push cursor state'
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
|
|
199
|
+
importRemoteCursor(base64State: string): void {
|
|
200
|
+
if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
|
|
201
|
+
try {
|
|
202
|
+
const data = decodeBase64(base64State);
|
|
203
|
+
if (this._onCursorUpdate) {
|
|
204
|
+
this._onCursorUpdate(data);
|
|
205
|
+
} else {
|
|
206
|
+
// Only keep the latest cursor state — older positions are useless
|
|
207
|
+
this.pendingCursorUpdate = data;
|
|
208
|
+
}
|
|
209
|
+
} catch (e) {
|
|
210
|
+
this.logger?.warn(
|
|
211
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
|
|
212
|
+
'Failed to apply remote cursor data'
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
private scheduleRemotePush(): void {
|
|
218
|
+
if (this.pushTimer) clearTimeout(this.pushTimer);
|
|
219
|
+
this.pushTimer = setTimeout(() => void this.pushToRemote(), this.remoteDebounceMs);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** SET path inside a parent row for the current snapshot. `@crdt`-only
|
|
223
|
+
* fields hold the snapshot directly (`<field>`); `@crdt @cursor`
|
|
224
|
+
* fields hold a `{ state, cursors }` object so the snapshot lives at
|
|
225
|
+
* `<field>.state` next to per-session cursor blobs. */
|
|
226
|
+
private statePath(): string {
|
|
227
|
+
return this.cursorsEnabled ? `${this.fieldName}.state` : this.fieldName;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Mirror the LoroDoc snapshot into the parent row locally. Runs on
|
|
231
|
+
* every local update and every remote import so reloads (online or
|
|
232
|
+
* offline) see the freshest content immediately. Failures are
|
|
233
|
+
* swallowed — a stale local write must never block user input. */
|
|
234
|
+
private async persistLocal(): Promise<void> {
|
|
235
|
+
if (!this.local || !this.recordId) return;
|
|
236
|
+
try {
|
|
237
|
+
await this.local.query(
|
|
238
|
+
`UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
|
|
239
|
+
{ id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
|
|
240
|
+
);
|
|
241
|
+
} catch (e) {
|
|
242
|
+
this.logger?.debug(
|
|
243
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::persistLocal' },
|
|
244
|
+
'Local CRDT persist failed (best-effort)'
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private async pushToRemote(): Promise<void> {
|
|
250
|
+
if (!this.remote || !this.recordId) return;
|
|
251
|
+
this.lastPushTime = Date.now();
|
|
252
|
+
try {
|
|
253
|
+
// The UPDATE on the parent fires the parent table's LIVE feed,
|
|
254
|
+
// so cross-browser receivers see this change directly in the LIVE
|
|
255
|
+
// payload — no separate `_00_rv` bump or sidecar UPSERT.
|
|
256
|
+
await this.remote.query(
|
|
257
|
+
`UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
|
|
258
|
+
{ id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
|
|
259
|
+
);
|
|
260
|
+
this.pushRetryCount = 0;
|
|
261
|
+
} catch (e) {
|
|
262
|
+
this.logger?.warn(
|
|
263
|
+
{ error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
|
|
264
|
+
'Failed to push CRDT state to remote'
|
|
265
|
+
);
|
|
266
|
+
// Bounded retry. Offline first-loads will exhaust this and stop
|
|
267
|
+
// hammering; the next user keystroke (or the next time a remote
|
|
268
|
+
// event lands once we're back online) will kick off another push.
|
|
269
|
+
if (this.pushRetryCount < 2) {
|
|
270
|
+
this.pushRetryCount++;
|
|
271
|
+
this.scheduleRemotePush();
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export function decodeBase64(b64: string): Uint8Array {
|
|
278
|
+
if (typeof atob === 'function') {
|
|
279
|
+
const binary = atob(b64);
|
|
280
|
+
const bytes = new Uint8Array(binary.length);
|
|
281
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
282
|
+
return bytes;
|
|
283
|
+
}
|
|
284
|
+
return new Uint8Array(Buffer.from(b64, 'base64'));
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export function encodeBase64(bytes: Uint8Array): string {
|
|
288
|
+
if (typeof btoa === 'function') {
|
|
289
|
+
let binary = '';
|
|
290
|
+
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
|
|
291
|
+
return btoa(binary);
|
|
292
|
+
}
|
|
293
|
+
return Buffer.from(bytes).toString('base64');
|
|
294
|
+
}
|