@spooky-sync/core 0.0.1-canary.206 → 0.0.1-canary.208
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +54 -2
- package/dist/index.js +144 -32
- package/package.json +3 -3
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +26 -4
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/index.ts +35 -15
- package/src/modules/sync/sync.live-removal.test.ts +40 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +71 -6
- package/src/services/tabs/coordinator.test.ts +118 -2
- package/src/services/tabs/coordinator.ts +25 -0
- package/src/services/tabs/protocol.ts +17 -1
- package/src/sp00ky.ts +12 -14
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +30 -1
package/dist/index.d.ts
CHANGED
|
@@ -537,15 +537,29 @@ declare class CacheModule implements StreamUpdateReceiver {
|
|
|
537
537
|
private versionLookups;
|
|
538
538
|
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
539
539
|
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
540
|
-
* follower only needs the circuit feed).
|
|
540
|
+
* follower only needs the circuit feed). A follower relays its own
|
|
541
|
+
* mutations to the leader the same way, see {@link setIngestRelay}. */
|
|
541
542
|
private ingestRelay;
|
|
543
|
+
/** See {@link setIngestRelay}. */
|
|
544
|
+
private relayLocalWritesOnly;
|
|
542
545
|
constructor(local: LocalStore, streamProcessor: StreamProcessorService, streamUpdateCallback: (update: StreamUpdate) => void, logger: Logger$1);
|
|
543
546
|
/**
|
|
544
547
|
* Implements StreamUpdateReceiver interface
|
|
545
548
|
* Called directly by StreamProcessor when views change
|
|
546
549
|
*/
|
|
547
550
|
onStreamUpdate(update: StreamUpdate): void;
|
|
548
|
-
|
|
551
|
+
/**
|
|
552
|
+
* Fan every committed ingest out to the other tabs. The leader relays
|
|
553
|
+
* everything (its sync fetches are the only copy the followers get). A
|
|
554
|
+
* follower relays with `localWritesOnly`: just the mutation path, which is
|
|
555
|
+
* the only thing it knows that the leader does not. Its sync-fetched
|
|
556
|
+
* batches are the leader's data coming back and must not be re-broadcast,
|
|
557
|
+
* or every follower registration would fan its whole working set to
|
|
558
|
+
* every tab.
|
|
559
|
+
*/
|
|
560
|
+
setIngestRelay(cb: ((tuples: CacheIngestTuple[]) => void) | null, opts?: {
|
|
561
|
+
localWritesOnly?: boolean;
|
|
562
|
+
}): void;
|
|
549
563
|
/**
|
|
550
564
|
* Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
|
|
551
565
|
* The rows are already in the shared store (the leader wrote them), so no
|
|
@@ -1035,6 +1049,15 @@ declare class DataModule<S extends SchemaStructure> {
|
|
|
1035
1049
|
* Rollback a failed optimistic update by restoring the previous record state
|
|
1036
1050
|
*/
|
|
1037
1051
|
rollbackUpdate(recordId: RecordId, tableName: string, beforeRecord: Record<string, unknown>): Promise<void>;
|
|
1052
|
+
/**
|
|
1053
|
+
* Force a re-materialize + notify of every active query on `tableName`.
|
|
1054
|
+
* Used after a DELETE landed in the local store (this tab's own, or one
|
|
1055
|
+
* relayed from another tab): the SSP may not emit a view update for a
|
|
1056
|
+
* DELETE ingest, and the re-materialize reads the store, which already
|
|
1057
|
+
* excludes the row. Each query is isolated so one failing re-materialize
|
|
1058
|
+
* can't stop the others.
|
|
1059
|
+
*/
|
|
1060
|
+
notifyTableQueries(tableName: string): Promise<void>;
|
|
1038
1061
|
/**
|
|
1039
1062
|
* Remove a record from all active query states and notify subscribers
|
|
1040
1063
|
*/
|
|
@@ -1085,6 +1108,14 @@ type FollowerToLeaderMessage = {
|
|
|
1085
1108
|
mutationId: string;
|
|
1086
1109
|
} | {
|
|
1087
1110
|
type: 'request-poll';
|
|
1111
|
+
}
|
|
1112
|
+
/** An optimistic write this follower committed to the SHARED store and
|
|
1113
|
+
* ingested into its own circuit. The leader ingests it (no DB write, the
|
|
1114
|
+
* row is already there) and fans it out to every OTHER follower as
|
|
1115
|
+
* `ingest-relay`, so a follower's write lands in every tab in one hop
|
|
1116
|
+
* instead of after the server round-trip. */ | {
|
|
1117
|
+
type: 'ingest';
|
|
1118
|
+
tuples: IngestTuple[];
|
|
1088
1119
|
};
|
|
1089
1120
|
type LeaderToFollowerMessage = {
|
|
1090
1121
|
type: 'db-ready';
|
|
@@ -1114,6 +1145,15 @@ type LeaderToFollowerMessage = {
|
|
|
1114
1145
|
recordId: string;
|
|
1115
1146
|
eventType: 'create' | 'update' | 'delete';
|
|
1116
1147
|
error: string;
|
|
1148
|
+
}
|
|
1149
|
+
/** The leader's drain pushed a mutation and deleted its outbox row from the
|
|
1150
|
+
* SHARED store. Every follower starts its settled-write grace so a row it
|
|
1151
|
+
* was rendering as a pending write does not blink out before its
|
|
1152
|
+
* `_00_list_ref` membership arrives. */ | {
|
|
1153
|
+
type: 'mutation-settled';
|
|
1154
|
+
mutationId: string;
|
|
1155
|
+
recordId: string;
|
|
1156
|
+
eventType: 'create' | 'update' | 'delete';
|
|
1117
1157
|
};
|
|
1118
1158
|
//#endregion
|
|
1119
1159
|
//#region src/services/tabs/coordinator.d.ts
|
|
@@ -1150,6 +1190,11 @@ declare class SyncForwarder {
|
|
|
1150
1190
|
unbind(): void;
|
|
1151
1191
|
private post;
|
|
1152
1192
|
mutationEnqueued(mutationId: string): void;
|
|
1193
|
+
/** An optimistic write this tab just ingested. Deliberately NOT queued while
|
|
1194
|
+
* detached: a new leader primes its circuit from the shared store, which
|
|
1195
|
+
* already holds the row, and replaying a stale tuple at it later would put
|
|
1196
|
+
* an older `_00_rv` in its version memo. */
|
|
1197
|
+
ingest(tuples: IngestTuple[]): void;
|
|
1153
1198
|
requestPoll(): void;
|
|
1154
1199
|
}
|
|
1155
1200
|
//#endregion
|
|
@@ -1381,6 +1426,13 @@ declare class Sp00kySync<S extends SchemaStructure> {
|
|
|
1381
1426
|
private onMutationDropped;
|
|
1382
1427
|
/** A forwarded outbox row from a follower: load + drain it. Idempotent. */
|
|
1383
1428
|
enqueueForwardedMutation(mutationId: string): Promise<void>;
|
|
1429
|
+
/**
|
|
1430
|
+
* Tuples another tab already committed to the shared store: feed them to
|
|
1431
|
+
* THIS tab's circuit (no local write). A DELETE additionally forces a
|
|
1432
|
+
* re-materialize of the table's queries, exactly as the writing tab does
|
|
1433
|
+
* for itself, because the SSP may not emit a view update for it.
|
|
1434
|
+
*/
|
|
1435
|
+
private applyRelayedIngest;
|
|
1384
1436
|
/** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
|
|
1385
1437
|
* and run the exact same handling the LIVE subscription would have. */
|
|
1386
1438
|
private applyRelayedListRefChange;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as renderOrderSql, c as serializeRow, i as projectedDataSql, l as resolveRelations, o as renderWhereSql, r as project, s as reviveRow, t as PROMOTION_OPEN_OPTIONS, u as stableKey } from "./sqlite-open.js";
|
|
2
2
|
import { DateTime, Duration, RecordId, Surreal, Uuid, applyDiagnostics, createRemoteEngines } from "surrealdb";
|
|
3
|
-
import { QueryBuilder, RecordId as RecordId$1, cyrb53 } from "@spooky-sync/query-builder";
|
|
3
|
+
import { QueryBuilder, RecordId as RecordId$1, baseFieldOfParam, cyrb53 } from "@spooky-sync/query-builder";
|
|
4
4
|
import pino from "pino";
|
|
5
5
|
import { applyPatch } from "fast-json-patch";
|
|
6
6
|
import init, { Sp00kyProcessor } from "@spooky-sync/ssp-wasm";
|
|
@@ -84,6 +84,10 @@ function cleanRecord(tableSchema, record) {
|
|
|
84
84
|
for (const [key, value] of Object.entries(record)) if (key === "id" || key.startsWith("_00_") || key in tableSchema) cleaned[key] = value;
|
|
85
85
|
return cleaned;
|
|
86
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Parse a RECORD's fields against the table schema. Anything the schema does not
|
|
89
|
+
* know is dropped, which is what keeps a stray field out of a write.
|
|
90
|
+
*/
|
|
87
91
|
function parseParams(tableSchema, params) {
|
|
88
92
|
const parsedParams = {};
|
|
89
93
|
for (const [key, value] of Object.entries(params)) {
|
|
@@ -92,6 +96,26 @@ function parseParams(tableSchema, params) {
|
|
|
92
96
|
}
|
|
93
97
|
return parsedParams;
|
|
94
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Parse a QUERY's params. Unlike a record's fields, a param name is not always a
|
|
101
|
+
* column: an `_or` branch binds under a synthetic `white__or0` (see
|
|
102
|
+
* `orParamName` in @spooky-sync/query-builder), and the surql that references it
|
|
103
|
+
* is already written. So resolve the column through the synthetic name, and keep
|
|
104
|
+
* a param we cannot type rather than dropping it.
|
|
105
|
+
*
|
|
106
|
+
* Dropping was the bug: `parseParams` kept only column-named params, so every
|
|
107
|
+
* `_or` query registered with `$or0`/`$or1` unbound and matched no rows at all,
|
|
108
|
+
* silently (rowCount 0, errorCount 0).
|
|
109
|
+
*/
|
|
110
|
+
function parseQueryParams(tableSchema, params) {
|
|
111
|
+
const parsedParams = {};
|
|
112
|
+
for (const [key, value] of Object.entries(params)) {
|
|
113
|
+
if (value === void 0) continue;
|
|
114
|
+
const column = tableSchema[key] ?? tableSchema[baseFieldOfParam(key)];
|
|
115
|
+
parsedParams[key] = column ? parseValue(key, column, value) : value;
|
|
116
|
+
}
|
|
117
|
+
return parsedParams;
|
|
118
|
+
}
|
|
95
119
|
function parseValue(name, column, value) {
|
|
96
120
|
if (column.recordId) {
|
|
97
121
|
if (value instanceof RecordId$1) return value;
|
|
@@ -4531,15 +4555,7 @@ var DataModule = class DataModule {
|
|
|
4531
4555
|
Category: "sp00ky-client::DataModule::delete"
|
|
4532
4556
|
}, "SSP delete-ingest failed; relying on query re-materialize to reflect the delete");
|
|
4533
4557
|
}
|
|
4534
|
-
|
|
4535
|
-
await this.notifyQuerySynced(queryHash);
|
|
4536
|
-
} catch (err) {
|
|
4537
|
-
this.logger.error({
|
|
4538
|
-
err,
|
|
4539
|
-
queryHash,
|
|
4540
|
-
Category: "sp00ky-client::DataModule::delete"
|
|
4541
|
-
}, "notifyQuerySynced failed after delete");
|
|
4542
|
-
}
|
|
4558
|
+
await this.notifyTableQueries(tableName);
|
|
4543
4559
|
const mutationEvent = {
|
|
4544
4560
|
type: "delete",
|
|
4545
4561
|
mutation_id: mutationId,
|
|
@@ -4610,6 +4626,26 @@ var DataModule = class DataModule {
|
|
|
4610
4626
|
}
|
|
4611
4627
|
}
|
|
4612
4628
|
/**
|
|
4629
|
+
* Force a re-materialize + notify of every active query on `tableName`.
|
|
4630
|
+
* Used after a DELETE landed in the local store (this tab's own, or one
|
|
4631
|
+
* relayed from another tab): the SSP may not emit a view update for a
|
|
4632
|
+
* DELETE ingest, and the re-materialize reads the store, which already
|
|
4633
|
+
* excludes the row. Each query is isolated so one failing re-materialize
|
|
4634
|
+
* can't stop the others.
|
|
4635
|
+
*/
|
|
4636
|
+
async notifyTableQueries(tableName) {
|
|
4637
|
+
for (const [queryHash, queryState] of this.activeQueries) if (queryState.config.tableName === tableName) try {
|
|
4638
|
+
await this.notifyQuerySynced(queryHash);
|
|
4639
|
+
} catch (err) {
|
|
4640
|
+
this.logger.error({
|
|
4641
|
+
err,
|
|
4642
|
+
queryHash,
|
|
4643
|
+
tableName,
|
|
4644
|
+
Category: "sp00ky-client::DataModule::notifyTableQueries"
|
|
4645
|
+
}, "notifyQuerySynced failed after delete");
|
|
4646
|
+
}
|
|
4647
|
+
}
|
|
4648
|
+
/**
|
|
4613
4649
|
* Remove a record from all active query states and notify subscribers
|
|
4614
4650
|
*/
|
|
4615
4651
|
removeRecordFromQueries(recordId) {
|
|
@@ -4723,7 +4759,7 @@ var DataModule = class DataModule {
|
|
|
4723
4759
|
...configRecord,
|
|
4724
4760
|
id: recordId,
|
|
4725
4761
|
plan,
|
|
4726
|
-
params:
|
|
4762
|
+
params: parseQueryParams(tableSchema.columns, params ?? configRecord.params),
|
|
4727
4763
|
membershipKey
|
|
4728
4764
|
};
|
|
4729
4765
|
if (membershipKey && !config.remoteArray?.length) {
|
|
@@ -6134,8 +6170,14 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6134
6170
|
switch (msg.type) {
|
|
6135
6171
|
case "sync-hello": break;
|
|
6136
6172
|
case "mutation-enqueued":
|
|
6173
|
+
this.listRefIdleStreak = 0;
|
|
6137
6174
|
this.enqueueForwardedMutation(msg.mutationId);
|
|
6138
6175
|
break;
|
|
6176
|
+
case "ingest":
|
|
6177
|
+
this.applyRelayedIngest(msg.tuples);
|
|
6178
|
+
hub.relayIngest(msg.tuples, tabId);
|
|
6179
|
+
this.listRefIdleStreak = 0;
|
|
6180
|
+
break;
|
|
6139
6181
|
case "request-poll":
|
|
6140
6182
|
this.listRefIdleStreak = 0;
|
|
6141
6183
|
break;
|
|
@@ -6180,6 +6222,12 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6180
6222
|
this.killRefLiveQuery();
|
|
6181
6223
|
forwarder.onLeaderMessage = (msg) => {
|
|
6182
6224
|
switch (msg.type) {
|
|
6225
|
+
case "ingest-relay":
|
|
6226
|
+
this.applyRelayedIngest(msg.tuples);
|
|
6227
|
+
break;
|
|
6228
|
+
case "mutation-settled":
|
|
6229
|
+
this.dataModule.noteWriteSettled(msg.recordId, msg.eventType);
|
|
6230
|
+
break;
|
|
6183
6231
|
case "list-ref-change":
|
|
6184
6232
|
this.applyRelayedListRefChange(msg).catch((err) => {
|
|
6185
6233
|
this.logger.error({
|
|
@@ -6225,6 +6273,24 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6225
6273
|
if (this.tabRole !== "leader") return;
|
|
6226
6274
|
await this.upQueue.enqueueFromDatabase(mutationId);
|
|
6227
6275
|
}
|
|
6276
|
+
/**
|
|
6277
|
+
* Tuples another tab already committed to the shared store: feed them to
|
|
6278
|
+
* THIS tab's circuit (no local write). A DELETE additionally forces a
|
|
6279
|
+
* re-materialize of the table's queries, exactly as the writing tab does
|
|
6280
|
+
* for itself, because the SSP may not emit a view update for it.
|
|
6281
|
+
*/
|
|
6282
|
+
applyRelayedIngest(tuples) {
|
|
6283
|
+
this.cache.applyRelayedIngest(tuples);
|
|
6284
|
+
const deletedTables = /* @__PURE__ */ new Set();
|
|
6285
|
+
for (const t of tuples) if (t.op === "DELETE") deletedTables.add(t.table);
|
|
6286
|
+
for (const table of deletedTables) this.dataModule.notifyTableQueries(table).catch((err) => {
|
|
6287
|
+
this.logger.warn({
|
|
6288
|
+
err,
|
|
6289
|
+
table,
|
|
6290
|
+
Category: "sp00ky-client::Sp00kySync::applyRelayedIngest"
|
|
6291
|
+
}, "Re-materialize after relayed delete failed");
|
|
6292
|
+
});
|
|
6293
|
+
}
|
|
6228
6294
|
/** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
|
|
6229
6295
|
* and run the exact same handling the LIVE subscription would have. */
|
|
6230
6296
|
async applyRelayedListRefChange(msg) {
|
|
@@ -6617,8 +6683,20 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6617
6683
|
}, "Live update is being processed");
|
|
6618
6684
|
const diff = createDiffFromDbOp(action, recordId, version, localArray);
|
|
6619
6685
|
const hash = extractIdPart(existing.config.id);
|
|
6620
|
-
if (existing.config.membershipKnown
|
|
6621
|
-
const
|
|
6686
|
+
if (existing.config.membershipKnown) {
|
|
6687
|
+
const membershipDiff = action === "DELETE" ? {
|
|
6688
|
+
added: [],
|
|
6689
|
+
updated: [],
|
|
6690
|
+
removed: [recordId]
|
|
6691
|
+
} : {
|
|
6692
|
+
added: [{
|
|
6693
|
+
id: recordId,
|
|
6694
|
+
version
|
|
6695
|
+
}],
|
|
6696
|
+
updated: [],
|
|
6697
|
+
removed: []
|
|
6698
|
+
};
|
|
6699
|
+
const next = applyRecordVersionDiff(existing.config.remoteArray ?? [], membershipDiff);
|
|
6622
6700
|
if (!recordVersionArraysEqual(next, existing.config.remoteArray ?? [])) await this.dataModule.updateQueryRemoteArray(hash, next);
|
|
6623
6701
|
}
|
|
6624
6702
|
await this.runSyncForQuery(hash, diff);
|
|
@@ -6737,7 +6815,14 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6737
6815
|
* vanish, and return, while every other client showed it throughout.
|
|
6738
6816
|
*/
|
|
6739
6817
|
handleMutationSettled(event) {
|
|
6740
|
-
|
|
6818
|
+
const recordId = encodeRecordId(event.record_id);
|
|
6819
|
+
this.dataModule.noteWriteSettled(recordId, event.type);
|
|
6820
|
+
this.hub?.broadcast({
|
|
6821
|
+
type: "mutation-settled",
|
|
6822
|
+
mutationId: encodeRecordId(event.mutation_id),
|
|
6823
|
+
recordId,
|
|
6824
|
+
eventType: event.type
|
|
6825
|
+
});
|
|
6741
6826
|
}
|
|
6742
6827
|
async handleRollback(event, error) {
|
|
6743
6828
|
const recordId = encodeRecordId(event.record_id);
|
|
@@ -7353,8 +7438,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
7353
7438
|
|
|
7354
7439
|
//#endregion
|
|
7355
7440
|
//#region src/modules/devtools/index.ts
|
|
7356
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
7357
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
7441
|
+
const CORE_VERSION = "0.0.1-canary.208";
|
|
7442
|
+
const WASM_VERSION = "0.0.1-canary.208";
|
|
7358
7443
|
const SURREAL_VERSION = "3.0.3";
|
|
7359
7444
|
var DevToolsService = class DevToolsService {
|
|
7360
7445
|
eventsHistory = [];
|
|
@@ -8883,8 +8968,11 @@ var CacheModule = class {
|
|
|
8883
8968
|
versionLookups = {};
|
|
8884
8969
|
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
8885
8970
|
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
8886
|
-
* follower only needs the circuit feed).
|
|
8971
|
+
* follower only needs the circuit feed). A follower relays its own
|
|
8972
|
+
* mutations to the leader the same way, see {@link setIngestRelay}. */
|
|
8887
8973
|
ingestRelay = null;
|
|
8974
|
+
/** See {@link setIngestRelay}. */
|
|
8975
|
+
relayLocalWritesOnly = false;
|
|
8888
8976
|
constructor(local, streamProcessor, streamUpdateCallback, logger) {
|
|
8889
8977
|
this.local = local;
|
|
8890
8978
|
this.streamProcessor = streamProcessor;
|
|
@@ -8904,8 +8992,18 @@ var CacheModule = class {
|
|
|
8904
8992
|
}, "Stream update received");
|
|
8905
8993
|
this.streamUpdateCallback(update);
|
|
8906
8994
|
}
|
|
8907
|
-
|
|
8995
|
+
/**
|
|
8996
|
+
* Fan every committed ingest out to the other tabs. The leader relays
|
|
8997
|
+
* everything (its sync fetches are the only copy the followers get). A
|
|
8998
|
+
* follower relays with `localWritesOnly`: just the mutation path, which is
|
|
8999
|
+
* the only thing it knows that the leader does not. Its sync-fetched
|
|
9000
|
+
* batches are the leader's data coming back and must not be re-broadcast,
|
|
9001
|
+
* or every follower registration would fan its whole working set to
|
|
9002
|
+
* every tab.
|
|
9003
|
+
*/
|
|
9004
|
+
setIngestRelay(cb, opts = {}) {
|
|
8908
9005
|
this.ingestRelay = cb;
|
|
9006
|
+
this.relayLocalWritesOnly = opts.localWritesOnly === true;
|
|
8909
9007
|
}
|
|
8910
9008
|
/**
|
|
8911
9009
|
* Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
|
|
@@ -8996,7 +9094,7 @@ var CacheModule = class {
|
|
|
8996
9094
|
});
|
|
8997
9095
|
const ingested = this.streamProcessor.ingestMany(bulk);
|
|
8998
9096
|
for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
|
|
8999
|
-
if (ingested.length > 0) this.ingestRelay?.(ingested);
|
|
9097
|
+
if (ingested.length > 0 && (!this.relayLocalWritesOnly || skipDbInsert)) this.ingestRelay?.(ingested);
|
|
9000
9098
|
this.logger.debug({
|
|
9001
9099
|
count: records.length,
|
|
9002
9100
|
Category: "sp00ky-client::CacheModule::saveBatch"
|
|
@@ -9037,7 +9135,7 @@ var CacheModule = class {
|
|
|
9037
9135
|
id,
|
|
9038
9136
|
record: recordData
|
|
9039
9137
|
}]);
|
|
9040
|
-
this.ingestRelay?.([{
|
|
9138
|
+
if (!this.relayLocalWritesOnly || skipDbDelete) this.ingestRelay?.([{
|
|
9041
9139
|
table,
|
|
9042
9140
|
op: "DELETE",
|
|
9043
9141
|
id,
|
|
@@ -10617,6 +10715,19 @@ var SyncForwarder = class {
|
|
|
10617
10715
|
mutationId
|
|
10618
10716
|
});
|
|
10619
10717
|
}
|
|
10718
|
+
/** An optimistic write this tab just ingested. Deliberately NOT queued while
|
|
10719
|
+
* detached: a new leader primes its circuit from the shared store, which
|
|
10720
|
+
* already holds the row, and replaying a stale tuple at it later would put
|
|
10721
|
+
* an older `_00_rv` in its version memo. */
|
|
10722
|
+
ingest(tuples) {
|
|
10723
|
+
if (!this.port) return;
|
|
10724
|
+
try {
|
|
10725
|
+
this.port.postMessage({
|
|
10726
|
+
type: "ingest",
|
|
10727
|
+
tuples
|
|
10728
|
+
});
|
|
10729
|
+
} catch {}
|
|
10730
|
+
}
|
|
10620
10731
|
requestPoll() {
|
|
10621
10732
|
this.post({ type: "request-poll" });
|
|
10622
10733
|
}
|
|
@@ -10865,6 +10976,8 @@ var TabsCoordinator = class {
|
|
|
10865
10976
|
const forwarder = this.forwarder;
|
|
10866
10977
|
await new Promise((resolve) => {
|
|
10867
10978
|
let adopted = false;
|
|
10979
|
+
let attached = false;
|
|
10980
|
+
const pending = [];
|
|
10868
10981
|
const previousHandler = forwarder.onLeaderMessage;
|
|
10869
10982
|
forwarder.onLeaderMessage = (msg) => {
|
|
10870
10983
|
if (msg.type === "db-ready" && !adopted) {
|
|
@@ -10875,11 +10988,18 @@ var TabsCoordinator = class {
|
|
|
10875
10988
|
leadershipId: msg.leadershipId
|
|
10876
10989
|
}).then(() => {
|
|
10877
10990
|
this.deps.hooks.becomeSyncFollower(forwarder);
|
|
10991
|
+
attached = true;
|
|
10992
|
+
const backlog = pending.splice(0);
|
|
10993
|
+
for (const m of backlog) forwarder.onLeaderMessage?.(m);
|
|
10878
10994
|
this.setRole("follower");
|
|
10879
10995
|
resolve();
|
|
10880
10996
|
});
|
|
10881
10997
|
return;
|
|
10882
10998
|
}
|
|
10999
|
+
if (!attached) {
|
|
11000
|
+
pending.push(msg);
|
|
11001
|
+
return;
|
|
11002
|
+
}
|
|
10883
11003
|
previousHandler?.(msg);
|
|
10884
11004
|
};
|
|
10885
11005
|
forwarder.rebind(syncPort);
|
|
@@ -12303,7 +12423,7 @@ var Sp00kyClient = class {
|
|
|
12303
12423
|
return new TabsCoordinator({
|
|
12304
12424
|
tabId,
|
|
12305
12425
|
fingerprint: computeTabsFingerprint({
|
|
12306
|
-
coreVersion: "0.0.1-canary.
|
|
12426
|
+
coreVersion: "0.0.1-canary.208",
|
|
12307
12427
|
schemaHash: hash53(this.config.schemaSurql),
|
|
12308
12428
|
endpoint: this.config.database.endpoint ?? "",
|
|
12309
12429
|
namespace: this.config.database.namespace,
|
|
@@ -12330,18 +12450,10 @@ var Sp00kyClient = class {
|
|
|
12330
12450
|
resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
|
|
12331
12451
|
becomeSyncFollower: (forwarder) => {
|
|
12332
12452
|
this.streamProcessor.setPersistenceEnabled(false);
|
|
12333
|
-
this.cache.setIngestRelay(
|
|
12453
|
+
this.cache.setIngestRelay((tuples) => forwarder.ingest(tuples), { localWritesOnly: true });
|
|
12334
12454
|
this.sync.setTabContext("follower", tabId);
|
|
12335
12455
|
this.dataModule.setTabId(tabId);
|
|
12336
12456
|
this.sync.demoteToFollower(forwarder);
|
|
12337
|
-
const inner = forwarder.onLeaderMessage;
|
|
12338
|
-
forwarder.onLeaderMessage = (msg) => {
|
|
12339
|
-
if (msg.type === "ingest-relay") {
|
|
12340
|
-
this.cache.applyRelayedIngest(msg.tuples);
|
|
12341
|
-
return;
|
|
12342
|
-
}
|
|
12343
|
-
inner?.(msg);
|
|
12344
|
-
};
|
|
12345
12457
|
},
|
|
12346
12458
|
becomeSyncSolo: () => {
|
|
12347
12459
|
this.streamProcessor.setPersistenceEnabled(true);
|
|
@@ -12748,7 +12860,7 @@ var Sp00kyClient = class {
|
|
|
12748
12860
|
async initQuery(table, q, ttl) {
|
|
12749
12861
|
const tableSchema = this.config.schema.tables.find((t) => t.name === table);
|
|
12750
12862
|
if (!tableSchema) throw new Error(`Table ${table} not found`);
|
|
12751
|
-
const params =
|
|
12863
|
+
const params = parseQueryParams(tableSchema.columns, q.selectQuery.vars ?? {});
|
|
12752
12864
|
const hash = await this.dataModule.query(table, q.selectQuery.query, params, ttl, q.selectQuery.plan);
|
|
12753
12865
|
if (!this.pendingQueryInits.has(hash)) {
|
|
12754
12866
|
const chain = this.finishQueryInit(hash, q, params).finally(() => {
|
|
@@ -12820,7 +12932,7 @@ var Sp00kyClient = class {
|
|
|
12820
12932
|
const tableName = q.tableName;
|
|
12821
12933
|
const tableSchema = this.config.schema.tables.find((t) => t.name === tableName);
|
|
12822
12934
|
if (!tableSchema) throw new Error(`Table ${tableName} not found`);
|
|
12823
|
-
const params =
|
|
12935
|
+
const params = parseQueryParams(tableSchema.columns, q.selectQuery.vars ?? {});
|
|
12824
12936
|
const hashKey = String(q.hash);
|
|
12825
12937
|
const marker = await this.dataModule.getPreloadMarker(hashKey);
|
|
12826
12938
|
if (!marker) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.208",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.208",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.208",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"blurhash": "^2.0.5",
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { CacheModule } from './index';
|
|
4
|
+
|
|
5
|
+
// The ingest relay is how one tab's circuit learns what another tab wrote.
|
|
6
|
+
// The leader relays every ingest (its sync fetches are the only copy the
|
|
7
|
+
// followers get); a follower relays with `localWritesOnly`, just its mutation
|
|
8
|
+
// path, because its sync-fetched batches are the leader's data coming back.
|
|
9
|
+
|
|
10
|
+
function makeLogger(): any {
|
|
11
|
+
const logger: any = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {}, trace: () => {} };
|
|
12
|
+
logger.child = () => logger;
|
|
13
|
+
return logger;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function setup() {
|
|
17
|
+
const local: any = { epoch: 1, execute: vi.fn(async () => {}), query: vi.fn(async () => []) };
|
|
18
|
+
const ssp: any = {
|
|
19
|
+
addReceiver: vi.fn(),
|
|
20
|
+
ingestMany: vi.fn((records: unknown[]) => records),
|
|
21
|
+
};
|
|
22
|
+
const cache = new CacheModule(local, ssp, () => {}, makeLogger());
|
|
23
|
+
const relay = vi.fn();
|
|
24
|
+
return { cache, local, ssp, relay };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const rec = (id: string, version: number) => ({
|
|
28
|
+
table: 'thread',
|
|
29
|
+
op: 'CREATE' as const,
|
|
30
|
+
record: { id: new RecordId('thread', id), title: 't' },
|
|
31
|
+
version,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe('CacheModule ingest relay', () => {
|
|
35
|
+
it('applyRelayedIngest updates the version memo and never re-relays', () => {
|
|
36
|
+
const { cache, ssp, relay } = setup();
|
|
37
|
+
cache.setIngestRelay(relay);
|
|
38
|
+
|
|
39
|
+
cache.applyRelayedIngest([
|
|
40
|
+
{ table: 'thread', op: 'CREATE', id: 'thread:a', record: { id: 'thread:a', _00_rv: 4 } },
|
|
41
|
+
]);
|
|
42
|
+
|
|
43
|
+
expect(ssp.ingestMany).toHaveBeenCalledTimes(1);
|
|
44
|
+
expect(cache.lookup('thread:a')).toBe(4);
|
|
45
|
+
expect(relay).not.toHaveBeenCalled();
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('localWritesOnly relays the mutation path (save + delete)', async () => {
|
|
49
|
+
const { cache, local, relay } = setup();
|
|
50
|
+
cache.setIngestRelay(relay, { localWritesOnly: true });
|
|
51
|
+
|
|
52
|
+
await cache.save(rec('a', 1), true);
|
|
53
|
+
expect(local.execute).not.toHaveBeenCalled();
|
|
54
|
+
expect(relay).toHaveBeenCalledTimes(1);
|
|
55
|
+
expect(relay.mock.calls[0][0][0]).toMatchObject({ table: 'thread', op: 'CREATE', id: 'thread:a' });
|
|
56
|
+
|
|
57
|
+
const before = { id: 'thread:a', title: 't' };
|
|
58
|
+
await cache.delete('thread', 'thread:a', true, before);
|
|
59
|
+
expect(relay).toHaveBeenCalledTimes(2);
|
|
60
|
+
expect(relay.mock.calls[1][0]).toEqual([
|
|
61
|
+
{ table: 'thread', op: 'DELETE', id: 'thread:a', record: before },
|
|
62
|
+
]);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('localWritesOnly stays silent for sync-fetched batches', async () => {
|
|
66
|
+
const { cache, local, relay } = setup();
|
|
67
|
+
cache.setIngestRelay(relay, { localWritesOnly: true });
|
|
68
|
+
|
|
69
|
+
await cache.saveBatch([rec('a', 1), rec('b', 1)], false);
|
|
70
|
+
expect(local.execute).toHaveBeenCalledTimes(1);
|
|
71
|
+
expect(relay).not.toHaveBeenCalled();
|
|
72
|
+
|
|
73
|
+
await cache.delete('thread', 'thread:a', false, {});
|
|
74
|
+
expect(relay).not.toHaveBeenCalled();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('an unscoped relay fires for every ingest', async () => {
|
|
78
|
+
const { cache, relay } = setup();
|
|
79
|
+
cache.setIngestRelay(relay);
|
|
80
|
+
|
|
81
|
+
await cache.saveBatch([rec('a', 1)], false);
|
|
82
|
+
await cache.save(rec('b', 1), true);
|
|
83
|
+
await cache.delete('thread', 'thread:a', false, {});
|
|
84
|
+
|
|
85
|
+
expect(relay).toHaveBeenCalledTimes(3);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('setIngestRelay(null) stops relaying', async () => {
|
|
89
|
+
const { cache, relay } = setup();
|
|
90
|
+
cache.setIngestRelay(relay, { localWritesOnly: true });
|
|
91
|
+
cache.setIngestRelay(null);
|
|
92
|
+
await cache.save(rec('a', 1), true);
|
|
93
|
+
expect(relay).not.toHaveBeenCalled();
|
|
94
|
+
});
|
|
95
|
+
});
|
|
@@ -33,8 +33,11 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
33
33
|
private versionLookups: Record<string, number> = {};
|
|
34
34
|
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
35
35
|
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
36
|
-
* follower only needs the circuit feed).
|
|
36
|
+
* follower only needs the circuit feed). A follower relays its own
|
|
37
|
+
* mutations to the leader the same way, see {@link setIngestRelay}. */
|
|
37
38
|
private ingestRelay: ((tuples: CacheIngestTuple[]) => void) | null = null;
|
|
39
|
+
/** See {@link setIngestRelay}. */
|
|
40
|
+
private relayLocalWritesOnly = false;
|
|
38
41
|
|
|
39
42
|
constructor(
|
|
40
43
|
private local: LocalStore,
|
|
@@ -64,8 +67,21 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
64
67
|
this.streamUpdateCallback(update);
|
|
65
68
|
}
|
|
66
69
|
|
|
67
|
-
|
|
70
|
+
/**
|
|
71
|
+
* Fan every committed ingest out to the other tabs. The leader relays
|
|
72
|
+
* everything (its sync fetches are the only copy the followers get). A
|
|
73
|
+
* follower relays with `localWritesOnly`: just the mutation path, which is
|
|
74
|
+
* the only thing it knows that the leader does not. Its sync-fetched
|
|
75
|
+
* batches are the leader's data coming back and must not be re-broadcast,
|
|
76
|
+
* or every follower registration would fan its whole working set to
|
|
77
|
+
* every tab.
|
|
78
|
+
*/
|
|
79
|
+
setIngestRelay(
|
|
80
|
+
cb: ((tuples: CacheIngestTuple[]) => void) | null,
|
|
81
|
+
opts: { localWritesOnly?: boolean } = {}
|
|
82
|
+
): void {
|
|
68
83
|
this.ingestRelay = cb;
|
|
84
|
+
this.relayLocalWritesOnly = opts.localWritesOnly === true;
|
|
69
85
|
}
|
|
70
86
|
|
|
71
87
|
/**
|
|
@@ -197,7 +213,11 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
197
213
|
// failed mid-batch is neither "known" here nor fanned out to followers.
|
|
198
214
|
const ingested = this.streamProcessor.ingestMany(bulk);
|
|
199
215
|
for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
|
|
200
|
-
|
|
216
|
+
// `skipDbInsert` is exactly the mutation path (the tx already wrote the
|
|
217
|
+
// row); sync-fetched batches pass false.
|
|
218
|
+
if (ingested.length > 0 && (!this.relayLocalWritesOnly || skipDbInsert)) {
|
|
219
|
+
this.ingestRelay?.(ingested as CacheIngestTuple[]);
|
|
220
|
+
}
|
|
201
221
|
|
|
202
222
|
this.logger.debug(
|
|
203
223
|
{ count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
|
|
@@ -239,7 +259,9 @@ export class CacheModule implements StreamUpdateReceiver {
|
|
|
239
259
|
// 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
|
|
240
260
|
delete this.versionLookups[id];
|
|
241
261
|
this.streamProcessor.ingestMany([{ table, op: 'DELETE', id, record: recordData }]);
|
|
242
|
-
this.
|
|
262
|
+
if (!this.relayLocalWritesOnly || skipDbDelete) {
|
|
263
|
+
this.ingestRelay?.([{ table, op: 'DELETE', id, record: recordData }]);
|
|
264
|
+
}
|
|
243
265
|
|
|
244
266
|
this.logger.debug(
|
|
245
267
|
{ table, id, Category: 'sp00ky-client::CacheModule::delete' },
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DataModule } from './index';
|
|
4
|
+
|
|
5
|
+
// A DELETE that landed in the local store (this tab's own, or one relayed
|
|
6
|
+
// from another tab) needs a forced re-materialize of the table's queries: the
|
|
7
|
+
// SSP may not emit a view update for it.
|
|
8
|
+
|
|
9
|
+
function makeLogger(): any {
|
|
10
|
+
const logger: any = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {}, trace: () => {} };
|
|
11
|
+
logger.child = () => logger;
|
|
12
|
+
return logger;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const schema = { tables: [{ name: 'comment', columns: {} }, { name: 'thread', columns: {} }] } as any;
|
|
16
|
+
|
|
17
|
+
function state(hash: string, tableName: string): any {
|
|
18
|
+
return {
|
|
19
|
+
config: { id: new RecordId('_00_query', hash), tableName, localArray: [], remoteArray: [] },
|
|
20
|
+
records: [],
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe('DataModule.notifyTableQueries', () => {
|
|
25
|
+
it('re-materializes only the queries on that table, isolating failures', async () => {
|
|
26
|
+
const local: any = { epoch: 1, query: vi.fn(async () => [[]]) };
|
|
27
|
+
const dm = new DataModule({ saveBatch: async () => {} } as any, local, schema, makeLogger(), 100);
|
|
28
|
+
(dm as any).activeQueries.set('c1', state('c1', 'comment'));
|
|
29
|
+
(dm as any).activeQueries.set('c2', state('c2', 'comment'));
|
|
30
|
+
(dm as any).activeQueries.set('t1', state('t1', 'thread'));
|
|
31
|
+
const notified: string[] = [];
|
|
32
|
+
dm.notifyQuerySynced = vi.fn(async (hash: string) => {
|
|
33
|
+
notified.push(hash);
|
|
34
|
+
if (hash === 'c1') throw new Error('boom');
|
|
35
|
+
}) as any;
|
|
36
|
+
|
|
37
|
+
await dm.notifyTableQueries('comment');
|
|
38
|
+
|
|
39
|
+
expect(notified.sort()).toEqual(['c1', 'c2']);
|
|
40
|
+
});
|
|
41
|
+
});
|