@spooky-sync/core 0.0.1-canary.205 → 0.0.1-canary.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +78 -6
- package/dist/index.js +151 -44
- package/dist/types.d.ts +8 -0
- package/package.json +3 -3
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +26 -4
- package/src/modules/data/data.membership.test.ts +62 -1
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.rebind.test.ts +23 -0
- package/src/modules/data/index.ts +82 -31
- package/src/modules/devtools/index.ts +6 -0
- package/src/modules/sync/sync.live-removal.test.ts +40 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +71 -6
- package/src/services/tabs/coordinator.test.ts +118 -2
- package/src/services/tabs/coordinator.ts +25 -0
- package/src/services/tabs/protocol.ts +17 -1
- package/src/sp00ky.ts +9 -11
- package/src/types.ts +8 -0
package/dist/index.d.ts
CHANGED
|
@@ -537,15 +537,29 @@ declare class CacheModule implements StreamUpdateReceiver {
|
|
|
537
537
|
private versionLookups;
|
|
538
538
|
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
539
539
|
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
540
|
-
* follower only needs the circuit feed).
|
|
540
|
+
* follower only needs the circuit feed). A follower relays its own
|
|
541
|
+
* mutations to the leader the same way, see {@link setIngestRelay}. */
|
|
541
542
|
private ingestRelay;
|
|
543
|
+
/** See {@link setIngestRelay}. */
|
|
544
|
+
private relayLocalWritesOnly;
|
|
542
545
|
constructor(local: LocalStore, streamProcessor: StreamProcessorService, streamUpdateCallback: (update: StreamUpdate) => void, logger: Logger$1);
|
|
543
546
|
/**
|
|
544
547
|
* Implements StreamUpdateReceiver interface
|
|
545
548
|
* Called directly by StreamProcessor when views change
|
|
546
549
|
*/
|
|
547
550
|
onStreamUpdate(update: StreamUpdate): void;
|
|
548
|
-
|
|
551
|
+
/**
|
|
552
|
+
* Fan every committed ingest out to the other tabs. The leader relays
|
|
553
|
+
* everything (its sync fetches are the only copy the followers get). A
|
|
554
|
+
* follower relays with `localWritesOnly`: just the mutation path, which is
|
|
555
|
+
* the only thing it knows that the leader does not. Its sync-fetched
|
|
556
|
+
* batches are the leader's data coming back and must not be re-broadcast,
|
|
557
|
+
* or every follower registration would fan its whole working set to
|
|
558
|
+
* every tab.
|
|
559
|
+
*/
|
|
560
|
+
setIngestRelay(cb: ((tuples: CacheIngestTuple[]) => void) | null, opts?: {
|
|
561
|
+
localWritesOnly?: boolean;
|
|
562
|
+
}): void;
|
|
549
563
|
/**
|
|
550
564
|
* Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
|
|
551
565
|
* The rows are already in the shared store (the leader wrote them), so no
|
|
@@ -606,6 +620,12 @@ declare class CacheModule implements StreamUpdateReceiver {
|
|
|
606
620
|
* Merges the functionality of QueryManager and MutationManager.
|
|
607
621
|
* Uses CacheModule for all storage operations.
|
|
608
622
|
*/
|
|
623
|
+
/** A `_00_window` row as read back: the id-set and whether the server vouched
|
|
624
|
+
* for it (which is what allows an empty set to count as known membership). */
|
|
625
|
+
interface DurableMembership {
|
|
626
|
+
ids: RecordVersionArray;
|
|
627
|
+
confirmed: boolean;
|
|
628
|
+
}
|
|
609
629
|
declare class DataModule<S extends SchemaStructure> {
|
|
610
630
|
private cache;
|
|
611
631
|
private local;
|
|
@@ -901,11 +921,25 @@ declare class DataModule<S extends SchemaStructure> {
|
|
|
901
921
|
* authoritative membership on this device. Any read error is treated as
|
|
902
922
|
* "unknown" so a broken row degrades to the predicate scan rather than
|
|
903
923
|
* rendering an empty list.
|
|
924
|
+
*
|
|
925
|
+
* `confirmed` is true only for rows written after the server itself vouched
|
|
926
|
+
* for the set (a non-empty id-set, or an empty one it reported a row count of
|
|
927
|
+
* zero for, or an empty one that followed a non-empty one in the same
|
|
928
|
+
* session). Rows written before the marker existed, including the `[]` rows a
|
|
929
|
+
* pre-`ea56f50e` client mirrored from an unflushed read, read as unconfirmed.
|
|
904
930
|
*/
|
|
905
|
-
getWindowMembership(key: string): Promise<
|
|
906
|
-
/**
|
|
907
|
-
*
|
|
908
|
-
|
|
931
|
+
getWindowMembership(key: string): Promise<DurableMembership | null>;
|
|
932
|
+
/**
|
|
933
|
+
* Persist the durable membership row. Best-effort: callers must not fail a
|
|
934
|
+
* sync round because the mirror write failed.
|
|
935
|
+
*
|
|
936
|
+
* `confirmed` says whether a cold start may trust this row even when it is
|
|
937
|
+
* empty. A confirmed empty is a real answer ("the server says this query has
|
|
938
|
+
* no rows") and stays empty across a reload; an unconfirmed empty is the
|
|
939
|
+
* retry budget's guess and falls back to the predicate scan on the next boot,
|
|
940
|
+
* exactly as every empty row did before the marker existed.
|
|
941
|
+
*/
|
|
942
|
+
writeWindowMembership(key: string, ids: RecordVersionArray, confirmed: boolean): Promise<void>;
|
|
909
943
|
/**
|
|
910
944
|
* Record ids with a mutation still in the outbox, split by direction.
|
|
911
945
|
*
|
|
@@ -1015,6 +1049,15 @@ declare class DataModule<S extends SchemaStructure> {
|
|
|
1015
1049
|
* Rollback a failed optimistic update by restoring the previous record state
|
|
1016
1050
|
*/
|
|
1017
1051
|
rollbackUpdate(recordId: RecordId, tableName: string, beforeRecord: Record<string, unknown>): Promise<void>;
|
|
1052
|
+
/**
|
|
1053
|
+
* Force a re-materialize + notify of every active query on `tableName`.
|
|
1054
|
+
* Used after a DELETE landed in the local store (this tab's own, or one
|
|
1055
|
+
* relayed from another tab): the SSP may not emit a view update for a
|
|
1056
|
+
* DELETE ingest, and the re-materialize reads the store, which already
|
|
1057
|
+
* excludes the row. Each query is isolated so one failing re-materialize
|
|
1058
|
+
* can't stop the others.
|
|
1059
|
+
*/
|
|
1060
|
+
notifyTableQueries(tableName: string): Promise<void>;
|
|
1018
1061
|
/**
|
|
1019
1062
|
* Remove a record from all active query states and notify subscribers
|
|
1020
1063
|
*/
|
|
@@ -1065,6 +1108,14 @@ type FollowerToLeaderMessage = {
|
|
|
1065
1108
|
mutationId: string;
|
|
1066
1109
|
} | {
|
|
1067
1110
|
type: 'request-poll';
|
|
1111
|
+
}
|
|
1112
|
+
/** An optimistic write this follower committed to the SHARED store and
|
|
1113
|
+
* ingested into its own circuit. The leader ingests it (no DB write, the
|
|
1114
|
+
* row is already there) and fans it out to every OTHER follower as
|
|
1115
|
+
* `ingest-relay`, so a follower's write lands in every tab in one hop
|
|
1116
|
+
* instead of after the server round-trip. */ | {
|
|
1117
|
+
type: 'ingest';
|
|
1118
|
+
tuples: IngestTuple[];
|
|
1068
1119
|
};
|
|
1069
1120
|
type LeaderToFollowerMessage = {
|
|
1070
1121
|
type: 'db-ready';
|
|
@@ -1094,6 +1145,15 @@ type LeaderToFollowerMessage = {
|
|
|
1094
1145
|
recordId: string;
|
|
1095
1146
|
eventType: 'create' | 'update' | 'delete';
|
|
1096
1147
|
error: string;
|
|
1148
|
+
}
|
|
1149
|
+
/** The leader's drain pushed a mutation and deleted its outbox row from the
|
|
1150
|
+
* SHARED store. Every follower starts its settled-write grace so a row it
|
|
1151
|
+
* was rendering as a pending write does not blink out before its
|
|
1152
|
+
* `_00_list_ref` membership arrives. */ | {
|
|
1153
|
+
type: 'mutation-settled';
|
|
1154
|
+
mutationId: string;
|
|
1155
|
+
recordId: string;
|
|
1156
|
+
eventType: 'create' | 'update' | 'delete';
|
|
1097
1157
|
};
|
|
1098
1158
|
//#endregion
|
|
1099
1159
|
//#region src/services/tabs/coordinator.d.ts
|
|
@@ -1130,6 +1190,11 @@ declare class SyncForwarder {
|
|
|
1130
1190
|
unbind(): void;
|
|
1131
1191
|
private post;
|
|
1132
1192
|
mutationEnqueued(mutationId: string): void;
|
|
1193
|
+
/** An optimistic write this tab just ingested. Deliberately NOT queued while
|
|
1194
|
+
* detached: a new leader primes its circuit from the shared store, which
|
|
1195
|
+
* already holds the row, and replaying a stale tuple at it later would put
|
|
1196
|
+
* an older `_00_rv` in its version memo. */
|
|
1197
|
+
ingest(tuples: IngestTuple[]): void;
|
|
1133
1198
|
requestPoll(): void;
|
|
1134
1199
|
}
|
|
1135
1200
|
//#endregion
|
|
@@ -1361,6 +1426,13 @@ declare class Sp00kySync<S extends SchemaStructure> {
|
|
|
1361
1426
|
private onMutationDropped;
|
|
1362
1427
|
/** A forwarded outbox row from a follower: load + drain it. Idempotent. */
|
|
1363
1428
|
enqueueForwardedMutation(mutationId: string): Promise<void>;
|
|
1429
|
+
/**
|
|
1430
|
+
* Tuples another tab already committed to the shared store: feed them to
|
|
1431
|
+
* THIS tab's circuit (no local write). A DELETE additionally forces a
|
|
1432
|
+
* re-materialize of the table's queries, exactly as the writing tab does
|
|
1433
|
+
* for itself, because the SSP may not emit a view update for it.
|
|
1434
|
+
*/
|
|
1435
|
+
private applyRelayedIngest;
|
|
1364
1436
|
/** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
|
|
1365
1437
|
* and run the exact same handling the LIVE subscription would have. */
|
|
1366
1438
|
private applyRelayedListRefChange;
|
package/dist/index.js
CHANGED
|
@@ -3321,12 +3321,6 @@ function phaseStatOf(samples, lastMs) {
|
|
|
3321
3321
|
count: samples.length
|
|
3322
3322
|
};
|
|
3323
3323
|
}
|
|
3324
|
-
/**
|
|
3325
|
-
* DataModule - Unified query and mutation management
|
|
3326
|
-
*
|
|
3327
|
-
* Merges the functionality of QueryManager and MutationManager.
|
|
3328
|
-
* Uses CacheModule for all storage operations.
|
|
3329
|
-
*/
|
|
3330
3324
|
var DataModule = class DataModule {
|
|
3331
3325
|
/** Tab identity baked into mutation ids (shared-tabs rollback routing);
|
|
3332
3326
|
* undefined in solo mode, where mutation-id falls back to a session id. */
|
|
@@ -4030,23 +4024,42 @@ var DataModule = class DataModule {
|
|
|
4030
4024
|
* authoritative membership on this device. Any read error is treated as
|
|
4031
4025
|
* "unknown" so a broken row degrades to the predicate scan rather than
|
|
4032
4026
|
* rendering an empty list.
|
|
4027
|
+
*
|
|
4028
|
+
* `confirmed` is true only for rows written after the server itself vouched
|
|
4029
|
+
* for the set (a non-empty id-set, or an empty one it reported a row count of
|
|
4030
|
+
* zero for, or an empty one that followed a non-empty one in the same
|
|
4031
|
+
* session). Rows written before the marker existed, including the `[]` rows a
|
|
4032
|
+
* pre-`ea56f50e` client mirrored from an unflushed read, read as unconfirmed.
|
|
4033
4033
|
*/
|
|
4034
4034
|
async getWindowMembership(key) {
|
|
4035
4035
|
try {
|
|
4036
4036
|
const row = await this.local.getById("_00_window", new RecordId("_00_window", key));
|
|
4037
4037
|
if (!row || typeof row !== "object") return null;
|
|
4038
4038
|
const ids = row.ids;
|
|
4039
|
-
|
|
4039
|
+
if (!Array.isArray(ids)) return null;
|
|
4040
|
+
return {
|
|
4041
|
+
ids,
|
|
4042
|
+
confirmed: row.confirmed === true
|
|
4043
|
+
};
|
|
4040
4044
|
} catch {
|
|
4041
4045
|
return null;
|
|
4042
4046
|
}
|
|
4043
4047
|
}
|
|
4044
|
-
/**
|
|
4045
|
-
*
|
|
4046
|
-
|
|
4048
|
+
/**
|
|
4049
|
+
* Persist the durable membership row. Best-effort: callers must not fail a
|
|
4050
|
+
* sync round because the mirror write failed.
|
|
4051
|
+
*
|
|
4052
|
+
* `confirmed` says whether a cold start may trust this row even when it is
|
|
4053
|
+
* empty. A confirmed empty is a real answer ("the server says this query has
|
|
4054
|
+
* no rows") and stays empty across a reload; an unconfirmed empty is the
|
|
4055
|
+
* retry budget's guess and falls back to the predicate scan on the next boot,
|
|
4056
|
+
* exactly as every empty row did before the marker existed.
|
|
4057
|
+
*/
|
|
4058
|
+
async writeWindowMembership(key, ids, confirmed) {
|
|
4047
4059
|
try {
|
|
4048
4060
|
await this.local.upsert("_00_window", new RecordId("_00_window", key), {
|
|
4049
4061
|
ids,
|
|
4062
|
+
confirmed,
|
|
4050
4063
|
updatedAt: Date.now()
|
|
4051
4064
|
}, "replace");
|
|
4052
4065
|
} catch (err) {
|
|
@@ -4205,9 +4218,12 @@ var DataModule = class DataModule {
|
|
|
4205
4218
|
}, "Query to update remote array not found");
|
|
4206
4219
|
return;
|
|
4207
4220
|
}
|
|
4221
|
+
let confirmed = remoteArray.length > 0 || queryState.config.remoteSeen === true;
|
|
4208
4222
|
if (remoteArray.length === 0 && !queryState.config.remoteSeen) {
|
|
4209
4223
|
const serverRowCount = opts?.serverRowCount;
|
|
4210
|
-
|
|
4224
|
+
const knownEmpty = serverRowCount === 0;
|
|
4225
|
+
confirmed = knownEmpty;
|
|
4226
|
+
if (!knownEmpty) {
|
|
4211
4227
|
const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
|
|
4212
4228
|
queryState.config.emptyReads = emptyReads;
|
|
4213
4229
|
if (!(serverRowCount === null || serverRowCount === void 0 ? emptyReads >= EMPTY_MEMBERSHIP_CONFIRMATIONS : false)) {
|
|
@@ -4228,7 +4244,7 @@ var DataModule = class DataModule {
|
|
|
4228
4244
|
queryState.config.remoteSeen = true;
|
|
4229
4245
|
queryState.config.emptyReads = 0;
|
|
4230
4246
|
}
|
|
4231
|
-
if (queryState.config.membershipKey) await this.writeWindowMembership(queryState.config.membershipKey, remoteArray);
|
|
4247
|
+
if (queryState.config.membershipKey) await this.writeWindowMembership(queryState.config.membershipKey, remoteArray, confirmed);
|
|
4232
4248
|
try {
|
|
4233
4249
|
await this.local.query(surql.seal(surql.updateSet("id", ["remoteArray"])), {
|
|
4234
4250
|
id: queryState.config.id,
|
|
@@ -4281,8 +4297,8 @@ var DataModule = class DataModule {
|
|
|
4281
4297
|
config.emptyReads = 0;
|
|
4282
4298
|
if (config.membershipKey) {
|
|
4283
4299
|
const durable = await this.getWindowMembership(config.membershipKey);
|
|
4284
|
-
if (durable
|
|
4285
|
-
config.remoteArray = durable;
|
|
4300
|
+
if (durable && (durable.ids.length > 0 || durable.confirmed)) {
|
|
4301
|
+
config.remoteArray = durable.ids;
|
|
4286
4302
|
config.membershipKnown = true;
|
|
4287
4303
|
}
|
|
4288
4304
|
}
|
|
@@ -4515,15 +4531,7 @@ var DataModule = class DataModule {
|
|
|
4515
4531
|
Category: "sp00ky-client::DataModule::delete"
|
|
4516
4532
|
}, "SSP delete-ingest failed; relying on query re-materialize to reflect the delete");
|
|
4517
4533
|
}
|
|
4518
|
-
|
|
4519
|
-
await this.notifyQuerySynced(queryHash);
|
|
4520
|
-
} catch (err) {
|
|
4521
|
-
this.logger.error({
|
|
4522
|
-
err,
|
|
4523
|
-
queryHash,
|
|
4524
|
-
Category: "sp00ky-client::DataModule::delete"
|
|
4525
|
-
}, "notifyQuerySynced failed after delete");
|
|
4526
|
-
}
|
|
4534
|
+
await this.notifyTableQueries(tableName);
|
|
4527
4535
|
const mutationEvent = {
|
|
4528
4536
|
type: "delete",
|
|
4529
4537
|
mutation_id: mutationId,
|
|
@@ -4594,6 +4602,26 @@ var DataModule = class DataModule {
|
|
|
4594
4602
|
}
|
|
4595
4603
|
}
|
|
4596
4604
|
/**
|
|
4605
|
+
* Force a re-materialize + notify of every active query on `tableName`.
|
|
4606
|
+
* Used after a DELETE landed in the local store (this tab's own, or one
|
|
4607
|
+
* relayed from another tab): the SSP may not emit a view update for a
|
|
4608
|
+
* DELETE ingest, and the re-materialize reads the store, which already
|
|
4609
|
+
* excludes the row. Each query is isolated so one failing re-materialize
|
|
4610
|
+
* can't stop the others.
|
|
4611
|
+
*/
|
|
4612
|
+
async notifyTableQueries(tableName) {
|
|
4613
|
+
for (const [queryHash, queryState] of this.activeQueries) if (queryState.config.tableName === tableName) try {
|
|
4614
|
+
await this.notifyQuerySynced(queryHash);
|
|
4615
|
+
} catch (err) {
|
|
4616
|
+
this.logger.error({
|
|
4617
|
+
err,
|
|
4618
|
+
queryHash,
|
|
4619
|
+
tableName,
|
|
4620
|
+
Category: "sp00ky-client::DataModule::notifyTableQueries"
|
|
4621
|
+
}, "notifyQuerySynced failed after delete");
|
|
4622
|
+
}
|
|
4623
|
+
}
|
|
4624
|
+
/**
|
|
4597
4625
|
* Remove a record from all active query states and notify subscribers
|
|
4598
4626
|
*/
|
|
4599
4627
|
removeRecordFromQueries(recordId) {
|
|
@@ -4712,8 +4740,8 @@ var DataModule = class DataModule {
|
|
|
4712
4740
|
};
|
|
4713
4741
|
if (membershipKey && !config.remoteArray?.length) {
|
|
4714
4742
|
const durable = await this.getWindowMembership(membershipKey);
|
|
4715
|
-
if (durable
|
|
4716
|
-
config.remoteArray = durable;
|
|
4743
|
+
if (durable && (durable.ids.length > 0 || durable.confirmed)) {
|
|
4744
|
+
config.remoteArray = durable.ids;
|
|
4717
4745
|
config.membershipKnown = true;
|
|
4718
4746
|
}
|
|
4719
4747
|
} else if (config.remoteArray?.length) config.membershipKnown = true;
|
|
@@ -6118,8 +6146,14 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6118
6146
|
switch (msg.type) {
|
|
6119
6147
|
case "sync-hello": break;
|
|
6120
6148
|
case "mutation-enqueued":
|
|
6149
|
+
this.listRefIdleStreak = 0;
|
|
6121
6150
|
this.enqueueForwardedMutation(msg.mutationId);
|
|
6122
6151
|
break;
|
|
6152
|
+
case "ingest":
|
|
6153
|
+
this.applyRelayedIngest(msg.tuples);
|
|
6154
|
+
hub.relayIngest(msg.tuples, tabId);
|
|
6155
|
+
this.listRefIdleStreak = 0;
|
|
6156
|
+
break;
|
|
6123
6157
|
case "request-poll":
|
|
6124
6158
|
this.listRefIdleStreak = 0;
|
|
6125
6159
|
break;
|
|
@@ -6164,6 +6198,12 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6164
6198
|
this.killRefLiveQuery();
|
|
6165
6199
|
forwarder.onLeaderMessage = (msg) => {
|
|
6166
6200
|
switch (msg.type) {
|
|
6201
|
+
case "ingest-relay":
|
|
6202
|
+
this.applyRelayedIngest(msg.tuples);
|
|
6203
|
+
break;
|
|
6204
|
+
case "mutation-settled":
|
|
6205
|
+
this.dataModule.noteWriteSettled(msg.recordId, msg.eventType);
|
|
6206
|
+
break;
|
|
6167
6207
|
case "list-ref-change":
|
|
6168
6208
|
this.applyRelayedListRefChange(msg).catch((err) => {
|
|
6169
6209
|
this.logger.error({
|
|
@@ -6209,6 +6249,24 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6209
6249
|
if (this.tabRole !== "leader") return;
|
|
6210
6250
|
await this.upQueue.enqueueFromDatabase(mutationId);
|
|
6211
6251
|
}
|
|
6252
|
+
/**
|
|
6253
|
+
* Tuples another tab already committed to the shared store: feed them to
|
|
6254
|
+
* THIS tab's circuit (no local write). A DELETE additionally forces a
|
|
6255
|
+
* re-materialize of the table's queries, exactly as the writing tab does
|
|
6256
|
+
* for itself, because the SSP may not emit a view update for it.
|
|
6257
|
+
*/
|
|
6258
|
+
applyRelayedIngest(tuples) {
|
|
6259
|
+
this.cache.applyRelayedIngest(tuples);
|
|
6260
|
+
const deletedTables = /* @__PURE__ */ new Set();
|
|
6261
|
+
for (const t of tuples) if (t.op === "DELETE") deletedTables.add(t.table);
|
|
6262
|
+
for (const table of deletedTables) this.dataModule.notifyTableQueries(table).catch((err) => {
|
|
6263
|
+
this.logger.warn({
|
|
6264
|
+
err,
|
|
6265
|
+
table,
|
|
6266
|
+
Category: "sp00ky-client::Sp00kySync::applyRelayedIngest"
|
|
6267
|
+
}, "Re-materialize after relayed delete failed");
|
|
6268
|
+
});
|
|
6269
|
+
}
|
|
6212
6270
|
/** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
|
|
6213
6271
|
* and run the exact same handling the LIVE subscription would have. */
|
|
6214
6272
|
async applyRelayedListRefChange(msg) {
|
|
@@ -6601,8 +6659,20 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6601
6659
|
}, "Live update is being processed");
|
|
6602
6660
|
const diff = createDiffFromDbOp(action, recordId, version, localArray);
|
|
6603
6661
|
const hash = extractIdPart(existing.config.id);
|
|
6604
|
-
if (existing.config.membershipKnown
|
|
6605
|
-
const
|
|
6662
|
+
if (existing.config.membershipKnown) {
|
|
6663
|
+
const membershipDiff = action === "DELETE" ? {
|
|
6664
|
+
added: [],
|
|
6665
|
+
updated: [],
|
|
6666
|
+
removed: [recordId]
|
|
6667
|
+
} : {
|
|
6668
|
+
added: [{
|
|
6669
|
+
id: recordId,
|
|
6670
|
+
version
|
|
6671
|
+
}],
|
|
6672
|
+
updated: [],
|
|
6673
|
+
removed: []
|
|
6674
|
+
};
|
|
6675
|
+
const next = applyRecordVersionDiff(existing.config.remoteArray ?? [], membershipDiff);
|
|
6606
6676
|
if (!recordVersionArraysEqual(next, existing.config.remoteArray ?? [])) await this.dataModule.updateQueryRemoteArray(hash, next);
|
|
6607
6677
|
}
|
|
6608
6678
|
await this.runSyncForQuery(hash, diff);
|
|
@@ -6721,7 +6791,14 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6721
6791
|
* vanish, and return, while every other client showed it throughout.
|
|
6722
6792
|
*/
|
|
6723
6793
|
handleMutationSettled(event) {
|
|
6724
|
-
|
|
6794
|
+
const recordId = encodeRecordId(event.record_id);
|
|
6795
|
+
this.dataModule.noteWriteSettled(recordId, event.type);
|
|
6796
|
+
this.hub?.broadcast({
|
|
6797
|
+
type: "mutation-settled",
|
|
6798
|
+
mutationId: encodeRecordId(event.mutation_id),
|
|
6799
|
+
recordId,
|
|
6800
|
+
eventType: event.type
|
|
6801
|
+
});
|
|
6725
6802
|
}
|
|
6726
6803
|
async handleRollback(event, error) {
|
|
6727
6804
|
const recordId = encodeRecordId(event.record_id);
|
|
@@ -7337,8 +7414,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
7337
7414
|
|
|
7338
7415
|
//#endregion
|
|
7339
7416
|
//#region src/modules/devtools/index.ts
|
|
7340
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
7341
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
7417
|
+
const CORE_VERSION = "0.0.1-canary.207";
|
|
7418
|
+
const WASM_VERSION = "0.0.1-canary.207";
|
|
7342
7419
|
const SURREAL_VERSION = "3.0.3";
|
|
7343
7420
|
var DevToolsService = class DevToolsService {
|
|
7344
7421
|
eventsHistory = [];
|
|
@@ -7439,6 +7516,9 @@ var DevToolsService = class DevToolsService {
|
|
|
7439
7516
|
data: q.records,
|
|
7440
7517
|
localArray: q.config.localArray,
|
|
7441
7518
|
remoteArray: q.config.remoteArray,
|
|
7519
|
+
membershipKnown: q.config.membershipKnown === true,
|
|
7520
|
+
remoteSeen: q.config.remoteSeen === true,
|
|
7521
|
+
emptyReads: q.config.emptyReads ?? 0,
|
|
7442
7522
|
timings: this.dataManager.phaseTimings(q)
|
|
7443
7523
|
});
|
|
7444
7524
|
});
|
|
@@ -8864,8 +8944,11 @@ var CacheModule = class {
|
|
|
8864
8944
|
versionLookups = {};
|
|
8865
8945
|
/** Shared-tabs leader: fan every committed ingest out to follower circuits.
|
|
8866
8946
|
* Fired AFTER the local tx (the rows are already in the shared store, so a
|
|
8867
|
-
* follower only needs the circuit feed).
|
|
8947
|
+
* follower only needs the circuit feed). A follower relays its own
|
|
8948
|
+
* mutations to the leader the same way, see {@link setIngestRelay}. */
|
|
8868
8949
|
ingestRelay = null;
|
|
8950
|
+
/** See {@link setIngestRelay}. */
|
|
8951
|
+
relayLocalWritesOnly = false;
|
|
8869
8952
|
constructor(local, streamProcessor, streamUpdateCallback, logger) {
|
|
8870
8953
|
this.local = local;
|
|
8871
8954
|
this.streamProcessor = streamProcessor;
|
|
@@ -8885,8 +8968,18 @@ var CacheModule = class {
|
|
|
8885
8968
|
}, "Stream update received");
|
|
8886
8969
|
this.streamUpdateCallback(update);
|
|
8887
8970
|
}
|
|
8888
|
-
|
|
8971
|
+
/**
|
|
8972
|
+
* Fan every committed ingest out to the other tabs. The leader relays
|
|
8973
|
+
* everything (its sync fetches are the only copy the followers get). A
|
|
8974
|
+
* follower relays with `localWritesOnly`: just the mutation path, which is
|
|
8975
|
+
* the only thing it knows that the leader does not. Its sync-fetched
|
|
8976
|
+
* batches are the leader's data coming back and must not be re-broadcast,
|
|
8977
|
+
* or every follower registration would fan its whole working set to
|
|
8978
|
+
* every tab.
|
|
8979
|
+
*/
|
|
8980
|
+
setIngestRelay(cb, opts = {}) {
|
|
8889
8981
|
this.ingestRelay = cb;
|
|
8982
|
+
this.relayLocalWritesOnly = opts.localWritesOnly === true;
|
|
8890
8983
|
}
|
|
8891
8984
|
/**
|
|
8892
8985
|
* Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
|
|
@@ -8977,7 +9070,7 @@ var CacheModule = class {
|
|
|
8977
9070
|
});
|
|
8978
9071
|
const ingested = this.streamProcessor.ingestMany(bulk);
|
|
8979
9072
|
for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
|
|
8980
|
-
if (ingested.length > 0) this.ingestRelay?.(ingested);
|
|
9073
|
+
if (ingested.length > 0 && (!this.relayLocalWritesOnly || skipDbInsert)) this.ingestRelay?.(ingested);
|
|
8981
9074
|
this.logger.debug({
|
|
8982
9075
|
count: records.length,
|
|
8983
9076
|
Category: "sp00ky-client::CacheModule::saveBatch"
|
|
@@ -9018,7 +9111,7 @@ var CacheModule = class {
|
|
|
9018
9111
|
id,
|
|
9019
9112
|
record: recordData
|
|
9020
9113
|
}]);
|
|
9021
|
-
this.ingestRelay?.([{
|
|
9114
|
+
if (!this.relayLocalWritesOnly || skipDbDelete) this.ingestRelay?.([{
|
|
9022
9115
|
table,
|
|
9023
9116
|
op: "DELETE",
|
|
9024
9117
|
id,
|
|
@@ -10598,6 +10691,19 @@ var SyncForwarder = class {
|
|
|
10598
10691
|
mutationId
|
|
10599
10692
|
});
|
|
10600
10693
|
}
|
|
10694
|
+
/** An optimistic write this tab just ingested. Deliberately NOT queued while
|
|
10695
|
+
* detached: a new leader primes its circuit from the shared store, which
|
|
10696
|
+
* already holds the row, and replaying a stale tuple at it later would put
|
|
10697
|
+
* an older `_00_rv` in its version memo. */
|
|
10698
|
+
ingest(tuples) {
|
|
10699
|
+
if (!this.port) return;
|
|
10700
|
+
try {
|
|
10701
|
+
this.port.postMessage({
|
|
10702
|
+
type: "ingest",
|
|
10703
|
+
tuples
|
|
10704
|
+
});
|
|
10705
|
+
} catch {}
|
|
10706
|
+
}
|
|
10601
10707
|
requestPoll() {
|
|
10602
10708
|
this.post({ type: "request-poll" });
|
|
10603
10709
|
}
|
|
@@ -10846,6 +10952,8 @@ var TabsCoordinator = class {
|
|
|
10846
10952
|
const forwarder = this.forwarder;
|
|
10847
10953
|
await new Promise((resolve) => {
|
|
10848
10954
|
let adopted = false;
|
|
10955
|
+
let attached = false;
|
|
10956
|
+
const pending = [];
|
|
10849
10957
|
const previousHandler = forwarder.onLeaderMessage;
|
|
10850
10958
|
forwarder.onLeaderMessage = (msg) => {
|
|
10851
10959
|
if (msg.type === "db-ready" && !adopted) {
|
|
@@ -10856,11 +10964,18 @@ var TabsCoordinator = class {
|
|
|
10856
10964
|
leadershipId: msg.leadershipId
|
|
10857
10965
|
}).then(() => {
|
|
10858
10966
|
this.deps.hooks.becomeSyncFollower(forwarder);
|
|
10967
|
+
attached = true;
|
|
10968
|
+
const backlog = pending.splice(0);
|
|
10969
|
+
for (const m of backlog) forwarder.onLeaderMessage?.(m);
|
|
10859
10970
|
this.setRole("follower");
|
|
10860
10971
|
resolve();
|
|
10861
10972
|
});
|
|
10862
10973
|
return;
|
|
10863
10974
|
}
|
|
10975
|
+
if (!attached) {
|
|
10976
|
+
pending.push(msg);
|
|
10977
|
+
return;
|
|
10978
|
+
}
|
|
10864
10979
|
previousHandler?.(msg);
|
|
10865
10980
|
};
|
|
10866
10981
|
forwarder.rebind(syncPort);
|
|
@@ -12284,7 +12399,7 @@ var Sp00kyClient = class {
|
|
|
12284
12399
|
return new TabsCoordinator({
|
|
12285
12400
|
tabId,
|
|
12286
12401
|
fingerprint: computeTabsFingerprint({
|
|
12287
|
-
coreVersion: "0.0.1-canary.
|
|
12402
|
+
coreVersion: "0.0.1-canary.207",
|
|
12288
12403
|
schemaHash: hash53(this.config.schemaSurql),
|
|
12289
12404
|
endpoint: this.config.database.endpoint ?? "",
|
|
12290
12405
|
namespace: this.config.database.namespace,
|
|
@@ -12311,18 +12426,10 @@ var Sp00kyClient = class {
|
|
|
12311
12426
|
resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
|
|
12312
12427
|
becomeSyncFollower: (forwarder) => {
|
|
12313
12428
|
this.streamProcessor.setPersistenceEnabled(false);
|
|
12314
|
-
this.cache.setIngestRelay(
|
|
12429
|
+
this.cache.setIngestRelay((tuples) => forwarder.ingest(tuples), { localWritesOnly: true });
|
|
12315
12430
|
this.sync.setTabContext("follower", tabId);
|
|
12316
12431
|
this.dataModule.setTabId(tabId);
|
|
12317
12432
|
this.sync.demoteToFollower(forwarder);
|
|
12318
|
-
const inner = forwarder.onLeaderMessage;
|
|
12319
|
-
forwarder.onLeaderMessage = (msg) => {
|
|
12320
|
-
if (msg.type === "ingest-relay") {
|
|
12321
|
-
this.cache.applyRelayedIngest(msg.tuples);
|
|
12322
|
-
return;
|
|
12323
|
-
}
|
|
12324
|
-
inner?.(msg);
|
|
12325
|
-
};
|
|
12326
12433
|
},
|
|
12327
12434
|
becomeSyncSolo: () => {
|
|
12328
12435
|
this.streamProcessor.setPersistenceEnabled(true);
|
package/dist/types.d.ts
CHANGED
|
@@ -874,6 +874,11 @@ interface QueryConfig {
|
|
|
874
874
|
* "never established" has to fall back to a predicate scan of the local store
|
|
875
875
|
* so a query first run on this device still paints offline. A
|
|
876
876
|
* `remoteArray.length === 0` check cannot tell those apart.
|
|
877
|
+
*
|
|
878
|
+
* On a cold start it is seeded from the durable `_00_window` row when that
|
|
879
|
+
* row is non-empty, or empty but `confirmed` (the server reported zero rows
|
|
880
|
+
* for the query). An unconfirmed empty row is ignored, so a device poisoned
|
|
881
|
+
* by an old client that mirrored unflushed reads still self-heals.
|
|
877
882
|
*/
|
|
878
883
|
membershipKnown?: boolean;
|
|
879
884
|
/**
|
|
@@ -889,6 +894,9 @@ interface QueryConfig {
|
|
|
889
894
|
* genuine transition and must be honoured, or removed rows resurrect.
|
|
890
895
|
*
|
|
891
896
|
* In-memory only: a fresh session must re-earn the right to believe empties.
|
|
897
|
+
* What does persist is the `confirmed` marker on the `_00_window` row, which
|
|
898
|
+
* an empty set earns when it arrives with a server row count of zero or after
|
|
899
|
+
* a non-empty set in the same session.
|
|
892
900
|
*/
|
|
893
901
|
remoteSeen?: boolean;
|
|
894
902
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.207",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.207",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.207",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"blurhash": "^2.0.5",
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { CacheModule } from './index';
|
|
4
|
+
|
|
5
|
+
// The ingest relay is how one tab's circuit learns what another tab wrote.
|
|
6
|
+
// The leader relays every ingest (its sync fetches are the only copy the
|
|
7
|
+
// followers get); a follower relays with `localWritesOnly`, just its mutation
|
|
8
|
+
// path, because its sync-fetched batches are the leader's data coming back.
|
|
9
|
+
|
|
10
|
+
function makeLogger(): any {
|
|
11
|
+
const logger: any = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {}, trace: () => {} };
|
|
12
|
+
logger.child = () => logger;
|
|
13
|
+
return logger;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function setup() {
|
|
17
|
+
const local: any = { epoch: 1, execute: vi.fn(async () => {}), query: vi.fn(async () => []) };
|
|
18
|
+
const ssp: any = {
|
|
19
|
+
addReceiver: vi.fn(),
|
|
20
|
+
ingestMany: vi.fn((records: unknown[]) => records),
|
|
21
|
+
};
|
|
22
|
+
const cache = new CacheModule(local, ssp, () => {}, makeLogger());
|
|
23
|
+
const relay = vi.fn();
|
|
24
|
+
return { cache, local, ssp, relay };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const rec = (id: string, version: number) => ({
|
|
28
|
+
table: 'thread',
|
|
29
|
+
op: 'CREATE' as const,
|
|
30
|
+
record: { id: new RecordId('thread', id), title: 't' },
|
|
31
|
+
version,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe('CacheModule ingest relay', () => {
|
|
35
|
+
it('applyRelayedIngest updates the version memo and never re-relays', () => {
|
|
36
|
+
const { cache, ssp, relay } = setup();
|
|
37
|
+
cache.setIngestRelay(relay);
|
|
38
|
+
|
|
39
|
+
cache.applyRelayedIngest([
|
|
40
|
+
{ table: 'thread', op: 'CREATE', id: 'thread:a', record: { id: 'thread:a', _00_rv: 4 } },
|
|
41
|
+
]);
|
|
42
|
+
|
|
43
|
+
expect(ssp.ingestMany).toHaveBeenCalledTimes(1);
|
|
44
|
+
expect(cache.lookup('thread:a')).toBe(4);
|
|
45
|
+
expect(relay).not.toHaveBeenCalled();
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('localWritesOnly relays the mutation path (save + delete)', async () => {
|
|
49
|
+
const { cache, local, relay } = setup();
|
|
50
|
+
cache.setIngestRelay(relay, { localWritesOnly: true });
|
|
51
|
+
|
|
52
|
+
await cache.save(rec('a', 1), true);
|
|
53
|
+
expect(local.execute).not.toHaveBeenCalled();
|
|
54
|
+
expect(relay).toHaveBeenCalledTimes(1);
|
|
55
|
+
expect(relay.mock.calls[0][0][0]).toMatchObject({ table: 'thread', op: 'CREATE', id: 'thread:a' });
|
|
56
|
+
|
|
57
|
+
const before = { id: 'thread:a', title: 't' };
|
|
58
|
+
await cache.delete('thread', 'thread:a', true, before);
|
|
59
|
+
expect(relay).toHaveBeenCalledTimes(2);
|
|
60
|
+
expect(relay.mock.calls[1][0]).toEqual([
|
|
61
|
+
{ table: 'thread', op: 'DELETE', id: 'thread:a', record: before },
|
|
62
|
+
]);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('localWritesOnly stays silent for sync-fetched batches', async () => {
|
|
66
|
+
const { cache, local, relay } = setup();
|
|
67
|
+
cache.setIngestRelay(relay, { localWritesOnly: true });
|
|
68
|
+
|
|
69
|
+
await cache.saveBatch([rec('a', 1), rec('b', 1)], false);
|
|
70
|
+
expect(local.execute).toHaveBeenCalledTimes(1);
|
|
71
|
+
expect(relay).not.toHaveBeenCalled();
|
|
72
|
+
|
|
73
|
+
await cache.delete('thread', 'thread:a', false, {});
|
|
74
|
+
expect(relay).not.toHaveBeenCalled();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('an unscoped relay fires for every ingest', async () => {
|
|
78
|
+
const { cache, relay } = setup();
|
|
79
|
+
cache.setIngestRelay(relay);
|
|
80
|
+
|
|
81
|
+
await cache.saveBatch([rec('a', 1)], false);
|
|
82
|
+
await cache.save(rec('b', 1), true);
|
|
83
|
+
await cache.delete('thread', 'thread:a', false, {});
|
|
84
|
+
|
|
85
|
+
expect(relay).toHaveBeenCalledTimes(3);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('setIngestRelay(null) stops relaying', async () => {
|
|
89
|
+
const { cache, relay } = setup();
|
|
90
|
+
cache.setIngestRelay(relay, { localWritesOnly: true });
|
|
91
|
+
cache.setIngestRelay(null);
|
|
92
|
+
await cache.save(rec('a', 1), true);
|
|
93
|
+
expect(relay).not.toHaveBeenCalled();
|
|
94
|
+
});
|
|
95
|
+
});
|