@spooky-sync/core 0.0.1-canary.204 → 0.0.1-canary.206
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +172 -44
- package/dist/index.js +636 -189
- package/dist/tabs-broker-worker.js +51 -13
- package/dist/types.d.ts +60 -13
- package/package.json +3 -3
- package/src/modules/cache/index.ts +21 -4
- package/src/modules/data/data.membership.test.ts +62 -1
- package/src/modules/data/data.rebind.test.ts +23 -0
- package/src/modules/data/index.ts +57 -17
- package/src/modules/devtools/index.ts +9 -1
- package/src/modules/sync/engine.ts +76 -58
- package/src/modules/sync/sync.ts +65 -8
- package/src/services/database/cache-engine.ts +33 -0
- package/src/services/database/sqlite-cache-engine.test.ts +58 -0
- package/src/services/database/sqlite-cache-engine.ts +71 -1
- package/src/services/database/surreal-cache-engine.ts +20 -0
- package/src/services/stream-processor/index.ts +471 -225
- package/src/services/stream-processor/stream-processor.batch.test.ts +3 -13
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +77 -67
- package/src/services/stream-processor/wasm-types.ts +32 -1
- package/src/services/tabs/broker.test.ts +49 -0
- package/src/services/tabs/coordinator.test.ts +6 -1
- package/src/services/tabs/coordinator.ts +35 -3
- package/src/services/tabs/tabs-broker-worker.ts +66 -12
- package/src/sp00ky.ts +54 -8
- package/src/types.ts +29 -13
package/dist/index.js
CHANGED
|
@@ -1616,6 +1616,21 @@ var SurrealCacheEngine = class extends LocalDatabaseService {
|
|
|
1616
1616
|
if (typeof id !== "string") return id;
|
|
1617
1617
|
return new RecordId(table, id.startsWith(`${table}:`) ? id.slice(table.length + 1) : id);
|
|
1618
1618
|
}
|
|
1619
|
+
/**
|
|
1620
|
+
* `(id, _00_rv)` of every row per table. Runs on the main thread against the
|
|
1621
|
+
* embedded engine (no worker here), which is why the circuit prime caps how
|
|
1622
|
+
* much it is willing to read through this path.
|
|
1623
|
+
*/
|
|
1624
|
+
async scanVersions(tables) {
|
|
1625
|
+
const out = {};
|
|
1626
|
+
for (const table of tables) try {
|
|
1627
|
+
const [rows] = await this.query(`SELECT id, _00_rv FROM ${table};`);
|
|
1628
|
+
out[table] = (rows ?? []).map((r) => [stableKey(r.id), Number(r._00_rv) || 0]);
|
|
1629
|
+
} catch {
|
|
1630
|
+
out[table] = [];
|
|
1631
|
+
}
|
|
1632
|
+
return out;
|
|
1633
|
+
}
|
|
1619
1634
|
async getById(table, id) {
|
|
1620
1635
|
const [row] = await this.query("SELECT * FROM ONLY $__id;", { __id: this.toRecordId(table, id) });
|
|
1621
1636
|
return row && typeof row === "object" ? row : null;
|
|
@@ -2136,8 +2151,10 @@ var PortSqliteTransport = class extends BaseTransport {
|
|
|
2136
2151
|
* on a fresh bucket throws "no such table". Keep in sync with the systemSchema
|
|
2137
2152
|
* block in `local-migrator.ts`.
|
|
2138
2153
|
*/
|
|
2154
|
+
const SNAPSHOT_TABLE = "_00_circuit_snapshot";
|
|
2139
2155
|
const SYSTEM_TABLES = [
|
|
2140
2156
|
"_00_stream_processor_state",
|
|
2157
|
+
"_00_circuit_snapshot",
|
|
2141
2158
|
"_00_query",
|
|
2142
2159
|
"_00_preload",
|
|
2143
2160
|
"_00_window",
|
|
@@ -2259,7 +2276,7 @@ var SqliteCacheEngine = class {
|
|
|
2259
2276
|
diag.freelistBytes = rows?.[0]?.freelist;
|
|
2260
2277
|
if (opts?.tableCounts) {
|
|
2261
2278
|
const { rows: tables } = await this.call("exec", { sql: "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" });
|
|
2262
|
-
const names = (tables ?? []).map((r) => r.name);
|
|
2279
|
+
const names = (tables ?? []).map((r) => r.name).filter((n) => n !== SNAPSHOT_TABLE);
|
|
2263
2280
|
if (names.length) {
|
|
2264
2281
|
const sql = names.map((n) => `SELECT '${n.replace(/'/g, "''")}' AS t, COUNT(*) AS n FROM "${n.replace(/"/g, "\"\"")}"`).join(" UNION ALL ");
|
|
2265
2282
|
const { rows: counts } = await this.call("exec", { sql });
|
|
@@ -2702,6 +2719,57 @@ var SqliteCacheEngine = class {
|
|
|
2702
2719
|
await this.ensureTable(table);
|
|
2703
2720
|
return (await this.execRows(`SELECT data FROM "${table}" WHERE id = ?`, [stableKey(id)]))[0] ?? null;
|
|
2704
2721
|
}
|
|
2722
|
+
/**
|
|
2723
|
+
* `(id, _00_rv)` of every row per table, straight off the worker: no body
|
|
2724
|
+
* parse, one round trip per table. A table that does not exist yet reads as
|
|
2725
|
+
* empty rather than failing the whole scan.
|
|
2726
|
+
*/
|
|
2727
|
+
async scanVersions(tables) {
|
|
2728
|
+
const out = {};
|
|
2729
|
+
for (const table of tables) try {
|
|
2730
|
+
const { rows } = await this.call("exec", { sql: `SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table.replace(/"/g, "\"\"")}"` });
|
|
2731
|
+
out[table] = (rows ?? []).map((r) => [r.id, Number(r.rv) || 0]);
|
|
2732
|
+
} catch {
|
|
2733
|
+
out[table] = [];
|
|
2734
|
+
}
|
|
2735
|
+
return out;
|
|
2736
|
+
}
|
|
2737
|
+
async getSnapshot(key) {
|
|
2738
|
+
const { rows } = await this.call("exec", {
|
|
2739
|
+
sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
|
|
2740
|
+
bind: [key, `${key}:meta`]
|
|
2741
|
+
});
|
|
2742
|
+
let bytes = null;
|
|
2743
|
+
let meta = null;
|
|
2744
|
+
for (const r of rows ?? []) if (r.id === key) {
|
|
2745
|
+
if (r.data instanceof Uint8Array) bytes = r.data;
|
|
2746
|
+
else if (r.data instanceof ArrayBuffer) bytes = new Uint8Array(r.data);
|
|
2747
|
+
else if (typeof r.data === "string") bytes = new TextEncoder().encode(r.data);
|
|
2748
|
+
} else if (typeof r.data === "string") try {
|
|
2749
|
+
meta = JSON.parse(r.data);
|
|
2750
|
+
} catch {
|
|
2751
|
+
meta = null;
|
|
2752
|
+
}
|
|
2753
|
+
return bytes && meta ? {
|
|
2754
|
+
bytes,
|
|
2755
|
+
meta
|
|
2756
|
+
} : null;
|
|
2757
|
+
}
|
|
2758
|
+
async putSnapshot(key, bytes, meta) {
|
|
2759
|
+
await this.call("batch", [{
|
|
2760
|
+
sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
|
|
2761
|
+
bind: [key, bytes]
|
|
2762
|
+
}, {
|
|
2763
|
+
sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
|
|
2764
|
+
bind: [`${key}:meta`, JSON.stringify(meta)]
|
|
2765
|
+
}]);
|
|
2766
|
+
}
|
|
2767
|
+
async deleteSnapshot(key) {
|
|
2768
|
+
await this.call("run", {
|
|
2769
|
+
sql: `DELETE FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
|
|
2770
|
+
bind: [key, `${key}:meta`]
|
|
2771
|
+
});
|
|
2772
|
+
}
|
|
2705
2773
|
async upsert(table, id, data, mode) {
|
|
2706
2774
|
await this.ensureTable(table);
|
|
2707
2775
|
const key = stableKey(id);
|
|
@@ -3253,12 +3321,6 @@ function phaseStatOf(samples, lastMs) {
|
|
|
3253
3321
|
count: samples.length
|
|
3254
3322
|
};
|
|
3255
3323
|
}
|
|
3256
|
-
/**
|
|
3257
|
-
* DataModule - Unified query and mutation management
|
|
3258
|
-
*
|
|
3259
|
-
* Merges the functionality of QueryManager and MutationManager.
|
|
3260
|
-
* Uses CacheModule for all storage operations.
|
|
3261
|
-
*/
|
|
3262
3324
|
var DataModule = class DataModule {
|
|
3263
3325
|
/** Tab identity baked into mutation ids (shared-tabs rollback routing);
|
|
3264
3326
|
* undefined in solo mode, where mutation-id falls back to a session id. */
|
|
@@ -3962,23 +4024,42 @@ var DataModule = class DataModule {
|
|
|
3962
4024
|
* authoritative membership on this device. Any read error is treated as
|
|
3963
4025
|
* "unknown" so a broken row degrades to the predicate scan rather than
|
|
3964
4026
|
* rendering an empty list.
|
|
4027
|
+
*
|
|
4028
|
+
* `confirmed` is true only for rows written after the server itself vouched
|
|
4029
|
+
* for the set (a non-empty id-set, or an empty one it reported a row count of
|
|
4030
|
+
* zero for, or an empty one that followed a non-empty one in the same
|
|
4031
|
+
* session). Rows written before the marker existed, including the `[]` rows a
|
|
4032
|
+
* pre-`ea56f50e` client mirrored from an unflushed read, read as unconfirmed.
|
|
3965
4033
|
*/
|
|
3966
4034
|
async getWindowMembership(key) {
|
|
3967
4035
|
try {
|
|
3968
4036
|
const row = await this.local.getById("_00_window", new RecordId("_00_window", key));
|
|
3969
4037
|
if (!row || typeof row !== "object") return null;
|
|
3970
4038
|
const ids = row.ids;
|
|
3971
|
-
|
|
4039
|
+
if (!Array.isArray(ids)) return null;
|
|
4040
|
+
return {
|
|
4041
|
+
ids,
|
|
4042
|
+
confirmed: row.confirmed === true
|
|
4043
|
+
};
|
|
3972
4044
|
} catch {
|
|
3973
4045
|
return null;
|
|
3974
4046
|
}
|
|
3975
4047
|
}
|
|
3976
|
-
/**
|
|
3977
|
-
*
|
|
3978
|
-
|
|
4048
|
+
/**
|
|
4049
|
+
* Persist the durable membership row. Best-effort: callers must not fail a
|
|
4050
|
+
* sync round because the mirror write failed.
|
|
4051
|
+
*
|
|
4052
|
+
* `confirmed` says whether a cold start may trust this row even when it is
|
|
4053
|
+
* empty. A confirmed empty is a real answer ("the server says this query has
|
|
4054
|
+
* no rows") and stays empty across a reload; an unconfirmed empty is the
|
|
4055
|
+
* retry budget's guess and falls back to the predicate scan on the next boot,
|
|
4056
|
+
* exactly as every empty row did before the marker existed.
|
|
4057
|
+
*/
|
|
4058
|
+
async writeWindowMembership(key, ids, confirmed) {
|
|
3979
4059
|
try {
|
|
3980
4060
|
await this.local.upsert("_00_window", new RecordId("_00_window", key), {
|
|
3981
4061
|
ids,
|
|
4062
|
+
confirmed,
|
|
3982
4063
|
updatedAt: Date.now()
|
|
3983
4064
|
}, "replace");
|
|
3984
4065
|
} catch (err) {
|
|
@@ -4137,9 +4218,12 @@ var DataModule = class DataModule {
|
|
|
4137
4218
|
}, "Query to update remote array not found");
|
|
4138
4219
|
return;
|
|
4139
4220
|
}
|
|
4221
|
+
let confirmed = remoteArray.length > 0 || queryState.config.remoteSeen === true;
|
|
4140
4222
|
if (remoteArray.length === 0 && !queryState.config.remoteSeen) {
|
|
4141
4223
|
const serverRowCount = opts?.serverRowCount;
|
|
4142
|
-
|
|
4224
|
+
const knownEmpty = serverRowCount === 0;
|
|
4225
|
+
confirmed = knownEmpty;
|
|
4226
|
+
if (!knownEmpty) {
|
|
4143
4227
|
const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
|
|
4144
4228
|
queryState.config.emptyReads = emptyReads;
|
|
4145
4229
|
if (!(serverRowCount === null || serverRowCount === void 0 ? emptyReads >= EMPTY_MEMBERSHIP_CONFIRMATIONS : false)) {
|
|
@@ -4160,7 +4244,7 @@ var DataModule = class DataModule {
|
|
|
4160
4244
|
queryState.config.remoteSeen = true;
|
|
4161
4245
|
queryState.config.emptyReads = 0;
|
|
4162
4246
|
}
|
|
4163
|
-
if (queryState.config.membershipKey) await this.writeWindowMembership(queryState.config.membershipKey, remoteArray);
|
|
4247
|
+
if (queryState.config.membershipKey) await this.writeWindowMembership(queryState.config.membershipKey, remoteArray, confirmed);
|
|
4164
4248
|
try {
|
|
4165
4249
|
await this.local.query(surql.seal(surql.updateSet("id", ["remoteArray"])), {
|
|
4166
4250
|
id: queryState.config.id,
|
|
@@ -4213,8 +4297,8 @@ var DataModule = class DataModule {
|
|
|
4213
4297
|
config.emptyReads = 0;
|
|
4214
4298
|
if (config.membershipKey) {
|
|
4215
4299
|
const durable = await this.getWindowMembership(config.membershipKey);
|
|
4216
|
-
if (durable
|
|
4217
|
-
config.remoteArray = durable;
|
|
4300
|
+
if (durable && (durable.ids.length > 0 || durable.confirmed)) {
|
|
4301
|
+
config.remoteArray = durable.ids;
|
|
4218
4302
|
config.membershipKnown = true;
|
|
4219
4303
|
}
|
|
4220
4304
|
}
|
|
@@ -4644,8 +4728,8 @@ var DataModule = class DataModule {
|
|
|
4644
4728
|
};
|
|
4645
4729
|
if (membershipKey && !config.remoteArray?.length) {
|
|
4646
4730
|
const durable = await this.getWindowMembership(membershipKey);
|
|
4647
|
-
if (durable
|
|
4648
|
-
config.remoteArray = durable;
|
|
4731
|
+
if (durable && (durable.ids.length > 0 || durable.confirmed)) {
|
|
4732
|
+
config.remoteArray = durable.ids;
|
|
4649
4733
|
config.membershipKnown = true;
|
|
4650
4734
|
}
|
|
4651
4735
|
} else if (config.remoteArray?.length) config.membershipKnown = true;
|
|
@@ -5394,6 +5478,8 @@ function recordVersionArraysEqual(a, b) {
|
|
|
5394
5478
|
|
|
5395
5479
|
//#endregion
|
|
5396
5480
|
//#region src/modules/sync/engine.ts
|
|
5481
|
+
/** Ids per remote `SELECT * FROM $ids` round trip. */
|
|
5482
|
+
const FETCH_CHUNK = 500;
|
|
5397
5483
|
/**
|
|
5398
5484
|
* SyncEngine handles the core sync operations: fetching remote records,
|
|
5399
5485
|
* caching them locally, and ingesting into DBSP.
|
|
@@ -5424,51 +5510,59 @@ var SyncEngine = class {
|
|
|
5424
5510
|
}, "SyncEngine.syncRecords diff");
|
|
5425
5511
|
let stillRemoteIds = [];
|
|
5426
5512
|
if (removed.length > 0) stillRemoteIds = await this.handleRemovedRecords(removed);
|
|
5427
|
-
const toFetch = [...added, ...updated]
|
|
5428
|
-
|
|
5429
|
-
|
|
5513
|
+
const toFetch = [...added, ...updated].filter((item) => {
|
|
5514
|
+
const localVersion = this.cache.lookup(encodeRecordId(item.id));
|
|
5515
|
+
return !(localVersion && item.version <= localVersion);
|
|
5516
|
+
});
|
|
5517
|
+
if (toFetch.length === 0) return {
|
|
5430
5518
|
remoteFetchMs: 0,
|
|
5431
5519
|
stillRemoteIds
|
|
5432
5520
|
};
|
|
5433
5521
|
const versionMap = /* @__PURE__ */ new Map();
|
|
5434
5522
|
for (const item of toFetch) versionMap.set(encodeRecordId(item.id), item.version);
|
|
5435
|
-
const
|
|
5436
|
-
|
|
5437
|
-
const
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
|
|
5523
|
+
const addedIds = new Set(added.map((item) => encodeRecordId(item.id)));
|
|
5524
|
+
let remoteFetchMs = 0;
|
|
5525
|
+
const remoteResults = [];
|
|
5526
|
+
for (let i = 0; i < toFetch.length; i += FETCH_CHUNK) {
|
|
5527
|
+
const idsToFetch = toFetch.slice(i, i + FETCH_CHUNK).map((x) => x.id);
|
|
5528
|
+
const remoteFetchStart = performance.now();
|
|
5529
|
+
const [chunkResults] = await this.remote.query("SELECT * FROM $idsToFetch", { idsToFetch });
|
|
5530
|
+
remoteFetchMs += performance.now() - remoteFetchStart;
|
|
5531
|
+
const cacheBatch = [];
|
|
5532
|
+
for (const record of chunkResults ?? []) {
|
|
5533
|
+
if (!record?.id) {
|
|
5534
|
+
this.logger.warn({
|
|
5535
|
+
record,
|
|
5536
|
+
idsToFetch,
|
|
5537
|
+
Category: "sp00ky-client::SyncEngine::syncRecords"
|
|
5538
|
+
}, "Remote record has no id (possibly deleted). Skipping record");
|
|
5539
|
+
continue;
|
|
5540
|
+
}
|
|
5541
|
+
remoteResults.push(record);
|
|
5542
|
+
const fullId = encodeRecordId(record.id);
|
|
5543
|
+
const table = record.id.table.toString();
|
|
5544
|
+
const version = versionMap.get(fullId) ?? 0;
|
|
5545
|
+
const localVersion = this.cache.lookup(fullId);
|
|
5546
|
+
if (localVersion && version <= localVersion) {
|
|
5547
|
+
this.logger.info({
|
|
5548
|
+
recordId: fullId,
|
|
5549
|
+
version,
|
|
5550
|
+
localVersion,
|
|
5551
|
+
Category: "sp00ky-client::SyncEngine::syncRecords"
|
|
5552
|
+
}, "Local version is higher than remote version. Skipping record");
|
|
5553
|
+
continue;
|
|
5554
|
+
}
|
|
5555
|
+
const tableSchema = this.schema.tables.find((t) => t.name === table);
|
|
5556
|
+
const cleanedRecord = tableSchema ? cleanRecord(tableSchema.columns, record) : record;
|
|
5557
|
+
cacheBatch.push({
|
|
5558
|
+
table,
|
|
5559
|
+
op: addedIds.has(fullId) ? "CREATE" : "UPDATE",
|
|
5560
|
+
record: cleanedRecord,
|
|
5561
|
+
version
|
|
5562
|
+
});
|
|
5461
5563
|
}
|
|
5462
|
-
|
|
5463
|
-
const cleanedRecord = tableSchema ? cleanRecord(tableSchema.columns, record) : record;
|
|
5464
|
-
cacheBatch.push({
|
|
5465
|
-
table,
|
|
5466
|
-
op: isAdded ? "CREATE" : "UPDATE",
|
|
5467
|
-
record: cleanedRecord,
|
|
5468
|
-
version
|
|
5469
|
-
});
|
|
5564
|
+
if (cacheBatch.length > 0) await this.cache.saveBatch(cacheBatch);
|
|
5470
5565
|
}
|
|
5471
|
-
if (cacheBatch.length > 0) await this.cache.saveBatch(cacheBatch);
|
|
5472
5566
|
this.events.emit(SyncEventTypes.RemoteDataIngested, { records: remoteResults });
|
|
5473
5567
|
return {
|
|
5474
5568
|
remoteFetchMs,
|
|
@@ -6005,14 +6099,37 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6005
6099
|
this.tabRole = role;
|
|
6006
6100
|
this.tabId = tabId;
|
|
6007
6101
|
}
|
|
6008
|
-
/**
|
|
6009
|
-
*
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6102
|
+
/** In-flight {@link resumeLeaderDuties}, so a second call joins the first
|
|
6103
|
+
* instead of double-draining the outbox. */
|
|
6104
|
+
leaderDutiesInFlight = null;
|
|
6105
|
+
/** Resolves once the in-browser circuit has been primed from the local
|
|
6106
|
+
* store. Every sync diff waits on it: diffing against an empty circuit
|
|
6107
|
+
* classifies the whole working set as missing and re-downloads it. */
|
|
6108
|
+
primeGate = () => Promise.resolve();
|
|
6109
|
+
/** The prime we last waited on. `whenPrimed` hands out one promise per
|
|
6110
|
+
* prime, so a new identity means a new prime (boot, bucket switch) ran. */
|
|
6111
|
+
settledPrime = null;
|
|
6112
|
+
setPrimeGate(gate) {
|
|
6113
|
+
this.primeGate = gate;
|
|
6114
|
+
this.settledPrime = null;
|
|
6115
|
+
}
|
|
6116
|
+
/**
|
|
6117
|
+
* Leader WIRING only, and deliberately synchronous.
|
|
6118
|
+
*
|
|
6119
|
+
* The coordinator publishes the leader role and tells the broker
|
|
6120
|
+
* `leader-ready` the moment the store is adopted, and the broker can mint a
|
|
6121
|
+
* follower's ports on the very next tick. So the follower-message handler
|
|
6122
|
+
* has to be live before this returns, or a mutation forwarded in that window
|
|
6123
|
+
* is dropped. Everything that can block (outbox reload, LIVE restart) moved
|
|
6124
|
+
* to {@link resumeLeaderDuties}: a promotion that waits on the network holds
|
|
6125
|
+
* `leader-ready` back, and a broker whose leader never reports ready serves
|
|
6126
|
+
* no follower ports and re-elects no one, which wedges the whole namespace.
|
|
6127
|
+
*/
|
|
6128
|
+
promoteToLeader(hub) {
|
|
6013
6129
|
this.tabRole = "leader";
|
|
6014
6130
|
this.hub = hub;
|
|
6015
6131
|
this.forwarder = null;
|
|
6132
|
+
this.leaderDutiesInFlight = null;
|
|
6016
6133
|
hub.onFollowerMessage = (tabId, msg) => {
|
|
6017
6134
|
switch (msg.type) {
|
|
6018
6135
|
case "sync-hello": break;
|
|
@@ -6024,7 +6141,17 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6024
6141
|
break;
|
|
6025
6142
|
}
|
|
6026
6143
|
};
|
|
6027
|
-
|
|
6144
|
+
}
|
|
6145
|
+
/** Leader duties: drain the shared outbox, own the single list_ref LIVE,
|
|
6146
|
+
* relay LIVE events and rollbacks to followers. Idempotent for a boot-time
|
|
6147
|
+
* leader; a runtime promotion (failover) reloads the outbox, which now
|
|
6148
|
+
* holds EVERY tab's rows, and restarts LIVE under this session. Runs in the
|
|
6149
|
+
* background off the promotion path, so however long it takes (or if it
|
|
6150
|
+
* never finishes) the tab is already a working leader. */
|
|
6151
|
+
resumeLeaderDuties() {
|
|
6152
|
+
if (this.leaderDutiesInFlight) return this.leaderDutiesInFlight;
|
|
6153
|
+
const run = (async () => {
|
|
6154
|
+
if (!this.isInit || this.tabRole !== "leader") return;
|
|
6028
6155
|
await this.upQueue.loadFromDatabase();
|
|
6029
6156
|
this.scheduler.syncUp();
|
|
6030
6157
|
if (this.currentUserId || this.anonLiveEnabled) {
|
|
@@ -6032,11 +6159,15 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6032
6159
|
await this.restartRefLiveQuery().catch((err) => {
|
|
6033
6160
|
this.logger.warn({
|
|
6034
6161
|
err,
|
|
6035
|
-
Category: "sp00ky-client::Sp00kySync::
|
|
6162
|
+
Category: "sp00ky-client::Sp00kySync::resumeLeaderDuties"
|
|
6036
6163
|
}, "LIVE restart failed on promotion; poll fallback covers it");
|
|
6037
6164
|
});
|
|
6038
6165
|
}
|
|
6039
|
-
}
|
|
6166
|
+
})();
|
|
6167
|
+
this.leaderDutiesInFlight = run;
|
|
6168
|
+
return run.finally(() => {
|
|
6169
|
+
if (this.leaderDutiesInFlight === run) this.leaderDutiesInFlight = null;
|
|
6170
|
+
});
|
|
6040
6171
|
}
|
|
6041
6172
|
/** Follower duties: no outbox drain, no LIVE. Mutations forward to the
|
|
6042
6173
|
* leader; everything else (registration, per-query sync, poll) runs
|
|
@@ -6045,6 +6176,7 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6045
6176
|
this.tabRole = "follower";
|
|
6046
6177
|
this.hub = null;
|
|
6047
6178
|
this.forwarder = forwarder;
|
|
6179
|
+
this.leaderDutiesInFlight = null;
|
|
6048
6180
|
this.killRefLiveQuery();
|
|
6049
6181
|
forwarder.onLeaderMessage = (msg) => {
|
|
6050
6182
|
switch (msg.type) {
|
|
@@ -6694,6 +6826,16 @@ var Sp00kySync = class Sp00kySync {
|
|
|
6694
6826
|
* means the single resulting UI update lands after this completes.
|
|
6695
6827
|
*/
|
|
6696
6828
|
async runSyncForQuery(hash, diff) {
|
|
6829
|
+
const prime = this.primeGate();
|
|
6830
|
+
if (prime !== this.settledPrime) {
|
|
6831
|
+
await prime;
|
|
6832
|
+
this.settledPrime = prime;
|
|
6833
|
+
const fresh = this.dataModule.getQueryByHash(hash);
|
|
6834
|
+
if (!fresh) return;
|
|
6835
|
+
const recomputed = new ArraySyncer(fresh.config.localArray, fresh.config.remoteArray).nextSet();
|
|
6836
|
+
if (!recomputed) return;
|
|
6837
|
+
diff = recomputed;
|
|
6838
|
+
}
|
|
6697
6839
|
if (diff.added.length > 0 || diff.updated.length > 0) {
|
|
6698
6840
|
const pendingDeletes = await this.getPendingDeleteIds();
|
|
6699
6841
|
if (pendingDeletes.size > 0) diff = {
|
|
@@ -7211,8 +7353,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
7211
7353
|
|
|
7212
7354
|
//#endregion
|
|
7213
7355
|
//#region src/modules/devtools/index.ts
|
|
7214
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
7215
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
7356
|
+
const CORE_VERSION = "0.0.1-canary.206";
|
|
7357
|
+
const WASM_VERSION = "0.0.1-canary.206";
|
|
7216
7358
|
const SURREAL_VERSION = "3.0.3";
|
|
7217
7359
|
var DevToolsService = class DevToolsService {
|
|
7218
7360
|
eventsHistory = [];
|
|
@@ -7313,6 +7455,9 @@ var DevToolsService = class DevToolsService {
|
|
|
7313
7455
|
data: q.records,
|
|
7314
7456
|
localArray: q.config.localArray,
|
|
7315
7457
|
remoteArray: q.config.remoteArray,
|
|
7458
|
+
membershipKnown: q.config.membershipKnown === true,
|
|
7459
|
+
remoteSeen: q.config.remoteSeen === true,
|
|
7460
|
+
emptyReads: q.config.emptyReads ?? 0,
|
|
7316
7461
|
timings: this.dataManager.phaseTimings(q)
|
|
7317
7462
|
});
|
|
7318
7463
|
});
|
|
@@ -7406,7 +7551,7 @@ var DevToolsService = class DevToolsService {
|
|
|
7406
7551
|
const info = this.unwrapInfo(res);
|
|
7407
7552
|
this.localTablesAt = Date.now();
|
|
7408
7553
|
if (info && info.tables) {
|
|
7409
|
-
const names = Object.keys(info.tables);
|
|
7554
|
+
const names = Object.keys(info.tables).filter((n) => n !== "_00_circuit_snapshot");
|
|
7410
7555
|
const changed = names.length !== this.localTables.length || names.some((n, i) => n !== this.localTables[i]);
|
|
7411
7556
|
this.localTables = names;
|
|
7412
7557
|
if (changed) this.notifyDevTools();
|
|
@@ -7936,13 +8081,29 @@ var AuthService = class {
|
|
|
7936
8081
|
|
|
7937
8082
|
//#endregion
|
|
7938
8083
|
//#region src/services/stream-processor/index.ts
|
|
7939
|
-
/**
|
|
7940
|
-
|
|
7941
|
-
|
|
7942
|
-
|
|
7943
|
-
|
|
7944
|
-
|
|
7945
|
-
|
|
8084
|
+
/** Storage key of the circuit snapshot inside the local store. */
|
|
8085
|
+
const CIRCUIT_SNAPSHOT_KEY = "circuit";
|
|
8086
|
+
/** Bump when the bytes `load_store_state` reads change shape. */
|
|
8087
|
+
const CIRCUIT_SNAPSHOT_FORMAT = 1;
|
|
8088
|
+
/** Rows per `ingest_many` call. Bounds the transient the wasm side allocates
|
|
8089
|
+
* to parse a batch (measured: one 7700-row call of 20 KB bodies peaked at
|
|
8090
|
+
* 500 MB, 128-row chunks at 335 MB, and under projection at 24 MB). */
|
|
8091
|
+
const INGEST_CHUNK = 128;
|
|
8092
|
+
/** Ids per `selectByIds` when priming bodies out of the local store. */
|
|
8093
|
+
const PRIME_CHUNK = 256;
|
|
8094
|
+
/** Rows the SurrealDB (main-thread, IndexedDB) engine is allowed to prime
|
|
8095
|
+
* through; above this the circuit boots empty as it always did. */
|
|
8096
|
+
const SURREAL_PRIME_ROW_CAP = 2e4;
|
|
8097
|
+
/** Below this many changed rows the checkpoint timer stays quiet. */
|
|
8098
|
+
const CHECKPOINT_MIN_ROWS = 50;
|
|
8099
|
+
/** Stable `table:id` for a row id that may already be a string. */
|
|
8100
|
+
function idString(id) {
|
|
8101
|
+
return typeof id === "string" ? id : encodeRecordId(id);
|
|
8102
|
+
}
|
|
8103
|
+
function chunks(items, size) {
|
|
8104
|
+
const out = [];
|
|
8105
|
+
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
|
8106
|
+
return out;
|
|
7946
8107
|
}
|
|
7947
8108
|
var StreamProcessorService = class {
|
|
7948
8109
|
logger;
|
|
@@ -7955,18 +8116,23 @@ var StreamProcessorService = class {
|
|
|
7955
8116
|
authId: "",
|
|
7956
8117
|
access: ""
|
|
7957
8118
|
};
|
|
7958
|
-
stateKeySuffix = "";
|
|
7959
8119
|
stateGeneration = 0;
|
|
7960
8120
|
persistState = true;
|
|
7961
8121
|
persistCircuit = false;
|
|
7962
8122
|
checkpointMs = 3e4;
|
|
7963
8123
|
checkpointTimer = null;
|
|
7964
8124
|
snapshotDirty = false;
|
|
7965
|
-
|
|
7966
|
-
|
|
8125
|
+
dirtyRows = 0;
|
|
8126
|
+
hideHandler = null;
|
|
8127
|
+
checkpointInFlight = null;
|
|
8128
|
+
projection = true;
|
|
8129
|
+
primed = Promise.resolve();
|
|
8130
|
+
schemaHash = null;
|
|
8131
|
+
widenQueue = Promise.resolve();
|
|
8132
|
+
widenPending = /* @__PURE__ */ new Map();
|
|
8133
|
+
constructor(events, db, logger) {
|
|
7967
8134
|
this.events = events;
|
|
7968
8135
|
this.db = db;
|
|
7969
|
-
this.persistenceClient = persistenceClient;
|
|
7970
8136
|
this.logger = logger.child({ name: "StreamProcessorService" });
|
|
7971
8137
|
}
|
|
7972
8138
|
/**
|
|
@@ -7998,31 +8164,41 @@ var StreamProcessorService = class {
|
|
|
7998
8164
|
for (const update of updates) for (const receiver of this.receivers) receiver.onStreamUpdate(update);
|
|
7999
8165
|
}
|
|
8000
8166
|
/**
|
|
8001
|
-
* Ingest a batch of record changes
|
|
8002
|
-
*
|
|
8003
|
-
*
|
|
8004
|
-
*
|
|
8005
|
-
*
|
|
8006
|
-
*
|
|
8167
|
+
* Ingest a batch of record changes, firing one coalesced `StreamUpdate` per
|
|
8168
|
+
* affected query once every record has been ingested. Use this whenever
|
|
8169
|
+
* multiple records land at once (sync fetching N rows, the boot prime).
|
|
8170
|
+
*
|
|
8171
|
+
* The batch is fed to the wasm side in chunks of {@link INGEST_CHUNK}: one
|
|
8172
|
+
* circuit step per chunk (a step walks every registered view, so per-record
|
|
8173
|
+
* ingest paid that fixed cost N times), but never the whole batch at once,
|
|
8174
|
+
* because the wasm side has to hold every parsed row of a call at the same
|
|
8175
|
+
* time and wasm32 dlmalloc never returns that peak.
|
|
8007
8176
|
*
|
|
8008
|
-
*
|
|
8009
|
-
*
|
|
8010
|
-
*
|
|
8177
|
+
* Returns the records that were ingested. A chunk that fails is reported
|
|
8178
|
+
* and skipped, not retried (a retry would double-apply whatever the failed
|
|
8179
|
+
* step already committed), and the remaining chunks still run.
|
|
8011
8180
|
*/
|
|
8012
8181
|
ingestMany(records) {
|
|
8013
|
-
if (records.length === 0) return;
|
|
8182
|
+
if (records.length === 0) return [];
|
|
8014
8183
|
if (!this.processor) {
|
|
8015
8184
|
this.logger.warn({ Category: "sp00ky-client::StreamProcessorService::ingestMany" }, "Not initialized, skipping ingest");
|
|
8016
|
-
return;
|
|
8185
|
+
return [];
|
|
8017
8186
|
}
|
|
8018
8187
|
const bulkIngest = this.processor.ingest_many;
|
|
8019
|
-
if (typeof bulkIngest
|
|
8020
|
-
this.
|
|
8021
|
-
count: records.length,
|
|
8022
|
-
Category: "sp00ky-client::StreamProcessorService::ingestMany"
|
|
8023
|
-
}, "Ingesting batch into ssp");
|
|
8188
|
+
if (typeof bulkIngest !== "function") {
|
|
8189
|
+
this.beginCoalescing();
|
|
8024
8190
|
try {
|
|
8025
|
-
const
|
|
8191
|
+
for (const record of records) this.ingest(record.table, record.op, record.id, record.record);
|
|
8192
|
+
} finally {
|
|
8193
|
+
this.flushCoalescing();
|
|
8194
|
+
}
|
|
8195
|
+
return records;
|
|
8196
|
+
}
|
|
8197
|
+
const ingested = [];
|
|
8198
|
+
this.beginCoalescing();
|
|
8199
|
+
try {
|
|
8200
|
+
for (const chunk of chunks(records, INGEST_CHUNK)) try {
|
|
8201
|
+
const items = chunk.map((record) => ({
|
|
8026
8202
|
table: record.table,
|
|
8027
8203
|
op: record.op,
|
|
8028
8204
|
id: record.id,
|
|
@@ -8031,13 +8207,7 @@ var StreamProcessorService = class {
|
|
|
8031
8207
|
const t0 = performance.now();
|
|
8032
8208
|
const rawUpdates = bulkIngest.call(this.processor, items) ?? [];
|
|
8033
8209
|
const materializationTimeMs = performance.now() - t0;
|
|
8034
|
-
this.
|
|
8035
|
-
count: records.length,
|
|
8036
|
-
rawUpdates: rawUpdates.length,
|
|
8037
|
-
materializationTimeMs,
|
|
8038
|
-
Category: "sp00ky-client::StreamProcessorService::ingestMany"
|
|
8039
|
-
}, "Ingesting batch into ssp done");
|
|
8040
|
-
if (rawUpdates.length > 0) this.dispatchUpdates(rawUpdates.map((u) => ({
|
|
8210
|
+
if (rawUpdates.length > 0) this.notifyUpdates(rawUpdates.map((u) => ({
|
|
8041
8211
|
queryHash: u.query_id,
|
|
8042
8212
|
localArray: u.result_data,
|
|
8043
8213
|
op: "CREATE",
|
|
@@ -8046,22 +8216,24 @@ var StreamProcessorService = class {
|
|
|
8046
8216
|
circuitStepMs: u.timing_circuit_step_ms,
|
|
8047
8217
|
transformMs: u.timing_transform_ms
|
|
8048
8218
|
})));
|
|
8219
|
+
ingested.push(...chunk);
|
|
8049
8220
|
} catch (e) {
|
|
8050
8221
|
this.logger.error({
|
|
8051
8222
|
error: e,
|
|
8052
|
-
count:
|
|
8223
|
+
count: chunk.length,
|
|
8053
8224
|
Category: "sp00ky-client::StreamProcessorService::ingestMany"
|
|
8054
|
-
}, "Ingesting
|
|
8225
|
+
}, "Ingesting chunk into ssp failed");
|
|
8055
8226
|
}
|
|
8056
|
-
this.markSnapshotDirty();
|
|
8057
|
-
return;
|
|
8058
|
-
}
|
|
8059
|
-
this.beginCoalescing();
|
|
8060
|
-
try {
|
|
8061
|
-
for (const record of records) this.ingest(record.table, record.op, record.id, record.record);
|
|
8062
8227
|
} finally {
|
|
8063
8228
|
this.flushCoalescing();
|
|
8064
8229
|
}
|
|
8230
|
+
this.logger.debug({
|
|
8231
|
+
count: records.length,
|
|
8232
|
+
ingested: ingested.length,
|
|
8233
|
+
Category: "sp00ky-client::StreamProcessorService::ingestMany"
|
|
8234
|
+
}, "Ingested batch into ssp");
|
|
8235
|
+
this.markSnapshotDirty(ingested.length);
|
|
8236
|
+
return ingested;
|
|
8065
8237
|
}
|
|
8066
8238
|
/**
|
|
8067
8239
|
* Open a coalescing window. While open, the per-record stream updates
|
|
@@ -8088,7 +8260,6 @@ var StreamProcessorService = class {
|
|
|
8088
8260
|
const buffered = Array.from(this.batchBuffer.values());
|
|
8089
8261
|
this.batchBuffer.clear();
|
|
8090
8262
|
if (buffered.length > 0) this.dispatchUpdates(buffered);
|
|
8091
|
-
this.markSnapshotDirty();
|
|
8092
8263
|
}
|
|
8093
8264
|
/**
|
|
8094
8265
|
* Initialize the WASM module and processor.
|
|
@@ -8100,7 +8271,7 @@ var StreamProcessorService = class {
|
|
|
8100
8271
|
try {
|
|
8101
8272
|
await init();
|
|
8102
8273
|
this.processor = new Sp00kyProcessor();
|
|
8103
|
-
|
|
8274
|
+
this.applyProjection();
|
|
8104
8275
|
this.isInitialized = true;
|
|
8105
8276
|
this.logger.info({ Category: "sp00ky-client::StreamProcessorService::init" }, "Initialized successfully");
|
|
8106
8277
|
} catch (e) {
|
|
@@ -8111,21 +8282,16 @@ var StreamProcessorService = class {
|
|
|
8111
8282
|
throw e;
|
|
8112
8283
|
}
|
|
8113
8284
|
}
|
|
8114
|
-
/** Route the persisted circuit snapshot to a per-bucket key. */
|
|
8115
|
-
setStateKeySuffix(bucketId) {
|
|
8116
|
-
this.stateKeySuffix = bucketId;
|
|
8117
|
-
}
|
|
8118
|
-
stateKey() {
|
|
8119
|
-
return this.stateKeySuffix ? `_00_stream_processor_state:${this.stateKeySuffix}` : "_00_stream_processor_state";
|
|
8120
|
-
}
|
|
8121
8285
|
/**
|
|
8122
8286
|
* Drop the current WASM processor and start a fresh, empty circuit. Used on
|
|
8123
8287
|
* local-bucket switches: the old circuit holds the previous user's rows AND
|
|
8124
8288
|
* views registered with the previous `$auth` context, so neither may survive.
|
|
8125
|
-
* Deliberately
|
|
8126
|
-
*
|
|
8127
|
-
*
|
|
8128
|
-
*
|
|
8289
|
+
* Deliberately loads nothing: the snapshot in the store being swapped away
|
|
8290
|
+
* from belongs to the previous bucket; the caller primes the new bucket's
|
|
8291
|
+
* circuit (`primeFromLocal`) once its store is open, and the DataModule
|
|
8292
|
+
* rebind re-registers every live view against this fresh processor. Caller
|
|
8293
|
+
* must re-seed `setPermissions` afterwards (a fresh circuit default-denies
|
|
8294
|
+
* every table).
|
|
8129
8295
|
*/
|
|
8130
8296
|
async reset() {
|
|
8131
8297
|
if (!this.isInitialized) return;
|
|
@@ -8133,8 +8299,11 @@ var StreamProcessorService = class {
|
|
|
8133
8299
|
this.batching = false;
|
|
8134
8300
|
this.batchBuffer.clear();
|
|
8135
8301
|
this.snapshotDirty = false;
|
|
8302
|
+
this.dirtyRows = 0;
|
|
8303
|
+
this.widenPending.clear();
|
|
8136
8304
|
const previous = this.processor;
|
|
8137
8305
|
this.processor = new Sp00kyProcessor();
|
|
8306
|
+
this.applyProjection();
|
|
8138
8307
|
this.freeProcessor(previous);
|
|
8139
8308
|
this.logger.info({ Category: "sp00ky-client::StreamProcessorService::reset" }, "Stream processor reset (fresh circuit)");
|
|
8140
8309
|
}
|
|
@@ -8174,9 +8343,9 @@ var StreamProcessorService = class {
|
|
|
8174
8343
|
if (!enabled) this.stopCheckpoints();
|
|
8175
8344
|
}
|
|
8176
8345
|
/**
|
|
8177
|
-
*
|
|
8178
|
-
*
|
|
8179
|
-
*
|
|
8346
|
+
* Snapshot persistence (`persistCircuit`). When on, the circuit's store is
|
|
8347
|
+
* written to the local store on a checkpoint interval and when the page
|
|
8348
|
+
* goes hidden, and restored by {@link primeFromLocal} on the next boot.
|
|
8180
8349
|
*/
|
|
8181
8350
|
configureCircuitPersistence(enabled, checkpointMs) {
|
|
8182
8351
|
this.persistCircuit = enabled;
|
|
@@ -8184,59 +8353,291 @@ var StreamProcessorService = class {
|
|
|
8184
8353
|
if (!enabled) this.stopCheckpoints();
|
|
8185
8354
|
}
|
|
8186
8355
|
/**
|
|
8187
|
-
*
|
|
8188
|
-
*
|
|
8189
|
-
* `persistCircuit` is off (the default).
|
|
8356
|
+
* Field projection (`circuitProjection`, default on). Takes effect on the
|
|
8357
|
+
* next processor (`init`/`reset`) and on rows written after that.
|
|
8190
8358
|
*/
|
|
8191
|
-
|
|
8359
|
+
configureProjection(enabled) {
|
|
8360
|
+
this.projection = enabled;
|
|
8361
|
+
this.applyProjection();
|
|
8362
|
+
}
|
|
8363
|
+
applyProjection() {
|
|
8364
|
+
if (!this.processor || typeof this.processor.set_projection !== "function") return;
|
|
8365
|
+
try {
|
|
8366
|
+
this.processor.set_projection(this.projection);
|
|
8367
|
+
} catch (e) {
|
|
8368
|
+
this.logger.warn({
|
|
8369
|
+
error: e,
|
|
8370
|
+
Category: "sp00ky-client::StreamProcessorService::applyProjection"
|
|
8371
|
+
}, "set_projection failed");
|
|
8372
|
+
}
|
|
8373
|
+
}
|
|
8374
|
+
/** Resolves once the boot-time prime has finished (or was skipped). */
|
|
8375
|
+
whenPrimed() {
|
|
8376
|
+
return this.primed;
|
|
8377
|
+
}
|
|
8378
|
+
/**
|
|
8379
|
+
* Fill the circuit from the LOCAL store, in the background.
|
|
8380
|
+
*
|
|
8381
|
+
* With a usable snapshot: install it under whatever views have registered
|
|
8382
|
+
* meanwhile (`load_store_state` re-primes them), then `reconcile` each table
|
|
8383
|
+
* against the store's `(id, rv)` list so rows deleted since the checkpoint
|
|
8384
|
+
* are stepped out and only rows added or changed since are read back and
|
|
8385
|
+
* ingested. Without one: read every row and ingest it, chunked.
|
|
8386
|
+
*
|
|
8387
|
+
* Either way the circuit ends up equal to the local store without touching
|
|
8388
|
+
* the network, so the first sync diff is a real delta rather than "fetch
|
|
8389
|
+
* everything". The returned promise never rejects; `whenPrimed` gates on it.
|
|
8390
|
+
*/
|
|
8391
|
+
primeFromLocal(ctx) {
|
|
8392
|
+
this.schemaHash = ctx.schemaHash;
|
|
8393
|
+
const run = this.runPrime(ctx).catch((e) => {
|
|
8394
|
+
this.logger.warn({
|
|
8395
|
+
error: e,
|
|
8396
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
8397
|
+
}, "Circuit prime failed; the circuit fills from sync instead");
|
|
8398
|
+
});
|
|
8399
|
+
this.primed = run;
|
|
8400
|
+
return run;
|
|
8401
|
+
}
|
|
8402
|
+
async runPrime(ctx) {
|
|
8403
|
+
const processor = this.processor;
|
|
8404
|
+
if (!processor || typeof this.db.scanVersions !== "function") return;
|
|
8405
|
+
const generation = this.stateGeneration;
|
|
8406
|
+
const epoch = this.db.epoch;
|
|
8407
|
+
const alive = () => this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
|
|
8408
|
+
const t0 = performance.now();
|
|
8409
|
+
let restored = false;
|
|
8410
|
+
if (this.persistCircuit && typeof this.db.getSnapshot === "function" && typeof processor.load_store_state === "function") {
|
|
8411
|
+
const snapshot = await this.db.getSnapshot(CIRCUIT_SNAPSHOT_KEY);
|
|
8412
|
+
if (!alive()) return;
|
|
8413
|
+
if (snapshot) {
|
|
8414
|
+
const { meta } = snapshot;
|
|
8415
|
+
if (meta.formatVersion !== CIRCUIT_SNAPSHOT_FORMAT || meta.schemaHash !== ctx.schemaHash) this.logger.info({
|
|
8416
|
+
meta,
|
|
8417
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
8418
|
+
}, "Circuit snapshot is from another format or schema; priming from rows");
|
|
8419
|
+
else try {
|
|
8420
|
+
const tLoad = performance.now();
|
|
8421
|
+
const updates = processor.load_store_state(snapshot.bytes);
|
|
8422
|
+
this.dispatchWasmUpdates(updates);
|
|
8423
|
+
restored = true;
|
|
8424
|
+
this.logger.info({
|
|
8425
|
+
bytes: snapshot.bytes.byteLength,
|
|
8426
|
+
views: updates.length,
|
|
8427
|
+
loadMs: Math.round(performance.now() - tLoad),
|
|
8428
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
8429
|
+
}, "Circuit snapshot restored");
|
|
8430
|
+
} catch (e) {
|
|
8431
|
+
this.logger.warn({
|
|
8432
|
+
error: e,
|
|
8433
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
8434
|
+
}, "Circuit snapshot unreadable; priming from rows");
|
|
8435
|
+
}
|
|
8436
|
+
}
|
|
8437
|
+
}
|
|
8438
|
+
const versions = await this.db.scanVersions(ctx.tables);
|
|
8439
|
+
if (!alive()) return;
|
|
8440
|
+
const total = Object.values(versions).reduce((n, v) => n + v.length, 0);
|
|
8441
|
+
if (!restored && this.db.engineKind !== "sqlite" && total > SURREAL_PRIME_ROW_CAP) {
|
|
8442
|
+
this.logger.warn({
|
|
8443
|
+
total,
|
|
8444
|
+
cap: SURREAL_PRIME_ROW_CAP,
|
|
8445
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
8446
|
+
}, "Too many cached rows to prime through the main-thread engine; circuit fills from sync");
|
|
8447
|
+
return;
|
|
8448
|
+
}
|
|
8449
|
+
let ingested = 0;
|
|
8450
|
+
let fetched = 0;
|
|
8451
|
+
let deleted = 0;
|
|
8452
|
+
for (const table of ctx.tables) {
|
|
8453
|
+
const entries = versions[table] ?? [];
|
|
8454
|
+
let toFetch;
|
|
8455
|
+
if (restored && typeof processor.reconcile === "function") {
|
|
8456
|
+
const result = processor.reconcile(table, entries);
|
|
8457
|
+
this.dispatchWasmUpdates(result.updates);
|
|
8458
|
+
deleted += result.deleted;
|
|
8459
|
+
toFetch = result.fetch;
|
|
8460
|
+
} else toFetch = entries.map(([id]) => id);
|
|
8461
|
+
for (const chunk of chunks(toFetch, PRIME_CHUNK)) {
|
|
8462
|
+
const rows = await this.db.selectByIds(table, chunk);
|
|
8463
|
+
if (!alive()) return;
|
|
8464
|
+
fetched += rows.length;
|
|
8465
|
+
ingested += this.ingestMany(rows.map((row) => ({
|
|
8466
|
+
table,
|
|
8467
|
+
op: "CREATE",
|
|
8468
|
+
id: idString(row.id),
|
|
8469
|
+
record: row
|
|
8470
|
+
}))).length;
|
|
8471
|
+
}
|
|
8472
|
+
if (entries.length > 0) ctx.onVersions?.(table, entries.filter(([id]) => !ctx.pendingIds.has(id)));
|
|
8473
|
+
}
|
|
8474
|
+
if (!restored && ingested > 0) this.markSnapshotDirty(Math.max(ingested, CHECKPOINT_MIN_ROWS));
|
|
8475
|
+
this.logger.info({
|
|
8476
|
+
restored,
|
|
8477
|
+
rows: total,
|
|
8478
|
+
fetched,
|
|
8479
|
+
ingested,
|
|
8480
|
+
deleted,
|
|
8481
|
+
ms: Math.round(performance.now() - t0),
|
|
8482
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
8483
|
+
}, "Circuit primed from the local store");
|
|
8484
|
+
}
|
|
8485
|
+
/** Publish wasm updates produced outside an ingest (restore, reconcile). */
|
|
8486
|
+
dispatchWasmUpdates(updates) {
|
|
8487
|
+
if (!updates || updates.length === 0) return;
|
|
8488
|
+
this.dispatchUpdates(updates.map((u) => ({
|
|
8489
|
+
queryHash: u.query_id,
|
|
8490
|
+
localArray: u.result_data,
|
|
8491
|
+
op: "CREATE"
|
|
8492
|
+
})));
|
|
8493
|
+
}
|
|
8494
|
+
/**
|
|
8495
|
+
* Record that the circuit changed by `rows` rows. Cheap; the snapshot is
|
|
8496
|
+
* deferred to the checkpoint timer and skipped entirely when `persistCircuit`
|
|
8497
|
+
* is off.
|
|
8498
|
+
*/
|
|
8499
|
+
markSnapshotDirty(rows = 1) {
|
|
8192
8500
|
if (!this.persistCircuit || !this.persistState) return;
|
|
8193
8501
|
this.snapshotDirty = true;
|
|
8502
|
+
this.dirtyRows += rows;
|
|
8194
8503
|
this.startCheckpoints();
|
|
8195
8504
|
}
|
|
8196
8505
|
startCheckpoints() {
|
|
8197
8506
|
if (this.checkpointTimer) return;
|
|
8198
8507
|
this.checkpointTimer = setInterval(() => {
|
|
8199
|
-
if (!this.snapshotDirty) return;
|
|
8200
|
-
this.
|
|
8201
|
-
this.saveState();
|
|
8508
|
+
if (!this.snapshotDirty || this.dirtyRows < CHECKPOINT_MIN_ROWS) return;
|
|
8509
|
+
this.checkpoint("interval");
|
|
8202
8510
|
}, this.checkpointMs);
|
|
8203
|
-
if (typeof window !== "undefined" && !this.
|
|
8204
|
-
this.
|
|
8511
|
+
if (typeof window !== "undefined" && !this.hideHandler) {
|
|
8512
|
+
this.hideHandler = () => {
|
|
8205
8513
|
if (!this.snapshotDirty) return;
|
|
8206
|
-
|
|
8207
|
-
this.
|
|
8514
|
+
if (typeof document !== "undefined" && document.visibilityState === "visible") return;
|
|
8515
|
+
this.checkpoint("hidden");
|
|
8208
8516
|
};
|
|
8209
|
-
window.addEventListener("
|
|
8517
|
+
window.addEventListener("visibilitychange", this.hideHandler);
|
|
8518
|
+
window.addEventListener("pagehide", this.hideHandler);
|
|
8210
8519
|
}
|
|
8211
8520
|
}
|
|
8212
|
-
/** Stop checkpointing and drop the
|
|
8521
|
+
/** Stop checkpointing and drop the visibility listeners. */
|
|
8213
8522
|
stopCheckpoints() {
|
|
8214
8523
|
if (this.checkpointTimer) {
|
|
8215
8524
|
clearInterval(this.checkpointTimer);
|
|
8216
8525
|
this.checkpointTimer = null;
|
|
8217
8526
|
}
|
|
8218
|
-
if (this.
|
|
8219
|
-
|
|
8527
|
+
if (this.hideHandler && typeof window !== "undefined") {
|
|
8528
|
+
window.removeEventListener("visibilitychange", this.hideHandler);
|
|
8529
|
+
window.removeEventListener("pagehide", this.hideHandler);
|
|
8530
|
+
}
|
|
8531
|
+
this.hideHandler = null;
|
|
8220
8532
|
this.snapshotDirty = false;
|
|
8533
|
+
this.dirtyRows = 0;
|
|
8534
|
+
}
|
|
8535
|
+
/**
|
|
8536
|
+
* Write the circuit's store to the local store as a snapshot. Compacts the
|
|
8537
|
+
* row arena first when dead bytes outweigh live ones. Serialised: a second
|
|
8538
|
+
* call while one is in flight joins it. No-op unless persistence is on, this
|
|
8539
|
+
* tab owns the store, and the engine can hold a snapshot.
|
|
8540
|
+
*/
|
|
8541
|
+
checkpoint(reason = "manual") {
|
|
8542
|
+
if (this.checkpointInFlight) return this.checkpointInFlight;
|
|
8543
|
+
const run = this.runCheckpoint(reason).finally(() => {
|
|
8544
|
+
this.checkpointInFlight = null;
|
|
8545
|
+
});
|
|
8546
|
+
this.checkpointInFlight = run;
|
|
8547
|
+
return run;
|
|
8221
8548
|
}
|
|
8222
|
-
async
|
|
8223
|
-
|
|
8549
|
+
async runCheckpoint(reason) {
|
|
8550
|
+
const processor = this.processor;
|
|
8551
|
+
if (!processor || !this.persistState || !this.persistCircuit || !this.schemaHash) return;
|
|
8552
|
+
if (typeof this.db.putSnapshot !== "function" || typeof processor.save_store_state !== "function") return;
|
|
8553
|
+
const generation = this.stateGeneration;
|
|
8554
|
+
const epoch = this.db.epoch;
|
|
8555
|
+
const rows = this.dirtyRows;
|
|
8556
|
+
this.snapshotDirty = false;
|
|
8557
|
+
this.dirtyRows = 0;
|
|
8224
8558
|
try {
|
|
8225
|
-
const
|
|
8226
|
-
|
|
8227
|
-
if (
|
|
8228
|
-
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
|
|
8233
|
-
|
|
8234
|
-
|
|
8559
|
+
const t0 = performance.now();
|
|
8560
|
+
let reclaimed = 0;
|
|
8561
|
+
if (typeof processor.compact === "function" && typeof processor.dead_bytes === "function" && typeof processor.live_bytes === "function") {
|
|
8562
|
+
const dead = processor.dead_bytes();
|
|
8563
|
+
if (dead > 1e6 && dead > processor.live_bytes()) reclaimed = processor.compact();
|
|
8564
|
+
}
|
|
8565
|
+
const bytes = processor.save_store_state();
|
|
8566
|
+
const meta = {
|
|
8567
|
+
formatVersion: CIRCUIT_SNAPSHOT_FORMAT,
|
|
8568
|
+
schemaHash: this.schemaHash,
|
|
8569
|
+
savedAt: Date.now(),
|
|
8570
|
+
maxRv: typeof processor.max_row_versions === "function" ? processor.max_row_versions() : void 0
|
|
8571
|
+
};
|
|
8572
|
+
if (generation !== this.stateGeneration || epoch !== this.db.epoch) return;
|
|
8573
|
+
await this.db.putSnapshot(CIRCUIT_SNAPSHOT_KEY, bytes, meta);
|
|
8574
|
+
this.logger.info({
|
|
8575
|
+
reason,
|
|
8576
|
+
rows,
|
|
8577
|
+
bytes: bytes.byteLength,
|
|
8578
|
+
reclaimed,
|
|
8579
|
+
ms: Math.round(performance.now() - t0),
|
|
8580
|
+
Category: "sp00ky-client::StreamProcessorService::checkpoint"
|
|
8581
|
+
}, "Circuit snapshot written");
|
|
8235
8582
|
} catch (e) {
|
|
8236
|
-
this.logger.
|
|
8583
|
+
this.logger.warn({
|
|
8237
8584
|
error: e,
|
|
8238
|
-
|
|
8239
|
-
|
|
8585
|
+
reason,
|
|
8586
|
+
Category: "sp00ky-client::StreamProcessorService::checkpoint"
|
|
8587
|
+
}, "Circuit snapshot failed");
|
|
8588
|
+
}
|
|
8589
|
+
}
|
|
8590
|
+
/**
|
|
8591
|
+
* Projection widening: a newly registered view evaluates fields that rows
|
|
8592
|
+
* already in the circuit were stored without. Merge just those fields in,
|
|
8593
|
+
* table by table, from the local store. The view registered against what
|
|
8594
|
+
* was present and converges as the merges step through.
|
|
8595
|
+
*/
|
|
8596
|
+
scheduleWiden(missing) {
|
|
8597
|
+
for (const [table, fields] of Object.entries(missing)) {
|
|
8598
|
+
const set = this.widenPending.get(table) ?? /* @__PURE__ */ new Set();
|
|
8599
|
+
for (const f of fields) set.add(f);
|
|
8600
|
+
this.widenPending.set(table, set);
|
|
8601
|
+
}
|
|
8602
|
+
this.widenQueue = this.widenQueue.then(() => this.runWiden()).catch((e) => {
|
|
8603
|
+
this.logger.warn({
|
|
8604
|
+
error: e,
|
|
8605
|
+
Category: "sp00ky-client::StreamProcessorService::widen"
|
|
8606
|
+
}, "Projection widening failed");
|
|
8607
|
+
});
|
|
8608
|
+
}
|
|
8609
|
+
async runWiden() {
|
|
8610
|
+
const processor = this.processor;
|
|
8611
|
+
if (!processor || typeof this.db.scanVersions !== "function") {
|
|
8612
|
+
this.widenPending.clear();
|
|
8613
|
+
return;
|
|
8614
|
+
}
|
|
8615
|
+
const generation = this.stateGeneration;
|
|
8616
|
+
const epoch = this.db.epoch;
|
|
8617
|
+
while (this.widenPending.size > 0) {
|
|
8618
|
+
const [table, fields] = this.widenPending.entries().next().value;
|
|
8619
|
+
this.widenPending.delete(table);
|
|
8620
|
+
const select = Array.from(fields);
|
|
8621
|
+
const versions = await this.db.scanVersions([table]);
|
|
8622
|
+
if (this.processor !== processor || generation !== this.stateGeneration || epoch !== this.db.epoch) return;
|
|
8623
|
+
const ids = (versions[table] ?? []).map(([id]) => id);
|
|
8624
|
+
let merged = 0;
|
|
8625
|
+
for (const chunk of chunks(ids, PRIME_CHUNK)) {
|
|
8626
|
+
const rows = await this.db.selectByIds(table, chunk, { select });
|
|
8627
|
+
if (this.processor !== processor || generation !== this.stateGeneration || epoch !== this.db.epoch) return;
|
|
8628
|
+
merged += this.ingestMany(rows.map((row) => ({
|
|
8629
|
+
table,
|
|
8630
|
+
op: "MERGE",
|
|
8631
|
+
id: idString(row.id),
|
|
8632
|
+
record: row
|
|
8633
|
+
}))).length;
|
|
8634
|
+
}
|
|
8635
|
+
this.logger.info({
|
|
8636
|
+
table,
|
|
8637
|
+
fields: select,
|
|
8638
|
+
merged,
|
|
8639
|
+
Category: "sp00ky-client::StreamProcessorService::widen"
|
|
8640
|
+
}, "Widened projected rows with newly evaluated fields");
|
|
8240
8641
|
}
|
|
8241
8642
|
}
|
|
8242
8643
|
/**
|
|
@@ -8277,25 +8678,6 @@ var StreamProcessorService = class {
|
|
|
8277
8678
|
Category: "sp00ky-client::StreamProcessorService::setSessionAuth"
|
|
8278
8679
|
}, "Session auth context updated");
|
|
8279
8680
|
}
|
|
8280
|
-
async saveState() {
|
|
8281
|
-
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
8282
|
-
const generation = this.stateGeneration;
|
|
8283
|
-
try {
|
|
8284
|
-
if (typeof this.processor.save_state === "function") {
|
|
8285
|
-
const state = this.processor.save_state();
|
|
8286
|
-
if (generation !== this.stateGeneration) return;
|
|
8287
|
-
if (state) {
|
|
8288
|
-
await this.persistenceClient.set(this.stateKey(), state);
|
|
8289
|
-
this.logger.trace({ Category: "sp00ky-client::StreamProcessorService::saveState" }, "State saved");
|
|
8290
|
-
}
|
|
8291
|
-
}
|
|
8292
|
-
} catch (e) {
|
|
8293
|
-
this.logger.error({
|
|
8294
|
-
error: e,
|
|
8295
|
-
Category: "sp00ky-client::StreamProcessorService::saveState"
|
|
8296
|
-
}, "Failed to save state");
|
|
8297
|
-
}
|
|
8298
|
-
}
|
|
8299
8681
|
/**
|
|
8300
8682
|
* Ingest a record change into the processor.
|
|
8301
8683
|
* Emits 'stream_update' event if materialized views are affected.
|
|
@@ -8329,7 +8711,7 @@ var StreamProcessorService = class {
|
|
|
8329
8711
|
const updates = rawUpdates.map((u) => ({
|
|
8330
8712
|
queryHash: u.query_id,
|
|
8331
8713
|
localArray: u.result_data,
|
|
8332
|
-
op,
|
|
8714
|
+
op: op === "MERGE" ? "UPDATE" : op,
|
|
8333
8715
|
materializationTimeMs,
|
|
8334
8716
|
storeApplyMs: u.timing_store_apply_ms,
|
|
8335
8717
|
circuitStepMs: u.timing_circuit_step_ms,
|
|
@@ -8390,7 +8772,7 @@ var StreamProcessorService = class {
|
|
|
8390
8772
|
snapshotMs: initialUpdate.timing_snapshot_ms ?? 0
|
|
8391
8773
|
}
|
|
8392
8774
|
};
|
|
8393
|
-
this.
|
|
8775
|
+
if (initialUpdate.missing_fields && Object.keys(initialUpdate.missing_fields).length > 0) this.scheduleWiden(initialUpdate.missing_fields);
|
|
8394
8776
|
this.logger.debug({
|
|
8395
8777
|
queryHash: queryPlan.queryHash,
|
|
8396
8778
|
surql: queryPlan.surql,
|
|
@@ -8413,7 +8795,6 @@ var StreamProcessorService = class {
|
|
|
8413
8795
|
if (!this.processor) return;
|
|
8414
8796
|
try {
|
|
8415
8797
|
this.processor.unregister_view(queryHash);
|
|
8416
|
-
this.markSnapshotDirty();
|
|
8417
8798
|
} catch (e) {
|
|
8418
8799
|
this.logger.error({
|
|
8419
8800
|
error: e,
|
|
@@ -8544,6 +8925,16 @@ var CacheModule = class {
|
|
|
8544
8925
|
lookup(recordId) {
|
|
8545
8926
|
return this.versionLookups[recordId] ?? 0;
|
|
8546
8927
|
}
|
|
8928
|
+
/**
|
|
8929
|
+
* Seed the version memo from rows the circuit was primed with out of the
|
|
8930
|
+
* local store, so the first post-reload sync diff does not re-download
|
|
8931
|
+
* bodies the browser already has. Only rows the prime actually put into the
|
|
8932
|
+
* circuit belong here: a memo entry with no circuit row would make the diff
|
|
8933
|
+
* flag the id forever while nothing ever fetches it.
|
|
8934
|
+
*/
|
|
8935
|
+
primeVersions(entries) {
|
|
8936
|
+
for (const [id, rv] of entries) if (rv > (this.versionLookups[id] ?? 0)) this.versionLookups[id] = rv;
|
|
8937
|
+
}
|
|
8547
8938
|
/** Drop the version cache on a bucket switch — a stale version would make
|
|
8548
8939
|
* the sync diff skip fetching a body the new bucket legitimately needs. */
|
|
8549
8940
|
clearVersionLookups() {
|
|
@@ -8592,9 +8983,10 @@ var CacheModule = class {
|
|
|
8592
8983
|
await this.local.execute(query, params, { epoch });
|
|
8593
8984
|
}
|
|
8594
8985
|
if (this.local.epoch !== epoch) throw new StaleEpochError();
|
|
8986
|
+
const versionOf = /* @__PURE__ */ new Map();
|
|
8595
8987
|
const bulk = populatedRecords.map((record) => {
|
|
8596
8988
|
const recordId = encodeRecordId(record.record.id);
|
|
8597
|
-
|
|
8989
|
+
versionOf.set(recordId, record.version);
|
|
8598
8990
|
return {
|
|
8599
8991
|
table: record.table,
|
|
8600
8992
|
op: record.op,
|
|
@@ -8602,8 +8994,9 @@ var CacheModule = class {
|
|
|
8602
8994
|
record: record.record
|
|
8603
8995
|
};
|
|
8604
8996
|
});
|
|
8605
|
-
this.streamProcessor.ingestMany(bulk);
|
|
8606
|
-
this.
|
|
8997
|
+
const ingested = this.streamProcessor.ingestMany(bulk);
|
|
8998
|
+
for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
|
|
8999
|
+
if (ingested.length > 0) this.ingestRelay?.(ingested);
|
|
8607
9000
|
this.logger.debug({
|
|
8608
9001
|
count: records.length,
|
|
8609
9002
|
Category: "sp00ky-client::CacheModule::saveBatch"
|
|
@@ -8638,7 +9031,12 @@ var CacheModule = class {
|
|
|
8638
9031
|
if (!skipDbDelete) await this.local.query("DELETE $id", { id: parseRecordIdString(id) }, { epoch });
|
|
8639
9032
|
if (this.local.epoch !== epoch) throw new StaleEpochError();
|
|
8640
9033
|
delete this.versionLookups[id];
|
|
8641
|
-
this.streamProcessor.
|
|
9034
|
+
this.streamProcessor.ingestMany([{
|
|
9035
|
+
table,
|
|
9036
|
+
op: "DELETE",
|
|
9037
|
+
id,
|
|
9038
|
+
record: recordData
|
|
9039
|
+
}]);
|
|
8642
9040
|
this.ingestRelay?.([{
|
|
8643
9041
|
table,
|
|
8644
9042
|
op: "DELETE",
|
|
@@ -10228,6 +10626,10 @@ var TabsCoordinator = class {
|
|
|
10228
10626
|
leadershipId = 0;
|
|
10229
10627
|
leaderTabId = null;
|
|
10230
10628
|
brokerRole = "pending";
|
|
10629
|
+
/** Wall time from `become-leader` to `leader-ready` on the last promotion.
|
|
10630
|
+
* Reported in the DevTools role-change event: a slow store adopt (OPFS pool
|
|
10631
|
+
* contention after the previous leader died) shows up here as seconds. */
|
|
10632
|
+
lastPromotionMs = null;
|
|
10231
10633
|
fingerprint;
|
|
10232
10634
|
bucketId;
|
|
10233
10635
|
broker;
|
|
@@ -10278,6 +10680,7 @@ var TabsCoordinator = class {
|
|
|
10278
10680
|
this.deps.logger.info({
|
|
10279
10681
|
role,
|
|
10280
10682
|
leadershipId: this.leadershipId,
|
|
10683
|
+
promotionMs: role === "leader" ? this.lastPromotionMs : void 0,
|
|
10281
10684
|
Category: "sp00ky-client::TabsCoordinator"
|
|
10282
10685
|
}, "Tab role changed");
|
|
10283
10686
|
for (const cb of this.roleListeners) cb(role);
|
|
@@ -10354,6 +10757,7 @@ var TabsCoordinator = class {
|
|
|
10354
10757
|
if (this.closed) return;
|
|
10355
10758
|
if (msg.leadershipId <= this.leadershipId && !msg.resumeHeld) return;
|
|
10356
10759
|
const previousRole = this.role;
|
|
10760
|
+
const startedAt = Date.now();
|
|
10357
10761
|
try {
|
|
10358
10762
|
if (previousRole === "follower") this.teardownFollower("promoted");
|
|
10359
10763
|
this.leadershipId = msg.leadershipId;
|
|
@@ -10375,7 +10779,8 @@ var TabsCoordinator = class {
|
|
|
10375
10779
|
resumeHeld: msg.resumeHeld
|
|
10376
10780
|
});
|
|
10377
10781
|
this.hub = new LeaderSyncHub(msg.leadershipId, this.deps.logger);
|
|
10378
|
-
|
|
10782
|
+
this.deps.hooks.becomeSyncLeader(this.hub);
|
|
10783
|
+
this.lastPromotionMs = Date.now() - startedAt;
|
|
10379
10784
|
this.setRole("leader");
|
|
10380
10785
|
this.broker.send({
|
|
10381
10786
|
type: "leader-ready",
|
|
@@ -10383,6 +10788,12 @@ var TabsCoordinator = class {
|
|
|
10383
10788
|
bucketId: this.bucketId,
|
|
10384
10789
|
leadershipId: msg.leadershipId
|
|
10385
10790
|
});
|
|
10791
|
+
this.deps.hooks.resumeSyncLeaderDuties().catch((e) => {
|
|
10792
|
+
this.deps.logger.error({
|
|
10793
|
+
err: e,
|
|
10794
|
+
Category: "sp00ky-client::TabsCoordinator"
|
|
10795
|
+
}, "Leader duties failed after promotion");
|
|
10796
|
+
});
|
|
10386
10797
|
} catch (e) {
|
|
10387
10798
|
const reason = e instanceof Error ? e.message : String(e);
|
|
10388
10799
|
this.deps.logger.error({
|
|
@@ -11821,8 +12232,9 @@ var Sp00kyClient = class {
|
|
|
11821
12232
|
else if (config.persistenceClient === "localstorage" || !config.persistenceClient) this.persistenceClient = new LocalStoragePersistenceClient(logger);
|
|
11822
12233
|
else this.persistenceClient = config.persistenceClient;
|
|
11823
12234
|
this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
|
|
11824
|
-
this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local,
|
|
11825
|
-
this.streamProcessor.configureCircuitPersistence(config.persistCircuit ??
|
|
12235
|
+
this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local, logger);
|
|
12236
|
+
this.streamProcessor.configureCircuitPersistence(config.persistCircuit ?? config.localEngine === "sqlite", config.circuitCheckpointMs);
|
|
12237
|
+
this.streamProcessor.configureProjection(config.circuitProjection ?? true);
|
|
11826
12238
|
this.migrator = new LocalMigrator(this.local, logger);
|
|
11827
12239
|
this.cache = new CacheModule(this.local, this.streamProcessor, (update) => {
|
|
11828
12240
|
this.dataModule.onStreamUpdate(update);
|
|
@@ -11838,6 +12250,7 @@ var Sp00kyClient = class {
|
|
|
11838
12250
|
downTimeoutMs: this.config.downTimeoutMs,
|
|
11839
12251
|
connectionSupervisor: this.connectionSupervisor
|
|
11840
12252
|
});
|
|
12253
|
+
this.sync.setPrimeGate(() => this.streamProcessor.whenPrimed());
|
|
11841
12254
|
this.featureFlags = new FeatureFlagModule({
|
|
11842
12255
|
dataModule: this.dataModule,
|
|
11843
12256
|
sync: this.sync,
|
|
@@ -11890,7 +12303,7 @@ var Sp00kyClient = class {
|
|
|
11890
12303
|
return new TabsCoordinator({
|
|
11891
12304
|
tabId,
|
|
11892
12305
|
fingerprint: computeTabsFingerprint({
|
|
11893
|
-
coreVersion: "0.0.1-canary.
|
|
12306
|
+
coreVersion: "0.0.1-canary.206",
|
|
11894
12307
|
schemaHash: hash53(this.config.schemaSurql),
|
|
11895
12308
|
endpoint: this.config.database.endpoint ?? "",
|
|
11896
12309
|
namespace: this.config.database.namespace,
|
|
@@ -11907,13 +12320,14 @@ var Sp00kyClient = class {
|
|
|
11907
12320
|
onLeaderLost: (reason) => engine.onLeaderLost(reason),
|
|
11908
12321
|
exposeClientPort: (clientId, port) => engine.exposeClientPort(clientId, port),
|
|
11909
12322
|
removeClientPort: (clientId) => engine.removeClientPort(clientId),
|
|
11910
|
-
becomeSyncLeader:
|
|
12323
|
+
becomeSyncLeader: (hub) => {
|
|
11911
12324
|
this.streamProcessor.setPersistenceEnabled(true);
|
|
11912
12325
|
this.cache.setIngestRelay((tuples) => hub.relayIngest(tuples));
|
|
11913
12326
|
this.sync.setTabContext("leader", tabId);
|
|
11914
12327
|
this.dataModule.setTabId(tabId);
|
|
11915
|
-
|
|
12328
|
+
this.sync.promoteToLeader(hub);
|
|
11916
12329
|
},
|
|
12330
|
+
resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
|
|
11917
12331
|
becomeSyncFollower: (forwarder) => {
|
|
11918
12332
|
this.streamProcessor.setPersistenceEnabled(false);
|
|
11919
12333
|
this.cache.setIngestRelay(null);
|
|
@@ -12007,6 +12421,13 @@ var Sp00kyClient = class {
|
|
|
12007
12421
|
if (this.tabsCoordinator) {
|
|
12008
12422
|
this.tabsCoordinator.onRoleChange((role) => {
|
|
12009
12423
|
this.sharedActive = role !== "solo";
|
|
12424
|
+
this.devTools.logEvent("TABS_ROLE_CHANGED", {
|
|
12425
|
+
role,
|
|
12426
|
+
tabId: this.tabsCoordinator?.tabId,
|
|
12427
|
+
leadershipId: this.tabsCoordinator?.leadershipId,
|
|
12428
|
+
leaderTabId: this.tabsCoordinator?.leaderTabId,
|
|
12429
|
+
promotionMs: this.tabsCoordinator?.lastPromotionMs
|
|
12430
|
+
});
|
|
12010
12431
|
});
|
|
12011
12432
|
try {
|
|
12012
12433
|
const role = await this.tabsCoordinator.start(bootBucket);
|
|
@@ -12044,9 +12465,9 @@ var Sp00kyClient = class {
|
|
|
12044
12465
|
}, "Blob cache failed to start; bucket files will not be cached locally");
|
|
12045
12466
|
}
|
|
12046
12467
|
})();
|
|
12047
|
-
this.streamProcessor.setStateKeySuffix(bootBucket);
|
|
12048
12468
|
await this.streamProcessor.init();
|
|
12049
12469
|
this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
|
|
12470
|
+
this.primeCircuit();
|
|
12050
12471
|
this.logger.debug({ Category: "sp00ky-client::Sp00kyClient::init" }, "StreamProcessor initialized");
|
|
12051
12472
|
const restoredUserId = await this.auth.restoreSessionFromToken();
|
|
12052
12473
|
const sessionId = this.mintSessionSalt();
|
|
@@ -12215,10 +12636,10 @@ var Sp00kyClient = class {
|
|
|
12215
12636
|
Category: "sp00ky-client::Sp00kyClient::doSwitchBucket"
|
|
12216
12637
|
}, "Blob cache bucket switch failed");
|
|
12217
12638
|
}
|
|
12218
|
-
this.streamProcessor.setStateKeySuffix(target);
|
|
12219
12639
|
await this.streamProcessor.reset();
|
|
12220
12640
|
this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
|
|
12221
12641
|
this.cache.clearVersionLookups();
|
|
12642
|
+
this.primeCircuit();
|
|
12222
12643
|
this.preloadedHashes.clear();
|
|
12223
12644
|
} finally {
|
|
12224
12645
|
reopen();
|
|
@@ -12250,6 +12671,7 @@ var Sp00kyClient = class {
|
|
|
12250
12671
|
this.crdtManager.closeAll();
|
|
12251
12672
|
this.crdtManager.dispose();
|
|
12252
12673
|
await this.blobs.close().catch(() => {});
|
|
12674
|
+
await this.streamProcessor.checkpoint("close").catch(() => {});
|
|
12253
12675
|
if (this.tabsCoordinator) await this.tabsCoordinator.stop();
|
|
12254
12676
|
await this.local.close();
|
|
12255
12677
|
await this.remote.close();
|
|
@@ -12537,6 +12959,31 @@ var Sp00kyClient = class {
|
|
|
12537
12959
|
* the former to `encodeRecordId` reads `.table` off a string and throws
|
|
12538
12960
|
* during boot.
|
|
12539
12961
|
*/
|
|
12962
|
+
/**
|
|
12963
|
+
* Prime the in-browser circuit from the local store. Builds the context the
|
|
12964
|
+
* stream processor needs: every synced table (the app schema plus the
|
|
12965
|
+
* server-written meta tables that sync down), a schema hash so a snapshot
|
|
12966
|
+
* projected under another schema is not trusted, and the ids whose local
|
|
12967
|
+
* `_00_rv` was bumped by an unsettled mutation.
|
|
12968
|
+
*/
|
|
12969
|
+
async primeCircuit() {
|
|
12970
|
+
const tables = [
|
|
12971
|
+
...this.config.schema.tables.map((t) => t.name),
|
|
12972
|
+
"_00_user_feature",
|
|
12973
|
+
"_00_app_release"
|
|
12974
|
+
];
|
|
12975
|
+
let pendingIds = /* @__PURE__ */ new Set();
|
|
12976
|
+
try {
|
|
12977
|
+
const pending = await this.dataModule.getPendingRecordIds();
|
|
12978
|
+
pendingIds = new Set([...pending.writes, ...pending.deletes]);
|
|
12979
|
+
} catch {}
|
|
12980
|
+
await this.streamProcessor.primeFromLocal({
|
|
12981
|
+
tables,
|
|
12982
|
+
schemaHash: String(hash53(this.config.schemaSurql)),
|
|
12983
|
+
pendingIds,
|
|
12984
|
+
onVersions: (_table, entries) => this.cache.primeVersions(entries)
|
|
12985
|
+
});
|
|
12986
|
+
}
|
|
12540
12987
|
sessionAuthId() {
|
|
12541
12988
|
const id = this.auth.currentUser?.id;
|
|
12542
12989
|
if (!id) return null;
|