@spooky-sync/core 0.0.1-canary.204 → 0.0.1-canary.206

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1616,6 +1616,21 @@ var SurrealCacheEngine = class extends LocalDatabaseService {
1616
1616
  if (typeof id !== "string") return id;
1617
1617
  return new RecordId(table, id.startsWith(`${table}:`) ? id.slice(table.length + 1) : id);
1618
1618
  }
1619
+ /**
1620
+ * `(id, _00_rv)` of every row per table. Runs on the main thread against the
1621
+ * embedded engine (no worker here), which is why the circuit prime caps how
1622
+ * much it is willing to read through this path.
1623
+ */
1624
+ async scanVersions(tables) {
1625
+ const out = {};
1626
+ for (const table of tables) try {
1627
+ const [rows] = await this.query(`SELECT id, _00_rv FROM ${table};`);
1628
+ out[table] = (rows ?? []).map((r) => [stableKey(r.id), Number(r._00_rv) || 0]);
1629
+ } catch {
1630
+ out[table] = [];
1631
+ }
1632
+ return out;
1633
+ }
1619
1634
  async getById(table, id) {
1620
1635
  const [row] = await this.query("SELECT * FROM ONLY $__id;", { __id: this.toRecordId(table, id) });
1621
1636
  return row && typeof row === "object" ? row : null;
@@ -2136,8 +2151,10 @@ var PortSqliteTransport = class extends BaseTransport {
2136
2151
  * on a fresh bucket throws "no such table". Keep in sync with the systemSchema
2137
2152
  * block in `local-migrator.ts`.
2138
2153
  */
2154
+ const SNAPSHOT_TABLE = "_00_circuit_snapshot";
2139
2155
  const SYSTEM_TABLES = [
2140
2156
  "_00_stream_processor_state",
2157
+ "_00_circuit_snapshot",
2141
2158
  "_00_query",
2142
2159
  "_00_preload",
2143
2160
  "_00_window",
@@ -2259,7 +2276,7 @@ var SqliteCacheEngine = class {
2259
2276
  diag.freelistBytes = rows?.[0]?.freelist;
2260
2277
  if (opts?.tableCounts) {
2261
2278
  const { rows: tables } = await this.call("exec", { sql: "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" });
2262
- const names = (tables ?? []).map((r) => r.name);
2279
+ const names = (tables ?? []).map((r) => r.name).filter((n) => n !== SNAPSHOT_TABLE);
2263
2280
  if (names.length) {
2264
2281
  const sql = names.map((n) => `SELECT '${n.replace(/'/g, "''")}' AS t, COUNT(*) AS n FROM "${n.replace(/"/g, "\"\"")}"`).join(" UNION ALL ");
2265
2282
  const { rows: counts } = await this.call("exec", { sql });
@@ -2702,6 +2719,57 @@ var SqliteCacheEngine = class {
2702
2719
  await this.ensureTable(table);
2703
2720
  return (await this.execRows(`SELECT data FROM "${table}" WHERE id = ?`, [stableKey(id)]))[0] ?? null;
2704
2721
  }
2722
+ /**
2723
+ * `(id, _00_rv)` of every row per table, straight off the worker: no body
2724
+ * parse, one round trip per table. A table that does not exist yet reads as
2725
+ * empty rather than failing the whole scan.
2726
+ */
2727
+ async scanVersions(tables) {
2728
+ const out = {};
2729
+ for (const table of tables) try {
2730
+ const { rows } = await this.call("exec", { sql: `SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table.replace(/"/g, "\"\"")}"` });
2731
+ out[table] = (rows ?? []).map((r) => [r.id, Number(r.rv) || 0]);
2732
+ } catch {
2733
+ out[table] = [];
2734
+ }
2735
+ return out;
2736
+ }
2737
+ async getSnapshot(key) {
2738
+ const { rows } = await this.call("exec", {
2739
+ sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
2740
+ bind: [key, `${key}:meta`]
2741
+ });
2742
+ let bytes = null;
2743
+ let meta = null;
2744
+ for (const r of rows ?? []) if (r.id === key) {
2745
+ if (r.data instanceof Uint8Array) bytes = r.data;
2746
+ else if (r.data instanceof ArrayBuffer) bytes = new Uint8Array(r.data);
2747
+ else if (typeof r.data === "string") bytes = new TextEncoder().encode(r.data);
2748
+ } else if (typeof r.data === "string") try {
2749
+ meta = JSON.parse(r.data);
2750
+ } catch {
2751
+ meta = null;
2752
+ }
2753
+ return bytes && meta ? {
2754
+ bytes,
2755
+ meta
2756
+ } : null;
2757
+ }
2758
+ async putSnapshot(key, bytes, meta) {
2759
+ await this.call("batch", [{
2760
+ sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
2761
+ bind: [key, bytes]
2762
+ }, {
2763
+ sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
2764
+ bind: [`${key}:meta`, JSON.stringify(meta)]
2765
+ }]);
2766
+ }
2767
+ async deleteSnapshot(key) {
2768
+ await this.call("run", {
2769
+ sql: `DELETE FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
2770
+ bind: [key, `${key}:meta`]
2771
+ });
2772
+ }
2705
2773
  async upsert(table, id, data, mode) {
2706
2774
  await this.ensureTable(table);
2707
2775
  const key = stableKey(id);
@@ -3253,12 +3321,6 @@ function phaseStatOf(samples, lastMs) {
3253
3321
  count: samples.length
3254
3322
  };
3255
3323
  }
3256
- /**
3257
- * DataModule - Unified query and mutation management
3258
- *
3259
- * Merges the functionality of QueryManager and MutationManager.
3260
- * Uses CacheModule for all storage operations.
3261
- */
3262
3324
  var DataModule = class DataModule {
3263
3325
  /** Tab identity baked into mutation ids (shared-tabs rollback routing);
3264
3326
  * undefined in solo mode, where mutation-id falls back to a session id. */
@@ -3962,23 +4024,42 @@ var DataModule = class DataModule {
3962
4024
  * authoritative membership on this device. Any read error is treated as
3963
4025
  * "unknown" so a broken row degrades to the predicate scan rather than
3964
4026
  * rendering an empty list.
4027
+ *
4028
+ * `confirmed` is true only for rows written after the server itself vouched
4029
+ * for the set (a non-empty id-set, or an empty one it reported a row count of
4030
+ * zero for, or an empty one that followed a non-empty one in the same
4031
+ * session). Rows written before the marker existed, including the `[]` rows a
4032
+ * pre-`ea56f50e` client mirrored from an unflushed read, read as unconfirmed.
3965
4033
  */
3966
4034
  async getWindowMembership(key) {
3967
4035
  try {
3968
4036
  const row = await this.local.getById("_00_window", new RecordId("_00_window", key));
3969
4037
  if (!row || typeof row !== "object") return null;
3970
4038
  const ids = row.ids;
3971
- return Array.isArray(ids) ? ids : null;
4039
+ if (!Array.isArray(ids)) return null;
4040
+ return {
4041
+ ids,
4042
+ confirmed: row.confirmed === true
4043
+ };
3972
4044
  } catch {
3973
4045
  return null;
3974
4046
  }
3975
4047
  }
3976
- /** Persist the durable membership row. Best-effort: callers must not fail a
3977
- * sync round because the mirror write failed. */
3978
- async writeWindowMembership(key, ids) {
4048
+ /**
4049
+ * Persist the durable membership row. Best-effort: callers must not fail a
4050
+ * sync round because the mirror write failed.
4051
+ *
4052
+ * `confirmed` says whether a cold start may trust this row even when it is
4053
+ * empty. A confirmed empty is a real answer ("the server says this query has
4054
+ * no rows") and stays empty across a reload; an unconfirmed empty is the
4055
+ * retry budget's guess and falls back to the predicate scan on the next boot,
4056
+ * exactly as every empty row did before the marker existed.
4057
+ */
4058
+ async writeWindowMembership(key, ids, confirmed) {
3979
4059
  try {
3980
4060
  await this.local.upsert("_00_window", new RecordId("_00_window", key), {
3981
4061
  ids,
4062
+ confirmed,
3982
4063
  updatedAt: Date.now()
3983
4064
  }, "replace");
3984
4065
  } catch (err) {
@@ -4137,9 +4218,12 @@ var DataModule = class DataModule {
4137
4218
  }, "Query to update remote array not found");
4138
4219
  return;
4139
4220
  }
4221
+ let confirmed = remoteArray.length > 0 || queryState.config.remoteSeen === true;
4140
4222
  if (remoteArray.length === 0 && !queryState.config.remoteSeen) {
4141
4223
  const serverRowCount = opts?.serverRowCount;
4142
- if (!(serverRowCount === 0)) {
4224
+ const knownEmpty = serverRowCount === 0;
4225
+ confirmed = knownEmpty;
4226
+ if (!knownEmpty) {
4143
4227
  const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
4144
4228
  queryState.config.emptyReads = emptyReads;
4145
4229
  if (!(serverRowCount === null || serverRowCount === void 0 ? emptyReads >= EMPTY_MEMBERSHIP_CONFIRMATIONS : false)) {
@@ -4160,7 +4244,7 @@ var DataModule = class DataModule {
4160
4244
  queryState.config.remoteSeen = true;
4161
4245
  queryState.config.emptyReads = 0;
4162
4246
  }
4163
- if (queryState.config.membershipKey) await this.writeWindowMembership(queryState.config.membershipKey, remoteArray);
4247
+ if (queryState.config.membershipKey) await this.writeWindowMembership(queryState.config.membershipKey, remoteArray, confirmed);
4164
4248
  try {
4165
4249
  await this.local.query(surql.seal(surql.updateSet("id", ["remoteArray"])), {
4166
4250
  id: queryState.config.id,
@@ -4213,8 +4297,8 @@ var DataModule = class DataModule {
4213
4297
  config.emptyReads = 0;
4214
4298
  if (config.membershipKey) {
4215
4299
  const durable = await this.getWindowMembership(config.membershipKey);
4216
- if (durable?.length) {
4217
- config.remoteArray = durable;
4300
+ if (durable && (durable.ids.length > 0 || durable.confirmed)) {
4301
+ config.remoteArray = durable.ids;
4218
4302
  config.membershipKnown = true;
4219
4303
  }
4220
4304
  }
@@ -4644,8 +4728,8 @@ var DataModule = class DataModule {
4644
4728
  };
4645
4729
  if (membershipKey && !config.remoteArray?.length) {
4646
4730
  const durable = await this.getWindowMembership(membershipKey);
4647
- if (durable?.length) {
4648
- config.remoteArray = durable;
4731
+ if (durable && (durable.ids.length > 0 || durable.confirmed)) {
4732
+ config.remoteArray = durable.ids;
4649
4733
  config.membershipKnown = true;
4650
4734
  }
4651
4735
  } else if (config.remoteArray?.length) config.membershipKnown = true;
@@ -5394,6 +5478,8 @@ function recordVersionArraysEqual(a, b) {
5394
5478
 
5395
5479
  //#endregion
5396
5480
  //#region src/modules/sync/engine.ts
5481
+ /** Ids per remote `SELECT * FROM $ids` round trip. */
5482
+ const FETCH_CHUNK = 500;
5397
5483
  /**
5398
5484
  * SyncEngine handles the core sync operations: fetching remote records,
5399
5485
  * caching them locally, and ingesting into DBSP.
@@ -5424,51 +5510,59 @@ var SyncEngine = class {
5424
5510
  }, "SyncEngine.syncRecords diff");
5425
5511
  let stillRemoteIds = [];
5426
5512
  if (removed.length > 0) stillRemoteIds = await this.handleRemovedRecords(removed);
5427
- const toFetch = [...added, ...updated];
5428
- const idsToFetch = toFetch.map((x) => x.id);
5429
- if (idsToFetch.length === 0) return {
5513
+ const toFetch = [...added, ...updated].filter((item) => {
5514
+ const localVersion = this.cache.lookup(encodeRecordId(item.id));
5515
+ return !(localVersion && item.version <= localVersion);
5516
+ });
5517
+ if (toFetch.length === 0) return {
5430
5518
  remoteFetchMs: 0,
5431
5519
  stillRemoteIds
5432
5520
  };
5433
5521
  const versionMap = /* @__PURE__ */ new Map();
5434
5522
  for (const item of toFetch) versionMap.set(encodeRecordId(item.id), item.version);
5435
- const remoteFetchStart = performance.now();
5436
- const [remoteResults] = await this.remote.query("SELECT * FROM $idsToFetch", { idsToFetch });
5437
- const remoteFetchMs = performance.now() - remoteFetchStart;
5438
- const cacheBatch = [];
5439
- for (const record of remoteResults) {
5440
- if (!record?.id) {
5441
- this.logger.warn({
5442
- record,
5443
- idsToFetch,
5444
- Category: "sp00ky-client::SyncEngine::syncRecords"
5445
- }, "Remote record has no id (possibly deleted). Skipping record");
5446
- continue;
5447
- }
5448
- const fullId = encodeRecordId(record.id);
5449
- const table = record.id.table.toString();
5450
- const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
5451
- const version = versionMap.get(fullId) ?? 0;
5452
- const localVersion = this.cache.lookup(fullId);
5453
- if (localVersion && version <= localVersion) {
5454
- this.logger.info({
5455
- recordId: fullId,
5456
- version,
5457
- localVersion,
5458
- Category: "sp00ky-client::SyncEngine::syncRecords"
5459
- }, "Local version is higher than remote version. Skipping record");
5460
- continue;
5523
+ const addedIds = new Set(added.map((item) => encodeRecordId(item.id)));
5524
+ let remoteFetchMs = 0;
5525
+ const remoteResults = [];
5526
+ for (let i = 0; i < toFetch.length; i += FETCH_CHUNK) {
5527
+ const idsToFetch = toFetch.slice(i, i + FETCH_CHUNK).map((x) => x.id);
5528
+ const remoteFetchStart = performance.now();
5529
+ const [chunkResults] = await this.remote.query("SELECT * FROM $idsToFetch", { idsToFetch });
5530
+ remoteFetchMs += performance.now() - remoteFetchStart;
5531
+ const cacheBatch = [];
5532
+ for (const record of chunkResults ?? []) {
5533
+ if (!record?.id) {
5534
+ this.logger.warn({
5535
+ record,
5536
+ idsToFetch,
5537
+ Category: "sp00ky-client::SyncEngine::syncRecords"
5538
+ }, "Remote record has no id (possibly deleted). Skipping record");
5539
+ continue;
5540
+ }
5541
+ remoteResults.push(record);
5542
+ const fullId = encodeRecordId(record.id);
5543
+ const table = record.id.table.toString();
5544
+ const version = versionMap.get(fullId) ?? 0;
5545
+ const localVersion = this.cache.lookup(fullId);
5546
+ if (localVersion && version <= localVersion) {
5547
+ this.logger.info({
5548
+ recordId: fullId,
5549
+ version,
5550
+ localVersion,
5551
+ Category: "sp00ky-client::SyncEngine::syncRecords"
5552
+ }, "Local version is higher than remote version. Skipping record");
5553
+ continue;
5554
+ }
5555
+ const tableSchema = this.schema.tables.find((t) => t.name === table);
5556
+ const cleanedRecord = tableSchema ? cleanRecord(tableSchema.columns, record) : record;
5557
+ cacheBatch.push({
5558
+ table,
5559
+ op: addedIds.has(fullId) ? "CREATE" : "UPDATE",
5560
+ record: cleanedRecord,
5561
+ version
5562
+ });
5461
5563
  }
5462
- const tableSchema = this.schema.tables.find((t) => t.name === table);
5463
- const cleanedRecord = tableSchema ? cleanRecord(tableSchema.columns, record) : record;
5464
- cacheBatch.push({
5465
- table,
5466
- op: isAdded ? "CREATE" : "UPDATE",
5467
- record: cleanedRecord,
5468
- version
5469
- });
5564
+ if (cacheBatch.length > 0) await this.cache.saveBatch(cacheBatch);
5470
5565
  }
5471
- if (cacheBatch.length > 0) await this.cache.saveBatch(cacheBatch);
5472
5566
  this.events.emit(SyncEventTypes.RemoteDataIngested, { records: remoteResults });
5473
5567
  return {
5474
5568
  remoteFetchMs,
@@ -6005,14 +6099,37 @@ var Sp00kySync = class Sp00kySync {
6005
6099
  this.tabRole = role;
6006
6100
  this.tabId = tabId;
6007
6101
  }
6008
- /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
6009
- * relay LIVE events and rollbacks to followers via `hub`. Idempotent for a
6010
- * boot-time leader; a runtime promotion (failover) reloads the outbox,
6011
- * which now holds EVERY tab's rows, and restarts LIVE under this session. */
6012
- async promoteToLeader(hub) {
6102
+ /** In-flight {@link resumeLeaderDuties}, so a second call joins the first
6103
+ * instead of double-draining the outbox. */
6104
+ leaderDutiesInFlight = null;
6105
+ /** Resolves once the in-browser circuit has been primed from the local
6106
+ * store. Every sync diff waits on it: diffing against an empty circuit
6107
+ * classifies the whole working set as missing and re-downloads it. */
6108
+ primeGate = () => Promise.resolve();
6109
+ /** The prime we last waited on. `whenPrimed` hands out one promise per
6110
+ * prime, so a new identity means a new prime (boot, bucket switch) ran. */
6111
+ settledPrime = null;
6112
+ setPrimeGate(gate) {
6113
+ this.primeGate = gate;
6114
+ this.settledPrime = null;
6115
+ }
6116
+ /**
6117
+ * Leader WIRING only, and deliberately synchronous.
6118
+ *
6119
+ * The coordinator publishes the leader role and tells the broker
6120
+ * `leader-ready` the moment the store is adopted, and the broker can mint a
6121
+ * follower's ports on the very next tick. So the follower-message handler
6122
+ * has to be live before this returns, or a mutation forwarded in that window
6123
+ * is dropped. Everything that can block (outbox reload, LIVE restart) moved
6124
+ * to {@link resumeLeaderDuties}: a promotion that waits on the network holds
6125
+ * `leader-ready` back, and a broker whose leader never reports ready serves
6126
+ * no follower ports and re-elects no one, which wedges the whole namespace.
6127
+ */
6128
+ promoteToLeader(hub) {
6013
6129
  this.tabRole = "leader";
6014
6130
  this.hub = hub;
6015
6131
  this.forwarder = null;
6132
+ this.leaderDutiesInFlight = null;
6016
6133
  hub.onFollowerMessage = (tabId, msg) => {
6017
6134
  switch (msg.type) {
6018
6135
  case "sync-hello": break;
@@ -6024,7 +6141,17 @@ var Sp00kySync = class Sp00kySync {
6024
6141
  break;
6025
6142
  }
6026
6143
  };
6027
- if (this.isInit) {
6144
+ }
6145
+ /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
6146
+ * relay LIVE events and rollbacks to followers. Idempotent for a boot-time
6147
+ * leader; a runtime promotion (failover) reloads the outbox, which now
6148
+ * holds EVERY tab's rows, and restarts LIVE under this session. Runs in the
6149
+ * background off the promotion path, so however long it takes (or if it
6150
+ * never finishes) the tab is already a working leader. */
6151
+ resumeLeaderDuties() {
6152
+ if (this.leaderDutiesInFlight) return this.leaderDutiesInFlight;
6153
+ const run = (async () => {
6154
+ if (!this.isInit || this.tabRole !== "leader") return;
6028
6155
  await this.upQueue.loadFromDatabase();
6029
6156
  this.scheduler.syncUp();
6030
6157
  if (this.currentUserId || this.anonLiveEnabled) {
@@ -6032,11 +6159,15 @@ var Sp00kySync = class Sp00kySync {
6032
6159
  await this.restartRefLiveQuery().catch((err) => {
6033
6160
  this.logger.warn({
6034
6161
  err,
6035
- Category: "sp00ky-client::Sp00kySync::promoteToLeader"
6162
+ Category: "sp00ky-client::Sp00kySync::resumeLeaderDuties"
6036
6163
  }, "LIVE restart failed on promotion; poll fallback covers it");
6037
6164
  });
6038
6165
  }
6039
- }
6166
+ })();
6167
+ this.leaderDutiesInFlight = run;
6168
+ return run.finally(() => {
6169
+ if (this.leaderDutiesInFlight === run) this.leaderDutiesInFlight = null;
6170
+ });
6040
6171
  }
6041
6172
  /** Follower duties: no outbox drain, no LIVE. Mutations forward to the
6042
6173
  * leader; everything else (registration, per-query sync, poll) runs
@@ -6045,6 +6176,7 @@ var Sp00kySync = class Sp00kySync {
6045
6176
  this.tabRole = "follower";
6046
6177
  this.hub = null;
6047
6178
  this.forwarder = forwarder;
6179
+ this.leaderDutiesInFlight = null;
6048
6180
  this.killRefLiveQuery();
6049
6181
  forwarder.onLeaderMessage = (msg) => {
6050
6182
  switch (msg.type) {
@@ -6694,6 +6826,16 @@ var Sp00kySync = class Sp00kySync {
6694
6826
  * means the single resulting UI update lands after this completes.
6695
6827
  */
6696
6828
  async runSyncForQuery(hash, diff) {
6829
+ const prime = this.primeGate();
6830
+ if (prime !== this.settledPrime) {
6831
+ await prime;
6832
+ this.settledPrime = prime;
6833
+ const fresh = this.dataModule.getQueryByHash(hash);
6834
+ if (!fresh) return;
6835
+ const recomputed = new ArraySyncer(fresh.config.localArray, fresh.config.remoteArray).nextSet();
6836
+ if (!recomputed) return;
6837
+ diff = recomputed;
6838
+ }
6697
6839
  if (diff.added.length > 0 || diff.updated.length > 0) {
6698
6840
  const pendingDeletes = await this.getPendingDeleteIds();
6699
6841
  if (pendingDeletes.size > 0) diff = {
@@ -7211,8 +7353,8 @@ function selfAllowlistedVariant(flag, userId) {
7211
7353
 
7212
7354
  //#endregion
7213
7355
  //#region src/modules/devtools/index.ts
7214
- const CORE_VERSION = "0.0.1-canary.204";
7215
- const WASM_VERSION = "0.0.1-canary.204";
7356
+ const CORE_VERSION = "0.0.1-canary.206";
7357
+ const WASM_VERSION = "0.0.1-canary.206";
7216
7358
  const SURREAL_VERSION = "3.0.3";
7217
7359
  var DevToolsService = class DevToolsService {
7218
7360
  eventsHistory = [];
@@ -7313,6 +7455,9 @@ var DevToolsService = class DevToolsService {
7313
7455
  data: q.records,
7314
7456
  localArray: q.config.localArray,
7315
7457
  remoteArray: q.config.remoteArray,
7458
+ membershipKnown: q.config.membershipKnown === true,
7459
+ remoteSeen: q.config.remoteSeen === true,
7460
+ emptyReads: q.config.emptyReads ?? 0,
7316
7461
  timings: this.dataManager.phaseTimings(q)
7317
7462
  });
7318
7463
  });
@@ -7406,7 +7551,7 @@ var DevToolsService = class DevToolsService {
7406
7551
  const info = this.unwrapInfo(res);
7407
7552
  this.localTablesAt = Date.now();
7408
7553
  if (info && info.tables) {
7409
- const names = Object.keys(info.tables);
7554
+ const names = Object.keys(info.tables).filter((n) => n !== "_00_circuit_snapshot");
7410
7555
  const changed = names.length !== this.localTables.length || names.some((n, i) => n !== this.localTables[i]);
7411
7556
  this.localTables = names;
7412
7557
  if (changed) this.notifyDevTools();
@@ -7936,13 +8081,29 @@ var AuthService = class {
7936
8081
 
7937
8082
  //#endregion
7938
8083
  //#region src/services/stream-processor/index.ts
7939
- /**
7940
- * Read a circuit snapshot out of the pre-`persistCircuit` persisted shape
7941
- * (`[[{ state }]]`, a raw SurrealDB result). Returns null for anything else.
7942
- */
7943
- function extractLegacyState(result) {
7944
- if (Array.isArray(result) && Array.isArray(result[0]) && typeof result[0][0]?.state === "string") return result[0][0].state;
7945
- return null;
8084
+ /** Storage key of the circuit snapshot inside the local store. */
8085
+ const CIRCUIT_SNAPSHOT_KEY = "circuit";
8086
+ /** Bump when the bytes `load_store_state` reads change shape. */
8087
+ const CIRCUIT_SNAPSHOT_FORMAT = 1;
8088
+ /** Rows per `ingest_many` call. Bounds the transient the wasm side allocates
8089
+ * to parse a batch (measured: one 7700-row call of 20 KB bodies peaked at
8090
+ * 500 MB, 128-row chunks at 335 MB, and under projection at 24 MB). */
8091
+ const INGEST_CHUNK = 128;
8092
+ /** Ids per `selectByIds` when priming bodies out of the local store. */
8093
+ const PRIME_CHUNK = 256;
8094
+ /** Rows the SurrealDB (main-thread, IndexedDB) engine is allowed to prime
8095
+ * through; above this the circuit boots empty as it always did. */
8096
+ const SURREAL_PRIME_ROW_CAP = 2e4;
8097
+ /** Below this many changed rows the checkpoint timer stays quiet. */
8098
+ const CHECKPOINT_MIN_ROWS = 50;
8099
+ /** Stable `table:id` for a row id that may already be a string. */
8100
+ function idString(id) {
8101
+ return typeof id === "string" ? id : encodeRecordId(id);
8102
+ }
8103
+ function chunks(items, size) {
8104
+ const out = [];
8105
+ for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
8106
+ return out;
7946
8107
  }
7947
8108
  var StreamProcessorService = class {
7948
8109
  logger;
@@ -7955,18 +8116,23 @@ var StreamProcessorService = class {
7955
8116
  authId: "",
7956
8117
  access: ""
7957
8118
  };
7958
- stateKeySuffix = "";
7959
8119
  stateGeneration = 0;
7960
8120
  persistState = true;
7961
8121
  persistCircuit = false;
7962
8122
  checkpointMs = 3e4;
7963
8123
  checkpointTimer = null;
7964
8124
  snapshotDirty = false;
7965
- pagehideHandler = null;
7966
- constructor(events, db, persistenceClient, logger) {
8125
+ dirtyRows = 0;
8126
+ hideHandler = null;
8127
+ checkpointInFlight = null;
8128
+ projection = true;
8129
+ primed = Promise.resolve();
8130
+ schemaHash = null;
8131
+ widenQueue = Promise.resolve();
8132
+ widenPending = /* @__PURE__ */ new Map();
8133
+ constructor(events, db, logger) {
7967
8134
  this.events = events;
7968
8135
  this.db = db;
7969
- this.persistenceClient = persistenceClient;
7970
8136
  this.logger = logger.child({ name: "StreamProcessorService" });
7971
8137
  }
7972
8138
  /**
@@ -7998,31 +8164,41 @@ var StreamProcessorService = class {
7998
8164
  for (const update of updates) for (const receiver of this.receivers) receiver.onStreamUpdate(update);
7999
8165
  }
8000
8166
  /**
8001
- * Ingest a batch of record changes as a single bulk operation, firing only
8002
- * one coalesced `StreamUpdate` per affected query once every record has been
8003
- * ingested (instead of one update per record). Use this whenever multiple
8004
- * records land at once — e.g. sync fetching N missing rows — so a list query
8005
- * re-runs and the UI re-renders once for the whole batch rather than
8006
- * row-by-row.
8167
+ * Ingest a batch of record changes, firing one coalesced `StreamUpdate` per
8168
+ * affected query once every record has been ingested. Use this whenever
8169
+ * multiple records land at once (sync fetching N rows, the boot prime).
8170
+ *
8171
+ * The batch is fed to the wasm side in chunks of {@link INGEST_CHUNK}: one
8172
+ * circuit step per chunk (a step walks every registered view, so per-record
8173
+ * ingest paid that fixed cost N times), but never the whole batch at once,
8174
+ * because the wasm side has to hold every parsed row of a call at the same
8175
+ * time and wasm32 dlmalloc never returns that peak.
8007
8176
  *
8008
- * Internally opens a coalescing window, ingests each record, then flushes;
8009
- * processor state is persisted once for the whole batch. No-op for an empty
8010
- * batch.
8177
+ * Returns the records that were ingested. A chunk that fails is reported
8178
+ * and skipped, not retried (a retry would double-apply whatever the failed
8179
+ * step already committed), and the remaining chunks still run.
8011
8180
  */
8012
8181
  ingestMany(records) {
8013
- if (records.length === 0) return;
8182
+ if (records.length === 0) return [];
8014
8183
  if (!this.processor) {
8015
8184
  this.logger.warn({ Category: "sp00ky-client::StreamProcessorService::ingestMany" }, "Not initialized, skipping ingest");
8016
- return;
8185
+ return [];
8017
8186
  }
8018
8187
  const bulkIngest = this.processor.ingest_many;
8019
- if (typeof bulkIngest === "function") {
8020
- this.logger.debug({
8021
- count: records.length,
8022
- Category: "sp00ky-client::StreamProcessorService::ingestMany"
8023
- }, "Ingesting batch into ssp");
8188
+ if (typeof bulkIngest !== "function") {
8189
+ this.beginCoalescing();
8024
8190
  try {
8025
- const items = records.map((record) => ({
8191
+ for (const record of records) this.ingest(record.table, record.op, record.id, record.record);
8192
+ } finally {
8193
+ this.flushCoalescing();
8194
+ }
8195
+ return records;
8196
+ }
8197
+ const ingested = [];
8198
+ this.beginCoalescing();
8199
+ try {
8200
+ for (const chunk of chunks(records, INGEST_CHUNK)) try {
8201
+ const items = chunk.map((record) => ({
8026
8202
  table: record.table,
8027
8203
  op: record.op,
8028
8204
  id: record.id,
@@ -8031,13 +8207,7 @@ var StreamProcessorService = class {
8031
8207
  const t0 = performance.now();
8032
8208
  const rawUpdates = bulkIngest.call(this.processor, items) ?? [];
8033
8209
  const materializationTimeMs = performance.now() - t0;
8034
- this.logger.debug({
8035
- count: records.length,
8036
- rawUpdates: rawUpdates.length,
8037
- materializationTimeMs,
8038
- Category: "sp00ky-client::StreamProcessorService::ingestMany"
8039
- }, "Ingesting batch into ssp done");
8040
- if (rawUpdates.length > 0) this.dispatchUpdates(rawUpdates.map((u) => ({
8210
+ if (rawUpdates.length > 0) this.notifyUpdates(rawUpdates.map((u) => ({
8041
8211
  queryHash: u.query_id,
8042
8212
  localArray: u.result_data,
8043
8213
  op: "CREATE",
@@ -8046,22 +8216,24 @@ var StreamProcessorService = class {
8046
8216
  circuitStepMs: u.timing_circuit_step_ms,
8047
8217
  transformMs: u.timing_transform_ms
8048
8218
  })));
8219
+ ingested.push(...chunk);
8049
8220
  } catch (e) {
8050
8221
  this.logger.error({
8051
8222
  error: e,
8052
- count: records.length,
8223
+ count: chunk.length,
8053
8224
  Category: "sp00ky-client::StreamProcessorService::ingestMany"
8054
- }, "Ingesting batch into ssp failed");
8225
+ }, "Ingesting chunk into ssp failed");
8055
8226
  }
8056
- this.markSnapshotDirty();
8057
- return;
8058
- }
8059
- this.beginCoalescing();
8060
- try {
8061
- for (const record of records) this.ingest(record.table, record.op, record.id, record.record);
8062
8227
  } finally {
8063
8228
  this.flushCoalescing();
8064
8229
  }
8230
+ this.logger.debug({
8231
+ count: records.length,
8232
+ ingested: ingested.length,
8233
+ Category: "sp00ky-client::StreamProcessorService::ingestMany"
8234
+ }, "Ingested batch into ssp");
8235
+ this.markSnapshotDirty(ingested.length);
8236
+ return ingested;
8065
8237
  }
8066
8238
  /**
8067
8239
  * Open a coalescing window. While open, the per-record stream updates
@@ -8088,7 +8260,6 @@ var StreamProcessorService = class {
8088
8260
  const buffered = Array.from(this.batchBuffer.values());
8089
8261
  this.batchBuffer.clear();
8090
8262
  if (buffered.length > 0) this.dispatchUpdates(buffered);
8091
- this.markSnapshotDirty();
8092
8263
  }
8093
8264
  /**
8094
8265
  * Initialize the WASM module and processor.
@@ -8100,7 +8271,7 @@ var StreamProcessorService = class {
8100
8271
  try {
8101
8272
  await init();
8102
8273
  this.processor = new Sp00kyProcessor();
8103
- await this.loadState();
8274
+ this.applyProjection();
8104
8275
  this.isInitialized = true;
8105
8276
  this.logger.info({ Category: "sp00ky-client::StreamProcessorService::init" }, "Initialized successfully");
8106
8277
  } catch (e) {
@@ -8111,21 +8282,16 @@ var StreamProcessorService = class {
8111
8282
  throw e;
8112
8283
  }
8113
8284
  }
8114
- /** Route the persisted circuit snapshot to a per-bucket key. */
8115
- setStateKeySuffix(bucketId) {
8116
- this.stateKeySuffix = bucketId;
8117
- }
8118
- stateKey() {
8119
- return this.stateKeySuffix ? `_00_stream_processor_state:${this.stateKeySuffix}` : "_00_stream_processor_state";
8120
- }
8121
8285
  /**
8122
8286
  * Drop the current WASM processor and start a fresh, empty circuit. Used on
8123
8287
  * local-bucket switches: the old circuit holds the previous user's rows AND
8124
8288
  * views registered with the previous `$auth` context, so neither may survive.
8125
- * Deliberately does NOT `loadState()` — a persisted snapshot references views
8126
- * under a dead sessionId salt; the DataModule rebind re-registers every live
8127
- * view against this fresh processor. Caller must re-seed `setPermissions`
8128
- * afterwards (a fresh circuit default-denies every table).
8289
+ * Deliberately loads nothing: the snapshot in the store being swapped away
8290
+ * from belongs to the previous bucket; the caller primes the new bucket's
8291
+ * circuit (`primeFromLocal`) once its store is open, and the DataModule
8292
+ * rebind re-registers every live view against this fresh processor. Caller
8293
+ * must re-seed `setPermissions` afterwards (a fresh circuit default-denies
8294
+ * every table).
8129
8295
  */
8130
8296
  async reset() {
8131
8297
  if (!this.isInitialized) return;
@@ -8133,8 +8299,11 @@ var StreamProcessorService = class {
8133
8299
  this.batching = false;
8134
8300
  this.batchBuffer.clear();
8135
8301
  this.snapshotDirty = false;
8302
+ this.dirtyRows = 0;
8303
+ this.widenPending.clear();
8136
8304
  const previous = this.processor;
8137
8305
  this.processor = new Sp00kyProcessor();
8306
+ this.applyProjection();
8138
8307
  this.freeProcessor(previous);
8139
8308
  this.logger.info({ Category: "sp00ky-client::StreamProcessorService::reset" }, "Stream processor reset (fresh circuit)");
8140
8309
  }
@@ -8174,9 +8343,9 @@ var StreamProcessorService = class {
8174
8343
  if (!enabled) this.stopCheckpoints();
8175
8344
  }
8176
8345
  /**
8177
- * Opt into snapshot persistence (`persistCircuit`). Off by default: see the
8178
- * `persistCircuit` field comment for why per-ingest snapshots were removed.
8179
- * Must be called before `init()` for a snapshot to be restored at boot.
8346
+ * Snapshot persistence (`persistCircuit`). When on, the circuit's store is
8347
+ * written to the local store on a checkpoint interval and when the page
8348
+ * goes hidden, and restored by {@link primeFromLocal} on the next boot.
8180
8349
  */
8181
8350
  configureCircuitPersistence(enabled, checkpointMs) {
8182
8351
  this.persistCircuit = enabled;
@@ -8184,59 +8353,291 @@ var StreamProcessorService = class {
8184
8353
  if (!enabled) this.stopCheckpoints();
8185
8354
  }
8186
8355
  /**
8187
- * Record that the circuit changed. Cheap and O(1), the expensive snapshot is
8188
- * deferred to the checkpoint timer, and skipped entirely when
8189
- * `persistCircuit` is off (the default).
8356
+ * Field projection (`circuitProjection`, default on). Takes effect on the
8357
+ * next processor (`init`/`reset`) and on rows written after that.
8190
8358
  */
8191
- markSnapshotDirty() {
8359
+ configureProjection(enabled) {
8360
+ this.projection = enabled;
8361
+ this.applyProjection();
8362
+ }
8363
+ applyProjection() {
8364
+ if (!this.processor || typeof this.processor.set_projection !== "function") return;
8365
+ try {
8366
+ this.processor.set_projection(this.projection);
8367
+ } catch (e) {
8368
+ this.logger.warn({
8369
+ error: e,
8370
+ Category: "sp00ky-client::StreamProcessorService::applyProjection"
8371
+ }, "set_projection failed");
8372
+ }
8373
+ }
8374
+ /** Resolves once the boot-time prime has finished (or was skipped). */
8375
+ whenPrimed() {
8376
+ return this.primed;
8377
+ }
8378
+ /**
8379
+ * Fill the circuit from the LOCAL store, in the background.
8380
+ *
8381
+ * With a usable snapshot: install it under whatever views have registered
8382
+ * meanwhile (`load_store_state` re-primes them), then `reconcile` each table
8383
+ * against the store's `(id, rv)` list so rows deleted since the checkpoint
8384
+ * are stepped out and only rows added or changed since are read back and
8385
+ * ingested. Without one: read every row and ingest it, chunked.
8386
+ *
8387
+ * Either way the circuit ends up equal to the local store without touching
8388
+ * the network, so the first sync diff is a real delta rather than "fetch
8389
+ * everything". The returned promise never rejects; `whenPrimed` gates on it.
8390
+ */
8391
+ primeFromLocal(ctx) {
8392
+ this.schemaHash = ctx.schemaHash;
8393
+ const run = this.runPrime(ctx).catch((e) => {
8394
+ this.logger.warn({
8395
+ error: e,
8396
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8397
+ }, "Circuit prime failed; the circuit fills from sync instead");
8398
+ });
8399
+ this.primed = run;
8400
+ return run;
8401
+ }
8402
+ async runPrime(ctx) {
8403
+ const processor = this.processor;
8404
+ if (!processor || typeof this.db.scanVersions !== "function") return;
8405
+ const generation = this.stateGeneration;
8406
+ const epoch = this.db.epoch;
8407
+ const alive = () => this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
8408
+ const t0 = performance.now();
8409
+ let restored = false;
8410
+ if (this.persistCircuit && typeof this.db.getSnapshot === "function" && typeof processor.load_store_state === "function") {
8411
+ const snapshot = await this.db.getSnapshot(CIRCUIT_SNAPSHOT_KEY);
8412
+ if (!alive()) return;
8413
+ if (snapshot) {
8414
+ const { meta } = snapshot;
8415
+ if (meta.formatVersion !== CIRCUIT_SNAPSHOT_FORMAT || meta.schemaHash !== ctx.schemaHash) this.logger.info({
8416
+ meta,
8417
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8418
+ }, "Circuit snapshot is from another format or schema; priming from rows");
8419
+ else try {
8420
+ const tLoad = performance.now();
8421
+ const updates = processor.load_store_state(snapshot.bytes);
8422
+ this.dispatchWasmUpdates(updates);
8423
+ restored = true;
8424
+ this.logger.info({
8425
+ bytes: snapshot.bytes.byteLength,
8426
+ views: updates.length,
8427
+ loadMs: Math.round(performance.now() - tLoad),
8428
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8429
+ }, "Circuit snapshot restored");
8430
+ } catch (e) {
8431
+ this.logger.warn({
8432
+ error: e,
8433
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8434
+ }, "Circuit snapshot unreadable; priming from rows");
8435
+ }
8436
+ }
8437
+ }
8438
+ const versions = await this.db.scanVersions(ctx.tables);
8439
+ if (!alive()) return;
8440
+ const total = Object.values(versions).reduce((n, v) => n + v.length, 0);
8441
+ if (!restored && this.db.engineKind !== "sqlite" && total > SURREAL_PRIME_ROW_CAP) {
8442
+ this.logger.warn({
8443
+ total,
8444
+ cap: SURREAL_PRIME_ROW_CAP,
8445
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8446
+ }, "Too many cached rows to prime through the main-thread engine; circuit fills from sync");
8447
+ return;
8448
+ }
8449
+ let ingested = 0;
8450
+ let fetched = 0;
8451
+ let deleted = 0;
8452
+ for (const table of ctx.tables) {
8453
+ const entries = versions[table] ?? [];
8454
+ let toFetch;
8455
+ if (restored && typeof processor.reconcile === "function") {
8456
+ const result = processor.reconcile(table, entries);
8457
+ this.dispatchWasmUpdates(result.updates);
8458
+ deleted += result.deleted;
8459
+ toFetch = result.fetch;
8460
+ } else toFetch = entries.map(([id]) => id);
8461
+ for (const chunk of chunks(toFetch, PRIME_CHUNK)) {
8462
+ const rows = await this.db.selectByIds(table, chunk);
8463
+ if (!alive()) return;
8464
+ fetched += rows.length;
8465
+ ingested += this.ingestMany(rows.map((row) => ({
8466
+ table,
8467
+ op: "CREATE",
8468
+ id: idString(row.id),
8469
+ record: row
8470
+ }))).length;
8471
+ }
8472
+ if (entries.length > 0) ctx.onVersions?.(table, entries.filter(([id]) => !ctx.pendingIds.has(id)));
8473
+ }
8474
+ if (!restored && ingested > 0) this.markSnapshotDirty(Math.max(ingested, CHECKPOINT_MIN_ROWS));
8475
+ this.logger.info({
8476
+ restored,
8477
+ rows: total,
8478
+ fetched,
8479
+ ingested,
8480
+ deleted,
8481
+ ms: Math.round(performance.now() - t0),
8482
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8483
+ }, "Circuit primed from the local store");
8484
+ }
8485
+ /** Publish wasm updates produced outside an ingest (restore, reconcile). */
8486
+ dispatchWasmUpdates(updates) {
8487
+ if (!updates || updates.length === 0) return;
8488
+ this.dispatchUpdates(updates.map((u) => ({
8489
+ queryHash: u.query_id,
8490
+ localArray: u.result_data,
8491
+ op: "CREATE"
8492
+ })));
8493
+ }
8494
+ /**
8495
+ * Record that the circuit changed by `rows` rows. Cheap; the snapshot is
8496
+ * deferred to the checkpoint timer and skipped entirely when `persistCircuit`
8497
+ * is off.
8498
+ */
8499
+ markSnapshotDirty(rows = 1) {
8192
8500
  if (!this.persistCircuit || !this.persistState) return;
8193
8501
  this.snapshotDirty = true;
8502
+ this.dirtyRows += rows;
8194
8503
  this.startCheckpoints();
8195
8504
  }
8196
8505
  startCheckpoints() {
8197
8506
  if (this.checkpointTimer) return;
8198
8507
  this.checkpointTimer = setInterval(() => {
8199
- if (!this.snapshotDirty) return;
8200
- this.snapshotDirty = false;
8201
- this.saveState();
8508
+ if (!this.snapshotDirty || this.dirtyRows < CHECKPOINT_MIN_ROWS) return;
8509
+ this.checkpoint("interval");
8202
8510
  }, this.checkpointMs);
8203
- if (typeof window !== "undefined" && !this.pagehideHandler) {
8204
- this.pagehideHandler = () => {
8511
+ if (typeof window !== "undefined" && !this.hideHandler) {
8512
+ this.hideHandler = () => {
8205
8513
  if (!this.snapshotDirty) return;
8206
- this.snapshotDirty = false;
8207
- this.saveState();
8514
+ if (typeof document !== "undefined" && document.visibilityState === "visible") return;
8515
+ this.checkpoint("hidden");
8208
8516
  };
8209
- window.addEventListener("pagehide", this.pagehideHandler);
8517
+ window.addEventListener("visibilitychange", this.hideHandler);
8518
+ window.addEventListener("pagehide", this.hideHandler);
8210
8519
  }
8211
8520
  }
8212
- /** Stop checkpointing and drop the `pagehide` listener. */
8521
+ /** Stop checkpointing and drop the visibility listeners. */
8213
8522
  stopCheckpoints() {
8214
8523
  if (this.checkpointTimer) {
8215
8524
  clearInterval(this.checkpointTimer);
8216
8525
  this.checkpointTimer = null;
8217
8526
  }
8218
- if (this.pagehideHandler && typeof window !== "undefined") window.removeEventListener("pagehide", this.pagehideHandler);
8219
- this.pagehideHandler = null;
8527
+ if (this.hideHandler && typeof window !== "undefined") {
8528
+ window.removeEventListener("visibilitychange", this.hideHandler);
8529
+ window.removeEventListener("pagehide", this.hideHandler);
8530
+ }
8531
+ this.hideHandler = null;
8220
8532
  this.snapshotDirty = false;
8533
+ this.dirtyRows = 0;
8534
+ }
8535
+ /**
8536
+ * Write the circuit's store to the local store as a snapshot. Compacts the
8537
+ * row arena first when dead bytes outweigh live ones. Serialised: a second
8538
+ * call while one is in flight joins it. No-op unless persistence is on, this
8539
+ * tab owns the store, and the engine can hold a snapshot.
8540
+ */
8541
+ checkpoint(reason = "manual") {
8542
+ if (this.checkpointInFlight) return this.checkpointInFlight;
8543
+ const run = this.runCheckpoint(reason).finally(() => {
8544
+ this.checkpointInFlight = null;
8545
+ });
8546
+ this.checkpointInFlight = run;
8547
+ return run;
8221
8548
  }
8222
- async loadState() {
8223
- if (!this.processor || !this.persistState || !this.persistCircuit) return;
8549
+ async runCheckpoint(reason) {
8550
+ const processor = this.processor;
8551
+ if (!processor || !this.persistState || !this.persistCircuit || !this.schemaHash) return;
8552
+ if (typeof this.db.putSnapshot !== "function" || typeof processor.save_store_state !== "function") return;
8553
+ const generation = this.stateGeneration;
8554
+ const epoch = this.db.epoch;
8555
+ const rows = this.dirtyRows;
8556
+ this.snapshotDirty = false;
8557
+ this.dirtyRows = 0;
8224
8558
  try {
8225
- const result = await this.persistenceClient.get(this.stateKey());
8226
- const state = typeof result === "string" ? result : extractLegacyState(result);
8227
- if (state) {
8228
- this.logger.info({
8229
- stateLength: state.length,
8230
- Category: "sp00ky-client::StreamProcessorService::loadState"
8231
- }, "Loading state from DB");
8232
- if (typeof this.processor.load_state === "function") this.processor.load_state(state);
8233
- else this.logger.warn({ Category: "sp00ky-client::StreamProcessorService::loadState" }, "load_state method not found on processor");
8234
- } else this.logger.info({ Category: "sp00ky-client::StreamProcessorService::loadState" }, "No saved state found");
8559
+ const t0 = performance.now();
8560
+ let reclaimed = 0;
8561
+ if (typeof processor.compact === "function" && typeof processor.dead_bytes === "function" && typeof processor.live_bytes === "function") {
8562
+ const dead = processor.dead_bytes();
8563
+ if (dead > 1e6 && dead > processor.live_bytes()) reclaimed = processor.compact();
8564
+ }
8565
+ const bytes = processor.save_store_state();
8566
+ const meta = {
8567
+ formatVersion: CIRCUIT_SNAPSHOT_FORMAT,
8568
+ schemaHash: this.schemaHash,
8569
+ savedAt: Date.now(),
8570
+ maxRv: typeof processor.max_row_versions === "function" ? processor.max_row_versions() : void 0
8571
+ };
8572
+ if (generation !== this.stateGeneration || epoch !== this.db.epoch) return;
8573
+ await this.db.putSnapshot(CIRCUIT_SNAPSHOT_KEY, bytes, meta);
8574
+ this.logger.info({
8575
+ reason,
8576
+ rows,
8577
+ bytes: bytes.byteLength,
8578
+ reclaimed,
8579
+ ms: Math.round(performance.now() - t0),
8580
+ Category: "sp00ky-client::StreamProcessorService::checkpoint"
8581
+ }, "Circuit snapshot written");
8235
8582
  } catch (e) {
8236
- this.logger.error({
8583
+ this.logger.warn({
8237
8584
  error: e,
8238
- Category: "sp00ky-client::StreamProcessorService::loadState"
8239
- }, "Failed to load state");
8585
+ reason,
8586
+ Category: "sp00ky-client::StreamProcessorService::checkpoint"
8587
+ }, "Circuit snapshot failed");
8588
+ }
8589
+ }
8590
+ /**
8591
+ * Projection widening: a newly registered view evaluates fields that rows
8592
+ * already in the circuit were stored without. Merge just those fields in,
8593
+ * table by table, from the local store. The view registered against what
8594
+ * was present and converges as the merges step through.
8595
+ */
8596
+ scheduleWiden(missing) {
8597
+ for (const [table, fields] of Object.entries(missing)) {
8598
+ const set = this.widenPending.get(table) ?? /* @__PURE__ */ new Set();
8599
+ for (const f of fields) set.add(f);
8600
+ this.widenPending.set(table, set);
8601
+ }
8602
+ this.widenQueue = this.widenQueue.then(() => this.runWiden()).catch((e) => {
8603
+ this.logger.warn({
8604
+ error: e,
8605
+ Category: "sp00ky-client::StreamProcessorService::widen"
8606
+ }, "Projection widening failed");
8607
+ });
8608
+ }
8609
+ async runWiden() {
8610
+ const processor = this.processor;
8611
+ if (!processor || typeof this.db.scanVersions !== "function") {
8612
+ this.widenPending.clear();
8613
+ return;
8614
+ }
8615
+ const generation = this.stateGeneration;
8616
+ const epoch = this.db.epoch;
8617
+ while (this.widenPending.size > 0) {
8618
+ const [table, fields] = this.widenPending.entries().next().value;
8619
+ this.widenPending.delete(table);
8620
+ const select = Array.from(fields);
8621
+ const versions = await this.db.scanVersions([table]);
8622
+ if (this.processor !== processor || generation !== this.stateGeneration || epoch !== this.db.epoch) return;
8623
+ const ids = (versions[table] ?? []).map(([id]) => id);
8624
+ let merged = 0;
8625
+ for (const chunk of chunks(ids, PRIME_CHUNK)) {
8626
+ const rows = await this.db.selectByIds(table, chunk, { select });
8627
+ if (this.processor !== processor || generation !== this.stateGeneration || epoch !== this.db.epoch) return;
8628
+ merged += this.ingestMany(rows.map((row) => ({
8629
+ table,
8630
+ op: "MERGE",
8631
+ id: idString(row.id),
8632
+ record: row
8633
+ }))).length;
8634
+ }
8635
+ this.logger.info({
8636
+ table,
8637
+ fields: select,
8638
+ merged,
8639
+ Category: "sp00ky-client::StreamProcessorService::widen"
8640
+ }, "Widened projected rows with newly evaluated fields");
8240
8641
  }
8241
8642
  }
8242
8643
  /**
@@ -8277,25 +8678,6 @@ var StreamProcessorService = class {
8277
8678
  Category: "sp00ky-client::StreamProcessorService::setSessionAuth"
8278
8679
  }, "Session auth context updated");
8279
8680
  }
8280
- async saveState() {
8281
- if (!this.processor || !this.persistState || !this.persistCircuit) return;
8282
- const generation = this.stateGeneration;
8283
- try {
8284
- if (typeof this.processor.save_state === "function") {
8285
- const state = this.processor.save_state();
8286
- if (generation !== this.stateGeneration) return;
8287
- if (state) {
8288
- await this.persistenceClient.set(this.stateKey(), state);
8289
- this.logger.trace({ Category: "sp00ky-client::StreamProcessorService::saveState" }, "State saved");
8290
- }
8291
- }
8292
- } catch (e) {
8293
- this.logger.error({
8294
- error: e,
8295
- Category: "sp00ky-client::StreamProcessorService::saveState"
8296
- }, "Failed to save state");
8297
- }
8298
- }
8299
8681
  /**
8300
8682
  * Ingest a record change into the processor.
8301
8683
  * Emits 'stream_update' event if materialized views are affected.
@@ -8329,7 +8711,7 @@ var StreamProcessorService = class {
8329
8711
  const updates = rawUpdates.map((u) => ({
8330
8712
  queryHash: u.query_id,
8331
8713
  localArray: u.result_data,
8332
- op,
8714
+ op: op === "MERGE" ? "UPDATE" : op,
8333
8715
  materializationTimeMs,
8334
8716
  storeApplyMs: u.timing_store_apply_ms,
8335
8717
  circuitStepMs: u.timing_circuit_step_ms,
@@ -8390,7 +8772,7 @@ var StreamProcessorService = class {
8390
8772
  snapshotMs: initialUpdate.timing_snapshot_ms ?? 0
8391
8773
  }
8392
8774
  };
8393
- this.markSnapshotDirty();
8775
+ if (initialUpdate.missing_fields && Object.keys(initialUpdate.missing_fields).length > 0) this.scheduleWiden(initialUpdate.missing_fields);
8394
8776
  this.logger.debug({
8395
8777
  queryHash: queryPlan.queryHash,
8396
8778
  surql: queryPlan.surql,
@@ -8413,7 +8795,6 @@ var StreamProcessorService = class {
8413
8795
  if (!this.processor) return;
8414
8796
  try {
8415
8797
  this.processor.unregister_view(queryHash);
8416
- this.markSnapshotDirty();
8417
8798
  } catch (e) {
8418
8799
  this.logger.error({
8419
8800
  error: e,
@@ -8544,6 +8925,16 @@ var CacheModule = class {
8544
8925
  lookup(recordId) {
8545
8926
  return this.versionLookups[recordId] ?? 0;
8546
8927
  }
8928
+ /**
8929
+ * Seed the version memo from rows the circuit was primed with out of the
8930
+ * local store, so the first post-reload sync diff does not re-download
8931
+ * bodies the browser already has. Only rows the prime actually put into the
8932
+ * circuit belong here: a memo entry with no circuit row would make the diff
8933
+ * flag the id forever while nothing ever fetches it.
8934
+ */
8935
+ primeVersions(entries) {
8936
+ for (const [id, rv] of entries) if (rv > (this.versionLookups[id] ?? 0)) this.versionLookups[id] = rv;
8937
+ }
8547
8938
  /** Drop the version cache on a bucket switch — a stale version would make
8548
8939
  * the sync diff skip fetching a body the new bucket legitimately needs. */
8549
8940
  clearVersionLookups() {
@@ -8592,9 +8983,10 @@ var CacheModule = class {
8592
8983
  await this.local.execute(query, params, { epoch });
8593
8984
  }
8594
8985
  if (this.local.epoch !== epoch) throw new StaleEpochError();
8986
+ const versionOf = /* @__PURE__ */ new Map();
8595
8987
  const bulk = populatedRecords.map((record) => {
8596
8988
  const recordId = encodeRecordId(record.record.id);
8597
- this.versionLookups[recordId] = record.version;
8989
+ versionOf.set(recordId, record.version);
8598
8990
  return {
8599
8991
  table: record.table,
8600
8992
  op: record.op,
@@ -8602,8 +8994,9 @@ var CacheModule = class {
8602
8994
  record: record.record
8603
8995
  };
8604
8996
  });
8605
- this.streamProcessor.ingestMany(bulk);
8606
- this.ingestRelay?.(bulk);
8997
+ const ingested = this.streamProcessor.ingestMany(bulk);
8998
+ for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
8999
+ if (ingested.length > 0) this.ingestRelay?.(ingested);
8607
9000
  this.logger.debug({
8608
9001
  count: records.length,
8609
9002
  Category: "sp00ky-client::CacheModule::saveBatch"
@@ -8638,7 +9031,12 @@ var CacheModule = class {
8638
9031
  if (!skipDbDelete) await this.local.query("DELETE $id", { id: parseRecordIdString(id) }, { epoch });
8639
9032
  if (this.local.epoch !== epoch) throw new StaleEpochError();
8640
9033
  delete this.versionLookups[id];
8641
- this.streamProcessor.ingest(table, "DELETE", id, recordData);
9034
+ this.streamProcessor.ingestMany([{
9035
+ table,
9036
+ op: "DELETE",
9037
+ id,
9038
+ record: recordData
9039
+ }]);
8642
9040
  this.ingestRelay?.([{
8643
9041
  table,
8644
9042
  op: "DELETE",
@@ -10228,6 +10626,10 @@ var TabsCoordinator = class {
10228
10626
  leadershipId = 0;
10229
10627
  leaderTabId = null;
10230
10628
  brokerRole = "pending";
10629
+ /** Wall time from `become-leader` to `leader-ready` on the last promotion.
10630
+ * Reported in the DevTools role-change event: a slow store adopt (OPFS pool
10631
+ * contention after the previous leader died) shows up here as seconds. */
10632
+ lastPromotionMs = null;
10231
10633
  fingerprint;
10232
10634
  bucketId;
10233
10635
  broker;
@@ -10278,6 +10680,7 @@ var TabsCoordinator = class {
10278
10680
  this.deps.logger.info({
10279
10681
  role,
10280
10682
  leadershipId: this.leadershipId,
10683
+ promotionMs: role === "leader" ? this.lastPromotionMs : void 0,
10281
10684
  Category: "sp00ky-client::TabsCoordinator"
10282
10685
  }, "Tab role changed");
10283
10686
  for (const cb of this.roleListeners) cb(role);
@@ -10354,6 +10757,7 @@ var TabsCoordinator = class {
10354
10757
  if (this.closed) return;
10355
10758
  if (msg.leadershipId <= this.leadershipId && !msg.resumeHeld) return;
10356
10759
  const previousRole = this.role;
10760
+ const startedAt = Date.now();
10357
10761
  try {
10358
10762
  if (previousRole === "follower") this.teardownFollower("promoted");
10359
10763
  this.leadershipId = msg.leadershipId;
@@ -10375,7 +10779,8 @@ var TabsCoordinator = class {
10375
10779
  resumeHeld: msg.resumeHeld
10376
10780
  });
10377
10781
  this.hub = new LeaderSyncHub(msg.leadershipId, this.deps.logger);
10378
- await this.deps.hooks.becomeSyncLeader(this.hub);
10782
+ this.deps.hooks.becomeSyncLeader(this.hub);
10783
+ this.lastPromotionMs = Date.now() - startedAt;
10379
10784
  this.setRole("leader");
10380
10785
  this.broker.send({
10381
10786
  type: "leader-ready",
@@ -10383,6 +10788,12 @@ var TabsCoordinator = class {
10383
10788
  bucketId: this.bucketId,
10384
10789
  leadershipId: msg.leadershipId
10385
10790
  });
10791
+ this.deps.hooks.resumeSyncLeaderDuties().catch((e) => {
10792
+ this.deps.logger.error({
10793
+ err: e,
10794
+ Category: "sp00ky-client::TabsCoordinator"
10795
+ }, "Leader duties failed after promotion");
10796
+ });
10386
10797
  } catch (e) {
10387
10798
  const reason = e instanceof Error ? e.message : String(e);
10388
10799
  this.deps.logger.error({
@@ -11821,8 +12232,9 @@ var Sp00kyClient = class {
11821
12232
  else if (config.persistenceClient === "localstorage" || !config.persistenceClient) this.persistenceClient = new LocalStoragePersistenceClient(logger);
11822
12233
  else this.persistenceClient = config.persistenceClient;
11823
12234
  this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
11824
- this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local, this.persistenceClient, logger);
11825
- this.streamProcessor.configureCircuitPersistence(config.persistCircuit ?? false, config.circuitCheckpointMs);
12235
+ this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local, logger);
12236
+ this.streamProcessor.configureCircuitPersistence(config.persistCircuit ?? config.localEngine === "sqlite", config.circuitCheckpointMs);
12237
+ this.streamProcessor.configureProjection(config.circuitProjection ?? true);
11826
12238
  this.migrator = new LocalMigrator(this.local, logger);
11827
12239
  this.cache = new CacheModule(this.local, this.streamProcessor, (update) => {
11828
12240
  this.dataModule.onStreamUpdate(update);
@@ -11838,6 +12250,7 @@ var Sp00kyClient = class {
11838
12250
  downTimeoutMs: this.config.downTimeoutMs,
11839
12251
  connectionSupervisor: this.connectionSupervisor
11840
12252
  });
12253
+ this.sync.setPrimeGate(() => this.streamProcessor.whenPrimed());
11841
12254
  this.featureFlags = new FeatureFlagModule({
11842
12255
  dataModule: this.dataModule,
11843
12256
  sync: this.sync,
@@ -11890,7 +12303,7 @@ var Sp00kyClient = class {
11890
12303
  return new TabsCoordinator({
11891
12304
  tabId,
11892
12305
  fingerprint: computeTabsFingerprint({
11893
- coreVersion: "0.0.1-canary.204",
12306
+ coreVersion: "0.0.1-canary.206",
11894
12307
  schemaHash: hash53(this.config.schemaSurql),
11895
12308
  endpoint: this.config.database.endpoint ?? "",
11896
12309
  namespace: this.config.database.namespace,
@@ -11907,13 +12320,14 @@ var Sp00kyClient = class {
11907
12320
  onLeaderLost: (reason) => engine.onLeaderLost(reason),
11908
12321
  exposeClientPort: (clientId, port) => engine.exposeClientPort(clientId, port),
11909
12322
  removeClientPort: (clientId) => engine.removeClientPort(clientId),
11910
- becomeSyncLeader: async (hub) => {
12323
+ becomeSyncLeader: (hub) => {
11911
12324
  this.streamProcessor.setPersistenceEnabled(true);
11912
12325
  this.cache.setIngestRelay((tuples) => hub.relayIngest(tuples));
11913
12326
  this.sync.setTabContext("leader", tabId);
11914
12327
  this.dataModule.setTabId(tabId);
11915
- await this.sync.promoteToLeader(hub);
12328
+ this.sync.promoteToLeader(hub);
11916
12329
  },
12330
+ resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
11917
12331
  becomeSyncFollower: (forwarder) => {
11918
12332
  this.streamProcessor.setPersistenceEnabled(false);
11919
12333
  this.cache.setIngestRelay(null);
@@ -12007,6 +12421,13 @@ var Sp00kyClient = class {
12007
12421
  if (this.tabsCoordinator) {
12008
12422
  this.tabsCoordinator.onRoleChange((role) => {
12009
12423
  this.sharedActive = role !== "solo";
12424
+ this.devTools.logEvent("TABS_ROLE_CHANGED", {
12425
+ role,
12426
+ tabId: this.tabsCoordinator?.tabId,
12427
+ leadershipId: this.tabsCoordinator?.leadershipId,
12428
+ leaderTabId: this.tabsCoordinator?.leaderTabId,
12429
+ promotionMs: this.tabsCoordinator?.lastPromotionMs
12430
+ });
12010
12431
  });
12011
12432
  try {
12012
12433
  const role = await this.tabsCoordinator.start(bootBucket);
@@ -12044,9 +12465,9 @@ var Sp00kyClient = class {
12044
12465
  }, "Blob cache failed to start; bucket files will not be cached locally");
12045
12466
  }
12046
12467
  })();
12047
- this.streamProcessor.setStateKeySuffix(bootBucket);
12048
12468
  await this.streamProcessor.init();
12049
12469
  this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
12470
+ this.primeCircuit();
12050
12471
  this.logger.debug({ Category: "sp00ky-client::Sp00kyClient::init" }, "StreamProcessor initialized");
12051
12472
  const restoredUserId = await this.auth.restoreSessionFromToken();
12052
12473
  const sessionId = this.mintSessionSalt();
@@ -12215,10 +12636,10 @@ var Sp00kyClient = class {
12215
12636
  Category: "sp00ky-client::Sp00kyClient::doSwitchBucket"
12216
12637
  }, "Blob cache bucket switch failed");
12217
12638
  }
12218
- this.streamProcessor.setStateKeySuffix(target);
12219
12639
  await this.streamProcessor.reset();
12220
12640
  this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
12221
12641
  this.cache.clearVersionLookups();
12642
+ this.primeCircuit();
12222
12643
  this.preloadedHashes.clear();
12223
12644
  } finally {
12224
12645
  reopen();
@@ -12250,6 +12671,7 @@ var Sp00kyClient = class {
12250
12671
  this.crdtManager.closeAll();
12251
12672
  this.crdtManager.dispose();
12252
12673
  await this.blobs.close().catch(() => {});
12674
+ await this.streamProcessor.checkpoint("close").catch(() => {});
12253
12675
  if (this.tabsCoordinator) await this.tabsCoordinator.stop();
12254
12676
  await this.local.close();
12255
12677
  await this.remote.close();
@@ -12537,6 +12959,31 @@ var Sp00kyClient = class {
12537
12959
  * the former to `encodeRecordId` reads `.table` off a string and throws
12538
12960
  * during boot.
12539
12961
  */
12962
+ /**
12963
+ * Prime the in-browser circuit from the local store. Builds the context the
12964
+ * stream processor needs: every synced table (the app schema plus the
12965
+ * server-written meta tables that sync down), a schema hash so a snapshot
12966
+ * projected under another schema is not trusted, and the ids whose local
12967
+ * `_00_rv` was bumped by an unsettled mutation.
12968
+ */
12969
+ async primeCircuit() {
12970
+ const tables = [
12971
+ ...this.config.schema.tables.map((t) => t.name),
12972
+ "_00_user_feature",
12973
+ "_00_app_release"
12974
+ ];
12975
+ let pendingIds = /* @__PURE__ */ new Set();
12976
+ try {
12977
+ const pending = await this.dataModule.getPendingRecordIds();
12978
+ pendingIds = new Set([...pending.writes, ...pending.deletes]);
12979
+ } catch {}
12980
+ await this.streamProcessor.primeFromLocal({
12981
+ tables,
12982
+ schemaHash: String(hash53(this.config.schemaSurql)),
12983
+ pendingIds,
12984
+ onVersions: (_table, entries) => this.cache.primeVersions(entries)
12985
+ });
12986
+ }
12540
12987
  sessionAuthId() {
12541
12988
  const id = this.auth.currentUser?.id;
12542
12989
  if (!id) return null;