@spooky-sync/core 0.0.1-canary.204 → 0.0.1-canary.205

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1616,6 +1616,21 @@ var SurrealCacheEngine = class extends LocalDatabaseService {
1616
1616
  if (typeof id !== "string") return id;
1617
1617
  return new RecordId(table, id.startsWith(`${table}:`) ? id.slice(table.length + 1) : id);
1618
1618
  }
1619
+ /**
1620
+ * `(id, _00_rv)` of every row per table. Runs on the main thread against the
1621
+ * embedded engine (no worker here), which is why the circuit prime caps how
1622
+ * much it is willing to read through this path.
1623
+ */
1624
+ async scanVersions(tables) {
1625
+ const out = {};
1626
+ for (const table of tables) try {
1627
+ const [rows] = await this.query(`SELECT id, _00_rv FROM ${table};`);
1628
+ out[table] = (rows ?? []).map((r) => [stableKey(r.id), Number(r._00_rv) || 0]);
1629
+ } catch {
1630
+ out[table] = [];
1631
+ }
1632
+ return out;
1633
+ }
1619
1634
  async getById(table, id) {
1620
1635
  const [row] = await this.query("SELECT * FROM ONLY $__id;", { __id: this.toRecordId(table, id) });
1621
1636
  return row && typeof row === "object" ? row : null;
@@ -2136,8 +2151,10 @@ var PortSqliteTransport = class extends BaseTransport {
2136
2151
  * on a fresh bucket throws "no such table". Keep in sync with the systemSchema
2137
2152
  * block in `local-migrator.ts`.
2138
2153
  */
2154
+ const SNAPSHOT_TABLE = "_00_circuit_snapshot";
2139
2155
  const SYSTEM_TABLES = [
2140
2156
  "_00_stream_processor_state",
2157
+ "_00_circuit_snapshot",
2141
2158
  "_00_query",
2142
2159
  "_00_preload",
2143
2160
  "_00_window",
@@ -2259,7 +2276,7 @@ var SqliteCacheEngine = class {
2259
2276
  diag.freelistBytes = rows?.[0]?.freelist;
2260
2277
  if (opts?.tableCounts) {
2261
2278
  const { rows: tables } = await this.call("exec", { sql: "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" });
2262
- const names = (tables ?? []).map((r) => r.name);
2279
+ const names = (tables ?? []).map((r) => r.name).filter((n) => n !== SNAPSHOT_TABLE);
2263
2280
  if (names.length) {
2264
2281
  const sql = names.map((n) => `SELECT '${n.replace(/'/g, "''")}' AS t, COUNT(*) AS n FROM "${n.replace(/"/g, "\"\"")}"`).join(" UNION ALL ");
2265
2282
  const { rows: counts } = await this.call("exec", { sql });
@@ -2702,6 +2719,57 @@ var SqliteCacheEngine = class {
2702
2719
  await this.ensureTable(table);
2703
2720
  return (await this.execRows(`SELECT data FROM "${table}" WHERE id = ?`, [stableKey(id)]))[0] ?? null;
2704
2721
  }
2722
+ /**
2723
+ * `(id, _00_rv)` of every row per table, straight off the worker: no body
2724
+ * parse, one round trip per table. A table that does not exist yet reads as
2725
+ * empty rather than failing the whole scan.
2726
+ */
2727
+ async scanVersions(tables) {
2728
+ const out = {};
2729
+ for (const table of tables) try {
2730
+ const { rows } = await this.call("exec", { sql: `SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table.replace(/"/g, "\"\"")}"` });
2731
+ out[table] = (rows ?? []).map((r) => [r.id, Number(r.rv) || 0]);
2732
+ } catch {
2733
+ out[table] = [];
2734
+ }
2735
+ return out;
2736
+ }
2737
+ async getSnapshot(key) {
2738
+ const { rows } = await this.call("exec", {
2739
+ sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
2740
+ bind: [key, `${key}:meta`]
2741
+ });
2742
+ let bytes = null;
2743
+ let meta = null;
2744
+ for (const r of rows ?? []) if (r.id === key) {
2745
+ if (r.data instanceof Uint8Array) bytes = r.data;
2746
+ else if (r.data instanceof ArrayBuffer) bytes = new Uint8Array(r.data);
2747
+ else if (typeof r.data === "string") bytes = new TextEncoder().encode(r.data);
2748
+ } else if (typeof r.data === "string") try {
2749
+ meta = JSON.parse(r.data);
2750
+ } catch {
2751
+ meta = null;
2752
+ }
2753
+ return bytes && meta ? {
2754
+ bytes,
2755
+ meta
2756
+ } : null;
2757
+ }
2758
+ async putSnapshot(key, bytes, meta) {
2759
+ await this.call("batch", [{
2760
+ sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
2761
+ bind: [key, bytes]
2762
+ }, {
2763
+ sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
2764
+ bind: [`${key}:meta`, JSON.stringify(meta)]
2765
+ }]);
2766
+ }
2767
+ async deleteSnapshot(key) {
2768
+ await this.call("run", {
2769
+ sql: `DELETE FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
2770
+ bind: [key, `${key}:meta`]
2771
+ });
2772
+ }
2705
2773
  async upsert(table, id, data, mode) {
2706
2774
  await this.ensureTable(table);
2707
2775
  const key = stableKey(id);
@@ -5394,6 +5462,8 @@ function recordVersionArraysEqual(a, b) {
5394
5462
 
5395
5463
  //#endregion
5396
5464
  //#region src/modules/sync/engine.ts
5465
+ /** Ids per remote `SELECT * FROM $ids` round trip. */
5466
+ const FETCH_CHUNK = 500;
5397
5467
  /**
5398
5468
  * SyncEngine handles the core sync operations: fetching remote records,
5399
5469
  * caching them locally, and ingesting into DBSP.
@@ -5424,51 +5494,59 @@ var SyncEngine = class {
5424
5494
  }, "SyncEngine.syncRecords diff");
5425
5495
  let stillRemoteIds = [];
5426
5496
  if (removed.length > 0) stillRemoteIds = await this.handleRemovedRecords(removed);
5427
- const toFetch = [...added, ...updated];
5428
- const idsToFetch = toFetch.map((x) => x.id);
5429
- if (idsToFetch.length === 0) return {
5497
+ const toFetch = [...added, ...updated].filter((item) => {
5498
+ const localVersion = this.cache.lookup(encodeRecordId(item.id));
5499
+ return !(localVersion && item.version <= localVersion);
5500
+ });
5501
+ if (toFetch.length === 0) return {
5430
5502
  remoteFetchMs: 0,
5431
5503
  stillRemoteIds
5432
5504
  };
5433
5505
  const versionMap = /* @__PURE__ */ new Map();
5434
5506
  for (const item of toFetch) versionMap.set(encodeRecordId(item.id), item.version);
5435
- const remoteFetchStart = performance.now();
5436
- const [remoteResults] = await this.remote.query("SELECT * FROM $idsToFetch", { idsToFetch });
5437
- const remoteFetchMs = performance.now() - remoteFetchStart;
5438
- const cacheBatch = [];
5439
- for (const record of remoteResults) {
5440
- if (!record?.id) {
5441
- this.logger.warn({
5442
- record,
5443
- idsToFetch,
5444
- Category: "sp00ky-client::SyncEngine::syncRecords"
5445
- }, "Remote record has no id (possibly deleted). Skipping record");
5446
- continue;
5447
- }
5448
- const fullId = encodeRecordId(record.id);
5449
- const table = record.id.table.toString();
5450
- const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
5451
- const version = versionMap.get(fullId) ?? 0;
5452
- const localVersion = this.cache.lookup(fullId);
5453
- if (localVersion && version <= localVersion) {
5454
- this.logger.info({
5455
- recordId: fullId,
5456
- version,
5457
- localVersion,
5458
- Category: "sp00ky-client::SyncEngine::syncRecords"
5459
- }, "Local version is higher than remote version. Skipping record");
5460
- continue;
5507
+ const addedIds = new Set(added.map((item) => encodeRecordId(item.id)));
5508
+ let remoteFetchMs = 0;
5509
+ const remoteResults = [];
5510
+ for (let i = 0; i < toFetch.length; i += FETCH_CHUNK) {
5511
+ const idsToFetch = toFetch.slice(i, i + FETCH_CHUNK).map((x) => x.id);
5512
+ const remoteFetchStart = performance.now();
5513
+ const [chunkResults] = await this.remote.query("SELECT * FROM $idsToFetch", { idsToFetch });
5514
+ remoteFetchMs += performance.now() - remoteFetchStart;
5515
+ const cacheBatch = [];
5516
+ for (const record of chunkResults ?? []) {
5517
+ if (!record?.id) {
5518
+ this.logger.warn({
5519
+ record,
5520
+ idsToFetch,
5521
+ Category: "sp00ky-client::SyncEngine::syncRecords"
5522
+ }, "Remote record has no id (possibly deleted). Skipping record");
5523
+ continue;
5524
+ }
5525
+ remoteResults.push(record);
5526
+ const fullId = encodeRecordId(record.id);
5527
+ const table = record.id.table.toString();
5528
+ const version = versionMap.get(fullId) ?? 0;
5529
+ const localVersion = this.cache.lookup(fullId);
5530
+ if (localVersion && version <= localVersion) {
5531
+ this.logger.info({
5532
+ recordId: fullId,
5533
+ version,
5534
+ localVersion,
5535
+ Category: "sp00ky-client::SyncEngine::syncRecords"
5536
+ }, "Local version is higher than remote version. Skipping record");
5537
+ continue;
5538
+ }
5539
+ const tableSchema = this.schema.tables.find((t) => t.name === table);
5540
+ const cleanedRecord = tableSchema ? cleanRecord(tableSchema.columns, record) : record;
5541
+ cacheBatch.push({
5542
+ table,
5543
+ op: addedIds.has(fullId) ? "CREATE" : "UPDATE",
5544
+ record: cleanedRecord,
5545
+ version
5546
+ });
5461
5547
  }
5462
- const tableSchema = this.schema.tables.find((t) => t.name === table);
5463
- const cleanedRecord = tableSchema ? cleanRecord(tableSchema.columns, record) : record;
5464
- cacheBatch.push({
5465
- table,
5466
- op: isAdded ? "CREATE" : "UPDATE",
5467
- record: cleanedRecord,
5468
- version
5469
- });
5548
+ if (cacheBatch.length > 0) await this.cache.saveBatch(cacheBatch);
5470
5549
  }
5471
- if (cacheBatch.length > 0) await this.cache.saveBatch(cacheBatch);
5472
5550
  this.events.emit(SyncEventTypes.RemoteDataIngested, { records: remoteResults });
5473
5551
  return {
5474
5552
  remoteFetchMs,
@@ -6005,14 +6083,37 @@ var Sp00kySync = class Sp00kySync {
6005
6083
  this.tabRole = role;
6006
6084
  this.tabId = tabId;
6007
6085
  }
6008
- /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
6009
- * relay LIVE events and rollbacks to followers via `hub`. Idempotent for a
6010
- * boot-time leader; a runtime promotion (failover) reloads the outbox,
6011
- * which now holds EVERY tab's rows, and restarts LIVE under this session. */
6012
- async promoteToLeader(hub) {
6086
+ /** In-flight {@link resumeLeaderDuties}, so a second call joins the first
6087
+ * instead of double-draining the outbox. */
6088
+ leaderDutiesInFlight = null;
6089
+ /** Resolves once the in-browser circuit has been primed from the local
6090
+ * store. Every sync diff waits on it: diffing against an empty circuit
6091
+ * classifies the whole working set as missing and re-downloads it. */
6092
+ primeGate = () => Promise.resolve();
6093
+ /** The prime we last waited on. `whenPrimed` hands out one promise per
6094
+ * prime, so a new identity means a new prime (boot, bucket switch) ran. */
6095
+ settledPrime = null;
6096
+ setPrimeGate(gate) {
6097
+ this.primeGate = gate;
6098
+ this.settledPrime = null;
6099
+ }
6100
+ /**
6101
+ * Leader WIRING only, and deliberately synchronous.
6102
+ *
6103
+ * The coordinator publishes the leader role and tells the broker
6104
+ * `leader-ready` the moment the store is adopted, and the broker can mint a
6105
+ * follower's ports on the very next tick. So the follower-message handler
6106
+ * has to be live before this returns, or a mutation forwarded in that window
6107
+ * is dropped. Everything that can block (outbox reload, LIVE restart) moved
6108
+ * to {@link resumeLeaderDuties}: a promotion that waits on the network holds
6109
+ * `leader-ready` back, and a broker whose leader never reports ready serves
6110
+ * no follower ports and re-elects no one, which wedges the whole namespace.
6111
+ */
6112
+ promoteToLeader(hub) {
6013
6113
  this.tabRole = "leader";
6014
6114
  this.hub = hub;
6015
6115
  this.forwarder = null;
6116
+ this.leaderDutiesInFlight = null;
6016
6117
  hub.onFollowerMessage = (tabId, msg) => {
6017
6118
  switch (msg.type) {
6018
6119
  case "sync-hello": break;
@@ -6024,7 +6125,17 @@ var Sp00kySync = class Sp00kySync {
6024
6125
  break;
6025
6126
  }
6026
6127
  };
6027
- if (this.isInit) {
6128
+ }
6129
+ /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
6130
+ * relay LIVE events and rollbacks to followers. Idempotent for a boot-time
6131
+ * leader; a runtime promotion (failover) reloads the outbox, which now
6132
+ * holds EVERY tab's rows, and restarts LIVE under this session. Runs in the
6133
+ * background off the promotion path, so however long it takes (or if it
6134
+ * never finishes) the tab is already a working leader. */
6135
+ resumeLeaderDuties() {
6136
+ if (this.leaderDutiesInFlight) return this.leaderDutiesInFlight;
6137
+ const run = (async () => {
6138
+ if (!this.isInit || this.tabRole !== "leader") return;
6028
6139
  await this.upQueue.loadFromDatabase();
6029
6140
  this.scheduler.syncUp();
6030
6141
  if (this.currentUserId || this.anonLiveEnabled) {
@@ -6032,11 +6143,15 @@ var Sp00kySync = class Sp00kySync {
6032
6143
  await this.restartRefLiveQuery().catch((err) => {
6033
6144
  this.logger.warn({
6034
6145
  err,
6035
- Category: "sp00ky-client::Sp00kySync::promoteToLeader"
6146
+ Category: "sp00ky-client::Sp00kySync::resumeLeaderDuties"
6036
6147
  }, "LIVE restart failed on promotion; poll fallback covers it");
6037
6148
  });
6038
6149
  }
6039
- }
6150
+ })();
6151
+ this.leaderDutiesInFlight = run;
6152
+ return run.finally(() => {
6153
+ if (this.leaderDutiesInFlight === run) this.leaderDutiesInFlight = null;
6154
+ });
6040
6155
  }
6041
6156
  /** Follower duties: no outbox drain, no LIVE. Mutations forward to the
6042
6157
  * leader; everything else (registration, per-query sync, poll) runs
@@ -6045,6 +6160,7 @@ var Sp00kySync = class Sp00kySync {
6045
6160
  this.tabRole = "follower";
6046
6161
  this.hub = null;
6047
6162
  this.forwarder = forwarder;
6163
+ this.leaderDutiesInFlight = null;
6048
6164
  this.killRefLiveQuery();
6049
6165
  forwarder.onLeaderMessage = (msg) => {
6050
6166
  switch (msg.type) {
@@ -6694,6 +6810,16 @@ var Sp00kySync = class Sp00kySync {
6694
6810
  * means the single resulting UI update lands after this completes.
6695
6811
  */
6696
6812
  async runSyncForQuery(hash, diff) {
6813
+ const prime = this.primeGate();
6814
+ if (prime !== this.settledPrime) {
6815
+ await prime;
6816
+ this.settledPrime = prime;
6817
+ const fresh = this.dataModule.getQueryByHash(hash);
6818
+ if (!fresh) return;
6819
+ const recomputed = new ArraySyncer(fresh.config.localArray, fresh.config.remoteArray).nextSet();
6820
+ if (!recomputed) return;
6821
+ diff = recomputed;
6822
+ }
6697
6823
  if (diff.added.length > 0 || diff.updated.length > 0) {
6698
6824
  const pendingDeletes = await this.getPendingDeleteIds();
6699
6825
  if (pendingDeletes.size > 0) diff = {
@@ -7211,8 +7337,8 @@ function selfAllowlistedVariant(flag, userId) {
7211
7337
 
7212
7338
  //#endregion
7213
7339
  //#region src/modules/devtools/index.ts
7214
- const CORE_VERSION = "0.0.1-canary.204";
7215
- const WASM_VERSION = "0.0.1-canary.204";
7340
+ const CORE_VERSION = "0.0.1-canary.205";
7341
+ const WASM_VERSION = "0.0.1-canary.205";
7216
7342
  const SURREAL_VERSION = "3.0.3";
7217
7343
  var DevToolsService = class DevToolsService {
7218
7344
  eventsHistory = [];
@@ -7406,7 +7532,7 @@ var DevToolsService = class DevToolsService {
7406
7532
  const info = this.unwrapInfo(res);
7407
7533
  this.localTablesAt = Date.now();
7408
7534
  if (info && info.tables) {
7409
- const names = Object.keys(info.tables);
7535
+ const names = Object.keys(info.tables).filter((n) => n !== "_00_circuit_snapshot");
7410
7536
  const changed = names.length !== this.localTables.length || names.some((n, i) => n !== this.localTables[i]);
7411
7537
  this.localTables = names;
7412
7538
  if (changed) this.notifyDevTools();
@@ -7936,13 +8062,29 @@ var AuthService = class {
7936
8062
 
7937
8063
  //#endregion
7938
8064
  //#region src/services/stream-processor/index.ts
7939
- /**
7940
- * Read a circuit snapshot out of the pre-`persistCircuit` persisted shape
7941
- * (`[[{ state }]]`, a raw SurrealDB result). Returns null for anything else.
7942
- */
7943
- function extractLegacyState(result) {
7944
- if (Array.isArray(result) && Array.isArray(result[0]) && typeof result[0][0]?.state === "string") return result[0][0].state;
7945
- return null;
8065
+ /** Storage key of the circuit snapshot inside the local store. */
8066
+ const CIRCUIT_SNAPSHOT_KEY = "circuit";
8067
+ /** Bump when the bytes `load_store_state` reads change shape. */
8068
+ const CIRCUIT_SNAPSHOT_FORMAT = 1;
8069
+ /** Rows per `ingest_many` call. Bounds the transient the wasm side allocates
8070
+ * to parse a batch (measured: one 7700-row call of 20 KB bodies peaked at
8071
+ * 500 MB, 128-row chunks at 335 MB, and under projection at 24 MB). */
8072
+ const INGEST_CHUNK = 128;
8073
+ /** Ids per `selectByIds` when priming bodies out of the local store. */
8074
+ const PRIME_CHUNK = 256;
8075
+ /** Rows the SurrealDB (main-thread, IndexedDB) engine is allowed to prime
8076
+ * through; above this the circuit boots empty as it always did. */
8077
+ const SURREAL_PRIME_ROW_CAP = 2e4;
8078
+ /** Below this many changed rows the checkpoint timer stays quiet. */
8079
+ const CHECKPOINT_MIN_ROWS = 50;
8080
+ /** Stable `table:id` for a row id that may already be a string. */
8081
+ function idString(id) {
8082
+ return typeof id === "string" ? id : encodeRecordId(id);
8083
+ }
8084
+ function chunks(items, size) {
8085
+ const out = [];
8086
+ for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
8087
+ return out;
7946
8088
  }
7947
8089
  var StreamProcessorService = class {
7948
8090
  logger;
@@ -7955,18 +8097,23 @@ var StreamProcessorService = class {
7955
8097
  authId: "",
7956
8098
  access: ""
7957
8099
  };
7958
- stateKeySuffix = "";
7959
8100
  stateGeneration = 0;
7960
8101
  persistState = true;
7961
8102
  persistCircuit = false;
7962
8103
  checkpointMs = 3e4;
7963
8104
  checkpointTimer = null;
7964
8105
  snapshotDirty = false;
7965
- pagehideHandler = null;
7966
- constructor(events, db, persistenceClient, logger) {
8106
+ dirtyRows = 0;
8107
+ hideHandler = null;
8108
+ checkpointInFlight = null;
8109
+ projection = true;
8110
+ primed = Promise.resolve();
8111
+ schemaHash = null;
8112
+ widenQueue = Promise.resolve();
8113
+ widenPending = /* @__PURE__ */ new Map();
8114
+ constructor(events, db, logger) {
7967
8115
  this.events = events;
7968
8116
  this.db = db;
7969
- this.persistenceClient = persistenceClient;
7970
8117
  this.logger = logger.child({ name: "StreamProcessorService" });
7971
8118
  }
7972
8119
  /**
@@ -7998,31 +8145,41 @@ var StreamProcessorService = class {
7998
8145
  for (const update of updates) for (const receiver of this.receivers) receiver.onStreamUpdate(update);
7999
8146
  }
8000
8147
  /**
8001
- * Ingest a batch of record changes as a single bulk operation, firing only
8002
- * one coalesced `StreamUpdate` per affected query once every record has been
8003
- * ingested (instead of one update per record). Use this whenever multiple
8004
- * records land at once — e.g. sync fetching N missing rows — so a list query
8005
- * re-runs and the UI re-renders once for the whole batch rather than
8006
- * row-by-row.
8148
+ * Ingest a batch of record changes, firing one coalesced `StreamUpdate` per
8149
+ * affected query once every record has been ingested. Use this whenever
8150
+ * multiple records land at once (sync fetching N rows, the boot prime).
8151
+ *
8152
+ * The batch is fed to the wasm side in chunks of {@link INGEST_CHUNK}: one
8153
+ * circuit step per chunk (a step walks every registered view, so per-record
8154
+ * ingest paid that fixed cost N times), but never the whole batch at once,
8155
+ * because the wasm side has to hold every parsed row of a call at the same
8156
+ * time and wasm32 dlmalloc never returns that peak.
8007
8157
  *
8008
- * Internally opens a coalescing window, ingests each record, then flushes;
8009
- * processor state is persisted once for the whole batch. No-op for an empty
8010
- * batch.
8158
+ * Returns the records that were ingested. A chunk that fails is reported
8159
+ * and skipped, not retried (a retry would double-apply whatever the failed
8160
+ * step already committed), and the remaining chunks still run.
8011
8161
  */
8012
8162
  ingestMany(records) {
8013
- if (records.length === 0) return;
8163
+ if (records.length === 0) return [];
8014
8164
  if (!this.processor) {
8015
8165
  this.logger.warn({ Category: "sp00ky-client::StreamProcessorService::ingestMany" }, "Not initialized, skipping ingest");
8016
- return;
8166
+ return [];
8017
8167
  }
8018
8168
  const bulkIngest = this.processor.ingest_many;
8019
- if (typeof bulkIngest === "function") {
8020
- this.logger.debug({
8021
- count: records.length,
8022
- Category: "sp00ky-client::StreamProcessorService::ingestMany"
8023
- }, "Ingesting batch into ssp");
8169
+ if (typeof bulkIngest !== "function") {
8170
+ this.beginCoalescing();
8024
8171
  try {
8025
- const items = records.map((record) => ({
8172
+ for (const record of records) this.ingest(record.table, record.op, record.id, record.record);
8173
+ } finally {
8174
+ this.flushCoalescing();
8175
+ }
8176
+ return records;
8177
+ }
8178
+ const ingested = [];
8179
+ this.beginCoalescing();
8180
+ try {
8181
+ for (const chunk of chunks(records, INGEST_CHUNK)) try {
8182
+ const items = chunk.map((record) => ({
8026
8183
  table: record.table,
8027
8184
  op: record.op,
8028
8185
  id: record.id,
@@ -8031,13 +8188,7 @@ var StreamProcessorService = class {
8031
8188
  const t0 = performance.now();
8032
8189
  const rawUpdates = bulkIngest.call(this.processor, items) ?? [];
8033
8190
  const materializationTimeMs = performance.now() - t0;
8034
- this.logger.debug({
8035
- count: records.length,
8036
- rawUpdates: rawUpdates.length,
8037
- materializationTimeMs,
8038
- Category: "sp00ky-client::StreamProcessorService::ingestMany"
8039
- }, "Ingesting batch into ssp done");
8040
- if (rawUpdates.length > 0) this.dispatchUpdates(rawUpdates.map((u) => ({
8191
+ if (rawUpdates.length > 0) this.notifyUpdates(rawUpdates.map((u) => ({
8041
8192
  queryHash: u.query_id,
8042
8193
  localArray: u.result_data,
8043
8194
  op: "CREATE",
@@ -8046,22 +8197,24 @@ var StreamProcessorService = class {
8046
8197
  circuitStepMs: u.timing_circuit_step_ms,
8047
8198
  transformMs: u.timing_transform_ms
8048
8199
  })));
8200
+ ingested.push(...chunk);
8049
8201
  } catch (e) {
8050
8202
  this.logger.error({
8051
8203
  error: e,
8052
- count: records.length,
8204
+ count: chunk.length,
8053
8205
  Category: "sp00ky-client::StreamProcessorService::ingestMany"
8054
- }, "Ingesting batch into ssp failed");
8206
+ }, "Ingesting chunk into ssp failed");
8055
8207
  }
8056
- this.markSnapshotDirty();
8057
- return;
8058
- }
8059
- this.beginCoalescing();
8060
- try {
8061
- for (const record of records) this.ingest(record.table, record.op, record.id, record.record);
8062
8208
  } finally {
8063
8209
  this.flushCoalescing();
8064
8210
  }
8211
+ this.logger.debug({
8212
+ count: records.length,
8213
+ ingested: ingested.length,
8214
+ Category: "sp00ky-client::StreamProcessorService::ingestMany"
8215
+ }, "Ingested batch into ssp");
8216
+ this.markSnapshotDirty(ingested.length);
8217
+ return ingested;
8065
8218
  }
8066
8219
  /**
8067
8220
  * Open a coalescing window. While open, the per-record stream updates
@@ -8088,7 +8241,6 @@ var StreamProcessorService = class {
8088
8241
  const buffered = Array.from(this.batchBuffer.values());
8089
8242
  this.batchBuffer.clear();
8090
8243
  if (buffered.length > 0) this.dispatchUpdates(buffered);
8091
- this.markSnapshotDirty();
8092
8244
  }
8093
8245
  /**
8094
8246
  * Initialize the WASM module and processor.
@@ -8100,7 +8252,7 @@ var StreamProcessorService = class {
8100
8252
  try {
8101
8253
  await init();
8102
8254
  this.processor = new Sp00kyProcessor();
8103
- await this.loadState();
8255
+ this.applyProjection();
8104
8256
  this.isInitialized = true;
8105
8257
  this.logger.info({ Category: "sp00ky-client::StreamProcessorService::init" }, "Initialized successfully");
8106
8258
  } catch (e) {
@@ -8111,21 +8263,16 @@ var StreamProcessorService = class {
8111
8263
  throw e;
8112
8264
  }
8113
8265
  }
8114
- /** Route the persisted circuit snapshot to a per-bucket key. */
8115
- setStateKeySuffix(bucketId) {
8116
- this.stateKeySuffix = bucketId;
8117
- }
8118
- stateKey() {
8119
- return this.stateKeySuffix ? `_00_stream_processor_state:${this.stateKeySuffix}` : "_00_stream_processor_state";
8120
- }
8121
8266
  /**
8122
8267
  * Drop the current WASM processor and start a fresh, empty circuit. Used on
8123
8268
  * local-bucket switches: the old circuit holds the previous user's rows AND
8124
8269
  * views registered with the previous `$auth` context, so neither may survive.
8125
- * Deliberately does NOT `loadState()` — a persisted snapshot references views
8126
- * under a dead sessionId salt; the DataModule rebind re-registers every live
8127
- * view against this fresh processor. Caller must re-seed `setPermissions`
8128
- * afterwards (a fresh circuit default-denies every table).
8270
+ * Deliberately loads nothing: the snapshot in the store being swapped away
8271
+ * from belongs to the previous bucket; the caller primes the new bucket's
8272
+ * circuit (`primeFromLocal`) once its store is open, and the DataModule
8273
+ * rebind re-registers every live view against this fresh processor. Caller
8274
+ * must re-seed `setPermissions` afterwards (a fresh circuit default-denies
8275
+ * every table).
8129
8276
  */
8130
8277
  async reset() {
8131
8278
  if (!this.isInitialized) return;
@@ -8133,8 +8280,11 @@ var StreamProcessorService = class {
8133
8280
  this.batching = false;
8134
8281
  this.batchBuffer.clear();
8135
8282
  this.snapshotDirty = false;
8283
+ this.dirtyRows = 0;
8284
+ this.widenPending.clear();
8136
8285
  const previous = this.processor;
8137
8286
  this.processor = new Sp00kyProcessor();
8287
+ this.applyProjection();
8138
8288
  this.freeProcessor(previous);
8139
8289
  this.logger.info({ Category: "sp00ky-client::StreamProcessorService::reset" }, "Stream processor reset (fresh circuit)");
8140
8290
  }
@@ -8174,9 +8324,9 @@ var StreamProcessorService = class {
8174
8324
  if (!enabled) this.stopCheckpoints();
8175
8325
  }
8176
8326
  /**
8177
- * Opt into snapshot persistence (`persistCircuit`). Off by default: see the
8178
- * `persistCircuit` field comment for why per-ingest snapshots were removed.
8179
- * Must be called before `init()` for a snapshot to be restored at boot.
8327
+ * Snapshot persistence (`persistCircuit`). When on, the circuit's store is
8328
+ * written to the local store on a checkpoint interval and when the page
8329
+ * goes hidden, and restored by {@link primeFromLocal} on the next boot.
8180
8330
  */
8181
8331
  configureCircuitPersistence(enabled, checkpointMs) {
8182
8332
  this.persistCircuit = enabled;
@@ -8184,59 +8334,291 @@ var StreamProcessorService = class {
8184
8334
  if (!enabled) this.stopCheckpoints();
8185
8335
  }
8186
8336
  /**
8187
- * Record that the circuit changed. Cheap and O(1), the expensive snapshot is
8188
- * deferred to the checkpoint timer, and skipped entirely when
8189
- * `persistCircuit` is off (the default).
8337
+ * Field projection (`circuitProjection`, default on). Takes effect on the
8338
+ * next processor (`init`/`reset`) and on rows written after that.
8339
+ */
8340
+ configureProjection(enabled) {
8341
+ this.projection = enabled;
8342
+ this.applyProjection();
8343
+ }
8344
+ applyProjection() {
8345
+ if (!this.processor || typeof this.processor.set_projection !== "function") return;
8346
+ try {
8347
+ this.processor.set_projection(this.projection);
8348
+ } catch (e) {
8349
+ this.logger.warn({
8350
+ error: e,
8351
+ Category: "sp00ky-client::StreamProcessorService::applyProjection"
8352
+ }, "set_projection failed");
8353
+ }
8354
+ }
8355
+ /** Resolves once the boot-time prime has finished (or was skipped). */
8356
+ whenPrimed() {
8357
+ return this.primed;
8358
+ }
8359
+ /**
8360
+ * Fill the circuit from the LOCAL store, in the background.
8361
+ *
8362
+ * With a usable snapshot: install it under whatever views have registered
8363
+ * meanwhile (`load_store_state` re-primes them), then `reconcile` each table
8364
+ * against the store's `(id, rv)` list so rows deleted since the checkpoint
8365
+ * are stepped out and only rows added or changed since are read back and
8366
+ * ingested. Without one: read every row and ingest it, chunked.
8367
+ *
8368
+ * Either way the circuit ends up equal to the local store without touching
8369
+ * the network, so the first sync diff is a real delta rather than "fetch
8370
+ * everything". The returned promise never rejects; `whenPrimed` gates on it.
8190
8371
  */
8191
- markSnapshotDirty() {
8372
+ primeFromLocal(ctx) {
8373
+ this.schemaHash = ctx.schemaHash;
8374
+ const run = this.runPrime(ctx).catch((e) => {
8375
+ this.logger.warn({
8376
+ error: e,
8377
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8378
+ }, "Circuit prime failed; the circuit fills from sync instead");
8379
+ });
8380
+ this.primed = run;
8381
+ return run;
8382
+ }
8383
+ async runPrime(ctx) {
8384
+ const processor = this.processor;
8385
+ if (!processor || typeof this.db.scanVersions !== "function") return;
8386
+ const generation = this.stateGeneration;
8387
+ const epoch = this.db.epoch;
8388
+ const alive = () => this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
8389
+ const t0 = performance.now();
8390
+ let restored = false;
8391
+ if (this.persistCircuit && typeof this.db.getSnapshot === "function" && typeof processor.load_store_state === "function") {
8392
+ const snapshot = await this.db.getSnapshot(CIRCUIT_SNAPSHOT_KEY);
8393
+ if (!alive()) return;
8394
+ if (snapshot) {
8395
+ const { meta } = snapshot;
8396
+ if (meta.formatVersion !== CIRCUIT_SNAPSHOT_FORMAT || meta.schemaHash !== ctx.schemaHash) this.logger.info({
8397
+ meta,
8398
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8399
+ }, "Circuit snapshot is from another format or schema; priming from rows");
8400
+ else try {
8401
+ const tLoad = performance.now();
8402
+ const updates = processor.load_store_state(snapshot.bytes);
8403
+ this.dispatchWasmUpdates(updates);
8404
+ restored = true;
8405
+ this.logger.info({
8406
+ bytes: snapshot.bytes.byteLength,
8407
+ views: updates.length,
8408
+ loadMs: Math.round(performance.now() - tLoad),
8409
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8410
+ }, "Circuit snapshot restored");
8411
+ } catch (e) {
8412
+ this.logger.warn({
8413
+ error: e,
8414
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8415
+ }, "Circuit snapshot unreadable; priming from rows");
8416
+ }
8417
+ }
8418
+ }
8419
+ const versions = await this.db.scanVersions(ctx.tables);
8420
+ if (!alive()) return;
8421
+ const total = Object.values(versions).reduce((n, v) => n + v.length, 0);
8422
+ if (!restored && this.db.engineKind !== "sqlite" && total > SURREAL_PRIME_ROW_CAP) {
8423
+ this.logger.warn({
8424
+ total,
8425
+ cap: SURREAL_PRIME_ROW_CAP,
8426
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8427
+ }, "Too many cached rows to prime through the main-thread engine; circuit fills from sync");
8428
+ return;
8429
+ }
8430
+ let ingested = 0;
8431
+ let fetched = 0;
8432
+ let deleted = 0;
8433
+ for (const table of ctx.tables) {
8434
+ const entries = versions[table] ?? [];
8435
+ let toFetch;
8436
+ if (restored && typeof processor.reconcile === "function") {
8437
+ const result = processor.reconcile(table, entries);
8438
+ this.dispatchWasmUpdates(result.updates);
8439
+ deleted += result.deleted;
8440
+ toFetch = result.fetch;
8441
+ } else toFetch = entries.map(([id]) => id);
8442
+ for (const chunk of chunks(toFetch, PRIME_CHUNK)) {
8443
+ const rows = await this.db.selectByIds(table, chunk);
8444
+ if (!alive()) return;
8445
+ fetched += rows.length;
8446
+ ingested += this.ingestMany(rows.map((row) => ({
8447
+ table,
8448
+ op: "CREATE",
8449
+ id: idString(row.id),
8450
+ record: row
8451
+ }))).length;
8452
+ }
8453
+ if (entries.length > 0) ctx.onVersions?.(table, entries.filter(([id]) => !ctx.pendingIds.has(id)));
8454
+ }
8455
+ if (!restored && ingested > 0) this.markSnapshotDirty(Math.max(ingested, CHECKPOINT_MIN_ROWS));
8456
+ this.logger.info({
8457
+ restored,
8458
+ rows: total,
8459
+ fetched,
8460
+ ingested,
8461
+ deleted,
8462
+ ms: Math.round(performance.now() - t0),
8463
+ Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
8464
+ }, "Circuit primed from the local store");
8465
+ }
8466
+ /** Publish wasm updates produced outside an ingest (restore, reconcile). */
8467
+ dispatchWasmUpdates(updates) {
8468
+ if (!updates || updates.length === 0) return;
8469
+ this.dispatchUpdates(updates.map((u) => ({
8470
+ queryHash: u.query_id,
8471
+ localArray: u.result_data,
8472
+ op: "CREATE"
8473
+ })));
8474
+ }
8475
+ /**
8476
+ * Record that the circuit changed by `rows` rows. Cheap; the snapshot is
8477
+ * deferred to the checkpoint timer and skipped entirely when `persistCircuit`
8478
+ * is off.
8479
+ */
8480
+ markSnapshotDirty(rows = 1) {
8192
8481
  if (!this.persistCircuit || !this.persistState) return;
8193
8482
  this.snapshotDirty = true;
8483
+ this.dirtyRows += rows;
8194
8484
  this.startCheckpoints();
8195
8485
  }
8196
8486
  startCheckpoints() {
8197
8487
  if (this.checkpointTimer) return;
8198
8488
  this.checkpointTimer = setInterval(() => {
8199
- if (!this.snapshotDirty) return;
8200
- this.snapshotDirty = false;
8201
- this.saveState();
8489
+ if (!this.snapshotDirty || this.dirtyRows < CHECKPOINT_MIN_ROWS) return;
8490
+ this.checkpoint("interval");
8202
8491
  }, this.checkpointMs);
8203
- if (typeof window !== "undefined" && !this.pagehideHandler) {
8204
- this.pagehideHandler = () => {
8492
+ if (typeof window !== "undefined" && !this.hideHandler) {
8493
+ this.hideHandler = () => {
8205
8494
  if (!this.snapshotDirty) return;
8206
- this.snapshotDirty = false;
8207
- this.saveState();
8495
+ if (typeof document !== "undefined" && document.visibilityState === "visible") return;
8496
+ this.checkpoint("hidden");
8208
8497
  };
8209
- window.addEventListener("pagehide", this.pagehideHandler);
8498
+ window.addEventListener("visibilitychange", this.hideHandler);
8499
+ window.addEventListener("pagehide", this.hideHandler);
8210
8500
  }
8211
8501
  }
8212
- /** Stop checkpointing and drop the `pagehide` listener. */
8502
+ /** Stop checkpointing and drop the visibility listeners. */
8213
8503
  stopCheckpoints() {
8214
8504
  if (this.checkpointTimer) {
8215
8505
  clearInterval(this.checkpointTimer);
8216
8506
  this.checkpointTimer = null;
8217
8507
  }
8218
- if (this.pagehideHandler && typeof window !== "undefined") window.removeEventListener("pagehide", this.pagehideHandler);
8219
- this.pagehideHandler = null;
8508
+ if (this.hideHandler && typeof window !== "undefined") {
8509
+ window.removeEventListener("visibilitychange", this.hideHandler);
8510
+ window.removeEventListener("pagehide", this.hideHandler);
8511
+ }
8512
+ this.hideHandler = null;
8220
8513
  this.snapshotDirty = false;
8514
+ this.dirtyRows = 0;
8515
+ }
8516
+ /**
8517
+ * Write the circuit's store to the local store as a snapshot. Compacts the
8518
+ * row arena first when dead bytes outweigh live ones. Serialised: a second
8519
+ * call while one is in flight joins it. No-op unless persistence is on, this
8520
+ * tab owns the store, and the engine can hold a snapshot.
8521
+ */
8522
+ checkpoint(reason = "manual") {
8523
+ if (this.checkpointInFlight) return this.checkpointInFlight;
8524
+ const run = this.runCheckpoint(reason).finally(() => {
8525
+ this.checkpointInFlight = null;
8526
+ });
8527
+ this.checkpointInFlight = run;
8528
+ return run;
8221
8529
  }
8222
- async loadState() {
8223
- if (!this.processor || !this.persistState || !this.persistCircuit) return;
8530
+ async runCheckpoint(reason) {
8531
+ const processor = this.processor;
8532
+ if (!processor || !this.persistState || !this.persistCircuit || !this.schemaHash) return;
8533
+ if (typeof this.db.putSnapshot !== "function" || typeof processor.save_store_state !== "function") return;
8534
+ const generation = this.stateGeneration;
8535
+ const epoch = this.db.epoch;
8536
+ const rows = this.dirtyRows;
8537
+ this.snapshotDirty = false;
8538
+ this.dirtyRows = 0;
8224
8539
  try {
8225
- const result = await this.persistenceClient.get(this.stateKey());
8226
- const state = typeof result === "string" ? result : extractLegacyState(result);
8227
- if (state) {
8228
- this.logger.info({
8229
- stateLength: state.length,
8230
- Category: "sp00ky-client::StreamProcessorService::loadState"
8231
- }, "Loading state from DB");
8232
- if (typeof this.processor.load_state === "function") this.processor.load_state(state);
8233
- else this.logger.warn({ Category: "sp00ky-client::StreamProcessorService::loadState" }, "load_state method not found on processor");
8234
- } else this.logger.info({ Category: "sp00ky-client::StreamProcessorService::loadState" }, "No saved state found");
8540
+ const t0 = performance.now();
8541
+ let reclaimed = 0;
8542
+ if (typeof processor.compact === "function" && typeof processor.dead_bytes === "function" && typeof processor.live_bytes === "function") {
8543
+ const dead = processor.dead_bytes();
8544
+ if (dead > 1e6 && dead > processor.live_bytes()) reclaimed = processor.compact();
8545
+ }
8546
+ const bytes = processor.save_store_state();
8547
+ const meta = {
8548
+ formatVersion: CIRCUIT_SNAPSHOT_FORMAT,
8549
+ schemaHash: this.schemaHash,
8550
+ savedAt: Date.now(),
8551
+ maxRv: typeof processor.max_row_versions === "function" ? processor.max_row_versions() : void 0
8552
+ };
8553
+ if (generation !== this.stateGeneration || epoch !== this.db.epoch) return;
8554
+ await this.db.putSnapshot(CIRCUIT_SNAPSHOT_KEY, bytes, meta);
8555
+ this.logger.info({
8556
+ reason,
8557
+ rows,
8558
+ bytes: bytes.byteLength,
8559
+ reclaimed,
8560
+ ms: Math.round(performance.now() - t0),
8561
+ Category: "sp00ky-client::StreamProcessorService::checkpoint"
8562
+ }, "Circuit snapshot written");
8235
8563
  } catch (e) {
8236
- this.logger.error({
8564
+ this.logger.warn({
8565
+ error: e,
8566
+ reason,
8567
+ Category: "sp00ky-client::StreamProcessorService::checkpoint"
8568
+ }, "Circuit snapshot failed");
8569
+ }
8570
+ }
8571
+ /**
8572
+ * Projection widening: a newly registered view evaluates fields that rows
8573
+ * already in the circuit were stored without. Merge just those fields in,
8574
+ * table by table, from the local store. The view registered against what
8575
+ * was present and converges as the merges step through.
8576
+ */
8577
+ scheduleWiden(missing) {
8578
+ for (const [table, fields] of Object.entries(missing)) {
8579
+ const set = this.widenPending.get(table) ?? /* @__PURE__ */ new Set();
8580
+ for (const f of fields) set.add(f);
8581
+ this.widenPending.set(table, set);
8582
+ }
8583
+ this.widenQueue = this.widenQueue.then(() => this.runWiden()).catch((e) => {
8584
+ this.logger.warn({
8237
8585
  error: e,
8238
- Category: "sp00ky-client::StreamProcessorService::loadState"
8239
- }, "Failed to load state");
8586
+ Category: "sp00ky-client::StreamProcessorService::widen"
8587
+ }, "Projection widening failed");
8588
+ });
8589
+ }
8590
+ async runWiden() {
8591
+ const processor = this.processor;
8592
+ if (!processor || typeof this.db.scanVersions !== "function") {
8593
+ this.widenPending.clear();
8594
+ return;
8595
+ }
8596
+ const generation = this.stateGeneration;
8597
+ const epoch = this.db.epoch;
8598
+ while (this.widenPending.size > 0) {
8599
+ const [table, fields] = this.widenPending.entries().next().value;
8600
+ this.widenPending.delete(table);
8601
+ const select = Array.from(fields);
8602
+ const versions = await this.db.scanVersions([table]);
8603
+ if (this.processor !== processor || generation !== this.stateGeneration || epoch !== this.db.epoch) return;
8604
+ const ids = (versions[table] ?? []).map(([id]) => id);
8605
+ let merged = 0;
8606
+ for (const chunk of chunks(ids, PRIME_CHUNK)) {
8607
+ const rows = await this.db.selectByIds(table, chunk, { select });
8608
+ if (this.processor !== processor || generation !== this.stateGeneration || epoch !== this.db.epoch) return;
8609
+ merged += this.ingestMany(rows.map((row) => ({
8610
+ table,
8611
+ op: "MERGE",
8612
+ id: idString(row.id),
8613
+ record: row
8614
+ }))).length;
8615
+ }
8616
+ this.logger.info({
8617
+ table,
8618
+ fields: select,
8619
+ merged,
8620
+ Category: "sp00ky-client::StreamProcessorService::widen"
8621
+ }, "Widened projected rows with newly evaluated fields");
8240
8622
  }
8241
8623
  }
8242
8624
  /**
@@ -8277,25 +8659,6 @@ var StreamProcessorService = class {
8277
8659
  Category: "sp00ky-client::StreamProcessorService::setSessionAuth"
8278
8660
  }, "Session auth context updated");
8279
8661
  }
8280
- async saveState() {
8281
- if (!this.processor || !this.persistState || !this.persistCircuit) return;
8282
- const generation = this.stateGeneration;
8283
- try {
8284
- if (typeof this.processor.save_state === "function") {
8285
- const state = this.processor.save_state();
8286
- if (generation !== this.stateGeneration) return;
8287
- if (state) {
8288
- await this.persistenceClient.set(this.stateKey(), state);
8289
- this.logger.trace({ Category: "sp00ky-client::StreamProcessorService::saveState" }, "State saved");
8290
- }
8291
- }
8292
- } catch (e) {
8293
- this.logger.error({
8294
- error: e,
8295
- Category: "sp00ky-client::StreamProcessorService::saveState"
8296
- }, "Failed to save state");
8297
- }
8298
- }
8299
8662
  /**
8300
8663
  * Ingest a record change into the processor.
8301
8664
  * Emits 'stream_update' event if materialized views are affected.
@@ -8329,7 +8692,7 @@ var StreamProcessorService = class {
8329
8692
  const updates = rawUpdates.map((u) => ({
8330
8693
  queryHash: u.query_id,
8331
8694
  localArray: u.result_data,
8332
- op,
8695
+ op: op === "MERGE" ? "UPDATE" : op,
8333
8696
  materializationTimeMs,
8334
8697
  storeApplyMs: u.timing_store_apply_ms,
8335
8698
  circuitStepMs: u.timing_circuit_step_ms,
@@ -8390,7 +8753,7 @@ var StreamProcessorService = class {
8390
8753
  snapshotMs: initialUpdate.timing_snapshot_ms ?? 0
8391
8754
  }
8392
8755
  };
8393
- this.markSnapshotDirty();
8756
+ if (initialUpdate.missing_fields && Object.keys(initialUpdate.missing_fields).length > 0) this.scheduleWiden(initialUpdate.missing_fields);
8394
8757
  this.logger.debug({
8395
8758
  queryHash: queryPlan.queryHash,
8396
8759
  surql: queryPlan.surql,
@@ -8413,7 +8776,6 @@ var StreamProcessorService = class {
8413
8776
  if (!this.processor) return;
8414
8777
  try {
8415
8778
  this.processor.unregister_view(queryHash);
8416
- this.markSnapshotDirty();
8417
8779
  } catch (e) {
8418
8780
  this.logger.error({
8419
8781
  error: e,
@@ -8544,6 +8906,16 @@ var CacheModule = class {
8544
8906
  lookup(recordId) {
8545
8907
  return this.versionLookups[recordId] ?? 0;
8546
8908
  }
8909
+ /**
8910
+ * Seed the version memo from rows the circuit was primed with out of the
8911
+ * local store, so the first post-reload sync diff does not re-download
8912
+ * bodies the browser already has. Only rows the prime actually put into the
8913
+ * circuit belong here: a memo entry with no circuit row would make the diff
8914
+ * flag the id forever while nothing ever fetches it.
8915
+ */
8916
+ primeVersions(entries) {
8917
+ for (const [id, rv] of entries) if (rv > (this.versionLookups[id] ?? 0)) this.versionLookups[id] = rv;
8918
+ }
8547
8919
  /** Drop the version cache on a bucket switch — a stale version would make
8548
8920
  * the sync diff skip fetching a body the new bucket legitimately needs. */
8549
8921
  clearVersionLookups() {
@@ -8592,9 +8964,10 @@ var CacheModule = class {
8592
8964
  await this.local.execute(query, params, { epoch });
8593
8965
  }
8594
8966
  if (this.local.epoch !== epoch) throw new StaleEpochError();
8967
+ const versionOf = /* @__PURE__ */ new Map();
8595
8968
  const bulk = populatedRecords.map((record) => {
8596
8969
  const recordId = encodeRecordId(record.record.id);
8597
- this.versionLookups[recordId] = record.version;
8970
+ versionOf.set(recordId, record.version);
8598
8971
  return {
8599
8972
  table: record.table,
8600
8973
  op: record.op,
@@ -8602,8 +8975,9 @@ var CacheModule = class {
8602
8975
  record: record.record
8603
8976
  };
8604
8977
  });
8605
- this.streamProcessor.ingestMany(bulk);
8606
- this.ingestRelay?.(bulk);
8978
+ const ingested = this.streamProcessor.ingestMany(bulk);
8979
+ for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
8980
+ if (ingested.length > 0) this.ingestRelay?.(ingested);
8607
8981
  this.logger.debug({
8608
8982
  count: records.length,
8609
8983
  Category: "sp00ky-client::CacheModule::saveBatch"
@@ -8638,7 +9012,12 @@ var CacheModule = class {
8638
9012
  if (!skipDbDelete) await this.local.query("DELETE $id", { id: parseRecordIdString(id) }, { epoch });
8639
9013
  if (this.local.epoch !== epoch) throw new StaleEpochError();
8640
9014
  delete this.versionLookups[id];
8641
- this.streamProcessor.ingest(table, "DELETE", id, recordData);
9015
+ this.streamProcessor.ingestMany([{
9016
+ table,
9017
+ op: "DELETE",
9018
+ id,
9019
+ record: recordData
9020
+ }]);
8642
9021
  this.ingestRelay?.([{
8643
9022
  table,
8644
9023
  op: "DELETE",
@@ -10228,6 +10607,10 @@ var TabsCoordinator = class {
10228
10607
  leadershipId = 0;
10229
10608
  leaderTabId = null;
10230
10609
  brokerRole = "pending";
10610
+ /** Wall time from `become-leader` to `leader-ready` on the last promotion.
10611
+ * Reported in the DevTools role-change event: a slow store adopt (OPFS pool
10612
+ * contention after the previous leader died) shows up here as seconds. */
10613
+ lastPromotionMs = null;
10231
10614
  fingerprint;
10232
10615
  bucketId;
10233
10616
  broker;
@@ -10278,6 +10661,7 @@ var TabsCoordinator = class {
10278
10661
  this.deps.logger.info({
10279
10662
  role,
10280
10663
  leadershipId: this.leadershipId,
10664
+ promotionMs: role === "leader" ? this.lastPromotionMs : void 0,
10281
10665
  Category: "sp00ky-client::TabsCoordinator"
10282
10666
  }, "Tab role changed");
10283
10667
  for (const cb of this.roleListeners) cb(role);
@@ -10354,6 +10738,7 @@ var TabsCoordinator = class {
10354
10738
  if (this.closed) return;
10355
10739
  if (msg.leadershipId <= this.leadershipId && !msg.resumeHeld) return;
10356
10740
  const previousRole = this.role;
10741
+ const startedAt = Date.now();
10357
10742
  try {
10358
10743
  if (previousRole === "follower") this.teardownFollower("promoted");
10359
10744
  this.leadershipId = msg.leadershipId;
@@ -10375,7 +10760,8 @@ var TabsCoordinator = class {
10375
10760
  resumeHeld: msg.resumeHeld
10376
10761
  });
10377
10762
  this.hub = new LeaderSyncHub(msg.leadershipId, this.deps.logger);
10378
- await this.deps.hooks.becomeSyncLeader(this.hub);
10763
+ this.deps.hooks.becomeSyncLeader(this.hub);
10764
+ this.lastPromotionMs = Date.now() - startedAt;
10379
10765
  this.setRole("leader");
10380
10766
  this.broker.send({
10381
10767
  type: "leader-ready",
@@ -10383,6 +10769,12 @@ var TabsCoordinator = class {
10383
10769
  bucketId: this.bucketId,
10384
10770
  leadershipId: msg.leadershipId
10385
10771
  });
10772
+ this.deps.hooks.resumeSyncLeaderDuties().catch((e) => {
10773
+ this.deps.logger.error({
10774
+ err: e,
10775
+ Category: "sp00ky-client::TabsCoordinator"
10776
+ }, "Leader duties failed after promotion");
10777
+ });
10386
10778
  } catch (e) {
10387
10779
  const reason = e instanceof Error ? e.message : String(e);
10388
10780
  this.deps.logger.error({
@@ -11821,8 +12213,9 @@ var Sp00kyClient = class {
11821
12213
  else if (config.persistenceClient === "localstorage" || !config.persistenceClient) this.persistenceClient = new LocalStoragePersistenceClient(logger);
11822
12214
  else this.persistenceClient = config.persistenceClient;
11823
12215
  this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
11824
- this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local, this.persistenceClient, logger);
11825
- this.streamProcessor.configureCircuitPersistence(config.persistCircuit ?? false, config.circuitCheckpointMs);
12216
+ this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local, logger);
12217
+ this.streamProcessor.configureCircuitPersistence(config.persistCircuit ?? config.localEngine === "sqlite", config.circuitCheckpointMs);
12218
+ this.streamProcessor.configureProjection(config.circuitProjection ?? true);
11826
12219
  this.migrator = new LocalMigrator(this.local, logger);
11827
12220
  this.cache = new CacheModule(this.local, this.streamProcessor, (update) => {
11828
12221
  this.dataModule.onStreamUpdate(update);
@@ -11838,6 +12231,7 @@ var Sp00kyClient = class {
11838
12231
  downTimeoutMs: this.config.downTimeoutMs,
11839
12232
  connectionSupervisor: this.connectionSupervisor
11840
12233
  });
12234
+ this.sync.setPrimeGate(() => this.streamProcessor.whenPrimed());
11841
12235
  this.featureFlags = new FeatureFlagModule({
11842
12236
  dataModule: this.dataModule,
11843
12237
  sync: this.sync,
@@ -11890,7 +12284,7 @@ var Sp00kyClient = class {
11890
12284
  return new TabsCoordinator({
11891
12285
  tabId,
11892
12286
  fingerprint: computeTabsFingerprint({
11893
- coreVersion: "0.0.1-canary.204",
12287
+ coreVersion: "0.0.1-canary.205",
11894
12288
  schemaHash: hash53(this.config.schemaSurql),
11895
12289
  endpoint: this.config.database.endpoint ?? "",
11896
12290
  namespace: this.config.database.namespace,
@@ -11907,13 +12301,14 @@ var Sp00kyClient = class {
11907
12301
  onLeaderLost: (reason) => engine.onLeaderLost(reason),
11908
12302
  exposeClientPort: (clientId, port) => engine.exposeClientPort(clientId, port),
11909
12303
  removeClientPort: (clientId) => engine.removeClientPort(clientId),
11910
- becomeSyncLeader: async (hub) => {
12304
+ becomeSyncLeader: (hub) => {
11911
12305
  this.streamProcessor.setPersistenceEnabled(true);
11912
12306
  this.cache.setIngestRelay((tuples) => hub.relayIngest(tuples));
11913
12307
  this.sync.setTabContext("leader", tabId);
11914
12308
  this.dataModule.setTabId(tabId);
11915
- await this.sync.promoteToLeader(hub);
12309
+ this.sync.promoteToLeader(hub);
11916
12310
  },
12311
+ resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
11917
12312
  becomeSyncFollower: (forwarder) => {
11918
12313
  this.streamProcessor.setPersistenceEnabled(false);
11919
12314
  this.cache.setIngestRelay(null);
@@ -12007,6 +12402,13 @@ var Sp00kyClient = class {
12007
12402
  if (this.tabsCoordinator) {
12008
12403
  this.tabsCoordinator.onRoleChange((role) => {
12009
12404
  this.sharedActive = role !== "solo";
12405
+ this.devTools.logEvent("TABS_ROLE_CHANGED", {
12406
+ role,
12407
+ tabId: this.tabsCoordinator?.tabId,
12408
+ leadershipId: this.tabsCoordinator?.leadershipId,
12409
+ leaderTabId: this.tabsCoordinator?.leaderTabId,
12410
+ promotionMs: this.tabsCoordinator?.lastPromotionMs
12411
+ });
12010
12412
  });
12011
12413
  try {
12012
12414
  const role = await this.tabsCoordinator.start(bootBucket);
@@ -12044,9 +12446,9 @@ var Sp00kyClient = class {
12044
12446
  }, "Blob cache failed to start; bucket files will not be cached locally");
12045
12447
  }
12046
12448
  })();
12047
- this.streamProcessor.setStateKeySuffix(bootBucket);
12048
12449
  await this.streamProcessor.init();
12049
12450
  this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
12451
+ this.primeCircuit();
12050
12452
  this.logger.debug({ Category: "sp00ky-client::Sp00kyClient::init" }, "StreamProcessor initialized");
12051
12453
  const restoredUserId = await this.auth.restoreSessionFromToken();
12052
12454
  const sessionId = this.mintSessionSalt();
@@ -12215,10 +12617,10 @@ var Sp00kyClient = class {
12215
12617
  Category: "sp00ky-client::Sp00kyClient::doSwitchBucket"
12216
12618
  }, "Blob cache bucket switch failed");
12217
12619
  }
12218
- this.streamProcessor.setStateKeySuffix(target);
12219
12620
  await this.streamProcessor.reset();
12220
12621
  this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
12221
12622
  this.cache.clearVersionLookups();
12623
+ this.primeCircuit();
12222
12624
  this.preloadedHashes.clear();
12223
12625
  } finally {
12224
12626
  reopen();
@@ -12250,6 +12652,7 @@ var Sp00kyClient = class {
12250
12652
  this.crdtManager.closeAll();
12251
12653
  this.crdtManager.dispose();
12252
12654
  await this.blobs.close().catch(() => {});
12655
+ await this.streamProcessor.checkpoint("close").catch(() => {});
12253
12656
  if (this.tabsCoordinator) await this.tabsCoordinator.stop();
12254
12657
  await this.local.close();
12255
12658
  await this.remote.close();
@@ -12537,6 +12940,31 @@ var Sp00kyClient = class {
12537
12940
  * the former to `encodeRecordId` reads `.table` off a string and throws
12538
12941
  * during boot.
12539
12942
  */
12943
+ /**
12944
+ * Prime the in-browser circuit from the local store. Builds the context the
12945
+ * stream processor needs: every synced table (the app schema plus the
12946
+ * server-written meta tables that sync down), a schema hash so a snapshot
12947
+ * projected under another schema is not trusted, and the ids whose local
12948
+ * `_00_rv` was bumped by an unsettled mutation.
12949
+ */
12950
+ async primeCircuit() {
12951
+ const tables = [
12952
+ ...this.config.schema.tables.map((t) => t.name),
12953
+ "_00_user_feature",
12954
+ "_00_app_release"
12955
+ ];
12956
+ let pendingIds = /* @__PURE__ */ new Set();
12957
+ try {
12958
+ const pending = await this.dataModule.getPendingRecordIds();
12959
+ pendingIds = new Set([...pending.writes, ...pending.deletes]);
12960
+ } catch {}
12961
+ await this.streamProcessor.primeFromLocal({
12962
+ tables,
12963
+ schemaHash: String(hash53(this.config.schemaSurql)),
12964
+ pendingIds,
12965
+ onVersions: (_table, entries) => this.cache.primeVersions(entries)
12966
+ });
12967
+ }
12540
12968
  sessionAuthId() {
12541
12969
  const id = this.auth.currentUser?.id;
12542
12970
  if (!id) return null;