@spooky-sync/core 0.0.1-canary.174 → 0.0.1-canary.176

package/dist/index.d.ts

@@ -848,7 +848,10 @@ declare class DataModule<S extends SchemaStructure> {
   getActiveQueries(): QueryState[];
   getActiveQueryHashes(): QueryHash[];
   updateQueryLocalArray(id: string, localArray: RecordVersionArray): Promise<void>;
-  updateQueryRemoteArray(hash: string, remoteArray: RecordVersionArray): Promise<void>;
+  updateQueryRemoteArray(hash: string, remoteArray: RecordVersionArray, opts?: {
+    /** `_00_query.rowCount` read in the same round trip; `null` = unknown. */
+    serverRowCount?: number | null;
+  }): Promise<void>;
   /**
    * Cancel every armed timer ahead of a local-bucket switch: stream-update
    * debounce timers (their pending updates carry the OLD bucket's id-sets) and

package/dist/index.js

@@ -2624,6 +2624,7 @@ var SqliteCacheEngine = class {
 		let sql = `SELECT data FROM "${plan.table}"`;
 		if (plan.where && plan.where.length > 0) sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
 		if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
+		else sql += ` ORDER BY id`;
 		if (plan.limit !== void 0) sql += ` LIMIT ${Number(plan.limit)}`;
 		if (plan.offset !== void 0) sql += ` OFFSET ${Number(plan.offset)}`;
 		const rows = await this.execRows(sql, bind);
@@ -3565,6 +3566,9 @@ var DataModule = class {
 				ordered.push(id);
 			}
 		}
+		const hasExplicitOrder = (config.plan?.orderBy?.length ?? 0) > 0;
+		const isWindow = buildWindowMaterialization(config.surql) !== null;
+		if (!hasExplicitOrder && !isWindow) ordered.sort();
 		return ordered.map((id) => parseRecordIdString(id));
 	}
 	async processStreamUpdate(update) {
@@ -4009,7 +4013,7 @@ var DataModule = class {
 			throw err;
 		}
 	}
-	async updateQueryRemoteArray(hash, remoteArray) {
+	async updateQueryRemoteArray(hash, remoteArray, opts) {
 		const queryState = this.getQueryByHash(hash);
 		if (!queryState) {
 			this.logger.warn({
@@ -4019,15 +4023,19 @@ var DataModule = class {
 			return;
 		}
 		if (remoteArray.length === 0 && !queryState.config.remoteSeen) {
-			const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
-			queryState.config.emptyReads = emptyReads;
-			if (emptyReads < EMPTY_MEMBERSHIP_CONFIRMATIONS) {
-				this.logger.debug({
-					hash,
-					emptyReads,
-					Category: "sp00ky-client::DataModule::updateQueryRemoteArray"
-				}, "Ignoring unconfirmed empty membership (server may not have flushed list_ref yet)");
-				return;
+			const serverRowCount = opts?.serverRowCount;
+			if (!(serverRowCount === 0)) {
+				const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
+				queryState.config.emptyReads = emptyReads;
+				if (!(serverRowCount === null || serverRowCount === void 0 ? emptyReads >= EMPTY_MEMBERSHIP_CONFIRMATIONS : false)) {
+					this.logger.debug({
+						hash,
+						emptyReads,
+						serverRowCount,
+						Category: "sp00ky-client::DataModule::updateQueryRemoteArray"
+					}, "Ignoring empty membership: the server still reports rows for this query");
+					return;
+				}
 			}
 		}
 		const epoch = this.local.epoch;
@@ -5107,6 +5115,24 @@ function buildListRefSelect(table) {
 	return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
 }
 /**
+* Build the select that says whether an EMPTY id-set means "this query has no
+* rows" or "the server has not published them yet".
+*
+* The SSP writes `rowCount` onto the `_00_query` row in the same statement that
+* registers the view — synchronously, and BEFORE it hands the view's initial
+* edges to the coalescing edge flusher. So the two are not interchangeable:
+* `rowCount > 0` with no edges is the flush window, and only `rowCount === 0`
+* is a genuinely empty query. Polling the edges alone cannot tell those apart
+* no matter how long it waits, which is why this is read alongside them rather
+* than a retry counter.
+*
+* Returns `NONE` (→ null) when the row is not readable or does not exist yet;
+* callers must treat that as "unknown", not as zero.
+*/
+function buildQueryRowCountSelect() {
+	return "SELECT VALUE rowCount FROM ONLY $in";
+}
+/**
 * Build the SurrealQL select for a query's SUBQUERY child edges — the
 * mirror of {@link buildListRefSelect}. `.related()` queries register a
 * correlated subquery; the SSP materializes each matched child as a
@@ -5978,14 +6004,14 @@ var Sp00kySync = class Sp00kySync {
 		const queryState = this.dataModule.getQueryByHash(queryHash);
 		if (!queryState) return false;
 		const listRefTbl = this.listRefTable();
-		const [items] = await this.remote.query(buildListRefSelect(listRefTbl), { in: queryState.config.id });
+		const [items, serverRowCount] = await this.remote.query(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()}`, { in: queryState.config.id });
 		if (!Array.isArray(items)) return false;
 		const fresh = items.map((item) => [encodeRecordId(item.out), item.version]);
 		const prevRemote = queryState.config.remoteArray ?? [];
 		const freshIds = new Set(fresh.map(([id]) => id));
 		const removedIds = prevRemote.filter(([id]) => !freshIds.has(id)).map(([id]) => id);
 		const changed = !recordVersionArraysEqual(fresh, queryState.config.remoteArray);
-		if (changed) await this.dataModule.updateQueryRemoteArray(queryHash, fresh);
+		if (changed) await this.dataModule.updateQueryRemoteArray(queryHash, fresh, { serverRowCount });
 		try {
 			await this.syncQuery(queryHash);
 		} catch (err) {
@@ -6467,7 +6493,7 @@ var Sp00kySync = class Sp00kySync {
 			ttl: queryState.config.ttl
 		} });
 		const listRefTbl = this.listRefTable();
-		const [items] = await this.remote.query(buildListRefSelect(listRefTbl), { in: queryState.config.id });
+		const [items, serverRowCount] = await this.remote.query(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()}`, { in: queryState.config.id });
 		this.logger.trace({
 			queryId: encodeRecordId(queryState.config.id),
 			items,
@@ -6479,7 +6505,7 @@ var Sp00kySync = class Sp00kySync {
 			array,
 			Category: "sp00ky-client::Sp00kySync::createRemoteQuery"
 		}, "createdRemoteQuery");
-		if (array) await this.dataModule.updateQueryRemoteArray(queryHash, array);
+		if (array) await this.dataModule.updateQueryRemoteArray(queryHash, array, { serverRowCount });
 		await this.syncSubqueryChildren(queryHash).catch((err) => {
 			this.logger.info({
 				err: err?.message ?? err,
@@ -6866,8 +6892,8 @@ function selfAllowlistedVariant(flag, userId) {
 
 //#endregion
 //#region src/modules/devtools/index.ts
-const CORE_VERSION = "0.0.1-canary.174";
-const WASM_VERSION = "0.0.1-canary.174";
+const CORE_VERSION = "0.0.1-canary.176";
+const WASM_VERSION = "0.0.1-canary.176";
 const SURREAL_VERSION = "3.0.3";
 var DevToolsService = class DevToolsService {
 	eventsHistory = [];
@@ -11298,7 +11324,7 @@ var Sp00kyClient = class {
 		return new TabsCoordinator({
 			tabId,
 			fingerprint: computeTabsFingerprint({
-				coreVersion: "0.0.1-canary.174",
+				coreVersion: "0.0.1-canary.176",
 				schemaHash: hash53(this.config.schemaSurql),
 				endpoint: this.config.database.endpoint ?? "",
 				namespace: this.config.database.namespace,

package/dist/sqlite-worker.js

@@ -60,6 +60,7 @@ async function executeSelect(plan, params, db) {
 	let sql = `SELECT data FROM "${plan.table}"`;
 	if (plan.where && plan.where.length > 0) sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
 	if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
+	else sql += ` ORDER BY id`;
 	if (plan.limit !== void 0) sql += ` LIMIT ${Number(plan.limit)}`;
 	if (plan.offset !== void 0) sql += ` OFFSET ${Number(plan.offset)}`;
 	const rows = execRows(db, sql, bind);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.174",
+  "version": "0.0.1-canary.176",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.174",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.174",
+    "@spooky-sync/query-builder": "0.0.1-canary.176",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.176",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "fast-json-patch": "^3.1.1",

package/src/modules/data/data.membership.test.ts

@@ -229,6 +229,53 @@ describe('membership-authoritative rendering', () => {
     });
   });
 
+  describe('render order', () => {
+    it('sorts an unordered query so both paints agree', async () => {
+      // `_00_list_ref` is selected without an ORDER BY, so membership arrives
+      // shuffled. The first paint came from the local scan in id order, so
+      // rendering server order here is what made lists visibly reorder about a
+      // second after load.
+      const { dm, hash, state } = setup();
+      state.config.plan = { table: 'thread', where: [['done', '=', false]] } as any;
+
+      const ids = await (dm as any).buildRenderIds(state.config, [
+        ['thread:c', 1],
+        ['thread:a', 1],
+        ['thread:b', 1],
+      ]);
+
+      expect(ids.map(String)).toEqual(['thread:a', 'thread:b', 'thread:c']);
+    });
+
+    it('leaves an explicitly ordered query to the engine', async () => {
+      const { dm, state } = setup();
+      state.config.plan = { table: 'thread', orderBy: [['created', 'desc']] } as any;
+
+      const ids = await (dm as any).buildRenderIds(state.config, [
+        ['thread:c', 1],
+        ['thread:a', 1],
+      ]);
+
+      // Untouched: the engine applies the ORDER BY over the id set.
+      expect(ids.map(String)).toEqual(['thread:c', 'thread:a']);
+    });
+
+    it('preserves the slice order of a windowed query', async () => {
+      // For a window the id-set order IS the window: sorting it would reorder
+      // rows within the page.
+      const { dm, state } = setup();
+      state.config.surql = 'SELECT * FROM thread LIMIT 50 START 100;';
+      state.config.plan = { table: 'thread' } as any;
+
+      const ids = await (dm as any).buildRenderIds(state.config, [
+        ['thread:c', 1],
+        ['thread:a', 1],
+      ]);
+
+      expect(ids.map(String)).toEqual(['thread:c', 'thread:a']);
+    });
+  });
+
   describe('durability across a reload', () => {
     it('updateQueryRemoteArray latches membership and writes _00_window', async () => {
       const { dm, state, local, hash } = setup({ membershipKey: 'stable-key' });
@@ -326,6 +373,57 @@ describe('membership-authoritative rendering', () => {
       expect(state.config.remoteArray).toEqual([]);
     });
 
+    it('ignores an empty id-set while the server still reports rows', async () => {
+      // The reported failure: registration returns before the SSP flushes the
+      // view's edges, and the poll 500ms later is still inside that window. A
+      // retry counter believes the second read and blanks the list ~2s after
+      // load; `rowCount` says the query has 26 rows, so no number of empty
+      // reads should ever be taken as "empty".
+      const { dm, state, local, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      for (let i = 0; i < 5; i++) {
+        await dm.updateQueryRemoteArray(hash, [], { serverRowCount: 26 });
+      }
+
+      expect(state.config.remoteArray).toEqual([['thread:a', 1]]);
+      expect(local.windowRows.has('stable-key')).toBe(false);
+    });
+
+    it('believes an empty id-set the moment the server reports zero rows', async () => {
+      // The other half: a genuinely empty query must not sit on a stale seed
+      // waiting for a retry budget to run out.
+      const { dm, state, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      await dm.updateQueryRemoteArray(hash, [], { serverRowCount: 0 });
+
+      expect(state.config.membershipKnown).toBe(true);
+      expect(state.config.remoteArray).toEqual([]);
+    });
+
+    it('falls back to the retry budget when the row count is unreadable', async () => {
+      // Older servers, or a row the client cannot select: unknown must not
+      // strand the device on its durable seed forever.
+      const { dm, state, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      await dm.updateQueryRemoteArray(hash, [], { serverRowCount: null });
+      expect(state.config.remoteArray).toEqual([['thread:a', 1]]);
+
+      await dm.updateQueryRemoteArray(hash, [], { serverRowCount: null });
+      expect(state.config.remoteArray).toEqual([]);
+    });
+
     it('does not seed membership from an empty durable row', async () => {
       // Self-heals devices poisoned before the guard existed: an empty durable
       // row is indistinguishable from "never had membership", so it must fall

package/src/modules/data/index.ts

@@ -591,6 +591,18 @@ export class DataModule<S extends SchemaStructure> {
         ordered.push(id);
       }
     }
+    // `_00_list_ref` is selected without an ORDER BY, so this id-set arrives in
+    // whatever order the server happened to return. For a query with its own
+    // ORDER BY that does not matter (the engine sorts), and for a window the
+    // id-set order IS the window's slice order and must be preserved. Anything
+    // else renders in server order while its first paint came from the local
+    // scan in id order — the same rows, visibly reshuffled a second later.
+    // Sorting here is what makes the two paints agree.
+    const hasExplicitOrder = (config.plan?.orderBy?.length ?? 0) > 0;
+    const isWindow = buildWindowMaterialization(config.surql) !== null;
+    if (!hasExplicitOrder && !isWindow) {
+      ordered.sort();
+    }
     return ordered.map((id) => parseRecordIdString(id));
   }
 
@@ -1197,7 +1209,14 @@ export class DataModule<S extends SchemaStructure> {
     }
   }
 
-  async updateQueryRemoteArray(hash: string, remoteArray: RecordVersionArray): Promise<void> {
+  async updateQueryRemoteArray(
+    hash: string,
+    remoteArray: RecordVersionArray,
+    opts?: {
+      /** `_00_query.rowCount` read in the same round trip; `null` = unknown. */
+      serverRowCount?: number | null;
+    }
+  ): Promise<void> {
     const queryState = this.getQueryByHash(hash);
     if (!queryState) {
       this.logger.warn(
@@ -1220,20 +1239,40 @@ export class DataModule<S extends SchemaStructure> {
     // row, and overwriting that with `[]` would blank the very rows the seed
     // exists to paint. The `_00_list_ref` poll re-reads within ~500ms and
     // delivers the real set.
-    // Bounded, though: a query seeded from the durable row on a cold start has
-    // `remoteSeen === false`, and if its window really did empty while this
-    // device was away, the server will keep answering `[]`. Believe it on the
-    // second such read — one poll cycle (~500ms) past the flush window — so
-    // stale rows can't render forever.
+    // `serverRowCount` is what makes the two cases separable. The SSP writes it
+    // onto the `_00_query` row in the same statement that registers the view,
+    // before it queues the view's initial edges — so `> 0` with no edges is the
+    // flush window and `=== 0` is a genuinely empty query. `null`/undefined
+    // means we could not read it (older server, row not visible yet).
+    //
+    // Retry counting cannot substitute for this: the poll runs 500ms after
+    // registration, well inside the flush window for a real collection, so
+    // "believe it the second time" blanked exactly the lists this guard exists
+    // to protect — reported as rows vanishing ~2s after a page load.
     if (remoteArray.length === 0 && !queryState.config.remoteSeen) {
-      const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
-      queryState.config.emptyReads = emptyReads;
-      if (emptyReads < EMPTY_MEMBERSHIP_CONFIRMATIONS) {
-        this.logger.debug(
-          { hash, emptyReads, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
-          'Ignoring unconfirmed empty membership (server may not have flushed list_ref yet)'
-        );
-        return;
+      const serverRowCount = opts?.serverRowCount;
+      const knownEmpty = serverRowCount === 0;
+      if (!knownEmpty) {
+        // Unknown row count still gets a bounded escape hatch, so a server that
+        // cannot report one never strands a device on a durable seed forever.
+        const emptyReads = (queryState.config.emptyReads ?? 0) + 1;
+        queryState.config.emptyReads = emptyReads;
+        const exhausted =
+          serverRowCount === null || serverRowCount === undefined
+            ? emptyReads >= EMPTY_MEMBERSHIP_CONFIRMATIONS
+            : false; // a positive row count is never "confirmed empty"
+        if (!exhausted) {
+          this.logger.debug(
+            {
+              hash,
+              emptyReads,
+              serverRowCount,
+              Category: 'sp00ky-client::DataModule::updateQueryRemoteArray',
+            },
+            'Ignoring empty membership: the server still reports rows for this query'
+          );
+          return;
+        }
       }
     }
 

package/src/modules/sync/sync.ts

@@ -19,6 +19,7 @@ import {
   applyRecordVersionDiff,
   ArraySyncer,
   buildListRefSelect,
+  buildQueryRowCountSelect,
   buildSubqueryListRefSelect,
   createDiffFromDbOp,
   diffRecordVersionArray,
@@ -853,10 +854,11 @@ export class Sp00kySync<S extends SchemaStructure> {
     const queryState = this.dataModule.getQueryByHash(queryHash);
     if (!queryState) return false;
     const listRefTbl = this.listRefTable();
-    const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
-      buildListRefSelect(listRefTbl),
-      { in: queryState.config.id }
-    );
+    const [items, serverRowCount] = await this.remote.query<
+      [{ out: RecordId<string>; version: number }[], number | null]
+    >(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()}`, {
+      in: queryState.config.id,
+    });
     if (!Array.isArray(items)) return false;
     const fresh: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
     // Capture which ids LEFT the query's window (present in the cached
@@ -881,7 +883,7 @@ export class Sp00kySync<S extends SchemaStructure> {
       // and notifies subscribers. We skip an explicit `notifyQuerySynced`
       // because that path races the stream-update path (can notify with stale
       // records).
-      await this.dataModule.updateQueryRemoteArray(queryHash, fresh);
+      await this.dataModule.updateQueryRemoteArray(queryHash, fresh, { serverRowCount });
     }
     // Run `syncQuery` every tick regardless: it's a no-op when localArray has
     // caught up to remoteArray (`if (!diff) return`, issues no query), but it
@@ -1636,12 +1638,14 @@ export class Sp00kySync<S extends SchemaStructure> {
     // sync. `parent IS NONE` excludes subquery entries; the
     // `localArray` cache only tracks primary records.
     const listRefTbl = this.listRefTable();
-    const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
-      buildListRefSelect(listRefTbl),
-      {
-        in: queryState.config.id,
-      }
-    );
+    // `rowCount` rides along: it is written by the SSP in the same statement
+    // that registers the view, BEFORE the edges are flushed, so it is the only
+    // way to tell "this query is empty" from "its edges have not landed yet".
+    const [items, serverRowCount] = await this.remote.query<
+      [{ out: RecordId<string>; version: number }[], number | null]
+    >(`${buildListRefSelect(listRefTbl)};\n${buildQueryRowCountSelect()}`, {
+      in: queryState.config.id,
+    });
 
     this.logger.trace(
       {
@@ -1665,7 +1669,7 @@ export class Sp00kySync<S extends SchemaStructure> {
 
     if (array) {
       /// Incantation existed already
-      await this.dataModule.updateQueryRemoteArray(queryHash, array);
+      await this.dataModule.updateQueryRemoteArray(queryHash, array, { serverRowCount });
     }
 
     // Pull the bodies of any `.related()` subquery children into the local

package/src/modules/sync/utils.ts

@@ -188,6 +188,25 @@ export function buildListRefSelect(table: string): string {
   return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
 }
 
+/**
+ * Build the select that says whether an EMPTY id-set means "this query has no
+ * rows" or "the server has not published them yet".
+ *
+ * The SSP writes `rowCount` onto the `_00_query` row in the same statement that
+ * registers the view — synchronously, and BEFORE it hands the view's initial
+ * edges to the coalescing edge flusher. So the two are not interchangeable:
+ * `rowCount > 0` with no edges is the flush window, and only `rowCount === 0`
+ * is a genuinely empty query. Polling the edges alone cannot tell those apart
+ * no matter how long it waits, which is why this is read alongside them rather
+ * than a retry counter.
+ *
+ * Returns `NONE` (→ null) when the row is not readable or does not exist yet;
+ * callers must treat that as "unknown", not as zero.
+ */
+export function buildQueryRowCountSelect(): string {
+  return 'SELECT VALUE rowCount FROM ONLY $in';
+}
+
 /**
  * Build the SurrealQL select for a query's SUBQUERY child edges — the
  * mirror of {@link buildListRefSelect}. `.related()` queries register a

package/src/services/database/sqlite-cache-engine.ts

@@ -720,6 +720,10 @@ export class SqliteCacheEngine implements LocalStore {
       sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
     }
     if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
+    // Deterministic fallback, in parity with `sqlite-select.ts`: without it an
+    // unordered query renders in insertion order here and in membership order
+    // after the server answers, which reshuffles the list on screen.
+    else sql += ` ORDER BY id`;
     if (plan.limit !== undefined) sql += ` LIMIT ${Number(plan.limit)}`;
     if (plan.offset !== undefined) sql += ` OFFSET ${Number(plan.offset)}`;
     const rows = await this.execRows(sql, bind);

package/src/services/database/sqlite-select.ts

@@ -103,6 +103,14 @@ export async function executeSelect(
     sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
   }
   if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
+  // A query with no ORDER BY still has to render in SOME order, and "whatever
+  // SQLite hands back" is insertion order — which disagrees with the order the
+  // same query gets once it renders from server membership, and disagrees with
+  // SurrealDB, whose natural order is by id. That mismatch is visible: the
+  // first paint comes from this scan and the second from membership, so an
+  // unordered list visibly reshuffled about a second after load. Ordering by
+  // id here makes the two agree and makes the result stable across reloads.
+  else sql += ` ORDER BY id`;
   if (plan.limit !== undefined) sql += ` LIMIT ${Number(plan.limit)}`;
   if (plan.offset !== undefined) sql += ` OFFSET ${Number(plan.offset)}`;
   const rows = execRows(db, sql, bind);