@spooky-sync/core 0.0.1-canary.325 → 0.0.1-canary.327

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.325",
+  "version": "0.0.1-canary.327",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -75,8 +75,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.325",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.325",
+    "@spooky-sync/query-builder": "0.0.1-canary.327",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.327",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "blurhash": "^2.0.5",

package/src/client/runtime.ts

@@ -7,9 +7,9 @@ import type { Lane, Saga } from '../kernel/saga';
 import { runSaga } from '../kernel/saga';
 import type { ClientState } from '../state/client-state';
 import { emptyState } from '../state/client-state';
-import { deriveStatus, isAuthoritative } from '../state/lifecycle';
+import { isAuthoritative } from '../state/lifecycle';
 import * as R from '../state/reducers';
-import { fetchingQueryCount, pendingMutationCount } from '../state/selectors';
+import { fetchingQueryCount, pendingMutationCount, queryStatus } from '../state/selectors';
 import type { SagaEnv } from '../query/env';
 import { route } from './router';
 
@@ -152,7 +152,7 @@ export class Runtime {
     for (const [hash, subs] of this.statusSubs) {
       const entry = next.queries.get(hash);
       if (!entry) continue;
-      const status = deriveStatus(entry.lifecycle);
+      const status = queryStatus(next, hash) ?? 'idle';
       if (this.lastStatus.get(hash) === status) continue;
       this.lastStatus.set(hash, status);
       for (const cb of subs) this.safely(() => cb(status));
@@ -255,7 +255,7 @@ export class Runtime {
     this.statusSubs.set(hash, set);
     const entry = this.stateValue.queries.get(hash);
     if (entry) {
-      const status = deriveStatus(entry.lifecycle);
+      const status = queryStatus(this.stateValue, hash) ?? 'idle';
       this.lastStatus.set(hash, status);
       if (options.immediate) this.safely(() => cb(status));
     }

package/src/client/sp00ky-client.ts

@@ -603,7 +603,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
   }
 
   /** Close a `@crdt` field. The last close resolves once its unsent edits
-   *  are queued in the outbox. */
+   *  are queued in the outbox; offline, once they are saved on the device. */
   closeCrdtField(table: string, recordId: string, field: string): Promise<void> {
     return this.services.crdt.close(table, recordId, field);
   }

package/src/modules/crdt/crdt.test.ts

@@ -25,16 +25,26 @@ class World {
   peers = new Map<string, string>();
   published = new Map<string, { rev: number; snapshot: Uint8Array; upto?: unknown; state_hash?: string }>();
   online = true;
+  /** Offline remote calls never answer (queued behind a reconnect) instead of failing. */
+  hang = false;
+  /** A slow device store: each local write takes this long (real timers). */
+  storeDelayMs = 0;
   private watchers = new Set<{ table: string; where: Record<string, unknown>; cb: (rows: any[]) => void }>();
 
   client(user: string, store = new Map<string, any>(), logger: any = silent) {
     const local = {
       getById: async (_t: string, id: any) => store.get(String(id)) ?? null,
-      upsert: async (_t: string, id: any, data: any) => void store.set(String(id), structuredClone(data)),
+      upsert: async (_t: string, id: any, data: any) => {
+        const copy = structuredClone(data);
+        if (this.storeDelayMs) await new Promise((resolve) => setTimeout(resolve, this.storeDelayMs));
+        store.set(String(id), copy);
+      },
     } as any;
     const remote = {
       subscribeConnection: () => () => {},
+      getStatus: () => (this.online ? 'connected' : 'disconnected'),
       query: async (sql: string, vars: any) => {
+        if (!this.online && this.hang) return new Promise(() => {});
         if (!this.online) throw new Error('not connected');
         if (sql.startsWith('CREATE')) {
           const owner = this.peers.get(vars.peer);
@@ -186,6 +196,44 @@ describe('CRDT fields', () => {
     expect(world.peers.get(oldPeer)).toBe('user:a');
   });
 
+  it('open, adopt a draft and close offline without waiting on the server', async () => {
+    const world = new World();
+    world.online = false;
+    world.hang = true;
+    const store = new Map<string, any>();
+    const a = world.client('user:a', store);
+    const draft = await a.manager.createDraft('doc', 'body');
+    draft.getDoc().getText('body').insert(0, 'written offline');
+    draft.getDoc().commit();
+
+    let done = false;
+    void (async () => {
+      const f = await a.manager.open('doc', 'doc:1', 'body', draft);
+      expect(text(f)).toBe('written offline');
+      await a.manager.close('doc', 'doc:1', 'body');
+      done = true;
+    })();
+    await realWait();
+    expect(done).toBe(true);
+    expect(world.log).toHaveLength(0);
+    // Kept on the device, to go out from the next open.
+    expect([...store.values()].some((row) => row.snapshot)).toBe(true);
+  });
+
+  it('a reopen right after a close starts from what the close saved', async () => {
+    vi.useRealTimers();
+    const world = new World();
+    world.storeDelayMs = 30;
+    const a = world.client('user:a');
+    const draft = await a.manager.createDraft('doc', 'body');
+    draft.getDoc().getText('body').insert(0, 'from the form');
+    draft.getDoc().commit();
+    await a.manager.open('doc', 'doc:1', 'body', draft);
+    void a.manager.close('doc', 'doc:1', 'body');
+    const reopened = await a.manager.open('doc', 'doc:1', 'body');
+    expect(text(reopened)).toBe('from the form');
+  });
+
   it('never import a row that fails the acceptance rule', async () => {
     const world = new World();
     const b = world.client('user:b');

package/src/modules/crdt/index.ts

@@ -100,15 +100,18 @@ const MAX_WAIT_MS = 2000;
 const PERSIST_DEBOUNCE_MS = 400;
 const PRESENCE_THROTTLE_MS = 250;
 const CLAIM_RETRY_MS = 5000;
-/** How long an open of a field with nothing local waits for the server's
- *  first answer before handing out the (still empty) document. */
-const FIRST_ANSWER_TIMEOUT_MS = 4000;
+/** Longest an open or close waits on the server (the first answer for a
+ *  field with nothing local, the last edits going out) before going on
+ *  without it. Offline it does not wait at all. */
+const SERVER_WAIT_MS = 4000;
 /** Pause after a fold is noticed before checking the published document. */
 const VERIFY_DELAY_MS = 1000;
 
 export class CrdtManager {
   private sessions = new Map<string, Session>();
   private opening = new Map<string, Promise<Session>>();
+  /** Closes still saving their field's local state, by session key. */
+  private saving = new Map<string, Promise<void>>();
   private ports: CrdtPorts | null = null;
   private logger: Logger;
   private peer: `${number}` = randomPeer();
@@ -178,7 +181,7 @@ export class CrdtManager {
         session = await inFlight;
         session.refs++;
       } else {
-        const pending = this.start(key, table, recordId, field, kind, cursor).finally(() => this.opening.delete(key));
+        const pending = this.start(key, table, recordId, field, kind, cursor, !!draft).finally(() => this.opening.delete(key));
         this.opening.set(key, pending);
         session = await pending;
       }
@@ -209,7 +212,8 @@ export class CrdtManager {
   }
 
   /** Close one open. The last close of a field resolves once its unsent
-   *  edits are in the outbox (or are left for a later session, offline). */
+   *  edits are in the outbox, or once they are saved on the device when the
+   *  server does not answer in time (offline: at once). */
   close(table: string, recordId: string, field: string): Promise<void> {
     const s = this.sessions.get(`${table}|${recordId}|${field}`);
     if (!s || --s.refs > 0) return Promise.resolve();
@@ -245,13 +249,18 @@ export class CrdtManager {
     recordId: string,
     field: string,
     kind: CrdtKind,
-    cursor: boolean
+    cursor: boolean,
+    /** Opened with a draft: the record was just created here, so the server
+     *  has nothing for it yet and the open does not wait on it. */
+    fresh = false
   ): Promise<Session> {
     if (!this.ports) throw new Error('CrdtManager: not attached to a client');
     const loro = await loadLoro();
     const doc = new loro.LoroDoc();
     doc.setPeerId(this.peer);
 
+    // A close of this field may still be saving: read what it saves.
+    await this.saving.get(key);
     const local = await this.readLocal(recordId, field);
     if (local.snapshot) this.tryImport(doc, local.snapshot, key);
 
@@ -288,8 +297,9 @@ export class CrdtManager {
 
     // Claimed now, so the first edits do not wait on the round trip.
     void this.ensureClaimed();
-    // First open on this device: start from the published document.
-    if (!local.snapshot) await this.refetchCanonical(session);
+    // First open on this device: start from the published document. Offline
+    // it is read once the connection is back instead of holding the open.
+    if (!local.snapshot && !fresh) await this.serverWait(this.refetchCanonical(session));
 
     session.disposers.push(
       doc.subscribeLocalUpdates(() => {
@@ -315,9 +325,7 @@ export class CrdtManager {
     // two clients doing that at once keep only one of them (with whatever was
     // typed into the other). With nothing local, hand the document out only
     // once the server's rows for it are in.
-    if (!session.handle.hasContent()) {
-      await Promise.race([log.settled, delay(FIRST_ANSWER_TIMEOUT_MS)]);
-    }
+    if (!session.handle.hasContent() && !fresh) await this.serverWait(log.settled);
     await session.ingesting;
     // Edits a previous session made but never sent (closed tab, offline).
     this.scheduleFlush(session, 0);
@@ -340,11 +348,25 @@ export class CrdtManager {
     if (s.verifyTimer) clearTimeout(s.verifyTimer);
     if (s.presenceSent) void this.dropPresence(s);
     if (flush) {
-      await this.persist(s);
-      await this.flush(s);
+      const saved = this.persist(s);
+      this.saving.set(s.key, saved);
+      await saved;
+      if (this.saving.get(s.key) === saved) this.saving.delete(s.key);
+      await this.serverWait(this.flush(s));
     }
   }
 
+  /**
+   * Wait for server-bound `work`, at most {@link SERVER_WAIT_MS}, and not at
+   * all while disconnected: a remote call made then queues behind the
+   * reconnect for up to the query timeout, and an offline "new thread" form
+   * must not hang on it. The work itself goes on either way.
+   */
+  private serverWait(work: Promise<unknown>): Promise<unknown> {
+    if (this.remote.getStatus() !== 'connected') return Promise.resolve();
+    return Promise.race([work, delay(SERVER_WAIT_MS)]);
+  }
+
   // --- receiving ---------------------------------------------------
 
   private async ingest(s: Session, rows: Record<string, unknown>[]): Promise<void> {

package/src/query/id-set-plan.test.ts

@@ -59,12 +59,12 @@ describe('buildIdSetPlan', () => {
 describe('buildIdSetSurql', () => {
   it('rewrites a non-offset query to the id-set source, keeping ORDER BY', () => {
     const out = buildIdSetSurql('SELECT * FROM thread WHERE done = false ORDER BY title ASC LIMIT 50;');
-    expect(out?.query).toBe('SELECT * FROM $__win ORDER BY title ASC');
+    expect(out?.query).toBe('SELECT * FROM (SELECT * FROM $__win) ORDER BY title ASC');
   });
 
   it('preserves the projection', () => {
     const out = buildIdSetSurql('SELECT id, title FROM thread WHERE done = false;');
-    expect(out?.query).toBe('SELECT id, title FROM $__win');
+    expect(out?.query).toBe('SELECT id, title FROM (SELECT * FROM $__win)');
   });
 
   it('returns null when there is no top-level FROM to replace', () => {

package/src/query/lifecycle.saga.ts

@@ -5,7 +5,7 @@ import { ACK_GRACE_MS, GC_CHUNK, GC_INTERVAL_MS, TTL_HEARTBEAT_FRACTION, VIEW_RE
 import type { IngestRecord } from '../services/stream-processor/index';
 import type { ClientState, QueryEntry, TabRole } from '../state/client-state';
 import * as R from '../state/reducers';
-import { evictable, retained, shortestTtlMs } from '../state/selectors';
+import { activelyHeld, evictable, retained, shortestTtlMs } from '../state/selectors';
 import type { SagaEnv } from './env';
 import { parseViewIndex, type ViewIndexRow } from './membership';
 import * as sql from './sql';
@@ -125,6 +125,33 @@ export function* gcTick(): Saga<void> {
   }
 }
 
+/**
+ * Forget rows the server just took out of `hash` (its membership or its
+ * subquery children) when nothing live still holds them: delete the local
+ * body and the in-browser circuit's copy, and re-render the query.
+ *
+ * Primary rows already stop rendering once they leave membership, but
+ * subquery children are rendered from the local store (a thread's comments
+ * are read back by the relation), so a comment deleted on another device
+ * stayed on screen until the hourly GC or a reload. Only a leader or solo tab
+ * deletes, and a leader relays it to its followers, as the GC does.
+ */
+export function* forgetDeparted(hash: string, departed: string[]): Saga<void> {
+  if (departed.length === 0) return;
+  const [gone, role] = (yield fx.state.read((s) => {
+    const held = activelyHeld(s);
+    return [departed.filter((id) => !id.startsWith('_00_') && s.versions.has(id) && !held.has(id)), s.tabRole];
+  })) as [string[], TabRole];
+  if (gone.length === 0 || role === 'follower') return;
+  const settled = (yield fx.all(gone.map((id) => fx.local.delete(tableOf(id), id)))) as Settled[];
+  const done = gone.filter((_, j) => settled[j].ok);
+  if (done.length === 0) return;
+  const records: IngestRecord[] = done.map((id) => ({ table: tableOf(id), op: 'DELETE', id, record: {} }));
+  yield fx.state.update(R.compose(R.deleteVersions(done), R.markDirty([hash])));
+  yield fx.ssp.ingest(records);
+  if (role === 'leader') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records } });
+}
+
 /**
  * Delete every body nothing retains, `GC_CHUNK` at a time. Each chunk is
  * re-checked against fresh state (a membership committed meanwhile may name

package/src/query/materialize.test.ts

@@ -15,7 +15,7 @@ describe('materializeEffect', () => {
   });
   it('surql + ids: id-set rewrite bound to __win; unparseable surql falls back to the raw query', () => {
     const e = materializeEffect({ surql: 'SELECT * FROM thing WHERE a = $a ORDER BY b LIMIT 3', params: { a: 1 } }, ['thing:1']);
-    expect(e).toMatchObject({ kind: 'local.query', sql: 'SELECT * FROM $__win ORDER BY b' });
+    expect(e).toMatchObject({ kind: 'local.query', sql: 'SELECT * FROM (SELECT * FROM $__win) ORDER BY b' });
     if (e.kind !== 'local.query') throw new Error();
     expect(e.vars).toEqual({ a: 1, __win: [new RecordId('thing', '1')] });
     const raw = materializeEffect({ surql: 'RETURN 1', params: {} }, ['thing:1']);

package/src/query/membership.saga.test.ts

@@ -147,6 +147,40 @@ describe('applySubqueryChildren', () => {
   });
 });
 
+describe('rows the server takes out of a query', () => {
+  it('a child nothing else holds is forgotten locally and the query re-renders; a held one stays', async () => {
+    const s = buildState(
+      [
+        buildEntry({ def: { hash: 'a' }, lifecycle: { phase: 'live' }, remoteArray: [['t:1', 1]], subqueryRemoteArray: [['c:1', 1], ['c:9', 1]] }),
+        buildEntry({ def: { hash: 'b' }, lifecycle: { phase: 'live' }, remoteArray: [['c:9', 1]] }),
+      ],
+      R.setVersions([['t:1', 1], ['c:1', 1], ['c:9', 1]])
+    );
+    const out = await runPure(applySubqueryChildren('a', []), {
+      state: { ...s, dirty: new Set<string>() },
+      handlers: { 'local.upsert': () => undefined, 'local.delete': () => undefined, 'ssp.ingest': () => undefined },
+    });
+    expect(out.log.filter((e) => e.kind === 'local.delete')).toEqual([expect.objectContaining({ table: 'c', id: 'c:1' })]);
+    expect(out.log.find((e) => e.kind === 'ssp.ingest')).toEqual(
+      expect.objectContaining({ records: [{ table: 'c', op: 'DELETE', id: 'c:1', record: {} }] })
+    );
+    expect(out.state.versions.has('c:1')).toBe(false);
+    expect(out.state.versions.has('c:9')).toBe(true);
+    expect(out.state.dirty.has('a')).toBe(true);
+  });
+  it('a row with a pending local write is kept, and a follower tab never deletes', async () => {
+    const base = buildState(
+      [buildEntry({ def: { hash: 'a' }, lifecycle: { phase: 'live' }, remoteArray: [['t:1', 1]], subqueryRemoteArray: [['c:1', 1]] })],
+      R.setVersions([['t:1', 1], ['c:1', 1]])
+    );
+    const handlers = { 'local.upsert': () => undefined, 'local.delete': () => undefined, 'ssp.ingest': () => undefined };
+    const pending = { ...base, outbox: [buildOutboxItem({ recordId: 'c:1' })] };
+    expect((await runPure(applySubqueryChildren('a', []), { state: pending, handlers })).log.some((e) => e.kind === 'local.delete')).toBe(false);
+    const follower = { ...base, tabRole: 'follower' as const };
+    expect((await runPure(applySubqueryChildren('a', []), { state: follower, handlers })).log.some((e) => e.kind === 'local.delete')).toBe(false);
+  });
+});
+
 describe('readMembership', () => {
   const qid = (h: string) => new RecordId('_00_query', h);
   const out1 = (id: string, v = 1) => ({ out: new RecordId('thing', id), version: v });

package/src/query/membership.saga.ts

@@ -13,6 +13,8 @@ import type { ClientState, QueryEntry } from '../state/client-state';
 import { isAuthoritative } from '../state/lifecycle';
 import * as R from '../state/reducers';
 import { encodeRecordId } from '../utils/index';
+import { departedIds } from '../state/selectors';
+import { forgetDeparted } from './lifecycle.saga';
 import { planListRefPollChunks, recordVersionArraysEqual } from '../sync/policy';
 import type { SagaEnv } from './env';
 import { listRefTable } from './env';
@@ -86,6 +88,7 @@ export function* applyMembership(
   }
   const wasAuthoritative = isAuthoritative(entry.lifecycle);
   yield fx.state.update(R.commitMembership(hash, remoteArray, meta?.present ?? true));
+  yield* forgetDeparted(hash, departedIds(entry.remoteArray, remoteArray));
   if (!wasAuthoritative) yield fx.emit({ type: 'query:authority', hash, known: true });
   const now = (yield fx.now()) as number;
   try {
@@ -126,6 +129,7 @@ export function* applySubqueryChildren(hash: QueryHash, children: RecordVersionA
   const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
   if (!entry || recordVersionArraysEqual(entry.subqueryRemoteArray, children)) return;
   yield fx.state.update(R.setSubqueryRemoteArray(hash, children));
+  yield* forgetDeparted(hash, departedIds(entry.subqueryRemoteArray, children));
   if (entry.lifecycle.phase !== 'cold') {
     try {
       yield fx.local.upsert(sql.VIEW_TABLE, sql.viewRecordId(entry.def.viewKey), { children }, 'merge');

package/src/query/window-query.test.ts

@@ -12,9 +12,9 @@ describe('scanner edge cases', () => {
     expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START')).toBeNull();
   });
   it('matches a keyword at the very start and at the end of the string', () => {
-    expect(buildWindowMaterialization('FROM t LIMIT 5 START 5')).toEqual({ query: ' FROM $__win' });
+    expect(buildWindowMaterialization('FROM t LIMIT 5 START 5')).toEqual({ query: ' FROM (SELECT * FROM $__win)' });
     expect(buildWindowMaterialization('SELECT * FROM t ORDER BY a LIMIT 5 START 5 ORDER')).toEqual({
-      query: 'SELECT * FROM $__win ORDER BY a',
+      query: 'SELECT * FROM (SELECT * FROM $__win) ORDER BY a',
     });
   });
 });
@@ -25,7 +25,7 @@ describe('buildWindowMaterialization', () => {
       'SELECT * FROM game WHERE database = $database ORDER BY sort_index asc, date desc LIMIT 30 START 30;';
     const r = buildWindowMaterialization(surql);
     expect(r).not.toBeNull();
-    expect(r!.query).toBe('SELECT * FROM $__win ORDER BY sort_index asc, date desc');
+    expect(r!.query).toBe('SELECT * FROM (SELECT * FROM $__win) ORDER BY sort_index asc, date desc');
   });
 
   it('returns null for START 0 (offset-free windows still work via the normal re-query)', () => {
@@ -44,7 +44,7 @@ describe('buildWindowMaterialization', () => {
   it('preserves a custom projection', () => {
     const surql = 'SELECT id, white, black FROM game ORDER BY date desc LIMIT 30 START 60;';
     expect(buildWindowMaterialization(surql)!.query).toBe(
-      'SELECT id, white, black FROM $__win ORDER BY date desc'
+      'SELECT id, white, black FROM (SELECT * FROM $__win) ORDER BY date desc'
     );
   });
 
@@ -58,13 +58,13 @@ describe('buildWindowMaterialization', () => {
       'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments ' +
       'FROM game ORDER BY sort_index asc LIMIT 30 START 90;';
     expect(buildWindowMaterialization(surql)!.query).toBe(
-      'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments FROM $__win ORDER BY sort_index asc'
+      'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments FROM (SELECT * FROM $__win) ORDER BY sort_index asc'
     );
   });
 
   it('does not treat a START inside a string literal as the offset', () => {
     const surql = "SELECT * FROM game WHERE note = 'LIMIT 30 START 30' ORDER BY date desc LIMIT 30 START 30;";
     const r = buildWindowMaterialization(surql);
-    expect(r!.query).toBe('SELECT * FROM $__win ORDER BY date desc');
+    expect(r!.query).toBe('SELECT * FROM (SELECT * FROM $__win) ORDER BY date desc');
   });
 });

package/src/query/window-query.ts

@@ -70,7 +70,20 @@ function renderIdSetSelect(surql: string, kw: TopLevelClauses, idsParam: string)
     orderBy = ' ' + surql.slice(orderByIndex, Math.min(...ends)).trim();
   }
 
-  return `${selectClause} FROM $${idsParam}${orderBy}`;
+  return `${selectClause} FROM ${idSetSource(selectClause, idsParam, orderBy !== '')}${orderBy}`;
+}
+
+/**
+ * The FROM source for an id-set read. The in-browser SurrealDB (wasm, 3.x)
+ * reads bare record ids without loading them: a field list over them throws
+ * ("Specify a database to use") and an ORDER BY is silently ignored, while a
+ * plain `SELECT *` works. Those read from `(SELECT * FROM $ids)` instead, the
+ * same rows as documents. The SQLite translator accepts both shapes.
+ */
+export function idSetSource(selectClause: string, idsParam: string, ordered = false): string {
+  return !ordered && /^SELECT\s+(VALUE\s+)?\*/i.test(selectClause.trim())
+    ? `$${idsParam}`
+    : `(SELECT * FROM $${idsParam})`;
 }
 
 /**

package/src/services/database/sqlite-cache-engine.test.ts

@@ -556,6 +556,17 @@ describe('create() tx result shaping (fast path parity)', () => {
     expect(outboxRow.mutationType).toBe('create');
     expect(outboxRow.data).toEqual(payload);
   });
+
+  // A raw query with a field list materializes from its id-set as
+  // `FROM (SELECT * FROM $__win)` (the wasm SurrealDB throws on a field list
+  // over bare record ids). This engine reads both shapes the same way.
+  it('reads a field list over a wrapped id-set like the bare one', () => {
+    const ids = ['_00_user_feature:a', '_00_user_feature:b'];
+    const bare = translateSurql('SELECT key, variant FROM $__win', { __win: ids }).ops[0];
+    const wrapped = translateSurql('SELECT key, variant FROM (SELECT * FROM $__win)', { __win: ids }).ops[0];
+    expect(wrapped).toEqual(bare);
+    expect(wrapped).toMatchObject({ kind: 'selectByIds', ids });
+  });
 });
 
 

package/src/services/database/surql-translate.ts

@@ -193,13 +193,14 @@ function translateStatement(stmt: string, vars: Record<string, unknown>): SqlOp
     return { kind: 'getById', id: idOperand(m[3], vars), select: projFields(m[2], !!m[1]), value };
   }
 
-  // SELECT [VALUE] <proj> FROM $arrayParam
-  m = /^SELECT\s+(VALUE\s+)?(.+?)\s+FROM\s+\$(\w+)$/i.exec(head);
+  // SELECT [VALUE] <proj> FROM $arrayParam, or FROM (SELECT * FROM $arrayParam)
+  // (the id-set read's shape for a field list, see `idSetSource`).
+  m = /^SELECT\s+(VALUE\s+)?(.+?)\s+FROM\s+(?:\$(\w+)|\(\s*SELECT\s+\*\s+FROM\s+\$(\w+)\s*\))$/i.exec(head);
   if (m) {
     const value = m[1] ? m[2].trim() : undefined;
     return {
       kind: 'selectByIds',
-      ids: (vars[m[3]] as unknown[]) ?? [],
+      ids: (vars[m[3] ?? m[4]] as unknown[]) ?? [],
       select: projFields(m[2], !!m[1]),
       value,
       limit,

package/src/services/database/surreal-cache-engine.ts

@@ -69,7 +69,9 @@ export class SurrealCacheEngine extends LocalDatabaseService implements LocalCac
     if (ids.length === 0) return [];
     const projection =
       opts?.select && opts.select.length > 0 ? ['id', ...opts.select].join(', ') : '*';
-    let sql = `SELECT ${projection} FROM $__ids`;
+    // As documents: over bare record ids the wasm engine throws on a field
+    // list and ignores ORDER BY (see `idSetSource`).
+    let sql = `SELECT ${projection} FROM (SELECT * FROM $__ids)`;
     if (opts?.orderBy && opts.orderBy.length > 0) {
       sql += ` ORDER BY ${opts.orderBy.map(([f, d]) => `${f} ${d}`).join(', ')}`;
     }

package/src/state/selectors.test.ts

@@ -77,6 +77,19 @@ describe('needed / planFetch / settled', () => {
     expect(S.settleFailed(s, 'a')).toBe(false);
     expect(S.settleFailed(R.applyLifecycle('a', { type: 'remote-failed' })(s), 'a')).toBe(true);
   });
+  it('queryStatus stays fetching until a live query is settled', () => {
+    const base = e('a', { lifecycle: { ...live, notified: true }, remoteArray: [['thing:1', 1]] });
+    const s = { ...buildState([base], R.setVersions([['thing:1', 1]])), dirty: new Set<string>() };
+    expect(S.queryStatus(s, 'a')).toBe('idle');
+    // Membership in, body still downloading (registration already ended its cycle).
+    expect(S.queryStatus(R.setVersions([['thing:1', 0]])(s), 'a')).toBe('fetching');
+    // Body landed, not yet shown to subscribers.
+    expect(S.queryStatus(R.markDirty(['a'])(s), 'a')).toBe('fetching');
+    // A cold query with no fetch in flight has nothing to wait for.
+    expect(S.queryStatus(buildState([e('c', { lifecycle: { phase: 'cold' } })]), 'c')).toBe('idle');
+    expect(S.queryStatus(R.applyLifecycle('a', { type: 'fetch-begin' })(s), 'a')).toBe('fetching');
+    expect(S.queryStatus(s, 'missing')).toBeUndefined();
+  });
   it('fetchingQueryCount counts entries with fetch depth', () => {
     const s = buildState([e('a', { lifecycle: { fetchDepth: 2 } }), e('b')]);
     expect(S.fetchingQueryCount(s)).toBe(1);

package/src/state/selectors.ts

@@ -12,9 +12,21 @@ export const activeHashes = (s: ClientState): QueryHash[] => [...s.queries.keys(
 export const hashesForTable = (s: ClientState, table: string): QueryHash[] =>
   [...s.queries].filter(([, e]) => e.def.tableName === table).map(([h]) => h);
 
+/**
+ * The status subscribers see. `fetching` while a fetch cycle is in flight,
+ * and also while a live query is not yet `settled`: the server's membership
+ * is in but its rows are still downloading or not yet shown. Registration
+ * ends its cycle as soon as membership is applied and the body fetch starts
+ * its own a moment later, so the fetch count alone read `idle` in between,
+ * and a binding that settles on "membership known and idle" told the app a
+ * row did not exist while it was on its way (a by-id screen showed "not
+ * found" for a moment, or for as long as the bodies took).
+ */
 export const queryStatus = (s: ClientState, hash: QueryHash): QueryStatus | undefined => {
   const e = s.queries.get(hash);
-  return e ? deriveStatus(e.lifecycle) : undefined;
+  if (!e) return undefined;
+  if (deriveStatus(e.lifecycle) === 'fetching') return 'fetching';
+  return e.lifecycle.phase === 'live' && !settled(s, hash) ? 'fetching' : 'idle';
 };
 
 export interface Overlay {
@@ -67,6 +79,30 @@ export function retained(s: ClientState): (id: string) => boolean {
   return (id) => id.startsWith('_00_') || known.has(id) || held.has(id);
 }
 
+/**
+ * Ids something live still needs: a mounted query's membership or subquery
+ * children, or a local write not yet settled. Unlike `retained`, durable view
+ * rows of queries nobody runs right now do not count: a row the server just
+ * took out of a live query must stop showing even if an old view named it.
+ */
+export function activelyHeld(s: ClientState): ReadonlySet<string> {
+  const held = new Set<string>();
+  for (const e of s.queries.values()) {
+    for (const [id] of e.remoteArray) held.add(id);
+    for (const [id] of e.subqueryRemoteArray) held.add(id);
+  }
+  for (const item of s.outbox) held.add(item.recordId);
+  for (const write of s.pendingWrites.values()) held.add(write.recordId);
+  return held;
+}
+
+/** Ids of `before` that `after` no longer names. */
+export function departedIds(before: RecordVersionArray, after: RecordVersionArray): string[] {
+  if (before.length === 0) return [];
+  const kept = new Set(after.map(([id]) => id));
+  return before.filter(([id]) => !kept.has(id)).map(([id]) => id);
+}
+
 export const pendingDeleteIds = (s: ClientState): ReadonlySet<string> =>
   new Set(s.outbox.filter((i) => i.type === 'delete').map((i) => i.recordId));