@spooky-sync/core 0.0.1-canary.202 → 0.0.1-canary.204

package/src/modules/sync/queue/queue-down.test.ts

@@ -105,3 +105,76 @@ describe('DownQueue.next failure handling', () => {
     expect(drained).toEqual(['b']);
   });
 });
+
+describe('DownQueue.takeNext (per-hash ordering under concurrency)', () => {
+  it('hands out events for distinct hashes so they can run in parallel', () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('b'));
+    q.push(register('c'));
+    const busy = new Set<string>();
+
+    const first = q.takeNext(busy)!;
+    busy.add(hashOf(first));
+    const second = q.takeNext(busy)!;
+    busy.add(hashOf(second));
+
+    expect([hashOf(first), hashOf(second)]).toEqual(['a', 'b']);
+    expect(q.size).toBe(1);
+  });
+
+  it('SKIPS an event whose hash is busy without reordering it', () => {
+    // The whole correctness argument: a `cleanup` must never overtake the
+    // `register` for the same query. A busy hash's event keeps its place in the
+    // queue and is simply passed over.
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('a'));
+    q.push(register('b'));
+
+    const busy = new Set(['a']);
+    const taken = q.takeNext(busy)!;
+    expect(hashOf(taken)).toBe('b');
+
+    // Once 'a' frees up, its two events come back out in their original order.
+    const rest = [q.takeNext(new Set())!, q.takeNext(new Set())!];
+    expect(rest.map(hashOf)).toEqual(['a', 'a']);
+    expect(q.size).toBe(0);
+  });
+
+  it('returns undefined when every remaining event is blocked', () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    expect(q.takeNext(new Set(['a']))).toBeUndefined();
+    // Blocked, not consumed.
+    expect(q.size).toBe(1);
+  });
+});
+
+describe('DownQueue.run', () => {
+  it('returns the error instead of throwing, and re-heads the event', async () => {
+    // A concurrent drain has other work in flight when one event fails; a
+    // rejection here would take that work down with it.
+    const q = makeQueue();
+    const event = register('a');
+    q.push(event);
+    const taken = q.takeNext(new Set())!;
+
+    const boom = new Error('nope');
+    const err = await q.run(taken, async () => {
+      throw boom;
+    });
+
+    expect(err).toBe(boom);
+    expect(q.size).toBe(1);
+    expect(hashOf(q.takeNext(new Set())!)).toBe('a');
+  });
+
+  it('returns undefined on success', async () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    const taken = q.takeNext(new Set())!;
+    await expect(q.run(taken, async () => {})).resolves.toBeUndefined();
+    expect(q.size).toBe(0);
+  });
+});

package/src/modules/sync/queue/queue-down.ts

@@ -91,28 +91,66 @@ export class DownQueue {
 
   async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
     const event = this.queue.shift();
-    if (event) {
-      try {
-        await fn(event);
-        this.failures.delete(event);
-      } catch (error) {
-        const attempts = (this.failures.get(event) ?? 0) + 1;
-        this.failures.set(event, attempts);
-        // Re-head so a transient failure keeps its ordering, but give up the
-        // head once it looks permanent and there is other work waiting — one
-        // unregisterable query must not stall every other query's registration.
-        const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
-        if (starvingOthers) {
-          this.queue.push(event);
-        } else {
-          this.queue.unshift(event);
-        }
-        this.logger.error(
-          { error, event, attempts, rotated: starvingOthers, Category: 'sp00ky-client::DownQueue::next' },
-          'Failed to process query'
-        );
-        throw error;
+    if (!event) return;
+    const error = await this.run(event, fn);
+    if (error !== undefined) throw error;
+  }
+
+  /**
+   * The next event whose hash is NOT already being processed, or `undefined`
+   * when every remaining event is blocked on a busy hash.
+   *
+   * Ordering only ever mattered PER HASH — a `cleanup` must not overtake the
+   * `register` for the same query — but the queue enforced it globally, so one
+   * registration RPC at a time was the ceiling for the whole client. An event
+   * for a busy hash keeps its place here (it is skipped, not reordered), so
+   * per-hash ordering is preserved exactly while independent hashes proceed
+   * concurrently.
+   */
+  takeNext(busy: ReadonlySet<string>): DownEvent | undefined {
+    for (let i = 0; i < this.queue.length; i++) {
+      const event = this.queue[i]!;
+      if (busy.has(event.payload.hash)) continue;
+      this.queue.splice(i, 1);
+      return event;
+    }
+    return undefined;
+  }
+
+  /**
+   * Process one event, applying the re-head / rotate failure policy.
+   *
+   * NEVER rejects: it RETURNS the error instead (`undefined` on success). A
+   * concurrent drain has other work in flight when one event fails, and a
+   * rejection would either take that work down with it or have to be caught at
+   * every call site. `next` reinstates the throwing contract for the serial
+   * callers that still want it.
+   */
+  async run(
+    event: DownEvent,
+    fn: (event: DownEvent) => Promise<void>
+  ): Promise<unknown | undefined> {
+    try {
+      await fn(event);
+      this.failures.delete(event);
+      return undefined;
+    } catch (error) {
+      const attempts = (this.failures.get(event) ?? 0) + 1;
+      this.failures.set(event, attempts);
+      // Re-head so a transient failure keeps its ordering, but give up the
+      // head once it looks permanent and there is other work waiting — one
+      // unregisterable query must not stall every other query's registration.
+      const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
+      if (starvingOthers) {
+        this.queue.push(event);
+      } else {
+        this.queue.unshift(event);
       }
+      this.logger.error(
+        { error, event, attempts, rotated: starvingOthers, Category: 'sp00ky-client::DownQueue::next' },
+        'Failed to process query'
+      );
+      return error;
     }
   }
 }

package/src/modules/sync/scheduler.retry.test.ts

@@ -32,11 +32,28 @@ function makeQueue<E>(items: E[]) {
     async next(fn: (event: E) => Promise<void>) {
       const event = this.queue.shift();
       if (!event) return;
+      const err = await this.run(event, fn);
+      if (err !== undefined) throw err;
+    },
+    // The concurrent down drain takes events itself and runs them, so the fake
+    // has to mirror that half of the real DownQueue too: skip events whose hash
+    // is already in flight, and hand the error back rather than throwing.
+    takeNext(busy: ReadonlySet<string>) {
+      for (let i = 0; i < this.queue.length; i++) {
+        const event = this.queue[i] as unknown as { payload?: { hash?: string } };
+        const hash = event?.payload?.hash;
+        if (hash !== undefined && busy.has(hash)) continue;
+        return this.queue.splice(i, 1)[0];
+      }
+      return undefined;
+    },
+    async run(event: E, fn: (event: E) => Promise<void>) {
       try {
         await fn(event);
+        return undefined;
       } catch (err) {
         this.queue.unshift(event);
-        throw err;
+        return err;
       }
     },
   };
@@ -154,3 +171,67 @@ describe('SyncScheduler retry', () => {
     expect(attempts).toBe(1);
   });
 });
+
+describe('SyncScheduler down concurrency', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  /** Scheduler over a down queue of `hashes`, whose handler blocks until the
+   *  returned `release` is called for that hash. Lets a test observe exactly
+   *  which events are in flight at the same moment. */
+  function makeConcurrent(hashes: string[]) {
+    const downQueue = makeQueue(hashes.map(downEvent));
+    const upQueue = makeQueue<UpEvent>([]);
+    const started: string[] = [];
+    const gates = new Map<string, () => void>();
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async (event) => {
+        const hash = (event as { payload: { hash: string } }).payload.hash;
+        started.push(hash);
+        await new Promise<void>((resolve) => gates.set(hash, resolve));
+      },
+      silentLogger
+    );
+    return { scheduler, started, release: (h: string) => gates.get(h)?.() };
+  }
+
+  it('runs several independent hashes at once instead of one at a time', async () => {
+    // The bug this whole change exists for: strictly serial draining meant one
+    // registration RPC at a time for the entire client.
+    const { scheduler, started, release } = makeConcurrent(['a', 'b', 'c', 'd', 'e']);
+    const pass = scheduler.syncDown();
+    await vi.advanceTimersByTimeAsync(0);
+
+    // Bounded by MAX_CONCURRENT_DOWN (4), not 1.
+    expect(started).toEqual(['a', 'b', 'c', 'd']);
+
+    // Freeing one slot immediately admits the next event.
+    release('a');
+    await vi.advanceTimersByTimeAsync(0);
+    expect(started).toEqual(['a', 'b', 'c', 'd', 'e']);
+
+    for (const h of ['b', 'c', 'd', 'e']) release(h);
+    await pass;
+  });
+
+  it('never overlaps two events for the SAME hash', async () => {
+    // Per-hash ordering is the one invariant the serial queue was providing.
+    const { scheduler, started, release } = makeConcurrent(['a', 'a', 'b']);
+    const pass = scheduler.syncDown();
+    await vi.advanceTimersByTimeAsync(0);
+
+    // The second 'a' is skipped while the first is in flight; 'b' goes ahead.
+    expect(started).toEqual(['a', 'b']);
+
+    release('a');
+    await vi.advanceTimersByTimeAsync(0);
+    expect(started).toEqual(['a', 'b', 'a']);
+
+    release('b');
+    release('a');
+    await pass;
+  });
+});

package/src/modules/sync/scheduler.ts

@@ -8,6 +8,17 @@ import { SyncQueueEventTypes } from './events/index';
  */
 /** Backoff for re-draining a queue that halted on an error. */
 const RETRY_BASE_MS = 500;
+/**
+ * How many down events may be in flight at once.
+ *
+ * The down queue was strictly serial: one `register`/`sync`/`cleanup` RPC at a
+ * time for the WHOLE client. Measured in production that drained at roughly one
+ * event per 8.8s, so a list that registers a query per scrolled-to window took
+ * minutes to fill — the rows were already cached, only the registration lagged.
+ * Ordering is a per-hash requirement, not a global one (see `takeNext`), so
+ * independent hashes can go in parallel. Bounded to stay polite to the SSP.
+ */
+const MAX_CONCURRENT_DOWN = 4;
 const RETRY_MAX_MS = 15_000;
 
 export class SyncScheduler {
@@ -197,14 +208,58 @@ export class SyncScheduler {
 
     this.isSyncingDown = true;
     let processedAny = false;
+    // Hashes currently in flight. `takeNext` skips their events, so per-hash
+    // ordering holds while independent hashes run concurrently.
+    const busy = new Set<string>();
+    const inFlight = new Set<Promise<void>>();
+    // First failure of this pass. Once set we stop TAKING work (mirroring the
+    // old "halt the pass and retry on a backoff") but still let what is already
+    // in flight finish, rather than abandoning RPCs mid-round.
+    let failure: unknown | undefined;
     try {
-      while (this.downQueue.size > 0 && !this.paused) {
-        if (this.upQueue.size > 0) break;
-        await this.downQueue.next(this.onProcessDown);
-        processedAny = true;
+      for (;;) {
+        if (this.paused) break;
+        // Yield to the up queue: stop taking new down work so a pending
+        // mutation gets its turn. The old code `break`-ed out and came back
+        // only on a backoff, so a steady trickle of local writes could starve
+        // registration indefinitely; draining what is in flight and re-arming
+        // immediately interleaves the two instead.
+        const yieldToUp = this.upQueue.size > 0;
+        while (
+          failure === undefined &&
+          !yieldToUp &&
+          inFlight.size < MAX_CONCURRENT_DOWN
+        ) {
+          const event = this.downQueue.takeNext(busy);
+          if (!event) break;
+          const hash = event.payload.hash;
+          busy.add(hash);
+          processedAny = true;
+          // `run` never rejects — it hands the error back — so one failing
+          // event cannot reject the `Promise.race` below and lose the rest.
+          const task: Promise<void> = this.downQueue
+            .run(event, this.onProcessDown)
+            .then((error) => {
+              if (error !== undefined && failure === undefined) failure = error;
+            })
+            .finally(() => {
+              busy.delete(hash);
+              inFlight.delete(task);
+            });
+          inFlight.add(task);
+        }
+        if (inFlight.size === 0) break;
+        // Wake as soon as ANY slot frees, so the pool refills continuously
+        // instead of in barriers.
+        await Promise.race(inFlight);
+        if (yieldToUp && inFlight.size === 0) break;
       }
+      if (failure !== undefined) throw failure;
       if (processedAny) this.onSyncOutcome?.(true);
       this.downRetryAttempt = 0;
+      // Re-arm if we stopped early for the up queue (or hit the pool cap with
+      // work still queued) — otherwise that work waits for a fresh enqueue.
+      if (this.downQueue.size > 0) this.scheduleDownRetry(RETRY_BASE_MS);
     } catch (error) {
       this.onSyncOutcome?.(false, error);
       this.scheduleDownRetry();

package/src/modules/sync/sync.ts

@@ -74,6 +74,15 @@ export interface Sp00kySyncOptions {
    * up-queue for the session. Defaults to 30000; `0` disables the timeout.
    */
   pushTimeoutMs?: number;
+  /**
+   * Max time a single down event (`register`/`sync`/`cleanup`) may take before
+   * it is treated as a network failure and retried. The mirror of
+   * {@link pushTimeoutMs} for the read side, which had no such guard: a
+   * `fn::query::register` that never settled held its slot in the down drain,
+   * and every later registration behind it, for the rest of the session.
+   * Defaults to 30000; `0` disables the timeout.
+   */
+  downTimeoutMs?: number;
   /**
    * Transport supervisor. Sync reads its state to report `connection` in
    * {@link SyncHealth} so a UI can show "reconnecting…" the instant the socket
@@ -225,6 +234,7 @@ export class Sp00kySync<S extends SchemaStructure> {
   private readonly degradeAfterFailures: number;
   /** Per-push RPC deadline; see {@link withPushTimeout}. */
   private readonly pushTimeoutMs: number;
+  private readonly downTimeoutMs: number;
   private consecutiveSyncFailures = 0;
   private syncHealthStatus: SyncHealthStatus = 'healthy';
   private lastSyncErrorKind: 'network' | 'application' | undefined;
@@ -461,6 +471,7 @@ export class Sp00kySync<S extends SchemaStructure> {
     this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
     this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
     this.pushTimeoutMs = Math.max(0, options?.pushTimeoutMs ?? 30_000);
+    this.downTimeoutMs = Math.max(0, options?.downTimeoutMs ?? 30_000);
     this.connectionSupervisor = options?.connectionSupervisor;
   }
 
@@ -473,6 +484,13 @@ export class Sp00kySync<S extends SchemaStructure> {
     if (this.isInit) throw new Error('Sp00kySync is already initialized');
     this.isInit = true;
     await this.scheduler.init({ loadOutbox: this.tabRole !== 'follower' });
+    // Boot is local-first now, so init() routinely runs BEFORE the socket is
+    // up. Treat that as "the socket we registered on is gone": otherwise the
+    // first `connected` takes the initial-connect branch, returns early, and
+    // never re-enqueues `register` for queries that registered while offline.
+    // They would still heal via the down-queue backoff, but slowly and only
+    // because every query happens to enqueue its own register.
+    if (this.remote.getStatus() !== 'connected') this.needsResubscribe = true;
     this.subscribeToReconnect();
     this.subscribeToConnectionState();
     void this.scheduler.syncUp();
@@ -1449,6 +1467,24 @@ export class Sp00kySync<S extends SchemaStructure> {
       { event, Category: 'sp00ky-client::Sp00kySync::processDownEvent' },
       'Processing down event'
     );
+    // Bounded for the same reason a push is (see withPushTimeout): an RPC that
+    // never settles would otherwise hold its slot in the concurrent down drain
+    // — and, before that drain existed, the WHOLE queue — for the rest of the
+    // session, with no retry, no error, and every dependent `useQuery` stuck
+    // loading. "timed out" in the message keeps `classifySyncError` treating it
+    // as a network failure, so `DownQueue.run` re-heads it for the next pass.
+    return this.withDownTimeout(this.runDownEvent(event), `${event.type} ${event.payload.hash}`);
+  }
+
+  private withDownTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
+    return withTimeout(
+      promise,
+      this.downTimeoutMs,
+      `Down event timed out after ${this.downTimeoutMs}ms (${label})`
+    );
+  }
+
+  private async runDownEvent(event: DownEvent): Promise<void> {
     switch (event.type) {
       case 'register':
         return this.registerQuery(event.payload.hash);

package/src/services/database/sqlite-cache-engine.ts

@@ -6,6 +6,7 @@ import {
   reviveRow,
   serializeRow,
   project,
+  projectedDataSql,
 } from './sqlite-plan-sql';
 import type { Logger } from '../logger/index';
 import type { Sp00kyConfig, StorageHealth } from '../../types';
@@ -772,7 +773,8 @@ export class SqliteCacheEngine implements LocalStore {
     }
     await this.ensureTable(plan.table);
     const bind: unknown[] = [];
-    const proj = 'data';
+    // Projection binds land in the SELECT clause, ahead of any WHERE binds.
+    const proj = plan.select ? projectedDataSql(plan.select, bind) : 'data';
     let sql = `SELECT ${proj} FROM "${plan.table}"`;
     if (plan.where && plan.where.length > 0) {
       sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
@@ -785,10 +787,8 @@ export class SqliteCacheEngine implements LocalStore {
     if (plan.limit !== undefined) sql += ` LIMIT ${Number(plan.limit)}`;
     if (plan.offset !== undefined) sql += ` OFFSET ${Number(plan.offset)}`;
     const rows = await this.execRows(sql, bind);
-    // Optional projection trimming to match `SELECT <fields>`.
-    const projected = plan.select ? rows.map((r) => project(r, plan.select!)) : rows;
-    await resolveRelations(projected, plan.relations, this);
-    return projected;
+    await resolveRelations(rows, plan.relations, this);
+    return rows;
   }
 
   async fetchRelation(req: RelationFetch): Promise<Row[]> {
@@ -816,14 +816,19 @@ export class SqliteCacheEngine implements LocalStore {
     await this.ensureTable(table);
     const keys = ids.map(stableKey);
     const placeholders = keys.map(() => '?').join(', ');
-    let sql = `SELECT data FROM "${table}" WHERE id IN (${placeholders})`;
+    // Projection binds land in the SELECT clause, so they go first. Must stay
+    // byte-identical to the worker path in sqlite-select.ts.
+    const bind: unknown[] = [];
+    const dataCol = opts?.select ? projectedDataSql(opts.select, bind) : 'data';
+    bind.push(...keys);
+    let sql = `SELECT ${dataCol} FROM "${table}" WHERE id IN (${placeholders})`;
     if (opts?.orderBy && opts.orderBy.length > 0) sql += renderOrderSql(opts.orderBy);
-    let rows = await this.execRows(sql, keys);
+    let rows = await this.execRows(sql, bind);
     if (!opts?.orderBy || opts.orderBy.length === 0) {
       const pos = new Map(keys.map((k, i) => [k, i]));
       rows = rows.sort((a, b) => (pos.get(stableKey(a.id)) ?? 0) - (pos.get(stableKey(b.id)) ?? 0));
     }
-    return opts?.select ? rows.map((r) => project(r, opts.select!)) : rows;
+    return rows;
   }
 
   async getById(table: string, id: Id): Promise<Row | null> {

package/src/services/database/sqlite-plan-sql.ts

@@ -83,6 +83,38 @@ export function reviveRow(json: string): Row {
   });
 }
 
+/**
+ * SELECT-clause expression yielding the row's `data` narrowed to `fields` (plus
+ * `id`), so SQLite never returns — and neither side ever parses — the fields the
+ * caller did not ask for. On a game list that is every row's `pgn`, which was
+ * being read, parsed and thrown away 50 rows at a time for eight rendered fields.
+ *
+ * Byte-identical to running {@link project} over the fully-parsed row, which is
+ * what both paths did before, and verified so against real SQLite:
+ * `json_each` walks the keys the row ACTUALLY has, so an ABSENT key stays absent
+ * rather than becoming an explicit null — the difference `json_object`/
+ * `json_extract` would have introduced. A stored null, nested objects and
+ * arrays, and `{__u8}` blob tags all round-trip unchanged, because
+ * `json_group_object` understands `json_each`'s `value` column as JSON rather
+ * than as text. `COALESCE` covers a row sharing none of the requested keys,
+ * where the subquery yields NULL and `reviveRow` would throw.
+ *
+ * Deliberately unaliased, so the emitted statement keeps the exact shape the
+ * callers already produce (`FROM "t" WHERE id IN (…)`).
+ *
+ * Binds one parameter per key, pushed onto `bind` — these land in the SELECT
+ * clause, so they must be bound BEFORE any WHERE parameters.
+ */
+export function projectedDataSql(fields: string[], bind: unknown[]): string {
+  const keys = ['id', ...fields];
+  for (const k of keys) bind.push(k);
+  const placeholders = keys.map(() => '?').join(', ');
+  return (
+    `COALESCE((SELECT json_group_object(je.key, je.value) ` +
+    `FROM json_each(data) je WHERE je.key IN (${placeholders})), '{}') AS data`
+  );
+}
+
 export function project(row: Row, fields: string[]): Row {
   const out: Row = {};
   for (const f of ['id', ...fields]) if (f in row) out[f] = row[f];

package/src/services/database/sqlite-projection.test.ts

@@ -0,0 +1,99 @@
+import { describe, it, expect } from 'vitest';
+import { DatabaseSync } from 'node:sqlite';
+import { projectedDataSql, project, reviveRow, serializeRow } from './sqlite-plan-sql';
+import type { Row } from './cache-engine';
+
+/**
+ * `projectedDataSql` is the one change in this area that alters what SQLite
+ * actually returns, and every row read goes through it — so it is verified
+ * against a REAL SQLite, not the scripted responder the parity tests use.
+ *
+ * The contract: byte-identical to parsing the whole row and running `project`
+ * over it, which is what both engine paths did before. The traps it has to
+ * clear are an ABSENT key (must stay absent, not become an explicit null — the
+ * difference `json_object`/`json_extract` would have introduced), a STORED
+ * null, nested objects/arrays, and the `{__u8}` tag that carries binary.
+ */
+
+function makeDb(rows: Row[]) {
+  const db = new DatabaseSync(':memory:');
+  db.exec('CREATE TABLE "game" (id TEXT PRIMARY KEY, data TEXT NOT NULL)');
+  const ins = db.prepare('INSERT INTO "game" (id,data) VALUES (?,?)');
+  for (const r of rows) ins.run(String(r.id), serializeRow(r));
+  return db;
+}
+
+/** What the code did before: read the whole row, parse it, then project. */
+function referenceRows(db: DatabaseSync, ids: string[], fields: string[]): Row[] {
+  const ph = ids.map(() => '?').join(', ');
+  return db
+    .prepare(`SELECT data FROM "game" WHERE id IN (${ph}) ORDER BY id ASC`)
+    .all(...ids)
+    .map((r) => project(reviveRow((r as { data: string }).data), fields));
+}
+
+/** What the code does now: let SQLite narrow the row before it is ever parsed. */
+function projectedRows(db: DatabaseSync, ids: string[], fields: string[]): Row[] {
+  const bind: unknown[] = [];
+  const col = projectedDataSql(fields, bind);
+  bind.push(...ids);
+  const ph = ids.map(() => '?').join(', ');
+  return db
+    .prepare(`SELECT ${col} FROM "game" WHERE id IN (${ph}) ORDER BY id ASC`)
+    .all(...(bind as string[]))
+    .map((r) => reviveRow((r as { data: string }).data));
+}
+
+const FIXTURE: Row[] = [
+  { id: 'game:1', white: 'player_name:PN_a', pgn: '[Event "x"]\n1. e4 e5', sort_index: -5 },
+  { id: 'game:2', white: null, pgn: 'heavy', sort_index: -3 }, // STORED null
+  { id: 'game:3', pgn: 'heavy', sort_index: -4 }, // 'white' ABSENT
+  { id: 'game:4', white: 'w', pgn: 'heavy', sort_index: -1, meta: { a: 1, b: [2, 3] } },
+];
+const IDS = ['game:1', 'game:2', 'game:3', 'game:4'];
+const FIELDS = ['white', 'sort_index', 'meta'];
+
+describe('projectedDataSql against real SQLite', () => {
+  it('is byte-identical to parse-then-project', () => {
+    const db = makeDb(FIXTURE);
+    expect(projectedRows(db, IDS, FIELDS)).toEqual(referenceRows(db, IDS, FIELDS));
+  });
+
+  it('never returns the fields that were not asked for', () => {
+    // The point of the change: `pgn` must not leave SQLite, so it is never
+    // parsed, never crosses the worker boundary, never reaches the store.
+    const db = makeDb(FIXTURE);
+    for (const row of projectedRows(db, IDS, FIELDS)) expect('pgn' in row).toBe(false);
+  });
+
+  it('keeps an absent key absent and a stored null null', () => {
+    const db = makeDb(FIXTURE);
+    const byId = new Map(projectedRows(db, IDS, FIELDS).map((r) => [r.id, r]));
+    expect('white' in byId.get('game:3')!).toBe(false); // absent stays absent
+    expect('white' in byId.get('game:2')!).toBe(true); // stored null is a real key
+    expect(byId.get('game:2')!.white).toBeNull();
+  });
+
+  it('round-trips nested objects and arrays as JSON, not as strings', () => {
+    const db = makeDb(FIXTURE);
+    const row = projectedRows(db, IDS, FIELDS).find((r) => r.id === 'game:4')!;
+    expect(row.meta).toEqual({ a: 1, b: [2, 3] });
+  });
+
+  it('round-trips binary through the {__u8} tag', () => {
+    const db = makeDb([{ id: 'game:9', blob: new Uint8Array([0, 1, 2]), pgn: 'heavy' }]);
+    const [row] = projectedRows(db, ['game:9'], ['blob']);
+    expect(row!.blob).toEqual(new Uint8Array([0, 1, 2]));
+  });
+
+  it('always includes id, even when it is not in the field list', () => {
+    const db = makeDb(FIXTURE);
+    for (const row of projectedRows(db, IDS, ['sort_index'])) expect(row.id).toBeTruthy();
+  });
+
+  it('yields an empty object, not NULL, for a row sharing none of the keys', () => {
+    // Without COALESCE the subquery returns NULL here and reviveRow throws.
+    const db = makeDb([{ id: 'game:8', pgn: 'heavy' }]);
+    expect(projectedRows(db, ['game:8'], ['nothing_matches'])).toEqual([{ id: 'game:8' }]);
+  });
+});

package/src/services/database/sqlite-select.ts

@@ -1,6 +1,12 @@
 import type { QueryPlan } from '@spooky-sync/query-builder';
 import { resolveRelations, stableKey } from './relation-resolver';
-import { renderOrderSql, renderWhereSql, reviveRow, project } from './sqlite-plan-sql';
+import {
+  renderOrderSql,
+  renderWhereSql,
+  reviveRow,
+  project,
+  projectedDataSql,
+} from './sqlite-plan-sql';
 import type { OrderBy, RelationFetch, Row, RowFetcher } from './cache-engine';
 
 /**
@@ -48,14 +54,18 @@ function selectByIds(
   ensureTable(db, table);
   const keys = ids.map(stableKey);
   const placeholders = keys.map(() => '?').join(', ');
-  let sql = `SELECT data FROM "${table}" WHERE id IN (${placeholders})`;
+  // Projection binds land in the SELECT clause, so they go first.
+  const bind: unknown[] = [];
+  const dataCol = opts?.select ? projectedDataSql(opts.select, bind) : 'data';
+  bind.push(...keys);
+  let sql = `SELECT ${dataCol} FROM "${table}" WHERE id IN (${placeholders})`;
   if (opts?.orderBy && opts.orderBy.length > 0) sql += renderOrderSql(opts.orderBy);
-  let rows = execRows(db, sql, keys);
+  let rows = execRows(db, sql, bind);
   if (!opts?.orderBy || opts.orderBy.length === 0) {
     const pos = new Map(keys.map((k, i) => [k, i]));
     rows = rows.sort((a, b) => (pos.get(stableKey(a.id)) ?? 0) - (pos.get(stableKey(b.id)) ?? 0));
   }
-  return opts?.select ? rows.map((r) => project(r, opts.select!)) : rows;
+  return rows;
 }
 
 /** Mirrors the engine's `fetchRelation` SQL exactly. */
@@ -98,7 +108,9 @@ export async function executeSelect(
   }
   ensureTable(db, plan.table);
   const bind: unknown[] = [];
-  let sql = `SELECT data FROM "${plan.table}"`;
+  // Projection binds land in the SELECT clause, ahead of any WHERE binds.
+  const dataCol = plan.select ? projectedDataSql(plan.select, bind) : 'data';
+  let sql = `SELECT ${dataCol} FROM "${plan.table}"`;
   if (plan.where && plan.where.length > 0) {
     sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
   }
@@ -114,8 +126,6 @@ export async function executeSelect(
   if (plan.limit !== undefined) sql += ` LIMIT ${Number(plan.limit)}`;
   if (plan.offset !== undefined) sql += ` OFFSET ${Number(plan.offset)}`;
   const rows = execRows(db, sql, bind);
-  // Optional projection trimming to match `SELECT <fields>`.
-  const projected = plan.select ? rows.map((r) => project(r, plan.select!)) : rows;
-  await resolveRelations(projected, plan.relations, fetcher);
-  return { rows: projected, relationFetches: counter.n };
+  await resolveRelations(rows, plan.relations, fetcher);
+  return { rows, relationFetches: counter.n };
 }