@spooky-sync/core 0.0.1-canary.286 → 0.0.1-canary.287

package/src/query/membership.saga.test.ts

@@ -74,7 +74,7 @@ describe('applyMembership', () => {
   });
   it('applied: commits, writes the view row, flips authority once, releases acked items, asks for bodies', async () => {
     const s = buildState(
-      [buildEntry({ def: { hash: 'a', viewKey: 'vk' }, lifecycle: { phase: 'cold' } })],
+      [buildEntry({ def: { hash: 'a', viewKey: 'vk' }, lifecycle: { phase: 'cold' }, subqueryRemoteArray: [['c:1', 1]] })],
       R.outboxReplace([buildOutboxItem({ id: 'm', recordId: 't:1', status: 'acked', ackedAt: 1 })])
     );
     const out = await runPure(applyMembership('a', [['t:1', 1]], ready(1)), { state: s, now: 42, handlers: { 'local.upsert': () => undefined } });
@@ -86,9 +86,11 @@ describe('applyMembership', () => {
       kind: 'local.upsert',
       table: '_00_view',
       id: new RecordId('_00_view', 'vk'),
-      data: { ids: [['t:1', 1]], confirmed: true, updatedAt: 42 },
+      data: { ids: [['t:1', 1]], children: [['c:1', 1]], confirmed: true, updatedAt: 42 },
       mode: 'replace',
     });
+    // The index vouches for what the row now holds: members and children.
+    expect(out.state.views.get('vk')).toEqual(['t:1', 'c:1']);
     expect(out.dispatched).toEqual([{ type: 'FetchRows' }]);
     const second = await runPure(applyMembership('a', [['t:1', 2]], ready(1)), {
       state: out.state,
@@ -116,6 +118,32 @@ describe('applySubqueryChildren', () => {
     const out = await runPure(applySubqueryChildren('a', [['c:2', 1]]), { state: s });
     expect(out.state.queries.get('a')!.subqueryRemoteArray).toEqual([['c:2', 1]]);
     expect(out.dispatched).toEqual([{ type: 'FetchRows' }]);
+    // Cold: no `_00_view` row to record them in, and the index is untouched.
+    expect(out.log.filter((e) => e.kind === 'local.upsert')).toEqual([]);
+    expect(out.state.views.size).toBe(0);
+  });
+  it('a query with a view row records the children there and in the index; a failed write is logged', async () => {
+    const s = buildState([buildEntry({ def: { hash: 'a', viewKey: 'vk' }, lifecycle: { phase: 'live' }, remoteArray: [['t:1', 1]] })]);
+    const out = await runPure(applySubqueryChildren('a', [['c:2', 1]]), { state: s, handlers: { 'local.upsert': () => undefined } });
+    expect(out.log.find((e) => e.kind === 'local.upsert')).toEqual({
+      kind: 'local.upsert',
+      table: '_00_view',
+      id: new RecordId('_00_view', 'vk'),
+      data: { children: [['c:2', 1]] },
+      mode: 'merge',
+    });
+    expect(out.state.views.get('vk')).toEqual(['t:1', 'c:2']);
+    expect(out.dispatched).toEqual([{ type: 'FetchRows' }]);
+    const failed = await runPure(applySubqueryChildren('a', [['c:3', 1]]), {
+      state: s,
+      handlers: {
+        'local.upsert': () => {
+          throw new Error('disk');
+        },
+      },
+    });
+    expect(failed.emitted).toEqual([expect.objectContaining({ level: 'debug', message: 'view row write failed' })]);
+    expect(failed.dispatched).toEqual([{ type: 'FetchRows' }]);
   });
 });
 

package/src/query/membership.saga.ts

@@ -89,7 +89,12 @@ export function* applyMembership(
   if (!wasAuthoritative) yield fx.emit({ type: 'query:authority', hash, known: true });
   const now = (yield fx.now()) as number;
   try {
-    yield fx.local.upsert(sql.VIEW_TABLE, sql.viewRecordId(entry.def.viewKey), sql.viewRow(remoteArray, true, now), 'replace');
+    yield fx.local.upsert(
+      sql.VIEW_TABLE,
+      sql.viewRecordId(entry.def.viewKey),
+      sql.viewRow(remoteArray, entry.subqueryRemoteArray, true, now),
+      'replace'
+    );
   } catch (err) {
     yield fx.emit({ type: 'log', level: 'debug', message: 'view row write failed', data: { hash, err } });
   }
@@ -112,11 +117,22 @@ export function* recoverLostView(hash: QueryHash): Saga<void> {
   yield fx.dispatch({ type: 'RegisterRemote', hash });
 }
 
-/** Replace the subquery child set; bodies follow through the fetch plan. */
+/**
+ * Replace the subquery child set; bodies follow through the fetch plan. A
+ * query with a `_00_view` row (past `cold`) records the children there too,
+ * so the GC keeps their bodies and a cold scan may paint them.
+ */
 export function* applySubqueryChildren(hash: QueryHash, children: RecordVersionArray): Saga<void> {
   const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
   if (!entry || recordVersionArraysEqual(entry.subqueryRemoteArray, children)) return;
   yield fx.state.update(R.setSubqueryRemoteArray(hash, children));
+  if (entry.lifecycle.phase !== 'cold') {
+    try {
+      yield fx.local.upsert(sql.VIEW_TABLE, sql.viewRecordId(entry.def.viewKey), { children }, 'merge');
+    } catch (err) {
+      yield fx.emit({ type: 'log', level: 'debug', message: 'view row write failed', data: { hash, err } });
+    }
+  }
   yield fx.dispatch({ type: 'FetchRows' });
 }
 

package/src/query/membership.ts

@@ -16,6 +16,36 @@ export function parseViewRow(row: unknown): DurableView | null {
   return { ids: ids as RecordVersionArray, confirmed: (row as { confirmed?: unknown }).confirmed === true };
 }
 
+/** One `_00_view` row as the view index and the GC see it. */
+export interface ViewIndexRow {
+  key: string;
+  /** Members and subquery children: what the row vouches for. */
+  ids: string[];
+  /** Last time a session resolved the query (0 when the row predates the field). */
+  updatedAt: number;
+}
+
+const pairIds = (pairs: unknown): string[] =>
+  Array.isArray(pairs) ? pairs.filter((p): p is [string, number] => Array.isArray(p) && typeof p[0] === 'string').map((p) => p[0]) : [];
+
+/** `SELECT * FROM _00_view` rows, keyed by view key; junk rows are skipped. */
+export function parseViewIndex(rows: unknown): ViewIndexRow[] {
+  if (!Array.isArray(rows)) return [];
+  const out: ViewIndexRow[] = [];
+  for (const row of rows as Array<Record<string, unknown> | null>) {
+    if (!row || typeof row !== 'object' || !Array.isArray(row.ids)) continue;
+    const id = row.id;
+    const key = typeof id === 'string' ? id.slice(id.indexOf(':') + 1) : String((id as { id?: unknown } | undefined)?.id ?? '');
+    if (!key) continue;
+    out.push({
+      key,
+      ids: [...new Set([...pairIds(row.ids), ...pairIds(row.children)])],
+      updatedAt: typeof row.updatedAt === 'number' ? row.updatedAt : 0,
+    });
+  }
+  return out;
+}
+
 /**
  * "Resolved before": the server answered this query on this device at some
  * point. An unconfirmed empty row cannot be told apart from one written before

package/src/query/render-set.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { buildRenderIds, resolveMembership } from './render-set';
+import { buildRenderIds, paintsCold, resolveMembership } from './render-set';
 
 const ov = (writes: string[] = [], deletes: string[] = []) => ({ writes: new Set(writes), deletes: new Set(deletes) });
 
@@ -34,3 +34,14 @@ describe('buildRenderIds', () => {
     expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: false, isWindow: true })).toEqual(['t:b', 't:a']);
   });
 });
+
+describe('paintsCold', () => {
+  it('only vouched bodies, optimistic writes and bookkeeping paint before the server answers', () => {
+    const vouched = new Set(['t:1']);
+    expect(paintsCold('t:1', vouched, ov())).toBe(true);
+    expect(paintsCold('t:2', vouched, ov(['t:2']))).toBe(true);
+    expect(paintsCold('_00_user_feature:f', vouched, ov())).toBe(true);
+    expect(paintsCold('t:3', vouched, ov([], ['t:3']))).toBe(false);
+    expect(paintsCold('t:3', vouched, ov())).toBe(false);
+  });
+});

package/src/query/render-set.ts

@@ -21,6 +21,17 @@ export function resolveMembership(input: RenderInput): RecordVersionArray | null
   return null;
 }
 
+/**
+ * Whether a body may paint before the server has answered (`cold`). One no
+ * durable view vouches for and no optimistic write touches is an orphan:
+ * deleted upstream while no view watched it, or dropped from every view since.
+ * The predicate scan would otherwise resurrect it until the server answers,
+ * and indefinitely offline.
+ */
+export function paintsCold(id: string, vouched: ReadonlySet<string>, overlay: Overlay): boolean {
+  return id.startsWith('_00_') || vouched.has(id) || overlay.writes.has(id);
+}
+
 export interface RenderOptions {
   hasExplicitOrder: boolean;
   isWindow: boolean;

package/src/query/sql.test.ts

@@ -33,7 +33,8 @@ describe('query sql builders (golden against the previous builders)', () => {
   });
   it('view rows and ids', () => {
     expect(sql.viewRecordId('k')).toEqual(new RecordId('_00_view', 'k'));
-    expect(sql.viewRow([['t:1', 1]], true, 5)).toEqual({ ids: [['t:1', 1]], confirmed: true, updatedAt: 5 });
+    expect(sql.viewRow([['t:1', 1]], [['c:1', 2]], true, 5)).toEqual({ ids: [['t:1', 1]], children: [['c:1', 2]], confirmed: true, updatedAt: 5 });
+    expect(sql.readViewRows()).toBe('SELECT * FROM _00_view');
     expect(sql.readLegacyViewRows()).toBe('SELECT * FROM _00_window');
     expect(sql.countViewRows()).toContain('_00_view');
     expect(sql.bodySelect()).toBe('SELECT * FROM $ids');

package/src/query/sql.ts

@@ -11,12 +11,26 @@ export const LEGACY_VIEW_TABLE = '_00_window';
 
 export const viewRecordId = (viewKey: string): RecordId<string> => new RecordId(VIEW_TABLE, viewKey);
 
-export const viewRow = (ids: RecordVersionArray, confirmed: boolean, now: number): Record<string, unknown> => ({
+/**
+ * The durable `_00_view` row: the confirmed membership plus the subquery
+ * children, which the GC must keep and a cold scan may paint as much as the
+ * members.
+ */
+export const viewRow = (
+  ids: RecordVersionArray,
+  children: RecordVersionArray,
+  confirmed: boolean,
+  now: number
+): Record<string, unknown> => ({
   ids,
+  children,
   confirmed,
   updatedAt: now,
 });
 
+/** Every `_00_view` row: the durable view index and the GC read it. */
+export const readViewRows = (): string => `SELECT * FROM ${VIEW_TABLE}`;
+
 /** Copy every legacy `_00_window` row into `_00_view` once. */
 export const readLegacyViewRows = (): string => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
 export const countViewRows = (): string => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;

package/src/state/client-state.ts

@@ -126,6 +126,12 @@ export interface ClientState {
   readonly membershipReread: ReadonlyMap<QueryHash, number>;
   /** Local body versions (`_00_rv`) by encoded record id. */
   readonly versions: ReadonlyMap<string, number>;
+  /**
+   * What each durable `_00_view` row vouches for, by view key: its members
+   * and subquery children. A body no row names is an orphan: it never paints
+   * before the server answers and the GC deletes it.
+   */
+  readonly views: ReadonlyMap<string, ReadonlyArray<string>>;
   readonly outbox: ReadonlyArray<OutboxItem>;
   readonly pendingWrites: ReadonlyMap<string, PendingWrite>;
   readonly failedCount: number;
@@ -169,6 +175,7 @@ export function emptyState(init: { tabId: string }): ClientState {
     registering: new Set(),
     membershipReread: new Map(),
     versions: new Map(),
+    views: new Map(),
     outbox: [],
     pendingWrites: new Map(),
     failedCount: 0,

package/src/state/reducers.test.ts

@@ -281,3 +281,22 @@ describe('identity / connection / compose', () => {
     expect(s6.failedCount).toBe(3);
   });
 });
+
+describe('durable view index', () => {
+  it('commit and children keep a view key in step with its `_00_view` row; cold children are not indexed', () => {
+    let s = buildState([e('a', { def: { viewKey: 'va' }, lifecycle: { phase: 'cold' }, subqueryRemoteArray: [['c:1', 1]] })]);
+    s = R.setSubqueryRemoteArray('a', [['c:1', 1], ['c:2', 1]])(s);
+    expect(s.views.size).toBe(0);
+    s = R.commitMembership('a', [['t:1', 1], ['c:1', 1]], true)(s);
+    expect(s.views.get('va')).toEqual(['t:1', 'c:1', 'c:2']);
+    s = R.setSubqueryRemoteArray('a', [['c:3', 1]])(s);
+    expect(s.views.get('va')).toEqual(['t:1', 'c:1', 'c:3']);
+    expect(R.setSubqueryRemoteArray('zz', [['c:9', 1]])(s)).toBe(s);
+    expect(R.clearBucketState()(s).views.size).toBe(0);
+  });
+  it('a reload takes the store, except entries of queries in state', () => {
+    const s = R.compose(R.commitMembership('a', [['t:new', 1]], true))(buildState([e('a', { def: { viewKey: 'va' } }), e('b', { def: { viewKey: 'vb' } })]));
+    const reloaded = R.reloadViews(new Map([['va', ['t:old']], ['vb', ['t:b']], ['vc', ['t:c']]]))(s);
+    expect(reloaded.views).toEqual(new Map([['va', ['t:new']], ['vb', ['t:b']], ['vc', ['t:c']]]));
+  });
+});

package/src/state/reducers.ts

@@ -90,6 +90,33 @@ export const setServerState =
  * set names: membership has caught up with the write, the overlay's job is
  * done (invariant I7).
  */
+/** What a `_00_view` row holding these arrays vouches for. */
+const vouchedIds = (members: RecordVersionArray, children: RecordVersionArray): string[] => [
+  ...new Set([...members.map(([id]) => id), ...children.map(([id]) => id)]),
+];
+
+const withView = (s: ClientState, viewKey: string, ids: ReadonlyArray<string>): ClientState => {
+  const views = new Map(s.views);
+  views.set(viewKey, ids);
+  return { ...s, views };
+};
+
+/**
+ * Replace the durable view index with a fresh `_00_view` read (boot, bucket
+ * switch, GC). An entry of a query in state is kept: its commits update the
+ * index as they write the row, so it is never older than the read.
+ */
+export const reloadViews =
+  (fromStore: ReadonlyMap<string, ReadonlyArray<string>>): Reducer =>
+  (s) => {
+    const views = new Map(fromStore);
+    for (const e of s.queries.values()) {
+      const held = s.views.get(e.def.viewKey);
+      if (held) views.set(e.def.viewKey, held);
+    }
+    return { ...s, views };
+  };
+
 export const commitMembership =
   (hash: QueryHash, remoteArray: RecordVersionArray, present: boolean): Reducer =>
   (s) => {
@@ -99,9 +126,11 @@ export const commitMembership =
       lifecycle: transition(e.lifecycle, { type: 'membership-applied', present }),
     }));
     if (withArray === s) return s;
+    const entry = withArray.queries.get(hash)!;
     const named = new Set(remoteArray.map(([id]) => id));
     const outbox = withArray.outbox.filter((item) => !(item.status === 'acked' && named.has(item.recordId)));
-    return { ...withArray, outbox, dirty: addAll(withArray.dirty, [hash]) };
+    const indexed = withView(withArray, entry.def.viewKey, vouchedIds(remoteArray, entry.subqueryRemoteArray));
+    return { ...indexed, outbox, dirty: addAll(withArray.dirty, [hash]) };
   };
 
 export const setLocalArray =
@@ -111,10 +140,18 @@ export const setLocalArray =
     return next === s ? s : { ...next, dirty: addAll(next.dirty, [hash]) };
   };
 
+/**
+ * Replace a query's subquery child set. A query past `cold` has a `_00_view`
+ * row, which records the children too (see `applySubqueryChildren`).
+ */
 export const setSubqueryRemoteArray =
   (hash: QueryHash, subqueryRemoteArray: RecordVersionArray): Reducer =>
-  (s) =>
-    withEntry(s, hash, (e) => ({ ...e, subqueryRemoteArray }));
+  (s) => {
+    const next = withEntry(s, hash, (e) => ({ ...e, subqueryRemoteArray }));
+    const entry = next.queries.get(hash);
+    if (!entry || entry.lifecycle.phase === 'cold') return next;
+    return withView(next, entry.def.viewKey, vouchedIds(entry.remoteArray, subqueryRemoteArray));
+  };
 
 export const setRecords =
   (hash: QueryHash, records: ReadonlyArray<Row>, changed: boolean, materializeMs: number | null): Reducer =>
@@ -441,6 +478,7 @@ export const rebindQuery =
 export const clearBucketState = (): Reducer => (s) => ({
   ...s,
   versions: new Map(),
+  views: new Map(),
   outbox: [],
   pendingWrites: new Map(),
   dirty: new Set(),

package/src/state/selectors.test.ts

@@ -147,3 +147,21 @@ describe('phaseTimings', () => {
     expect(t.registration.parseMs).toBeNull();
   });
 });
+
+describe('view index selectors', () => {
+  it('vouched is the union of every view row, memoized on the index', () => {
+    const s = R.reloadViews(new Map([['v1', ['t:1', 't:2']], ['v2', ['t:2', 'c:1']]]))(buildState());
+    expect([...S.vouched(s)]).toEqual(['t:1', 't:2', 'c:1']);
+    expect(S.vouched({ ...s, dirty: new Set(['x']) })).toBe(S.vouched(s));
+  });
+  it('retained: vouched, held in memory (members, children), in the outbox or a debounced write, or bookkeeping', () => {
+    const s = R.compose(
+      R.reloadViews(new Map([['v', ['t:vouched']]])),
+      R.outboxReplace([buildOutboxItem({ recordId: 't:outbox' })]),
+      (st) => ({ ...st, pendingWrites: new Map([['k', { key: 'k', table: 't', recordId: 't:debounced', data: {}, before: null, firstAt: 0 }]]) })
+    )(buildState([e('a', { remoteArray: [['t:member', 1]], subqueryRemoteArray: [['t:child', 1]] })]));
+    const keep = S.retained(s);
+    for (const id of ['t:vouched', 't:member', 't:child', 't:outbox', 't:debounced', '_00_view:x']) expect(keep(id)).toBe(true);
+    expect(keep('t:orphan')).toBe(false);
+  });
+});

package/src/state/selectors.ts

@@ -35,6 +35,38 @@ export function overlay(s: ClientState): Overlay {
   return { writes, deletes };
 }
 
+const vouchedCache = new WeakMap<ClientState['views'], ReadonlySet<string>>();
+
+/**
+ * Every id some durable `_00_view` row vouches for. Memoized on the index,
+ * which only changes when a view row does.
+ */
+export function vouched(s: ClientState): ReadonlySet<string> {
+  const hit = vouchedCache.get(s.views);
+  if (hit) return hit;
+  const out = new Set<string>();
+  for (const ids of s.views.values()) for (const id of ids) out.add(id);
+  vouchedCache.set(s.views, out);
+  return out;
+}
+
+/**
+ * Bodies the GC must keep: `_00_` bookkeeping, anything a durable view
+ * vouches for, anything a query in memory names (members or children), and
+ * anything a write is still in flight for.
+ */
+export function retained(s: ClientState): (id: string) => boolean {
+  const held = new Set<string>();
+  for (const e of s.queries.values()) {
+    for (const [id] of e.remoteArray) held.add(id);
+    for (const [id] of e.subqueryRemoteArray) held.add(id);
+  }
+  for (const item of s.outbox) held.add(item.recordId);
+  for (const write of s.pendingWrites.values()) held.add(write.recordId);
+  const known = vouched(s);
+  return (id) => id.startsWith('_00_') || known.has(id) || held.has(id);
+}
+
 export const pendingDeleteIds = (s: ClientState): ReadonlySet<string> =>
   new Set(s.outbox.filter((i) => i.type === 'delete').map((i) => i.recordId));
 

package/src/sync/tabs.saga.test.ts

@@ -43,6 +43,10 @@ describe('tabMessage', () => {
     expect(out.state.versions.get('other:1')).toBe(0);
     expect(out.state.versions.has('thing:1')).toBe(false);
     expect(out.state.dirty.has('a')).toBe(true);
+    // A relayed delete (another tab's GC) may drop a body this tab names: the fetch plan decides.
+    expect(out.dispatched).toEqual([{ type: 'FetchRows' }]);
+    const writesOnly = await runPure(tabMessage(env, { ...msg, records: msg.records.filter((r) => r.op !== 'DELETE') }), { state: s(), handlers: { 'ssp.ingest': () => undefined } });
+    expect(writesOnly.dispatched).toEqual([]);
     const failing = await runPure(tabMessage(env, msg), {
       state: s(),
       handlers: {

package/src/sync/tabs.saga.ts

@@ -60,6 +60,9 @@ export function* tabMessage(env: SagaEnv, raw: unknown): Saga<void> {
           ...[...new Set(msg.records.map((r) => r.table))].map((t) => R.markTableDirty(t))
         )
       );
+      // A body another tab deleted (its orphan GC) may still be one this tab
+      // names: with the version gone, the fetch plan pulls it back.
+      if (msg.records.some((r) => r.op === 'DELETE')) yield fx.dispatch({ type: 'FetchRows' });
       return;
     }
     case 'membership-dirty':