@spooky-sync/core 0.0.1-canary.331 → 0.0.1-canary.333

package/src/modules/crdt/index.ts

@@ -80,9 +80,14 @@ interface Session {
   verifyTimer: ReturnType<typeof setTimeout> | null;
   disposers: Array<() => void>;
   flushTimer: ReturnType<typeof setTimeout> | null;
-  firstDirtyAt: number;
+  /** When the armed flush timer fires. */
+  flushDueAt: number;
+  /** When this session last sent edits (0: not yet). */
+  lastSentAt: number;
   persistTimer: ReturnType<typeof setTimeout> | null;
   flushing: Promise<void> | null;
+  /** A flush asked for while one was running: it runs once that one is done. */
+  flushQueued: Promise<void> | null;
   refetching: boolean;
   presenceTimer: ReturnType<typeof setTimeout> | null;
   presencePending: Uint8Array | null;
@@ -99,8 +104,6 @@ interface Session {
 const LOCAL_TABLE = '_00_crdt_local';
 const LOG_SUFFIX = '__crdt_log';
 const PRESENCE_SUFFIX = '__crdt_presence';
-/** Longest a burst of typing waits before its edits go out. */
-const MAX_WAIT_MS = 2000;
 const PERSIST_DEBOUNCE_MS = 400;
 const PRESENCE_THROTTLE_MS = 250;
 const CLAIM_RETRY_MS = 5000;
@@ -129,7 +132,10 @@ export class CrdtManager {
     private local: LocalStore,
     private remote: RemoteDatabaseService,
     logger: Logger,
-    private debounceMs: number = 300
+    /** First edit after a quiet spell: wait this long to batch the burst. */
+    private debounceMs: number = 50,
+    /** Shortest gap between two sends of one field, while edits keep coming. */
+    private sendIntervalMs: number = 200
   ) {
     this.logger = logger.child({ service: 'CrdtManager' });
     this.connectionUnsubscribes.push(
@@ -289,9 +295,11 @@ export class CrdtManager {
       verifyTimer: null,
       disposers: [],
       flushTimer: null,
-      firstDirtyAt: 0,
+      flushDueAt: 0,
+      lastSentAt: 0,
       persistTimer: null,
       flushing: null,
+      flushQueued: null,
       refetching: false,
       presenceTimer: null,
       presencePending: null,
@@ -309,9 +317,11 @@ export class CrdtManager {
 
     // Claimed now, so the first edits do not wait on the round trip.
     void this.ensureClaimed();
-    // First open on this device: start from the published document. Offline
-    // it is read once the connection is back instead of holding the open.
-    if (!local.snapshot && !fresh) await this.serverWait(this.refetchCanonical(session));
+    // First open on this device: start from the published document. Read at
+    // the same time as the log query below registers, not before it: each is
+    // a round trip, and the import order does not matter (ops whose
+    // predecessors are still missing wait inside Loro until they arrive).
+    const canonical = !local.snapshot && !fresh ? this.refetchCanonical(session) : null;
 
     session.disposers.push(
       doc.subscribeLocalUpdates(() => {
@@ -323,20 +333,22 @@ export class CrdtManager {
       })
     );
     const ports = this.ports;
-    const log = await ports.watch(`${table}${LOG_SUFFIX}`, { doc: recordId, field }, (rows) => {
+    const logWatch = ports.watch(`${table}${LOG_SUFFIX}`, { doc: recordId, field }, (rows) => {
       session.ingesting = session.ingesting.then(() => this.ingest(session, rows));
     });
+    const presenceWatch = cursor
+      ? ports.watch(`${table}${PRESENCE_SUFFIX}`, { doc: recordId, field }, (rows) => this.ingestPresence(session, rows))
+      : null;
+    const [log, presence] = await Promise.all([logWatch, presenceWatch]);
     session.disposers.push(log.stop);
+    if (presence) session.disposers.push(presence.stop);
     void log.settled.then(() => {
       session.logSettled = true;
       void session.ingesting.then(() => this.checkLoaded(session));
     });
-    if (cursor) {
-      const presence = await ports.watch(`${table}${PRESENCE_SUFFIX}`, { doc: recordId, field }, (rows) =>
-        this.ingestPresence(session, rows)
-      );
-      session.disposers.push(presence.stop);
-    }
+    // Offline the published document is read once the connection is back
+    // instead of holding the open.
+    if (canonical) await this.serverWait(canonical);
     // An editor binding an empty document creates its root structure, and
     // two clients doing that at once keep only one of them (with whatever was
     // typed into the other). With nothing local, hand the document out only
@@ -527,19 +539,37 @@ export class CrdtManager {
 
   // --- sending -----------------------------------------------------
 
-  private scheduleFlush(s: Session, delay = this.debounceMs): void {
+  /**
+   * Arm the send. Edits go out on a steady cadence while they keep coming:
+   * the first one after a quiet spell waits `debounceMs` (one row for the
+   * burst), later ones at least `sendIntervalMs` after the previous send. An
+   * armed timer is never pushed back, so continuous typing cannot hold the
+   * edits; it is only brought forward (`delay`, e.g. 0 on reconnect).
+   */
+  private scheduleFlush(s: Session, delay?: number): void {
     const now = Date.now();
-    if (!s.flushTimer) s.firstDirtyAt = now;
-    else clearTimeout(s.flushTimer);
-    const wait = Math.max(0, Math.min(delay, s.firstDirtyAt + MAX_WAIT_MS - now));
+    const due = delay !== undefined ? now + delay : Math.max(now + this.debounceMs, s.lastSentAt + this.sendIntervalMs);
+    if (s.flushTimer) {
+      if (s.flushDueAt <= due) return;
+      clearTimeout(s.flushTimer);
+    }
+    s.flushDueAt = due;
     s.flushTimer = setTimeout(() => {
       s.flushTimer = null;
       void this.flush(s);
-    }, wait);
+    }, due - now);
   }
 
-  private async flush(s: Session): Promise<void> {
-    if (s.flushing) return s.flushing;
+  /** Send what is unsent. A call while a send runs queues one more after it,
+   *  so edits made during that send are not left waiting for the next edit. */
+  private flush(s: Session): Promise<void> {
+    if (s.flushing) {
+      s.flushQueued ??= s.flushing.then(() => {
+        s.flushQueued = null;
+        return this.flush(s);
+      });
+      return s.flushQueued;
+    }
     s.flushing = this.sendOwnEdits(s).finally(() => {
       s.flushing = null;
     });
@@ -557,6 +587,7 @@ export class CrdtManager {
       .map((peer) => ({ peer, end: vv.get(peer as `${number}`) ?? 0, from: s.local.peers[peer]?.enqueued ?? 0 }))
       .filter((p) => p.end > p.from);
     if (due.length === 0 || !this.ports) return;
+    s.lastSentAt = Date.now();
     if (!(await this.ensureClaimed())) {
       this.retryLater(s);
       return;
@@ -581,7 +612,12 @@ export class CrdtManager {
       }
       s.local.peers[peer] = { enqueued: end };
     }
-    await this.persist(s);
+    // The outbox marks reach the device with the next persist. An open field
+    // sends every few hundred ms while typing, and a persist exports the whole
+    // document, so it rides the persist debounce; a crash in between only
+    // re-sends rows, which import as no-ops. A closing field saves now.
+    if (this.sessions.get(s.key) === s) this.schedulePersist(s);
+    else await this.persist(s);
   }
 
   private retryLater(s: Session): void {

package/src/query/anon-query-isolation.integration.test.ts

@@ -58,14 +58,18 @@ function ownerGuard(): string {
 
 const sha256 = (text: string) => createHash('sha256').update(text).digest('hex');
 
-/** Just the `_00_query` block: the rest of the file needs tables and modules this test does not. */
+/**
+ * Just the session guard (its rule's user branch asks it) and the `_00_query`
+ * block: the rest of the file needs tables and modules this test does not.
+ */
 function queryTableSchema(): string {
   const meta = cliSource('meta_tables_remote.surql');
+  const guard = meta.indexOf('-- SPOOKY SESSION GUARD');
   const start = meta.indexOf('DEFINE TABLE OVERWRITE _00_query SCHEMALESS');
   const end = meta.indexOf('-- SPOOKY LIST REF');
-  if (start < 0 || end < start)
+  if (guard < 0 || start < guard || end < start)
     throw new Error('_00_query block not found in meta_tables_remote.surql');
-  return meta.slice(start, end);
+  return meta.slice(guard, meta.indexOf('-- SPOOKY INCANTATION')) + meta.slice(start, end);
 }
 
 const ACCESS = `

package/src/query/membership.saga.test.ts

@@ -4,7 +4,7 @@ import { runPure } from '../testing/run-pure';
 import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
 import * as R from '../state/reducers';
 import { defaultEnv } from './env';
-import { applyMembership, applySubqueryChildren, markMembershipDirty, readDirtyMembership, readMembership, recoverLostView } from './membership.saga';
+import { applyEdgeDeltas, applyMembership, applySubqueryChildren, markMembershipDirty, readDirtyMembership, readMembership, recoverLostView } from './membership.saga';
 import type { StatementResult } from '../kernel/effects';
 
 const env = defaultEnv({ tables: [] } as any);
@@ -405,3 +405,75 @@ describe('dirty helpers', () => {
     expect(stuck.timers.get('membership')).toEqual({ ms: 50, event: { type: 'ReadDirtyMembership' } });
   });
 });
+
+describe('applyEdgeDeltas', () => {
+  const handlers = { 'local.upsert': () => undefined };
+  const live = (over: Record<string, unknown> = {}) =>
+    buildEntry({ def: { hash: 'a', viewKey: 'vk' }, lifecycle: { phase: 'live', remote: 'registered' }, remoteArray: [['t:1', 1]], ...over });
+
+  it('commits a joining row to a live view without reading the edges', async () => {
+    const out = await runPure(applyEdgeDeltas([{ hash: 'a', id: 't:2', version: 1 }, { hash: 'a', id: 't:1', version: 3 }]), {
+      state: buildState([live()]),
+      handlers,
+    });
+    expect([...out.result]).toEqual(['a']);
+    expect(out.state.queries.get('a')!.remoteArray).toEqual([['t:1', 3], ['t:2', 1]]);
+    expect(out.log.some((e) => e.kind === 'remote.query')).toBe(false);
+    // The durable view row follows, and the bodies are asked for.
+    expect(out.log.filter((e) => e.kind === 'local.upsert')).toHaveLength(1);
+    expect(out.dispatched).toContainEqual({ type: 'FetchRows' });
+  });
+
+  it('takes the fast path for a planned query with no LIMIT too', async () => {
+    const out = await runPure(applyEdgeDeltas([{ hash: 'a', id: 't:2', version: 1 }]), {
+      state: buildState([live({ def: { hash: 'a', viewKey: 'vk', plan: { table: 't' } } })]),
+      handlers,
+    });
+    expect([...out.result]).toEqual(['a']);
+  });
+
+  it('settles a delta it already holds without writing anything', async () => {
+    const out = await runPure(applyEdgeDeltas([{ hash: 'a', id: 't:1', version: 1 }]), { state: buildState([live()]) });
+    expect([...out.result]).toEqual(['a']);
+    expect(out.log.some((e) => e.kind === 'local.upsert')).toBe(false);
+  });
+
+  it('leaves to the read a view whose membership is not known and current', async () => {
+    const deltas = [{ hash: 'a', id: 't:2', version: 1 }];
+    for (const entry of [
+      live({ lifecycle: { phase: 'cold' } }),
+      live({ lifecycle: { phase: 'cached' } }),
+      live({ lifecycle: { phase: 'view-lost' } }),
+      live({ serverState: 'materializing' }),
+      live({ viewLostRetryAt: 9_000 }),
+      // A bounded view: the row this one pushes out must leave in the same render.
+      live({ def: { hash: 'a', surql: 'SELECT * FROM t ORDER BY at DESC LIMIT 10' } }),
+      live({ def: { hash: 'a', plan: { table: 't', limit: 10 } } }),
+    ]) {
+      const out = await runPure(applyEdgeDeltas(deltas), { state: buildState([entry]) });
+      expect(out.result.size).toBe(0);
+      expect(out.state.queries.get('a')!.remoteArray).toEqual([['t:1', 1]]);
+    }
+    expect((await runPure(applyEdgeDeltas([{ hash: 'zz', id: 't:2', version: 1 }]), { state: buildState([live()]) })).result.size).toBe(0);
+  });
+});
+
+describe('readMembership after a LIVE delta', () => {
+  const qid = (h: string) => new RecordId('_00_query', h);
+  it('a read sent before a row joined does not take it back', async () => {
+    const s = buildState([buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'live' }, remoteArray: [['thing:1', 1]], serverState: 'ready' })]);
+    const out = await runPure(readMembership(env, ['a']), {
+      state: s,
+      handlers: {
+        // The server answers for the moment it ran the read; the delta lands
+        // while the answer travels back.
+        'remote.query': (_e: any, ctx: any) => {
+          ctx.state = R.commitMembership('a', [['thing:1', 1], ['thing:2', 1]], true)(ctx.state);
+          return [ok([{ out: new RecordId('thing', '1'), version: 1 }]), ok({ rowCount: 1, state: 'ready' }), ok([])];
+        },
+        'local.upsert': () => undefined,
+      },
+    });
+    expect(out.state.queries.get('a')!.remoteArray).toEqual([['thing:1', 1], ['thing:2', 1]]);
+  });
+});

package/src/query/membership.saga.ts

@@ -1,4 +1,4 @@
-import type { QueryHash, RecordVersionArray, ServerViewMeta } from '../types';
+import type { EdgeDelta, QueryHash, RecordVersionArray, ServerViewMeta } from '../types';
 import type { Saga } from '../kernel/saga';
 import type { Settled, StatementResult } from '../kernel/effects';
 import { fx } from '../kernel/effects';
@@ -22,6 +22,8 @@ import * as sql from './sql';
 import {
   FAILED,
   decideMembershipOutcome,
+  rebaseRead,
+  withDeltas,
   snapshotFromSingle,
   snapshotsFromBatch,
   suspectHashes,
@@ -148,10 +150,41 @@ const stmt = (results: unknown, i: number): unknown => {
   return r && r.status === 'OK' ? r.result : FAILED;
 };
 
+/**
+ * Commit LIVE edge deltas (rows joining a view, or moving to a newer
+ * version) straight to membership, without re-reading the edges: the
+ * notification already says what the read would. Returns the hashes it
+ * settled; any other change still goes through {@link readMembership}.
+ *
+ * Only for a query whose membership is known and current (`live`, not
+ * materializing, no view-lost recovery pending): before that the read decides
+ * what the set is. Removals are never applied here, so a view whose edges
+ * are being rewritten never loses (and garbage-collects) bodies mid-rewrite.
+ * Nor for a query with a `LIMIT`: a row joining it pushes another out, and
+ * rendering the joiner before the read removes the leaver would show one row
+ * too many (the id-set render drops the `LIMIT`).
+ */
+export function* applyEdgeDeltas(deltas: EdgeDelta[]): Saga<Set<QueryHash>> {
+  const settled = new Set<QueryHash>();
+  const byHash = new Map<QueryHash, EdgeDelta[]>();
+  for (const d of deltas) byHash.set(d.hash, [...(byHash.get(d.hash) ?? []), d]);
+  for (const [hash, list] of byHash) {
+    const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+    if (!entry || entry.lifecycle.phase !== 'live' || entry.serverState === 'materializing' || entry.viewLostRetryAt !== null) continue;
+    if (entry.def.plan ? entry.def.plan.limit != null : /\bLIMIT\b/i.test(entry.def.surql)) continue;
+    const next = withDeltas(entry.remoteArray, list);
+    // Never empty, so always applied.
+    if (next !== entry.remoteArray) yield* applyMembership(hash, next);
+    settled.add(hash);
+  }
+  return settled;
+}
+
 /**
  * Read the server membership of many queries in as few round trips as the
- * row budget allows, then apply each changed set. This is the ONLY place a
- * server id-set enters state: registration, the poll, LIVE dirt and view-lost
+ * row budget allows, then apply each changed set. Every server id-set enters
+ * state here, except the rows LIVE reports joining (see
+ * {@link applyEdgeDeltas}): registration, the poll, LIVE dirt and view-lost
  * recovery all come through here.
  */
 export function* readMembership(
@@ -226,9 +259,10 @@ export function* readMembership(
   for (const [hash, snap] of snapshots) {
     const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
     if (!current) continue;
+    const primary = rebaseRead(snap.primary, byHash.get(hash)!.remoteArray, current.remoteArray);
     const metaChanged = current.serverState !== (snap.meta.present ? snap.meta.state : null);
-    if (!recordVersionArraysEqual(snap.primary, current.remoteArray) || metaChanged || current.lifecycle.phase === 'cold' || current.lifecycle.phase === 'cached') {
-      const outcome = yield* applyMembership(hash, snap.primary, snap.meta);
+    if (!recordVersionArraysEqual(primary, current.remoteArray) || metaChanged || current.lifecycle.phase === 'cold' || current.lifecycle.phase === 'cached') {
+      const outcome = yield* applyMembership(hash, primary, snap.meta);
       if (outcome === 'applied') changed = true;
       if (outcome === 'ignored' && snap.meta.present && snap.meta.state === 'materializing') stillMaterializing.push(hash);
     }

package/src/query/membership.test.ts

@@ -6,9 +6,11 @@ import {
   isResolvedBefore,
   metaFromRow,
   parseViewRow,
+  rebaseRead,
   snapshotFromSingle,
   snapshotsFromBatch,
   suspectHashes,
+  withDeltas,
 } from './membership';
 
 describe('durable view row', () => {
@@ -116,3 +118,34 @@ describe('snapshots', () => {
     expect(suspectHashes(nullCount, new Map(), 0)).toEqual([]);
   });
 });
+
+describe('withDeltas', () => {
+  it('adds a joining row and moves a held one to a newer version', () => {
+    const held: Array<[string, number]> = [['t:1', 1], ['t:2', 1]];
+    expect(withDeltas(held, [{ id: 't:3', version: 1 }, { id: 't:2', version: 4 }])).toEqual([['t:1', 1], ['t:2', 4], ['t:3', 1]]);
+  });
+  it('hands back the held set itself when nothing moves', () => {
+    const held: Array<[string, number]> = [['t:1', 5]];
+    expect(withDeltas(held, [{ id: 't:1', version: 5 }, { id: 't:1', version: 2 }])).toBe(held);
+    expect(withDeltas(held, [])).toBe(held);
+  });
+});
+
+describe('rebaseRead', () => {
+  it('is the read itself when state did not move while it was in flight', () => {
+    const before: Array<[string, number]> = [['t:1', 1]];
+    const read: Array<[string, number]> = [['t:2', 1]];
+    expect(rebaseRead(read, before, before)).toBe(read);
+  });
+  it('keeps a row that joined (or moved) after the read was sent', () => {
+    const before: Array<[string, number]> = [['t:1', 1]];
+    const now: Array<[string, number]> = [['t:1', 3], ['t:2', 1]];
+    // The server answered before either change.
+    expect(rebaseRead([['t:1', 1]], before, now)).toEqual([['t:1', 3], ['t:2', 1]]);
+    // ...or after them: nothing is doubled or moved back.
+    expect(rebaseRead([['t:1', 4], ['t:2', 2]], before, now)).toEqual([['t:1', 4], ['t:2', 2]]);
+  });
+  it('drops a row that left in the meantime', () => {
+    expect(rebaseRead([['t:1', 1], ['t:2', 1]], [['t:1', 1], ['t:2', 1]], [['t:2', 1]])).toEqual([['t:2', 1]]);
+  });
+});

package/src/query/membership.ts

@@ -104,6 +104,39 @@ export function decideMembershipOutcome(input: MembershipDecisionInput): Members
   return knownEmpty ? 'applied' : 'ignored';
 }
 
+/**
+ * Commit LIVE edge deltas onto a held id-set: each names a row that joined
+ * (or moved to a newer version). Returns `held` itself when nothing moves.
+ */
+export function withDeltas(held: RecordVersionArray, deltas: ReadonlyArray<{ id: string; version: number }>): RecordVersionArray {
+  const next = new Map(held);
+  let moved = false;
+  for (const { id, version } of deltas) {
+    if ((next.get(id) ?? -1) >= version) continue;
+    next.set(id, version);
+    moved = true;
+  }
+  return moved ? Array.from(next.entries()) : held;
+}
+
+/**
+ * A membership read answers for the moment the server ran it, which may be
+ * before a LIVE delta that was committed while the read was in flight. Carry
+ * what changed in state since the read was sent (`before` -> `now`) over onto
+ * the read, so a late answer never takes back a row that just joined.
+ */
+export function rebaseRead(read: RecordVersionArray, before: RecordVersionArray, now: RecordVersionArray): RecordVersionArray {
+  if (before === now) return read;
+  const was = new Map(before);
+  const is = new Map(now);
+  const out = new Map(read);
+  for (const [id] of before) if (!is.has(id)) out.delete(id);
+  for (const [id, version] of now) {
+    if (version > (was.get(id) ?? -1) && version > (out.get(id) ?? -1)) out.set(id, version);
+  }
+  return Array.from(out.entries());
+}
+
 // ---- list_ref snapshots ------------------------------------------------------
 
 /**