@spooky-sync/core 0.0.1-canary.322 → 0.0.1-canary.324

package/dist/types.d.ts

@@ -451,6 +451,16 @@ interface LocalStore extends LocalCacheEngine {
    * without it boots the circuit empty, as before.
    */
   scanVersions?(tables: string[]): Promise<Record<string, [string, number][]>>;
+  /**
+   * Reset the cached `_00_rv` of every table whose column set (`fingerprints`,
+   * see `columnFingerprints`) changed since the store last recorded it, except
+   * the rows in `keep`, and record the new sets. Returns the tables reset.
+   * For engines that keep bodies across schema changes: a body landed under an
+   * older schema lacks the columns it did not know, at the server's version,
+   * and would never be fetched again. OPTIONAL: an engine that rebuilds its
+   * store on a schema change (SurrealDB) does not need it.
+   */
+  resetChangedTables?(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]>;
   /**
    * Circuit snapshot storage, keyed. Lives in the same durable store as the
    * rows (OPFS SQLite), so it is per-bucket by construction, atomic, and
@@ -1570,11 +1580,11 @@ interface Sp00kyConfig<S extends SchemaStructure> {
   streamDebounceTime?: number;
   /**
    * Debounce time in milliseconds for syncing collaborative (CRDT) field
-   * changes to the remote database. Local writes happen immediately on
-   * every keystroke (so reload/offline works), but the remote UPSERT is
-   * coalesced over this window. Lower = snappier remote propagation +
-   * more network traffic; higher = less traffic + more lag for other
-   * collaborators. Defaults to 500ms.
+   * edits to the server. Local state is kept on every change (so reload and
+   * offline work), but a field's own new ops go out as one edit-log row per
+   * pause of this length (at most 2 s apart while typing continues).
+   * Lower = snappier propagation and more rows; higher = fewer rows and more
+   * lag for other collaborators. Defaults to 300ms.
    */
   crdtDebounceMs?: number;
   /**
@@ -1632,7 +1642,8 @@ interface Sp00kyConfig<S extends SchemaStructure> {
    * page gets live `useQuery` updates over world-readable tables. Requires the
    * server to be deployed with `anonymousLiveQueries: true` in `sp00ky.yml`
    * (this flag must match it). Defaults to `false`: anonymous clients can read
-   * one-shot but never sync live.
+   * one-shot but never sync live, and a `preload` made while signed out returns
+   * at once rather than wait for membership that never comes.
    */
   enableAnonymousLiveQueries?: boolean;
   /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.322",
+  "version": "0.0.1-canary.324",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -75,13 +75,13 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.322",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.322",
+    "@spooky-sync/query-builder": "0.0.1-canary.324",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.324",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "blurhash": "^2.0.5",
     "fast-json-patch": "^3.1.1",
-    "loro-crdt": "^1.5.6",
+    "loro-crdt": "1.11.0",
     "pino": "^10.1.0",
     "surrealdb": "2.1.0"
   },

package/src/boot/preload.saga.test.ts

@@ -21,6 +21,7 @@ describe('preload', () => {
   });
   it('never resolved: waits for settled (membership + bodies + first render), or fails when the registration failed', async () => {
     const hash = await sha256Hex(queryHashInput(input, null));
+    const signedIn = buildState([], R.setIdentity({ userId: 'user:alice' }));
     const settleIt = (ctx: any) => {
       ctx.state = R.compose(
         R.commitMembership(hash, [['thing:1', 1]], true),
@@ -29,10 +30,12 @@ describe('preload', () => {
       )(ctx.state);
     };
     const out = await runPure(preload(env, input), {
+      state: signedIn,
       handlers: { 'local.getById': () => null, 'ssp.register': sspOk, 'state.wait': (_e, ctx) => settleIt(ctx) },
     });
     expect(out.result).toEqual({ hash, waited: true });
     const failed = runPure(preload(env, input), {
+      state: signedIn,
       handlers: {
         'local.getById': () => null,
         'ssp.register': sspOk,
@@ -41,6 +44,7 @@ describe('preload', () => {
     });
     await expect(failed).rejects.toBeInstanceOf(PreloadFailedError);
     const gone = runPure(preload(env, input), {
+      state: signedIn,
       handlers: { 'local.getById': () => null, 'ssp.register': sspOk, 'state.wait': (_e, ctx) => void (ctx.state = R.removeQuery(hash)(ctx.state)) },
     });
     await expect(gone).rejects.toThrow(/could not settle/);
@@ -48,4 +52,29 @@ describe('preload', () => {
     const cached = await runPure(preload(env, input), { state: already });
     expect(cached.result.waited).toBe(false);
   });
+  it('signed out without anonymous live queries: registers but never waits, since no membership can reach this session', async () => {
+    const out = await runPure(preload(env, input), {
+      handlers: { 'local.getById': () => null, 'ssp.register': sspOk },
+    });
+    expect(out.result.waited).toBe(false);
+    expect(out.log.filter((e) => e.kind === 'state.wait')).toHaveLength(0);
+    expect(out.log.some((e) => e.kind === 'dispatch' && (e as any).event?.type === 'EnsureRegistered')).toBe(true);
+
+    // With anonymous live queries a signed-out session is sent its membership
+    // on `_00_list_ref_anon`, so it waits like a signed-in one.
+    const hash = await sha256Hex(queryHashInput(input, null));
+    const anon = await runPure(preload(defaultEnv(env.schema, { anonLive: true }), input), {
+      handlers: {
+        'local.getById': () => null,
+        'ssp.register': sspOk,
+        'state.wait': (_e, ctx) =>
+          void (ctx.state = R.compose(
+            R.commitMembership(hash, [['thing:1', 1]], true),
+            R.setVersions([['thing:1', 1]]),
+            R.setRecords(hash, [{ id: 'thing:1' }], true, 1)
+          )(ctx.state)),
+      },
+    });
+    expect(anon.result).toEqual({ hash, waited: true });
+  });
 });

package/src/boot/preload.saga.ts

@@ -3,6 +3,7 @@ import type { Saga } from '../kernel/saga';
 import { fx } from '../kernel/effects';
 import { settleFailed, settled } from '../state/selectors';
 import type { SagaEnv } from '../query/env';
+import { receivesMembership } from '../query/env';
 import type { RegisterInput } from '../query/register.saga';
 import { registerLocal } from '../query/register.saga';
 
@@ -19,11 +20,19 @@ export class PreloadFailedError extends Error {
  * Never resolved: blocks until the server's membership and every body are
  * local and the first materialization ran. The entry is evicted like any
  * other query a ttl after it was last watched (or registered).
+ *
+ * A session that cannot be sent membership (signed out, without anonymous
+ * live queries) does not wait either: a view with rows would never settle,
+ * and an app that gates its first paint on a preload showed its loading
+ * screen forever, signed-out landing page included.
  */
 export function* preload(env: SagaEnv, input: RegisterInput): Saga<{ hash: QueryHash; waited: boolean }> {
   const hash = yield* registerLocal(env, input);
-  const phase = (yield fx.state.read((s) => s.queries.get(hash)?.lifecycle.phase)) as string | undefined;
-  if (phase !== 'cold') return { hash, waited: false };
+  const [phase, reachable] = (yield fx.state.read((s) => [s.queries.get(hash)?.lifecycle.phase, receivesMembership(env, s)])) as [
+    string | undefined,
+    boolean,
+  ];
+  if (phase !== 'cold' || !reachable) return { hash, waited: false };
   yield fx.state.wait((s) => settled(s, hash) || settleFailed(s, hash));
   const failed = (yield fx.state.read((s) => settleFailed(s, hash))) as boolean;
   if (failed) throw new PreloadFailedError(hash);

package/src/client/services.ts

@@ -21,6 +21,7 @@ import { TabsCoordinator, type CoordinatorHooks } from '../services/tabs/coordin
 import type { LeaderSyncHub, SyncForwarder } from '../services/tabs/coordinator';
 import { computeTabsFingerprint, hash53 } from '../services/tabs/protocol';
 import type { SqliteCacheEngine } from '../services/database/sqlite-cache-engine';
+import { columnFingerprints } from '../services/database/sqlite-column-reset';
 import type { BlobCache } from '../services/blobs/index';
 import { MemoryBlobStore, createBlobCache, resolveBlobBudget } from '../services/blobs/index';
 import type { Adapters } from '../kernel/interpreter';
@@ -175,7 +176,7 @@ export function createServices<S extends SchemaStructure>(config: Sp00kyConfig<S
   streamProcessor.configureCircuitPersistence(config.persistCircuit ?? config.localEngine === 'sqlite', config.circuitCheckpointMs);
   streamProcessor.configureProjection(config.circuitProjection ?? true);
   const migrator = new LocalMigrator(local, logger);
-  const crdt = new CrdtManager(config.schema, local, remote, logger, config.crdtDebounceMs ?? 500);
+  const crdt = new CrdtManager(config.schema, local, remote, logger, config.crdtDebounceMs ?? 300);
   const auth = new AuthService(config.schema, remote, persistence, logger);
   const tabId = typeof crypto !== 'undefined' && crypto.randomUUID ? crypto.randomUUID() : `tab_${Math.random().toString(36).slice(2)}`;
   return {
@@ -308,6 +309,7 @@ export function createAdapters<S extends SchemaStructure>(config: Sp00kyConfig<S
         tables: [...config.schema.tables.map((t) => t.name), '_00_user_feature', '_00_app_release'],
         schemaHash: String(hash53(config.schemaSurql)),
         pendingIds: new Set(pendingIds),
+        columnFingerprints: columnFingerprints(config.schema.tables),
         onVersions: (_table, entries) => host.dispatch({ type: 'VersionsPrimed', entries }),
       }),
     'ssp.reset': () => s.streamProcessor.reset(),

package/src/client/sp00ky-client.test.ts

@@ -18,6 +18,7 @@ const flush = async (n = 6) => {
 function makeClient(over: Parameters<typeof fakeAdapters>[0] = {}) {
   const services = fakeServiceBundle<any>();
   const a = fakeAdapters({
+    ...over,
     services: {
       'hint.read': () => 'u1',
       'local.usesSurqlSchema': () => false,
@@ -26,7 +27,6 @@ function makeClient(over: Parameters<typeof fakeAdapters>[0] = {}) {
       'remote.connect': () => new Promise(() => {}),
       ...over.services,
     },
-    ...over,
   });
   const runtime = new Runtime({ env: defaultEnv(schema, { materializeDebounceMs: 1 }), adapters: a.adapters, logger: services.logger, tabId: 'tab-test' });
   const client = new Sp00kyClient<any>(config, { services, runtime });
@@ -155,7 +155,10 @@ describe('Sp00kyClient facade', () => {
 
   it('preload: cold waits for membership + bodies; cached returns at once; abort rejects', async () => {
     const edges = [{ out: new RecordId('thing', '1'), version: 1 }];
+    // Signed in: a signed-out session without anonymous live queries is never
+    // sent membership, so its preload does not wait (see `preload`).
     const { client, a } = makeClient({
+      services: { 'auth.restoreSession': async () => 'user:u1', 'auth.sessionAuthId': () => 'user:u1' },
       remote: {
         queryResponses: async (sql: string) => {
           if (sql.startsWith('fn::query::register')) return [ok(null), ok(edges), ok({ rowCount: 1, state: 'ready' }), ok([])];

package/src/client/sp00ky-client.ts

@@ -142,6 +142,29 @@ export class Sp00kyClient<S extends SchemaStructure> {
         this.runtime.subscribe(hash, cb, options),
     };
     const remoteRegisterPort = { enqueueDownEvent: () => void this.runtime.dispatch({ type: 'EnsureRegistered' }) };
+    // CRDT edit logs ride ordinary queries and the ordinary outbox.
+    s.crdt.attach({
+      watch: async (table, where, onRows) => {
+        const { hash } = await (this.query as any)(table, {}).where(where).build().run();
+        const stopRows = this.runtime.subscribe(hash, onRows, { immediate: true });
+        let known = false;
+        let idle = false;
+        let done!: () => void;
+        const settled = new Promise<void>((resolve) => (done = resolve));
+        const check = () => known && idle && done();
+        const stopAuthority = this.runtime.subscribeAuthority(hash, (k) => ((known = k), check()), { immediate: true });
+        const stopStatus = this.runtime.subscribeStatus(hash, (st) => ((idle = st === 'idle'), check()), { immediate: true });
+        return {
+          stop: () => {
+            stopRows();
+            stopAuthority();
+            stopStatus();
+          },
+          settled,
+        };
+      },
+      create: (id, data) => this.create(id, data),
+    });
     this.featureFlags = new FeatureFlagModule<S>({ dataModule: liveQueryPort, sync: remoteRegisterPort, auth: s.auth, logger: s.logger });
     this.appReleases = new AppReleaseModule<S>({ dataModule: liveQueryPort, sync: remoteRegisterPort, auth: s.auth, logger: s.logger });
     this.webPush = new WebPushModule({
@@ -567,12 +590,22 @@ export class Sp00kyClient<S extends SchemaStructure> {
     return this.appReleases.release(app, options);
   }
 
-  async openCrdtField(table: string, recordId: string, field: string, fallbackText?: string): Promise<CrdtField> {
-    return this.services.crdt.open(table, recordId, field, fallbackText);
+  /** Open a `@crdt` field for editing. `draft` (from {@link createCrdtDraft})
+   *  hands over edits made before the record existed. */
+  async openCrdtField(table: string, recordId: string, field: string, draft?: CrdtField): Promise<CrdtField> {
+    return this.services.crdt.open(table, recordId, field, draft);
+  }
+
+  /** A detached `@crdt` document for a record not created yet; see
+   *  {@link openCrdtField}. */
+  async createCrdtDraft(table: string, field: string): Promise<CrdtField> {
+    return this.services.crdt.createDraft(table, field);
   }
 
-  closeCrdtField(table: string, recordId: string, field: string): void {
-    this.services.crdt.close(table, recordId, field);
+  /** Close a `@crdt` field. The last close resolves once its unsent edits
+   *  are queued in the outbox. */
+  closeCrdtField(table: string, recordId: string, field: string): Promise<void> {
+    return this.services.crdt.close(table, recordId, field);
   }
 
   bucket<B extends BucketNames<S>>(name: B): BucketHandle {

package/src/modules/crdt/acceptance.ts

@@ -0,0 +1,75 @@
+import type * as Loro from 'loro-crdt';
+
+/**
+ * Which edit-log rows may enter a document.
+ *
+ * Every client and the SSP fold (`packages/ssp-node/src/crdt/acceptance.rs`)
+ * apply this exact rule before importing a row, so they import the same rows
+ * and converge on the same document: a row one side accepted and another
+ * refused would leave them different forever (a CRDT never forgets an op).
+ *
+ * - the blob decodes as a Loro **update** (a snapshot could smuggle in
+ *   arbitrary history from any peer);
+ * - it is no larger than the column allows;
+ * - every op in it was made by the row's `peer`, which the database already
+ *   checked belongs to the row's author (`_00_crdt_peer`).
+ *
+ * Loro's decoder can trap on crafted input with a valid checksum, so a raw
+ * header check rejects non-update blobs before decoding, and decoding runs
+ * under `try`.
+ */
+
+export type Rejection =
+  | 'empty'
+  | 'too_large'
+  | 'bad_peer'
+  | 'header'
+  | 'undecodable'
+  | 'not_an_update'
+  | 'no_ops'
+  | 'foreign_peer';
+
+export interface Accepted {
+  peer: `${number}`;
+  /** Exclusive end counter of the row's ops for `peer`. */
+  end: number;
+}
+
+export type Verdict = { ok: true; accepted: Accepted } | { ok: false; reason: Rejection };
+
+/** The largest blob a row may carry (the generated column's cap). */
+export const MAX_DELTA_BYTES = 256 * 1024;
+
+const PEER = /^\d{1,20}$/;
+/** `"loro"`: every Loro blob starts with it. */
+const MAGIC = [0x6c, 0x6f, 0x72, 0x6f];
+/** Header bytes 20..22 (big-endian) carry the encoding mode; 4 is an update. */
+const MODE_UPDATES = 4;
+
+export function check(
+  loro: Pick<typeof Loro, 'decodeImportBlobMeta'>,
+  blob: Uint8Array | undefined,
+  peer: unknown,
+  maxBytes = MAX_DELTA_BYTES
+): Verdict {
+  if (!blob || blob.length === 0) return { ok: false, reason: 'empty' };
+  if (blob.length > maxBytes) return { ok: false, reason: 'too_large' };
+  if (typeof peer !== 'string' || !PEER.test(peer)) return { ok: false, reason: 'bad_peer' };
+  if (blob.length < 22 || MAGIC.some((b, i) => blob[i] !== b)) return { ok: false, reason: 'header' };
+  if (((blob[20] << 8) | blob[21]) !== MODE_UPDATES) return { ok: false, reason: 'not_an_update' };
+  let meta: ReturnType<typeof Loro.decodeImportBlobMeta>;
+  try {
+    meta = loro.decodeImportBlobMeta(blob, true);
+  } catch {
+    return { ok: false, reason: 'undecodable' };
+  }
+  if (meta.mode !== 'update') return { ok: false, reason: 'not_an_update' };
+  const start = meta.partialStartVersionVector.toJSON();
+  const authors: Array<[string, number]> = [];
+  for (const [p, end] of meta.partialEndVersionVector.toJSON()) {
+    if (end > (start.get(p) ?? 0)) authors.push([String(p), end]);
+  }
+  if (authors.length === 0) return { ok: false, reason: 'no_ops' };
+  if (authors.length > 1 || BigInt(authors[0][0]) !== BigInt(peer)) return { ok: false, reason: 'foreign_peer' };
+  return { ok: true, accepted: { peer: peer as `${number}`, end: authors[0][1] } };
+}

package/src/modules/crdt/bytes.ts

@@ -0,0 +1,50 @@
+/**
+ * Byte columns reach the client in several shapes depending on the engine and
+ * the path: `Uint8Array` (SQLite, a fresh local write), `ArrayBuffer` (the
+ * surrealdb SDK's CBOR decode), a typed-array view, or a `number[]` (bytes
+ * nested in a FLEXIBLE object on the SurrealDB-WASM engine). Normalize all of
+ * them; anything else is not bytes.
+ */
+export function asBytes(value: unknown): Uint8Array | undefined {
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  if (ArrayBuffer.isView(value)) {
+    return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+  }
+  if (Array.isArray(value) && value.length > 0 && value.every((n) => typeof n === 'number')) {
+    return Uint8Array.from(value as number[]);
+  }
+  return undefined;
+}
+
+export function decodeBase64(b64: string): Uint8Array {
+  if (typeof atob === 'function') {
+    const binary = atob(b64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+    return bytes;
+  }
+  return new Uint8Array(Buffer.from(b64, 'base64'));
+}
+
+export function encodeBase64(bytes: Uint8Array): string {
+  if (typeof btoa === 'function') {
+    let binary = '';
+    for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+    return btoa(binary);
+  }
+  return Buffer.from(bytes).toString('base64');
+}
+
+/**
+ * A random Loro peer id below 2^53 (so it is exact as a JS number), as the
+ * decimal string Loro uses for `PeerID`. Each tab picks a new one per session
+ * and claims it in `_00_crdt_peer` before any of its edits are sent.
+ */
+export function randomPeer(): `${number}` {
+  const words = new Uint32Array(2);
+  crypto.getRandomValues(words);
+  const high = words[0] & 0x1fffff; // 21 bits
+  const id = high * 0x100000000 + words[1];
+  return String(id === 0 ? 1 : id) as `${number}`;
+}