@spooky-sync/core 0.0.1-canary.38 → 0.0.1-canary.381

package/src/query/register.saga.ts

@@ -0,0 +1,293 @@
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import { RecordId } from 'surrealdb';
+import type { QueryHash, QueryTimeToLive } from '../types';
+import type { Saga } from '../kernel/saga';
+import type { RegisterResult, StatementResult } from '../kernel/effects';
+import { fx } from '../kernel/effects';
+import { AUTH_READY_MAX_ATTEMPTS, AUTH_READY_RETRY_MS, REGISTER_MAX_RETRIES, backoffMs } from '../kernel/constants';
+import type { ClientState, QueryEntry } from '../state/client-state';
+import { emptyTelemetry } from '../state/client-state';
+import { seedLifecycle } from '../state/lifecycle';
+import * as R from '../state/reducers';
+import { desiredRegistrations } from '../state/selectors';
+import { parseDuration } from '../utils/index';
+import type { SagaEnv } from './env';
+import { listRefTable } from './env';
+import { queryHashInput, viewKeyInput } from './hash';
+import { FAILED, isResolvedBefore, parseViewRow, snapshotFromSingle } from './membership';
+import { applyMembership, applySubqueryChildren, markMembershipDirty } from './membership.saga';
+import * as sql from './sql';
+
+export interface RegisterInput {
+  tableName: string;
+  surql: string;
+  params: Record<string, unknown>;
+  ttl: QueryTimeToLive;
+  plan?: QueryPlan;
+}
+
+/**
+ * Register a query locally: compute its keys, read the durable `_00_view`
+ * row, build the SSP local view and publish the entry. Returns as soon as the
+ * entry exists (the first paint comes from `materialize`); the remote
+ * registration is dispatched, never awaited.
+ */
+export function* registerLocal(env: SagaEnv, input: RegisterInput): Saga<QueryHash> {
+  const sessionId = (yield fx.state.read((s) => s.sessionId)) as string | null;
+  const hash = (yield fx.hash(queryHashInput(input, sessionId))) as string;
+  const status = (yield fx.state.read((s) => (s.queries.has(hash) ? 'active' : s.registering.has(hash) ? 'pending' : 'new'))) as
+    | 'active'
+    | 'pending'
+    | 'new';
+  if (status === 'active') return hash;
+  if (status === 'pending') {
+    yield fx.state.wait((s) => !s.registering.has(hash));
+    return hash;
+  }
+  yield fx.state.update(R.beginRegistering(hash));
+  try {
+    const viewKey = (yield fx.hash(viewKeyInput(input))) as string;
+    let viewRow: unknown = null;
+    try {
+      viewRow = yield fx.local.getById(sql.VIEW_TABLE, sql.viewRecordId(viewKey));
+    } catch {
+      viewRow = null;
+    }
+    const view = parseViewRow(viewRow);
+    const now = (yield fx.now()) as number;
+    const reg = (yield fx.ssp.register({
+      queryHash: hash,
+      surql: input.surql,
+      params: input.params,
+      ttl: input.ttl,
+      tableName: input.tableName,
+    })) as RegisterResult;
+    const entry: QueryEntry = {
+      def: {
+        id: new RecordId('_00_query', hash),
+        hash,
+        viewKey,
+        surql: input.surql,
+        params: input.params,
+        plan: input.plan,
+        ttl: input.ttl,
+        ttlMs: parseDuration(input.ttl),
+        tableName: input.tableName,
+        createdAt: now,
+      },
+      lifecycle: seedLifecycle(isResolvedBefore(view)),
+      remoteArray: view?.ids ?? [],
+      localArray: reg.localArray,
+      subqueryRemoteArray: [],
+      records: [],
+      serverState: null,
+      subscribers: 0,
+      lastSubscriberLeftAt: now,
+      lastHeartbeatAt: null,
+      lastPolledAt: null,
+      registerAttempts: 0,
+      viewLostCount: 0,
+      viewLostRetryAt: null,
+      telemetry: { ...emptyTelemetry(), registrationTimings: reg.timings },
+    };
+    yield fx.state.update(R.putQuery(entry));
+    yield fx.dispatch({ type: 'EnsureRegistered' });
+    return hash;
+  } finally {
+    yield fx.state.update(R.endRegistering(hash));
+  }
+}
+
+/**
+ * Make the remote match the desired set: start a registration for every
+ * query that has none. Idempotent. With `requireAuth` (after a reconnect or
+ * while degraded) it first waits for `$auth.id` to be visible on the socket:
+ * a register issued before that stamps the view with an empty identity.
+ */
+export function* ensureRegistered(env: SagaEnv, opts: { requireAuth?: boolean; attempt?: number } = {}): Saga<void> {
+  const [hashes, userId] = (yield fx.state.read((s) => [desiredRegistrations(s), s.userId])) as [QueryHash[], string | null];
+  if (hashes.length === 0) return;
+  if (opts.requireAuth && userId !== null) {
+    let authed = false;
+    try {
+      const res = (yield fx.remote.query('RETURN $auth.id', undefined, env.remoteTimeoutMs)) as StatementResult[];
+      authed = res[0]?.status === 'OK' && res[0].result != null;
+    } catch {
+      authed = false;
+    }
+    if (!authed) {
+      const attempt = (opts.attempt ?? 0) + 1;
+      if (attempt >= AUTH_READY_MAX_ATTEMPTS) {
+        yield fx.emit({ type: 'log', level: 'warn', message: 'auth identity never became visible; not registering', data: { attempt } });
+        return;
+      }
+      yield fx.timer.set('ensure-registered', AUTH_READY_RETRY_MS, { type: 'EnsureRegistered', requireAuth: true, attempt });
+      return;
+    }
+  }
+  for (const hash of hashes) yield fx.dispatch({ type: 'RegisterRemote', hash });
+}
+
+const stmt = (results: StatementResult[], i: number): StatementResult | undefined => results[i];
+
+/**
+ * The SSP answers a registration whose shape is not on the query allowlist
+ * with HTTP 403 `{"error":"not_allowlisted"}`. It reaches the client as the
+ * `fn::query::register` statement error carrying SurrealDB's http error text
+ * (status reason and/or body), and no retry can change the answer.
+ */
+export const isNotAllowlisted = (error: unknown): boolean =>
+  /not_allowlisted|\b403\b|forbidden/i.test(String((error as { message?: unknown })?.message ?? error));
+
+/**
+ * The entry no longer names the remote id this registration sent: a salt
+ * rotation (`rebindQueries`) re-keyed it while the request was out, and
+ * seeded a fresh lifecycle for the new id.
+ */
+const movedOff = (current: QueryEntry, sent: QueryEntry): boolean => String(current.def.id) !== String(sent.def.id);
+
+/**
+ * Release the id a superseded registration sent and register the new one.
+ * The re-key's own `RegisterRemote` for this hash joined the run still in
+ * flight (the lane dedupes per hash), so it is armed again on a timer, which
+ * fires once this run has left the lane.
+ */
+function* handOff(hash: QueryHash, sent: QueryEntry): Saga<void> {
+  yield fx.dispatch({ type: 'ReleaseSuperseded', ids: [sent.def.id] });
+  yield fx.timer.set(`register:${hash}`, 0, { type: 'RegisterRemote', hash });
+}
+
+/**
+ * One remote registration: `fn::query::register` plus the edge/meta/children
+ * read in ONE request, then membership application. Concurrent with every
+ * other registration; retried on its own backoff; stops when the entry is
+ * gone.
+ *
+ * An answer that lands after the query moved to a new id applies nothing:
+ * the membership and the `registered` flag belong to the id it sent, which
+ * this page no longer uses, so it releases that id and leaves the new one to
+ * the registration the re-key dispatched. Marking the new id registered from
+ * the old id's answer meant the new id was never registered, and nothing
+ * ever named the old one again.
+ */
+export function* registerRemote(env: SagaEnv, hash: QueryHash, retry = false): Saga<void> {
+  const state = (yield fx.state.read((s) => s)) as ClientState;
+  const entry = state.queries.get(hash);
+  if (!entry) return;
+  const { remote } = entry.lifecycle;
+  if (remote === 'registered' || remote === 'failed' || (remote === 'registering' && !retry)) return;
+  // A query that already shows rows (cached or live) only re-verifies them
+  // here: the cycle holds `fetching` but is not traffic the user waits on.
+  const background = entry.lifecycle.phase !== 'cold';
+  let superseded = false;
+  yield fx.state.update(
+    R.compose(R.applyLifecycle(hash, { type: 'remote-registering' }), R.applyLifecycle(hash, { type: 'fetch-begin', background }))
+  );
+  try {
+    const table = listRefTable(env, state);
+    const results = (yield fx.remote.query(
+      sql.registerSelect(table),
+      sql.registerVars({
+        id: entry.def.id,
+        surql: entry.def.surql,
+        params: { ...entry.def.params },
+        ttl: entry.def.ttl,
+        ...(state.ownerSecret ? { owner: state.ownerSecret } : {}),
+      }),
+      env.remoteTimeoutMs
+    )) as StatementResult[];
+    const answered = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+    if (!answered) return;
+    superseded = movedOff(answered, entry);
+    if (superseded) {
+      yield* handOff(hash, entry);
+      return;
+    }
+    const register = stmt(results, 0);
+    if (!register || register.status === 'ERR') throw new Error(register?.status === 'ERR' ? register.error : 'register returned nothing');
+    const answer = (r: StatementResult | undefined): unknown => (r?.status === 'OK' ? r.result : FAILED);
+    const snap = snapshotFromSingle(
+      answer(stmt(results, 1)) as never,
+      answer(stmt(results, 2)) as never,
+      answer(stmt(results, 3)) as never
+    );
+    if (!snap) {
+      // Registered, but the read-back did not answer. That says nothing about
+      // the row, so read membership again rather than guess: a guess of
+      // "gone" re-registered the query, and under load the next read-back
+      // failed the same way.
+      yield fx.state.update(R.compose(R.applyLifecycle(hash, { type: 'remote-registered' }), R.resetRegisterAttempts(hash)));
+      yield fx.emit({ type: 'log', level: 'debug', message: 'registration read-back did not answer; re-reading membership', data: { hash } });
+      yield* markMembershipDirty([hash]);
+      yield fx.dispatch({ type: 'SyncOutcome', ok: false, error: 'registration read-back failed' });
+      return;
+    }
+    const outcome = yield* applyMembership(hash, snap.primary, snap.meta);
+    if (outcome === 'ignored' && snap.meta.present && snap.meta.state === 'materializing') {
+      yield fx.state.update(R.compose(R.markMembershipDirty([hash]), R.setMembershipReread(hash, 1)));
+      yield fx.timer.set('membership', 150, { type: 'ReadDirtyMembership' });
+    }
+    yield* applySubqueryChildren(hash, snap.subquery);
+    yield fx.state.update(R.compose(R.applyLifecycle(hash, { type: 'remote-registered' }), R.resetRegisterAttempts(hash)));
+    yield fx.dispatch({ type: 'SyncOutcome', ok: true });
+  } catch (error) {
+    const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+    if (!current) return;
+    // A request that timed out may still land: release its id all the same.
+    superseded = movedOff(current, entry);
+    if (superseded) {
+      yield* handOff(hash, entry);
+      return;
+    }
+    if (isNotAllowlisted(error)) {
+      yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-failed' }));
+      yield fx.emit({
+        type: 'log',
+        level: 'error',
+        message: 'query refused: not allowlisted',
+        data: {
+          hash,
+          table: entry.def.tableName,
+          surql: entry.def.surql,
+          params: Object.keys(entry.def.params),
+          hint: 'Add or regenerate the query allowlist: spky generate, then spky deploy (or restart spky dev).',
+        },
+      });
+      yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
+      return;
+    }
+    const attempts = current.registerAttempts + 1;
+    yield fx.state.update(R.bumpRegisterAttempts(hash));
+    yield fx.emit({ type: 'log', level: 'warn', message: 'remote registration failed', data: { hash, attempts, error } });
+    yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
+    if (attempts >= REGISTER_MAX_RETRIES) {
+      yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-failed' }));
+    } else {
+      yield fx.timer.set(`register:${hash}`, backoffMs(attempts), { type: 'RegisterRemote', hash, retry: true });
+    }
+  } finally {
+    // A re-key seeded a fresh lifecycle: this cycle's begin is not in it.
+    const stillThere = (yield fx.state.read((s) => s.queries.has(hash))) as boolean;
+    if (stillThere && !superseded) yield fx.state.update(R.applyLifecycle(hash, { type: 'fetch-end', background }));
+  }
+}
+
+/**
+ * Hand back remote ids this page registered and no longer uses: a query
+ * moved to a new salt (sign-up, sign-in, sign-out, impersonation) while its
+ * old id was registered or on its way. Nothing names the old id again, not
+ * the heartbeat, not the pagehide beacon, so it used to hold a live view
+ * until its TTL ran out, and the SSP fanned every change to a row it held out
+ * to it. Best-effort: a row the caller may no longer touch (another
+ * principal's) or a failed request is left to the TTL sweep, as before.
+ */
+export function* releaseSuperseded(env: SagaEnv, ids: ReadonlyArray<QueryEntry['def']['id']>): Saga<void> {
+  if (ids.length === 0) return;
+  const sessionIds = (yield fx.service('remote.sessionIds')) as readonly string[];
+  const batch = sql.releaseBatch(ids, sessionIds);
+  try {
+    yield fx.remote.query(batch.sql, batch.vars, env.remoteTimeoutMs);
+  } catch (error) {
+    yield fx.emit({ type: 'log', level: 'debug', message: 'superseded view release failed; the TTL sweep reclaims it', data: { ids: ids.map(String), error } });
+  }
+}

package/src/query/render-set.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { buildRenderIds, paintsCold, resolveMembership } from './render-set';
+
+const ov = (writes: string[] = [], deletes: string[] = []) => ({ writes: new Set(writes), deletes: new Set(deletes) });
+
+describe('resolveMembership', () => {
+  it('cold scans, cold window uses the local view, anything else uses membership', () => {
+    const remoteArray: [string, number][] = [['t:1', 1]];
+    const localArray: [string, number][] = [['t:2', 1]];
+    expect(resolveMembership({ phase: 'cold', remoteArray, localArray, isWindow: false })).toBeNull();
+    expect(resolveMembership({ phase: 'cold', remoteArray, localArray, isWindow: true })).toBe(localArray);
+    expect(resolveMembership({ phase: 'cached', remoteArray, localArray, isWindow: false })).toBe(remoteArray);
+    expect(resolveMembership({ phase: 'view-lost', remoteArray, localArray, isWindow: true })).toBe(remoteArray);
+  });
+});
+
+describe('buildRenderIds', () => {
+  const opts = { hasExplicitOrder: false, isWindow: false };
+  it('membership minus deletes, sorted', () => {
+    expect(buildRenderIds([['t:b', 1], ['t:a', 1], ['t:c', 1]], [], ov([], ['t:c']), opts)).toEqual(['t:a', 't:b']);
+  });
+  it('adds pending writes the local view admits, never twice, never deleted', () => {
+    const ids = buildRenderIds(
+      [['t:1', 1]],
+      [['t:1', 1], ['t:2', 1], ['t:3', 1], ['t:4', 1]],
+      ov(['t:1', 't:2', 't:3', 't:9'], ['t:3']),
+      opts
+    );
+    expect(ids).toEqual(['t:1', 't:2']);
+  });
+  it('keeps server order for windows and explicit orderBy; dedupes membership', () => {
+    const m: [string, number][] = [['t:b', 1], ['t:a', 1], ['t:b', 1]];
+    expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: true, isWindow: false })).toEqual(['t:b', 't:a']);
+    expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: false, isWindow: true })).toEqual(['t:b', 't:a']);
+  });
+});
+
+describe('paintsCold', () => {
+  it('only vouched bodies, optimistic writes and bookkeeping paint before the server answers', () => {
+    const vouched = new Set(['t:1']);
+    expect(paintsCold('t:1', vouched, ov())).toBe(true);
+    expect(paintsCold('t:2', vouched, ov(['t:2']))).toBe(true);
+    expect(paintsCold('_00_user_feature:f', vouched, ov())).toBe(true);
+    expect(paintsCold('t:3', vouched, ov([], ['t:3']))).toBe(false);
+    expect(paintsCold('t:3', vouched, ov())).toBe(false);
+  });
+});

package/src/query/render-set.ts

@@ -0,0 +1,71 @@
+import type { RecordVersionArray } from '../types';
+import type { QueryPhase } from '../state/lifecycle';
+import type { Overlay } from '../state/selectors';
+
+export interface RenderInput {
+  phase: QueryPhase;
+  remoteArray: RecordVersionArray;
+  localArray: RecordVersionArray;
+  isWindow: boolean;
+}
+
+/**
+ * The id-set a query renders from, or `null` when it must fall back to a
+ * local predicate scan (a cold query that was never resolved on this device).
+ * A cold window has no usable scan (re-applying `START m` against the shared
+ * store returns the wrong rows), so it renders the SSP's local window instead.
+ */
+export function resolveMembership(input: RenderInput): RecordVersionArray | null {
+  if (input.phase !== 'cold') return input.remoteArray;
+  if (input.isWindow) return input.localArray;
+  return null;
+}
+
+/**
+ * Whether a body may paint before the server has answered (`cold`). One no
+ * durable view vouches for and no optimistic write touches is an orphan:
+ * deleted upstream while no view watched it, or dropped from every view since.
+ * The predicate scan would otherwise resurrect it until the server answers,
+ * and indefinitely offline.
+ */
+export function paintsCold(id: string, vouched: ReadonlySet<string>, overlay: Overlay): boolean {
+  return id.startsWith('_00_') || vouched.has(id) || overlay.writes.has(id);
+}
+
+export interface RenderOptions {
+  hasExplicitOrder: boolean;
+  isWindow: boolean;
+}
+
+/**
+ *   render = (membership ∪ (writes ∩ localView)) − deletes
+ *
+ * `writes`/`deletes` are the outbox overlay (pending and acked items). The
+ * middle term keeps optimistic writes visible without re-admitting stale rows:
+ * `localView` (the SSP's local id-set) says whether the written row matches
+ * the query per local truth. Sorted unless the query orders itself or is a
+ * window (whose id order is the slice order).
+ */
+export function buildRenderIds(
+  membership: RecordVersionArray,
+  localView: RecordVersionArray,
+  overlay: Overlay,
+  opts: RenderOptions
+): string[] {
+  const ordered: string[] = [];
+  const seen = new Set<string>();
+  for (const [id] of membership) {
+    if (overlay.deletes.has(id) || seen.has(id)) continue;
+    seen.add(id);
+    ordered.push(id);
+  }
+  if (overlay.writes.size > 0) {
+    for (const [id] of localView) {
+      if (!overlay.writes.has(id) || overlay.deletes.has(id) || seen.has(id)) continue;
+      seen.add(id);
+      ordered.push(id);
+    }
+  }
+  if (!opts.hasExplicitOrder && !opts.isWindow) ordered.sort();
+  return ordered;
+}