@spooky-sync/core 0.0.1-canary.234 → 0.0.1-canary.236

package/AGENTS.md

@@ -23,7 +23,7 @@ Local mutations are applied optimistically, ingested into a DBSP layer that driv
 
 ## Key exports (`src/index.ts`)
 
-- `Sp00kyClient<S>` — main class. Methods: `init()`, `create(id, payload)`, `update(table, id, payload, options?)`, `delete(table, idOrSelector)`, `query(table, opts?)`, `run(backend, route, payload)`, `bucket(name)`, `useRemote(fn)`, `authenticate(token)`, `signOut()`. Plus `pendingMutationCount` and `subscribeToPendingMutations(cb)`.
+- `Sp00kyClient<S>` — main class. Methods: `init()`, `create(id, payload)`, `update(table, id, payload, options?)`, `delete(table, idOrSelector)`, `query(table, opts?)`, `run(backend, route, payload)`, `bucket(name)`, `remoteQuery(sql, vars?)`, `useRemote(fn)`, `authenticate(token)`, `signOut()`. Plus `pendingMutationCount` and `subscribeToPendingMutations(cb)`.
 - `BucketHandle` — file storage handle (`put`, `get`, `delete`, `exists`), plus the cache-aware `read`, `url`, `pin`/`unpin`, `evict`, `prefetch`. `get` is always remote; `read`/`url` go through the blob cache.
 - `services/blobs/` — durable cache for bucket bytes: OPFS holds the files, `_00_blob` holds a manifest that reconcile rebuilds from disk (so a wiped local store costs metadata, not the offline cache). Nothing expires on a timer; eviction is LRU under a byte budget only, skipping pinned and on-screen entries. Config: `blobCache` in `types.ts`.
 - `AuthService` — token management, sign-in/sign-out events.
@@ -37,7 +37,8 @@ Local mutations are applied optimistically, ingested into a DBSP layer that driv
 - **Mutations are optimistic.** `db.create` / `db.update` / `db.delete` return immediately; the queue drains in the background. Inspect progress via `pendingMutationCount`.
 - **Record IDs are full strings.** `db.create('thread:abc123', {...})` — *not* `db.create('thread', { id: 'abc123' })`. The first arg is `'<table>:<id>'`. Generate IDs with `Uuid` from `surrealdb` (re-exported by `client-solid`).
 - **`db.update` takes `(table, id, payload, options?)`**. The `debounced` option (`{ debounced: true }` or `{ debounced: { delay, key } }`) coalesces rapid updates — use it for CRDT text/title fields.
-- **`db.useRemote(fn)` bypasses the cache.** Use only for queries that can't be expressed via the query builder (e.g. graph traversals across `RELATE` edges). Results are not synced into the local cache.
+- **`db.remoteQuery(sql, vars?)` is the one-shot remote read.** It runs one SurrealQL statement through the client's remote path (connect gate, per-statement timeout, concurrency limit) and returns the SurrealDB result array; results are not synced into the local cache. Use it for counts and other reads the query builder can't express.
+- **`db.useRemote(fn)` hands out the bare SDK client.** Same cache bypass, but it skips the connect gate (a call during a reconnect fails with "Specify a namespace to use"). Reserve it for SDK features `remoteQuery` cannot reach (transactions, `live`).
 - **`@parent` columns are auto-populated.** Defined in the `.surql` schema as `record<user>` with the `-- @parent` annotation; never write them yourself — they're set from the auth context server-side.
 
 ## Module map (under `src/modules/`)

package/dist/index.d.ts

@@ -30,14 +30,39 @@ declare abstract class AbstractDatabaseService {
   getClient(): Surreal$1;
   getEvents(): DatabaseEventSystem;
   tx(): Promise<SurrealTransaction>;
-  private queryQueue;
   /**
-   * Execute a query with serialized execution to prevent WASM transaction issues.
+   * How many statements may be in flight at once.
    *
-   * Serialization means every query waits on the previous one, so a call that
-   * never settles blocks the whole chain forever. {@link queryTimeoutMs} bounds
-   * each link: on expiry this promise rejects and the chain moves on, even
-   * though the underlying RPC is still parked in the SDK's pending map.
+   * `1` serializes every statement behind the previous one. The local engines
+   * need that: overlapping ops trip the WASM / SQLite transaction layer. The
+   * remote service raises it (`RemoteDatabaseService`): a SurrealDB socket
+   * multiplexes requests by id, and serializing them capped the client at one
+   * statement per round trip (~5/s at 170ms RTT), so the `_00_list_ref` poll
+   * alone could hold every registration behind it and one slow one-shot read
+   * stalled the whole client. The scheduler's `MAX_CONCURRENT_DOWN` only means
+   * something when this is above 1.
+   */
+  protected maxConcurrentQueries: number;
+  private inFlightQueries;
+  private queryWaiters;
+  /** FIFO slot acquisition, so `maxConcurrentQueries = 1` keeps strict order. */
+  private acquireQuerySlot;
+  private releaseQuerySlot;
+  /**
+   * Hook run right before a statement is sent, inside its slot. The remote
+   * service uses it to hold statements while a connect is in flight, so a
+   * query never runs on a socket that is open but has not had its namespace
+   * and token applied yet. The default does nothing.
+   */
+  protected beforeQuery(): Promise<void>;
+  /**
+   * Execute a statement under {@link maxConcurrentQueries}.
+   *
+   * A statement that never settles would hold its slot forever, and with a
+   * limit of 1 that blocks every later statement behind it. {@link
+   * queryTimeoutMs} bounds each one: on expiry this promise rejects and the
+   * slot is released, even though the underlying RPC is still parked in the
+   * SDK's pending map.
    */
   query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T>;
   execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>): Promise<T>;
@@ -117,6 +142,19 @@ declare class RemoteDatabaseService extends AbstractDatabaseService {
    * `disconnected`, which is what drives the supervisor's revive loop.
    */
   forceClose(): Promise<void>;
+  /**
+   * Hold statements while a connect is in flight.
+   *
+   * `connect()` opens the socket, then selects the namespace/database, then
+   * authenticates - three round trips. The SDK considers the connection ready
+   * after the first, so a query queued during the other two ran on a socket
+   * with no namespace ("Specify a namespace to use") or, worse, no identity.
+   * Seen 2026-09-06 after the supervisor rebuilt a socket: a collection
+   * recount failed with exactly that error. Waiting on the in-flight connect
+   * costs nothing when no connect is running; a failed connect is not
+   * swallowed here - the query then fails on its own, as before.
+   */
+  protected beforeQuery(): Promise<void>;
   /**
    * Open (or re-open) the remote connection.
    *
@@ -1389,6 +1427,7 @@ declare class Sp00kySync<S extends SchemaStructure> {
   private listRefPollInFlight;
   readonly refSyncIntervalMs: number;
   private listRefIdleStreak;
+  private listRefPolledAt;
   private stillRemoteStreaks;
   private lastLiveEventAt;
   private _liveRetryCount;
@@ -1567,9 +1606,10 @@ declare class Sp00kySync<S extends SchemaStructure> {
   private startListRefPoll;
   private stopListRefPoll;
   /**
-   * One poll cycle: refetch `_00_list_ref` for every active query. Returns
-   * whether ANY query's remoteArray actually changed — the scheduler uses this
-   * to drive the adaptive idle backoff.
+   * One poll cycle: refetch `_00_list_ref` for the active queries that are due,
+   * one round trip per chunk (see `planListRefPollChunks`). Returns whether ANY
+   * query's remoteArray actually changed — the scheduler uses this to drive the
+   * adaptive idle backoff.
    *
    * Also the ONLY health signal that runs while the page is idle. Sync health is
    * otherwise activity-driven (mutations/registrations via the scheduler,
@@ -1582,14 +1622,37 @@ declare class Sp00kySync<S extends SchemaStructure> {
    */
   private pollListRefForActiveQueries;
   /**
-   * Pull the upstream list_ref entries for `queryHash`, diff them
-   * against the local `remoteArray` cache, sync any added/updated rows
-   * through the SyncEngine, then persist the new remoteArray. This is
-   * the same shape `createRemoteQuery` does for its initial fetch and
-   * what `handleRemoteListRefChange` does per-LIVE-event — we reuse
-   * it on a timer as a fallback for missed LIVE notifications.
+   * The server's current `_00_list_ref` state for a chunk of queries in ONE
+   * round trip: primary edges, subquery-child edges and `_00_query.rowCount`.
+   * Every hash whose query still exists gets an entry (empty arrays and
+   * `rowCount: null` when nothing was readable), so a caller can tell "no
+   * edges" from "not asked". A hash is left out only when the select did not
+   * come back as an array at all — the old per-query code skipped the apply in
+   * that case too, since a non-array is not a membership set.
+   *
+   * Guarded against a batch that silently matches nothing: if the select
+   * returned no edge at all while the client holds edges for a query the
+   * server still reports rows for, the batch is not trusted and the chunk is
+   * re-read one query at a time with the single-query select. An empty set
+   * taken at face value would be recorded as the removal of every row.
    */
-  private refetchListRefForQuery;
+  private fetchListRefSnapshots;
+  /**
+   * Single-query form of {@link fetchListRefSnapshots}: the selects the
+   * registration path has always used, in one statement batch. `null` when
+   * the primary select did not come back as an array.
+   */
+  private fetchListRefSnapshot;
+  /**
+   * Land one poll snapshot for `queryHash`: diff the server's edges against the
+   * cached `remoteArray`, sync any added/updated rows through the SyncEngine,
+   * persist the new remoteArray and converge the subquery children. This is
+   * the same shape `createRemoteQuery` does for its initial fetch and what
+   * `handleRemoteListRefChange` does per-LIVE-event — we reuse it on a timer
+   * as a fallback for missed LIVE notifications. Returns whether membership
+   * changed.
+   */
+  private applyListRefSnapshot;
   /**
    * Resolve the current `_00_list_ref` table name for the active auth
    * context. Public so the `createRemoteQuery` initial-fetch path can
@@ -1765,6 +1828,13 @@ declare class Sp00kySync<S extends SchemaStructure> {
    * query to `fetching` or skew its DevTools timings.
    */
   private syncSubqueryChildren;
+  /**
+   * Converge a `.related()` query's child bodies onto `fresh` (the server's
+   * current child edges). Shared by the registration path above and the poll,
+   * which reads the child edges in its batched round trip. See
+   * {@link syncSubqueryChildren} for the deletion-safety rules.
+   */
+  private applySubqueryChildren;
   heartbeatQuery(queryHash: string): Promise<void>;
   private cleanupQuery;
 }
@@ -2691,14 +2761,20 @@ declare class Sp00kyClient<S extends SchemaStructure> {
   query<Table extends TableNames<S>>(table: Table, options: QueryOptions<TableModel<GetTable<S, Table>>, false>, ttl?: QueryTimeToLive): QueryBuilder<S, Table, Sp00kyQueryResultPromise>;
   private initQuery;
   /**
-   * Background tail of {@link initQuery}: instant-hydrate (opt-in via
-   * `config.instantHydrate`, and only when the query is cold) followed by
-   * enqueuing the `register` down-event. Never rejects — both halves catch and
-   * log, so `void`-ing the returned promise can't produce an unhandled
-   * rejection. By default (hydrate off) the register lifecycle is the single
-   * freshness path; the one-shot fetch is an optimization apps enable
-   * explicitly, and it runs regardless of preload state — cache-first delivery
-   * never depends on WHY rows are cached.
+   * Background tail of {@link initQuery}: enqueue the `register` down-event,
+   * then instant-hydrate (opt-in via `config.instantHydrate`, and only when
+   * the query is cold). Never rejects — both halves catch and log, so
+   * `void`-ing the returned promise can't produce an unhandled rejection. By
+   * default (hydrate off) the register lifecycle is the single freshness path;
+   * the one-shot fetch is an optimization apps enable explicitly, and it runs
+   * regardless of preload state — cache-first delivery never depends on WHY
+   * rows are cached.
+   *
+   * Register goes first on purpose. The hydrate used to run strictly before
+   * the enqueue, so a windowed list's one-shot read (a full-table scan when
+   * the app has no matching index) held its own registration back by seconds
+   * per window. Now whichever answer lands first paints; a hydrate that
+   * arrives after the registration is dropped (`applyHydration`).
    */
   private finishQueryInit;
   /**
@@ -2769,6 +2845,15 @@ declare class Sp00kyClient<S extends SchemaStructure> {
    */
   isLocalReady(): boolean;
   useRemote<T>(fn: (client: Surreal) => Promise<T> | T): Promise<T>;
+  /**
+   * Run one raw SurrealQL statement on the remote through the client's own
+   * query path: the connect gate (it never runs on a socket whose namespace
+   * and token are not applied yet), the per-statement timeout and the
+   * concurrency limit. Prefer this over {@link useRemote} for one-shot reads
+   * such as counts; `useRemote` hands out the bare SDK client and raced a
+   * reconnect with "Specify a namespace to use".
+   */
+  remoteQuery<T extends unknown[]>(sql: string, vars?: Record<string, unknown>): Promise<T>;
   /**
    * Mint the salt used for query-id hashing, so two sessions registering the
    * same logical query get distinct `_00_query` rows.