@spooky-sync/core 0.0.1-canary.238 → 0.0.1-canary.239

package/AGENTS.md

@@ -23,7 +23,7 @@ Local mutations are applied optimistically, ingested into a DBSP layer that driv
 
 ## Key exports (`src/index.ts`)
 
-- `Sp00kyClient<S>` — main class. Methods: `init()`, `create(id, payload)`, `update(table, id, payload, options?)`, `delete(table, idOrSelector)`, `query(table, opts?)`, `run(backend, route, payload)`, `bucket(name)`, `remoteQuery(sql, vars?)`, `useRemote(fn)`, `authenticate(token)`, `signOut()`. Plus `pendingMutationCount` and `subscribeToPendingMutations(cb)`.
+- `Sp00kyClient<S>` — main class. Methods: `init()`, `create(id, payload)`, `update(table, id, payload, options?)`, `delete(table, idOrSelector)`, `query(table, opts?)`, `run(backend, route, payload)`, `bucket(name)`, `remoteQuery(sql, vars?)`, `useRemote(fn)`, `authenticate(token)`, `signOut()`. Plus `pendingMutationCount` / `subscribeToPendingMutations(cb)` (outbox depth) and `fetchingQueryCount` / `subscribeToFetchActivity(cb)` (queries mid-fetch, aggregate) for traffic indicators.
 - `BucketHandle` — file storage handle (`put`, `get`, `delete`, `exists`), plus the cache-aware `read`, `url`, `pin`/`unpin`, `evict`, `prefetch`. `get` is always remote; `read`/`url` go through the blob cache.
 - `services/blobs/` — durable cache for bucket bytes: OPFS holds the files, `_00_blob` holds a manifest that reconcile rebuilds from disk (so a wiped local store costs metadata, not the offline cache). Nothing expires on a timer; eviction is LRU under a byte budget only, skipping pinned and on-screen entries. Config: `blobCache` in `types.ts`.
 - `AuthService` — token management, sign-in/sign-out events.

package/dist/index.d.ts

@@ -739,6 +739,12 @@ declare class DataModule<S extends SchemaStructure> {
   private debounceTimers;
   private pendingStreamUpdates;
   private fetchDepth;
+  /**
+   * Observers of the AGGREGATE fetch activity: how many queries are mid-fetch
+   * right now. What a "downloading" indicator in the app chrome hangs off,
+   * instead of subscribing to every query's status one by one.
+   */
+  private activitySubscriptions;
   private logger;
   /**
    * Optional observer notified whenever a query's fetch status changes.
@@ -848,6 +854,18 @@ declare class DataModule<S extends SchemaStructure> {
   beginFetching(queryHash: string): void;
   /** Leave a fetch cycle started with {@link beginFetching}; emits `idle` on the last exit. */
   endFetching(queryHash: string): void;
+  /** How many queries are mid-fetch right now (see {@link subscribeActivity}). */
+  get fetchingQueryCount(): number;
+  /**
+   * Observe the number of queries currently fetching. Fires on every change of
+   * that number (a query entering or leaving its outermost fetch cycle), and
+   * synchronously with the current value when `immediate` is set. One
+   * subscription per indicator, however many queries the page holds.
+   */
+  subscribeActivity(callback: (fetching: number) => void, options?: {
+    immediate?: boolean;
+  }): () => void;
+  private notifyActivity;
   /**
    * Subscribe to mutations (for sync)
    */
@@ -2715,6 +2733,15 @@ declare class Sp00kyClient<S extends SchemaStructure> {
    *  suite can guard the pre-emptive path against regression. */
   get liveRetryCount(): number;
   subscribeToPendingMutations(cb: (count: number) => void): () => void;
+  /** Number of queries mid-fetch right now. See {@link subscribeToFetchActivity}. */
+  get fetchingQueryCount(): number;
+  /**
+   * Observe how many queries are fetching from the server. Fires immediately
+   * with the current count and again on every change, so one subscription can
+   * drive a "downloading" indicator for the whole app. Pair with
+   * {@link subscribeToPendingMutations} for the "uploading" half.
+   */
+  subscribeToFetchActivity(cb: (fetching: number) => void): () => void;
   /** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
   get syncHealth(): SyncHealth;
   /**

package/dist/index.js

@@ -148,7 +148,12 @@ const NETWORK_ERROR_PATTERNS = [
 	"econnreset",
 	"enotfound",
 	"epipe",
-	"abort"
+	"abort",
+	"transaction conflict",
+	"resource busy",
+	"can be retried",
+	"specify a namespace",
+	"specify a database"
 ];
 function classifySyncError(error) {
 	const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
@@ -3558,6 +3563,12 @@ var DataModule = class DataModule {
 	debounceTimers = /* @__PURE__ */ new Map();
 	pendingStreamUpdates = /* @__PURE__ */ new Map();
 	fetchDepth = /* @__PURE__ */ new Map();
+	/**
+	* Observers of the AGGREGATE fetch activity: how many queries are mid-fetch
+	* right now. What a "downloading" indicator in the app chrome hangs off,
+	* instead of subscribing to every query's status one by one.
+	*/
+	activitySubscriptions = /* @__PURE__ */ new Set();
 	logger;
 	/**
 	* Optional observer notified whenever a query's fetch status changes.
@@ -3775,7 +3786,10 @@ var DataModule = class DataModule {
 	beginFetching(queryHash) {
 		const depth = this.fetchDepth.get(queryHash) ?? 0;
 		this.fetchDepth.set(queryHash, depth + 1);
-		if (depth === 0) this.setQueryStatus(queryHash, "fetching");
+		if (depth === 0) {
+			this.setQueryStatus(queryHash, "fetching");
+			this.notifyActivity();
+		}
 	}
 	/** Leave a fetch cycle started with {@link beginFetching}; emits `idle` on the last exit. */
 	endFetching(queryHash) {
@@ -3783,10 +3797,40 @@ var DataModule = class DataModule {
 		if (depth <= 1) {
 			this.fetchDepth.delete(queryHash);
 			this.setQueryStatus(queryHash, "idle");
+			this.notifyActivity();
 			return;
 		}
 		this.fetchDepth.set(queryHash, depth - 1);
 	}
+	/** How many queries are mid-fetch right now (see {@link subscribeActivity}). */
+	get fetchingQueryCount() {
+		return this.fetchDepth.size;
+	}
+	/**
+	* Observe the number of queries currently fetching. Fires on every change of
+	* that number (a query entering or leaving its outermost fetch cycle), and
+	* synchronously with the current value when `immediate` is set. One
+	* subscription per indicator, however many queries the page holds.
+	*/
+	subscribeActivity(callback, options = {}) {
+		this.activitySubscriptions.add(callback);
+		if (options.immediate) callback(this.fetchDepth.size);
+		return () => {
+			this.activitySubscriptions.delete(callback);
+		};
+	}
+	notifyActivity() {
+		if (this.activitySubscriptions.size === 0) return;
+		const n = this.fetchDepth.size;
+		for (const cb of this.activitySubscriptions) try {
+			cb(n);
+		} catch (err) {
+			this.logger.warn({
+				err,
+				Category: "sp00ky-client::DataModule::notifyActivity"
+			}, "Activity subscriber threw");
+		}
+	}
 	/**
 	* Subscribe to mutations (for sync)
 	*/
@@ -4612,7 +4656,9 @@ var DataModule = class DataModule {
 		for (const timer of this.debounceTimers.values()) clearTimeout(timer);
 		this.debounceTimers.clear();
 		this.pendingStreamUpdates.clear();
+		const wasFetching = this.fetchDepth.size > 0;
 		this.fetchDepth.clear();
+		if (wasFetching) this.notifyActivity();
 		for (const queryState of this.activeQueries.values()) if (queryState.ttlTimer) {
 			clearTimeout(queryState.ttlTimer);
 			queryState.ttlTimer = null;
@@ -8341,8 +8387,8 @@ function selfAllowlistedVariant(flag, userId) {
 
 //#endregion
 //#region src/modules/devtools/index.ts
-const CORE_VERSION = "0.0.1-canary.238";
-const WASM_VERSION = "0.0.1-canary.238";
+const CORE_VERSION = "0.0.1-canary.239";
+const WASM_VERSION = "0.0.1-canary.239";
 const SURREAL_VERSION = "3.0.3";
 var DevToolsService = class DevToolsService {
 	eventsHistory = [];
@@ -13232,6 +13278,19 @@ var Sp00kyClient = class {
 	subscribeToPendingMutations(cb) {
 		return this.sync.subscribeToPendingMutations(cb);
 	}
+	/** Number of queries mid-fetch right now. See {@link subscribeToFetchActivity}. */
+	get fetchingQueryCount() {
+		return this.dataModule.fetchingQueryCount;
+	}
+	/**
+	* Observe how many queries are fetching from the server. Fires immediately
+	* with the current count and again on every change, so one subscription can
+	* drive a "downloading" indicator for the whole app. Pair with
+	* {@link subscribeToPendingMutations} for the "uploading" half.
+	*/
+	subscribeToFetchActivity(cb) {
+		return this.dataModule.subscribeActivity(cb, { immediate: true });
+	}
 	/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
 	get syncHealth() {
 		return this.sync.syncHealth;
@@ -13358,7 +13417,7 @@ var Sp00kyClient = class {
 		return new TabsCoordinator({
 			tabId,
 			fingerprint: computeTabsFingerprint({
-				coreVersion: "0.0.1-canary.238",
+				coreVersion: "0.0.1-canary.239",
 				schemaHash: hash53(this.config.schemaSurql),
 				endpoint: this.config.database.endpoint ?? "",
 				namespace: this.config.database.namespace,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.238",
+  "version": "0.0.1-canary.239",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.238",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.238",
+    "@spooky-sync/query-builder": "0.0.1-canary.239",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.239",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "blurhash": "^2.0.5",

package/src/modules/data/data.activity.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect, vi } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState } from '../../types';
+
+/**
+ * The aggregate fetch-activity channel (`subscribeActivity`): one number, how
+ * many queries are inside their outermost fetch cycle, so an app-chrome
+ * "downloading" indicator needs a single subscription instead of one per query.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeDataModule(): DataModule<any> {
+  return new DataModule({} as any, {} as any, { tables: [] } as any, makeLogger(), 100);
+}
+
+function makeQueryState(hash: string): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [],
+      remoteArray: [],
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+describe('DataModule fetch activity', () => {
+  it('emits the current count immediately and on every change', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    (dm as any).activeQueries.set('b', makeQueryState('b'));
+    const seen: number[] = [];
+    dm.subscribeActivity((n) => seen.push(n), { immediate: true });
+    expect(seen).toEqual([0]);
+
+    dm.beginFetching('a');
+    dm.beginFetching('b');
+    expect(seen).toEqual([0, 1, 2]);
+    expect(dm.fetchingQueryCount).toBe(2);
+
+    dm.endFetching('a');
+    dm.endFetching('b');
+    expect(seen).toEqual([0, 1, 2, 1, 0]);
+  });
+
+  it('counts a query once however deep its fetch cycles nest', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    const seen: number[] = [];
+    dm.subscribeActivity((n) => seen.push(n));
+    dm.beginFetching('a');
+    dm.beginFetching('a'); // overlapping poll round on the same hash
+    dm.endFetching('a');
+    expect(seen).toEqual([1], 'the inner cycle is silent');
+    dm.endFetching('a');
+    expect(seen).toEqual([1, 0]);
+  });
+
+  it('stops notifying after unsubscribe and survives a throwing subscriber', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    const bad = vi.fn(() => {
+      throw new Error('boom');
+    });
+    const good = vi.fn();
+    dm.subscribeActivity(bad);
+    const off = dm.subscribeActivity(good);
+    dm.beginFetching('a');
+    expect(good).toHaveBeenCalledWith(1);
+    off();
+    dm.endFetching('a');
+    expect(good).toHaveBeenCalledTimes(1);
+    expect(bad).toHaveBeenCalledTimes(2);
+  });
+
+  it('drops to zero when a bucket switch quiesces in-flight fetches', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    const seen: number[] = [];
+    dm.subscribeActivity((n) => seen.push(n));
+    dm.beginFetching('a');
+    dm.quiesce();
+    expect(seen).toEqual([1, 0]);
+    expect(dm.fetchingQueryCount).toBe(0);
+  });
+});

package/src/modules/data/index.ts

@@ -119,6 +119,12 @@ export class DataModule<S extends SchemaStructure> {
   // back to `idle` only on 1→0, so an inner cycle finishing can't emit a
   // premature idle mid-registration.
   private fetchDepth: Map<QueryHash, number> = new Map();
+  /**
+   * Observers of the AGGREGATE fetch activity: how many queries are mid-fetch
+   * right now. What a "downloading" indicator in the app chrome hangs off,
+   * instead of subscribing to every query's status one by one.
+   */
+  private activitySubscriptions: Set<(fetching: number) => void> = new Set();
   private logger: Logger;
   /**
    * Optional observer notified whenever a query's fetch status changes.
@@ -446,6 +452,7 @@ export class DataModule<S extends SchemaStructure> {
     this.fetchDepth.set(queryHash, depth + 1);
     if (depth === 0) {
       this.setQueryStatus(queryHash, 'fetching');
+      this.notifyActivity();
     }
   }
 
@@ -455,11 +462,49 @@ export class DataModule<S extends SchemaStructure> {
     if (depth <= 1) {
       this.fetchDepth.delete(queryHash);
       this.setQueryStatus(queryHash, 'idle');
+      this.notifyActivity();
       return;
     }
     this.fetchDepth.set(queryHash, depth - 1);
   }
 
+  /** How many queries are mid-fetch right now (see {@link subscribeActivity}). */
+  get fetchingQueryCount(): number {
+    return this.fetchDepth.size;
+  }
+
+  /**
+   * Observe the number of queries currently fetching. Fires on every change of
+   * that number (a query entering or leaving its outermost fetch cycle), and
+   * synchronously with the current value when `immediate` is set. One
+   * subscription per indicator, however many queries the page holds.
+   */
+  subscribeActivity(
+    callback: (fetching: number) => void,
+    options: { immediate?: boolean } = {}
+  ): () => void {
+    this.activitySubscriptions.add(callback);
+    if (options.immediate) callback(this.fetchDepth.size);
+    return () => {
+      this.activitySubscriptions.delete(callback);
+    };
+  }
+
+  private notifyActivity(): void {
+    if (this.activitySubscriptions.size === 0) return;
+    const n = this.fetchDepth.size;
+    for (const cb of this.activitySubscriptions) {
+      try {
+        cb(n);
+      } catch (err) {
+        this.logger.warn(
+          { err, Category: 'sp00ky-client::DataModule::notifyActivity' },
+          'Activity subscriber threw'
+        );
+      }
+    }
+  }
+
   /**
    * Subscribe to mutations (for sync)
    */
@@ -1664,7 +1709,9 @@ export class DataModule<S extends SchemaStructure> {
     }
     this.debounceTimers.clear();
     this.pendingStreamUpdates.clear();
+    const wasFetching = this.fetchDepth.size > 0;
     this.fetchDepth.clear();
+    if (wasFetching) this.notifyActivity();
     for (const queryState of this.activeQueries.values()) {
       if (queryState.ttlTimer) {
         clearTimeout(queryState.ttlTimer);

package/src/sp00ky.ts

@@ -401,6 +401,21 @@ export class Sp00kyClient<S extends SchemaStructure> {
     return this.sync.subscribeToPendingMutations(cb);
   }
 
+  /** Number of queries mid-fetch right now. See {@link subscribeToFetchActivity}. */
+  get fetchingQueryCount(): number {
+    return this.dataModule.fetchingQueryCount;
+  }
+
+  /**
+   * Observe how many queries are fetching from the server. Fires immediately
+   * with the current count and again on every change, so one subscription can
+   * drive a "downloading" indicator for the whole app. Pair with
+   * {@link subscribeToPendingMutations} for the "uploading" half.
+   */
+  subscribeToFetchActivity(cb: (fetching: number) => void): () => void {
+    return this.dataModule.subscribeActivity(cb, { immediate: true });
+  }
+
   /** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
   get syncHealth(): SyncHealth {
     return this.sync.syncHealth;

package/src/utils/error-classification.test.ts

@@ -29,6 +29,16 @@ describe('classifySyncError', () => {
     expect(classifySyncError(new Error(message))).toBe('network');
   });
 
+  it.each([
+    // SurrealDB optimistic-concurrency conflict: retryable by definition.
+    'Transaction conflict: Resource busy: . This transaction can be retried',
+    // Session state not re-applied yet after the SDK's reconnect handshake.
+    'Specify a namespace to use',
+    'Specify a database to use',
+  ])('classifies transient server state %j as network (retry, never rollback)', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
   it.each([
     'Permission denied',
     'There was a problem with the database: record already exists',

package/src/utils/error-classification.ts

@@ -19,6 +19,24 @@ const NETWORK_ERROR_PATTERNS = [
   'enotfound',
   'epipe',
   'abort',
+  // Transient server-side states that are NOT the mutation's fault and clear
+  // on their own, so the outbox must retry them rather than roll the write
+  // back. Seen 2026-09-08 on whitepawn: a 1200-game import lost 752 games and
+  // ~1500 player_name rows because every queued CREATE that hit one of these
+  // during a DB stall + WebSocket reconnect was classified `application`,
+  // rolled back locally and discarded - silent data loss.
+  //
+  // - SurrealDB optimistic-concurrency conflicts: "Transaction conflict:
+  //   Resource busy: . This transaction can be retried".
+  // - A socket whose namespace/identity has not been re-applied yet after the
+  //   SDK's own reconnect handshake: "Specify a namespace to use" /
+  //   "Specify a database to use". The statement is fine; the session is not
+  //   ready.
+  'transaction conflict',
+  'resource busy',
+  'can be retried',
+  'specify a namespace',
+  'specify a database',
 ];
 
 export function classifySyncError(error: unknown): 'network' | 'application' {