@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

package/src/utils/blurhash.ts

@@ -0,0 +1,90 @@
+import { encode, decode, isBlurhashValid } from 'blurhash';
+
+export { encode as encodeBlurhash, decode as decodeBlurhash, isBlurhashValid };
+
+/**
+ * Blurhash generation settings. `true` enables with the defaults below, `false`
+ * disables. Resolution order for a put: per-call option > client config >
+ * default ON. See {@link Sp00kyConfig.blurhash}.
+ */
+export type BlurhashSetting = boolean | BlurhashEncodeOptions;
+
+export interface BlurhashEncodeOptions {
+  /** Horizontal detail components, 1-9. Defaults to 4. */
+  componentX?: number;
+  /** Vertical detail components, 1-9. Defaults to 3. */
+  componentY?: number;
+}
+
+/**
+ * Where an image's blurhash lives: a tiny sidecar object in the same bucket.
+ * Buckets have no per-object metadata channel (`put` is just `.put($content)`),
+ * so the hash for `covers/x_t.webp` is the text object `covers/x_t.webp.bh`.
+ */
+export function blurhashSidecarPath(path: string): string {
+  return `${path}.bh`;
+}
+
+/** Extensions `bucket.put` treats as images worth hashing. */
+export const BLURHASH_IMAGE_EXTENSIONS = [
+  'webp',
+  'png',
+  'jpg',
+  'jpeg',
+  'gif',
+  'avif',
+  'bmp',
+] as const;
+
+const IMAGE_EXT_RE = new RegExp(`\\.(${BLURHASH_IMAGE_EXTENSIONS.join('|')})$`, 'i');
+
+export function isImagePath(path: string): boolean {
+  return IMAGE_EXT_RE.test(path);
+}
+
+/** Longest edge of the downscale the hash is computed from. Blurhash carries at
+ *  most 9x9 DCT components, so anything past ~32px is wasted decode work. */
+const ENCODE_MAX_EDGE = 32;
+
+/**
+ * Decode `content` as an image and compute its blurhash. Browser-only: returns
+ * null (never throws) when image decoding is unavailable (node, workers without
+ * canvas), when the bytes are not a decodable image, or on any other failure —
+ * a missing hash must never break the upload that triggered it.
+ */
+export async function encodeImageToBlurhash(
+  content: string | Uint8Array | Blob,
+  options?: BlurhashEncodeOptions
+): Promise<string | null> {
+  if (typeof createImageBitmap !== 'function') return null;
+  let bitmap: ImageBitmap | null = null;
+  try {
+    const blob =
+      content instanceof Blob
+        ? content
+        : new Blob([content as unknown as BlobPart]);
+    bitmap = await createImageBitmap(blob);
+    const scale = Math.min(1, ENCODE_MAX_EDGE / Math.max(bitmap.width, bitmap.height));
+    const width = Math.max(1, Math.round(bitmap.width * scale));
+    const height = Math.max(1, Math.round(bitmap.height * scale));
+    const canvas =
+      typeof OffscreenCanvas !== 'undefined'
+        ? new OffscreenCanvas(width, height)
+        : typeof document !== 'undefined'
+          ? Object.assign(document.createElement('canvas'), { width, height })
+          : null;
+    if (!canvas) return null;
+    const ctx = canvas.getContext('2d') as
+      | OffscreenCanvasRenderingContext2D
+      | CanvasRenderingContext2D
+      | null;
+    if (!ctx) return null;
+    ctx.drawImage(bitmap, 0, 0, width, height);
+    const { data } = ctx.getImageData(0, 0, width, height);
+    return encode(data, width, height, options?.componentX ?? 4, options?.componentY ?? 3);
+  } catch {
+    return null;
+  } finally {
+    bitmap?.close();
+  }
+}

package/src/utils/edges.ts

@@ -0,0 +1,39 @@
+import { RecordId } from 'surrealdb';
+import type { InlineRow } from '../types';
+import { encodeRecordId } from './index';
+
+/**
+ * Reading `_00_list_ref` edges as a LIVE notification delivers them. Pure, so
+ * both the full client and the wasm-free live feed (`@spooky-sync/core/live`)
+ * decode the same wire shape the same way.
+ */
+
+/** The hash a `_00_list_ref` edge belongs to: the id part of its `in` (`_00_query:<hash>`). */
+export function hashOfEdge(value: unknown): string | null {
+  const inId = (value as { in?: unknown } | null)?.in;
+  if (!inId) return null;
+  const str = inId instanceof RecordId ? String(inId.id) : String(inId).replace(/^_00_query:/, '');
+  return str.length > 0 ? str : null;
+}
+
+/**
+ * The row an edge notification carries, when the subscription joined it on.
+ *
+ * Without `FETCH out` the edge's `out` is a record id and there is nothing to
+ * land, so this returns null and the caller falls back to fetching the body.
+ * It also returns null for a row the session may not read (the join yields
+ * `out: null`) and for an edge whose target has been deleted.
+ */
+export function rowOfEdge(value: unknown): InlineRow | null {
+  const edge = value as { out?: unknown; version?: unknown } | null;
+  const out = edge?.out;
+  if (!out || typeof out !== 'object' || Array.isArray(out) || out instanceof RecordId) return null;
+  // Must be a real RecordId: the landing path keys off `row.id` being one, and
+  // would otherwise record the version for a body it never wrote.
+  const rid = (out as { id?: unknown }).id;
+  if (!(rid instanceof RecordId)) return null;
+  const id = encodeRecordId(rid);
+  const version = typeof edge?.version === 'number' ? edge.version : null;
+  if (version === null) return null;
+  return { id, version, record: out as Record<string, unknown> };
+}

package/src/utils/error-classification.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect } from 'vitest';
+import { classifySyncError } from './error-classification';
+
+describe('classifySyncError', () => {
+  it('classifies surreal ConnectionUnavailableError as network', () => {
+    // Thrown synchronously by the WS engine's send() while the socket is down
+    // but reconnect hasn't fired. Message contains "connected", not "connection".
+    const err = new Error(
+      'You must be connected to a SurrealDB instance before performing this operation'
+    );
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it('classifies surreal CallTerminatedError as network', () => {
+    const err = new Error(
+      'The call has been terminated because the connection was closed'
+    );
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it.each([
+    'WebSocket connection failed',
+    'fetch failed',
+    'connect ECONNREFUSED 127.0.0.1:8666',
+    'request timed out',
+    'The operation was aborted',
+    'socket hang up',
+  ])('classifies %j as network', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
+  it.each([
+    // SurrealDB optimistic-concurrency conflict: retryable by definition.
+    'Transaction conflict: Resource busy: . This transaction can be retried',
+    // Session state not re-applied yet after the SDK's reconnect handshake.
+    'Specify a namespace to use',
+    'Specify a database to use',
+  ])('classifies transient server state %j as network (retry, never rollback)', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
+  it.each([
+    'Permission denied',
+    'There was a problem with the database: record already exists',
+    'Parse error: unexpected token',
+  ])('classifies %j as application', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('application');
+  });
+
+  it('handles non-Error values', () => {
+    expect(classifySyncError('connection reset')).toBe('network');
+    expect(classifySyncError('boom')).toBe('application');
+  });
+});

package/src/utils/error-classification.ts

@@ -1,5 +1,12 @@
 const NETWORK_ERROR_PATTERNS = [
   'connection',
+  // surreal's ConnectionUnavailableError reads "You must be connected to a
+  // SurrealDB instance..." — it contains "connected", not "connection", so it
+  // slips past the pattern above. The WS engine throws it synchronously from
+  // `send()` while the socket is down but reconnect hasn't fired yet, so it's the
+  // canonical error on an idle-dropped socket; classify it as network.
+  'must be connected',
+  'connectionunavailable',
   'timeout',
   'timed out',
   'websocket',
@@ -12,6 +19,24 @@ const NETWORK_ERROR_PATTERNS = [
   'enotfound',
   'epipe',
   'abort',
+  // Transient server-side states that are NOT the mutation's fault and clear
+  // on their own, so the outbox must retry them rather than roll the write
+  // back. Seen 2026-09-08 on whitepawn: a 1200-game import lost 752 games and
+  // ~1500 player_name rows because every queued CREATE that hit one of these
+  // during a DB stall + WebSocket reconnect was classified `application`,
+  // rolled back locally and discarded - silent data loss.
+  //
+  // - SurrealDB optimistic-concurrency conflicts: "Transaction conflict:
+  //   Resource busy: . This transaction can be retried".
+  // - A socket whose namespace/identity has not been re-applied yet after the
+  //   SDK's own reconnect handshake: "Specify a namespace to use" /
+  //   "Specify a database to use". The statement is fine; the session is not
+  //   ready.
+  'transaction conflict',
+  'resource busy',
+  'can be retried',
+  'specify a namespace',
+  'specify a database',
 ];
 
 export function classifySyncError(error: unknown): 'network' | 'application' {

package/src/utils/index.ts

@@ -1,7 +1,7 @@
-import { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
+import type { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
 import { Uuid, RecordId, Duration } from 'surrealdb';
-import { Logger } from '../services/logger/index';
-import { QueryTimeToLive } from '../types';
+import type { Logger } from '../services/logger/index';
+import type { QueryTimeToLive } from '../types';
 
 export * from './surql';
 export * from './parser';
@@ -19,7 +19,7 @@ export const compareRecordIds = (
 };
 
 export const encodeRecordId = (recordId: RecordId<string>): string => {
-  return `${recordId.table.toString()}:${recordId.id}`;
+  return `${recordId.table.name}:${recordId.id}`;
 };
 
 export const extractIdPart = (id: string | RecordId<string>): string => {
@@ -39,7 +39,7 @@ export const extractTablePart = (id: string | RecordId<string>): string => {
   if (typeof id === 'string') {
     return id.split(':')[0];
   }
-  return id.table.toString();
+  return id.table.name;
 };
 
 export const parseRecordIdString = (id: string): RecordId<string> => {
@@ -51,6 +51,17 @@ export function generateId(): string {
   return Uuid.v4().toString().replace(/-/g, '');
 }
 
+/**
+ * 32 random bytes, hex: the secret a page proves it owns its anonymous views
+ * with (`$config.owner` on `fn::query::register`). `getRandomValues` rather
+ * than `randomUUID`, which a page served over plain http does not have.
+ */
+export function mintOwnerSecret(): string {
+  const bytes = new Uint8Array(32);
+  globalThis.crypto.getRandomValues(bytes);
+  return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
+}
+
 export function generateNewTableId<S extends SchemaStructure, T extends TableNames<S>>(
   tableName: T
 ): RecordId {
@@ -74,12 +85,12 @@ export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames
   for (const field of Object.keys(table.columns)) {
     const column = table.columns[field] as any;
     const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
-    if ((column.recordId || relation) && encoded[field] != null) {
+    if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
       if (encoded[field] instanceof RecordId) {
-        encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
+        encoded[field] = `${encoded[field].table.name}:${encoded[field].id}`;
       } else if (
         relation &&
-        (encoded[field] instanceof Object || encoded[field] instanceof Array)
+        (encoded[field] instanceof Object || Array.isArray(encoded[field]))
       ) {
         if (Array.isArray(encoded[field])) {
           encoded[field] = encoded[field].map((item) =>
@@ -97,15 +108,25 @@ export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames
 
 // ==================== TIME/DURATION UTILITIES ====================
 
+/**
+ * Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
+ * releases expose it under `milliseconds` (current) or the older private
+ * `_milliseconds` field, so read whichever is set; returns 0 when neither is.
+ */
+function durationMillis(duration: Duration): number {
+  const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
+  return Number(d.milliseconds || d._milliseconds || 0);
+}
+
 /**
  * Parse duration string or Duration object to milliseconds
  */
 export function parseDuration(duration: QueryTimeToLive | Duration): number {
   if (duration instanceof Duration) {
-    const ms = (duration as any).milliseconds || (duration as any)._milliseconds;
-    if (ms) return Number(ms);
+    const ms = durationMillis(duration);
+    if (ms) return ms;
     const str = duration.toString();
-    if (str !== '[object Object]') return parseDuration(str as any);
+    if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
     return 600000;
   }
 
@@ -117,7 +138,7 @@ export function parseDuration(duration: QueryTimeToLive | Duration): number {
 
   const match = duration.match(/^(\d+)([smh])$/);
   if (!match) return 600000;
-  const val = parseInt(match[1], 10);
+  const val = Number.parseInt(match[1], 10);
   const unit = match[2];
   switch (unit) {
     case 's':
@@ -137,6 +158,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
   return new Uint8Array(buffer);
 }
 
+// ==================== TEXT UTILITIES ====================
+
+/**
+ * Convert plain text to simple HTML paragraphs.
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
+ */
+export function textToHtml(text: string): string {
+  return text
+    .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+    .split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
+}
+
 // ==================== DATABASE UTILITIES ====================
 
 /**
@@ -154,6 +187,9 @@ export async function withRetry<T>(
       return await operation();
     } catch (err: any) {
       lastError = err;
+      // A deadline expiry is explicitly not retryable: the op is still running
+      // in the engine, and a retry only queues a second copy behind it.
+      if (err?.retryable === false) throw err;
       if (
         err?.message?.includes('Can not open transaction') ||
         err?.message?.includes('transaction') ||
@@ -177,3 +213,44 @@ export async function withRetry<T>(
   }
   throw lastError;
 }
+
+/**
+ * Reject after `timeoutMs` if `promise` hasn't settled.
+ *
+ * Exists because a WebSocket RPC has no deadline of its own: the SurrealDB SDK
+ * parks each call in a pending map and only rejects it when the socket reports
+ * a close. On a half-open socket (peer gone, no `close` event, `readyState`
+ * still OPEN) the call never settles at all, which wedges anything serialized
+ * behind it.
+ *
+ * `message` MUST contain "timed out" so `classifySyncError` treats the
+ * rejection as `network` — that's what makes the sync queues retry the
+ * operation instead of rolling the mutation back as an application error.
+ *
+ * Non-positive `timeoutMs` returns `promise` unchanged. The underlying promise
+ * is NOT cancelled (nothing can cancel an in-flight RPC); its eventual
+ * settlement is absorbed, so attach no expectations to it after a timeout.
+ */
+export function withTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+  message: string | (() => Error)
+): Promise<T> {
+  if (!(timeoutMs > 0)) return promise;
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(
+      () => reject(typeof message === 'function' ? message() : new Error(message)),
+      timeoutMs
+    );
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    );
+  });
+}

package/src/utils/parser.test.ts

@@ -1,125 +1,90 @@
-import { describe, it, expect } from 'vitest';
-import { RecordId } from 'surrealdb';
-import { parseParams, cleanRecord } from './parser';
+import { describe, expect, it } from 'vitest';
+import { RecordId } from '@spooky-sync/query-builder';
+import { absentOptionalColumns, parseParams, parseQueryParams } from './parser';
 
-describe('parseParams', () => {
-  it('passes through plain values unchanged', () => {
-    const schema = {
-      name: { type: 'string' as const, optional: false },
-      age: { type: 'number' as const, optional: false },
-    };
-    const result = parseParams(schema, { name: 'Alice', age: 30 });
-    expect(result).toEqual({ name: 'Alice', age: 30 });
-  });
-
-  it('converts string to RecordId for recordId columns', () => {
-    const schema = {
-      owner: { type: 'string' as const, optional: false, recordId: true },
-    };
-    const result = parseParams(schema, { owner: 'user:123' });
-    expect(result.owner).toBeInstanceOf(RecordId);
-    expect(result.owner.table.toString()).toBe('user');
-    expect(result.owner.id).toBe('123');
-  });
+const columns = {
+  white: { recordId: true },
+  black: { recordId: true },
+  owner: { recordId: true },
+  created_at: { dateTime: true },
+  result: {},
+} as any;
 
-  it('passes through existing RecordId for recordId columns', () => {
-    const schema = {
-      owner: { type: 'string' as const, optional: false, recordId: true },
-    };
-    const rid = new RecordId('user', '456');
-    const result = parseParams(schema, { owner: rid });
-    expect(result.owner).toBe(rid);
-  });
-
-  it('converts string to Date for dateTime columns', () => {
-    const schema = {
-      createdAt: { type: 'string' as const, optional: false, dateTime: true },
-    };
-    const result = parseParams(schema, { createdAt: '2024-01-01T00:00:00Z' });
-    expect(result.createdAt).toBeInstanceOf(Date);
-    expect(result.createdAt.toISOString()).toBe('2024-01-01T00:00:00.000Z');
-  });
+describe('parseQueryParams', () => {
+  // The regression this exists for: an `_or` group binds under synthetic names,
+  // and dropping them left `(white = $white__or0 OR black = $black__or1)` with
+  // nothing bound, so every OR query registered and matched no rows at all.
+  it('keeps an _or branch param and types it through its field', () => {
+    const parsed = parseQueryParams(columns, {
+      owner: 'user:abc',
+      white__or0: 'player_name:PN_1',
+      black__or1: 'player_name:PN_1',
+    });
 
-  it('converts number (timestamp) to Date for dateTime columns', () => {
-    const schema = {
-      createdAt: { type: 'string' as const, optional: false, dateTime: true },
-    };
-    const ts = 1704067200000; // 2024-01-01T00:00:00.000Z
-    const result = parseParams(schema, { createdAt: ts });
-    expect(result.createdAt).toBeInstanceOf(Date);
-    expect(result.createdAt.getTime()).toBe(ts);
+    expect(Object.keys(parsed).sort()).toEqual(['black__or1', 'owner', 'white__or0']);
+    expect(parsed.white__or0).toBeInstanceOf(RecordId);
+    expect(String(parsed.white__or0)).toBe('player_name:PN_1');
+    expect(parsed.black__or1).toBeInstanceOf(RecordId);
   });
 
-  it('passes through existing Date for dateTime columns', () => {
-    const schema = {
-      createdAt: { type: 'string' as const, optional: false, dateTime: true },
-    };
-    const date = new Date('2024-01-01');
-    const result = parseParams(schema, { createdAt: date });
-    expect(result.createdAt).toBe(date);
+  it('passes a param it cannot resolve through untouched instead of dropping it', () => {
+    const parsed = parseQueryParams(columns, { in: 'anything', limit: 30 });
+    expect(parsed).toEqual({ in: 'anything', limit: 30 });
   });
 
-  it('skips undefined values', () => {
-    const schema = {
-      name: { type: 'string' as const, optional: false },
-      age: { type: 'number' as const, optional: true },
-    };
-    const result = parseParams(schema, { name: 'Alice', age: undefined });
-    expect(result).toEqual({ name: 'Alice' });
-    expect('age' in result).toBe(false);
-  });
-
-  it('throws on invalid recordId value', () => {
-    const schema = {
-      owner: { type: 'string' as const, optional: false, recordId: true },
-    };
-    expect(() => parseParams(schema, { owner: 12345 })).toThrow('Invalid value for owner');
+  it('still types and keeps plain column params, and skips undefined', () => {
+    const parsed = parseQueryParams(columns, {
+      owner: 'user:abc',
+      created_at: '2026-01-01T00:00:00.000Z',
+      result: undefined,
+    });
+    expect(parsed.owner).toBeInstanceOf(RecordId);
+    expect(parsed.created_at).toBeInstanceOf(Date);
+    expect('result' in parsed).toBe(false);
   });
+});
 
-  it('throws on invalid dateTime value', () => {
-    const schema = {
-      createdAt: { type: 'string' as const, optional: false, dateTime: true },
-    };
-    expect(() => parseParams(schema, { createdAt: true })).toThrow(
-      'Invalid value for createdAt'
-    );
+describe('parseParams', () => {
+  // Records keep the old behaviour: a field the schema does not know is dropped
+  // rather than written.
+  it('drops a field that is not a column', () => {
+    const parsed = parseParams(columns, { result: '1-0', nonsense: 'x', white__or0: 'y' });
+    expect(parsed).toEqual({ result: '1-0' });
   });
 });
 
-describe('cleanRecord', () => {
-  const tableSchema = {
-    name: { type: 'string' as const, optional: false },
-    age: { type: 'number' as const, optional: false },
-  };
+describe('absentOptionalColumns', () => {
+  const schema = {
+    title: { type: 'string', optional: false },
+    club: { type: 'string', recordId: true, optional: true },
+    note: { type: 'string', optional: true },
+    _00_local: { type: 'string', optional: true },
+    id: { type: 'string', optional: true },
+  } as any;
 
-  it('keeps schema fields and strips non-schema fields', () => {
-    const record = { id: 'user:1', name: 'Alice', age: 30, _internal: true, computed_score: 99 };
-    const result = cleanRecord(tableSchema, record);
-    expect(result).toEqual({ id: 'user:1', name: 'Alice', age: 30 });
+  // The bug: `UPDATE broadcast:x UNSET club` leaves no `club` key in the body,
+  // and a MERGE that does not name it keeps the stale local value.
+  it('names every optional column the row lacks, as undefined', () => {
+    const cleared = absentOptionalColumns(schema, { id: 'broadcast:x', title: 't', note: 'n' });
+    expect(cleared).toStrictEqual({ club: undefined });
+    expect(Object.prototype.hasOwnProperty.call(cleared, 'club')).toBe(true);
   });
 
-  it('always preserves id even though it is not in schema', () => {
-    const record = { id: 'user:2', extra: 'gone' };
-    const result = cleanRecord(tableSchema, record);
-    expect(result).toEqual({ id: 'user:2' });
+  it('treats an explicit undefined (a decoded NONE) as absent', () => {
+    expect(absentOptionalColumns(schema, { title: 't', club: undefined, note: 'n' })).toStrictEqual(
+      { club: undefined }
+    );
   });
 
-  it('does not coerce or transform values', () => {
-    const date = new Date('2024-01-01');
-    const schema = { created: { type: 'string' as const, optional: false, dateTime: true } };
-    const record = { id: 'x:1', created: date };
-    const result = cleanRecord(schema, record);
-    expect(result.created).toBe(date);
+  it('never names a required column, id, or _00_* bookkeeping', () => {
+    expect(absentOptionalColumns(schema, {})).toStrictEqual({ club: undefined, note: undefined });
   });
 
-  it('returns only id when record has no schema fields', () => {
-    const record = { id: 'user:3', unknown1: 'a', unknown2: 'b' };
-    const result = cleanRecord(tableSchema, record);
-    expect(result).toEqual({ id: 'user:3' });
+  it('keeps a present value, even a falsy one', () => {
+    expect(absentOptionalColumns(schema, { club: null, note: '' })).toStrictEqual({});
   });
 
-  it('handles empty record', () => {
-    const result = cleanRecord(tableSchema, {});
-    expect(result).toEqual({});
+  it('is empty without a schema', () => {
+    expect(absentOptionalColumns(undefined, { a: 1 })).toStrictEqual({});
   });
 });