@spooky-sync/core 0.0.1-canary.38 → 0.0.1-canary.381

package/src/utils/blurhash.ts

@@ -0,0 +1,90 @@
+import { encode, decode, isBlurhashValid } from 'blurhash';
+
+export { encode as encodeBlurhash, decode as decodeBlurhash, isBlurhashValid };
+
+/**
+ * Blurhash generation settings. `true` enables with the defaults below, `false`
+ * disables. Resolution order for a put: per-call option > client config >
+ * default ON. See {@link Sp00kyConfig.blurhash}.
+ */
+export type BlurhashSetting = boolean | BlurhashEncodeOptions;
+
+export interface BlurhashEncodeOptions {
+  /** Horizontal detail components, 1-9. Defaults to 4. */
+  componentX?: number;
+  /** Vertical detail components, 1-9. Defaults to 3. */
+  componentY?: number;
+}
+
+/**
+ * Where an image's blurhash lives: a tiny sidecar object in the same bucket.
+ * Buckets have no per-object metadata channel (`put` is just `.put($content)`),
+ * so the hash for `covers/x_t.webp` is the text object `covers/x_t.webp.bh`.
+ */
+export function blurhashSidecarPath(path: string): string {
+  return `${path}.bh`;
+}
+
+/** Extensions `bucket.put` treats as images worth hashing. */
+export const BLURHASH_IMAGE_EXTENSIONS = [
+  'webp',
+  'png',
+  'jpg',
+  'jpeg',
+  'gif',
+  'avif',
+  'bmp',
+] as const;
+
+const IMAGE_EXT_RE = new RegExp(`\\.(${BLURHASH_IMAGE_EXTENSIONS.join('|')})$`, 'i');
+
+export function isImagePath(path: string): boolean {
+  return IMAGE_EXT_RE.test(path);
+}
+
+/** Longest edge of the downscale the hash is computed from. Blurhash carries at
+ *  most 9x9 DCT components, so anything past ~32px is wasted decode work. */
+const ENCODE_MAX_EDGE = 32;
+
+/**
+ * Decode `content` as an image and compute its blurhash. Browser-only: returns
+ * null (never throws) when image decoding is unavailable (node, workers without
+ * canvas), when the bytes are not a decodable image, or on any other failure —
+ * a missing hash must never break the upload that triggered it.
+ */
+export async function encodeImageToBlurhash(
+  content: string | Uint8Array | Blob,
+  options?: BlurhashEncodeOptions
+): Promise<string | null> {
+  if (typeof createImageBitmap !== 'function') return null;
+  let bitmap: ImageBitmap | null = null;
+  try {
+    const blob =
+      content instanceof Blob
+        ? content
+        : new Blob([content as unknown as BlobPart]);
+    bitmap = await createImageBitmap(blob);
+    const scale = Math.min(1, ENCODE_MAX_EDGE / Math.max(bitmap.width, bitmap.height));
+    const width = Math.max(1, Math.round(bitmap.width * scale));
+    const height = Math.max(1, Math.round(bitmap.height * scale));
+    const canvas =
+      typeof OffscreenCanvas !== 'undefined'
+        ? new OffscreenCanvas(width, height)
+        : typeof document !== 'undefined'
+          ? Object.assign(document.createElement('canvas'), { width, height })
+          : null;
+    if (!canvas) return null;
+    const ctx = canvas.getContext('2d') as
+      | OffscreenCanvasRenderingContext2D
+      | CanvasRenderingContext2D
+      | null;
+    if (!ctx) return null;
+    ctx.drawImage(bitmap, 0, 0, width, height);
+    const { data } = ctx.getImageData(0, 0, width, height);
+    return encode(data, width, height, options?.componentX ?? 4, options?.componentY ?? 3);
+  } catch {
+    return null;
+  } finally {
+    bitmap?.close();
+  }
+}

package/src/utils/edges.ts

@@ -0,0 +1,57 @@
+import { RecordId } from 'surrealdb';
+import type { EdgeDelta, InlineRow } from '../types';
+import { encodeRecordId } from './index';
+
+/**
+ * Reading `_00_list_ref` edges as a LIVE notification delivers them. Pure, so
+ * both the full client and the wasm-free live feed (`@spooky-sync/core/live`)
+ * decode the same wire shape the same way.
+ */
+
+/** The hash a `_00_list_ref` edge belongs to: the id part of its `in` (`_00_query:<hash>`). */
+export function hashOfEdge(value: unknown): string | null {
+  const inId = (value as { in?: unknown } | null)?.in;
+  if (!inId) return null;
+  const str = inId instanceof RecordId ? String(inId.id) : String(inId).replace(/^_00_query:/, '');
+  return str.length > 0 ? str : null;
+}
+
+/**
+ * The row an edge notification carries, when the subscription joined it on.
+ *
+ * Without `FETCH out` the edge's `out` is a record id and there is nothing to
+ * land, so this returns null and the caller falls back to fetching the body.
+ * It also returns null for a row the session may not read (the join yields
+ * `out: null`) and for an edge whose target has been deleted.
+ */
+export function rowOfEdge(value: unknown): InlineRow | null {
+  const edge = value as { out?: unknown; version?: unknown } | null;
+  const out = edge?.out;
+  if (!out || typeof out !== 'object' || Array.isArray(out) || out instanceof RecordId) return null;
+  // Must be a real RecordId: the landing path keys off `row.id` being one, and
+  // would otherwise record the version for a body it never wrote.
+  const rid = (out as { id?: unknown }).id;
+  if (!(rid instanceof RecordId)) return null;
+  const id = encodeRecordId(rid);
+  const version = typeof edge?.version === 'number' ? edge.version : null;
+  if (version === null) return null;
+  return { id, version, record: out as Record<string, unknown> };
+}
+
+/**
+ * The membership change a `CREATE` or `UPDATE` notification reports: the
+ * edge's target joined the view (or its version moved). Null for anything a
+ * re-read must settle instead: a `DELETE` (a row leaving, or every edge of a
+ * view being rewritten), a subquery child (`parent` set), or an edge missing
+ * its target or version. With `FETCH out` the target is the joined row, whose
+ * `id` is the record id.
+ */
+export function deltaOfEdge(action: string, value: unknown): EdgeDelta | null {
+  if (action !== 'CREATE' && action !== 'UPDATE') return null;
+  const hash = hashOfEdge(value);
+  const edge = value as { out?: unknown; version?: unknown; parent?: unknown };
+  if (!hash || edge.parent != null || typeof edge.version !== 'number') return null;
+  const out = edge.out instanceof RecordId ? edge.out : (edge.out as { id?: unknown } | null | undefined)?.id;
+  if (!(out instanceof RecordId)) return null;
+  return { hash, id: encodeRecordId(out), version: edge.version };
+}

package/src/utils/error-classification.test.ts

@@ -0,0 +1,116 @@
+import { describe, it, expect } from 'vitest';
+import { classifySyncError, isRefusedSessionError } from './error-classification';
+
+describe('classifySyncError', () => {
+  it('classifies a proxy with no database behind it as network', () => {
+    // Over HTTP the SDK's message is the response body. These are Traefik's, while
+    // a database restarts or moves in a version upgrade; a rollback would lose the
+    // write.
+    for (const body of ['Bad Gateway', 'Service Unavailable', 'Gateway Timeout', '404 page not found\n']) {
+      expect(classifySyncError(new Error(body))).toBe('network');
+    }
+    // Only as the whole message: a statement's own error can use the words.
+    expect(classifySyncError(new Error('Thrown error: bad gateway to the partner API'))).toBe('application');
+  });
+
+  it('classifies surreal ConnectionUnavailableError as network', () => {
+    // Thrown synchronously by the WS engine's send() while the socket is down
+    // but reconnect hasn't fired. Message contains "connected", not "connection".
+    const err = new Error(
+      'You must be connected to a SurrealDB instance before performing this operation'
+    );
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it('classifies surreal CallTerminatedError as network', () => {
+    const err = new Error(
+      'The call has been terminated because the connection was closed'
+    );
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it.each([
+    'WebSocket connection failed',
+    'fetch failed',
+    'connect ECONNREFUSED 127.0.0.1:8666',
+    'request timed out',
+    'The operation was aborted',
+    'socket hang up',
+  ])('classifies %j as network', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
+  it.each([
+    // SurrealDB optimistic-concurrency conflict: retryable by definition.
+    'Transaction conflict: Resource busy: . This transaction can be retried',
+    // Session state not re-applied yet after the SDK's reconnect handshake.
+    'Specify a namespace to use',
+    'Specify a database to use',
+  ])('classifies transient server state %j as network (retry, never rollback)', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
+  // Verbatim from SurrealDB 3.1.5 and 3.3.4: the write was refused because
+  // sp00ky's own ingest event could not reach the scheduler / SSP. The outbox
+  // must keep these queued, never roll them back (see ingest-outage.integration.test.ts).
+  it.each([
+    // Refused connection and a post past the event TIMEOUT read the same.
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: error sending request for url (http://scheduler:9667/ingest)',
+    'Error while processing event _00_item_delete: There was an error processing a remote HTTP request: error sending request for url (http://host.docker.internal:50466/ingest)',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 500 Internal Server Error',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 502 Bad Gateway',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 503 Service Unavailable',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 408 Request Timeout',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 429 Too Many Requests',
+    'Error while processing event _00_item_mutation: failed to lookup address information: Name or service not known',
+    // An app event that wrote to a synced table carries the inner event.
+    'Error while processing event app_fanout: Error while processing event _00_b_mutation: There was an error processing a remote HTTP request: 503 Service Unavailable',
+  ])('classifies a failed sp00ky ingest post %j as network (retry, never rollback)', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
+  it.each([
+    'Permission denied',
+    // Verbatim from SurrealDB 3.3.4: a record user running a statement it may not.
+    'IAM error: Not enough permissions to perform this action',
+    'There was a problem with the database: record already exists',
+    'Parse error: unexpected token',
+    "Couldn't coerce value for field `title` of `item:1`: Expected `string` but found `1`",
+    // An app's own event: its endpoint, its THROW, its business.
+    'Error while processing event notify_crm: There was an error processing a remote HTTP request: 500 Internal Server Error',
+    'Error while processing event notify_crm: There was an error processing a remote HTTP request: error sending request for url (http://crm.example/hook)',
+    'Error while processing event cf_guard: An error occurred: conformance guard: write refused',
+    // A THROW inside a sp00ky event is not a failed post.
+    'Error while processing event _00_item_mutation: An error occurred: inner',
+    // A 4xx from /ingest no retry fixes: payload rejected, too large, secret mismatch.
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 400 Bad Request',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 401 Unauthorized',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 413 Payload Too Large',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 422 Unprocessable Entity',
+  ])('classifies %j as application', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('application');
+  });
+
+  it('handles non-Error values', () => {
+    expect(classifySyncError('connection reset')).toBe('network');
+    expect(classifySyncError('boom')).toBe('application');
+  });
+});
+
+describe('isRefusedSessionError', () => {
+  it.each([
+    'There was a problem with authentication',
+    'An error occurred: There was a problem with authentication: this session has ended, sign in again',
+    'The session has expired',
+  ])('takes %j as a refused session', (message) => {
+    expect(isRefusedSessionError(new Error(message))).toBe(true);
+  });
+
+  it.each([
+    'There was a problem with the database: Invalid token',
+    'There was a problem with the underlying connection',
+    'Permission denied',
+  ])('does not take %j as one', (message) => {
+    expect(isRefusedSessionError(new Error(message))).toBe(false);
+  });
+});

package/src/utils/error-classification.ts

@@ -1,5 +1,12 @@
 const NETWORK_ERROR_PATTERNS = [
   'connection',
+  // surreal's ConnectionUnavailableError reads "You must be connected to a
+  // SurrealDB instance..." — it contains "connected", not "connection", so it
+  // slips past the pattern above. The WS engine throws it synchronously from
+  // `send()` while the socket is down but reconnect hasn't fired yet, so it's the
+  // canonical error on an idle-dropped socket; classify it as network.
+  'must be connected',
+  'connectionunavailable',
   'timeout',
   'timed out',
   'websocket',
@@ -12,12 +19,67 @@ const NETWORK_ERROR_PATTERNS = [
   'enotfound',
   'epipe',
   'abort',
+  // Transient server-side states that are NOT the mutation's fault and clear
+  // on their own, so the outbox must retry them rather than roll the write
+  // back. Seen 2026-09-08 on whitepawn: a 1200-game import lost 752 games and
+  // ~1500 player_name rows because every queued CREATE that hit one of these
+  // during a DB stall + WebSocket reconnect was classified `application`,
+  // rolled back locally and discarded - silent data loss.
+  //
+  // - SurrealDB optimistic-concurrency conflicts: "Transaction conflict:
+  //   Resource busy: . This transaction can be retried".
+  // - A socket whose namespace/identity has not been re-applied yet after the
+  //   SDK's own reconnect handshake: "Specify a namespace to use" /
+  //   "Specify a database to use". The statement is fine; the session is not
+  //   ready.
+  'transaction conflict',
+  'resource busy',
+  'can be retried',
+  'specify a namespace',
+  'specify a database',
 ];
 
+/**
+ * A write SurrealDB refused because sp00ky's OWN ingest post failed: every
+ * synced table's `_00_<table>_mutation` / `_00_<table>_delete` event posts to
+ * the scheduler or SSP inside the write's transaction (`sp00ky.rs::ingest_post`),
+ * so a scheduler that is down, slow past the event TIMEOUT, or answering 5xx
+ * aborts the write. That is an outage, not the mutation's fault, and it clears
+ * on its own; classified `application` it was rolled back and discarded, so
+ * every write queued during a scheduler restart or a cold SSP was lost.
+ *
+ * Wording on SurrealDB 3.1.5 and 3.3.4 (the event name may be nested inside an
+ * app event that wrote to a synced table):
+ *   "Error while processing event _00_item_mutation: There was an error
+ *    processing a remote HTTP request: error sending request for url (...)"
+ *    (refused connection or TIMEOUT), "...: 503 Service Unavailable" (5xx),
+ *   "Error while processing event _00_item_mutation: failed to lookup address
+ *    information: ..." (DNS).
+ *
+ * Only `_00_` events count: an app's own event posting to its own endpoint
+ * stays `application`, as does a 4xx from `/ingest` other than 408/429 (a
+ * malformed or oversized payload, a secret mismatch), which no retry fixes and
+ * which would otherwise hold the outbox's serial lane forever.
+ */
+const SP00KY_EVENT_POST_FAILED =
+  /error while processing event _00_[^:\s]*: (?:there was an error processing a remote http request: (?:error sending request|5\d\d\b|408\b|429\b)|failed to lookup address information)/;
+
+/**
+ * What the proxy in front of a database answers while the database is away: a
+ * restart, or the seconds of a version upgrade. Over HTTP the SDK's error message
+ * IS the response body, so these arrive as bare words, and an `application`
+ * reading rolls the write back. Anchored: a statement can mention these words,
+ * but SurrealDB never answers with nothing else.
+ */
+const PROXY_UNAVAILABLE = /^(?:bad gateway|service unavailable|gateway timeout|404 page not found)\s*$/;
+
 export function classifySyncError(error: unknown): 'network' | 'application' {
   const message =
     error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
 
+  if (SP00KY_EVENT_POST_FAILED.test(message)) return 'network';
+  if (PROXY_UNAVAILABLE.test(message)) return 'network';
+
   for (const pattern of NETWORK_ERROR_PATTERNS) {
     if (message.includes(pattern)) {
       return 'network';
@@ -26,3 +88,15 @@ export function classifySyncError(error: unknown): 'network' | 'application' {
 
   return 'application';
 }
+
+/**
+ * Whether the server refused the SESSION rather than the statement: its own
+ * wording for a token it will not accept ("There was a problem with
+ * authentication"), which is also what an app's AUTHENTICATE clause and
+ * sp00ky's session guard throw, and a session past its `DURATION FOR SESSION`.
+ */
+export function isRefusedSessionError(error: unknown): boolean {
+  const message =
+    error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
+  return message.includes('problem with authentication') || message.includes('session has expired');
+}

package/src/utils/index.ts

@@ -19,7 +19,7 @@ export const compareRecordIds = (
 };
 
 export const encodeRecordId = (recordId: RecordId<string>): string => {
-  return `${recordId.table.toString()}:${recordId.id}`;
+  return `${recordId.table.name}:${recordId.id}`;
 };
 
 export const extractIdPart = (id: string | RecordId<string>): string => {
@@ -39,7 +39,7 @@ export const extractTablePart = (id: string | RecordId<string>): string => {
   if (typeof id === 'string') {
     return id.split(':')[0];
   }
-  return id.table.toString();
+  return id.table.name;
 };
 
 export const parseRecordIdString = (id: string): RecordId<string> => {
@@ -51,6 +51,17 @@ export function generateId(): string {
   return Uuid.v4().toString().replace(/-/g, '');
 }
 
+/**
+ * 32 random bytes, hex: the secret a page proves it owns its anonymous views
+ * with (`$config.owner` on `fn::query::register`). `getRandomValues` rather
+ * than `randomUUID`, which a page served over plain http does not have.
+ */
+export function mintOwnerSecret(): string {
+  const bytes = new Uint8Array(32);
+  globalThis.crypto.getRandomValues(bytes);
+  return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
+}
+
 export function generateNewTableId<S extends SchemaStructure, T extends TableNames<S>>(
   tableName: T
 ): RecordId {
@@ -76,7 +87,7 @@ export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames
     const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
     if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
       if (encoded[field] instanceof RecordId) {
-        encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
+        encoded[field] = `${encoded[field].table.name}:${encoded[field].id}`;
       } else if (
         relation &&
         (encoded[field] instanceof Object || Array.isArray(encoded[field]))
@@ -97,15 +108,25 @@ export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames
 
 // ==================== TIME/DURATION UTILITIES ====================
 
+/**
+ * Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
+ * releases expose it under `milliseconds` (current) or the older private
+ * `_milliseconds` field, so read whichever is set; returns 0 when neither is.
+ */
+function durationMillis(duration: Duration): number {
+  const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
+  return Number(d.milliseconds || d._milliseconds || 0);
+}
+
 /**
  * Parse duration string or Duration object to milliseconds
  */
 export function parseDuration(duration: QueryTimeToLive | Duration): number {
   if (duration instanceof Duration) {
-    const ms = (duration as any).milliseconds || (duration as any)._milliseconds;
-    if (ms) return Number(ms);
+    const ms = durationMillis(duration);
+    if (ms) return ms;
     const str = duration.toString();
-    if (str !== '[object Object]') return parseDuration(str as any);
+    if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
     return 600000;
   }
 
@@ -137,6 +158,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
   return new Uint8Array(buffer);
 }
 
+// ==================== TEXT UTILITIES ====================
+
+/**
+ * Convert plain text to simple HTML paragraphs.
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
+ */
+export function textToHtml(text: string): string {
+  return text
+    .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+    .split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
+}
+
 // ==================== DATABASE UTILITIES ====================
 
 /**
@@ -154,6 +187,9 @@ export async function withRetry<T>(
       return await operation();
     } catch (err: any) {
       lastError = err;
+      // A deadline expiry is explicitly not retryable: the op is still running
+      // in the engine, and a retry only queues a second copy behind it.
+      if (err?.retryable === false) throw err;
       if (
         err?.message?.includes('Can not open transaction') ||
         err?.message?.includes('transaction') ||
@@ -177,3 +213,44 @@ export async function withRetry<T>(
   }
   throw lastError;
 }
+
+/**
+ * Reject after `timeoutMs` if `promise` hasn't settled.
+ *
+ * Exists because a WebSocket RPC has no deadline of its own: the SurrealDB SDK
+ * parks each call in a pending map and only rejects it when the socket reports
+ * a close. On a half-open socket (peer gone, no `close` event, `readyState`
+ * still OPEN) the call never settles at all, which wedges anything serialized
+ * behind it.
+ *
+ * `message` MUST contain "timed out" so `classifySyncError` treats the
+ * rejection as `network` — that's what makes the sync queues retry the
+ * operation instead of rolling the mutation back as an application error.
+ *
+ * Non-positive `timeoutMs` returns `promise` unchanged. The underlying promise
+ * is NOT cancelled (nothing can cancel an in-flight RPC); its eventual
+ * settlement is absorbed, so attach no expectations to it after a timeout.
+ */
+export function withTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+  message: string | (() => Error)
+): Promise<T> {
+  if (!(timeoutMs > 0)) return promise;
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(
+      () => reject(typeof message === 'function' ? message() : new Error(message)),
+      timeoutMs
+    );
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    );
+  });
+}