@foretag/tanstack-db-surrealdb 0.5.7 → 0.6.0

package/README.md

@@ -1,137 +1,313 @@
-# TanstackDB SurrealDB Collections
+# @foretag/tanstack-db-surrealdb
 
-Add Offline / Local First Caching & Syncing to your SurrealDB app with TanstackDB and Loro (CRDTs). Designed for SurrealDB v3 and SurrealDB JS SDK v2.
+TanStack DB collection adapter for SurrealDB JS with:
 
-- Local / Offline first applications with TanstackDB and Loro
-- High performance with Low resource consumption
-- Works with Web, Desktop or Native (WASM based)
-- Support for React, Svelte, Vue and any Framework!
+- Realtime replication (`LIVE`)
+- Local-first writes
+- Optional E2EE envelopes (`version/algorithm/key_id/nonce/ciphertext`)
+- Optional Loro CRDT replication (`json`, `richtext`)
+- Query-driven sync modes (`eager`, `on-demand`, `progressive`)
 
-## Installation
+## Install
 
-### NPM
 ```sh
-# NPM
 npm install @foretag/tanstack-db-surrealdb
-# Bun
-bun install @foretag/tanstack-db-surrealdb
+# or
+bun add @foretag/tanstack-db-surrealdb
 ```
 
-## Usage
+## Quick Start
+
 ```ts
-// db.ts
-import { QueryClient } from '@tanstack/svelte-query'; // Can also use '@tanstack/react-query' etc.
+import { createCollection } from '@tanstack/db';
+import { QueryClient } from '@tanstack/query-core';
 import { Surreal } from 'surrealdb';
+import { surrealCollectionOptions } from '@foretag/tanstack-db-surrealdb';
 
-export const db = new Surreal();
-await db.connect('ws://localhost:8000/rpc');
-await db.use({ ns: 'ns', db: 'db' });
+const db = new Surreal();
+const queryClient = new QueryClient();
 
-export const queryClient = new QueryClient();
+type Product = { id: string; name: string; price: number };
 
-// collections/products.ts
-import { eq } from '@tanstack/db';
-import { db, queryClient } from '../db';
-import { createCollection } from '@tanstack/db';
-import { surrealCollectionOptions } from '@foretag/tanstack-db-surrealdb';
-import { useLiveQuery } from '@tanstack/react-db';
-
-// Collection Type, could also be generated
-type Product = {
-	id: string;
-	name: string;
-	price: number;
-	store?: string;
+export const products = createCollection(
+  surrealCollectionOptions<Product>({
+    db,
+    table: { name: 'product' },
+    queryClient,
+    queryKey: ['product'],
+    syncMode: 'eager',
+  }),
+);
+```
+
+## Adapter API
+
+```ts
+type SurrealCollectionOptions<T> = {
+  db: Surreal;
+  table: Table | { name: string; relation?: boolean } | string;
+  queryClient: QueryClient;
+  queryKey: readonly unknown[];
+  syncMode?: 'eager' | 'on-demand' | 'progressive';
+  e2ee?: {
+    enabled: boolean;
+    crypto: CryptoProvider;
+    aad?: (ctx: { table: string; id: string; kind: 'base'|'update'|'snapshot'; baseTable?: string }) => Uint8Array;
+  };
+  crdt?: {
+    enabled: boolean;
+    profile: 'json' | 'richtext';
+    updatesTable: Table | { name: string } | string;
+    snapshotsTable?: Table | { name: string } | string;
+    // Optional overrides. If omitted, adapter uses built-in handlers for `profile`.
+    materialize?: (doc: LoroDoc, id: string) => T;
+    applyLocalChange?: (doc: LoroDoc, change: { type: 'insert'|'update'|'delete'; value: T }) => void;
+    persistMaterializedView?: boolean;
+    actor?: string | ((ctx: { id: string; change?: { type: 'insert'|'update'|'delete'; value: T } }) => string | undefined);
+    localActorId?: string; // deprecated
+  };
 };
+```
 
-export const products = createCollection(
-	surrealCollectionOptions<Product>({
-		db,
-		queryKey: ['products'],
-		queryClient,
-		useLoro: true, // Optional if you need CRDTs
-		table: { name: 'products' },
-	}),
-)
-
-const data = useLiveQuery((q) =>
-	q
-		.from({ products })
-		.where(({ products }) => eq(products.store, '123'))
-)
+## E2EE
+
+Envelope fields stored in Surreal records:
+
+```ts
+type EncryptedEnvelope = {
+  version: number;
+  algorithm: string;
+  key_id: string;
+  nonce: string;
+  ciphertext: string;
+};
 ```
 
-For syncModes, please see [Example](https://github.com/ForetagInc/tanstack-db-surrealdb/blob/master/examples/syncMode.ts)
+Default AAD:
 
-## Vite / Next.JS
+- Base records: `<table>:<record_id>`
+- CRDT updates/snapshots: `<updates_or_snapshots_table>:<base_table>:<doc_id>`
 
-### Vite
-```sh
-bun install vite-plugin-wasm vite-plugin-top-level-await -D
+Included provider:
+
+- `WebCryptoAESGCM` (`AES-256-GCM`, versioned envelope)
+
+## CRDT Profiles
+
+CRDT is managed by profile by default:
+
+- `profile: 'json'` uses built-in JSON handlers
+- `profile: 'richtext'` uses built-in richtext handlers
+
+Advanced overrides are still available:
+
+- `createLoroProfile('json' | 'richtext')`
+- `materialize` and `applyLocalChange` in `crdt` options
+
+For CRDT loop-prevention metadata, prefer `crdt.actor` so actor identity can be resolved per doc/write. `localActorId` remains only for backwards compatibility.
+
+## CRDT Table Requirements
+
+For `crdt.enabled: true`, users must provide:
+
+- Base table (`table`) for record identity and optional materialized metadata.
+- Updates table (`crdt.updatesTable`) as append-only CRDT log.
+
+Optional:
+
+- Snapshots table (`crdt.snapshotsTable`) for compaction and faster hydration.
+
+If `crdt.updatesTable` is missing, CRDT mode cannot function.
+
+## SQL Templates
+
+### Plain
+
+```sql
+DEFINE TABLE note SCHEMAFULL;
+DEFINE FIELD title ON note TYPE string;
+DEFINE FIELD body ON note TYPE string;
+DEFINE FIELD updated_at ON note TYPE datetime VALUE time::now();
+DEFINE INDEX note_updated ON note FIELDS updated_at;
+```
+
+### E2EE-only
+
+```sql
+DEFINE TABLE secret_note SCHEMAFULL;
+DEFINE FIELD owner ON secret_note TYPE record<account>;
+DEFINE FIELD updated_at ON secret_note TYPE datetime;
+DEFINE FIELD version ON secret_note TYPE int;
+DEFINE FIELD algorithm ON secret_note TYPE string;
+DEFINE FIELD key_id ON secret_note TYPE string;
+DEFINE FIELD nonce ON secret_note TYPE string;
+DEFINE FIELD ciphertext ON secret_note TYPE string;
+DEFINE INDEX secret_note_owner_updated ON secret_note FIELDS owner, updated_at;
+```
+
+### CRDT-only
+
+```sql
+DEFINE TABLE doc SCHEMAFULL;
+DEFINE FIELD owner ON doc TYPE record<account>;
+DEFINE FIELD updated_at ON doc TYPE datetime;
+DEFINE INDEX doc_owner_updated ON doc FIELDS owner, updated_at;
+
+-- Necessary for CRDT updates
+DEFINE TABLE crdt_update SCHEMAFULL;
+DEFINE FIELD doc ON crdt_update TYPE record<doc>;
+DEFINE FIELD ts ON crdt_update TYPE datetime;
+DEFINE FIELD update_bytes ON crdt_update TYPE string;
+DEFINE FIELD actor ON crdt_update TYPE string;
+DEFINE INDEX crdt_doc_ts ON crdt_update FIELDS doc, ts;
+
+DEFINE TABLE crdt_snapshot SCHEMAFULL;
+DEFINE FIELD doc ON crdt_snapshot TYPE record<doc>;
+DEFINE FIELD ts ON crdt_snapshot TYPE datetime;
+DEFINE FIELD snapshot_bytes ON crdt_snapshot TYPE string;
+DEFINE INDEX snap_doc_ts ON crdt_snapshot FIELDS doc, ts;
 ```
 
-`vite.config.ts`
+### CRDT + E2EE
+
+```sql
+DEFINE TABLE secure_doc SCHEMAFULL;
+DEFINE FIELD owner ON secure_doc TYPE record<account>;
+DEFINE FIELD updated_at ON secure_doc TYPE datetime;
+DEFINE INDEX secure_doc_owner_updated ON secure_doc FIELDS owner, updated_at;
+
+DEFINE TABLE crdt_update SCHEMAFULL;
+DEFINE FIELD doc ON crdt_update TYPE record<secure_doc>;
+DEFINE FIELD ts ON crdt_update TYPE datetime;
+DEFINE FIELD actor ON crdt_update TYPE string;
+DEFINE FIELD version ON crdt_update TYPE int;
+DEFINE FIELD algorithm ON crdt_update TYPE string;
+DEFINE FIELD key_id ON crdt_update TYPE string;
+DEFINE FIELD nonce ON crdt_update TYPE string;
+DEFINE FIELD ciphertext ON crdt_update TYPE string;
+DEFINE INDEX crdt_doc_ts ON crdt_update FIELDS doc, ts;
+```
+
+If a single `crdt_update` table is shared across multiple base tables, use a union type such as `record<doc> | record<sheet>`.
+
+## Usage Snippets
+
+### E2EE-only secret table
 
 ```ts
-// Plugins
-import wasm from 'vite-plugin-wasm';
-import topLevelAwait from 'vite-plugin-top-level-await';
+const provider = await WebCryptoAESGCM.fromRawKey(rawKey, { kid: 'org-key-2026-01' });
 
-export default defineConfig({
-  plugins: [...otherConfigures, wasm(), topLevelAwait()],
-});
+const secrets = createCollection(
+  surrealCollectionOptions<{ id: string; title: string; body: string }>({
+    db,
+    table: { name: 'secret_note' },
+    queryClient,
+    queryKey: ['secret-note'],
+    syncMode: 'eager',
+    e2ee: { enabled: true, crypto: provider },
+  }),
+);
 ```
 
-### NextJS
-`next.config.js`
+### CRDT richtext docs
 
 ```ts
-module.exports = {
-	webpack: function (config) {
-		config.experiments = {
-			layers: true,
-			asyncWebAssembly: true,
-		};
-		return config;
-	},
+const docs = createCollection(
+  surrealCollectionOptions<{ id: string; content: string; title?: string }>({
+    db,
+    table: { name: 'doc' },
+    queryClient,
+    queryKey: ['doc'],
+    syncMode: 'on-demand',
+    crdt: {
+      enabled: true,
+      profile: 'richtext',
+      updatesTable: { name: 'crdt_update' },
+      snapshotsTable: { name: 'crdt_snapshot' },
+      actor: ({ id }) => id.startsWith('team-a') ? 'device:team-a:abc' : 'device:team-b:abc',
+    },
+  }),
+);
+```
+
+### RecordId model example
+
+```ts
+import { RecordId } from 'surrealdb';
+
+type CalendarEvent = {
+  id: RecordId<'calendar_event'>;
+  owner: RecordId<'account'>;
+  title: string;
+  start_at: string;
 };
+
+await calendarEvents.insert({
+  id: new RecordId('calendar_event', 'evt-001'),
+  owner: new RecordId('account', 'user-123'),
+  title: 'Planning',
+  start_at: '2026-02-23T10:00:00.000Z',
+});
 ```
 
-## CRDTs
+Full runnable example: `examples/record-id.ts`.
 
-If you need to use CRDTs for your application consider adding the following fields to the specific tables and set `useLoro: true` for the respective table. Please note these fields are opinionated, therefore fixed and required:
+### On-demand drive listing (query-driven)
 
-```sql
-DEFINE FIELD OVERWRITE sync_deleted ON <table>
-	TYPE bool
-	DEFAULT false
-	COMMENT 'Tombstone for CRDTs';
-
-DEFINE FIELD OVERWRITE updated_at ON <table>
-	TYPE datetime
-	VALUE time::now();
+```ts
+import { createLiveQueryCollection, eq } from '@tanstack/db';
+
+const files = createCollection(
+  surrealCollectionOptions<{ id: string; owner: string; updated_at: string; name: string }>({
+    db,
+    table: { name: 'file' },
+    queryClient,
+    queryKey: ['file'],
+    syncMode: 'on-demand',
+  }),
+);
+
+const ownerFiles = createLiveQueryCollection((q) =>
+  q
+    .from({ files })
+    .where(({ files }) => eq(files.owner, 'account:abc'))
+    .select(({ files }) => files),
+);
+
+await ownerFiles.preload();
 ```
 
-> While using SurrealDB as a Web Database, please remember to allow `SELECT` & `UPDATE` permissions for the `sync_deleted` and `updated_at` fields for the respective access.
+## Key Wrapping / Multi-Principal Access
+
+This adapter expects key management to be provided by your app or KMS. For production shared access (users, teams, orgs), keep using wrapped keys:
+
+- Encrypt entity data with a data key.
+- Wrap that data key for each authorized principal (user/team/service/device).
+- Resolve the active key by `kid` at decrypt time.
+- Rotate by issuing a new `kid` and re-wrapping/re-encrypting progressively.
+
+The adapter consumes derived keys through `CryptoProvider`; it does not manage wrapping policy for you.
+
+## Testing
+
+Unit tests (`bun test`) cover:
+
+- id/query translation behavior
+- modern eager + on-demand sync controls
+- E2EE envelope/AAD behavior
+- CRDT update append, snapshot hydration, and actor loop prevention
 
-## FAQ
+Real SurrealDB integration tests are available and run against a live instance:
 
-<details>
-	<summary><strong>When do I need CRDTs?</strong></summary>
-	<p>In most cases Tanstack DB is sufficient to handle CRUD operations. However, if you need to implement a distributed system that is offline first, CRDTs are the way to go. Think: Google Docs, Figma Pages, Notion Blocks etc. We recommend you check out <a href='https://www.loro.dev/' target='_blank'>Loro</a> for a deeper understanding.</p>
-</details>
+1. Copy `.env.example` to `.env` and fill connection/auth values.
+2. Run `bun run test:integration`.
 
-<details>
-	<summary><strong>How do I achieve type safety?</strong></summary>
-	<p>Using Codegen tools that generate types from your SurrealDB Schema, this means you don't have to manually maintain types for each Collection.</p>
-</details>
+Required env:
 
-<details>
-	<summary><strong>Can I use GraphQL alongside this Library?</strong></summary>
-	<p>GraphQL workflow is in the works as SurrealDB's own implementation of the GraphQL protocol matures, we'll be able to provide a seamless integration. Since this library only targets TanstackDB, you can also use GraphQL for direct querying through Tanstack Query.</p>
-</details>
+- `SURREAL_URL`
+- `SURREAL_NAMESPACE`
+- `SURREAL_DATABASE`
+- `SURREAL_USERNAME`
+- `SURREAL_PASSWORD`
 
-<details>
-	<summary><strong>Can I reduce the package sizes?</strong></summary>
-	<p>They can be reduced, but these steps are very unique based on use-case. Loro ships a WASM binary thats 1 MB Gzipped in size, it's one of the tradeoffs of using this approach. The maintainers up-stream are working on reducing the size of the WASM binary.</p>
-</details>
+`SURREAL_REQUIRE_LIVE=true` (default) enforces LIVE query assertions; set it to `false` if you intentionally use a connection without LIVE support.

package/dist/index.d.mts

@@ -1,12 +1,67 @@
 import { StandardSchemaV1 } from '@standard-schema/spec';
-import { IR, UtilsRecord, OperationConfig, Transaction, CollectionConfig } from '@tanstack/db';
-import { Container } from 'loro-crdt';
-import { RecordId, Surreal } from 'surrealdb';
+import { CollectionConfig, UtilsRecord, LoadSubsetOptions, OperationConfig, Transaction } from '@tanstack/db';
+import { Table, Surreal, RecordId } from 'surrealdb';
 import { QueryClient } from '@tanstack/query-core';
+import { LoroDoc } from 'loro-crdt';
 
-type RecordIdLike = RecordId | string;
-declare const eqRecordId: (field: unknown, value: RecordIdLike) => IR.BasicExpression<boolean>;
+type EncryptInput = {
+    plaintext: Bytes;
+    aad?: Bytes;
+    v?: number;
+    alg?: string;
+    kid?: string;
+};
+type DecryptInput = {
+    envelope: EncryptedEnvelope;
+    aad?: Bytes;
+};
+interface CryptoProvider {
+    encrypt(input: EncryptInput): Promise<EncryptedEnvelope>;
+    decrypt(input: DecryptInput): Promise<Bytes>;
+}
+
+/**
+ * @deprecated Use SurrealE2EEOptions from ../types.
+ */
+interface E2EEConfig<TItem extends object> {
+    enabled: boolean;
+    serialize?: (item: TItem) => Bytes;
+    deserialize?: (bytes: Bytes) => TItem;
+    crypto: CryptoProvider;
+    fields?: {
+        ciphertext: string;
+        nonce: string;
+        version: string;
+    };
+    aad?: (ctx: {
+        table: string;
+        id: string;
+    }) => Bytes;
+}
+type EncryptionEnvelope = EncryptedEnvelope;
+type EncryptionAADContext = AADContext;
+type AdapterE2EEOptions = SurrealE2EEOptions;
+
+type KeyResolver = (kid: string) => Promise<CryptoKey> | CryptoKey;
+type WebCryptoAESGCMOptions = {
+    alg?: string;
+    version?: number;
+    kid?: string;
+    resolveKey?: KeyResolver;
+};
+declare class WebCryptoAESGCM implements CryptoProvider {
+    private readonly alg;
+    private readonly version;
+    private readonly kid;
+    private readonly resolveKey;
+    constructor(key: CryptoKey, options?: WebCryptoAESGCMOptions);
+    static fromRawKey(rawKey: Bytes, options?: Omit<WebCryptoAESGCMOptions, 'resolveKey'>): Promise<WebCryptoAESGCM>;
+    private keyFor;
+    encrypt(input: EncryptInput): Promise<EncryptedEnvelope>;
+    decrypt({ envelope, aad }: DecryptInput): Promise<Bytes>;
+}
 
+type Bytes = Uint8Array;
 type WithId<T> = T & {
     id: string | RecordId;
 };
@@ -18,36 +73,96 @@ type TableOptions = {
     name: string;
     relation?: boolean;
 };
-type SyncMode = 'eager' | 'on-demand';
-type SurrealCollectionConfig = {
-    id?: string;
+type TableLike = Table | TableOptions | string;
+type SurrealSubset = LoadSubsetOptions;
+type EncryptedEnvelope = {
+    v: number;
+    alg: string;
+    kid: string;
+    n: string;
+    ct: string;
+};
+type EnvelopeKind = 'base' | 'update' | 'snapshot';
+type AADContext = {
+    table: string;
+    id: string;
+    kind: EnvelopeKind;
+    baseTable?: string;
+};
+type SurrealE2EEOptions = {
+    enabled: boolean;
+    crypto: CryptoProvider;
+    aad?: (ctx: AADContext) => Bytes;
+};
+type LocalChange<T> = {
+    type: 'insert' | 'update' | 'delete';
+    value: T;
+};
+type CRDTActorContext<T> = {
+    id: string;
+    change?: LocalChange<T>;
+};
+type SurrealCRDTOptions<T extends object> = {
+    enabled: boolean;
+    profile: 'json' | 'richtext';
+    updatesTable: TableLike;
+    snapshotsTable?: TableLike;
+    materialize?: (doc: LoroDoc, id: string) => T;
+    applyLocalChange?: (doc: LoroDoc, change: LocalChange<T>) => void;
+    persistMaterializedView?: boolean;
+    actor?: string | ((ctx: CRDTActorContext<T>) => string | undefined);
+    /** @deprecated Use `actor` instead. */
+    localActorId?: string;
+};
+type AdapterSyncMode = 'eager' | 'on-demand' | 'progressive';
+type SurrealCollectionOptions<T extends object> = Omit<CollectionConfig<T>, 'onInsert' | 'onUpdate' | 'onDelete' | 'sync' | 'getKey' | 'syncMode'> & {
     db: Surreal;
-    table: TableOptions;
-    syncMode?: SyncMode;
+    table: TableLike;
     queryKey: readonly unknown[];
     queryClient: QueryClient;
-    useLoro?: boolean;
-    onError?: (e: unknown) => void;
+    syncMode?: AdapterSyncMode;
+    e2ee?: SurrealE2EEOptions;
+    crdt?: SurrealCRDTOptions<T>;
+    onError?: (error: unknown) => void;
+};
+type SurrealCollectionOptionsReturn<T extends {
+    id: string | RecordId;
+}> = CollectionConfig<T, string, StandardSchemaV1<Omit<T, 'id'> & {
+    id?: T['id'];
+}, T>, UtilsRecord> & {
+    schema: StandardSchemaV1<Omit<T, 'id'> & {
+        id?: T['id'];
+    }, T>;
+    utils: UtilsRecord;
 };
 
 declare const toRecordKeyString: (rid: RecordId | string) => string;
 
+interface CRDTProfileAdapter<T extends object> {
+    materialize: (doc: LoroDoc, id: string) => T;
+    applyLocalChange: (doc: LoroDoc, change: LocalChange<T>) => void;
+}
+
+type LoroProfile = 'json' | 'richtext';
+declare const materializeLoroJson: <T extends object>(doc: LoroDoc, id: string) => T;
+declare const applyLoroJsonChange: <T extends object>(doc: LoroDoc, change: LocalChange<T>) => void;
+declare const materializeLoroRichtext: <T extends object>(doc: LoroDoc, id: string) => T;
+declare const applyLoroRichtextChange: <T extends object>(doc: LoroDoc, change: LocalChange<T>) => void;
+declare const createLoroProfile: <T extends object = Record<string, unknown>>(profile: LoroProfile) => CRDTProfileAdapter<T>;
+
 type MutationInput<T extends {
     id: string | RecordId;
 }> = Omit<T, 'id'> & {
     id?: T['id'];
 };
-
+declare function surrealCollectionOptions<T extends SyncedTable<object>>(config: SurrealCollectionOptions<T>): CollectionConfig<T, string, StandardSchemaV1<MutationInput<T>, T>, UtilsRecord> & {
+    schema: StandardSchemaV1<MutationInput<T>, T>;
+    utils: UtilsRecord;
+};
 declare module '@tanstack/db' {
     interface Collection<T extends object = Record<string, unknown>, TKey extends string | number = string | number, TUtils extends UtilsRecord = UtilsRecord, TSchema extends StandardSchemaV1 = StandardSchemaV1, TInsertInput extends object = T> {
         delete(keys: Array<TKey | RecordId | string> | TKey | RecordId | string, config?: OperationConfig): Transaction<any>;
     }
 }
-declare function surrealCollectionOptions<T extends SyncedTable<object>, S extends Record<string, Container> = {
-    [k: string]: never;
-}>({ id, useLoro, onError, db, queryClient, queryKey, syncMode, ...config }: SurrealCollectionConfig): CollectionConfig<T, string, StandardSchemaV1<MutationInput<T>, T>, UtilsRecord> & {
-    schema: StandardSchemaV1<MutationInput<T>, T>;
-    utils: UtilsRecord;
-};
 
-export { eqRecordId, surrealCollectionOptions, toRecordKeyString };
+export { type AADContext, type AdapterE2EEOptions, type AdapterSyncMode, type Bytes, type CRDTActorContext, type CryptoProvider, type DecryptInput, type E2EEConfig, type EncryptInput, type EncryptedEnvelope, type EncryptionAADContext, type EncryptionEnvelope, type EnvelopeKind, type LocalChange, type LoroProfile, type SurrealCRDTOptions, type SurrealCollectionOptions, type SurrealCollectionOptionsReturn, type SurrealE2EEOptions, type SurrealSubset, type SyncedTable, type TableLike, type TableOptions, WebCryptoAESGCM, type WithId, applyLoroJsonChange, applyLoroRichtextChange, createLoroProfile, materializeLoroJson, materializeLoroRichtext, surrealCollectionOptions, toRecordKeyString };