@tallyui/storage-sqlite 0.2.0

package/src/storage-instance.test.ts

@@ -0,0 +1,265 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import type { RxJsonSchema, RxDocumentData, RxStorageInstanceCreationParams } from 'rxdb';
+import { createMockSQLiteDatabase } from './mock-sqlite';
+import { getRxStorageSQLite } from './rx-storage-sqlite';
+import type { SQLiteStorageSettings } from './types';
+import type { RxStorageInstanceSQLite } from './storage-instance';
+
+// Simple doc type for testing
+interface TestDoc {
+  id: string;
+  name: string;
+  age: number;
+}
+
+const testSchema: RxJsonSchema<RxDocumentData<TestDoc>> = {
+  version: 0,
+  primaryKey: 'id',
+  type: 'object',
+  properties: {
+    id: { type: 'string', maxLength: 100 },
+    name: { type: 'string' },
+    age: { type: 'number' },
+    _deleted: { type: 'boolean' },
+    _rev: { type: 'string' },
+    _meta: {
+      type: 'object',
+      properties: {
+        lwt: { type: 'number' },
+      },
+      required: ['lwt'],
+    },
+    _attachments: { type: 'object' },
+  },
+  required: ['id', 'name', 'age', '_deleted', '_rev', '_meta', '_attachments'],
+};
+
+function makeDoc(
+  overrides: Partial<TestDoc & RxDocumentData<TestDoc>> = {},
+): RxDocumentData<TestDoc> {
+  return {
+    id: 'doc1',
+    name: 'Alice',
+    age: 30,
+    _deleted: false,
+    _rev: '1-abc',
+    _meta: { lwt: Date.now() },
+    _attachments: {},
+    ...overrides,
+  } as RxDocumentData<TestDoc>;
+}
+
+describe('RxStorageInstanceSQLite', () => {
+  let storage: ReturnType<typeof getRxStorageSQLite>;
+  let instance: RxStorageInstanceSQLite<TestDoc>;
+
+  beforeEach(async () => {
+    const db = createMockSQLiteDatabase();
+    storage = getRxStorageSQLite(db);
+    instance = (await storage.createStorageInstance({
+      databaseInstanceToken: 'test-token',
+      databaseName: 'testdb',
+      collectionName: 'testcol',
+      schema: testSchema,
+      options: { database: db },
+      multiInstance: false,
+      devMode: false,
+    })) as RxStorageInstanceSQLite<TestDoc>;
+  });
+
+  afterEach(async () => {
+    await instance.close();
+  });
+
+  describe('bulkWrite', () => {
+    it('should insert a new document', async () => {
+      const doc = makeDoc({ id: 'doc1', name: 'Alice', age: 30 });
+      const result = await instance.bulkWrite(
+        [{ document: doc }],
+        'test-insert',
+      );
+      expect(result.error).toHaveLength(0);
+
+      // Verify we can read it back
+      const found = await instance.findDocumentsById(['doc1'], false);
+      expect(found).toHaveLength(1);
+      expect(found[0].name).toBe('Alice');
+    });
+
+    it('should insert multiple documents', async () => {
+      const doc1 = makeDoc({ id: 'doc1', name: 'Alice', age: 30, _meta: { lwt: 1000 } });
+      const doc2 = makeDoc({ id: 'doc2', name: 'Bob', age: 25, _meta: { lwt: 1001 } });
+      const result = await instance.bulkWrite(
+        [{ document: doc1 }, { document: doc2 }],
+        'test-multi-insert',
+      );
+      expect(result.error).toHaveLength(0);
+
+      const found = await instance.findDocumentsById(['doc1', 'doc2'], false);
+      expect(found).toHaveLength(2);
+    });
+
+    it('should update an existing document when previous matches', async () => {
+      const doc1 = makeDoc({ id: 'doc1', name: 'Alice', _rev: '1-aaa', _meta: { lwt: 1000 } });
+      await instance.bulkWrite([{ document: doc1 }], 'insert');
+
+      const updated = makeDoc({
+        id: 'doc1',
+        name: 'Alice Updated',
+        _rev: '2-bbb',
+        _meta: { lwt: 2000 },
+      });
+      const result = await instance.bulkWrite(
+        [{ previous: doc1, document: updated }],
+        'update',
+      );
+      expect(result.error).toHaveLength(0);
+
+      const found = await instance.findDocumentsById(['doc1'], false);
+      expect(found).toHaveLength(1);
+      expect(found[0].name).toBe('Alice Updated');
+      expect(found[0]._rev).toBe('2-bbb');
+    });
+
+    it('should return a conflict error when previous rev does not match', async () => {
+      const doc1 = makeDoc({ id: 'doc1', name: 'Alice', _rev: '1-aaa', _meta: { lwt: 1000 } });
+      await instance.bulkWrite([{ document: doc1 }], 'insert');
+
+      const wrongPrevious = makeDoc({
+        id: 'doc1',
+        name: 'Alice',
+        _rev: '1-wrong',
+        _meta: { lwt: 1000 },
+      });
+      const updated = makeDoc({
+        id: 'doc1',
+        name: 'Alice Updated',
+        _rev: '2-bbb',
+        _meta: { lwt: 2000 },
+      });
+      const result = await instance.bulkWrite(
+        [{ previous: wrongPrevious, document: updated }],
+        'conflict-update',
+      );
+      expect(result.error).toHaveLength(1);
+      expect(result.error[0].status).toBe(409);
+    });
+
+    it('should emit events on the change stream', async () => {
+      const events: any[] = [];
+      const sub = instance.changeStream().subscribe((eventBulk) => {
+        events.push(...eventBulk.events);
+      });
+
+      const doc = makeDoc({ id: 'doc1', name: 'Alice', _meta: { lwt: 1000 } });
+      await instance.bulkWrite([{ document: doc }], 'insert');
+
+      expect(events).toHaveLength(1);
+      expect(events[0].operation).toBe('INSERT');
+      expect(events[0].documentId).toBe('doc1');
+
+      sub.unsubscribe();
+    });
+  });
+
+  describe('findDocumentsById', () => {
+    it('should return empty array for non-existent ids', async () => {
+      const found = await instance.findDocumentsById(['nonexistent'], false);
+      expect(found).toHaveLength(0);
+    });
+
+    it('should not return deleted documents when withDeleted is false', async () => {
+      const doc = makeDoc({ id: 'doc1', _deleted: true, _meta: { lwt: 1000 } });
+      await instance.bulkWrite([{ document: doc }], 'insert-deleted');
+
+      const found = await instance.findDocumentsById(['doc1'], false);
+      expect(found).toHaveLength(0);
+    });
+
+    it('should return deleted documents when withDeleted is true', async () => {
+      const doc = makeDoc({ id: 'doc1', _deleted: true, _meta: { lwt: 1000 } });
+      await instance.bulkWrite([{ document: doc }], 'insert-deleted');
+
+      const found = await instance.findDocumentsById(['doc1'], true);
+      expect(found).toHaveLength(1);
+    });
+  });
+
+  describe('getChangedDocumentsSince', () => {
+    it('should return documents ordered by lwt', async () => {
+      const doc1 = makeDoc({ id: 'a', name: 'First', _meta: { lwt: 100 } });
+      const doc2 = makeDoc({ id: 'b', name: 'Second', _meta: { lwt: 200 } });
+      const doc3 = makeDoc({ id: 'c', name: 'Third', _meta: { lwt: 300 } });
+      await instance.bulkWrite(
+        [{ document: doc1 }, { document: doc2 }, { document: doc3 }],
+        'insert-all',
+      );
+
+      const result = await instance.getChangedDocumentsSince(10);
+      expect(result.documents).toHaveLength(3);
+      expect(result.documents[0].name).toBe('First');
+      expect(result.documents[2].name).toBe('Third');
+      expect(result.checkpoint).toEqual({ id: 'c', lwt: 300 });
+    });
+
+    it('should respect the checkpoint for pagination', async () => {
+      const doc1 = makeDoc({ id: 'a', name: 'First', _meta: { lwt: 100 } });
+      const doc2 = makeDoc({ id: 'b', name: 'Second', _meta: { lwt: 200 } });
+      const doc3 = makeDoc({ id: 'c', name: 'Third', _meta: { lwt: 300 } });
+      await instance.bulkWrite(
+        [{ document: doc1 }, { document: doc2 }, { document: doc3 }],
+        'insert-all',
+      );
+
+      const result = await instance.getChangedDocumentsSince(10, { id: 'a', lwt: 100 });
+      expect(result.documents).toHaveLength(2);
+      expect(result.documents[0].name).toBe('Second');
+      expect(result.documents[1].name).toBe('Third');
+    });
+  });
+
+  describe('cleanup', () => {
+    it('should remove deleted documents older than minimumDeletedTime', async () => {
+      const oldLwt = Date.now() - 100000;
+      const doc = makeDoc({
+        id: 'doc1',
+        _deleted: true,
+        _meta: { lwt: oldLwt },
+      });
+      await instance.bulkWrite([{ document: doc }], 'insert-deleted');
+
+      // Cleanup with a 1000ms threshold
+      const result = await instance.cleanup(1000);
+      expect(result).toBe(true);
+
+      // Document should be gone
+      const found = await instance.findDocumentsById(['doc1'], true);
+      expect(found).toHaveLength(0);
+    });
+  });
+
+  describe('remove', () => {
+    it('should drop the table and close', async () => {
+      const doc = makeDoc({ id: 'doc1', _meta: { lwt: 1000 } });
+      await instance.bulkWrite([{ document: doc }], 'insert');
+
+      await instance.remove();
+
+      // After remove, creating a new instance should have no data
+      const db = createMockSQLiteDatabase();
+      const newStorage = getRxStorageSQLite(db);
+      const newInstance = await newStorage.createStorageInstance({
+        databaseInstanceToken: 'test-token-2',
+        databaseName: 'testdb',
+        collectionName: 'testcol',
+        schema: testSchema,
+        options: { database: db },
+        multiInstance: false,
+        devMode: false,
+      });
+      const found = await newInstance.findDocumentsById(['doc1'], true);
+      expect(found).toHaveLength(0);
+      await newInstance.close();
+    });
+  });
+});

package/src/storage-instance.ts

@@ -0,0 +1,333 @@
+import type {
+  BulkWriteRow,
+  EventBulk,
+  PreparedQuery,
+  RxDocumentData,
+  RxJsonSchema,
+  RxStorageBulkWriteResponse,
+  RxStorageChangeEvent,
+  RxStorageCountResult,
+  RxStorageDefaultCheckpoint,
+  RxStorageInstance,
+  RxStorageInstanceCreationParams,
+  RxStorageQueryResult,
+  StringKeys,
+} from 'rxdb';
+import { categorizeBulkWriteRows, getQueryMatcher, getSortComparator } from 'rxdb';
+import { now, ensureNotFalsy, randomToken } from 'rxdb/plugins/utils';
+import { Subject, type Observable } from 'rxjs';
+import type { SQLiteDatabase, SQLiteStorageSettings } from './types';
+import type { RxStorageSQLite } from './rx-storage-sqlite';
+import { buildQuerySQL, buildCountSQL } from './mango-to-sql';
+
+export interface SQLiteStorageInternals {
+  database: SQLiteDatabase;
+  tableName: string;
+}
+
+function sanitizeTableName(databaseName: string, collectionName: string, schemaVersion: number): string {
+  return `${databaseName}_${collectionName}_v${schemaVersion}`.replace(/[^a-zA-Z0-9_]/g, '_');
+}
+
+export class RxStorageInstanceSQLite<RxDocType>
+  implements RxStorageInstance<
+    RxDocType,
+    SQLiteStorageInternals,
+    SQLiteStorageSettings,
+    RxStorageDefaultCheckpoint
+  >
+{
+  readonly databaseName: string;
+  readonly collectionName: string;
+  readonly schema: Readonly<RxJsonSchema<RxDocumentData<RxDocType>>>;
+  readonly internals: SQLiteStorageInternals;
+  readonly options: Readonly<SQLiteStorageSettings>;
+  readonly primaryPath: StringKeys<RxDocumentData<RxDocType>>;
+  private changes$ = new Subject<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, RxStorageDefaultCheckpoint>>();
+  private closed = false;
+
+  constructor(
+    readonly storage: RxStorageSQLite,
+    params: RxStorageInstanceCreationParams<RxDocType, SQLiteStorageSettings>,
+    private database: SQLiteDatabase,
+  ) {
+    this.databaseName = params.databaseName;
+    this.collectionName = params.collectionName;
+    this.schema = params.schema;
+    this.options = params.options;
+    this.primaryPath = (
+      typeof params.schema.primaryKey === 'string'
+        ? params.schema.primaryKey
+        : params.schema.primaryKey.key
+    ) as StringKeys<RxDocumentData<RxDocType>>;
+
+    const tableName = sanitizeTableName(params.databaseName, params.collectionName, params.schema.version);
+    this.internals = { database, tableName };
+
+    this.createTable();
+  }
+
+  /**
+   * Create the SQLite table for this collection if it does not exist.
+   * Stores documents as JSON with indexed columns for _deleted, _rev, _meta.lwt and the primary key.
+   */
+  private createTable(): void {
+    this.database.execSync(
+      `CREATE TABLE IF NOT EXISTS "${this.internals.tableName}" (` +
+      `  id TEXT PRIMARY KEY NOT NULL,` +
+      `  data TEXT NOT NULL,` +
+      `  _deleted INTEGER NOT NULL DEFAULT 0,` +
+      `  _meta_lwt REAL NOT NULL DEFAULT 0` +
+      `)`
+    );
+  }
+
+  /**
+   * Read documents from SQLite by their primary keys.
+   * Returns a Map<primaryKey, RxDocumentData> for use with categorizeBulkWriteRows.
+   */
+  private getDocsByIdMap(ids: string[]): Map<string, RxDocumentData<RxDocType>> {
+    const map = new Map<string, RxDocumentData<RxDocType>>();
+    if (ids.length === 0) return map;
+
+    // Batch query using IN clause
+    const placeholders = ids.map(() => '?').join(',');
+    const rows = this.database.getAllSync<{ id: string; data: string }>(
+      `SELECT id, data FROM "${this.internals.tableName}" WHERE id IN (${placeholders})`,
+      ids
+    );
+    for (const row of rows) {
+      const doc = JSON.parse(row.data) as RxDocumentData<RxDocType>;
+      map.set(row.id, doc);
+    }
+    return map;
+  }
+
+  async bulkWrite(
+    documentWrites: BulkWriteRow<RxDocType>[],
+    context: string,
+  ): Promise<RxStorageBulkWriteResponse<RxDocType>> {
+    if (this.closed) {
+      throw new Error('RxStorageInstanceSQLite: storage is closed');
+    }
+
+    // Collect all document IDs we need to check for conflicts
+    const ids = documentWrites.map(
+      (row) => (row.document as any)[this.primaryPath] as string
+    );
+    const docsInDb = this.getDocsByIdMap(ids);
+
+    // Use RxDB's built-in categorization for conflict detection and event generation
+    const categorized = categorizeBulkWriteRows<RxDocType>(
+      this,
+      this.primaryPath as any,
+      docsInDb,
+      documentWrites,
+      context,
+    );
+
+    const error = categorized.errors;
+
+    // Perform inserts
+    for (const row of categorized.bulkInsertDocs) {
+      const doc = row.document;
+      const docId = (doc as any)[this.primaryPath] as string;
+      const json = JSON.stringify(doc);
+      this.database.runSync(
+        `INSERT OR REPLACE INTO "${this.internals.tableName}" (id, data, _deleted, _meta_lwt) VALUES (?, ?, ?, ?)`,
+        [docId, json, doc._deleted ? 1 : 0, doc._meta.lwt]
+      );
+    }
+
+    // Perform updates
+    for (const row of categorized.bulkUpdateDocs) {
+      const doc = row.document;
+      const docId = (doc as any)[this.primaryPath] as string;
+      const json = JSON.stringify(doc);
+      this.database.runSync(
+        `UPDATE "${this.internals.tableName}" SET data = ?, _deleted = ?, _meta_lwt = ? WHERE id = ?`,
+        [json, doc._deleted ? 1 : 0, doc._meta.lwt, docId]
+      );
+    }
+
+    // Emit events for change stream
+    if (categorized.eventBulk.events.length > 0) {
+      const lastState = ensureNotFalsy(categorized.newestRow).document;
+      categorized.eventBulk.checkpoint = {
+        id: (lastState as any)[this.primaryPath],
+        lwt: lastState._meta.lwt,
+      };
+      this.changes$.next(categorized.eventBulk);
+    }
+
+    return { error };
+  }
+
+  async findDocumentsById(
+    ids: string[],
+    withDeleted: boolean,
+  ): Promise<RxDocumentData<RxDocType>[]> {
+    if (ids.length === 0) return [];
+
+    const placeholders = ids.map(() => '?').join(',');
+    let sql = `SELECT data FROM "${this.internals.tableName}" WHERE id IN (${placeholders})`;
+    if (!withDeleted) {
+      sql += ' AND _deleted = 0';
+    }
+    const rows = this.database.getAllSync<{ data: string }>(sql, ids);
+    return rows.map((row) => JSON.parse(row.data) as RxDocumentData<RxDocType>);
+  }
+
+  async query(
+    preparedQuery: PreparedQuery<RxDocType>,
+  ): Promise<RxStorageQueryResult<RxDocType>> {
+    const { query } = preparedQuery;
+
+    // Try the Mango-to-SQL path first. If the SQLite database supports
+    // json_extract (all modern versions do), this is the fast path.
+    // Fall back to in-memory filtering if the SQL query fails.
+    try {
+      const { sql, params } = buildQuerySQL(this.internals.tableName, query);
+      const rows = this.database.getAllSync<{ data: string }>(sql, params);
+      const documents = rows.map(
+        (row) => JSON.parse(row.data) as RxDocumentData<RxDocType>,
+      );
+      return { documents };
+    } catch {
+      // Fallback: load all non-deleted docs and filter in-memory
+      return this.queryInMemory(query);
+    }
+  }
+
+  async count(
+    preparedQuery: PreparedQuery<RxDocType>,
+  ): Promise<RxStorageCountResult> {
+    const { query } = preparedQuery;
+
+    // Try the SQL COUNT path first.
+    try {
+      const { sql, params } = buildCountSQL(this.internals.tableName, query);
+      const rows = this.database.getAllSync<{ count: number }>(sql, params);
+      if (rows.length > 0 && typeof rows[0].count === 'number') {
+        return { count: rows[0].count, mode: 'fast' };
+      }
+    } catch {
+      // fall through to in-memory
+    }
+
+    // Fallback: do a full query and count the results
+    const result = await this.queryInMemory(query);
+    return { count: result.documents.length, mode: 'slow' };
+  }
+
+  /**
+   * In-memory query fallback using RxDB's built-in query matcher and sort comparator.
+   * Used when the Mango-to-SQL translation fails (e.g., unsupported operators).
+   */
+  private async queryInMemory(
+    query: PreparedQuery<RxDocType>['query'],
+  ): Promise<RxStorageQueryResult<RxDocType>> {
+    const rows = this.database.getAllSync<{ data: string }>(
+      `SELECT data FROM "${this.internals.tableName}" WHERE _deleted = 0`,
+    );
+
+    let documents = rows.map(
+      (row) => JSON.parse(row.data) as RxDocumentData<RxDocType>,
+    );
+
+    const queryMatcher = getQueryMatcher(this.schema, query);
+    documents = documents.filter((doc) => queryMatcher(doc));
+
+    const sortComparator = getSortComparator(this.schema, query);
+    documents.sort(sortComparator);
+
+    const skip = query.skip ?? 0;
+    const limit = query.limit ?? Infinity;
+    documents = documents.slice(skip, skip + limit);
+
+    return { documents };
+  }
+
+  getAttachmentData(
+    _documentId: string,
+    _attachmentId: string,
+    _digest: string,
+  ): Promise<string> {
+    throw new Error('Attachments not supported by SQLite storage');
+  }
+
+  async getChangedDocumentsSince(
+    limit: number,
+    checkpoint?: RxStorageDefaultCheckpoint,
+  ): Promise<{
+    documents: RxDocumentData<RxDocType>[];
+    checkpoint: RxStorageDefaultCheckpoint;
+  }> {
+    let sql: string;
+    let params: any[];
+
+    if (checkpoint) {
+      // Get documents changed after the checkpoint.
+      // Order by lwt then by id to break ties deterministically.
+      sql = `SELECT data FROM "${this.internals.tableName}" ` +
+        `WHERE (_meta_lwt > ?) OR (_meta_lwt = ? AND id > ?) ` +
+        `ORDER BY _meta_lwt ASC, id ASC LIMIT ?`;
+      params = [checkpoint.lwt, checkpoint.lwt, checkpoint.id, limit];
+    } else {
+      sql = `SELECT data FROM "${this.internals.tableName}" ` +
+        `ORDER BY _meta_lwt ASC, id ASC LIMIT ?`;
+      params = [limit];
+    }
+
+    const rows = this.database.getAllSync<{ data: string }>(sql, params);
+    const documents = rows.map(
+      (row) => JSON.parse(row.data) as RxDocumentData<RxDocType>
+    );
+
+    const lastDoc = documents.length > 0 ? documents[documents.length - 1] : undefined;
+    const newCheckpoint: RxStorageDefaultCheckpoint = lastDoc
+      ? {
+          id: (lastDoc as any)[this.primaryPath],
+          lwt: lastDoc._meta.lwt,
+        }
+      : checkpoint ?? { id: '', lwt: 0 };
+
+    return {
+      documents,
+      checkpoint: newCheckpoint,
+    };
+  }
+
+  changeStream(): Observable<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, RxStorageDefaultCheckpoint>> {
+    return this.changes$.asObservable();
+  }
+
+  async cleanup(minimumDeletedTime: number): Promise<boolean> {
+    const maxDeletionTime = now() - minimumDeletedTime;
+    this.database.runSync(
+      `DELETE FROM "${this.internals.tableName}" WHERE _deleted = 1 AND _meta_lwt < ?`,
+      [maxDeletionTime]
+    );
+    return true;
+  }
+
+  async close(): Promise<void> {
+    if (this.closed) return;
+    this.closed = true;
+    this.changes$.complete();
+  }
+
+  async remove(): Promise<void> {
+    this.database.execSync(`DROP TABLE IF EXISTS "${this.internals.tableName}"`);
+    await this.close();
+  }
+}
+
+export async function createSQLiteStorageInstance<RxDocType>(
+  storage: RxStorageSQLite,
+  params: RxStorageInstanceCreationParams<RxDocType, SQLiteStorageSettings>,
+  database: SQLiteDatabase,
+): Promise<RxStorageInstanceSQLite<RxDocType>> {
+  const instance = new RxStorageInstanceSQLite(storage, params, database);
+  return instance;
+}

package/src/types.ts

@@ -0,0 +1,9 @@
+export interface SQLiteDatabase {
+  execSync(source: string): void;
+  getAllSync<T>(source: string, params?: any[]): T[];
+  runSync(source: string, params?: any[]): { changes: number; lastInsertRowId: number };
+}
+
+export interface SQLiteStorageSettings {
+  database: SQLiteDatabase;
+}