@actual-app/core 26.3.0

package/src/platform/server/sqlite/index.api.ts

@@ -0,0 +1,2 @@
+// oxlint-disable-next-line no-restricted-imports
+export * from './index.electron';

package/src/platform/server/sqlite/index.electron.ts

@@ -0,0 +1,134 @@
+// @ts-strict-ignore
+import SQL from 'better-sqlite3';
+import { v4 as uuidv4 } from 'uuid';
+
+import { readFile, removeFile } from '../fs';
+import { logger } from '../log';
+
+import { normalise } from './normalise';
+import { unicodeLike } from './unicodeLike';
+
+function verifyParamTypes(sql, arr) {
+  arr.forEach(val => {
+    if (typeof val !== 'string' && typeof val !== 'number' && val !== null) {
+      logger.log(sql, arr);
+      throw new Error('Invalid field type ' + val + ' for sql ' + sql);
+    }
+  });
+}
+
+export async function init() {
+  // No need to initialise on electron
+}
+
+export function prepare(db, sql) {
+  return db.prepare(sql);
+}
+
+export function runQuery(
+  db: SQL.Database,
+  sql: string | SQL.Statement,
+  params: (string | number)[] = [],
+  fetchAll = false,
+) {
+  if (params) {
+    verifyParamTypes(sql, params);
+  }
+
+  let stmt: SQL.Statement;
+  try {
+    stmt = typeof sql === 'string' ? db.prepare(sql) : sql;
+  } catch (e) {
+    logger.log('error', sql);
+    throw e;
+  }
+
+  if (fetchAll) {
+    try {
+      const result = stmt.all(...params);
+      return result;
+    } catch (e) {
+      logger.log('error', sql);
+      throw e;
+    }
+  } else {
+    const info = stmt.run(...params);
+    return { changes: info.changes, insertId: info.lastInsertRowid };
+  }
+}
+
+export function execQuery(db: SQL.Database, sql: string) {
+  db.exec(sql);
+}
+
+export function transaction(db: SQL.Database, fn: () => void) {
+  db.transaction(fn)();
+}
+
+// **Important**: this is an unsafe function since sqlite executes
+// executes statements sequentially. It would be easy for other code
+// to run statements in between our transaction and get caught up in
+// it. This is rarely used, and only needed for specific cases (like
+// batch importing a bunch of data). Don't use this.
+let transactionDepth = 0;
+export async function asyncTransaction(
+  db: SQL.Database,
+  fn: () => Promise<void>,
+) {
+  // Support nested transactions by "coalescing" them into the parent
+  // one if one is already started
+  if (transactionDepth === 0) {
+    db.exec('BEGIN TRANSACTION');
+  }
+  transactionDepth++;
+
+  try {
+    await fn();
+  } finally {
+    transactionDepth--;
+    // We always commit because rollback is more dangerous - any
+    // queries that ran *in-between* this async function would be
+    // lost. Right now we are only using transactions for speed
+    // purposes unfortunately
+    if (transactionDepth === 0) {
+      db.exec('COMMIT');
+    }
+  }
+}
+
+function regexp(regex: string, text: string | null) {
+  return new RegExp(regex).test(text || '') ? 1 : 0;
+}
+
+export function openDatabase(pathOrBuffer: string | Buffer): SQL.Database {
+  const db = new SQL(pathOrBuffer);
+  // Define Unicode-aware LOWER, UPPER, and LIKE implementation.
+  // This is necessary because better-sqlite3 uses SQLite build without ICU support.
+  db.function('UNICODE_LOWER', { deterministic: true }, (arg: string | null) =>
+    arg?.toLowerCase(),
+  );
+  db.function('UNICODE_UPPER', { deterministic: true }, (arg: string | null) =>
+    arg?.toUpperCase(),
+  );
+  db.function('UNICODE_LIKE', { deterministic: true }, unicodeLike);
+  db.function('REGEXP', { deterministic: true }, regexp);
+  db.function('NORMALISE', { deterministic: true }, normalise);
+  return db;
+}
+
+export function closeDatabase(db: SQL.Database) {
+  db.close();
+}
+
+export async function exportDatabase(db: SQL.Database) {
+  // electron does not support better-sqlite serialize since v21
+  // save to file and read in the raw data.
+  const name = `${process.env.ACTUAL_DATA_DIR}/backup-for-export-${uuidv4()}.db`;
+
+  await db.backup(name);
+
+  const data = await readFile(name, 'binary');
+  await removeFile(name);
+
+  return data;
+}

package/src/platform/server/sqlite/index.test.ts

@@ -0,0 +1,108 @@
+// @ts-strict-ignore
+import { patchFetchForSqlJS } from '../../../mocks/util';
+
+import { execQuery, init, openDatabase, runQuery, transaction } from './index';
+
+beforeAll(async () => {
+  const baseURL = `${__dirname}/../../../../../../node_modules/@jlongster/sql.js/dist/`;
+  patchFetchForSqlJS(baseURL);
+
+  return init({ baseURL });
+});
+
+const initSQL = `
+CREATE TABLE numbers (id TEXT PRIMARY KEY, number INTEGER);
+CREATE TABLE textstrings (id TEXT PRIMARY KEY, string TEXT);
+`;
+
+describe('Web sqlite', () => {
+  it('should rollback transactions', async () => {
+    const db = await openDatabase();
+    execQuery(db, initSQL);
+
+    runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 4)");
+
+    let rows = runQuery(db, 'SELECT * FROM numbers', null, true);
+    expect(rows.length).toBe(1);
+    // @ts-expect-error Property 'number' does not exist on type 'unknown'
+    expect(rows[0].number).toBe(4);
+
+    const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => null);
+    expect(() => {
+      transaction(db, () => {
+        runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id2', 5)");
+        runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id3', 6)");
+        // Insert an invalid one that will error
+        runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 1)");
+      });
+    }).toThrow(/constraint failed/);
+    consoleSpy.mockRestore();
+
+    // Nothing should have changed in the db
+    rows = runQuery(db, 'SELECT * FROM numbers', null, true);
+    expect(rows.length).toBe(1);
+    // @ts-expect-error Property 'number' does not exist on type 'unknown'
+    expect(rows[0].number).toBe(4);
+  });
+
+  it('should support nested transactions', async () => {
+    const db = await openDatabase();
+    execQuery(db, initSQL);
+
+    runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 4)");
+
+    let rows = runQuery(db, 'SELECT * FROM numbers', null, true);
+    expect(rows.length).toBe(1);
+    // @ts-expect-error Property 'number' does not exist on type 'unknown'
+    expect(rows[0].number).toBe(4);
+
+    transaction(db, () => {
+      runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id2', 5)");
+      runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id3', 6)");
+
+      // Only this transaction should fail
+      const consoleSpy = vi
+        .spyOn(console, 'log')
+        .mockImplementation(() => null);
+      expect(() => {
+        transaction(db, () => {
+          runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id4', 7)");
+          // Insert an invalid one that will error
+          runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 1)");
+        });
+      }).toThrow(/constraint failed/);
+      consoleSpy.mockRestore();
+    });
+
+    // Nothing should have changed in the db
+    rows = runQuery(db, 'SELECT * FROM numbers', null, true);
+    expect(rows.length).toBe(3);
+    // @ts-expect-error Property 'number' does not exist on type 'unknown'
+    expect(rows[0].number).toBe(4);
+    // @ts-expect-error Property 'number' does not exist on type 'unknown'
+    expect(rows[1].number).toBe(5);
+    // @ts-expect-error Property 'number' does not exist on type 'unknown'
+    expect(rows[2].number).toBe(6);
+  });
+
+  it('should match regex on text fields', async () => {
+    const db = await openDatabase();
+    execQuery(db, initSQL);
+
+    runQuery(
+      db,
+      "INSERT INTO textstrings (id, string) VALUES ('id1', 'not empty string')",
+    );
+    runQuery(db, "INSERT INTO textstrings (id) VALUES ('id2')");
+
+    const rows = runQuery(
+      db,
+      'SELECT id FROM textstrings where REGEXP("n.", string)',
+      null,
+      true,
+    );
+    expect(rows.length).toBe(1);
+    // @ts-expect-error Property 'id' does not exist on type 'unknown'
+    expect(rows[0].id).toBe('id1');
+  });
+});

package/src/platform/server/sqlite/index.ts

@@ -0,0 +1,241 @@
+// @ts-strict-ignore
+import initSqlJS from '@jlongster/sql.js';
+import type { Database, SqlJsStatic, Statement } from '@jlongster/sql.js';
+
+import { logger } from '../log';
+
+import { normalise } from './normalise';
+import { unicodeLike } from './unicodeLike';
+
+// Types exported from sql.js (and Emscripten) are incomplete, so we need to redefine them here
+type FSStream = (typeof FS)['FSStream'] & {
+  node: (typeof FS)['FSNode'] & {
+    contents: {
+      readIfFallback: () => Promise<unknown>;
+    };
+  };
+};
+type FS = Omit<typeof FS, 'lookupPath' | 'open' | 'close'> & {
+  lookupPath: (
+    path: string,
+    opts?: { follow?: boolean },
+  ) => { node: (typeof FS)['FSNode'] & { link?: string } };
+  open: (path: string, flags: string, mode?: number) => FSStream;
+  close: (stream: FSStream) => void;
+};
+export type SqlJsModule = SqlJsStatic & {
+  FS: FS;
+  reset_filesystem: () => void;
+  register_for_idb: (idb: IDBDatabase) => void;
+};
+
+let SQL: SqlJsModule | null = null;
+
+export async function init({
+  baseURL = process.env.PUBLIC_URL,
+}: { baseURL?: string } = {}) {
+  // `initSqlJS` doesn't actually return a real promise, so make sure
+  // we're returning a real one for correct semantics
+  return new Promise((resolve, reject) => {
+    initSqlJS({
+      locateFile: file => baseURL + file,
+    }).then(
+      sql => {
+        SQL = sql as SqlJsModule;
+        resolve(undefined);
+      },
+      err => {
+        reject(err);
+      },
+    );
+  });
+}
+
+export function _getModule() {
+  if (SQL == null) {
+    throw new Error('_getModule: sql.js must be initialized first');
+  }
+  return SQL;
+}
+
+function verifyParamTypes(
+  sql: string | Statement,
+  arr: (string | number)[] = [],
+) {
+  arr.forEach(val => {
+    if (typeof val !== 'string' && typeof val !== 'number' && val !== null) {
+      throw new Error('Invalid field type ' + val + ' for sql ' + sql);
+    }
+  });
+}
+
+export function prepare(db: Database, sql: string) {
+  return db.prepare(sql);
+}
+
+export function runQuery(
+  db: Database,
+  sql: string | Statement,
+  params?: (string | number)[],
+  fetchAll?: false,
+): { changes: unknown };
+export function runQuery<T>(
+  db: Database,
+  sql: string | Statement,
+  params: (string | number)[],
+  fetchAll: true,
+): T[];
+export function runQuery<T>(
+  db: Database,
+  sql: string | Statement,
+  params: (string | number)[] = [],
+  fetchAll = false,
+): T[] | { changes: unknown } {
+  if (params) {
+    verifyParamTypes(sql, params);
+  }
+
+  const stmt = typeof sql === 'string' ? db.prepare(sql) : sql;
+
+  if (fetchAll) {
+    try {
+      stmt.bind(params);
+      const rows = [];
+
+      while (stmt.step()) {
+        rows.push(stmt.getAsObject());
+      }
+
+      if (typeof sql === 'string') {
+        stmt.free();
+      } else {
+        stmt.reset();
+      }
+      return rows;
+    } catch (e) {
+      logger.log(sql);
+      throw e;
+    }
+  } else {
+    stmt.run(params);
+    return { changes: db.getRowsModified() };
+  }
+}
+
+export function execQuery(db: Database, sql: string) {
+  db.exec(sql);
+}
+
+let transactionDepth = 0;
+
+export function transaction(db: Database, fn: () => void) {
+  let before, after, undo;
+  if (transactionDepth > 0) {
+    before = 'SAVEPOINT __actual_sp';
+    after = 'RELEASE __actual_sp';
+    undo = 'ROLLBACK TO __actual_sp';
+  } else {
+    before = 'BEGIN';
+    after = 'COMMIT';
+    undo = 'ROLLBACK';
+  }
+
+  execQuery(db, before);
+  transactionDepth++;
+
+  try {
+    fn();
+    execQuery(db, after);
+  } catch (ex) {
+    execQuery(db, undo);
+
+    if (undo !== 'ROLLBACK') {
+      execQuery(db, after);
+    }
+
+    throw ex;
+  } finally {
+    transactionDepth--;
+  }
+}
+
+// See the comment about this function in index.electron.js. You
+// shouldn't normally use this. I'd like to get rid of it.
+export async function asyncTransaction(db: Database, fn: () => Promise<void>) {
+  // Support nested transactions by "coalescing" them into the parent
+  // one if one is already started
+  if (transactionDepth === 0) {
+    db.exec('BEGIN TRANSACTION');
+  }
+  transactionDepth++;
+
+  try {
+    await fn();
+  } finally {
+    transactionDepth--;
+    // We always commit because rollback is more dangerous - any
+    // queries that ran *in-between* this async function would be
+    // lost. Right now we are only using transactions for speed
+    // purposes unfortunately
+    if (transactionDepth === 0) {
+      db.exec('COMMIT');
+    }
+  }
+}
+
+function regexp(regex: string, text: string) {
+  return new RegExp(regex).test(text || '') ? 1 : 0;
+}
+
+export async function openDatabase(pathOrBuffer?: string | Uint8Array) {
+  let db = null;
+  if (pathOrBuffer) {
+    if (typeof pathOrBuffer !== 'string') {
+      db = new SQL.Database(pathOrBuffer);
+    } else {
+      const path = pathOrBuffer;
+      if (path !== ':memory:') {
+        if (typeof SharedArrayBuffer === 'undefined') {
+          const stream = SQL.FS.open(SQL.FS.readlink(path), 'a+');
+          await stream.node.contents.readIfFallback();
+          SQL.FS.close(stream);
+        }
+
+        db = new SQL.Database(
+          path.includes('/blocked') ? path : SQL.FS.readlink(path),
+          // @ts-expect-error 2nd argument missed in sql.js types
+          { filename: true },
+        );
+        db.exec(`
+          PRAGMA journal_mode=MEMORY;
+          PRAGMA cache_size=-10000;
+        `);
+      }
+    }
+  }
+
+  if (db === null) {
+    db = new SQL.Database();
+  }
+
+  // Define Unicode-aware LOWER, UPPER, and LIKE implementation.
+  // This is necessary because sql.js uses SQLite build without ICU support.
+  //
+  // Note that this function should ideally be created with a deterministic flag
+  // to allow SQLite to better optimize calls to it by factoring them out of inner loops
+  // but SQL.js does not support this: https://github.com/sql-js/sql.js/issues/551
+  db.create_function('UNICODE_LOWER', arg => arg?.toLowerCase());
+  db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase());
+  db.create_function('UNICODE_LIKE', unicodeLike);
+  db.create_function('REGEXP', regexp);
+  db.create_function('NORMALISE', normalise);
+  return db;
+}
+
+export function closeDatabase(db: Database) {
+  db.close();
+}
+
+export async function exportDatabase(db: Database) {
+  return db.export();
+}

package/src/platform/server/sqlite/normalise.ts

@@ -0,0 +1,9 @@
+import { getNormalisedString } from '../../../shared/normalisation';
+
+export function normalise(value: string | null): string | null {
+  if (!value) {
+    return null;
+  }
+
+  return getNormalisedString(value);
+}

package/src/platform/server/sqlite/unicodeLike.test.ts

@@ -0,0 +1,58 @@
+import { unicodeLike } from './unicodeLike';
+
+describe('unicode LIKE functionality', () => {
+  it('empty pattern should not match to a value', () => {
+    const result = unicodeLike(null, 'value');
+
+    expect(result).toBe(0);
+  });
+
+  it('empty pattern should not match to null', () => {
+    const result = unicodeLike(null, null);
+
+    expect(result).toBe(0);
+  });
+
+  it('should match special characters', () => {
+    // oxlint-disable-next-line no-template-curly-in-string
+    const result = unicodeLike('.*+^${}()|[]\\', '.*+^${}()|[]\\');
+
+    expect(result).toBe(1);
+  });
+
+  it('should use ? as the single character placeholder', () => {
+    const result = unicodeLike('t?st', 'test');
+
+    expect(result).toBe(1);
+  });
+
+  it('should use % as the zero-or-more characters placeholder', () => {
+    const result = unicodeLike('t%st', 'te123st');
+
+    expect(result).toBe(1);
+  });
+
+  it('should ignore case for unicode', () => {
+    const result = unicodeLike('á', 'Ábcdefg');
+
+    expect(result).toBe(1);
+  });
+
+  it('should ignore case for ascii', () => {
+    const result = unicodeLike('a', 'Abcdefg');
+
+    expect(result).toBe(1);
+  });
+
+  it('should treat null value as empty string', () => {
+    const result = unicodeLike('%', null);
+
+    expect(result).toBe(1);
+  });
+
+  it('should not match null value to the string "null"', () => {
+    const result = unicodeLike('null', null);
+
+    expect(result).toBe(0);
+  });
+});

package/src/platform/server/sqlite/unicodeLike.ts

@@ -0,0 +1,31 @@
+import { LRUCache } from 'lru-cache';
+
+const likePatternCache = new LRUCache<string, RegExp>({ max: 500 });
+
+export function unicodeLike(
+  pattern: string | null,
+  value: string | null,
+): number {
+  if (!pattern) {
+    return 0;
+  }
+
+  if (!value) {
+    value = '';
+  }
+
+  let cachedRegExp = likePatternCache.get(pattern);
+  if (!cachedRegExp) {
+    // we don't escape ? and % because we don't know
+    // whether they originate from the user input or from our query compiler.
+    // Maybe improve the query compiler to correctly process these characters?
+    const processedPattern = pattern
+      .replace(/[.*+^${}()|[\]\\]/g, '\\$&')
+      .replaceAll('?', '.')
+      .replaceAll('%', '.*');
+    cachedRegExp = new RegExp(processedPattern, 'i');
+    likePatternCache.set(pattern, cachedRegExp);
+  }
+
+  return cachedRegExp.test(value) ? 1 : 0;
+}

package/src/server/__mocks__/post.ts

@@ -0,0 +1,9 @@
+// @ts-strict-ignore
+export {
+  handleRequest as post,
+  handleRequestBinary as postBinary,
+} from '../tests/mockSyncServer';
+
+export const get = function () {
+  throw new Error('get unimplemented');
+};