@actual-app/core 26.3.0

package/src/platform/server/fs/index.ts

@@ -0,0 +1,416 @@
+// @ts-strict-ignore
+import { SQLiteFS } from 'absurd-sql';
+import IndexedDBBackend from 'absurd-sql/dist/indexeddb-backend';
+
+import * as connection from '../connection';
+import * as idb from '../indexeddb';
+import { logger } from '../log';
+import { _getModule } from '../sqlite';
+import type { SqlJsModule } from '../sqlite';
+
+import { join } from './path-join';
+
+let FS: SqlJsModule['FS'] = null;
+let BFS = null;
+const NO_PERSIST = false;
+
+export const bundledDatabasePath: string = '/default-db.sqlite';
+export const migrationsPath: string = '/migrations';
+export const demoBudgetPath: string = '/demo-budget';
+export { join };
+export { getDocumentDir, getBudgetDir, _setDocumentDir } from './shared';
+export const getDataDir = () => process.env.ACTUAL_DATA_DIR;
+
+export const pathToId = function (filepath: string): string {
+  return filepath.replace(/^\//, '').replace(/\//g, '-');
+};
+
+function _exists(filepath: string): boolean {
+  try {
+    FS.readlink(filepath);
+    return true;
+  } catch {}
+
+  try {
+    FS.stat(filepath);
+    return true;
+  } catch {}
+  return false;
+}
+
+function _mkdirRecursively(dir) {
+  const parts = dir.split('/').filter(str => str !== '');
+  let path = '';
+  for (const part of parts) {
+    path += '/' + part;
+    if (!_exists(path)) {
+      FS.mkdir(path);
+    }
+  }
+}
+
+function _createFile(filepath: string) {
+  // This can create the file. Check if it exists, if not create a
+  // symlink if it's a sqlite file. Otherwise store in idb
+
+  if (!NO_PERSIST && filepath.startsWith('/documents')) {
+    if (filepath.endsWith('.sqlite')) {
+      // If it doesn't exist, we need to create a symlink
+      if (!_exists(filepath)) {
+        FS.symlink('/blocked/' + pathToId(filepath), filepath);
+      }
+    } else {
+      // The contents are actually stored in IndexedDB. We only write to
+      // the in-memory fs to take advantage of the file hierarchy
+      FS.writeFile(filepath, '!$@) this should never read !$@)');
+    }
+  }
+
+  return filepath;
+}
+
+async function _readFile(
+  filepath: string,
+  opts: { encoding: 'utf8' },
+): Promise<string>;
+async function _readFile(
+  filepath: string,
+  opts?: { encoding: 'binary' },
+): Promise<Uint8Array>;
+async function _readFile(
+  filepath: string,
+  opts?: { encoding: 'utf8' } | { encoding: 'binary' },
+): Promise<string | Uint8Array> {
+  // We persist stuff in /documents, but don't need to handle sqlite
+  // file specifically because those are symlinked to a separate
+  // filesystem and will be handled in the BlockedFS
+  if (
+    !NO_PERSIST &&
+    filepath.startsWith('/documents') &&
+    !filepath.endsWith('.sqlite')
+  ) {
+    if (!_exists(filepath)) {
+      throw new Error('File does not exist: ' + filepath);
+    }
+
+    // Grab contents from IDB
+    const { store } = idb.getStore(await idb.getDatabase(), 'files');
+    const item = await idb.get(store, filepath);
+
+    if (item == null) {
+      throw new Error('File does not exist: ' + filepath);
+    }
+
+    if (opts?.encoding === 'utf8' && ArrayBuffer.isView(item.contents)) {
+      return String.fromCharCode.apply(
+        null,
+        new Uint16Array(item.contents.buffer),
+      );
+    }
+
+    return item.contents;
+  } else {
+    if (opts?.encoding === 'utf8') {
+      return FS.readFile(resolveLink(filepath), { encoding: 'utf8' });
+    } else if (opts?.encoding === 'binary') {
+      return FS.readFile(resolveLink(filepath), { encoding: 'binary' });
+    } else {
+      return FS.readFile(resolveLink(filepath));
+    }
+  }
+}
+
+function resolveLink(path: string): string {
+  try {
+    const { node } = FS.lookupPath(path, { follow: false });
+    return node.link ? FS.readlink(path) : path;
+  } catch {
+    return path;
+  }
+}
+
+async function _writeFile(filepath: string, contents): Promise<boolean> {
+  if (contents instanceof ArrayBuffer) {
+    contents = new Uint8Array(contents);
+  } else if (ArrayBuffer.isView(contents)) {
+    contents = new Uint8Array(contents.buffer);
+  }
+
+  // We always create the file if it doesn't exist, and this function
+  // setups up the file depending on its type
+  _createFile(filepath);
+
+  if (!NO_PERSIST && filepath.startsWith('/documents')) {
+    const isDb = filepath.endsWith('.sqlite');
+
+    // Write to IDB
+    const { store } = idb.getStore(await idb.getDatabase(), 'files');
+
+    if (isDb) {
+      // We never write the contents of the database to idb ourselves.
+      // It gets handled via a symlink to the blocked fs (created by
+      // `_createFile` above). However, we still need to record an
+      // entry for the db file so the fs gets properly constructed on
+      // startup
+      await idb.set(store, { filepath, contents: '' });
+
+      // Actually persist the data by going the FS, which will pass
+      // the data through the symlink to the blocked fs. For some
+      // reason we need to resolve symlinks ourselves.
+      await Promise.resolve();
+      FS.writeFile(resolveLink(filepath), contents);
+    } else {
+      await idb.set(store, { filepath, contents });
+    }
+  } else {
+    FS.writeFile(resolveLink(filepath), contents);
+  }
+  return true;
+}
+
+async function _copySqlFile(
+  frompath: string,
+  topath: string,
+): Promise<boolean> {
+  _createFile(topath);
+
+  const { store } = idb.getStore(await idb.getDatabase(), 'files');
+  await idb.set(store, { filepath: topath, contents: '' });
+  const fromitem = await idb.get(store, frompath);
+  const fromDbPath = pathToId(fromitem.filepath);
+  const toDbPath = pathToId(topath);
+
+  const fromfile = BFS.backend.createFile(fromDbPath);
+  const tofile = BFS.backend.createFile(toDbPath);
+
+  try {
+    fromfile.open();
+    tofile.open();
+    const fileSize = fromfile.meta.size;
+    const blockSize = fromfile.meta.blockSize;
+
+    const buffer = new ArrayBuffer(blockSize);
+    const bufferView = new Uint8Array(buffer);
+
+    for (let i = 0; i < fileSize; i += blockSize) {
+      const bytesToRead = Math.min(blockSize, fileSize - i);
+      fromfile.read(bufferView, 0, bytesToRead, i);
+      tofile.write(bufferView, 0, bytesToRead, i);
+    }
+  } catch (error) {
+    tofile.close();
+    fromfile.close();
+    await _removeFile(toDbPath);
+    logger.error('Failed to copy database file', error);
+    return false;
+  } finally {
+    tofile.close();
+    fromfile.close();
+  }
+
+  return true;
+}
+
+async function _removeFile(filepath: string) {
+  if (!NO_PERSIST && filepath.startsWith('/documents')) {
+    const isDb = filepath.endsWith('.sqlite');
+
+    // Remove from IDB
+    const { store } = idb.getStore(await idb.getDatabase(), 'files');
+    await idb.del(store, filepath);
+
+    // If this is the database, is has been symlinked and we want to
+    // remove the actual contents
+    if (isDb) {
+      const linked = resolveLink(filepath);
+      // Be resilient to fs corruption: don't throw an error by trying
+      // to remove a file that doesn't exist. For some reason the db
+      // file is gone? It's ok, just ignore it
+      if (_exists(linked)) {
+        FS.unlink(linked);
+      }
+    }
+  }
+
+  // Finally, remove any in-memory instance
+  FS.unlink(filepath);
+}
+
+// Load files from the server that should exist by default
+async function populateDefaultFilesystem() {
+  const index = await (
+    await fetch(process.env.PUBLIC_URL + 'data-file-index.txt')
+  ).text();
+  const files = index
+    .split('\n')
+    .map(name => name.trim())
+    .filter(name => name !== '');
+  const fetchFile = url => fetch(url).then(res => res.arrayBuffer());
+
+  // This is hardcoded. We know we must create the migrations
+  // directory, it's not worth complicating the index to support
+  // creating arbitrary folders.
+  await mkdir('/migrations');
+  await mkdir('/demo-budget');
+
+  await Promise.all(
+    files.map(async file => {
+      const contents = await fetchFile(process.env.PUBLIC_URL + 'data/' + file);
+      await _writeFile('/' + file, contents);
+    }),
+  );
+}
+
+const populateFileHierarchy = async function () {
+  const { store } = idb.getStore(await idb.getDatabase(), 'files');
+  const req = store.getAllKeys();
+  const paths: string[] = await new Promise((resolve, reject) => {
+    // @ts-expect-error fix me
+    req.onsuccess = e => resolve(e.target.result);
+    req.onerror = e => reject(e);
+  });
+
+  for (const path of paths) {
+    _mkdirRecursively(basename(path));
+    _createFile(path);
+  }
+};
+
+export const init = async function () {
+  const Module = _getModule();
+  FS = Module.FS;
+
+  // When a user "uploads" a file, we just put it in memory in this
+  // dir and the backend takes it from there
+  FS.mkdir('/uploads');
+
+  // Files in /documents are actually read/written from idb.
+  // Everything in there is automatically persisted
+  FS.mkdir('/documents');
+
+  // Files in /blocked are handled by the BlockedFS, which is a
+  // special fs that persists files in blocks. This is necessary
+  // for sqlite3
+  FS.mkdir('/blocked');
+
+  // Jest doesn't support workers. Right now we disable the blocked fs
+  // backend under testing and just test that the directory structure
+  // is created correctly. We assume the the absurd-sql project tests
+  // the blocked fs enough. Additionally, we don't populate the
+  // default files in testing.
+  if (process.env.NODE_ENV !== 'test') {
+    const backend = new IndexedDBBackend(() => {
+      connection.send('fallback-write-error');
+    });
+    BFS = new SQLiteFS(FS, backend);
+    Module.register_for_idb(BFS);
+
+    FS.mount(BFS, {}, '/blocked');
+
+    await populateDefaultFilesystem();
+  }
+
+  await populateFileHierarchy();
+};
+
+export const basename = function (filepath) {
+  const parts = filepath.split('/');
+  return parts.slice(0, -1).join('/');
+};
+
+export const listDir = async function (filepath) {
+  const paths = FS.readdir(filepath);
+  return paths.filter(p => p !== '.' && p !== '..');
+};
+
+export const exists = async function (filepath) {
+  return _exists(filepath);
+};
+
+export const mkdir = async function (filepath) {
+  FS.mkdir(filepath);
+};
+
+export const size = async function (filepath) {
+  const attrs = FS.stat(resolveLink(filepath));
+  return attrs.size;
+};
+
+export const copyFile = async function (
+  frompath: string,
+  topath: string,
+): Promise<boolean> {
+  let result = false;
+  try {
+    const contents = await _readFile(frompath);
+    result = await _writeFile(topath, contents);
+  } catch (error) {
+    if (frompath.endsWith('.sqlite') || topath.endsWith('.sqlite')) {
+      try {
+        result = await _copySqlFile(frompath, topath);
+      } catch (secondError) {
+        throw new Error(
+          `Failed to copy SQL file from ${frompath} to ${topath}: ${secondError.message}`,
+        );
+      }
+    } else {
+      throw error;
+    }
+  }
+  return result;
+};
+
+export async function readFile(
+  filepath: string,
+  encoding?: 'utf8',
+): Promise<string>;
+export async function readFile(
+  filepath: string,
+  encoding: 'binary',
+): Promise<Uint8Array>;
+export async function readFile(
+  filepath: string,
+  encoding: 'binary' | 'utf8' = 'utf8',
+) {
+  if (encoding === 'utf8') {
+    return _readFile(filepath, { encoding });
+  }
+
+  return _readFile(filepath, { encoding });
+}
+
+export const writeFile = async function (filepath: string, contents) {
+  return _writeFile(filepath, contents);
+};
+
+export const removeFile = async function (filepath: string) {
+  return _removeFile(filepath);
+};
+
+export const removeDir = async function (filepath) {
+  FS.rmdir(filepath);
+};
+
+export const removeDirRecursively = async function (dirpath) {
+  if (await exists(dirpath)) {
+    for (const file of await listDir(dirpath)) {
+      const fullpath = join(dirpath, file);
+      // `true` here means to not follow symlinks
+      const attr = FS.stat(fullpath, true);
+
+      if (FS.isDir(attr.mode)) {
+        await removeDirRecursively(fullpath);
+      } else {
+        await removeFile(fullpath);
+      }
+    }
+
+    await removeDir(dirpath);
+  }
+};
+
+export const getModifiedTime = async (filepath: string): Promise<Date> => {
+  throw new Error(
+    'getModifiedTime not supported on the web (only used for backups)',
+  );
+};

package/src/platform/server/fs/path-join.api.ts

@@ -0,0 +1 @@
+export { join } from 'path';

package/src/platform/server/fs/path-join.electron.ts

@@ -0,0 +1 @@
+export { join } from 'path';

package/src/platform/server/fs/path-join.ts

@@ -0,0 +1,97 @@
+// @ts-strict-ignore
+// This code is pulled from
+// https://github.com/browserify/path-browserify/blob/master/index.js#L33
+
+// Resolves . and .. elements in a path with directory names
+function normalizeStringPosix(path, allowAboveRoot) {
+  let res = '';
+  let lastSegmentLength = 0;
+  let lastSlash = -1;
+  let dots = 0;
+  let code;
+  for (let i = 0; i <= path.length; ++i) {
+    if (i < path.length) code = path.charCodeAt(i);
+    else if (code === 47 /*/*/) break;
+    else code = 47 /*/*/;
+    if (code === 47 /*/*/) {
+      if (lastSlash === i - 1 || dots === 1) {
+        // NOOP
+      } else if (lastSlash !== i - 1 && dots === 2) {
+        if (
+          res.length < 2 ||
+          lastSegmentLength !== 2 ||
+          res.charCodeAt(res.length - 1) !== 46 /*.*/ ||
+          res.charCodeAt(res.length - 2) !== 46 /*.*/
+        ) {
+          if (res.length > 2) {
+            const lastSlashIndex = res.lastIndexOf('/');
+            if (lastSlashIndex !== res.length - 1) {
+              if (lastSlashIndex === -1) {
+                res = '';
+                lastSegmentLength = 0;
+              } else {
+                res = res.slice(0, lastSlashIndex);
+                lastSegmentLength = res.length - 1 - res.lastIndexOf('/');
+              }
+              lastSlash = i;
+              dots = 0;
+              continue;
+            }
+          } else if (res.length === 2 || res.length === 1) {
+            res = '';
+            lastSegmentLength = 0;
+            lastSlash = i;
+            dots = 0;
+            continue;
+          }
+        }
+        if (allowAboveRoot) {
+          if (res.length > 0) res += '/..';
+          else res = '..';
+          lastSegmentLength = 2;
+        }
+      } else {
+        if (res.length > 0) res += '/' + path.slice(lastSlash + 1, i);
+        else res = path.slice(lastSlash + 1, i);
+        lastSegmentLength = i - lastSlash - 1;
+      }
+      lastSlash = i;
+      dots = 0;
+    } else if (code === 46 /*.*/ && dots !== -1) {
+      ++dots;
+    } else {
+      dots = -1;
+    }
+  }
+  return res;
+}
+
+function normalizePath(path) {
+  if (path.length === 0) return '.';
+
+  const isAbsolute = path.charCodeAt(0) === 47; /*/*/
+  const trailingSeparator = path.charCodeAt(path.length - 1) === 47; /*/*/
+
+  // Normalize the path
+  path = normalizeStringPosix(path, !isAbsolute);
+
+  if (path.length === 0 && !isAbsolute) path = '.';
+  if (path.length > 0 && trailingSeparator) path += '/';
+
+  if (isAbsolute) return '/' + path;
+  return path;
+}
+
+export const join = (...args: string[]) => {
+  if (args.length === 0) return '.';
+  let joined;
+  for (let i = 0; i < args.length; ++i) {
+    const arg = args[i];
+    if (arg.length > 0) {
+      if (joined === undefined) joined = arg;
+      else joined += '/' + arg;
+    }
+  }
+  if (joined === undefined) return '.';
+  return normalizePath(joined);
+};

package/src/platform/server/fs/shared.ts

@@ -0,0 +1,33 @@
+// @ts-strict-ignore
+import { join } from './path-join';
+
+let documentDir;
+export const _setDocumentDir = dir => (documentDir = dir);
+
+export const getDocumentDir = () => {
+  if (!documentDir) {
+    throw new Error('Document directory is not set');
+  }
+  return documentDir;
+};
+
+export const getBudgetDir = id => {
+  if (!id) {
+    throw new Error('getDocumentDir: id is falsy: ' + id);
+  }
+
+  // TODO: This should be better
+  //
+  // A cheesy safe guard. The id is generated from the budget name,
+  // so it provides an entry point for the user to accidentally (or
+  // intentionally) access other parts of the system. Always
+  // restrict it to only access files within the budget directory by
+  // never allowing slashes.
+  if (id.match(/[^A-Za-z0-9\-_]/)) {
+    throw new Error(
+      `Invalid budget id "${id}". Check the id of your budget in the Advanced section of the settings page.`,
+    );
+  }
+
+  return join(getDocumentDir(), id);
+};

package/src/platform/server/indexeddb/index.ts

@@ -0,0 +1,115 @@
+import { logger } from '../log';
+
+let openedDb: null | ReturnType<typeof _openDatabase> = _openDatabase();
+
+// The web version uses IndexedDB to store data
+function _openDatabase() {
+  return new Promise<IDBDatabase>((resolve, reject) => {
+    const dbVersion = 9;
+    const openRequest = indexedDB.open('actual', dbVersion);
+
+    openRequest.onupgradeneeded = function (e) {
+      const db = (e.target as IDBOpenDBRequest).result;
+
+      // Remove old stores
+      if (db.objectStoreNames.contains('filesystem')) {
+        db.deleteObjectStore('filesystem');
+      }
+      if (db.objectStoreNames.contains('messages')) {
+        db.deleteObjectStore('messages');
+      }
+
+      // Create new stores
+      if (!db.objectStoreNames.contains('asyncStorage')) {
+        db.createObjectStore('asyncStorage');
+      }
+      if (!db.objectStoreNames.contains('files')) {
+        db.createObjectStore('files', { keyPath: 'filepath' });
+      }
+    };
+
+    openRequest.onblocked = e => logger.log('blocked', e);
+
+    openRequest.onerror = () => {
+      logger.log('openRequest error');
+      reject(new Error('indexeddb-failure: Could not open IndexedDB'));
+    };
+
+    openRequest.onsuccess = function (e) {
+      const db = (e.target as IDBOpenDBRequest).result;
+
+      db.onversionchange = () => {
+        // TODO: Notify the user somehow
+        db.close();
+      };
+
+      db.onerror = function (event) {
+        const error = (event.target as IDBOpenDBRequest)?.error;
+        logger.log('Database error:', error);
+
+        if (event.target && error) {
+          if (error.name === 'QuotaExceededError') {
+            throw new Error('indexeddb-quota-error');
+          }
+        }
+      };
+      resolve(db);
+    };
+  });
+}
+
+type Data = { filepath: string; contents: string };
+
+export const getStore = function (db: IDBDatabase, name: string) {
+  const trans = db.transaction([name], 'readwrite');
+  return { trans, store: trans.objectStore(name) };
+};
+
+export const get = async function (
+  store: IDBObjectStore,
+  key: IDBValidKey | IDBKeyRange,
+) {
+  return new Promise<Data>((resolve, reject) => {
+    const req = store.get(key);
+    req.onsuccess = () => {
+      resolve(req.result);
+    };
+    req.onerror = e => reject(e);
+  });
+};
+
+export const set = async function (store: IDBObjectStore, item: Data) {
+  return new Promise((resolve, reject) => {
+    const req = store.put(item);
+    req.onsuccess = () => resolve(undefined);
+    req.onerror = e => reject(e);
+  });
+};
+
+export const del = async function (store: IDBObjectStore, key: string) {
+  return new Promise((resolve, reject) => {
+    const req = store.delete(key);
+    req.onsuccess = () => resolve(undefined);
+    req.onerror = e => reject(e);
+  });
+};
+
+export const getDatabase = function () {
+  return openedDb;
+};
+
+export const openDatabase = function () {
+  if (openedDb == null) {
+    openedDb = _openDatabase();
+  }
+  return openedDb;
+};
+
+export const closeDatabase = async function () {
+  if (openedDb) {
+    await openedDb.then(db => {
+      db.close();
+    });
+    openedDb = null;
+  }
+};

package/src/platform/server/log/index.ts

@@ -0,0 +1,43 @@
+let verboseMode = true;
+
+export function setVerboseMode(verbose: boolean) {
+  verboseMode = verbose;
+}
+
+export function isVerboseMode(): boolean {
+  return verboseMode;
+}
+
+export const logger = {
+  info: (...args: unknown[]) => {
+    if (verboseMode) {
+      console.log(...args);
+    }
+  },
+  warn: (...args: unknown[]) => {
+    console.warn(...args);
+  },
+  log: (...args: unknown[]) => {
+    if (verboseMode) {
+      console.log(...args);
+    }
+  },
+  error: (...args: unknown[]) => {
+    console.error(...args);
+  },
+  debug: (...args: unknown[]) => {
+    if (verboseMode) {
+      console.debug(...args);
+    }
+  },
+  group: (...args: unknown[]) => {
+    if (verboseMode) {
+      console.group(...args);
+    }
+  },
+  groupEnd: () => {
+    if (verboseMode) {
+      console.groupEnd();
+    }
+  },
+};