@actual-app/core 26.3.0

package/src/server/sheet.ts

@@ -0,0 +1,280 @@
+// @ts-strict-ignore
+import type { Database } from '@jlongster/sql.js';
+
+import { captureBreadcrumb } from '../platform/exceptions';
+import { logger } from '../platform/server/log';
+import * as sqlite from '../platform/server/sqlite';
+import { sheetForMonth } from '../shared/months';
+import * as Platform from '../shared/platform';
+
+import type * as DbModule from './db';
+import type {
+  DbPreference,
+  DbReflectBudget,
+  DbZeroBudget,
+  DbZeroBudgetMonth,
+} from './db';
+import { Spreadsheet } from './spreadsheet/spreadsheet';
+import { resolveName } from './spreadsheet/util';
+
+let globalSheet: Spreadsheet;
+let globalOnChange;
+let globalCacheDb;
+
+export function get(): Spreadsheet {
+  return globalSheet;
+}
+
+async function updateSpreadsheetCache(rawDb, names: string[]) {
+  sqlite.transaction(rawDb, () => {
+    names.forEach(name => {
+      const node = globalSheet._getNode(name);
+
+      // Don't cache query nodes yet
+      if (node.sql == null) {
+        sqlite.runQuery(
+          rawDb,
+          'INSERT OR REPLACE INTO kvcache (key, value) VALUES (?, ?)',
+          [name, JSON.stringify(node.value)],
+        );
+      }
+    });
+  });
+}
+
+function setCacheStatus(
+  mainDb: Database,
+  cacheDb: Database,
+  { clean }: { clean: boolean },
+) {
+  if (clean) {
+    // Generate random number and stick in both places
+    const num = Math.random() * 10000000;
+    sqlite.runQuery(
+      cacheDb,
+      'INSERT OR REPLACE INTO kvcache_key (id, key) VALUES (1, ?)',
+      [num],
+    );
+
+    if (mainDb) {
+      sqlite.runQuery(
+        mainDb,
+        'INSERT OR REPLACE INTO kvcache_key (id, key) VALUES (1, ?)',
+        [num],
+      );
+    }
+  } else {
+    sqlite.runQuery(cacheDb, 'DELETE FROM kvcache_key');
+  }
+}
+
+function isCacheDirty(mainDb: Database, cacheDb: Database): boolean {
+  let rows = sqlite.runQuery<{ key?: number }>(
+    cacheDb,
+    'SELECT key FROM kvcache_key WHERE id = 1',
+    [],
+    true,
+  );
+  const num = rows.length === 0 ? null : rows[0].key;
+
+  if (num == null) {
+    return true;
+  }
+
+  if (mainDb) {
+    const rows = sqlite.runQuery<{ key?: number }>(
+      mainDb,
+      'SELECT key FROM kvcache_key WHERE id = 1',
+      [],
+      true,
+    );
+    if (rows.length === 0 || rows[0].key !== num) {
+      return true;
+    }
+  }
+
+  // Always also check if there is anything in `kvcache`. We ask for one item;
+  // if we didn't get back anything it's empty so there is no cache
+  rows = sqlite.runQuery(cacheDb, 'SELECT * FROM kvcache LIMIT 1', [], true);
+  return rows.length === 0;
+}
+
+export async function loadSpreadsheet(
+  db,
+  onSheetChange?,
+): Promise<Spreadsheet> {
+  const cacheEnabled = process.env.NODE_ENV !== 'test';
+  const mainDb = db.getDatabase();
+  let cacheDb;
+
+  if (!Platform.isBrowser && cacheEnabled) {
+    // Desktop apps use a separate database for the cache. This is because it is
+    // much more likely to directly work with files on desktop, and this makes
+    // it a lot clearer what the true filesize of the main db is (and avoid
+    // copying the cache data around).
+    const cachePath = db
+      .getDatabasePath()
+      .replace(/db\.sqlite$/, 'cache.sqlite');
+    globalCacheDb = cacheDb = await sqlite.openDatabase(cachePath);
+
+    sqlite.execQuery(
+      cacheDb,
+      `
+        CREATE TABLE IF NOT EXISTS kvcache (key TEXT PRIMARY KEY, value TEXT);
+        CREATE TABLE IF NOT EXISTS kvcache_key (id INTEGER PRIMARY KEY, key REAL)
+      `,
+    );
+  } else {
+    // All other platforms use the same database for cache
+    cacheDb = mainDb;
+  }
+
+  let sheet;
+  if (cacheEnabled) {
+    sheet = new Spreadsheet(
+      updateSpreadsheetCache.bind(null, cacheDb),
+      setCacheStatus.bind(null, mainDb, cacheDb),
+    );
+  } else {
+    sheet = new Spreadsheet();
+  }
+
+  captureBreadcrumb({
+    message: 'loading spreadsheet',
+    category: 'server',
+  });
+
+  globalSheet = sheet;
+  globalOnChange = onSheetChange;
+
+  if (onSheetChange) {
+    sheet.addEventListener('change', onSheetChange);
+  }
+
+  if (cacheEnabled && !isCacheDirty(mainDb, cacheDb)) {
+    const cachedRows = sqlite.runQuery<{ key?: number; value: string }>(
+      cacheDb,
+      'SELECT * FROM kvcache',
+      [],
+      true,
+    );
+    logger.log(`Loaded spreadsheet from cache (${cachedRows.length} items)`);
+
+    for (const row of cachedRows) {
+      const parsed = JSON.parse(row.value);
+      sheet.load(row.key, parsed);
+    }
+  } else {
+    logger.log('Loading fresh spreadsheet');
+    await loadUserBudgets(db);
+  }
+
+  captureBreadcrumb({
+    message: 'loaded spreadsheet',
+    category: 'server',
+  });
+
+  return sheet;
+}
+
+export function unloadSpreadsheet(): void {
+  if (globalSheet) {
+    // TODO: Should wait for the sheet to finish
+    globalSheet.unload();
+    globalSheet = null;
+  }
+
+  if (globalCacheDb) {
+    sqlite.closeDatabase(globalCacheDb);
+    globalCacheDb = null;
+  }
+}
+
+export async function reloadSpreadsheet(db): Promise<Spreadsheet> {
+  if (globalSheet) {
+    unloadSpreadsheet();
+    return loadSpreadsheet(db, globalOnChange);
+  }
+}
+
+export async function loadUserBudgets(db: typeof DbModule): Promise<void> {
+  const sheet = globalSheet;
+
+  // TODO: Clear out the cache here so make sure future loads of the app
+  // don't load any extra values that aren't set here
+
+  const { value: budgetType = 'envelope' } =
+    (await db.first<Pick<DbPreference, 'value'>>(
+      'SELECT value from preferences WHERE id = ?',
+      ['budgetType'],
+    )) ?? {};
+
+  const table = budgetType === 'tracking' ? 'reflect_budgets' : 'zero_budgets';
+  const budgets = await db.all<DbReflectBudget | DbZeroBudget>(`
+      SELECT * FROM ${table} b
+      LEFT JOIN categories c ON c.id = b.category
+      WHERE c.tombstone = 0
+    `);
+
+  sheet.startTransaction();
+
+  // Load all the budget amounts and carryover values
+  for (const budget of budgets) {
+    if (budget.month && budget.category) {
+      const sheetName = `budget${budget.month}`;
+      sheet.set(`${sheetName}!budget-${budget.category}`, budget.amount);
+      sheet.set(
+        `${sheetName}!carryover-${budget.category}`,
+        budget.carryover === 1 ? true : false,
+      );
+      sheet.set(`${sheetName}!goal-${budget.category}`, budget.goal);
+      sheet.set(`${sheetName}!long-goal-${budget.category}`, budget.long_goal);
+    }
+  }
+
+  // For zero-based budgets, load the buffered amounts
+  if (budgetType !== 'tracking') {
+    const budgetMonths = await db.all<DbZeroBudgetMonth>(
+      'SELECT * FROM zero_budget_months',
+    );
+    for (const budgetMonth of budgetMonths) {
+      const sheetName = sheetForMonth(budgetMonth.id);
+      sheet.set(`${sheetName}!buffered`, budgetMonth.buffered);
+    }
+  }
+
+  sheet.endTransaction();
+}
+
+export function getCell(sheet: string, name: string) {
+  return globalSheet._getNode(resolveName(sheet, name));
+}
+
+export function getCellValue(
+  sheet: string,
+  name: string,
+): string | number | boolean {
+  return globalSheet.getValue(resolveName(sheet, name));
+}
+
+export function startTransaction(): void {
+  if (globalSheet) {
+    globalSheet.startTransaction();
+  }
+}
+
+export function endTransaction(): void {
+  if (globalSheet) {
+    globalSheet.endTransaction();
+  }
+}
+
+export function waitOnSpreadsheet(): Promise<void> {
+  return new Promise(resolve => {
+    if (globalSheet) {
+      globalSheet.onFinish(resolve);
+    } else {
+      resolve(undefined);
+    }
+  });
+}

package/src/server/spreadsheet/__snapshots__/spreadsheet.test.ts.snap

@@ -0,0 +1,5 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Spreadsheet > querying deep join works 1`] = `"=from transactions where acct.offbudget = 0 and (description.transfer_acct.offbudget = null or description.transfer_acct.offbudget = 1) select { acct.offbudget, description.transfer_acct.offbudget as foo, amount }"`;
+
+exports[`Spreadsheet > querying transactions works 1`] = `"=from transactions select { amount, category }"`;

package/src/server/spreadsheet/app.ts

@@ -0,0 +1,54 @@
+import type { QueryState } from '../../shared/query';
+import { createApp } from '../app';
+import * as sheet from '../sheet';
+
+import { resolveName, unresolveName } from './util';
+
+export type SpreadsheetHandlers = {
+  'get-cell': typeof getCell;
+  'get-cell-names': typeof getCellNames;
+  'create-query': typeof createQuery;
+};
+
+// Expose functions to the client
+export const app = createApp<SpreadsheetHandlers>();
+app.method('get-cell', getCell);
+app.method('get-cell-names', getCellNames);
+app.method('create-query', createQuery);
+
+async function getCell({
+  sheetName,
+  name,
+}: {
+  sheetName: string;
+  name: string;
+}) {
+  const node = sheet.get()._getNode(resolveName(sheetName, name));
+  return { name: node.name, value: node.value };
+}
+
+async function getCellNames({ sheetName }: { sheetName: string }) {
+  const names = [];
+  for (const name of sheet.get().getNodes().keys()) {
+    const { sheet: nodeSheet, name: nodeName } = unresolveName(name);
+    if (nodeSheet === sheetName) {
+      names.push(nodeName);
+    }
+  }
+  return names;
+}
+
+async function createQuery({
+  sheetName,
+  name,
+  query,
+}: {
+  sheetName: string;
+  name: string;
+  query: QueryState;
+}) {
+  // Always run it regardless of cache. We don't know anything has changed
+  // between the cache value being saved and now
+  sheet.get().createQuery(sheetName, name, query);
+  return 'ok';
+}

package/src/server/spreadsheet/globals.ts

@@ -0,0 +1,13 @@
+export function number(v: unknown): number {
+  if (typeof v === 'number') {
+    return v;
+  } else if (typeof v === 'string') {
+    const parsed = parseFloat(v);
+    if (isNaN(parsed)) {
+      return 0;
+    }
+    return parsed;
+  }
+
+  return 0;
+}

package/src/server/spreadsheet/graph-data-structure.ts

@@ -0,0 +1,165 @@
+// @ts-strict-ignore
+export function Graph() {
+  const graph = {
+    addNode,
+    removeNode,
+    adjacent,
+    adjacentIncoming,
+    addEdge,
+    removeEdge,
+    removeIncomingEdges,
+    topologicalSort,
+    generateDOT,
+    getEdges,
+  };
+
+  const edges = new Map();
+  const incomingEdges = new Map();
+
+  function getEdges() {
+    return { edges, incomingEdges };
+  }
+
+  function addNode(node) {
+    edges.set(node, adjacent(node));
+    incomingEdges.set(node, adjacentIncoming(node));
+    return graph;
+  }
+
+  function removeIncomingEdges(node) {
+    const incoming = adjacentIncoming(node);
+    incomingEdges.set(node, new Set());
+
+    const iter = incoming.values();
+    let cur = iter.next();
+    while (!cur.done) {
+      removeEdge(cur.value, node);
+      cur = iter.next();
+    }
+  }
+
+  function removeNode(node) {
+    removeIncomingEdges(node);
+    edges.delete(node);
+    incomingEdges.delete(node);
+    return graph;
+  }
+
+  function adjacent(node) {
+    return edges.get(node) || new Set();
+  }
+
+  function adjacentIncoming(node) {
+    return incomingEdges.get(node) || new Set();
+  }
+
+  // Adds an edge from node u to node v.
+  // Implicitly adds the nodes if they were not already added.
+  function addEdge(node1, node2) {
+    addNode(node1);
+    addNode(node2);
+    adjacent(node1).add(node2);
+    adjacentIncoming(node2).add(node1);
+    return graph;
+  }
+
+  // Removes the edge from node u to node v.
+  // Does not remove the nodes.
+  // Does nothing if the edge does not exist.
+  function removeEdge(node1, node2) {
+    if (edges.has(node1)) {
+      adjacent(node1).delete(node2);
+    }
+    if (incomingEdges.has(node2)) {
+      adjacentIncoming(node2).delete(node1);
+    }
+    return graph;
+  }
+
+  function topologicalSort(sourceNodes) {
+    const visited = new Set();
+    const sorted = [];
+
+    sourceNodes.forEach(name => {
+      if (!visited.has(name)) {
+        topologicalSortIterable(name, visited, sorted);
+      }
+    });
+
+    return sorted;
+  }
+
+  function topologicalSortIterable(name, visited, sorted) {
+    const stackTrace: StackItem[] = [];
+
+    stackTrace.push({
+      count: -1,
+      value: name,
+      parent: '',
+      level: 0,
+    });
+
+    while (stackTrace.length > 0) {
+      const current = stackTrace.slice(-1)[0];
+
+      const adjacents = adjacent(current.value);
+      if (current.count === -1) {
+        current.count = adjacents.size;
+      }
+
+      if (current.count > 0) {
+        const iter = adjacents.values();
+        let cur = iter.next();
+        while (!cur.done) {
+          if (!visited.has(cur.value)) {
+            stackTrace.push({
+              count: -1,
+              parent: current.value,
+              value: cur.value,
+              level: current.level + 1,
+            });
+          } else {
+            current.count--;
+          }
+          cur = iter.next();
+        }
+      } else {
+        if (!visited.has(current.value)) {
+          visited.add(current.value);
+          sorted.unshift(current.value);
+        }
+
+        const removed = stackTrace.pop();
+        for (let i = 0; i < stackTrace.length; i++) {
+          if (stackTrace[i].value === removed.parent) {
+            stackTrace[i].count--;
+          }
+        }
+      }
+    }
+  }
+
+  function generateDOT() {
+    const edgeStrings = [];
+    edges.forEach(function (adj, edge) {
+      if (adj.length !== 0) {
+        edgeStrings.push(`${edge} -> {${adj.join(',')}}`);
+      }
+    });
+
+    return `
+    digraph G {
+      ${edgeStrings.join('\n').replace(/!/g, '_')}
+    }
+    `;
+  }
+
+  return graph;
+}
+
+type StackItem = {
+  count: number;
+  value: string;
+  parent: string;
+  level: number;
+};

package/src/server/spreadsheet/scratch

@@ -0,0 +1,60 @@
+const query = (
+  from t in transactions
+  where t.amount == 700
+  select { t.amount, t.description }
+);
+
+const sqlgen = require("./sqlgen");
+const sqlinterp = require("./sqlinterp");
+const { Database } = require("sqlite3");
+const db = new Database("db.sqlite");
+
+const shemas = {};
+
+function makeRow(columns, data) {
+  return t.toObj(columns.map((name, i) => {
+    return [name, data[i]];
+  }));
+}
+
+function run(query, cb) {
+  console.log("running")
+  db.each(sqlgen(query) + " LIMIT 10", function(err, row) {
+    console.log(row);
+  }, cb);
+}
+
+db.addListener("preupdate", function(type, db, table, oldValues, newValues) {
+  let existed, exists;
+  debugger;
+  if(oldValues) {
+    const row = makeRow(schemas[table], oldValues);
+    existed = sqlinterp(query, row);
+  }
+  if(newValues) {
+    const row = makeRow(schemas[table], newValues);
+    exists = sqlinterp(query, row);
+  }
+
+  if(existed || exists) {
+    console.log("Running query...");
+    run(query, function() {});
+  }
+});
+
+db.all("PRAGMA table_info(transactions)", function(err, rows) {
+  schemas["transactions"] = rows.map(r => r.name);
+
+  run(query, function() {
+    console.log("\nCHANGING\n");
+
+    db.run("INSERT INTO transactions (acct, category, amount, description, date)" +
+           ' VALUES (1, 1, 3300, "foobar", 1456808400000)');
+    db.run("INSERT INTO transactions (acct, category, amount, description, date)" +
+           ' VALUES (1, 1, 700, "foobar", 1456808400000)');
+    db.run("INSERT INTO transactions (acct, category, amount, description, date)" +
+           ' VALUES (1, 1, -500, "foobar", 1456808400000)');
+    db.run("INSERT INTO transactions (acct, category, amount, description, date)" +
+           ' VALUES (1, 1, 200, "foobar", 1456808400000)');
+  });
+});