@actual-app/api 1.1.1 → 2.0.0

package/{query.js → app/query.js}

@@ -8,6 +8,7 @@ class Query {
       calculation: false,
       rawMode: false,
       withDead: false,
+      validateRefs: true,
       limit: null,
       offset: null,
       ...state
@@ -85,6 +86,10 @@ class Query {
     return new Query({ ...this.state, withDead: true });
   }
 
+  withoutValidatedRefs() {
+    return new Query({ ...this.state, validateRefs: false });
+  }
+
   options(opts) {
     return new Query({ ...this.state, tableOptions: opts });
   }
@@ -94,6 +99,24 @@ class Query {
   }
 }
 
+function getPrimaryOrderBy(query, defaultOrderBy) {
+  let orderExprs = query.serialize().orderExpressions;
+  if (orderExprs.length === 0) {
+    if (defaultOrderBy) {
+      return { order: 'asc', ...defaultOrderBy };
+    }
+    return null;
+  }
+
+  let firstOrder = orderExprs[0];
+  if (typeof firstOrder === 'string') {
+    return { field: firstOrder, order: 'asc' };
+  }
+  // Handle this form: { field: 'desc' }
+  let [field] = Object.keys(firstOrder);
+  return { field, order: firstOrder[field] };
+}
+
 module.exports = function q(table) {
   return new Query({ table });
-}
+};

package/index.js

@@ -0,0 +1,10 @@
+const { init, shutdown } = require('./local');
+const methods = require('./methods');
+const utils = require('./utils');
+
+module.exports = {
+  init,
+  shutdown,
+  utils,
+  ...methods
+};

package/injected.js

@@ -0,0 +1,5 @@
+// TODO: comment on why it works this way
+
+let send;
+
+module.exports = { send };

package/local.js

@@ -0,0 +1,28 @@
+let injected = require('./injected');
+let actualApp;
+
+async function init({ budgetId, config } = {}) {
+  if (actualApp) {
+    return;
+  }
+
+  process.traceProcessWarnings = true;
+
+  global.fetch = require('node-fetch');
+  let bundle = require('./app/bundle.api.js');
+
+  await bundle.init({ budgetId, config });
+  actualApp = bundle.lib;
+
+  injected.send = bundle.lib.send;
+  return bundle.lib;
+}
+
+async function shutdown() {
+  if (actualApp) {
+    await actualApp.send('close-budget');
+    actualApp = null;
+  }
+}
+
+module.exports = { init, shutdown };

package/{api.js → methods.js}

@@ -1,12 +1,20 @@
-const {
-  init,
-  send,
-  disconnect,
-  runWithBudget,
-  runImport
-} = require('./connection');
-const q = require('./query');
-const utils = require('./utils');
+const q = require('./app/query');
+const injected = require('./injected');
+
+function send(name, args) {
+  return injected.send(name, args);
+}
+
+async function runImport(name, func) {
+  await send('api/start-import', { budgetName: name });
+  try {
+    await func();
+  } catch (e) {
+    await send('api/abort-import');
+    throw e;
+  }
+  await send('api/finish-import');
+}
 
 async function loadBudget(budgetId) {
   return send('api/load-budget', { id: budgetId });
@@ -158,11 +166,7 @@ function deletePayeeRule(id) {
 }
 
 module.exports = {
-  init,
-  disconnect,
-  runWithBudget,
   runImport,
-  utils,
 
   runQuery,
   q,

package/migrations/1548957970627_remove-db-version.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+DROP TABLE db_version;
+
+COMMIT;

package/migrations/1550601598648_payees.sql

@@ -0,0 +1,23 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE payees
+  (id TEXT PRIMARY KEY,
+   name TEXT,
+   category TEXT,
+   tombstone INTEGER DEFAULT 0,
+   transfer_acct TEXT);
+
+CREATE TABLE payee_rules
+  (id TEXT PRIMARY KEY,
+   payee_id TEXT,
+   type TEXT,
+   value TEXT,
+   tombstone INTEGER DEFAULT 0);
+
+CREATE INDEX payee_rules_lowercase_index ON payee_rules(LOWER(value));
+
+CREATE TABLE payee_mapping
+  (id TEXT PRIMARY KEY,
+   targetId TEXT);
+
+COMMIT;

package/migrations/1555786194328_remove_category_group_unique.sql

@@ -0,0 +1,25 @@
+BEGIN TRANSACTION;
+
+CREATE TEMPORARY TABLE category_groups_tmp
+   (id TEXT PRIMARY KEY,
+    name TEXT UNIQUE,
+    is_income INTEGER DEFAULT 0,
+    sort_order REAL,
+    tombstone INTEGER DEFAULT 0);
+
+INSERT INTO category_groups_tmp SELECT * FROM category_groups;
+
+DROP TABLE category_groups;
+
+CREATE TABLE category_groups
+   (id TEXT PRIMARY KEY,
+    name TEXT,
+    is_income INTEGER DEFAULT 0,
+    sort_order REAL,
+    tombstone INTEGER DEFAULT 0);
+
+INSERT INTO category_groups SELECT * FROM category_groups_tmp;
+
+DROP TABLE category_groups_tmp;
+
+COMMIT;

package/migrations/1561751833510_indexes.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+CREATE INDEX trans_category_date ON transactions(category, date);
+CREATE INDEX trans_category ON transactions(category);
+CREATE INDEX trans_date ON transactions(date);
+
+COMMIT;

package/migrations/1567699552727_budget.sql

@@ -0,0 +1,38 @@
+BEGIN TRANSACTION;
+
+DELETE FROM spreadsheet_cells WHERE
+  name NOT LIKE '%!budget\_%' ESCAPE '\' AND
+  name NOT LIKE '%!carryover\_%' ESCAPE '\' AND
+  name NOT LIKE '%!buffered';
+
+UPDATE OR REPLACE spreadsheet_cells SET name = REPLACE(name, '_', '-');
+
+UPDATE OR REPLACE spreadsheet_cells SET
+  name =
+    SUBSTR(name, 1, 28) ||
+    '-' ||
+    SUBSTR(name, 29, 4) ||
+    '-' ||
+    SUBSTR(name, 33, 4) ||
+    '-' ||
+    SUBSTR(name, 37, 4) ||
+    '-' ||
+    SUBSTR(name, 41, 12)
+WHERE name LIKE '%!budget-%' AND LENGTH(name) = 52;
+
+UPDATE OR REPLACE spreadsheet_cells SET
+  name =
+    SUBSTR(name, 1, 31) ||
+    '-' ||
+    SUBSTR(name, 32, 4) ||
+    '-' ||
+    SUBSTR(name, 36, 4) ||
+    '-' ||
+    SUBSTR(name, 40, 4) ||
+    '-' ||
+    SUBSTR(name, 44, 12)
+WHERE name LIKE '%!carryover-%' AND LENGTH(name) = 55;
+
+UPDATE spreadsheet_cells SET expr = SUBSTR(expr, 2) WHERE name LIKE '%!carryover-%';
+
+COMMIT;

package/migrations/1582384163573_cleared.sql

@@ -0,0 +1,6 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE transactions ADD COLUMN cleared INTEGER DEFAULT 1;
+ALTER TABLE transactions ADD COLUMN pending INTEGER DEFAULT 0;
+
+COMMIT;

package/migrations/1597756566448_rules.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE rules
+  (id TEXT PRIMARY KEY,
+   stage TEXT,
+   conditions TEXT,
+   actions TEXT,
+   tombstone INTEGER DEFAULT 0);
+
+COMMIT;

package/migrations/1608652596043_parent_field.sql

@@ -0,0 +1,13 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE transactions ADD COLUMN parent_id TEXT;
+
+UPDATE transactions SET
+  parent_id = CASE
+    WHEN isChild THEN SUBSTR(id, 1, INSTR(id, '/') - 1)
+    ELSE NULL
+  END;
+
+CREATE INDEX trans_parent_id ON transactions(parent_id);
+
+COMMIT;

package/migrations/1608652596044_trans_views.sql

@@ -0,0 +1,56 @@
+BEGIN TRANSACTION;
+
+DROP VIEW IF EXISTS v_transactions_layer2;
+CREATE VIEW v_transactions_layer2 AS
+SELECT
+  t.id AS id,
+  t.isParent AS is_parent,
+  t.isChild AS is_child,
+  t.acct AS account,
+  CASE WHEN t.isChild = 0 THEN NULL ELSE t.parent_id END AS parent_id,
+  CASE WHEN t.isParent = 1 THEN NULL ELSE cm.transferId END AS category,
+  pm.targetId AS payee,
+  t.imported_description AS imported_payee,
+  IFNULL(t.amount, 0) AS amount,
+  t.notes AS notes,
+  t.date AS date,
+  t.financial_id AS imported_id,
+  t.error AS error,
+  t.starting_balance_flag AS starting_balance_flag,
+  t.transferred_id AS transfer_id,
+  t.sort_order AS sort_order,
+  t.cleared AS cleared,
+  t.tombstone AS tombstone
+FROM transactions t
+LEFT JOIN category_mapping cm ON cm.id = t.category
+LEFT JOIN payee_mapping pm ON pm.id = t.description
+WHERE
+  t.date IS NOT NULL AND
+  t.acct IS NOT NULL;
+
+CREATE INDEX trans_sorted ON transactions(date desc, starting_balance_flag, sort_order desc, id);
+
+DROP VIEW IF EXISTS v_transactions_layer1;
+CREATE VIEW v_transactions_layer1 AS
+SELECT t.* FROM v_transactions_layer2 t
+LEFT JOIN transactions t2 ON (t.is_child = 1 AND t2.id = t.parent_id)
+WHERE IFNULL(t.tombstone, 0) = 0 AND IFNULL(t2.tombstone, 0) = 0;
+
+DROP VIEW IF EXISTS v_transactions;
+CREATE VIEW v_transactions AS
+SELECT t.* FROM v_transactions_layer1 t
+ORDER BY t.date desc, t.starting_balance_flag, t.sort_order desc, t.id;
+
+
+DROP VIEW IF EXISTS v_categories;
+CREATE VIEW v_categories AS
+SELECT
+  id,
+  name,
+  is_income,
+  cat_group AS "group",
+  sort_order,
+  tombstone
+FROM categories;
+
+COMMIT;

package/migrations/1612625548236_optimize.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+CREATE INDEX messages_crdt_search ON messages_crdt(dataset, row, column, timestamp);
+
+ANALYZE;
+
+COMMIT;

package/migrations/1614782639336_trans_views2.sql

@@ -0,0 +1,33 @@
+BEGIN TRANSACTION;
+
+-- This adds the isChild/parent_id constraint in `where`
+DROP VIEW IF EXISTS v_transactions_layer2;
+CREATE VIEW v_transactions_layer2 AS
+SELECT
+  t.id AS id,
+  t.isParent AS is_parent,
+  t.isChild AS is_child,
+  t.acct AS account,
+  CASE WHEN t.isChild = 0 THEN NULL ELSE t.parent_id END AS parent_id,
+  CASE WHEN t.isParent = 1 THEN NULL ELSE cm.transferId END AS category,
+  pm.targetId AS payee,
+  t.imported_description AS imported_payee,
+  IFNULL(t.amount, 0) AS amount,
+  t.notes AS notes,
+  t.date AS date,
+  t.financial_id AS imported_id,
+  t.error AS error,
+  t.starting_balance_flag AS starting_balance_flag,
+  t.transferred_id AS transfer_id,
+  t.sort_order AS sort_order,
+  t.cleared AS cleared,
+  t.tombstone AS tombstone
+FROM transactions t
+LEFT JOIN category_mapping cm ON cm.id = t.category
+LEFT JOIN payee_mapping pm ON pm.id = t.description
+WHERE
+  t.date IS NOT NULL AND
+  t.acct IS NOT NULL AND
+  (t.isChild = 0 OR t.parent_id IS NOT NULL);
+
+COMMIT;

package/migrations/1615745967948_meta.sql

@@ -0,0 +1,10 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE __meta__ (key TEXT PRIMARY KEY, value TEXT);
+
+DROP VIEW IF EXISTS v_transactions_layer2;
+DROP VIEW IF EXISTS v_transactions_layer1;
+DROP VIEW IF EXISTS v_transactions;
+DROP VIEW IF EXISTS v_categories;
+
+COMMIT;

package/migrations/1616167010796_accounts_order.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE accounts ADD COLUMN sort_order REAL;
+
+COMMIT;

package/migrations/1618975177358_schedules.sql

@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE schedules
+  (id TEXT PRIMARY KEY,
+   rule TEXT,
+   active INTEGER DEFAULT 0,
+   completed INTEGER DEFAULT 0,
+   posts_transaction INTEGER DEFAULT 0,
+   tombstone INTEGER DEFAULT 0);
+
+CREATE TABLE schedules_next_date
+  (id TEXT PRIMARY KEY,
+   schedule_id TEXT,
+   local_next_date INTEGER,
+   local_next_date_ts INTEGER,
+   base_next_date INTEGER,
+   base_next_date_ts INTEGER);
+
+CREATE TABLE schedules_json_paths
+  (schedule_id TEXT PRIMARY KEY,
+   payee TEXT,
+   account TEXT,
+   amount TEXT,
+   date TEXT);
+
+ALTER TABLE transactions ADD COLUMN schedule TEXT;
+
+COMMIT;

package/migrations/1632571489012_remove_cache.js

@@ -0,0 +1,135 @@
+export default async function runMigration(db, uuid) {
+  function getValue(node) {
+    return node.expr != null ? node.expr : node.cachedValue;
+  }
+
+  db.execQuery(`
+CREATE TABLE zero_budget_months
+  (id TEXT PRIMARY KEY,
+   buffered INTEGER DEFAULT 0); 
+
+CREATE TABLE zero_budgets
+  (id TEXT PRIMARY KEY,
+   month INTEGER,
+   category TEXT,
+   amount INTEGER DEFAULT 0,
+   carryover INTEGER DEFAULT 0);
+
+CREATE TABLE reflect_budgets
+  (id TEXT PRIMARY KEY,
+   month INTEGER,
+   category TEXT,
+   amount INTEGER DEFAULT 0,
+   carryover INTEGER DEFAULT 0);
+
+CREATE TABLE notes
+  (id TEXT PRIMARY KEY,
+   note TEXT);
+
+CREATE TABLE kvcache (key TEXT PRIMARY KEY, value TEXT);
+CREATE TABLE kvcache_key (id INTEGER PRIMARY KEY, key REAL);
+`);
+
+  // Migrate budget amounts and carryover
+  let budget = db.runQuery(
+    `SELECT * FROM spreadsheet_cells WHERE name LIKE 'budget%!budget-%'`,
+    [],
+    true
+  );
+  db.transaction(() => {
+    budget.map(monthBudget => {
+      let match = monthBudget.name.match(
+        /^(budget-report|budget)(\d+)!budget-(.+)$/
+      );
+      if (match == null) {
+        console.log('Warning: invalid budget month name', monthBudget.name);
+        return;
+      }
+
+      let type = match[1];
+      let month = match[2].slice(0, 4) + '-' + match[2].slice(4);
+      let dbmonth = parseInt(match[2]);
+      let cat = match[3];
+
+      let amount = parseInt(getValue(monthBudget));
+      if (isNaN(amount)) {
+        amount = 0;
+      }
+
+      let sheetName = monthBudget.name.split('!')[0];
+      let carryover = db.runQuery(
+        'SELECT * FROM spreadsheet_cells WHERE name = ?',
+        [`${sheetName}!carryover-${cat}`],
+        true
+      );
+
+      let table = type === 'budget-report' ? 'reflect_budgets' : 'zero_budgets';
+      db.runQuery(
+        `INSERT INTO ${table} (id, month, category, amount, carryover) VALUES (?, ?, ?, ?, ?)`,
+        [
+          `${month}-${cat}`,
+          dbmonth,
+          cat,
+          amount,
+          carryover.length > 0 && getValue(carryover[0]) === 'true' ? 1 : 0
+        ]
+      );
+    });
+  });
+
+  // Migrate buffers
+  let buffers = db.runQuery(
+    `SELECT * FROM spreadsheet_cells WHERE name LIKE 'budget%!buffered'`,
+    [],
+    true
+  );
+  db.transaction(() => {
+    buffers.map(buffer => {
+      let match = buffer.name.match(/^budget(\d+)!buffered$/);
+      if (match) {
+        let month = match[1].slice(0, 4) + '-' + match[1].slice(4);
+        let amount = parseInt(getValue(buffer));
+        if (isNaN(amount)) {
+          amount = 0;
+        }
+
+        db.runQuery(
+          `INSERT INTO zero_budget_months (id, buffered) VALUES (?, ?)`,
+          [month, amount]
+        );
+      }
+    });
+  });
+
+  // Migrate notes
+  let notes = db.runQuery(
+    `SELECT * FROM spreadsheet_cells WHERE name LIKE 'notes!%'`,
+    [],
+    true
+  );
+
+  let parseNote = str => {
+    try {
+      let value = JSON.parse(str);
+      return value && value !== '' ? value : null;
+    } catch (e) {
+      return null;
+    }
+  };
+
+  db.transaction(() => {
+    notes.forEach(note => {
+      let parsed = parseNote(getValue(note));
+      if (parsed) {
+        let [, id] = note.name.split('!');
+        db.runQuery(`INSERT INTO notes (id, note) VALUES (?, ?)`, [id, parsed]);
+      }
+    });
+  });
+
+  db.execQuery(`
+    DROP TABLE spreadsheet_cells;
+    ANALYZE;
+    VACUUM;
+  `);
+}

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@actual-app/api",
-  "version": "1.1.1",
+  "version": "2.0.0",
   "description": "An API for Actual",
   "main": "api.js",
   "license": "MIT",
   "dependencies": {
-    "node-ipc": "9.1.1",
-    "uuid": "3.3.2"
+    "better-sqlite3": "^7.5.0",
+    "uuid": "3.3.2",
+    "node-fetch": "^1.6.3"
   }
 }

package/test.js

@@ -0,0 +1,8 @@
+let api = require('./index');
+
+async function run() {
+  let app = await api.init({ config: { dataDir: '/tmp' } });
+  await app.send('create-budget', { testMode: true });
+}
+
+run();