@actual-app/core 26.3.0

package/src/server/transactions/import/qif2json.ts

@@ -0,0 +1,110 @@
+// @ts-strict-ignore
+type Division = {
+  category?: string;
+  subcategory?: string;
+  description?: string;
+  amount?: number;
+};
+
+type QIFTransaction = {
+  date?: string;
+  amount?: string;
+  number?: string;
+  memo?: string;
+  address?: string[];
+  clearedStatus?: string;
+  category?: string;
+  subcategory?: string;
+  payee?: string;
+  division?: Division[];
+};
+
+export function qif2json(qif, options: { dateFormat?: string } = {}) {
+  const lines = qif.split('\n').filter(Boolean);
+  let line = lines.shift();
+  const type = /!Type:([^$]*)$/.exec(line.trim());
+  const data: {
+    dateFormat: string | undefined;
+    type?;
+    transactions: QIFTransaction[];
+  } = {
+    dateFormat: options.dateFormat,
+    transactions: [],
+  };
+  const transactions = data.transactions;
+  let transaction: QIFTransaction = {};
+
+  if (!type || !type.length) {
+    throw new Error('File does not appear to be a valid qif file: ' + line);
+  }
+  data.type = type[1];
+
+  let division: Division = {};
+
+  while ((line = lines.shift())) {
+    line = line.trim();
+    if (line === '^') {
+      transactions.push(transaction);
+      transaction = {};
+      continue;
+    }
+    switch (line[0]) {
+      case 'D':
+        transaction.date = line.substring(1);
+        break;
+      case 'T':
+        transaction.amount = line.substring(1);
+        break;
+      case 'N':
+        transaction.number = line.substring(1);
+        break;
+      case 'M':
+        transaction.memo = line.substring(1);
+        break;
+      case 'A':
+        transaction.address = (transaction.address || []).concat(
+          line.substring(1),
+        );
+        break;
+      case 'P':
+        transaction.payee = line.substring(1).replace(/&/g, '&');
+        break;
+      case 'L':
+        const lArray = line.substring(1).split(':');
+        transaction.category = lArray[0];
+        if (lArray[1] !== undefined) {
+          transaction.subcategory = lArray[1];
+        }
+        break;
+      case 'C':
+        transaction.clearedStatus = line.substring(1);
+        break;
+      case 'S':
+        const sArray = line.substring(1).split(':');
+        division.category = sArray[0];
+        if (sArray[1] !== undefined) {
+          division.subcategory = sArray[1];
+        }
+        break;
+      case 'E':
+        division.description = line.substring(1);
+        break;
+      case '$':
+        division.amount = parseFloat(line.substring(1));
+        if (!(transaction.division instanceof Array)) {
+          transaction.division = [];
+        }
+        transaction.division.push(division);
+        division = {};
+        break;
+
+      default:
+        throw new Error('Unknown Detail Code: ' + line[0]);
+    }
+  }
+
+  if (Object.keys(transaction).length) {
+    transactions.push(transaction);
+  }
+  return data;
+}

package/src/server/transactions/import/xmlcamt2json.ts

@@ -0,0 +1,168 @@
+// @ts-strict-ignore
+import { parseStringPromise } from 'xml2js';
+
+type DateRef = { DtTm: string } | { Dt: string };
+type Amt = { _: string };
+
+type Ntry = {
+  AcctSvcrRef?: string;
+  Amt?: Amt;
+  CdtDbtInd: 'CRDT' | 'DBIT';
+  ValDt?: DateRef;
+  BookgDt?: DateRef;
+  NtryDtls?: NtryDtls;
+  AddtlNtryInf?: string;
+  NtryRef?: string;
+};
+
+type NtryDtls = {
+  TxDtls: TxDtls | TxDtls[];
+};
+
+type TxDtls = {
+  RltdPties?: {
+    Cdtr: {
+      Nm: string;
+    };
+    Dbtr: {
+      Nm: string;
+    };
+  };
+  RmtInf?: {
+    Ustrd: string | string[];
+  };
+};
+
+type TransactionCAMT = {
+  amount: number;
+  date: string;
+  payee_name: string | null;
+  imported_payee: string | null;
+  notes: string | null;
+  imported_id?: string;
+};
+
+function findKeys(obj: object, key: string): unknown[] {
+  let result = [];
+  for (const i in obj) {
+    if (!obj.hasOwnProperty(i)) continue;
+    if (i === key) {
+      if (Array.isArray(obj[i])) {
+        result = result.concat(obj[i]);
+      } else {
+        result.push(obj[i]);
+      }
+    }
+    if (typeof obj[i] === 'object') {
+      result = result.concat(findKeys(obj[i], key));
+    }
+  }
+  return result;
+}
+
+function getPayeeNameFromTxDtls(
+  TxDtls: TxDtls,
+  isDebit: boolean,
+): string | null {
+  if (TxDtls?.RltdPties) {
+    const key = isDebit ? TxDtls.RltdPties.Cdtr : TxDtls.RltdPties.Dbtr;
+    const Nm = findKeys(key, 'Nm');
+    return Nm.length > 0 ? (Nm[0] as string) : null;
+  }
+  return null;
+}
+
+function getNotesFromTxDtls(TxDtls: TxDtls): string | null {
+  if (TxDtls?.RmtInf) {
+    const Ustrd = TxDtls.RmtInf.Ustrd;
+    return Array.isArray(Ustrd) ? Ustrd.join(' ') : Ustrd;
+  }
+  return null;
+}
+
+function convertToNumberOrNull(value: string): number | null {
+  const number = Number(value);
+  return isNaN(number) ? null : number;
+}
+
+function getDtOrDtTm(Date: DateRef | null): string | null {
+  if (!Date) {
+    return null;
+  }
+  if ('DtTm' in Date) {
+    return Date.DtTm.slice(0, 10);
+  }
+  return Date?.Dt;
+}
+
+export async function xmlCAMT2json(
+  content: string,
+): Promise<TransactionCAMT[]> {
+  const data = await parseStringPromise(content, { explicitArray: false });
+  const entries = findKeys(data, 'Ntry') as Ntry[];
+
+  const transactions: TransactionCAMT[] = [];
+
+  for (const entry of entries) {
+    /*
+      For (camt.052/054) could filter on entry.Sts= BOOK or PDNG, currently importing all entries
+    */
+
+    const id = entry.AcctSvcrRef;
+
+    const amount = convertToNumberOrNull(entry.Amt?._);
+    const isDebit = entry.CdtDbtInd === 'DBIT';
+
+    const date = getDtOrDtTm(entry.ValDt) || getDtOrDtTm(entry.BookgDt);
+
+    if (Array.isArray(entry.NtryDtls?.TxDtls)) {
+      // we add subtransactions as normal transactions as importing split with subtransactions is not supported
+      // amount, and payee_name are not processed correctly for subtransaction.
+      entry.NtryDtls.TxDtls.forEach((TxDtls: TxDtls) => {
+        const subPayee = getPayeeNameFromTxDtls(TxDtls, isDebit);
+        const subNotes = getNotesFromTxDtls(TxDtls);
+        const Amt = findKeys(TxDtls, 'Amt') as Amt[];
+        const amount = Amt.length > 0 ? convertToNumberOrNull(Amt[0]._) : null;
+        transactions.push({
+          amount: isDebit ? -amount : amount,
+          date,
+          payee_name: subPayee,
+          imported_payee: subPayee,
+          notes: subNotes,
+        });
+      });
+    } else {
+      let payee_name: string | null;
+      let notes: string | null;
+      payee_name = getPayeeNameFromTxDtls(entry.NtryDtls?.TxDtls, isDebit);
+      if (!payee_name && entry.AddtlNtryInf) {
+        payee_name = entry.AddtlNtryInf;
+      }
+      notes = getNotesFromTxDtls(entry.NtryDtls?.TxDtls);
+      if (!notes && entry.AddtlNtryInf && entry.AddtlNtryInf !== payee_name) {
+        notes = entry.AddtlNtryInf;
+      }
+      if (!payee_name && !notes && entry.NtryRef) {
+        notes = entry.NtryRef;
+      }
+      if (payee_name && notes && payee_name.includes(notes)) {
+        notes = null;
+      }
+
+      const transaction: TransactionCAMT = {
+        amount: isDebit ? -amount : amount,
+        date,
+        payee_name,
+        imported_payee: payee_name,
+        notes,
+      };
+      if (id) {
+        transaction.imported_id = id;
+      }
+      transactions.push(transaction);
+    }
+  }
+  return transactions.filter(
+    trans => trans.date != null && trans.amount != null,
+  );
+}

package/src/server/transactions/index.ts

@@ -0,0 +1,196 @@
+// @ts-strict-ignore
+
+import * as connection from '../../platform/server/connection';
+import type { Diff } from '../../shared/util';
+import type { PayeeEntity, TransactionEntity } from '../../types/models';
+import * as db from '../db';
+import { incrFetch, whereIn } from '../db/util';
+import { batchMessages } from '../sync';
+
+import * as rules from './transaction-rules';
+import * as transfer from './transfer';
+
+async function idsWithChildren(ids: string[]) {
+  const whereIds = whereIn(ids, 'parent_id');
+  const rows = await db.all<Pick<db.DbViewTransactionInternal, 'id'>>(
+    `SELECT id FROM v_transactions_internal WHERE ${whereIds}`,
+  );
+  const set = new Set(ids);
+  for (const row of rows) {
+    set.add(row.id);
+  }
+  return [...set];
+}
+
+async function getTransactionsByIds(
+  ids: string[],
+): Promise<TransactionEntity[]> {
+  // TODO: convert to whereIn
+  //
+  // or better yet, use ActualQL
+  return incrFetch(
+    (query, params) => db.selectWithSchema('transactions', query, params),
+    ids,
+
+    id => `id = '${id}'`,
+    where => `SELECT * FROM v_transactions_internal WHERE ${where}`,
+  );
+}
+
+export async function batchUpdateTransactions({
+  added,
+  deleted,
+  updated,
+  learnCategories = false,
+  detectOrphanPayees = true,
+  runTransfers = true,
+}: Partial<Diff<TransactionEntity>> & {
+  learnCategories?: boolean;
+  detectOrphanPayees?: boolean;
+  runTransfers?: boolean;
+}) {
+  // Track the ids of each type of transaction change (see below for why)
+  let addedIds = [];
+  const updatedIds = updated ? updated.map(u => u.id) : [];
+  const deletedIds = deleted
+    ? await idsWithChildren(deleted.map(d => d.id))
+    : [];
+
+  const oldPayees = new Set<PayeeEntity['id']>();
+  const accounts = await db.all<db.DbAccount>(
+    'SELECT * FROM accounts WHERE tombstone = 0',
+  );
+
+  // We need to get all the payees of updated transactions _before_
+  // making changes
+  if (updated) {
+    const descUpdatedIds = updated
+      .filter(update => update.payee)
+      .map(update => update.id);
+
+    const transactions = await getTransactionsByIds(descUpdatedIds);
+
+    for (let i = 0; i < transactions.length; i++) {
+      oldPayees.add(transactions[i].payee);
+    }
+  }
+
+  // Apply all the updates. We can batch this now! This is important
+  // and makes bulk updates much faster
+  await batchMessages(async () => {
+    if (added) {
+      addedIds = await Promise.all(
+        added.map(async t => {
+          // Offbudget account transactions and parent transactions should not have categories.
+          const account = accounts.find(acct => acct.id === t.account);
+          if (t.is_parent || account?.offbudget === 1) {
+            t.category = null;
+          }
+          return db.insertTransaction(t);
+        }),
+      );
+    }
+
+    if (deleted) {
+      await Promise.all(
+        // It's important to use `deletedIds` and not `deleted` here
+        // because we've expanded it to include children above. The
+        // inconsistency of the delete APIs is annoying and should
+        // be fixed (it should only take an id)
+        deletedIds.map(async id => {
+          await db.deleteTransaction({ id });
+        }),
+      );
+    }
+
+    if (updated) {
+      await Promise.all(
+        updated.map(async t => {
+          if (t.account) {
+            // Moving transactions off budget should always clear the
+            // category. Parent transactions should not have categories.
+            const account = accounts.find(acct => acct.id === t.account);
+            if (t.is_parent || account?.offbudget === 1) {
+              t.category = null;
+            }
+          }
+
+          await db.updateTransaction(t);
+        }),
+      );
+    }
+  });
+
+  // Get all of the full transactions that were changed. This is
+  // needed to run any cascading logic that depends on the full
+  // transaction. Things like transfers, analyzing rule updates, and
+  // more
+  const allAdded = await getTransactionsByIds(addedIds);
+  const allUpdated = await getTransactionsByIds(updatedIds);
+  const allDeleted = await getTransactionsByIds(deletedIds);
+
+  // Post-processing phase: first do any updates to transfers.
+  // Transfers update the transactions and we need to return updates
+  // to the client so that can apply them. Note that added
+  // transactions just return the full transaction.
+  const resultAdded = allAdded;
+  const resultUpdated = allUpdated;
+  let transfersUpdated: Awaited<ReturnType<typeof transfer.onUpdate>>[];
+
+  if (runTransfers) {
+    await batchMessages(async () => {
+      await Promise.all(allAdded.map(t => transfer.onInsert(t)));
+
+      // Return any updates from here
+      transfersUpdated = (
+        await Promise.all(allUpdated.map(t => transfer.onUpdate(t)))
+      ).filter(Boolean);
+
+      await Promise.all(allDeleted.map(t => transfer.onDelete(t)));
+    });
+  }
+
+  if (learnCategories) {
+    // Analyze any updated categories and update rules to learn from
+    // the user's activity
+    const ids = new Set([
+      ...(added ? added.filter(add => add.category).map(add => add.id) : []),
+      ...(updated
+        ? updated.filter(update => update.category).map(update => update.id)
+        : []),
+    ]);
+    await rules.updateCategoryRules(
+      allAdded.concat(allUpdated).filter(trans => ids.has(trans.id)),
+    );
+  }
+
+  if (detectOrphanPayees) {
+    // Look for any orphaned payees and notify the user about merging
+    // them
+
+    if (updated) {
+      const newPayeeIds = updated.map(u => u.payee).filter(Boolean);
+      if (newPayeeIds.length > 0) {
+        const allOrphaned = new Set(await db.getOrphanedPayees());
+
+        const orphanedIds = [...oldPayees].filter(id => allOrphaned.has(id));
+
+        if (orphanedIds.length > 0) {
+          connection.send('orphaned-payees', {
+            orphanedIds,
+            updatedPayeeIds: newPayeeIds,
+          });
+        }
+      }
+    }
+  }
+
+  return {
+    added: resultAdded,
+    updated: runTransfers ? transfersUpdated : resultUpdated,
+    deleted: allDeleted,
+    errors: ((added || []) as Partial<TransactionEntity>[])
+      .concat(updated || [])
+      .flatMap(t => t._ruleErrors || []),
+  };
+}