@actual-app/core 26.3.0

package/src/server/transactions/import/ofx2json.test.ts

@@ -0,0 +1,33 @@
+import { html2Plain } from './ofx2json';
+
+describe('html2Plain', () => {
+  test('regular text works', async () => {
+    expect(html2Plain('Hello, world!')).toBe('Hello, world!');
+    expect(html2Plain('Hello, <b>world</b>!')).toBe('Hello, <b>world</b>!');
+  });
+
+  test('brackets are unescaped', async () => {
+    expect(html2Plain('Hello, &lt;world&gt;!')).toBe('Hello, <world>!');
+  });
+  test('apostrophes are unescaped', async () => {
+    expect(html2Plain('Hello, &#39;world&#39;!')).toBe("Hello, 'world'!");
+  });
+  test('quotes are unescaped', async () => {
+    expect(html2Plain('Hello, &quot;world&quot;!')).toBe('Hello, "world"!');
+  });
+  test('ampersands are unescaped', async () => {
+    expect(html2Plain('Hello, &amp;world&amp;!')).toBe('Hello, &world&!');
+    expect(html2Plain('Hello, &#038;world&#038;!')).toBe('Hello, &world&!');
+  });
+  test('no double unescaping with other entities', async () => {
+    expect(html2Plain('Hello, &amp;#038;world&amp;#038;!')).toBe(
+      'Hello, &#038;world&#038;!',
+    );
+    expect(html2Plain('Hello, &#038;amp;world&#038;amp;!')).toBe(
+      'Hello, &amp;world&amp;!',
+    );
+    expect(html2Plain('Hello, &amp;quot;world&amp;quot;!')).toBe(
+      'Hello, &quot;world&quot;!',
+    );
+  });
+});

package/src/server/transactions/import/ofx2json.ts

@@ -0,0 +1,157 @@
+// @ts-strict-ignore
+import { parseStringPromise } from 'xml2js';
+
+import { dayFromDate } from '../../../shared/months';
+
+type OFXTransaction = {
+  amount: string;
+  fitId: string;
+  name: string;
+  date: string;
+  memo: string;
+  type: string;
+};
+
+type OFXParseResult = {
+  headers: Record<string, unknown>;
+  transactions: OFXTransaction[];
+};
+
+function sgml2Xml(sgml) {
+  return sgml
+    .replace(/&/g, '&#038;') // Replace ampersands
+    .replace(/&amp;/g, '&#038;')
+    .replace(/>\s+</g, '><') // remove whitespace inbetween tag close/open
+    .replace(/\s+</g, '<') // remove whitespace before a close tag
+    .replace(/>\s+/g, '>') // remove whitespace after a close tag
+    .replace(/\.(?=[^<>]*>)/g, '') // Remove dots in tag names
+    .replace(/<(\w+?)>([^<]+)/g, '<$1>$2</<added>$1>') // Add a new end-tags for the ofx elements
+    .replace(/<\/<added>(\w+?)>(<\/\1>)?/g, '</$1>'); // Remove duplicate end-tags
+}
+
+export function html2Plain(value) {
+  return value
+    ?.replace(/&lt;/g, '<') // lessthan
+    .replace(/&gt;/g, '>') // greaterthan
+    .replace(/&#39;/g, "'")
+    .replace(/&quot;/g, '"')
+    .replace(/(&amp;|&#038;)/g, '&'); // ampersands
+}
+
+async function parseXml(content) {
+  return await parseStringPromise(content, {
+    explicitArray: false,
+    trim: true,
+  });
+}
+
+function getStmtTrn(data) {
+  const ofx = data?.['OFX'];
+  if (ofx?.['CREDITCARDMSGSRSV1'] != null) {
+    return getCcStmtTrn(ofx);
+  } else if (ofx?.['INVSTMTMSGSRSV1'] != null) {
+    return getInvStmtTrn(ofx);
+  } else {
+    return getBankStmtTrn(ofx);
+  }
+}
+
+function getBankStmtTrn(ofx) {
+  // Somes values could be an array or a single object.
+  // xml2js serializes single item to an object and multiple to an array.
+  const msg = ofx?.['BANKMSGSRSV1'];
+  const stmtTrnRs = getAsArray(msg?.['STMTTRNRS']);
+  const result = stmtTrnRs.flatMap(s => {
+    const stmtRs = s?.['STMTRS'];
+    const tranList = stmtRs?.['BANKTRANLIST'];
+    const stmtTrn = tranList?.['STMTTRN'];
+    return getAsArray(stmtTrn);
+  });
+  return result;
+}
+
+function getCcStmtTrn(ofx) {
+  // Some values could be an array or a single object.
+  // xml2js serializes single item to an object and multiple to an array.
+  const msg = ofx?.['CREDITCARDMSGSRSV1'];
+  const stmtTrnRs = getAsArray(msg?.['CCSTMTTRNRS']);
+  const result = stmtTrnRs.flatMap(s => {
+    const stmtRs = s?.['CCSTMTRS'];
+    const tranList = stmtRs?.['BANKTRANLIST'];
+    const stmtTrn = tranList?.['STMTTRN'];
+    return getAsArray(stmtTrn);
+  });
+  return result;
+}
+
+function getInvStmtTrn(ofx) {
+  // Somes values could be an array or a single object.
+  // xml2js serializes single item to an object and multiple to an array.
+  const msg = ofx?.['INVSTMTMSGSRSV1'];
+  const stmtTrnRs = getAsArray(msg?.['INVSTMTTRNRS']);
+  const result = stmtTrnRs.flatMap(s => {
+    const stmtRs = s?.['INVSTMTRS'];
+    const tranList = stmtRs?.['INVTRANLIST'];
+    const stmtTrn = tranList?.['INVBANKTRAN']?.flatMap(t => t?.['STMTTRN']);
+    return getAsArray(stmtTrn);
+  });
+  return result;
+}
+
+function getAsArray(value) {
+  return Array.isArray(value) ? value : value === undefined ? [] : [value];
+}
+
+function mapOfxTransaction(stmtTrn): OFXTransaction {
+  // YYYYMMDDHHMMSS format. We just need the date.
+  const dtPosted = stmtTrn['DTPOSTED'];
+  const transactionDate = dtPosted
+    ? new Date(
+        Number(dtPosted.substring(0, 4)), // year
+        Number(dtPosted.substring(4, 6)) - 1, // month (zero-based index)
+        Number(dtPosted.substring(6, 8)), // date
+      )
+    : null;
+
+  return {
+    amount: stmtTrn['TRNAMT'],
+    type: stmtTrn['TRNTYPE'],
+    fitId: stmtTrn['FITID'],
+    date: dayFromDate(transactionDate),
+    name: html2Plain(stmtTrn['NAME']),
+    memo: html2Plain(stmtTrn['MEMO']),
+  };
+}
+
+export async function ofx2json(ofx: string): Promise<OFXParseResult> {
+  // firstly, split into the header attributes and the footer sgml
+  const contents = ofx.split(/<OFX\s?>/, 2);
+
+  // firstly, parse the headers
+  const headerString = contents[0].split(/\r?\n/);
+  const headers = {};
+  headerString.forEach(attrs => {
+    if (attrs) {
+      const headAttr = attrs.split(/:/, 2);
+      headers[headAttr[0]] = headAttr[1];
+    }
+  });
+
+  // make the SGML and the XML
+  const content = `<OFX>${contents[1]}`;
+
+  // Parse the XML/SGML portion of the file into an object
+  // Try as XML first, and if that fails do the SGML->XML mangling
+  let dataParsed = null;
+  try {
+    dataParsed = await parseXml(content);
+  } catch {
+    const sanitized = sgml2Xml(content);
+    dataParsed = await parseXml(sanitized);
+  }
+
+  return {
+    headers,
+    transactions: getStmtTrn(dataParsed).map(mapOfxTransaction),
+  };
+}

package/src/server/transactions/import/parse-file.test.ts

@@ -0,0 +1,224 @@
+// @ts-strict-ignore
+import * as d from 'date-fns';
+
+import { amountToInteger } from '../../../shared/util';
+import { reconcileTransactions } from '../../accounts/sync';
+import * as db from '../../db';
+import * as prefs from '../../prefs';
+
+import { parseFile } from './parse-file';
+
+beforeEach(global.emptyDatabase());
+
+// libofx spits out errors that contain the entire
+// source code of the file in the stack which makes
+// it hard to test.
+const old = console.warn;
+beforeAll(() => {
+  console.warn = vi.fn();
+});
+afterAll(() => {
+  console.warn = old;
+});
+
+type Transaction = {
+  id: string;
+  amount: number;
+  date: string;
+  payee_name: string;
+  imported_payee: string;
+  notes: string | null;
+};
+
+async function getTransactions(accountId: string): Promise<Transaction[]> {
+  return db.runQuery(
+    'SELECT * FROM transactions WHERE acct = ?',
+    [accountId],
+    true,
+  );
+}
+
+async function importFileWithRealTime(
+  accountId,
+  filepath,
+  dateFormat?: string,
+  options?: { importNotes: boolean },
+) {
+  // Emscripten requires a real Date.now!
+  global.restoreDateNow();
+  const { errors, transactions: originalTransactions } = await parseFile(
+    filepath,
+    options,
+  );
+  global.restoreFakeDateNow();
+
+  let transactions = originalTransactions;
+  if (transactions) {
+    // oxlint-disable-next-line typescript/no-explicit-any
+    transactions = (transactions as any[]).map(trans => ({
+      ...trans,
+      amount: amountToInteger(trans.amount),
+      date: dateFormat
+        ? d.format(d.parse(trans.date, dateFormat, new Date()), 'yyyy-MM-dd')
+        : trans.date,
+    }));
+  }
+  if (errors.length > 0) {
+    return { errors, added: [] };
+  }
+
+  const { added } = await reconcileTransactions(accountId, transactions);
+  return { errors, added };
+}
+
+describe('File import', () => {
+  test('qif import works', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/data.qif',
+      'MM/dd/yy',
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  });
+
+  test('ofx import works', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/data.ofx',
+      null,
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  }, 45000);
+
+  test('ofx import works (credit card)', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/credit-card.ofx',
+      null,
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  }, 45000);
+
+  test('qfx import works', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/data.qfx',
+      null,
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  }, 45000);
+
+  test('import notes are respected when importing', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    // Test with importNotes enabled
+    const { errors: errorsWithNotes } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/data.ofx',
+      null,
+      { importNotes: true },
+    );
+    expect(errorsWithNotes.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot(
+      'transactions with notes',
+    );
+
+    // Clear transactions
+    db.runQuery('DELETE FROM transactions WHERE acct = ?', ['one']);
+
+    // Test with importNotes disabled
+    const { errors: errorsWithoutNotes } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/data.ofx',
+      null,
+      { importNotes: false },
+    );
+    expect(errorsWithoutNotes.length).toBe(0);
+    const transactionsWithoutNotes = await getTransactions('one');
+    expect(transactionsWithoutNotes.every(t => t.notes === null)).toBe(true);
+  }, 45000);
+
+  test('matches extensions correctly (case-insensitive, etc)', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    let res = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/best.data-ever$.QFX',
+    );
+    expect(res.errors.length).toBe(0);
+
+    res = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/big.data.QiF',
+      'MM/dd/yy',
+    );
+    expect(res.errors.length).toBe(0);
+
+    res = await importFileWithRealTime('one', 'foo.txt');
+    expect(res.errors.length).toBe(1);
+    expect(res.errors[0].message).toBe('Invalid file type');
+  }, 45000);
+
+  test('handles non-ASCII characters', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/8859-1.qfx',
+      'yyyy-MM-dd',
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  });
+
+  test('handles html escaped plaintext', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/html-vals.qfx',
+      'yyyy-MM-dd',
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  });
+
+  test('CAMT.053 import works', async () => {
+    await prefs.loadPrefs();
+    await db.insertAccount({ id: 'one', name: 'one' });
+
+    const { errors } = await importFileWithRealTime(
+      'one',
+      __dirname + '/../../../mocks/files/camt/camt.053.xml',
+      null,
+      { importNotes: true },
+    );
+    expect(errors.length).toBe(0);
+    expect(await getTransactions('one')).toMatchSnapshot();
+  });
+});

package/src/server/transactions/import/parse-file.ts

@@ -0,0 +1,286 @@
+// @ts-strict-ignore
+import { parse as csv2json } from 'csv-parse/sync';
+
+import * as fs from '../../../platform/server/fs';
+import { logger } from '../../../platform/server/log';
+import { looselyParseAmount } from '../../../shared/util';
+
+import { ofx2json } from './ofx2json';
+import { qif2json } from './qif2json';
+import { xmlCAMT2json } from './xmlcamt2json';
+
+/**
+ * Parse OFX amount strings to numbers.
+ * Handles various OFX amount formats including currency symbols, parentheses, and multiple decimal places.
+ * Returns null for invalid amounts instead of NaN.
+ */
+function parseOfxAmount(amount: string): number | null {
+  if (!amount || typeof amount !== 'string') {
+    return null;
+  }
+
+  // Handle parentheses for negative amounts (e.g., "(30.00)" -> "-30.00")
+  let cleaned = amount.trim();
+  if (cleaned.startsWith('(') && cleaned.endsWith(')')) {
+    cleaned = '-' + cleaned.slice(1, -1);
+  }
+
+  // Remove currency symbols and other non-numeric characters except decimal point and minus sign
+  cleaned = cleaned.replace(/[^\d.-]/g, '');
+
+  // Handle multiple decimal points by keeping only the first one
+  const decimalIndex = cleaned.indexOf('.');
+  if (decimalIndex !== -1) {
+    const beforeDecimal = cleaned.slice(0, decimalIndex);
+    const afterDecimal = cleaned.slice(decimalIndex + 1).replace(/\./g, '');
+    cleaned = beforeDecimal + '.' + afterDecimal;
+  }
+
+  // Ensure we have a valid number format
+  if (!cleaned || cleaned === '-' || cleaned === '.') {
+    return null;
+  }
+
+  const parsed = parseFloat(cleaned);
+  return isNaN(parsed) ? null : parsed;
+}
+
+type StructuredTransaction = {
+  amount: number;
+  date: string;
+  payee_name: string;
+  imported_payee: string;
+  notes: string;
+};
+
+// CSV files return raw data that are not guaranteed to be StructuredTransactions
+type CsvTransaction = Record<string, string> | string[];
+
+type Transaction = StructuredTransaction | CsvTransaction;
+
+type ParseError = { message: string; internal: string };
+export type ParseFileResult = {
+  errors: ParseError[];
+  transactions?: Transaction[];
+};
+
+export type ParseFileOptions = {
+  hasHeaderRow?: boolean;
+  delimiter?: string;
+  fallbackMissingPayeeToMemo?: boolean;
+  swapPayeeAndMemo?: boolean;
+  skipStartLines?: number;
+  skipEndLines?: number;
+  importNotes?: boolean;
+};
+
+export async function parseFile(
+  filepath: string,
+  options: ParseFileOptions = {},
+): Promise<ParseFileResult> {
+  const errors = Array<ParseError>();
+  const m = filepath.match(/\.[^.]*$/);
+
+  if (m) {
+    const ext = m[0];
+
+    switch (ext.toLowerCase()) {
+      case '.qif':
+        return parseQIF(filepath, options);
+      case '.csv':
+      case '.tsv':
+        return parseCSV(filepath, options);
+      case '.ofx':
+      case '.qfx':
+        return parseOFX(filepath, options);
+      case '.xml':
+        return parseCAMT(filepath, options);
+      default:
+    }
+  }
+
+  errors.push({
+    message: 'Invalid file type',
+    internal: '',
+  });
+  return { errors, transactions: [] };
+}
+
+async function parseCSV(
+  filepath: string,
+  options: ParseFileOptions,
+): Promise<ParseFileResult> {
+  const errors = Array<ParseError>();
+  let contents = await fs.readFile(filepath);
+
+  const skipStart = Math.max(0, options.skipStartLines || 0);
+  const skipEnd = Math.max(0, options.skipEndLines || 0);
+
+  if (skipStart > 0 || skipEnd > 0) {
+    const lines = contents.split(/\r?\n/);
+
+    if (skipStart + skipEnd >= lines.length) {
+      errors.push({
+        message: 'Cannot skip more lines than exist in the file',
+        internal: `Attempted to skip ${skipStart} start + ${skipEnd} end lines from ${lines.length} total lines`,
+      });
+      return { errors, transactions: [] };
+    }
+
+    const startLine = skipStart;
+    const endLine = skipEnd > 0 ? lines.length - skipEnd : lines.length;
+    contents = lines.slice(startLine, endLine).join('\r\n');
+  }
+
+  let data: ReturnType<typeof csv2json>;
+  try {
+    data = csv2json(contents, {
+      columns: options?.hasHeaderRow,
+      bom: true,
+      delimiter: options?.delimiter || ',',
+
+      quote: '"',
+      trim: true,
+      relax_column_count: true,
+      skip_empty_lines: true,
+    });
+  } catch (err) {
+    errors.push({
+      message: 'Failed parsing: ' + err.message,
+      internal: err.message,
+    });
+    return { errors, transactions: [] };
+  }
+
+  return { errors, transactions: data };
+}
+
+async function parseQIF(
+  filepath: string,
+  options: ParseFileOptions = {},
+): Promise<ParseFileResult> {
+  const errors = Array<ParseError>();
+  const contents = await fs.readFile(filepath);
+
+  let data: ReturnType<typeof qif2json>;
+  try {
+    data = qif2json(contents);
+  } catch (err) {
+    errors.push({
+      message: "Failed parsing: doesn't look like a valid QIF file.",
+      internal: err.stack,
+    });
+    return { errors, transactions: [] };
+  }
+
+  const swap = options.swapPayeeAndMemo;
+
+  return {
+    errors: [],
+    transactions: data.transactions
+      .map(trans => {
+        const payeeSource = swap ? trans.memo : trans.payee;
+        const memoSource = swap ? trans.payee : trans.memo;
+        const fallbackUsed = !payeeSource && swap;
+
+        return {
+          amount:
+            trans.amount != null ? looselyParseAmount(trans.amount) : null,
+          date: trans.date,
+          payee_name: payeeSource || (fallbackUsed ? memoSource : null),
+          imported_payee: payeeSource || (fallbackUsed ? memoSource : null),
+          notes:
+            options.importNotes && !fallbackUsed ? memoSource || null : null,
+        };
+      })
+      .filter(trans => trans.date != null && trans.amount != null),
+  };
+}
+
+async function parseOFX(
+  filepath: string,
+  options: ParseFileOptions,
+): Promise<ParseFileResult> {
+  const errors = Array<ParseError>();
+  const contents = await fs.readFile(filepath);
+
+  let data: Awaited<ReturnType<typeof ofx2json>>;
+  try {
+    data = await ofx2json(contents);
+  } catch (err) {
+    errors.push({
+      message: 'Failed importing file',
+      internal: err.stack,
+    });
+    return { errors };
+  }
+
+  // Banks don't always implement the OFX standard properly
+  // If no payee is available try and fallback to memo
+  const useMemoFallback = options.fallbackMissingPayeeToMemo;
+  const swap = options.swapPayeeAndMemo;
+
+  return {
+    errors,
+    transactions: data.transactions.map(trans => {
+      const parsedAmount = parseOfxAmount(trans.amount);
+      if (parsedAmount === null) {
+        errors.push({
+          message: `Invalid amount format: ${trans.amount}`,
+          internal: `Failed to parse amount: ${trans.amount}`,
+        });
+      }
+
+      const payeeSource = swap ? trans.memo : trans.name;
+      const memoSource = swap ? trans.name : trans.memo;
+      const fallbackUsed = !payeeSource && useMemoFallback;
+
+      return {
+        amount: parsedAmount || 0,
+        imported_id: trans.fitId,
+        date: trans.date,
+        payee_name: payeeSource || (fallbackUsed ? memoSource : null),
+        imported_payee: payeeSource || (fallbackUsed ? memoSource : null),
+        notes: options.importNotes && !fallbackUsed ? memoSource || null : null,
+      };
+    }),
+  };
+}
+
+async function parseCAMT(
+  filepath: string,
+  options: ParseFileOptions = {},
+): Promise<ParseFileResult> {
+  const errors = Array<ParseError>();
+  const contents = await fs.readFile(filepath);
+
+  let data: Awaited<ReturnType<typeof xmlCAMT2json>>;
+  try {
+    data = await xmlCAMT2json(contents);
+  } catch (err) {
+    logger.error(err);
+    errors.push({
+      message: 'Failed importing file',
+      internal: err.stack,
+    });
+    return { errors };
+  }
+
+  const swap = options.swapPayeeAndMemo;
+
+  return {
+    errors,
+    transactions: data.map(trans => {
+      const payeeSource = swap ? trans.notes : trans.payee_name;
+      const memoSource = swap ? trans.payee_name : trans.notes;
+      const fallbackUsed = !payeeSource && swap;
+
+      return {
+        ...trans,
+        payee_name: payeeSource || (fallbackUsed ? memoSource : null),
+        imported_payee: payeeSource || (fallbackUsed ? memoSource : null),
+        notes: options.importNotes && !fallbackUsed ? memoSource || null : null,
+      };
+    }),
+  };
+}