@actual-app/core 26.3.0

package/src/server/importers/index.ts

@@ -0,0 +1,58 @@
+// @ts-strict-ignore
+import { logger } from '../../platform/server/log';
+import { handlers } from '../main';
+
+import { importActual } from './actual';
+import * as YNAB4 from './ynab4';
+import * as YNAB5 from './ynab5';
+
+export type ImportableBudgetType = 'ynab4' | 'ynab5' | 'actual';
+
+type Importer = {
+  parseFile(buffer: Buffer): unknown;
+  getBudgetName(filepath: string, data: unknown): string | null;
+  doImport(data: unknown): Promise<void>;
+};
+
+const importers: Record<Exclude<ImportableBudgetType, 'actual'>, Importer> = {
+  ynab4: YNAB4,
+  ynab5: YNAB5,
+};
+
+export async function handleBudgetImport(
+  type: ImportableBudgetType,
+  filepath: string,
+  buffer: Buffer,
+) {
+  if (type === 'actual') {
+    return importActual(filepath, buffer);
+  }
+  const importer = importers[type];
+  try {
+    let data;
+    let budgetName: string;
+    try {
+      data = importer.parseFile(buffer);
+      budgetName = importer.getBudgetName(filepath, data);
+    } catch (e) {
+      logger.error('failed to parse file', e);
+    }
+    if (!budgetName) {
+      return { error: 'not-' + type };
+    }
+
+    try {
+      await handlers['api/start-import']({ budgetName });
+    } catch (e) {
+      logger.error('failed to start import', e);
+      return { error: 'unknown' };
+    }
+    await importer.doImport(data);
+  } catch (e) {
+    await handlers['api/abort-import']();
+    logger.error('failed to run import', e);
+    return { error: 'unknown' };
+  }
+
+  await handlers['api/finish-import']();
+}

package/src/server/importers/ynab4-types.ts

@@ -0,0 +1,163 @@
+export type YFull = {
+  masterCategories: MasterCategory[];
+  payees: Payee[];
+  monthlyBudgets: MonthlyBudget[];
+  fileMetaData: FileMetaData;
+  transactions: Transaction[];
+  scheduledTransactions: ScheduledTransaction[];
+  // accountMappings: [];
+  budgetMetaData: BudgetMetaData;
+  accounts: Account[];
+};
+
+export type MasterCategory = {
+  entityType: string;
+  expanded: boolean;
+  note?: string;
+  name: string;
+  type: string;
+  deleteable: boolean;
+  subCategories?: SubCategory[];
+  entityVersion: string;
+  entityId: string;
+  sortableIndex: number;
+
+  // speculative
+  isTombstone?: boolean;
+};
+
+export type SubCategory = {
+  entityType: string;
+  name: string;
+  note?: string;
+  type: string;
+  // cachedBalance: null;
+  masterCategoryId: string;
+  entityVersion: string;
+  entityId: string;
+  sortableIndex: number;
+
+  // speculative
+  isTombstone?: boolean;
+};
+
+export type Payee = {
+  entityType: string;
+  autoFillCategoryId?: string;
+  autoFillAmount: number;
+  name: string;
+  renameConditions?: RenameCondition[];
+  autoFillMemo?: string;
+  targetAccountId?: string;
+  // locations: null;
+  enabled: boolean;
+  entityVersion: string;
+  entityId: string;
+
+  // speculative
+  isTombstone?: boolean;
+};
+
+export type RenameCondition = {
+  entityType: string;
+  parentPayeeId: string;
+  operator: string;
+  operand: string;
+  entityVersion: string;
+  entityId: string;
+};
+
+export type MonthlyBudget = {
+  entityType: string;
+  monthlySubCategoryBudgets: MonthlySubCategoryBudget[];
+  month: string;
+  entityVersion: string;
+  entityId: string;
+};
+
+export type MonthlySubCategoryBudget = {
+  entityType: string;
+  categoryId: string;
+  budgeted: number;
+  overspendingHandling?: string | undefined | null;
+  entityVersion: string;
+  entityId: string;
+  parentMonthlyBudgetId: string;
+
+  // speculative
+  isTombstone?: boolean;
+};
+
+export type FileMetaData = {
+  entityType: string;
+  budgetDataVersion: string;
+  currentKnowledge: string;
+};
+
+export type Transaction = {
+  entityType: string;
+  entityId: string;
+  categoryId: string;
+  payeeId: string;
+  amount: number;
+  date: string;
+  accountId: string;
+  entityVersion: string;
+  cleared: string;
+  accepted: boolean;
+  isTombstone?: boolean;
+  memo?: string;
+  dateEnteredFromSchedule?: string;
+  // speculative:
+  subTransactions?: SubTransaction[];
+  transferTransactionId?: string;
+  targetAccountId?: string;
+};
+
+// speculative, not in the test data
+export type SubTransaction = Omit<Transaction, 'subTransactions'>;
+
+export type ScheduledTransaction = {
+  entityType: string;
+  entityId: string;
+  categoryId: string;
+  payeeId: string;
+  amount: number;
+  date: string;
+  isTombstone?: boolean;
+  accountId: string;
+  entityVersion: string;
+  memo: string;
+  twiceAMonthStartDay: number;
+  cleared: string;
+  frequency: string;
+  accepted: boolean;
+};
+
+export type BudgetMetaData = {
+  entityType: string;
+  strictBudget: string;
+  currencyISOSymbol?: string;
+  entityVersion: string;
+  currencyLocale: string;
+  budgetType: string;
+  dateLocale: string;
+  entityId: string;
+};
+
+export type Account = {
+  entityType: string;
+  // lastReconciledDate: null;
+  lastEnteredCheckNumber: number;
+  lastReconciledBalance: number;
+  accountType: string;
+  hidden: boolean;
+  sortableIndex: number;
+  onBudget: boolean;
+  accountName: string;
+  entityVersion: string;
+  entityId: string;
+
+  // speculative
+  isTombstone?: boolean;
+};

package/src/server/importers/ynab4.ts

@@ -0,0 +1,470 @@
+// @ts-strict-ignore
+import AdmZip from 'adm-zip';
+import normalizePathSep from 'slash';
+import { v4 as uuidv4 } from 'uuid';
+
+import { logger } from '../../platform/server/log';
+import * as monthUtils from '../../shared/months';
+import { amountToInteger, groupBy, sortByKey } from '../../shared/util';
+import { send } from '../main-app';
+
+import type * as YNAB4 from './ynab4-types';
+
+// Importer
+
+async function importAccounts(
+  data: YNAB4.YFull,
+  entityIdMap: Map<string, string>,
+) {
+  const accounts = sortByKey(data.accounts, 'sortableIndex');
+
+  return Promise.all(
+    accounts.map(async account => {
+      if (!account.isTombstone) {
+        const id = await send('api/account-create', {
+          account: {
+            name: account.accountName,
+            offbudget: account.onBudget ? false : true,
+            closed: account.hidden ? true : false,
+          },
+        });
+        entityIdMap.set(account.entityId, id);
+      }
+    }),
+  );
+}
+
+async function importCategories(
+  data: YNAB4.YFull,
+  entityIdMap: Map<string, string>,
+) {
+  const masterCategories = sortByKey(data.masterCategories, 'sortableIndex');
+
+  await Promise.all(
+    masterCategories.map(async masterCategory => {
+      if (
+        masterCategory.type === 'OUTFLOW' &&
+        !masterCategory.isTombstone &&
+        masterCategory.subCategories &&
+        masterCategory.subCategories.some(cat => !cat.isTombstone)
+      ) {
+        const id = await send('api/category-group-create', {
+          group: {
+            name: masterCategory.name,
+            is_income: false,
+          },
+        });
+        entityIdMap.set(masterCategory.entityId, id);
+        if (masterCategory.note) {
+          void send('notes-save', {
+            id,
+            note: masterCategory.note,
+          });
+        }
+
+        if (masterCategory.subCategories) {
+          const subCategories = sortByKey(
+            masterCategory.subCategories,
+            'sortableIndex',
+          );
+          subCategories.reverse();
+
+          // This can't be done in parallel because sort order depends
+          // on insertion order
+          for (const category of subCategories) {
+            if (!category.isTombstone) {
+              let categoryName = category.name;
+
+              // Hidden categories have the parent category entity id
+              // appended to the end of the sub category name.
+              // The format is 'MasterCategory ` SubCategory ` entityId'.
+              // Remove the id to shorten the name.
+              if (masterCategory.name === 'Hidden Categories') {
+                const categoryNameParts = categoryName.split(' ` ');
+
+                // Remove the last part, which is the entityId.
+                categoryNameParts.pop();
+
+                // Join the remaining parts with a slash between them.
+                categoryName = categoryNameParts.join('/').trim();
+              }
+
+              const id = await send('api/category-create', {
+                category: {
+                  name: categoryName,
+                  group_id: entityIdMap.get(category.masterCategoryId),
+                },
+              });
+              entityIdMap.set(category.entityId, id);
+              if (category.note) {
+                void send('notes-save', {
+                  id,
+                  note: category.note,
+                });
+              }
+            }
+          }
+        }
+      }
+    }),
+  );
+}
+
+async function importPayees(
+  data: YNAB4.YFull,
+  entityIdMap: Map<string, string>,
+) {
+  for (const payee of data.payees) {
+    if (!payee.isTombstone) {
+      const id = await send('api/payee-create', {
+        payee: {
+          name: payee.name,
+          transfer_acct: entityIdMap.get(payee.targetAccountId) || null,
+        },
+      });
+
+      // TODO: import payee rules
+
+      entityIdMap.set(payee.entityId, id);
+    }
+  }
+}
+
+async function importTransactions(
+  data: YNAB4.YFull,
+  entityIdMap: Map<string, string>,
+) {
+  const categories = await send('api/categories-get', {
+    grouped: false,
+  });
+  const incomeCategoryId: string = categories.find(
+    cat => cat.name === 'Income',
+  ).id;
+  const accounts = await send('api/accounts-get');
+  const payees = await send('api/payees-get');
+
+  function getCategory(id: string) {
+    if (id == null || id === 'Category/__Split__') {
+      return null;
+    } else if (
+      id === 'Category/__ImmediateIncome__' ||
+      id === 'Category/__DeferredIncome__'
+    ) {
+      return incomeCategoryId;
+    }
+    return entityIdMap.get(id);
+  }
+
+  function isOffBudget(acctId: string) {
+    const acct = accounts.find(acct => acct.id === acctId);
+    if (!acct) {
+      throw new Error('Could not find account for transaction when importing');
+    }
+    return acct.offbudget;
+  }
+
+  // Go ahead and generate ids for all of the transactions so we can
+  // reliably resolve transfers
+  for (const transaction of data.transactions) {
+    entityIdMap.set(transaction.entityId, uuidv4());
+
+    if (transaction.subTransactions) {
+      for (const subTransaction of transaction.subTransactions) {
+        entityIdMap.set(subTransaction.entityId, uuidv4());
+      }
+    }
+  }
+
+  const transactionsGrouped = groupBy(data.transactions, 'accountId');
+
+  await Promise.all(
+    [...transactionsGrouped.keys()].map(async accountId => {
+      const transactions = transactionsGrouped.get(accountId);
+
+      const toImport = transactions
+        .map(transaction => {
+          if (transaction.isTombstone) {
+            return null;
+          }
+
+          const id = entityIdMap.get(transaction.entityId);
+
+          function transferProperties(t: YNAB4.SubTransaction) {
+            const transferId = entityIdMap.get(t.transferTransactionId) || null;
+
+            let payee = null;
+            let imported_payee = null;
+            if (transferId) {
+              payee = payees.find(
+                p => p.transfer_acct === entityIdMap.get(t.targetAccountId),
+              ).id;
+            } else {
+              payee = entityIdMap.get(t.payeeId);
+              imported_payee = data.payees.find(
+                p => p.entityId === t.payeeId,
+              )?.name;
+            }
+
+            return {
+              transfer_id: transferId,
+              payee,
+              imported_payee,
+            };
+          }
+
+          const newTransaction = {
+            id,
+            amount: amountToInteger(transaction.amount),
+            category: isOffBudget(entityIdMap.get(accountId))
+              ? null
+              : getCategory(transaction.categoryId),
+            date: transaction.date,
+            notes: transaction.memo || null,
+            cleared:
+              transaction.cleared === 'Cleared' ||
+              transaction.cleared === 'Reconciled',
+            reconciled: transaction.cleared === 'Reconciled',
+            ...transferProperties(transaction),
+
+            subtransactions:
+              transaction.subTransactions &&
+              transaction.subTransactions
+                .filter(st => !st.isTombstone)
+                .map(t => {
+                  return {
+                    id: entityIdMap.get(t.entityId),
+                    amount: amountToInteger(t.amount),
+                    category: getCategory(t.categoryId),
+                    notes: t.memo || null,
+                    ...transferProperties(t),
+                  };
+                }),
+          };
+
+          return newTransaction;
+        })
+        .filter(x => x);
+
+      await send('api/transactions-add', {
+        accountId: entityIdMap.get(accountId),
+        transactions: toImport,
+        learnCategories: true,
+        runTransfers: false,
+      });
+    }),
+  );
+}
+
+function fillInBudgets(
+  data: YNAB4.YFull,
+  categoryBudgets: YNAB4.MonthlySubCategoryBudget[],
+) {
+  // YNAB only contains entries for categories that have been actually
+  // budgeted. That would be fine except that we need to set the
+  // "carryover" flag on each month when carrying debt across months.
+  // To make sure our system has a chance to set this flag on each
+  // category, make sure a budget exists for every category of every
+  // month.
+  const budgets: {
+    budgeted: number;
+    categoryId: string;
+    overspendingHandling?: string;
+  }[] = [...categoryBudgets];
+  data.masterCategories.forEach(masterCategory => {
+    if (masterCategory.subCategories) {
+      masterCategory.subCategories.forEach(category => {
+        if (!budgets.find(b => b.categoryId === category.entityId)) {
+          budgets.push({
+            budgeted: 0,
+            categoryId: category.entityId,
+          });
+        }
+      });
+    }
+  });
+  return budgets;
+}
+
+async function importBudgets(
+  data: YNAB4.YFull,
+  entityIdMap: Map<string, string>,
+) {
+  const budgets = sortByKey(data.monthlyBudgets, 'month');
+
+  await send('api/batch-budget-start');
+  try {
+    for (const budget of budgets) {
+      const filled = fillInBudgets(
+        data,
+        budget.monthlySubCategoryBudgets.filter(b => !b.isTombstone),
+      );
+
+      await Promise.all(
+        filled.map(async catBudget => {
+          const amount = amountToInteger(catBudget.budgeted);
+          const catId = entityIdMap.get(catBudget.categoryId);
+          const month = monthUtils.monthFromDate(budget.month);
+          if (!catId) {
+            return;
+          }
+
+          await send('api/budget-set-amount', {
+            month,
+            categoryId: catId,
+            amount,
+          });
+
+          if (catBudget.overspendingHandling === 'AffectsBuffer') {
+            await send('api/budget-set-carryover', {
+              month,
+              categoryId: catId,
+              flag: false,
+            });
+          } else if (catBudget.overspendingHandling === 'Confined') {
+            await send('api/budget-set-carryover', {
+              month,
+              categoryId: catId,
+              flag: true,
+            });
+          }
+        }),
+      );
+    }
+  } finally {
+    await send('api/batch-budget-end');
+  }
+}
+
+function estimateRecentness(str: string) {
+  // The "recentness" is the total amount of changes that this device
+  // is aware of, which is estimated by summing up all of the version
+  // numbers that its aware of. This works because version numbers are
+  // increasing integers.
+  return str.split(',').reduce((total, version) => {
+    const [_, number] = version.split('-');
+    return total + parseInt(number);
+  }, 0);
+}
+
+function findLatestDevice(zipped: AdmZip, entries: AdmZip.IZipEntry[]): string {
+  let devices = entries
+    .map(entry => {
+      const contents = zipped.readFile(entry).toString('utf8');
+
+      let data;
+      try {
+        data = JSON.parse(contents);
+      } catch {
+        return null;
+      }
+
+      if (data.hasFullKnowledge) {
+        return {
+          deviceGUID: data.deviceGUID,
+          shortName: data.shortDeviceId,
+          recentness: estimateRecentness(data.knowledge),
+        };
+      }
+
+      return null;
+    })
+    .filter(x => x);
+
+  devices = sortByKey(devices, 'recentness');
+  return devices[devices.length - 1].deviceGUID;
+}
+
+export async function doImport(data: YNAB4.YFull) {
+  const entityIdMap = new Map<string, string>();
+
+  logger.log('Importing Accounts...');
+  await importAccounts(data, entityIdMap);
+
+  logger.log('Importing Categories...');
+  await importCategories(data, entityIdMap);
+
+  logger.log('Importing Payees...');
+  await importPayees(data, entityIdMap);
+
+  logger.log('Importing Transactions...');
+  await importTransactions(data, entityIdMap);
+
+  logger.log('Importing Budgets...');
+  await importBudgets(data, entityIdMap);
+
+  logger.log('Setting up...');
+}
+
+export function getBudgetName(filepath) {
+  let unixFilepath = normalizePathSep(filepath);
+
+  if (!/\.zip/.test(unixFilepath)) {
+    return null;
+  }
+
+  unixFilepath = unixFilepath.replace(/\.zip$/, '').replace(/.ynab4$/, '');
+
+  // Most budgets are named like "Budget~51938D82.ynab4" but sometimes
+  // they are only "Budget.ynab4". We only want to grab the name
+  // before the ~ if it exists.
+  const m = unixFilepath.match(/([^/~]+)[^/]*$/);
+  if (!m) {
+    return null;
+  }
+  return m[1];
+}
+
+function getFile(entries: AdmZip.IZipEntry[], path: string) {
+  const files = entries.filter(e => e.entryName === path);
+  if (files.length === 0) {
+    throw new Error('Could not find file: ' + path);
+  }
+  if (files.length >= 2) {
+    throw new Error('File name matches multiple files: ' + path);
+  }
+  return files[0];
+}
+
+function join(...paths: string[]): string {
+  return paths.slice(1).reduce(
+    (full, path) => {
+      return full + '/' + path.replace(/^\//, '');
+    },
+    paths[0].replace(/\/$/, ''),
+  );
+}
+
+export function parseFile(buffer: Buffer): YNAB4.YFull {
+  const zipped = new AdmZip(buffer);
+  const entries = zipped.getEntries();
+
+  let root = '';
+  const dirMatch = entries[0].entryName.match(/([^/]*\.ynab4)/);
+  if (dirMatch) {
+    root = dirMatch[1] + '/';
+  }
+
+  const metaStr = zipped.readFile(getFile(entries, root + 'Budget.ymeta'));
+  const meta = JSON.parse(metaStr.toString('utf8'));
+  const budgetPath = join(root, meta.relativeDataFolderName);
+
+  const deviceFiles = entries.filter(e =>
+    e.entryName.startsWith(join(budgetPath, 'devices')),
+  );
+  const deviceGUID = findLatestDevice(zipped, deviceFiles);
+
+  const yfullPath = join(budgetPath, deviceGUID, 'Budget.yfull');
+  let contents;
+  try {
+    contents = zipped.readFile(getFile(entries, yfullPath)).toString('utf8');
+  } catch (e) {
+    logger.log(e);
+    throw new Error('Error reading Budget.yfull file');
+  }
+
+  try {
+    return JSON.parse(contents);
+  } catch {
+    throw new Error('Error parsing Budget.yfull file');
+  }
+}