@actual-app/core 26.3.0

package/src/server/tests/mockSyncServer.ts

@@ -0,0 +1,119 @@
+// @ts-strict-ignore
+import { makeClock, merkle, SyncProtoBuf, Timestamp } from '@actual-app/crdt';
+import type { Clock } from '@actual-app/crdt';
+
+import type { Message } from '../sync';
+
+import { basic as defaultMockData } from './mockData.json';
+
+const handlers = {};
+let currentMockData = defaultMockData;
+let currentClock = makeClock(new Timestamp(0, 0, '0000000000000000'));
+let currentMessages: {
+  timestamp: string;
+  is_encrypted: boolean;
+  content: Uint8Array;
+}[] = [];
+
+// Ugh, this is duplicated...
+function deserializeValue(value) {
+  const type = value[0];
+  switch (type) {
+    case '0':
+      return null;
+    case 'N':
+      return parseFloat(value.slice(2));
+    case 'S':
+      return value.slice(2);
+    default:
+  }
+
+  throw new Error('Invalid type key for value: ' + value);
+}
+
+handlers['/'] = () => {
+  return 'development';
+};
+
+handlers['/sync/sync'] = async (data: Uint8Array): Promise<Uint8Array> => {
+  const requestPb = SyncProtoBuf.SyncRequest.deserializeBinary(data);
+  const since = requestPb.getSince();
+  const messages = requestPb.getMessagesList();
+
+  const newMessages = currentMessages.filter(msg => msg.timestamp > since);
+
+  messages.forEach(msg => {
+    if (!currentMessages.find(m => m.timestamp === msg.getTimestamp())) {
+      currentMessages.push({
+        timestamp: msg.getTimestamp(),
+        is_encrypted: msg.getIsencrypted(),
+        content: msg.getContent_asU8(),
+      });
+
+      currentClock.merkle = merkle.insert(
+        currentClock.merkle,
+        Timestamp.parse(msg.getTimestamp()),
+      );
+    }
+  });
+
+  currentClock.merkle = merkle.prune(currentClock.merkle);
+
+  const responsePb = new SyncProtoBuf.SyncResponse();
+  responsePb.setMerkle(JSON.stringify(currentClock.merkle));
+
+  newMessages.forEach(msg => {
+    const envelopePb = new SyncProtoBuf.MessageEnvelope();
+    envelopePb.setTimestamp(msg.timestamp);
+    envelopePb.setIsencrypted(msg.is_encrypted);
+    envelopePb.setContent(msg.content);
+    responsePb.addMessages(envelopePb);
+  });
+
+  return responsePb.serializeBinary();
+};
+
+handlers['/gocardless/accounts'] = () => {
+  // Ignore the parameters and just return the accounts.
+  return { accounts: currentMockData.accounts };
+};
+
+export const filterMockData = func => {
+  const copied = JSON.parse(JSON.stringify(defaultMockData));
+  currentMockData = func(copied);
+};
+
+export const reset = () => {
+  currentMockData = defaultMockData;
+  currentClock = makeClock(new Timestamp(0, 0, '0000000000000000'));
+  currentMessages = [];
+};
+
+export const getClock = (): Clock => {
+  return currentClock;
+};
+
+export const getMessages = (): Message[] => {
+  return currentMessages.map(msg => {
+    const { timestamp, content } = msg;
+    const fields = SyncProtoBuf.Message.deserializeBinary(content);
+
+    return {
+      timestamp: Timestamp.parse(timestamp),
+      dataset: fields.getDataset(),
+      row: fields.getRow(),
+      column: fields.getColumn(),
+      value: deserializeValue(fields.getValue()),
+    };
+  });
+};
+
+export const handleRequest = (url, data) => {
+  url = url.replace(/http(s)?:\/\/[^/]*/, '');
+  if (!handlers[url]) {
+    throw new Error('No url handler for ' + url);
+  }
+  return Promise.resolve(handlers[url](data));
+};
+
+export { handlers, handleRequest as handleRequestBinary };

package/src/server/tools/app.ts

@@ -0,0 +1,152 @@
+// @ts-strict-ignore
+import { q } from '../../shared/query';
+import type { TransactionEntity } from '../../types/models';
+import { createApp } from '../app';
+import { aqlQuery } from '../aql';
+import * as db from '../db';
+import { runMutator } from '../mutators';
+import { batchUpdateTransactions } from '../transactions';
+
+export type ToolsHandlers = {
+  'tools/fix-split-transactions': typeof fixSplitTransactions;
+};
+
+export const app = createApp<ToolsHandlers>();
+
+app.method('tools/fix-split-transactions', fixSplitTransactions);
+
+async function fixSplitTransactions(): Promise<{
+  numBlankPayees: number;
+  numCleared: number;
+  numDeleted: number;
+  numTransfersFixed: number;
+  numNonParentErrorsFixed: number;
+  numParentTransactionsWithCategoryFixed: number;
+  mismatchedSplits: TransactionEntity[];
+}> {
+  // 1. Check for child transactions that have a blank payee, and set
+  //    the payee to whatever the parent has
+  const blankPayeeRows = await db.all<
+    db.DbViewTransactionInternal & {
+      parentPayee: db.DbViewTransactionInternal['payee'];
+    }
+  >(`
+    SELECT t.*, p.payee AS parentPayee FROM v_transactions_internal t
+    LEFT JOIN v_transactions_internal p ON t.parent_id = p.id
+    WHERE t.is_child = 1 AND t.payee IS NULL AND p.payee IS NOT NULL
+  `);
+
+  await runMutator(async () => {
+    const updated = blankPayeeRows.map(row => ({
+      id: row.id,
+      payee: row.parentPayee,
+    }));
+    await batchUpdateTransactions({ updated });
+  });
+
+  // 2. Make sure the "cleared" flag is synced up with the parent
+  // transactions
+  const clearedRows = await db.all<
+    Pick<db.DbViewTransactionInternal, 'id'> &
+      Pick<db.DbViewTransactionInternal, 'cleared'>
+  >(`
+    SELECT t.id, p.cleared FROM v_transactions_internal t
+    LEFT JOIN v_transactions_internal p ON t.parent_id = p.id
+    WHERE t.is_child = 1 AND t.cleared != p.cleared
+  `);
+
+  await runMutator(async () => {
+    const updated = clearedRows.map(row => ({
+      id: row.id,
+      cleared: row.cleared === 1,
+    }));
+    await batchUpdateTransactions({ updated });
+  });
+
+  // 3. Mark the `tombstone` field as true on any child transactions
+  //    that have a dead parent
+  const deletedRows = await db.all<db.DbViewTransactionInternal>(`
+    SELECT t.* FROM v_transactions_internal t
+    LEFT JOIN v_transactions_internal p ON t.parent_id = p.id
+    WHERE t.is_child = 1 AND t.tombstone = 0 AND (p.tombstone = 1 OR p.id IS NULL)
+  `);
+
+  await runMutator(async () => {
+    const updated = deletedRows.map(row => ({ id: row.id, tombstone: true }));
+    await batchUpdateTransactions({ updated });
+  });
+
+  const splitTransactions = (
+    await aqlQuery(
+      q('transactions')
+        .options({ splits: 'grouped' })
+        .filter({
+          is_parent: true,
+        })
+        .select('*'),
+    )
+  ).data;
+
+  const mismatchedSplits = splitTransactions.filter(t => {
+    const subValue = t.subtransactions.reduce((acc, st) => acc + st.amount, 0);
+
+    return subValue !== t.amount;
+  });
+
+  // 5. Fix transfers that should not have categories
+  const brokenTransfers = await db.all<
+    Pick<db.DbViewTransactionInternal, 'id'>
+  >(`
+    SELECT t1.id
+    FROM v_transactions_internal t1
+           JOIN accounts a1 ON t1.account = a1.id
+           JOIN v_transactions_internal t2 ON t1.transfer_id = t2.id
+           JOIN accounts a2 ON t2.account = a2.id
+    WHERE a1.offbudget = a2.offbudget
+      AND t1.category IS NOT NULL
+  `);
+
+  await runMutator(async () => {
+    const updated = brokenTransfers.map(row => ({
+      id: row.id,
+      category: null,
+    }));
+    await batchUpdateTransactions({ updated });
+  });
+
+  // 6. Remove transaction errors from non-parent transactions
+  const errorRows = await db.all<Pick<db.DbViewTransactionInternal, 'id'>>(`
+    SELECT id FROM v_transactions_internal WHERE error IS NOT NULL AND is_parent = 0
+  `);
+
+  await runMutator(async () => {
+    const updated = errorRows.map(({ id }) => ({ id, error: null }));
+    await batchUpdateTransactions({ updated });
+  });
+
+  // 7. Clear categories of parent transactions
+  const parentTransactionsWithCategory = await db.all<
+    Pick<db.DbViewTransactionInternal, 'id'>
+  >(`
+    SELECT id FROM transactions WHERE isParent = 1 AND category IS NOT NULL
+  `);
+
+  await runMutator(async () => {
+    const updated = parentTransactionsWithCategory.map(({ id }) => ({
+      id,
+      category: null,
+    }));
+    await batchUpdateTransactions({ updated });
+  });
+
+  return {
+    numBlankPayees: blankPayeeRows.length,
+    numCleared: clearedRows.length,
+    numDeleted: deletedRows.length,
+    numTransfersFixed: brokenTransfers.length,
+    numNonParentErrorsFixed: errorRows.length,
+    numParentTransactionsWithCategoryFixed:
+      parentTransactionsWithCategory.length,
+    mismatchedSplits,
+  };
+}

package/src/server/transactions/__snapshots__/transaction-rules.test.ts.snap

@@ -0,0 +1,173 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Learning categories > updates rules correctly even if multiple rules exist 1`] = `
+[
+  Rule {
+    "actions": [
+      Action {
+        "field": "category",
+        "handlebarsTemplate": undefined,
+        "op": "set",
+        "options": undefined,
+        "rawValue": "food",
+        "type": "id",
+        "value": "food",
+      },
+    ],
+    "conditions": [
+      Condition {
+        "field": "payee",
+        "op": "is",
+        "options": undefined,
+        "rawValue": "foo",
+        "type": "id",
+        "unparsedValue": "foo",
+        "value": "foo",
+      },
+    ],
+    "conditionsOp": "and",
+    "id": "id2",
+    "stage": null,
+  },
+  Rule {
+    "actions": [
+      Action {
+        "field": "category",
+        "handlebarsTemplate": undefined,
+        "op": "set",
+        "options": undefined,
+        "rawValue": "food",
+        "type": "id",
+        "value": "food",
+      },
+    ],
+    "conditions": [
+      Condition {
+        "field": "payee",
+        "op": "is",
+        "options": undefined,
+        "rawValue": "foo",
+        "type": "id",
+        "unparsedValue": "foo",
+        "value": "foo",
+      },
+    ],
+    "conditionsOp": "and",
+    "id": "id3",
+    "stage": null,
+  },
+  Rule {
+    "actions": [
+      Action {
+        "field": "category",
+        "handlebarsTemplate": undefined,
+        "op": "set",
+        "options": undefined,
+        "rawValue": "beer",
+        "type": "id",
+        "value": "beer",
+      },
+    ],
+    "conditions": [
+      Condition {
+        "field": "payee",
+        "op": "is",
+        "options": undefined,
+        "rawValue": null,
+        "type": "id",
+        "unparsedValue": null,
+        "value": null,
+      },
+    ],
+    "conditionsOp": "and",
+    "id": "id4",
+    "stage": null,
+  },
+]
+`;
+
+exports[`Learning categories > updates rules correctly even if multiple rules exist 2`] = `
+[
+  Rule {
+    "actions": [
+      Action {
+        "field": "category",
+        "handlebarsTemplate": undefined,
+        "op": "set",
+        "options": undefined,
+        "rawValue": "beer",
+        "type": "id",
+        "value": "beer",
+      },
+    ],
+    "conditions": [
+      Condition {
+        "field": "payee",
+        "op": "is",
+        "options": undefined,
+        "rawValue": "foo",
+        "type": "id",
+        "unparsedValue": "foo",
+        "value": "foo",
+      },
+    ],
+    "conditionsOp": "and",
+    "id": "id2",
+    "stage": null,
+  },
+  Rule {
+    "actions": [
+      Action {
+        "field": "category",
+        "handlebarsTemplate": undefined,
+        "op": "set",
+        "options": undefined,
+        "rawValue": "beer",
+        "type": "id",
+        "value": "beer",
+      },
+    ],
+    "conditions": [
+      Condition {
+        "field": "payee",
+        "op": "is",
+        "options": undefined,
+        "rawValue": "foo",
+        "type": "id",
+        "unparsedValue": "foo",
+        "value": "foo",
+      },
+    ],
+    "conditionsOp": "and",
+    "id": "id3",
+    "stage": null,
+  },
+  Rule {
+    "actions": [
+      Action {
+        "field": "category",
+        "handlebarsTemplate": undefined,
+        "op": "set",
+        "options": undefined,
+        "rawValue": "beer",
+        "type": "id",
+        "value": "beer",
+      },
+    ],
+    "conditions": [
+      Condition {
+        "field": "payee",
+        "op": "is",
+        "options": undefined,
+        "rawValue": null,
+        "type": "id",
+        "unparsedValue": null,
+        "value": null,
+      },
+    ],
+    "conditionsOp": "and",
+    "id": "id4",
+    "stage": null,
+  },
+]
+`;