@actual-app/core 26.3.0

package/src/server/sync/make-test-message.ts

@@ -0,0 +1,19 @@
+// @ts-strict-ignore
+import { SyncProtoBuf } from '@actual-app/crdt';
+
+import * as encryption from '../encryption';
+
+function randomString() {
+  return encryption.randomBytes(12).toString();
+}
+
+export async function makeTestMessage(keyId) {
+  const messagePb = new SyncProtoBuf.Message();
+  messagePb.setDataset(randomString());
+  messagePb.setRow(randomString());
+  messagePb.setColumn(randomString());
+  messagePb.setValue(randomString());
+  const binaryMsg = messagePb.serializeBinary();
+
+  return await encryption.encrypt(binaryMsg, keyId);
+}

package/src/server/sync/migrate.test.ts

@@ -0,0 +1,169 @@
+// @ts-strict-ignore
+import { Timestamp } from '@actual-app/crdt';
+import fc from 'fast-check';
+
+import * as arbs from '../../mocks/arbitrary-schema';
+import { execTracer } from '../../shared/test-helpers';
+import { convertInputType, schema, schemaConfig } from '../aql';
+import * as db from '../db';
+
+import { listen, unlisten } from './migrate';
+
+import { addSyncListener, sendMessages } from './index';
+import type { Message } from './index';
+
+beforeEach(() => {
+  listen();
+  return global.emptyDatabase()();
+});
+
+afterEach(() => {
+  unlisten();
+});
+
+const tableSchema = schema.transactions;
+const fields = Object.keys(tableSchema);
+
+function toInternalField(publicField) {
+  return schemaConfig.views.transactions.fields[publicField];
+}
+
+const messageArb: fc.Arbitrary<Message> = fc
+  .oneof(...fields.filter(f => f !== 'id').map(field => fc.constant(field)))
+  .chain(field => {
+    const value = arbs
+      .typeArbitrary(tableSchema[field])
+      .map(v => convertInputType(v, tableSchema[field].type));
+
+    const timestamp = fc
+      .noShrink(
+        fc.noBias(
+          fc.date({
+            min: new Date('2020-01-01T00:00:00.000Z'),
+            max: new Date('2020-05-01T00:00:00.000Z'),
+          }),
+        ),
+      )
+      .map(date => date.toISOString() + '-0000-0123456789ABCDEF')
+      .map(ts => Timestamp.parse(ts));
+
+    return fc.record<Message>({
+      timestamp,
+      dataset: fc.constant('transactions'),
+      column: fc.constant(toInternalField(field) || field),
+      row: fc.oneof(
+        fc.integer({ min: 0, max: 5 }).map(i => `id${i}`),
+        fc.integer({ min: 0, max: 5 }).chain(i => {
+          return fc.integer({ min: 0, max: 5 }).map(j => `id${i}/child${j}`);
+        }),
+      ),
+      value,
+    });
+  });
+
+describe('sync migrations', () => {
+  it('should set the parent_id', async () => {
+    const tracer = execTracer();
+    tracer.start();
+
+    const cleanup = addSyncListener((oldValues, newValues) => {
+      const transactionsMap = newValues.get('transactions') as Map<
+        string,
+        unknown
+      >;
+      tracer.event('applied', [...transactionsMap.keys()]);
+    });
+
+    await db.insert('transactions', {
+      id: 'trans1/child1',
+      isChild: 1,
+      amount: 4500,
+    });
+    tracer.expectNow('applied', ['trans1/child1']);
+    tracer.expectNow('applied', ['trans1/child1']);
+
+    const transactions = db.runQuery<db.DbTransaction>(
+      'SELECT * FROM transactions',
+      [],
+      true,
+    );
+    expect(transactions.length).toBe(1);
+    expect(transactions[0].parent_id).toBe('trans1');
+
+    cleanup();
+    tracer.end();
+  });
+
+  it('child transactions should always have a parent_id', async () => {
+    await fc.assert(
+      fc
+        .asyncProperty(fc.array(messageArb, { maxLength: 100 }), async msgs => {
+          const tracer = execTracer();
+          tracer.start();
+          const cleanup = addSyncListener((oldValues, newValues) => {
+            const ts = newValues.get('transactions') as Map<
+              string,
+              { isChild: number; parent_id: string | null; id: string }
+            >;
+            if (
+              !ts ||
+              ![...ts.values()].find(
+                t =>
+                  t.isChild === 1 && t.parent_id == null && t.id.includes('/'),
+              )
+            ) {
+              tracer.event('applied');
+            }
+          });
+
+          await sendMessages(msgs);
+          await tracer.expect('applied');
+
+          const transactions = await db.all<db.DbTransaction>(
+            'SELECT * FROM transactions',
+            [],
+          );
+          for (const trans of transactions) {
+            const transMsgs = msgs
+              .filter(msg => msg.row === trans.id)
+              .sort((m1, m2) => {
+                const t1 = m1.timestamp.toString();
+                const t2 = m2.timestamp.toString();
+                if (t1 < t2) {
+                  return 1;
+                } else if (t1 > t2) {
+                  return -1;
+                }
+                return 0;
+              });
+            const msg = transMsgs.find(m => m.column === 'parent_id');
+
+            if (
+              trans.isChild === 1 &&
+              trans.id.includes('/') &&
+              (msg == null || msg.value == null)
+            ) {
+              // This is a child transaction didn't have a `parent_id`
+              // set in the messages. It should have gotten set from
+              // the `id`
+              const [parentId] = trans.id.split('/');
+              expect(parentId).not.toBe(null);
+              expect(trans.parent_id).toBe(parentId);
+            } else if (msg) {
+              // At least one message set `parent_id`
+              expect(trans.parent_id).toBe(msg.value);
+            } else {
+              // `parent_id` should never have been set
+              expect(trans.parent_id).toBe(null);
+            }
+          }
+
+          cleanup();
+          tracer.end();
+        })
+        .beforeEach(() => {
+          return db.execQuery(`DELETE FROM transactions`);
+        }),
+    );
+  });
+});

package/src/server/sync/migrate.ts

@@ -0,0 +1,48 @@
+// @ts-strict-ignore
+import { Timestamp } from '@actual-app/crdt';
+
+import { addSyncListener, applyMessages } from './index';
+import type { Message } from './index';
+
+function migrateParentIds(_oldValues, newValues) {
+  newValues.forEach((items, table) => {
+    if (table === 'transactions') {
+      const toApply: Message[] = [];
+
+      items.forEach(newValue => {
+        if (
+          newValue.isChild === 1 &&
+          newValue.parent_id == null &&
+          newValue.id.includes('/')
+        ) {
+          const parentId = newValue.id.split('/')[0];
+
+          toApply.push({
+            dataset: 'transactions',
+            row: newValue.id,
+            column: 'parent_id',
+            value: parentId,
+            timestamp: Timestamp.send(),
+          });
+        }
+      });
+
+      if (toApply.length > 0) {
+        void applyMessages(toApply);
+      }
+    }
+  });
+}
+
+let _unlisten = null;
+export function listen() {
+  unlisten();
+  _unlisten = addSyncListener(migrateParentIds);
+}
+
+export function unlisten() {
+  if (_unlisten) {
+    _unlisten();
+    _unlisten = null;
+  }
+}

package/src/server/sync/repair.ts

@@ -0,0 +1,39 @@
+// @ts-strict-ignore
+import { getClock, merkle, serializeClock, Timestamp } from '@actual-app/crdt';
+
+import * as db from '../db';
+
+export function rebuildMerkleHash(): {
+  numMessages: number;
+  trie: merkle.TrieNode;
+} {
+  const rows: { timestamp: string }[] = db.runQuery(
+    'SELECT timestamp FROM messages_crdt',
+    [],
+    true,
+  );
+  let trie = merkle.emptyTrie();
+
+  for (let i = 0; i < rows.length; i++) {
+    trie = merkle.insert(trie, Timestamp.parse(rows[i].timestamp));
+  }
+
+  return {
+    numMessages: rows.length,
+    trie,
+  };
+}
+
+export async function repairSync(): Promise<void> {
+  const rebuilt = rebuildMerkleHash();
+  const clock = getClock();
+
+  // Save it locally
+  clock.merkle = rebuilt.trie;
+
+  // Persist it in the db
+  db.runQuery(
+    db.cache('INSERT OR REPLACE INTO messages_clock (id, clock) VALUES (1, ?)'),
+    [serializeClock(clock)],
+  );
+}

package/src/server/sync/reset.ts

@@ -0,0 +1,91 @@
+// @ts-strict-ignore
+import { captureException } from '../../platform/exceptions';
+import * as asyncStorage from '../../platform/server/asyncStorage';
+import * as connection from '../../platform/server/connection';
+import * as cloudStorage from '../cloud-storage';
+import * as db from '../db';
+import { runMutator } from '../mutators';
+import * as prefs from '../prefs';
+
+export async function resetSync(
+  keyState?,
+): Promise<{ error?: { reason: string; meta?: unknown } }> {
+  if (!keyState) {
+    // If we aren't resetting the key, make sure our key is up-to-date
+    // so we don't accidentally upload a file encrypted with the wrong
+    // key (or not encrypted at all)
+    const { valid, error } = await cloudStorage.checkKey();
+    if (error) {
+      return { error };
+    } else if (!valid) {
+      return { error: { reason: 'file-has-new-key' } };
+    }
+  }
+
+  const { error } = await cloudStorage.resetSyncState(keyState);
+  if (error) {
+    return { error };
+  }
+
+  await runMutator(async () => {
+    // TOOD: We could automatically generate the list of tables to
+    // cleanup by looking at the schema
+    //
+    // Be VERY careful here since we are bulk deleting data. It should
+    // never delete any data that doesn't have `tombstone = 1`
+    db.execQuery(`
+      DELETE FROM messages_crdt;
+      DELETE FROM messages_clock;
+      DELETE FROM transactions WHERE tombstone = 1;
+      DELETE FROM accounts WHERE tombstone = 1;
+      DELETE FROM payees WHERE tombstone = 1;
+      DELETE FROM categories WHERE tombstone = 1;
+      DELETE FROM category_groups WHERE tombstone = 1;
+      DELETE FROM schedules WHERE tombstone = 1;
+      DELETE FROM rules WHERE tombstone = 1;
+      ANALYZE;
+      VACUUM;
+    `);
+    await db.loadClock();
+  });
+
+  await prefs.savePrefs({
+    groupId: null,
+    lastSyncedTimestamp: null,
+    lastUploaded: null,
+  });
+
+  if (keyState) {
+    const { key } = keyState;
+    const { cloudFileId } = prefs.getPrefs();
+
+    // The key has changed, we need to update our local data to
+    // store the new key
+
+    // Persist key in async storage
+    const keys = JSON.parse(
+      (await asyncStorage.getItem(`encrypt-keys`)) || '{}',
+    );
+    keys[cloudFileId] = key.serialize();
+    await asyncStorage.setItem('encrypt-keys', JSON.stringify(keys));
+
+    // Save the key id in prefs
+    await prefs.savePrefs({ encryptKeyId: key.getId() });
+  }
+
+  // Finally, upload the file to make it the "true" version that all
+  // other clients need to pull down to get back in sync
+  try {
+    await cloudStorage.upload();
+  } catch (e) {
+    if (e.reason) {
+      return { error: e };
+    }
+    captureException(e);
+    return { error: { reason: 'upload-failure' } };
+  } finally {
+    connection.send('prefs-updated');
+  }
+
+  return {};
+}