@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240725112650

package/src/util/config/compound-config-collector.ts

@@ -1,4 +1,3 @@
-import * as t from 'ts-codec';
 import { configFile, normalizeConnection } from '@powersync/service-types';
 import { ConfigCollector } from './collectors/config-collector.js';
 import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
@@ -10,7 +9,7 @@ import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-co
 import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
 import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
-import { logger, schema } from '@powersync/lib-services-framework';
+import { logger } from '@powersync/lib-services-framework';
 
 const POWERSYNC_DEV_KID = 'powersync-dev';
 
@@ -29,12 +28,6 @@ export type CompoundConfigCollectorOptions = {
   syncRulesCollectors: SyncRulesCollector[];
 };
 
-export type ConfigCollectorGenerics = {
-  SERIALIZED: configFile.SerializedPowerSyncConfig;
-  DESERIALIZED: configFile.PowerSyncConfig;
-  RESOLVED: ResolvedPowerSyncConfig;
-};
-
 const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
   configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
   syncRulesCollectors: [
@@ -44,56 +37,15 @@ const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
   ]
 };
 
-export class CompoundConfigCollector<Generics extends ConfigCollectorGenerics = ConfigCollectorGenerics> {
+export class CompoundConfigCollector {
   constructor(protected options: CompoundConfigCollectorOptions = DEFAULT_COLLECTOR_OPTIONS) {}
 
-  /**
-   * The default ts-codec for validations and decoding
-   */
-  get codec(): t.AnyCodec {
-    return configFile.powerSyncConfig;
-  }
-
   /**
    * Collects and resolves base config
    */
-  async collectConfig(runnerConfig: RunnerConfig = {}): Promise<Generics['RESOLVED']> {
-    const baseConfig = await this.collectBaseConfig(runnerConfig);
-    const baseResolvedConfig = await this.resolveBaseConfig(baseConfig, runnerConfig);
-    return this.resolveConfig(baseConfig, baseResolvedConfig, runnerConfig);
-  }
-
-  /**
-   * Collects the base PowerSyncConfig from various registered collectors.
-   * @throws if no collector could return a configuration.
-   */
-  protected async collectBaseConfig(runner_config: RunnerConfig): Promise<Generics['DESERIALIZED']> {
-    for (const collector of this.options.configCollectors) {
-      try {
-        const baseConfig = await collector.collectSerialized(runner_config);
-        if (baseConfig) {
-          const decoded = this.decode(baseConfig);
-          this.validate(decoded);
-          return decoded;
-        }
-        logger.debug(
-          `Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`
-        );
-      } catch (ex) {
-        // An error in a collector is a hard stop
-        throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
-      }
-    }
-    throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
-  }
+  async collectConfig(runner_config: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
+    const baseConfig = await this.collectBaseConfig(runner_config);
 
-  /**
-   * Performs the resolving of the common (shared) base configuration
-   */
-  protected async resolveBaseConfig(
-    baseConfig: Generics['DESERIALIZED'],
-    runnerConfig: RunnerConfig = {}
-  ): Promise<ResolvedPowerSyncConfig> {
     const connections = baseConfig.replication?.connections ?? [];
     if (connections.length > 1) {
       throw new Error('Only a single replication connection is supported currently');
@@ -141,7 +93,7 @@ export class CompoundConfigCollector<Generics extends ConfigCollectorGenerics =
       devKey = await auth.KeySpec.importKey(baseDevKey);
     }
 
-    const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
+    const sync_rules = await this.collectSyncRules(baseConfig, runner_config);
 
     let jwt_audiences: string[] = baseConfig.client_auth?.audience ?? [];
 
@@ -178,17 +130,25 @@ export class CompoundConfigCollector<Generics extends ConfigCollectorGenerics =
   }
 
   /**
-   * Perform any additional resolving from {@link ResolvedPowerSyncConfig}
-   * to the extended {@link Generics['RESOLVED']}
-   *
+   * Collects the base PowerSyncConfig from various registered collectors.
+   * @throws if no collector could return a configuration.
    */
-  protected async resolveConfig(
-    baseConfig: Generics['DESERIALIZED'],
-    resolvedBaseConfig: ResolvedPowerSyncConfig,
-    runnerConfig: RunnerConfig = {}
-  ): Promise<Generics['RESOLVED']> {
-    // The base version has ResolvedPowerSyncConfig == Generics['RESOLVED']
-    return resolvedBaseConfig;
+  protected async collectBaseConfig(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig> {
+    for (const collector of this.options.configCollectors) {
+      try {
+        const baseConfig = await collector.collect(runner_config);
+        if (baseConfig) {
+          return baseConfig;
+        }
+        logger.debug(
+          `Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`
+        );
+      } catch (ex) {
+        // An error in a collector is a hard stop
+        throw new Error(`Could not collect config using ${collector.name} method. Caught exception: ${ex}`);
+      }
+    }
+    throw new Error('PowerSyncConfig could not be collected using any of the registered config collectors.');
   }
 
   protected async collectSyncRules(
@@ -213,28 +173,4 @@ export class CompoundConfigCollector<Generics extends ConfigCollectorGenerics =
       present: false
     };
   }
-
-  /**
-   * Validates input config
-   * ts-codec itself doesn't give great validation errors, so we use json schema for that
-   */
-  protected validate(config: Generics['DESERIALIZED']) {
-    // ts-codec itself doesn't give great validation errors, so we use json schema for that
-    const validator = schema
-      .parseJSONSchema(t.generateJSONSchema(this.codec, { allowAdditional: true, parsers: [configFile.portParser] }))
-      .validator();
-
-    const valid = validator.validate(config);
-    if (!valid.valid) {
-      throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
-    }
-  }
-
-  protected decode(encoded: Generics['SERIALIZED']): Generics['DESERIALIZED'] {
-    try {
-      return this.codec.decode(encoded);
-    } catch (ex) {
-      throw new Error(`Failed to decode PowerSync config: ${ex}`);
-    }
-  }
 }

package/test/src/__snapshots__/sync.test.ts.snap

@@ -1,5 +1,90 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
+[
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": -93886621n,
+          "op": "CLEAR",
+          "op_id": "2",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "4",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 499012468,
+          "count": 4,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "2",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 1859363232n,
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1b\\"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "3",
+          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+        },
+        {
+          "checksum": 3028503153n,
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2b\\"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "4",
+          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+        },
+      ],
+      "has_more": false,
+      "next_after": "4",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "4",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > expired token 1`] = `
 [
   {

package/test/src/bucket_validation.test.ts

@@ -0,0 +1,142 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { describe, expect, test } from 'vitest';
+import { reduceBucket, validateBucket } from './bucket_validation.js';
+
+// This tests the reduceBucket function.
+// While this function is not used directly in the service implementation,
+// it is an important part of validating consistency in other tests.
+describe('bucket validation', () => {
+  const ops1: OplogEntry[] = [
+    {
+      op_id: '1',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 2634521662,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: '{"id":"t1"}'
+    },
+    {
+      op_id: '2',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    },
+    {
+      op_id: '3',
+      op: 'REMOVE',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 4228978084,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: null
+    },
+    {
+      op_id: '4',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    }
+  ];
+
+  test('reduce 1', () => {
+    expect(reduceBucket(ops1)).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(ops1))).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(ops1);
+  });
+
+  test('reduce 2', () => {
+    const bucket: OplogEntry[] = [
+      ...ops1,
+
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '5'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ];
+
+    expect(reduceBucket(bucket)).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(bucket))).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(bucket);
+  });
+});

package/test/src/bucket_validation.ts

@@ -0,0 +1,116 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/utils.js';
+import { expect } from 'vitest';
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}
+
+/**
+ * Validate this property, as described in /docs/bucket-properties.md:
+ *
+ * $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
+ *
+ * We test that a client syncing the entire bucket in one go (left side of the equation),
+ * ends up with the same result as another client syncing up to operation id_i, then sync
+ * the rest.
+ */
+export function validateBucket(bucket: OplogEntry[]) {
+  const r1 = reduceBucket(bucket);
+  for (let i = 0; i <= bucket.length; i++) {
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const b3 = bucket.slice(i + 1);
+    const r3 = r2.concat(b3);
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+
+  // This is the same check, just implemented differently
+  validateCompactedBucket(bucket, bucket);
+}
+
+/**
+ * Validate these properties for a bucket $B$ and its compacted version $B'$,:
+ * as described in /docs/bucket-properties.md:
+ *
+ * 1. $r(B) = r(B')$
+ * 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
+ *
+ * The first one is that the result of syncing the original bucket is the same as
+ * syncing the compacted bucket.
+ *
+ * The second property is that result of syncing the entire original bucket, is the same
+ * as syncing any partial version of that (up to op $c_i$), and then continue syncing
+ * using the compacted bucket.
+ */
+export function validateCompactedBucket(bucket: OplogEntry[], compacted: OplogEntry[]) {
+  // r(B_{[..c]})
+  const r1 = reduceBucket(bucket);
+  // r(B) = r(B')
+  expect(reduceBucket(compacted)).toEqual(r1);
+
+  for (let i = 0; i < bucket.length; i++) {
+    // r(B_{[..c_i]})
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const c_i = BigInt(bucket[i].op_id);
+    // B'_{[c_i+1..c]}
+    const b3 = compacted.filter((op) => BigInt(op.op_id) > c_i);
+    // r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}
+    const r3 = r2.concat(b3);
+    // r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]})
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+}