@powersync/service-module-postgres-storage 0.1.0

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -0,0 +1,332 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`sync - postgres > compacting data - invalidate checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > compacting data - invalidate checkpoint 2`] = `
+[
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": -93886621n,
+          "op": "CLEAR",
+          "op_id": "2",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "4",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 499012468,
+          "count": 4,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "2",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 1859363232n,
+          "data": "{"id":"t1","description":"Test 1b"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "3",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+        {
+          "checksum": 3028503153n,
+          "data": "{"id":"t2","description":"Test 2b"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "4",
+          "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd",
+        },
+      ],
+      "has_more": false,
+      "next_after": "4",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "4",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > expired token 1`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - postgres > expiring token 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 0,
+          "count": 0,
+        },
+      ],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > expiring token 2`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - postgres > sync global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+        {
+          "checksum": 3280762209n,
+          "data": "{"id":"t2","description":"Test 2"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > sync legacy non-raw data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -852817836,
+          "count": 1,
+        },
+      ],
+      "last_op_id": "1",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3442149460n,
+          "data": {
+            "description": "Test
+"string"",
+            "id": "t1",
+            "large_num": 12345678901234567890n,
+          },
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > sync updates to global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 0,
+          "count": 0,
+        },
+      ],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > sync updates to global data 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 920318466,
+          "count": 1,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > sync updates to global data 3`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "1",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3280762209n,
+          "data": "{"id":"t2","description":"Test 2"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;

package/test/src/env.ts

@@ -0,0 +1,6 @@
+import { utils } from '@powersync/lib-services-framework';
+
+export const env = utils.collectEnvironmentVariables({
+  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'),
+  CI: utils.type.boolean.default('false')
+});

package/test/src/migrations.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest';
+
+import { register } from '@powersync/service-core-tests';
+import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js';
+import { env } from './env.js';
+import { POSTGRES_STORAGE_FACTORY } from './util.js';
+
+const MIGRATION_AGENT_FACTORY = () => {
+  return new PostgresMigrationAgent({ type: 'postgresql', uri: env.PG_STORAGE_TEST_URL, sslmode: 'disable' });
+};
+
+describe('Migrations', () => {
+  register.registerMigrationTests(MIGRATION_AGENT_FACTORY);
+
+  it('Should have tables declared', async () => {
+    const { db } = await POSTGRES_STORAGE_FACTORY();
+
+    const tables = await db.sql`
+      SELECT
+        table_schema,
+        table_name
+      FROM
+        information_schema.tables
+      WHERE
+        table_type = 'BASE TABLE'
+        AND table_schema NOT IN ('pg_catalog', 'information_schema')
+      ORDER BY
+        table_schema,
+        table_name;
+    `.rows<{ table_schema: string; table_name: string }>();
+
+    expect(tables.find((t) => t.table_name == 'sync_rules')).exist;
+  });
+});

package/test/src/setup.ts

@@ -0,0 +1,16 @@
+import { container } from '@powersync/lib-services-framework';
+import { Metrics } from '@powersync/service-core';
+import { beforeAll } from 'vitest';
+
+beforeAll(async () => {
+  // Executes for every test file
+  container.registerDefaults();
+
+  // The metrics need to be initialized before they can be used
+  await Metrics.initialise({
+    disable_telemetry_sharing: true,
+    powersync_instance_id: 'test',
+    internal_metrics_endpoint: 'unused.for.tests.com'
+  });
+  Metrics.getInstance().resetCounters();
+});

package/test/src/storage.test.ts

@@ -0,0 +1,131 @@
+import { storage } from '@powersync/service-core';
+import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
+import { POSTGRES_STORAGE_FACTORY } from './util.js';
+
+describe('Sync Bucket Validation', register.registerBucketValidationTests);
+
+describe('Postgres Sync Bucket Storage', () => {
+  register.registerDataStorageTests(POSTGRES_STORAGE_FACTORY);
+
+  /**
+   * The split of returned results can vary depending on storage drivers.
+   * The large rows here are 2MB large while the default chunk limit is 1mb.
+   * The Postgres storage driver will detect if the next row will increase the batch
+   * over the limit and separate that row into a new batch (or single row batch) if applicable.
+   */
+  test('large batch (2)', async () => {
+    // Test syncing a batch of data that is small in count,
+    // but large enough in size to be split over multiple returned chunks.
+    // Similar to the above test, but splits over 1MB chunks.
+    const sync_rules = test_utils.testRules(
+      `
+    bucket_definitions:
+      global:
+        data:
+          - SELECT id, description FROM "%"
+    `
+    );
+    using factory = await POSTGRES_STORAGE_FACTORY();
+    const bucketStorage = factory.getInstance(sync_rules);
+
+    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      const sourceTable = TEST_TABLE;
+
+      const largeDescription = '0123456789'.repeat(2_000_00);
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test1',
+          description: 'test1'
+        },
+        afterReplicaId: test_utils.rid('test1')
+      });
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large1',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large1')
+      });
+
+      // Large enough to split the returned batch
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'large2',
+          description: largeDescription
+        },
+        afterReplicaId: test_utils.rid('large2')
+      });
+
+      await batch.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 'test3',
+          description: 'test3'
+        },
+        afterReplicaId: test_utils.rid('test3')
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const options: storage.BucketDataBatchOptions = {};
+
+    const batch1 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options)
+    );
+    expect(test_utils.getBatchData(batch1)).toEqual([
+      { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
+    ]);
+    expect(test_utils.getBatchMeta(batch1)).toEqual({
+      after: '0',
+      has_more: true,
+      next_after: '1'
+    });
+
+    const batch2 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch2)).toEqual([
+      { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
+    ]);
+    expect(test_utils.getBatchMeta(batch2)).toEqual({
+      after: '1',
+      has_more: true,
+      next_after: '2'
+    });
+
+    const batch3 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch3)).toEqual([
+      { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
+    ]);
+    expect(test_utils.getBatchMeta(batch3)).toEqual({
+      after: '2',
+      has_more: true,
+      next_after: '3'
+    });
+
+    const batch4 = await test_utils.fromAsync(
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch3[0].batch.next_after]]), options)
+    );
+    expect(test_utils.getBatchData(batch4)).toEqual([
+      { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
+    ]);
+    expect(test_utils.getBatchMeta(batch4)).toEqual({
+      after: '3',
+      has_more: false,
+      next_after: '4'
+    });
+  });
+});

package/test/src/storage_compacting.test.ts

@@ -0,0 +1,5 @@
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { POSTGRES_STORAGE_FACTORY } from './util.js';
+
+describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY, {}));

package/test/src/storage_sync.test.ts

@@ -0,0 +1,12 @@
+import { register } from '@powersync/service-core-tests';
+import { describe } from 'vitest';
+import { POSTGRES_STORAGE_FACTORY } from './util.js';
+
+/**
+ * Bucket compacting is not yet implemented.
+ * This causes the internal compacting test to fail.
+ * Other tests have been verified manually.
+ */
+describe('sync - postgres', () => {
+  register.registerSyncTests(POSTGRES_STORAGE_FACTORY);
+});

package/test/src/util.ts

@@ -0,0 +1,34 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { normalizePostgresStorageConfig } from '../../src//types/types.js';
+import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js';
+import { PostgresTestStorageFactoryGenerator } from '../../src/storage/PostgresTestStorageFactoryGenerator.js';
+import { env } from './env.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export const TEST_URI = env.PG_STORAGE_TEST_URL;
+
+const BASE_CONFIG = {
+  type: 'postgresql' as const,
+  uri: TEST_URI,
+  sslmode: 'disable' as const
+};
+
+export const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG);
+
+/**
+ * Vitest tries to load the migrations via .ts files which fails.
+ * For tests this links to the relevant .js files correctly
+ */
+class TestPostgresMigrationAgent extends PostgresMigrationAgent {
+  getInternalScriptsDir(): string {
+    return path.resolve(__dirname, '../../dist/migrations/scripts');
+  }
+}
+
+export const POSTGRES_STORAGE_FACTORY = PostgresTestStorageFactoryGenerator({
+  url: env.PG_STORAGE_TEST_URL,
+  migrationAgent: (config) => new TestPostgresMigrationAgent(config)
+});

package/test/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "baseUrl": "./",
+    "noEmit": true,
+    "esModuleInterop": true,
+    "declarationDir": "dist/@types",
+    "tsBuildInfoFile": "dist/.tsbuildinfo",
+    "lib": ["ES2022", "esnext.disposable"],
+    "skipLibCheck": true,
+    "sourceMap": true
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../"
+    }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,36 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "declarationDir": "dist/@types",
+    "tsBuildInfoFile": "dist/.tsbuildinfo",
+    "rootDir": "src",
+    "target": "ES2022",
+    "lib": ["ES2022", "esnext.disposable"],
+    "skipLibCheck": true
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../../packages/types"
+    },
+    {
+      "path": "../../packages/jsonbig"
+    },
+    {
+      "path": "../../packages/jpgwire"
+    },
+    {
+      "path": "../../packages/sync-rules"
+    },
+    {
+      "path": "../../packages/service-core"
+    },
+    {
+      "path": "../../libs/lib-services"
+    },
+    {
+      "path": "../../libs/lib-postgres"
+    }
+  ]
+}

package/vitest.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    setupFiles: './test/src/setup.ts',
+    poolOptions: {
+      threads: {
+        singleThread: true
+      }
+    },
+    pool: 'threads'
+  }
+});