@powersync/service-core 0.0.0-dev-20240620165206

package/src/util/utils.ts

@@ -0,0 +1,122 @@
+import crypto from 'crypto';
+import * as pgwire from '@powersync/service-jpgwire';
+import { pgwireRows } from '@powersync/service-jpgwire';
+
+import * as storage from '../storage/storage-index.js';
+import { BucketChecksum, OpId } from './protocol-types.js';
+import { retriedQuery } from './pgwire_utils.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export type ChecksumMap = Map<string, BucketChecksum>;
+
+export function hashData(type: string, id: string, data: string): number {
+  const hash = crypto.createHash('sha256');
+  hash.update(`put.${type}.${id}.${data}`);
+  const buffer = hash.digest();
+  return buffer.readUInt32LE(0);
+}
+
+export function hashDelete(sourceKey: string) {
+  const hash = crypto.createHash('sha256');
+  hash.update(`delete.${sourceKey}`);
+  const buffer = hash.digest();
+  return buffer.readUInt32LE(0);
+}
+
+export function timestampToOpId(ts: bigint): OpId {
+  // Dynamic values are passed in in some cases, so we make extra sure that the
+  // number is a bigint and not number or Long.
+  if (typeof ts != 'bigint') {
+    throw new Error(`bigint expected, got: ${ts} (${typeof ts})`);
+  }
+  return ts.toString(10);
+}
+
+export function checksumsDiff(previous: ChecksumMap, current: ChecksumMap) {
+  // All changed ones
+  const updatedBuckets = new Map<string, BucketChecksum>();
+
+  const toRemove = new Set<string>(previous.keys());
+
+  for (let checksum of current.values()) {
+    const p = previous.get(checksum.bucket);
+    if (p == null) {
+      // Added
+      updatedBuckets.set(checksum.bucket, checksum);
+    } else {
+      toRemove.delete(checksum.bucket);
+      if (checksum.checksum != p.checksum || checksum.count != p.count) {
+        // Updated
+        updatedBuckets.set(checksum.bucket, checksum);
+      } else {
+        // No change
+      }
+    }
+  }
+
+  return {
+    updatedBuckets: [...updatedBuckets.values()],
+    removedBuckets: [...toRemove]
+  };
+}
+
+export function addChecksums(a: number, b: number) {
+  return (a + b) & 0xffffffff;
+}
+
+export function addBucketChecksums(a: BucketChecksum, b: BucketChecksum | null): BucketChecksum {
+  if (b == null) {
+    return a;
+  } else {
+    return {
+      bucket: a.bucket,
+      count: a.count + b.count,
+      checksum: addChecksums(a.checksum, b.checksum)
+    };
+  }
+}
+
+export async function getClientCheckpoint(
+  db: pgwire.PgClient,
+  bucketStorage: storage.BucketStorageFactory,
+  options?: { timeout?: number }
+): Promise<OpId> {
+  const start = Date.now();
+
+  const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
+
+  // This old API needs a persisted checkpoint id.
+  // Since we don't use LSNs anymore, the only way to get that is to wait.
+
+  const timeout = options?.timeout ?? 50_000;
+
+  logger.info(`Waiting for LSN checkpoint: ${lsn}`);
+  while (Date.now() - start < timeout) {
+    const cp = await bucketStorage.getActiveCheckpoint();
+    if (!cp.hasSyncRules()) {
+      throw new Error('No sync rules available');
+    }
+    if (cp.lsn >= lsn) {
+      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
+      return cp.checkpoint;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 30));
+  }
+
+  throw new Error('Timeout while waiting for checkpoint');
+}
+
+export async function createWriteCheckpoint(
+  db: pgwire.PgClient,
+  bucketStorage: storage.BucketStorageFactory,
+  user_id: string
+): Promise<bigint> {
+  const [{ lsn }] = pgwireRows(
+    await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`)
+  );
+
+  const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
+  logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
+  return id;
+}

package/test/src/__snapshots__/pg_test.test.ts.snap

@@ -0,0 +1,256 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`pg data types > schema 1`] = `
+[
+  {
+    "name": "public",
+    "tables": [
+      {
+        "columns": [
+          {
+            "name": "id",
+            "pg_type": "int4",
+            "type": "integer",
+          },
+          {
+            "name": "text",
+            "pg_type": "text",
+            "type": "text",
+          },
+          {
+            "name": "uuid",
+            "pg_type": "uuid",
+            "type": "uuid",
+          },
+          {
+            "name": "varchar",
+            "pg_type": "varchar",
+            "type": "character varying(255)",
+          },
+          {
+            "name": "bool",
+            "pg_type": "bool",
+            "type": "boolean",
+          },
+          {
+            "name": "bytea",
+            "pg_type": "bytea",
+            "type": "bytea",
+          },
+          {
+            "name": "int2",
+            "pg_type": "int2",
+            "type": "smallint",
+          },
+          {
+            "name": "int4",
+            "pg_type": "int4",
+            "type": "integer",
+          },
+          {
+            "name": "int8",
+            "pg_type": "int8",
+            "type": "bigint",
+          },
+          {
+            "name": "float4",
+            "pg_type": "float4",
+            "type": "real",
+          },
+          {
+            "name": "float8",
+            "pg_type": "float8",
+            "type": "double precision",
+          },
+          {
+            "name": "numeric",
+            "pg_type": "numeric",
+            "type": "numeric",
+          },
+          {
+            "name": "json",
+            "pg_type": "json",
+            "type": "json",
+          },
+          {
+            "name": "jsonb",
+            "pg_type": "jsonb",
+            "type": "jsonb",
+          },
+          {
+            "name": "pg_lsn",
+            "pg_type": "pg_lsn",
+            "type": "pg_lsn",
+          },
+          {
+            "name": "date",
+            "pg_type": "date",
+            "type": "date",
+          },
+          {
+            "name": "time",
+            "pg_type": "time",
+            "type": "time without time zone",
+          },
+          {
+            "name": "timestamp",
+            "pg_type": "timestamp",
+            "type": "timestamp without time zone",
+          },
+          {
+            "name": "timestamptz",
+            "pg_type": "timestamptz",
+            "type": "timestamp with time zone",
+          },
+          {
+            "name": "interval",
+            "pg_type": "interval",
+            "type": "interval",
+          },
+          {
+            "name": "macaddr",
+            "pg_type": "macaddr",
+            "type": "macaddr",
+          },
+          {
+            "name": "inet",
+            "pg_type": "inet",
+            "type": "inet",
+          },
+          {
+            "name": "oid",
+            "pg_type": "oid",
+            "type": "oid",
+          },
+        ],
+        "name": "test_data",
+      },
+      {
+        "columns": [
+          {
+            "name": "id",
+            "pg_type": "int4",
+            "type": "integer",
+          },
+          {
+            "name": "text",
+            "pg_type": "text[]",
+            "type": "text[]",
+          },
+          {
+            "name": "uuid",
+            "pg_type": "uuid[]",
+            "type": "uuid[]",
+          },
+          {
+            "name": "varchar",
+            "pg_type": "varchar[]",
+            "type": "character varying(255)[]",
+          },
+          {
+            "name": "bool",
+            "pg_type": "bool[]",
+            "type": "boolean[]",
+          },
+          {
+            "name": "bytea",
+            "pg_type": "bytea[]",
+            "type": "bytea[]",
+          },
+          {
+            "name": "int2",
+            "pg_type": "int2[]",
+            "type": "smallint[]",
+          },
+          {
+            "name": "int4",
+            "pg_type": "int4[]",
+            "type": "integer[]",
+          },
+          {
+            "name": "int8",
+            "pg_type": "int8[]",
+            "type": "bigint[]",
+          },
+          {
+            "name": "float4",
+            "pg_type": "float4[]",
+            "type": "real[]",
+          },
+          {
+            "name": "float8",
+            "pg_type": "float8[]",
+            "type": "double precision[]",
+          },
+          {
+            "name": "numeric",
+            "pg_type": "numeric[]",
+            "type": "numeric[]",
+          },
+          {
+            "name": "json",
+            "pg_type": "json[]",
+            "type": "json[]",
+          },
+          {
+            "name": "jsonb",
+            "pg_type": "jsonb[]",
+            "type": "jsonb[]",
+          },
+          {
+            "name": "pg_lsn",
+            "pg_type": "pg_lsn[]",
+            "type": "pg_lsn[]",
+          },
+          {
+            "name": "date",
+            "pg_type": "date[]",
+            "type": "date[]",
+          },
+          {
+            "name": "time",
+            "pg_type": "time[]",
+            "type": "time without time zone[]",
+          },
+          {
+            "name": "timestamp",
+            "pg_type": "timestamp[]",
+            "type": "timestamp without time zone[]",
+          },
+          {
+            "name": "timestamptz",
+            "pg_type": "timestamptz[]",
+            "type": "timestamp with time zone[]",
+          },
+          {
+            "name": "interval",
+            "pg_type": "interval[]",
+            "type": "interval[]",
+          },
+          {
+            "name": "macaddr",
+            "pg_type": "macaddr[]",
+            "type": "macaddr[]",
+          },
+          {
+            "name": "inet",
+            "pg_type": "inet[]",
+            "type": "inet[]",
+          },
+          {
+            "name": "oid",
+            "pg_type": "oid[]",
+            "type": "oid[]",
+          },
+          {
+            "name": "multidimensional",
+            "pg_type": "text[]",
+            "type": "text[]",
+          },
+        ],
+        "name": "test_data_arrays",
+      },
+    ],
+  },
+]
+`;

package/test/src/__snapshots__/sync.test.ts.snap

@@ -0,0 +1,247 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`sync - mongodb > expired token 1`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - mongodb > expiring token 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 0,
+          "count": 0,
+        },
+      ],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > expiring token 2`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - mongodb > sync global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+        },
+        {
+          "checksum": 3280762209n,
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync legacy non-raw data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -852817836,
+          "count": 1,
+        },
+      ],
+      "last_op_id": "1",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3442149460n,
+          "data": {
+            "description": "Test
+\\"string\\"",
+            "id": "t1",
+            "large_num": 12345678901234567890n,
+          },
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 0,
+          "count": 0,
+        },
+      ],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to global data 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 920318466,
+          "count": 1,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466n,
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to global data 3`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "1",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 3280762209n,
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;