@powersync/service-module-postgres-storage 0.12.0 → 0.13.0

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -2189,3 +2189,1098 @@ exports[`sync - postgres > storage v2 > sync updates to parameter query only 2`]
   },
 ]
 `;
+
+exports[`sync - postgres > storage v3 > compacting data - invalidate checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > compacting data - invalidate checkpoint 2`] = `
+[
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": -93886621,
+          "op": "CLEAR",
+          "op_id": "2",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "4",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": 499012468,
+          "count": 4,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "2",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": 1859363232,
+          "data": "{"id":"t1","description":"Test 1b"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "3",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+        {
+          "checksum": 3028503153,
+          "data": "{"id":"t2","description":"Test 2b"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "4",
+          "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd",
+        },
+      ],
+      "has_more": false,
+      "next_after": "4",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "4",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#test|0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "test",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#test|0[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 2`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "2#test|0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "test",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "2#test|0[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > expired token 1`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > expiring token 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > expiring token 2`] = `
+[
+  {
+    "token_expires_in": 0,
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sends checkpoint complete line for empty checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": -1221282404,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": 3073684892,
+          "data": "{"id":"t1","description":"sync"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  null,
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [],
+      "write_checkpoint": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync buckets in order 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#b0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 2,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+        {
+          "bucket": "1#b1[]",
+          "checksum": -1382098757,
+          "count": 1,
+          "priority": 1,
+          "subscriptions": [
+            {
+              "default": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b0",
+        },
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b1",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#b1[]",
+      "data": [
+        {
+          "checksum": 2912868539,
+          "data": "{"id":"earlier","description":"Test 2"}",
+          "object_id": "earlier",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 1,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#b0[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+        {
+          "checksum": 3280762209,
+          "data": "{"id":"t2","description":"Test 2"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#b0a[]",
+          "checksum": -659831575,
+          "count": 2000,
+          "priority": 2,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+        {
+          "bucket": "1#b0b[]",
+          "checksum": -659831575,
+          "count": 2000,
+          "priority": 2,
+          "subscriptions": [
+            {
+              "default": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#b1[]",
+          "checksum": -1096116670,
+          "count": 1,
+          "priority": 1,
+          "subscriptions": [
+            {
+              "default": 2,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "4001",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b0a",
+        },
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b0b",
+        },
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b1",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#b1[]",
+      "data": undefined,
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "4001",
+      "priority": 1,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#b0a[]",
+      "data": undefined,
+      "has_more": true,
+      "next_after": "2000",
+    },
+  },
+  {
+    "data": {
+      "after": "2000",
+      "bucket": "1#b0a[]",
+      "data": undefined,
+      "has_more": true,
+      "next_after": "4000",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "4004",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#b0a[]",
+          "checksum": 883076828,
+          "count": 2001,
+          "priority": 2,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+        {
+          "bucket": "1#b0b[]",
+          "checksum": 883076828,
+          "count": 2001,
+          "priority": 2,
+          "subscriptions": [
+            {
+              "default": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#b1[]",
+          "checksum": 1841937527,
+          "count": 2,
+          "priority": 1,
+          "subscriptions": [
+            {
+              "default": 2,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "1",
+      "bucket": "1#b1[]",
+      "data": undefined,
+      "has_more": false,
+      "next_after": "4002",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "4004",
+      "priority": 1,
+    },
+  },
+  {
+    "data": {
+      "after": "4000",
+      "bucket": "1#b0a[]",
+      "data": undefined,
+      "has_more": false,
+      "next_after": "4003",
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#b0b[]",
+      "data": undefined,
+      "has_more": true,
+      "next_after": "1999",
+    },
+  },
+  {
+    "data": {
+      "after": "1999",
+      "bucket": "1#b0b[]",
+      "data": undefined,
+      "has_more": true,
+      "next_after": "3999",
+    },
+  },
+  {
+    "data": {
+      "after": "3999",
+      "bucket": "1#b0b[]",
+      "data": undefined,
+      "has_more": false,
+      "next_after": "4004",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "4004",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync legacy non-raw data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": -852817836,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": 3442149460n,
+          "data": {
+            "description": "Test
+"string"",
+            "id": "t1",
+            "large_num": 12345678901234567890n,
+          },
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to data query only 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#by_user["user1"]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to data query only 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#by_user["user1"]",
+          "checksum": 1418351250,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#by_user["user1"]",
+      "data": [
+        {
+          "checksum": 1418351250,
+          "data": "{"id":"list1","user_id":"user1","name":"User 1"}",
+          "object_id": "list1",
+          "object_type": "lists",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to global data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to global data 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to global data 3`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "1",
+      "bucket": "1#mybucket[]",
+      "data": [
+        {
+          "checksum": 3280762209,
+          "data": "{"id":"t2","description":"Test 2"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to parameter query + data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [],
+      "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to parameter query + data 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#by_user["user1"]",
+          "checksum": 1418351250,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#by_user["user1"]",
+      "data": [
+        {
+          "checksum": 1418351250,
+          "data": "{"id":"list1","user_id":"user1","name":"User 1"}",
+          "object_id": "list1",
+          "object_type": "lists",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to parameter query only 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [],
+      "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > storage v3 > sync updates to parameter query only 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "1#by_user["user1"]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;