@powersync/service-core 0.0.0-dev-20240620165410 → 0.0.0-dev-20240708103353

package/src/routes/endpoints/sync-stream.ts

@@ -1,13 +1,13 @@
-import { Readable } from 'stream';
-import { SyncParameters, normalizeTokenParameters } from '@powersync/service-sync-rules';
 import { errors, logger, router, schema } from '@powersync/lib-services-framework';
+import { RequestParameters } from '@powersync/service-sync-rules';
+import { Readable } from 'stream';
 
 import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 
+import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { Metrics } from '../../metrics/Metrics.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -30,10 +30,7 @@ export const syncStreamed = routeDefinition({
     }
 
     const params: util.StreamingSyncRequest = payload.params;
-    const syncParams: SyncParameters = normalizeTokenParameters(
-      payload.context.token_payload!.parameters ?? {},
-      payload.params.parameters ?? {}
-    );
+    const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {});
 
     const storage = system.storage;
     // Sanity check before we start the stream

package/src/routes/hooks.ts

@@ -25,7 +25,6 @@ export const createRequestQueueHook = (params: CreateRequestQueueParams): fastif
         status: 429,
         method: request.method,
         path: request.url,
-        route: request.routerPath,
         queue_overflow: true
       });
       return reply.status(429).send();

package/src/storage/BucketStorage.ts

@@ -367,7 +367,12 @@ export interface SaveInsert {
 export interface SaveUpdate {
   tag: 'update';
   sourceTable: SourceTable;
+
+  /**
+   * This is only present when the id has changed, and will only contain replica identity columns.
+   */
   before?: SqliteRow;
+
   /**
    * A null value means null column.
    *

package/src/storage/mongo/MongoBucketBatch.ts

@@ -187,6 +187,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
         }
         const currentData = current_data_lookup.get(op.internalBeforeKey) ?? null;
         if (currentData != null) {
+          // If it will be used again later, it will be set again using nextData below
           current_data_lookup.delete(op.internalBeforeKey);
         }
         const nextData = this.saveOperation(persistedBatch!, op, currentData, op_seq);
@@ -242,6 +243,10 @@ export class MongoBucketBatch implements BucketStorageBatch {
         // Not an error if we re-apply a transaction
         existing_buckets = [];
         existing_lookups = [];
+        // Log to help with debugging if there was a consistency issue
+        logger.warn(
+          `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
+        );
       } else {
         const data = bson.deserialize((result.data as mongo.Binary).buffer, BSON_DESERIALIZE_OPTIONS) as SqliteRow;
         existing_buckets = result.buckets;
@@ -254,6 +259,10 @@ export class MongoBucketBatch implements BucketStorageBatch {
         // Not an error if we re-apply a transaction
         existing_buckets = [];
         existing_lookups = [];
+        // Log to help with debugging if there was a consistency issue
+        logger.warn(
+          `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
+        );
       } else {
         existing_buckets = result.buckets;
         existing_lookups = result.lookups;
@@ -292,7 +301,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
     }
 
     // 2. Save bucket data
-    if (beforeId != null && beforeId != afterId) {
+    if (beforeId != null && (afterId == null || !beforeId.equals(afterId))) {
       // Source ID updated
       if (sourceTable.syncData) {
         // Delete old record
@@ -422,7 +431,7 @@ export class MongoBucketBatch implements BucketStorageBatch {
       };
     }
 
-    if (beforeId != afterId) {
+    if (afterId == null || !beforeId.equals(afterId)) {
       // Either a delete (afterId == null), or replaced the old replication id
       batch.deleteCurrentData(before_key);
     }

package/src/sync/sync.ts

@@ -1,5 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { SyncParameters } from '@powersync/service-sync-rules';
+import { RequestParameters } from '@powersync/service-sync-rules';
 import { Semaphore } from 'async-mutex';
 import { AbortError } from 'ix/aborterror.js';
 
@@ -7,10 +7,10 @@ import * as auth from '../auth/auth-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
+import { logger } from '@powersync/lib-services-framework';
+import { Metrics } from '../metrics/Metrics.js';
 import { mergeAsyncIterables } from './merge.js';
 import { TokenStreamOptions, tokenStream } from './util.js';
-import { Metrics } from '../metrics/Metrics.js';
-import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Maximum number of connections actively fetching data.
@@ -21,7 +21,7 @@ const syncSemaphore = new Semaphore(MAX_ACTIVE_CONNECTIONS);
 export interface SyncStreamParameters {
   storage: storage.BucketStorageFactory;
   params: util.StreamingSyncRequest;
-  syncParams: SyncParameters;
+  syncParams: RequestParameters;
   token: auth.JwtPayload;
   /**
    * If this signal is aborted, the stream response ends as soon as possible, without error.
@@ -71,7 +71,7 @@ export async function* streamResponse(
 async function* streamResponseInner(
   storage: storage.BucketStorageFactory,
   params: util.StreamingSyncRequest,
-  syncParams: SyncParameters,
+  syncParams: RequestParameters,
   signal: AbortSignal
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
   // Bucket state of bucket id -> op_id.

package/test/src/data_storage.test.ts

@@ -1,4 +1,4 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
 import { SourceTable } from '../../src/storage/SourceTable.js';
@@ -289,12 +289,7 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const parameters = {
-      token_parameters: {
-        user_id: 'u1'
-      },
-      user_parameters: {}
-    };
+    const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
 
@@ -358,12 +353,7 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const parameters = {
-      token_parameters: {
-        user_id: 'unknown'
-      },
-      user_parameters: {}
-    };
+    const parameters = new RequestParameters({ sub: 'unknown' }, {});
 
     const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
 
@@ -442,12 +432,7 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const parameters = {
-      token_parameters: {
-        user_id: 'u1'
-      },
-      user_parameters: {}
-    };
+    const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     // Test intermediate values - could be moved to sync_rules.test.ts
     const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
@@ -897,6 +882,183 @@ bucket_definitions:
     ]);
   });
 
+  test('changed data with replica identity full', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const sourceTable = makeTestTable('test', ['id', 'description']);
+
+    // Pre-setup
+    const result1 = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable,
+        tag: 'insert',
+        after: {
+          id: 'test1',
+          description: 'test1a'
+        }
+      });
+    });
+
+    const checkpoint1 = result1?.flushed_op ?? '0';
+
+    const result2 = await storage.startBatch({}, async (batch) => {
+      // Unchanged, but has a before id
+      await batch.save({
+        sourceTable,
+        tag: 'update',
+        before: {
+          id: 'test1',
+          description: 'test1a'
+        },
+        after: {
+          id: 'test1',
+          description: 'test1b'
+        }
+      });
+    });
+
+    const result3 = await storage.startBatch({}, async (batch) => {
+      // Delete
+      await batch.save({
+        sourceTable,
+        tag: 'delete',
+        before: {
+          id: 'test1',
+          description: 'test1b'
+        },
+        after: undefined
+      });
+    });
+
+    const checkpoint3 = result3!.flushed_op;
+
+    const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
+    const data = batch[0].data.map((d) => {
+      return {
+        op: d.op,
+        object_id: d.object_id,
+        data: d.data,
+        subkey: d.subkey
+      };
+    });
+
+    // Operations must be in this order
+    expect(data).toEqual([
+      // 2
+      // The REMOVE is expected because the subkey changes
+      {
+        op: 'REMOVE',
+        object_id: 'test1',
+        data: null,
+        subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+      },
+      {
+        op: 'PUT',
+        object_id: 'test1',
+        data: JSON.stringify({ id: 'test1', description: 'test1b' }),
+        subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
+      },
+      // 3
+      {
+        op: 'REMOVE',
+        object_id: 'test1',
+        data: null,
+        subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
+      }
+    ]);
+  });
+
+  test('unchanged data with replica identity full', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const sourceTable = makeTestTable('test', ['id', 'description']);
+
+    // Pre-setup
+    const result1 = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable,
+        tag: 'insert',
+        after: {
+          id: 'test1',
+          description: 'test1a'
+        }
+      });
+    });
+
+    const checkpoint1 = result1?.flushed_op ?? '0';
+
+    const result2 = await storage.startBatch({}, async (batch) => {
+      // Unchanged, but has a before id
+      await batch.save({
+        sourceTable,
+        tag: 'update',
+        before: {
+          id: 'test1',
+          description: 'test1a'
+        },
+        after: {
+          id: 'test1',
+          description: 'test1a'
+        }
+      });
+    });
+
+    const result3 = await storage.startBatch({}, async (batch) => {
+      // Delete
+      await batch.save({
+        sourceTable,
+        tag: 'delete',
+        before: {
+          id: 'test1',
+          description: 'test1a'
+        },
+        after: undefined
+      });
+    });
+
+    const checkpoint3 = result3!.flushed_op;
+
+    const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
+    const data = batch[0].data.map((d) => {
+      return {
+        op: d.op,
+        object_id: d.object_id,
+        data: d.data,
+        subkey: d.subkey
+      };
+    });
+
+    // Operations must be in this order
+    expect(data).toEqual([
+      // 2
+      {
+        op: 'PUT',
+        object_id: 'test1',
+        data: JSON.stringify({ id: 'test1', description: 'test1a' }),
+        subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+      },
+      // 3
+      {
+        op: 'REMOVE',
+        object_id: 'test1',
+        data: null,
+        subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+      }
+    ]);
+  });
+
   test('large batch', async () => {
     // Test syncing a batch of data that is small in count,
     // but large enough in size to be split over multiple returned batches.

package/test/src/slow_tests.test.ts

@@ -62,7 +62,7 @@ function defineSlowTests(factory: StorageFactory) {
 bucket_definitions:
   global:
     data:
-      - SELECT id, description FROM "test_data"
+      - SELECT * FROM "test_data"
 `;
       const syncRules = await f.updateSyncRules({ content: syncRuleContent });
       const storage = f.getInstance(syncRules.parsed());
@@ -76,7 +76,10 @@ bucket_definitions:
       walStream = new WalStream(options);
 
       await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+      await pool.query(
+        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
+      );
+      await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
 
       await walStream.initReplication(replicationConnection);
       await storage.autoActivate();
@@ -88,14 +91,17 @@ bucket_definitions:
 
       while (!abort && Date.now() - start < TEST_DURATION_MS) {
         const bg = async () => {
-          for (let j = 0; j < 5 && !abort; j++) {
-            const n = Math.floor(Math.random() * 50);
+          for (let j = 0; j < 1 && !abort; j++) {
+            const n = 1;
             let statements: pgwire.Statement[] = [];
             for (let i = 0; i < n; i++) {
               const description = `test${i}`;
               statements.push({
-                statement: `INSERT INTO test_data(description) VALUES($1) returning id as test_id`,
-                params: [{ type: 'varchar', value: description }]
+                statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
+                params: [
+                  { type: 'varchar', value: description },
+                  { type: 'float8', value: Math.random() }
+                ]
               });
             }
             const results = await pool.query(...statements);
@@ -104,6 +110,24 @@ bucket_definitions:
             });
             await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
 
+            if (Math.random() > 0.5) {
+              const updateStatements: pgwire.Statement[] = ids.map((id) => {
+                return {
+                  statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
+                  params: [
+                    { type: 'uuid', value: id },
+                    { type: 'float8', value: Math.random() }
+                  ]
+                };
+              });
+
+              await pool.query(...updateStatements);
+              if (Math.random() > 0.5) {
+                // Special case - an update that doesn't change data
+                await pool.query(...updateStatements);
+              }
+            }
+
             const deleteStatements: pgwire.Statement[] = ids.map((id) => {
               return {
                 statement: `DELETE FROM test_data WHERE id = $1`,
@@ -129,6 +153,21 @@ bucket_definitions:
           return bson.deserialize((doc.data as mongo.Binary).buffer) as SqliteRow;
         });
         expect(transformed).toEqual([]);
+
+        // Check that each PUT has a REMOVE
+        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
+        let active = new Set<string>();
+        for (let op of ops) {
+          const key = op.source_key.toHexString();
+          if (op.op == 'PUT') {
+            active.add(key);
+          } else if (op.op == 'REMOVE') {
+            active.delete(key);
+          }
+        }
+        if (active.size > 0) {
+          throw new Error(`${active.size} rows not removed`);
+        }
       }
 
       abortController.abort();

package/test/src/sync.test.ts

@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import { streamResponse } from '../../src/sync/sync.js';
 import * as timers from 'timers/promises';
 import { lsnMakeComparable } from '@powersync/service-jpgwire';
+import { RequestParameters } from '@powersync/service-sync-rules';
 
 describe('sync - mongodb', function () {
   defineTests(MONGO_STORAGE_FACTORY);
@@ -77,7 +78,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
-      syncParams: { token_parameters: {}, user_parameters: {} },
+      syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
     });
 
@@ -117,7 +118,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: false
       },
-      syncParams: { token_parameters: {}, user_parameters: {} },
+      syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
     });
 
@@ -145,7 +146,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
-      syncParams: { token_parameters: {}, user_parameters: {} },
+      syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: 0 } as any
     });
 
@@ -171,7 +172,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
-      syncParams: { token_parameters: {}, user_parameters: {} },
+      syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: Date.now() / 1000 + 10 } as any
     });
     const iter = stream[Symbol.asyncIterator]();
@@ -231,7 +232,7 @@ function defineTests(factory: StorageFactory) {
         include_checksum: true,
         raw_data: true
       },
-      syncParams: { token_parameters: {}, user_parameters: {} },
+      syncParams: new RequestParameters({ sub: '' }, {}),
       token: { exp: exp } as any
     });
     const iter = stream[Symbol.asyncIterator]();