@powersync/service-core 0.2.2 → 0.4.0

package/src/sync/util.ts

@@ -1,7 +1,7 @@
 import * as timers from 'timers/promises';
 
-import * as util from '@/util/util-index.js';
-import { Metrics } from '@/metrics/Metrics.js';
+import * as util from '../util/util-index.js';
+import { Metrics } from '../metrics/Metrics.js';
 
 export type TokenStreamOptions = {
   /**

package/src/system/CorePowerSyncSystem.ts

@@ -1,26 +1,47 @@
 import * as pgwire from '@powersync/service-jpgwire';
-import * as micro from '@journeyapps-platform/micro';
+import { LifeCycledSystem, container, logger } from '@powersync/lib-services-framework';
 
-import * as auth from '@/auth/auth-index.js';
-import * as storage from '@/storage/storage-index.js';
-import * as utils from '@/util/util-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as utils from '../util/util-index.js';
 
-export abstract class CorePowerSyncSystem extends micro.system.MicroSystem {
+export abstract class CorePowerSyncSystem extends LifeCycledSystem {
   abstract storage: storage.BucketStorageFactory;
-  abstract client_keystore: auth.KeyStore;
-  abstract dev_client_keystore: auth.KeyStore;
   abstract pgwire_pool?: pgwire.PgClient;
+  closed: boolean;
 
   protected stopHandlers: Set<() => void> = new Set();
 
-  closed: boolean;
-
   constructor(public config: utils.ResolvedPowerSyncConfig) {
     super();
     this.closed = false;
   }
 
-  abstract addTerminationHandler(): void;
+  get client_keystore() {
+    return this.config.client_keystore;
+  }
+
+  get dev_client_keystore() {
+    return this.config.dev_client_keystore;
+  }
+
+  /**
+   * Adds a termination handler which will call handlers registered via
+   * [addStopHandler].
+   * This should be called after the server is started and it's termination handler is added.
+   * This is so that the handler is run before the server's handler, allowing streams to be interrupted on exit
+   */
+  addTerminationHandler() {
+    container.terminationHandler.handleTerminationSignal(async () => {
+      // Close open streams, so that they don't block the server from closing.
+      // Note: This does not work well when streaming requests are queued. In that case, the server still doesn't
+      // close in the 30-second timeout.
+      this.closed = true;
+      logger.info(`Closing ${this.stopHandlers.size} streams`);
+      for (let handler of this.stopHandlers) {
+        handler();
+      }
+    });
+  }
 
   addStopHandler(handler: () => void): () => void {
     if (this.closed) {

package/src/system/system-index.ts

@@ -0,0 +1 @@
+export * from './CorePowerSyncSystem.js';

package/src/util/alerting.ts

@@ -1,5 +1,3 @@
-import * as micro from '@journeyapps-platform/micro';
-
 let globalTags: Record<string, string> = {};
 
 export function setTags(tags: Record<string, string>) {
@@ -9,9 +7,3 @@ export function setTags(tags: Record<string, string>) {
 export function getGlobalTags() {
   return globalTags;
 }
-
-export function captureException(error: any, options?: micro.alerts.CaptureOptions) {
-  micro.alerts.captureException(error, {
-    ...options
-  });
-}

package/src/util/config/collectors/config-collector.ts

@@ -1,8 +1,10 @@
 import * as t from 'ts-codec';
+import * as yaml from 'yaml';
+
 import { configFile } from '@powersync/service-types';
-import * as micro from '@journeyapps-platform/micro';
+import { schema } from '@powersync/lib-services-framework';
+
 import { RunnerConfig } from '../types.js';
-import * as yaml from 'yaml';
 
 export enum ConfigFileFormat {
   YAML = 'yaml',
@@ -22,7 +24,7 @@ export enum ConfigFileFormat {
 const YAML_ENV_PREFIX = 'PS_';
 
 // ts-codec itself doesn't give great validation errors, so we use json schema for that
-const configSchemaValidator = micro.schema
+const configSchemaValidator = schema
   .parseJSONSchema(
     t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] })
   )

package/src/util/config/collectors/impl/filesystem-config-collector.ts

@@ -1,8 +1,9 @@
 import * as fs from 'fs/promises';
-import * as micro from '@journeyapps-platform/micro';
+import * as path from 'path';
 
 import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
 import { RunnerConfig } from '../../types.js';
+import { logger } from '@powersync/lib-services-framework';
 
 export class FileSystemConfigCollector extends ConfigCollector {
   get name(): string {
@@ -15,16 +16,17 @@ export class FileSystemConfigCollector extends ConfigCollector {
       return null;
     }
 
+    const resolvedPath = path.resolve(process.cwd(), config_path);
+
     // Check if file exists
     try {
-      await fs.access(config_path, fs.constants.F_OK);
+      await fs.access(resolvedPath, fs.constants.F_OK);
     } catch (ex) {
-      throw new Error(`Config file path ${config_path} was specified, but the file does not exist.`);
+      throw new Error(`Config file path ${resolvedPath} was specified, but the file does not exist.`);
     }
 
-    micro.logger.info(`Collecting PowerSync configuration from File: ${config_path}`);
-
-    const content = await fs.readFile(config_path, 'utf-8');
+    logger.info(`Collecting PowerSync configuration from File: ${resolvedPath}`);
+    const content = await fs.readFile(resolvedPath, 'utf-8');
 
     let contentType: ConfigFileFormat | undefined;
     switch (true) {

package/src/util/config/compound-config-collector.ts

@@ -1,8 +1,7 @@
-import * as micro from '@journeyapps-platform/micro';
 import { configFile, normalizeConnection } from '@powersync/service-types';
 import { ConfigCollector } from './collectors/config-collector.js';
 import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
-import * as auth from '@/auth/auth-index.js';
+import * as auth from '../../auth/auth-index.js';
 import { SyncRulesCollector } from './sync-rules/sync-collector.js';
 import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
 import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
@@ -10,6 +9,7 @@ import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-co
 import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
 import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
+import { logger } from '@powersync/lib-services-framework';
 
 const POWERSYNC_DEV_KID = 'powersync-dev';
 
@@ -140,7 +140,7 @@ export class CompoundConfigCollector {
         if (baseConfig) {
           return baseConfig;
         }
-        micro.logger.debug(
+        logger.debug(
           `Could not collect PowerSync config with ${collector.name} method. Moving on to next method if available.`
         );
       } catch (ex) {
@@ -161,7 +161,7 @@ export class CompoundConfigCollector {
         if (config) {
           return config;
         }
-        micro.logger.debug(
+        logger.debug(
           `Could not collect sync rules with ${collector.name} method. Moving on to next method if available.`
         );
       } catch (ex) {

package/src/util/env.ts

@@ -1,4 +1,4 @@
-import { utils } from '@journeyapps-platform/micro';
+import { utils } from '@powersync/lib-services-framework';
 
 import { ServiceRunner } from './config/types.js';
 
@@ -22,7 +22,9 @@ export const env = utils.collectEnvironmentVariables({
   /**
    * Port for metrics
    */
-  METRICS_PORT: utils.type.number.optional()
+  METRICS_PORT: utils.type.number.optional(),
+
+  NODE_ENV: utils.type.string.optional()
 });
 
 export type Env = typeof env;

package/src/util/memory-tracking.ts

@@ -1,4 +1,4 @@
-import * as micro from '@journeyapps-platform/micro';
+import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Track and log memory usage.
@@ -57,7 +57,7 @@ export function trackMemoryUsage() {
         )
       )`.replaceAll(/\s+/g, ' ');
 
-      micro.logger.info(output);
+      logger.info(output);
     }
   }, 50);
 }

package/src/util/pgwire_utils.ts

@@ -4,9 +4,9 @@ import * as bson from 'bson';
 import * as uuid from 'uuid';
 import * as pgwire from '@powersync/service-jpgwire';
 import { SqliteJsonValue, SqliteRow, ToastableSqliteRow, toSyncRulesRow } from '@powersync/service-sync-rules';
-import * as micro from '@journeyapps-platform/micro';
 
-import * as replication from '@/replication/replication-index.js';
+import * as replication from '../replication/replication-index.js';
+import { logger } from '@powersync/lib-services-framework';
 
 /**
  * pgwire message -> SQLite row.
@@ -133,7 +133,7 @@ export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) {
       if (tries == 1) {
         throw e;
       }
-      micro.logger.warn('Query error, retrying', e);
+      logger.warn('Query error, retrying', e);
     }
   }
 }

package/src/util/utils.ts

@@ -1,11 +1,11 @@
 import crypto from 'crypto';
 import * as pgwire from '@powersync/service-jpgwire';
 import { pgwireRows } from '@powersync/service-jpgwire';
-import * as micro from '@journeyapps-platform/micro';
 
-import * as storage from '@/storage/storage-index.js';
+import * as storage from '../storage/storage-index.js';
 import { BucketChecksum, OpId } from './protocol-types.js';
 import { retriedQuery } from './pgwire_utils.js';
+import { logger } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
@@ -90,14 +90,14 @@ export async function getClientCheckpoint(
 
   const timeout = options?.timeout ?? 50_000;
 
-  micro.logger.info(`Waiting for LSN checkpoint: ${lsn}`);
+  logger.info(`Waiting for LSN checkpoint: ${lsn}`);
   while (Date.now() - start < timeout) {
     const cp = await bucketStorage.getActiveCheckpoint();
     if (!cp.hasSyncRules()) {
       throw new Error('No sync rules available');
     }
     if (cp.lsn >= lsn) {
-      micro.logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
+      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
       return cp.checkpoint;
     }
 
@@ -117,6 +117,6 @@ export async function createWriteCheckpoint(
   );
 
   const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
-  micro.logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
+  logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
   return id;
 }

package/test/src/auth.test.ts

@@ -262,7 +262,8 @@ describe('JWT Auth', () => {
     expect(errors).toEqual([]);
     expect(keys.length).toBeGreaterThanOrEqual(1);
 
-    const invalid = new RemoteJWKSCollector('https://localhost/.well-known/jwks.json', {
+    // The localhost hostname fails to resolve correctly on MacOS https://github.com/nodejs/help/issues/2163
+    const invalid = new RemoteJWKSCollector('https://127.0.0.1/.well-known/jwks.json', {
       block_local_ip: true
     });
     expect(invalid.getKeys()).rejects.toThrow('IPs in this range are not supported');
@@ -278,7 +279,8 @@ describe('JWT Auth', () => {
     expect(errors).toEqual([]);
     expect(keys.length).toBeGreaterThanOrEqual(1);
 
-    const invalid = new RemoteJWKSCollector('https://localhost/.well-known/jwks.json');
+    // The localhost hostname fails to resolve correctly on MacOS https://github.com/nodejs/help/issues/2163
+    const invalid = new RemoteJWKSCollector('https://127.0.0.1/.well-known/jwks.json');
     // Should try and fetch
     expect(invalid.getKeys()).rejects.toThrow('ECONNREFUSED');
   });

package/test/src/data_storage.test.ts

@@ -1,4 +1,4 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
 import { SourceTable } from '../../src/storage/SourceTable.js';
@@ -289,12 +289,7 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const parameters = {
-      token_parameters: {
-        user_id: 'u1'
-      },
-      user_parameters: {}
-    };
+    const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
 
@@ -358,12 +353,7 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const parameters = {
-      token_parameters: {
-        user_id: 'unknown'
-      },
-      user_parameters: {}
-    };
+    const parameters = new RequestParameters({ sub: 'unknown' }, {});
 
     const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
 
@@ -442,12 +432,7 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const parameters = {
-      token_parameters: {
-        user_id: 'u1'
-      },
-      user_parameters: {}
-    };
+    const parameters = new RequestParameters({ sub: 'u1' }, {});
 
     // Test intermediate values - could be moved to sync_rules.test.ts
     const q1 = sync_rules.bucket_descriptors[0].parameter_queries[0];
@@ -897,6 +882,183 @@ bucket_definitions:
     ]);
   });
 
+  test('changed data with replica identity full', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const sourceTable = makeTestTable('test', ['id', 'description']);
+
+    // Pre-setup
+    const result1 = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable,
+        tag: 'insert',
+        after: {
+          id: 'test1',
+          description: 'test1a'
+        }
+      });
+    });
+
+    const checkpoint1 = result1?.flushed_op ?? '0';
+
+    const result2 = await storage.startBatch({}, async (batch) => {
+      // Unchanged, but has a before id
+      await batch.save({
+        sourceTable,
+        tag: 'update',
+        before: {
+          id: 'test1',
+          description: 'test1a'
+        },
+        after: {
+          id: 'test1',
+          description: 'test1b'
+        }
+      });
+    });
+
+    const result3 = await storage.startBatch({}, async (batch) => {
+      // Delete
+      await batch.save({
+        sourceTable,
+        tag: 'delete',
+        before: {
+          id: 'test1',
+          description: 'test1b'
+        },
+        after: undefined
+      });
+    });
+
+    const checkpoint3 = result3!.flushed_op;
+
+    const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
+    const data = batch[0].data.map((d) => {
+      return {
+        op: d.op,
+        object_id: d.object_id,
+        data: d.data,
+        subkey: d.subkey
+      };
+    });
+
+    // Operations must be in this order
+    expect(data).toEqual([
+      // 2
+      // The REMOVE is expected because the subkey changes
+      {
+        op: 'REMOVE',
+        object_id: 'test1',
+        data: null,
+        subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+      },
+      {
+        op: 'PUT',
+        object_id: 'test1',
+        data: JSON.stringify({ id: 'test1', description: 'test1b' }),
+        subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
+      },
+      // 3
+      {
+        op: 'REMOVE',
+        object_id: 'test1',
+        data: null,
+        subkey: '6544e3899293153fa7b38331/500e9b68-a2fd-51ff-9c00-313e2fb9f562'
+      }
+    ]);
+  });
+
+  test('unchanged data with replica identity full', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test"
+`);
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const sourceTable = makeTestTable('test', ['id', 'description']);
+
+    // Pre-setup
+    const result1 = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable,
+        tag: 'insert',
+        after: {
+          id: 'test1',
+          description: 'test1a'
+        }
+      });
+    });
+
+    const checkpoint1 = result1?.flushed_op ?? '0';
+
+    const result2 = await storage.startBatch({}, async (batch) => {
+      // Unchanged, but has a before id
+      await batch.save({
+        sourceTable,
+        tag: 'update',
+        before: {
+          id: 'test1',
+          description: 'test1a'
+        },
+        after: {
+          id: 'test1',
+          description: 'test1a'
+        }
+      });
+    });
+
+    const result3 = await storage.startBatch({}, async (batch) => {
+      // Delete
+      await batch.save({
+        sourceTable,
+        tag: 'delete',
+        before: {
+          id: 'test1',
+          description: 'test1a'
+        },
+        after: undefined
+      });
+    });
+
+    const checkpoint3 = result3!.flushed_op;
+
+    const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
+    const data = batch[0].data.map((d) => {
+      return {
+        op: d.op,
+        object_id: d.object_id,
+        data: d.data,
+        subkey: d.subkey
+      };
+    });
+
+    // Operations must be in this order
+    expect(data).toEqual([
+      // 2
+      {
+        op: 'PUT',
+        object_id: 'test1',
+        data: JSON.stringify({ id: 'test1', description: 'test1a' }),
+        subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+      },
+      // 3
+      {
+        op: 'REMOVE',
+        object_id: 'test1',
+        data: null,
+        subkey: '6544e3899293153fa7b38331/740ba9f2-8b0f-53e3-bb17-5f38a9616f0e'
+      }
+    ]);
+  });
+
   test('large batch', async () => {
     // Test syncing a batch of data that is small in count,
     // but large enough in size to be split over multiple returned batches.

package/test/src/env.ts

@@ -1,8 +1,8 @@
-import * as micro from '@journeyapps-platform/micro';
+import { utils } from '@powersync/lib-services-framework';
 
-export const env = micro.utils.collectEnvironmentVariables({
-  MONGO_TEST_URL: micro.utils.type.string.default('mongodb://localhost:27017/powersync_test'),
-  PG_TEST_URL: micro.utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
-  CI: micro.utils.type.boolean.default('false'),
-  SLOW_TESTS: micro.utils.type.boolean.default('false')
+export const env = utils.collectEnvironmentVariables({
+  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
+  PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'),
+  CI: utils.type.boolean.default('false'),
+  SLOW_TESTS: utils.type.boolean.default('false')
 });

package/test/src/setup.ts

@@ -0,0 +1,7 @@
+import { container } from '@powersync/lib-services-framework';
+import { beforeAll } from 'vitest';
+
+beforeAll(() => {
+  // Your setup code here
+  container.registerDefaults();
+});

package/test/src/slow_tests.test.ts

@@ -62,7 +62,7 @@ function defineSlowTests(factory: StorageFactory) {
 bucket_definitions:
   global:
     data:
-      - SELECT id, description FROM "test_data"
+      - SELECT * FROM "test_data"
 `;
       const syncRules = await f.updateSyncRules({ content: syncRuleContent });
       const storage = f.getInstance(syncRules.parsed());
@@ -76,7 +76,10 @@ bucket_definitions:
       walStream = new WalStream(options);
 
       await pool.query(`DROP TABLE IF EXISTS test_data`);
-      await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`);
+      await pool.query(
+        `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text, num decimal)`
+      );
+      await pool.query(`ALTER TABLE test_data REPLICA IDENTITY FULL`);
 
       await walStream.initReplication(replicationConnection);
       await storage.autoActivate();
@@ -88,14 +91,17 @@ bucket_definitions:
 
       while (!abort && Date.now() - start < TEST_DURATION_MS) {
         const bg = async () => {
-          for (let j = 0; j < 5 && !abort; j++) {
-            const n = Math.floor(Math.random() * 50);
+          for (let j = 0; j < 1 && !abort; j++) {
+            const n = 1;
             let statements: pgwire.Statement[] = [];
             for (let i = 0; i < n; i++) {
               const description = `test${i}`;
               statements.push({
-                statement: `INSERT INTO test_data(description) VALUES($1) returning id as test_id`,
-                params: [{ type: 'varchar', value: description }]
+                statement: `INSERT INTO test_data(description, num) VALUES($1, $2) returning id as test_id`,
+                params: [
+                  { type: 'varchar', value: description },
+                  { type: 'float8', value: Math.random() }
+                ]
               });
             }
             const results = await pool.query(...statements);
@@ -104,6 +110,24 @@ bucket_definitions:
             });
             await new Promise((resolve) => setTimeout(resolve, Math.random() * 30));
 
+            if (Math.random() > 0.5) {
+              const updateStatements: pgwire.Statement[] = ids.map((id) => {
+                return {
+                  statement: `UPDATE test_data SET num = $2 WHERE id = $1`,
+                  params: [
+                    { type: 'uuid', value: id },
+                    { type: 'float8', value: Math.random() }
+                  ]
+                };
+              });
+
+              await pool.query(...updateStatements);
+              if (Math.random() > 0.5) {
+                // Special case - an update that doesn't change data
+                await pool.query(...updateStatements);
+              }
+            }
+
             const deleteStatements: pgwire.Statement[] = ids.map((id) => {
               return {
                 statement: `DELETE FROM test_data WHERE id = $1`,
@@ -129,6 +153,21 @@ bucket_definitions:
           return bson.deserialize((doc.data as mongo.Binary).buffer) as SqliteRow;
         });
         expect(transformed).toEqual([]);
+
+        // Check that each PUT has a REMOVE
+        const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray();
+        let active = new Set<string>();
+        for (let op of ops) {
+          const key = op.source_key.toHexString();
+          if (op.op == 'PUT') {
+            active.add(key);
+          } else if (op.op == 'REMOVE') {
+            active.delete(key);
+          }
+        }
+        if (active.size > 0) {
+          throw new Error(`${active.size} rows not removed`);
+        }
       }
 
       abortController.abort();