@powersync/service-core 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219110735

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.0-dev-20241128134723",
+  "version": "0.0.0-dev-20241219110735",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -26,18 +26,17 @@
     "jose": "^4.15.1",
     "lodash": "^4.17.21",
     "lru-cache": "^10.2.2",
-    "mongodb": "^6.7.0",
     "node-fetch": "^3.3.2",
-    "ts-codec": "^1.2.2",
+    "ts-codec": "^1.3.0",
     "uri-js": "^4.4.1",
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.2.0",
+    "@powersync/lib-services-framework": "0.0.0-dev-20241219110735",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-rsocket-router": "0.0.14",
-    "@powersync/service-sync-rules": "0.21.0",
-    "@powersync/service-types": "0.0.0-dev-20241128134723"
+    "@powersync/service-rsocket-router": "0.0.0-dev-20241219110735",
+    "@powersync/service-sync-rules": "0.0.0-dev-20241219110735",
+    "@powersync/service-types": "0.0.0-dev-20241219110735"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",

package/src/auth/KeySpec.ts

@@ -2,7 +2,8 @@ import * as jose from 'jose';
 
 export const HS_ALGORITHMS = ['HS256', 'HS384', 'HS512'];
 export const RSA_ALGORITHMS = ['RS256', 'RS384', 'RS512'];
-export const SUPPORTED_ALGORITHMS = [...HS_ALGORITHMS, ...RSA_ALGORITHMS];
+export const OKP_ALGORITHMS = ['EdDSA'];
+export const SUPPORTED_ALGORITHMS = [...HS_ALGORITHMS, ...RSA_ALGORITHMS, ...OKP_ALGORITHMS];
 
 export interface KeyOptions {
   /**
@@ -38,17 +39,19 @@ export class KeySpec {
     return this.source.kid;
   }
 
-  matchesAlgorithm(jwtAlg: string) {
+  matchesAlgorithm(jwtAlg: string): boolean {
     if (this.source.alg) {
-      return jwtAlg == this.source.alg;
-    } else if (this.source.kty == 'RSA') {
+      return jwtAlg === this.source.alg;
+    } else if (this.source.kty === 'RSA') {
       return RSA_ALGORITHMS.includes(jwtAlg);
-    } else if (this.source.kty == 'oct') {
+    } else if (this.source.kty === 'oct') {
       return HS_ALGORITHMS.includes(jwtAlg);
-    } else {
-      // We don't support 'ec' yet
-      return false;
+    } else if (this.source.kty === 'OKP') {
+      return OKP_ALGORITHMS.includes(jwtAlg);
     }
+    
+    // 'EC' is unsupported
+    return false;
   }
 
   async isValidSignature(token: string): Promise<boolean> {
@@ -56,7 +59,7 @@ export class KeySpec {
       await jose.compactVerify(token, this.key);
       return true;
     } catch (e) {
-      if (e.code == 'ERR_JWS_SIGNATURE_VERIFICATION_FAILED') {
+      if (e.code === 'ERR_JWS_SIGNATURE_VERIFICATION_FAILED') {
         return false;
       } else {
         // Token format error most likely

package/src/auth/RemoteJWKSCollector.ts

@@ -73,8 +73,8 @@ export class RemoteJWKSCollector implements KeyCollector {
 
     let keys: KeySpec[] = [];
     for (let keyData of data.keys) {
-      if (keyData.kty != 'RSA') {
-        // Only RSA public keys supported here
+      if (keyData.kty != 'RSA' && keyData.kty != 'OKP') {
+        // HS (oct) keys not allowed because they are symmetric
         continue;
       }
 

package/src/entry/commands/compact-action.ts

@@ -1,9 +1,11 @@
 import { Command } from 'commander';
 
-import { logger } from '@powersync/lib-services-framework';
+import { container, logger } from '@powersync/lib-services-framework';
 import * as v8 from 'v8';
-import * as storage from '../../storage/storage-index.js';
+import * as system from '../../system/system-index.js';
 import * as utils from '../../util/util-index.js';
+
+import { modules } from '../../index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'compact';
@@ -29,23 +31,25 @@ export function registerCompactAction(program: Command) {
 
   return compactCommand.description('Compact storage').action(async (options) => {
     const buckets = options.buckets?.split(',');
-    if (buckets != null) {
+    if (buckets == null) {
       logger.info('Compacting storage for all buckets...');
     } else {
-      logger.info(`Compacting storage for ${buckets.join(', ')}...`);
+      logger.info(`Compacting storage for ${buckets?.join(', ')}...`);
     }
-    const runnerConfig = extractRunnerOptions(options);
-    const configuration = await utils.loadConfig(runnerConfig);
-    logger.info('Successfully loaded configuration...');
-    const { storage: storageConfig } = configuration;
+    const config = await utils.loadConfig(extractRunnerOptions(options));
+    const serviceContext = new system.ServiceContextContainer(config);
+
+    // Register modules in order to allow custom module compacting
+    const moduleManager = container.getImplementation(modules.ModuleManager);
+    await moduleManager.initialize(serviceContext);
+
     logger.info('Connecting to storage...');
-    const psdb = storage.createPowerSyncMongo(storageConfig);
-    const client = psdb.client;
-    await client.connect();
+
     try {
-      const bucketStorage = new storage.MongoBucketStorage(psdb, {
-        slot_name_prefix: configuration.slot_name_prefix
-      });
+      // Start the storage engine in order to create the appropriate BucketStorage
+      await serviceContext.lifeCycleEngine.start();
+      const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
+
       const active = await bucketStorage.getActiveSyncRulesContent();
       if (active == null) {
         logger.info('No active instance to compact');
@@ -57,9 +61,10 @@ export function registerCompactAction(program: Command) {
       logger.info('Successfully compacted storage.');
     } catch (e) {
       logger.error(`Failed to compact: ${e.toString()}`);
+      // Indirectly triggers lifeCycleEngine.stop
       process.exit(1);
     } finally {
-      await client.close();
+      // Indirectly triggers lifeCycleEngine.stop
       process.exit(0);
     }
   });

package/src/entry/commands/migrate-action.ts

@@ -1,7 +1,9 @@
-import { logger } from '@powersync/lib-services-framework';
+import { container, logger, migrations } from '@powersync/lib-services-framework';
 import { Command } from 'commander';
 
-import * as migrations from '../../migrations/migrations-index.js';
+import * as modules from '../../modules/modules-index.js';
+import * as system from '../../system/system-index.js';
+import * as utils from '../../util/util-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'migrate';
@@ -15,12 +17,23 @@ export function registerMigrationAction(program: Command) {
     .description('Run migrations')
     .argument('<direction>', 'Migration direction. `up` or `down`')
     .action(async (direction: migrations.Direction, options) => {
+      const config = await utils.loadConfig(extractRunnerOptions(options));
+      const serviceContext = new system.ServiceContextContainer(config);
+
+      // Register modules in order to allow custom module migrations
+      const moduleManager = container.getImplementation(modules.ModuleManager);
+      await moduleManager.initialize(serviceContext);
+
       try {
-        await migrations.migrate({
+        await serviceContext.migrations.migrate({
           direction,
-          runner_config: extractRunnerOptions(options)
+          // Give the migrations access to the service context
+          migrationContext: {
+            service_context: serviceContext
+          }
         });
 
+        await serviceContext.migrations[Symbol.asyncDispose]();
         process.exit(0);
       } catch (e) {
         logger.error(`Migration failure`, e);

package/src/index.ts

@@ -6,9 +6,6 @@ export * as api from './api/api-index.js';
 export * from './auth/auth-index.js';
 export * as auth from './auth/auth-index.js';
 
-export * from './db/db-index.js';
-export * as db from './db/db-index.js';
-
 export * from './entry/entry-index.js';
 export * as entry from './entry/entry-index.js';
 
@@ -18,8 +15,8 @@ export * as framework from '@powersync/lib-services-framework';
 export * from './metrics/Metrics.js';
 export * as metrics from './metrics/Metrics.js';
 
+export * from './migrations/migrations-index.js';
 export * as migrations from './migrations/migrations-index.js';
-export * from './migrations/migrations.js';
 
 export * from './modules/modules-index.js';
 export * as modules from './modules/modules-index.js';

package/src/migrations/PowerSyncMigrationManager.ts

@@ -0,0 +1,43 @@
+import * as framework from '@powersync/lib-services-framework';
+import fs from 'fs/promises';
+import path from 'path';
+import * as system from '../system/system-index.js';
+
+/**
+ * PowerSync service migrations each have this context available to the `up` and `down` methods.
+ */
+export interface PowerSyncMigrationContext {
+  service_context: system.ServiceContext;
+}
+
+export interface PowerSyncMigrationGenerics extends framework.MigrationAgentGenerics {
+  MIGRATION_CONTEXT: PowerSyncMigrationContext;
+}
+
+export type PowerSyncMigrationFunction = framework.MigrationFunction<PowerSyncMigrationContext>;
+
+export abstract class AbstractPowerSyncMigrationAgent extends framework.AbstractMigrationAgent<PowerSyncMigrationGenerics> {
+  abstract getInternalScriptsDir(): string;
+
+  async loadInternalMigrations(): Promise<framework.Migration<PowerSyncMigrationContext>[]> {
+    const migrationsDir = this.getInternalScriptsDir();
+    const files = await fs.readdir(migrationsDir);
+    const migrations = files.filter((file) => {
+      // Vitest resolves ts files
+      return ['.js', '.ts'].includes(path.extname(file));
+    });
+
+    return await Promise.all(
+      migrations.map(async (migration) => {
+        const module = await import(path.resolve(migrationsDir, migration));
+        return {
+          name: path.basename(migration).replace(path.extname(migration), ''),
+          up: module.up,
+          down: module.down
+        };
+      })
+    );
+  }
+}
+
+export type PowerSyncMigrationManager = framework.MigrationManager<PowerSyncMigrationGenerics>;

package/src/migrations/ensure-automatic-migrations.ts

@@ -0,0 +1,15 @@
+import * as framework from '@powersync/lib-services-framework';
+import * as system from '../system/system-index.js';
+
+export const ensureAutomaticMigrations = async (options: { serviceContext: system.ServiceContext }) => {
+  const { serviceContext } = options;
+  if (serviceContext.configuration.migrations?.disable_auto_migration) {
+    return;
+  }
+  await serviceContext.migrations.migrate({
+    direction: framework.migrations.Direction.Up,
+    migrationContext: {
+      service_context: serviceContext
+    }
+  });
+};

package/src/migrations/migrations-index.ts

@@ -1,3 +1,2 @@
-export * from './definitions.js';
-export * from './executor.js';
-export * from './migrations.js';
+export * from './ensure-automatic-migrations.js';
+export * from './PowerSyncMigrationManager.js';

package/src/routes/RouterEngine.ts

@@ -4,6 +4,7 @@ import * as api from '../api/api-index.js';
 
 import { ADMIN_ROUTES } from './endpoints/admin.js';
 import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { PROBES_ROUTES } from './endpoints/probes.js';
 import { syncStreamReactive } from './endpoints/socket-route.js';
 import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
 import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
@@ -47,7 +48,7 @@ export class RouterEngine {
 
     // Default routes
     this.routes = {
-      api_routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES],
+      api_routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...SYNC_RULES_ROUTES, ...PROBES_ROUTES],
       stream_routes: [...SYNC_STREAM_ROUTES],
       socket_routes: [syncStreamReactive]
     };

package/src/storage/BucketStorage.ts

@@ -8,13 +8,49 @@ import {
   SqliteRow,
   ToastableSqliteRow
 } from '@powersync/service-sync-rules';
+import { BSON } from 'bson';
 import * as util from '../util/util-index.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
-import { BatchedCustomWriteCheckpointOptions, ReplicaId } from './storage-index.js';
+import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = BSON.UUID | BSON.Document | any;
+
+export enum SyncRuleState {
+  /**
+   * New sync rules - needs to be processed (initial replication).
+   *
+   * While multiple sets of sync rules _can_ be in PROCESSING,
+   * it's generally pointless, so we only keep one in that state.
+   */
+  PROCESSING = 'PROCESSING',
+
+  /**
+   * Sync rule processing is done, and can be used for sync.
+   *
+   * Only one set of sync rules should be in ACTIVE state.
+   */
+  ACTIVE = 'ACTIVE',
+  /**
+   * This state is used when the sync rules has been replaced,
+   * and replication is or should be stopped.
+   */
+  STOP = 'STOP',
+  /**
+   * After sync rules have been stopped, the data needs to be
+   * deleted. Once deleted, the state is TERMINATED.
+   */
+  TERMINATED = 'TERMINATED'
+}
 export interface BucketStorageFactoryListener extends DisposableListener {
   syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
   replicationEvent: (event: ReplicationEventPayload) => void;
@@ -211,6 +247,13 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
    * database, for example from MongoDB.
    */
   storeCurrentData: boolean;
+
+  /**
+   * Set to true for initial replication.
+   *
+   * This will avoid creating new operations for rows previously replicated.
+   */
+  skipExistingRows?: boolean;
 }
 
 export interface SyncRulesBucketStorageListener extends DisposableListener {

package/src/storage/storage-index.ts

@@ -1,21 +1,9 @@
 export * from './BucketStorage.js';
-export * from './MongoBucketStorage.js';
+export * from './ChecksumCache.js';
 export * from './ReplicationEventPayload.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
-
-export * from './mongo/config.js';
-export * from './mongo/db.js';
-export * from './mongo/models.js';
-export * from './mongo/MongoBucketBatch.js';
-export * from './mongo/MongoIdSequence.js';
-export * from './mongo/MongoPersistedSyncRules.js';
-export * from './mongo/MongoPersistedSyncRulesContent.js';
-export * from './mongo/MongoStorageProvider.js';
-export * from './mongo/MongoSyncBucketStorage.js';
-export * from './mongo/MongoSyncRulesLock.js';
-export * from './mongo/OperationBatch.js';
-export * from './mongo/PersistedBatch.js';
-export * from './mongo/util.js';
+export * from './StorageProvider.js';
 export * from './WriteCheckpointAPI.js';
+

package/src/sync/sync.ts

@@ -318,9 +318,18 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         // Send the object as is, will most likely be encoded as a BSON document
         send_data = { data: r };
       } else if (raw_data) {
-        // Data is a raw string - we can use the more efficient JSON.stringify.
+        /**
+         * Data is a raw string - we can use the more efficient JSON.stringify.
+         * FIXME: ensure that the data is actually a string
+         */
         const response: util.StreamingSyncData = {
-          data: r
+          data: {
+            ...r,
+            data: r.data.map((entry) => ({
+              ...entry,
+              data: typeof entry.data == 'string' ? entry.data : JSONBig.stringify(entry.data)
+            }))
+          }
         };
         send_data = JSON.stringify(response);
       } else {
@@ -375,7 +384,7 @@ function transformLegacyResponse(bucketData: util.SyncBucketData): any {
     data: bucketData.data.map((entry) => {
       return {
         ...entry,
-        data: entry.data == null ? null : new JsonContainer(entry.data as string),
+        data: entry.data == null ? null : typeof entry.data == 'string' ? new JsonContainer(entry.data) : entry.data,
         checksum: BigInt(entry.checksum)
       };
     })

package/src/system/ServiceContext.ts

@@ -1,6 +1,8 @@
-import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
+import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
 
+import { framework } from '../index.js';
 import * as metrics from '../metrics/Metrics.js';
+import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
 import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
@@ -13,6 +15,7 @@ export interface ServiceContext {
   replicationEngine: replication.ReplicationEngine | null;
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
+  migrations: PowerSyncMigrationManager;
 }
 
 /**
@@ -30,13 +33,19 @@ export class ServiceContextContainer implements ServiceContext {
     this.storageEngine = new storage.StorageEngine({
       configuration
     });
+
+    const migrationManager = new MigrationManager();
+    container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
+
+    this.lifeCycleEngine.withLifecycle(migrationManager, {
+      // Migrations should be executed before the system starts
+      start: () => migrationManager[Symbol.asyncDispose]()
+    });
+
     this.lifeCycleEngine.withLifecycle(this.storageEngine, {
       start: (storageEngine) => storageEngine.start(),
       stop: (storageEngine) => storageEngine.shutDown()
     });
-
-    // Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
-    this.storageEngine.registerProvider(new storage.MongoStorageProvider());
   }
 
   get replicationEngine(): replication.ReplicationEngine | null {
@@ -51,6 +60,10 @@ export class ServiceContextContainer implements ServiceContext {
     return container.getOptional(metrics.Metrics);
   }
 
+  get migrations(): PowerSyncMigrationManager {
+    return container.getImplementation(framework.ContainerImplementation.MIGRATION_MANAGER);
+  }
+
   /**
    * Allows for registering core and generic implementations of services/helpers.
    * This uses the framework container under the hood.

package/src/util/config/types.ts

@@ -30,8 +30,8 @@ export type SyncRulesConfig = {
 
 export type ResolvedPowerSyncConfig = {
   base_config: PowerSyncConfig;
-  connections?: configFile.DataSourceConfig[];
-  storage: configFile.StorageConfig;
+  connections?: configFile.GenericDataSourceConfig[];
+  storage: configFile.GenericStorageConfig;
   dev: {
     demo_auth: boolean;
     demo_password?: string;

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, OpId } from './protocol-types.js';
+import { BucketChecksum, OpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
@@ -144,3 +144,61 @@ export function checkpointUserId(user_id: string | undefined, client_id: string
   }
   return `${user_id}/${client_id}`;
 }
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ *
+ * Used for tests.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}