@powersync/service-core 0.0.0-dev-20241007145127 → 0.0.0-dev-20241015210820

package/dist/storage/write-checkpoint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"write-checkpoint.js","sourceRoot":"","sources":["../../src/storage/write-checkpoint.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,mBAYX;AAZD,WAAY,mBAAmB;IAC7B;;;OAGG;IACH,wCAAiB,CAAA;IACjB;;;;OAIG;IACH,0CAAmB,CAAA;AACrB,CAAC,EAZW,mBAAmB,KAAnB,mBAAmB,QAY9B;AAsDD,MAAM,CAAC,MAAM,6BAA6B,GAAG,mBAAmB,CAAC,OAAO,CAAC"}

package/dist/util/protocol-types.d.ts

@@ -71,7 +71,7 @@ export type StreamingSyncLine = StreamingSyncData | StreamingSyncCheckpoint | St
  * 64-bit unsigned number, as a base-10 string.
  */
 export type OpId = string;
-export interface Checkpoint {
+interface Checkpoint {
     last_op_id: OpId;
     write_checkpoint?: OpId;
     buckets: BucketChecksum[];
@@ -119,3 +119,4 @@ export interface BucketChecksum {
      */
     count: number;
 }
+export {};

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.0-dev-20241007145127",
+  "version": "0.0.0-dev-20241015210820",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -33,11 +33,11 @@
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.0.0-dev-20241007145127",
+    "@powersync/lib-services-framework": "0.0.0-dev-20241015210820",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-rsocket-router": "0.0.0-dev-20241007145127",
-    "@powersync/service-sync-rules": "0.0.0-dev-20241007145127",
-    "@powersync/service-types": "0.0.0-dev-20241007145127"
+    "@powersync/service-rsocket-router": "0.0.0-dev-20241015210820",
+    "@powersync/service-sync-rules": "0.0.0-dev-20241015210820",
+    "@powersync/service-types": "0.0.0-dev-20241015210820"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",

package/src/api/RouteAPI.ts

@@ -1,6 +1,6 @@
 import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
 import * as types from '@powersync/service-types';
-import { ParseSyncRulesOptions } from '../storage/BucketStorage.js';
+import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/BucketStorage.js';
 
 export interface PatternResult {
   schema: string;
@@ -10,6 +10,10 @@ export interface PatternResult {
   table?: types.TableInfo;
 }
 
+export interface ReplicationLagOptions {
+  bucketStorage: SyncRulesBucketStorage;
+}
+
 /**
  *  Describes all the methods currently required to service the sync API endpoints.
  */
@@ -17,7 +21,7 @@ export interface RouteAPI {
   /**
    * @returns basic identification of the connection
    */
-  getSourceConfig(): Promise<types.configFile.DataSourceConfig>;
+  getSourceConfig(): Promise<types.configFile.ResolvedDataSourceConfig>;
 
   /**
    * Checks the current connection status of the data source.
@@ -42,9 +46,8 @@ export interface RouteAPI {
   /**
    * @returns The replication lag: that is the amount of data which has not been
    *          replicated yet, in bytes.
-   * @param {string} syncRulesId An identifier representing which set of sync rules the lag is required for.
    */
-  getReplicationLag(syncRulesId: string): Promise<number | undefined>;
+  getReplicationLag(options: ReplicationLagOptions): Promise<number | undefined>;
 
   /**
    * Get the current LSN or equivalent replication HEAD position identifier

package/src/api/diagnostics.ts

@@ -57,7 +57,7 @@ export async function getSyncRulesStatus(
   // This method can run under some situations if no connection is configured yet.
   // It will return a default tag in such a case. This default tag is not module specific.
   const tag = sourceConfig.tag ?? DEFAULT_TAG;
-  const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
+  using systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
   const status = await systemStorage?.getStatus();
   let replication_lag_bytes: number | undefined = undefined;
 
@@ -78,7 +78,9 @@ export async function getSyncRulesStatus(
 
     if (systemStorage) {
       try {
-        replication_lag_bytes = await apiHandler.getReplicationLag(systemStorage.slot_name);
+        replication_lag_bytes = await apiHandler.getReplicationLag({
+          bucketStorage: systemStorage
+        });
       } catch (e) {
         // Ignore
         logger.warn(`Unable to get replication lag`, e);

package/src/api/schema.ts

@@ -16,9 +16,9 @@ export async function getConnectionsSchema(api: api.RouteAPI): Promise<internal_
   return {
     connections: [
       {
-        schemas: await api.getConnectionSchema(),
-        tag: baseConfig.tag!,
-        id: baseConfig.id
+        id: baseConfig.id,
+        tag: baseConfig.tag,
+        schemas: await api.getConnectionSchema()
       }
     ],
     defaultConnectionTag: baseConfig.tag!,

package/src/entry/commands/compact-action.ts

@@ -36,13 +36,15 @@ export function registerCompactAction(program: Command) {
     const client = psdb.client;
     await client.connect();
     try {
-      const bucketStorage = new storage.MongoBucketStorage(psdb, { slot_name_prefix: configuration.slot_name_prefix });
+      const bucketStorage = new storage.MongoBucketStorage(psdb, {
+        slot_name_prefix: configuration.slot_name_prefix
+      });
       const active = await bucketStorage.getActiveSyncRulesContent();
       if (active == null) {
         logger.info('No active instance to compact');
         return;
       }
-      const p = bucketStorage.getInstance(active);
+      using p = bucketStorage.getInstance(active);
       logger.info('Performing compaction...');
       await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
       logger.info('Successfully compacted storage.');

package/src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts

@@ -0,0 +1,37 @@
+import * as storage from '../../../storage/storage-index.js';
+import * as utils from '../../../util/util-index.js';
+
+const INDEX_NAME = 'user_sync_rule_unique';
+
+export const up = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    await db.custom_write_checkpoints.createIndex(
+      {
+        user_id: 1,
+        sync_rules_id: 1
+      },
+      { name: INDEX_NAME, unique: true }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down = async (context: utils.MigrationContext) => {
+  const { runner_config } = context;
+  const config = await utils.loadConfig(runner_config);
+
+  const db = storage.createPowerSyncMongo(config.storage);
+
+  try {
+    if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
+      await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/replication/AbstractReplicationJob.ts

@@ -1,7 +1,7 @@
-import * as storage from '../storage/storage-index.js';
-import { ErrorRateLimiter } from './ErrorRateLimiter.js';
 import { container, logger } from '@powersync/lib-services-framework';
 import winston from 'winston';
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
 
 export interface AbstractReplicationJobOptions {
   id: string;
@@ -16,7 +16,7 @@ export abstract class AbstractReplicationJob {
   protected isReplicatingPromise: Promise<void> | null = null;
 
   protected constructor(protected options: AbstractReplicationJobOptions) {
-    this.logger = logger.child({ name: `ReplicationJob: ${options.id}` });
+    this.logger = logger.child({ name: `ReplicationJob: ${this.id}` });
   }
 
   /**
@@ -52,7 +52,7 @@ export abstract class AbstractReplicationJob {
    *  Safely stop the replication process
    */
   public async stop(): Promise<void> {
-    this.logger.info(`Stopping ${this.id} replication job for sync rule iteration: ${this.storage.group_id}`);
+    this.logger.info(`Stopping replication job for sync rule iteration: ${this.storage.group_id}`);
     this.abortController.abort();
     await this.isReplicatingPromise;
   }

package/src/replication/AbstractReplicator.ts

@@ -1,10 +1,10 @@
+import { container, logger } from '@powersync/lib-services-framework';
 import { hrtime } from 'node:process';
+import winston from 'winston';
 import * as storage from '../storage/storage-index.js';
-import { container, logger } from '@powersync/lib-services-framework';
+import { StorageEngine } from '../storage/storage-index.js';
 import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
-import winston from 'winston';
 import { AbstractReplicationJob } from './AbstractReplicationJob.js';
-import { StorageEngine } from '../storage/storage-index.js';
 import { ErrorRateLimiter } from './ErrorRateLimiter.js';
 
 // 5 minutes
@@ -192,6 +192,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
       try {
         await job.stop();
         await this.terminateSyncRules(job.storage);
+        job.storage[Symbol.dispose]();
       } catch (e) {
         // This will be retried
         this.logger.warn('Failed to terminate old replication job}', e);
@@ -202,7 +203,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     const stopped = await this.storage.getStoppedSyncRules();
     for (let syncRules of stopped) {
       try {
-        const syncRuleStorage = this.storage.getInstance(syncRules);
+        using syncRuleStorage = this.storage.getInstance(syncRules);
         await this.terminateSyncRules(syncRuleStorage);
       } catch (e) {
         this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);

package/src/replication/ReplicationEngine.ts

@@ -1,5 +1,5 @@
-import { AbstractReplicator } from './AbstractReplicator.js';
 import { logger } from '@powersync/lib-services-framework';
+import { AbstractReplicator } from './AbstractReplicator.js';
 
 export class ReplicationEngine {
   private readonly replicators: Map<string, AbstractReplicator> = new Map();

package/src/replication/ReplicationModule.ts

@@ -58,6 +58,10 @@ export abstract class ReplicationModule<TConfig extends DataSourceConfig> extend
       return;
     }
 
+    if (!matchingConfig.length) {
+      return;
+    }
+
     if (matchingConfig.length > 1) {
       this.logger.warning(
         `Multiple data sources of type ${this.type} found in the configuration. Only the first will be used.`

package/src/replication/replication-index.ts

@@ -1,5 +1,5 @@
-export * from './ErrorRateLimiter.js';
 export * from './AbstractReplicationJob.js';
 export * from './AbstractReplicator.js';
+export * from './ErrorRateLimiter.js';
 export * from './ReplicationEngine.js';
 export * from './ReplicationModule.js';

package/src/routes/configure-fastify.ts

@@ -8,7 +8,7 @@ import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
 import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
 import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
 import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
-import { RouteDefinition } from './router.js';
+import { RouteDefinition, RouterServiceContext } from './router.js';
 
 /**
  * A list of route definitions to be registered as endpoints.
@@ -56,6 +56,19 @@ export const DEFAULT_ROUTE_OPTIONS = {
  */
 export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
   const { service_context, routes = DEFAULT_ROUTE_OPTIONS } = options;
+
+  const generateContext = async () => {
+    const { routerEngine } = service_context;
+    if (!routerEngine) {
+      throw new Error(`RouterEngine has not been registered`);
+    }
+
+    return {
+      user_id: undefined,
+      service_context: service_context as RouterServiceContext
+    };
+  };
+
   /**
    * Fastify creates an encapsulated context for each `.register` call.
    * Creating a separate context here to separate the concurrency limits for Admin APIs
@@ -63,16 +76,7 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
    * https://github.com/fastify/fastify/blob/main/docs/Reference/Encapsulation.md
    */
   server.register(async function (childContext) {
-    registerFastifyRoutes(
-      childContext,
-      async () => {
-        return {
-          user_id: undefined,
-          service_context
-        };
-      },
-      routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
-    );
+    registerFastifyRoutes(childContext, generateContext, routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes);
     // Limit the active concurrent requests
     childContext.addHook(
       'onRequest',
@@ -84,12 +88,7 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
   server.register(async function (childContext) {
     registerFastifyRoutes(
       childContext,
-      async () => {
-        return {
-          user_id: undefined,
-          service_context
-        };
-      },
+      generateContext,
       routes.sync_stream?.routes ?? DEFAULT_ROUTE_OPTIONS.sync_stream.routes
     );
     // Limit the active concurrent requests

package/src/routes/configure-rsocket.ts

@@ -8,7 +8,7 @@ import { ServiceContext } from '../system/ServiceContext.js';
 import { generateContext, getTokenFromHeader } from './auth.js';
 import { syncStreamReactive } from './endpoints/socket-route.js';
 import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
-import { Context } from './router.js';
+import { Context, RouterServiceContext } from './router.js';
 
 export type RSockerRouterConfig = {
   service_context: ServiceContext;
@@ -36,12 +36,17 @@ export function configureRSocket(router: ReactiveSocketRouter<Context>, options:
           if (context?.token_payload == null) {
             throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
           }
+
+          if (!service_context.routerEngine) {
+            throw new Error(`RouterEngine has not been registered`);
+          }
+
           return {
             token,
             user_agent,
             ...context,
             token_errors: token_errors,
-            service_context
+            service_context: service_context as RouterServiceContext
           };
         } else {
           throw new errors.AuthorizationError('No token provided');

package/src/routes/endpoints/admin.ts

@@ -137,8 +137,8 @@ export const reprocess = routeDefinition({
       connections: [
         {
           // Previously the connection was asserted with `!`
-          tag: baseConfig!.tag!,
-          id: baseConfig!.id,
+          tag: baseConfig.tag,
+          id: baseConfig.id,
           slot_name: new_rules.slot_name
         }
       ]

package/src/routes/endpoints/checkpointing.ts

@@ -1,5 +1,5 @@
-import * as t from 'ts-codec';
 import { logger, router, schema } from '@powersync/lib-services-framework';
+import * as t from 'ts-codec';
 
 import * as util from '../../util/util-index.js';
 import { authUser } from '../auth.js';
@@ -63,7 +63,10 @@ export const writeCheckpoint2 = routeDefinition({
       storageEngine: { activeBucketStorage }
     } = service_context;
 
-    const writeCheckpoint = await activeBucketStorage.createWriteCheckpoint(full_user_id, { '1': currentCheckpoint });
+    const writeCheckpoint = await activeBucketStorage.createManagedWriteCheckpoint({
+      user_id: full_user_id,
+      heads: { '1': currentCheckpoint }
+    });
     logger.info(`Write checkpoint 2: ${JSON.stringify({ currentCheckpoint, id: String(full_user_id) })}`);
 
     return {

package/src/routes/endpoints/sync-rules.ts

@@ -3,6 +3,7 @@ import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules';
 import type { FastifyPluginAsync } from 'fastify';
 import * as t from 'ts-codec';
 
+import * as system from '../../system/system-index.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
 import { RouteAPI } from '../../api/RouteAPI.js';

package/src/routes/router.ts

@@ -1,13 +1,19 @@
 import { router } from '@powersync/lib-services-framework';
 import * as auth from '../auth/auth-index.js';
 import { ServiceContext } from '../system/ServiceContext.js';
+import { RouterEngine } from './RouterEngine.js';
 
+/**
+ * The {@link RouterEngine} must be provided for these routes
+ */
+export type RouterServiceContext = ServiceContext & { routerEngine: RouterEngine };
 /**
  * Common context for routes
  */
 export type Context = {
   user_id?: string;
-  service_context: ServiceContext;
+
+  service_context: RouterServiceContext;
 
   token_payload?: auth.JwtPayload;
   token_errors?: string[];

package/src/runner/teardown.ts

@@ -4,11 +4,11 @@
 // 2. Delete the storage
 
 import { container, logger } from '@powersync/lib-services-framework';
+import timers from 'timers/promises';
 import * as modules from '../modules/modules-index.js';
-import * as system from '../system/system-index.js';
 import * as storage from '../storage/storage-index.js';
+import * as system from '../system/system-index.js';
 import * as utils from '../util/util-index.js';
-import timers from 'timers/promises';
 
 export async function teardown(runnerConfig: utils.RunnerConfig) {
   try {
@@ -51,7 +51,7 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
 
       // Mark the sync rules as terminated
       for (let syncRules of combinedSyncRules) {
-        const syncRulesStorage = storageFactory.getInstance(syncRules);
+        using syncRulesStorage = storageFactory.getInstance(syncRules);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here
         await syncRulesStorage.terminate({ clearStorage: false });
       }

package/src/storage/BucketStorage.ts

@@ -1,3 +1,4 @@
+import { DisposableListener, DisposableObserverClient } from '@powersync/lib-services-framework';
 import {
   EvaluatedParameters,
   EvaluatedRow,
@@ -8,11 +9,19 @@ import {
   ToastableSqliteRow
 } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
-import { SourceTable } from './SourceTable.js';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
-import { ReplicaId } from './storage-index.js';
+import { SourceTable } from './SourceTable.js';
+import { BatchedCustomWriteCheckpointOptions, ReplicaId, WriteCheckpointAPI } from './storage-index.js';
+
+export interface BucketStorageFactoryListener extends DisposableListener {
+  syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
+  replicationEvent: (event: ReplicationEventPayload) => void;
+}
 
-export interface BucketStorageFactory {
+export interface BucketStorageFactory
+  extends DisposableObserverClient<BucketStorageFactoryListener>,
+    WriteCheckpointAPI {
   /**
    * Update sync rules from configuration, if changed.
    */
@@ -81,10 +90,9 @@ export interface BucketStorageFactory {
    */
   getActiveCheckpoint(): Promise<ActiveCheckpoint>;
 
-  createWriteCheckpoint(user_id: string, lsns: Record<string, string>): Promise<bigint>;
-
-  lastWriteCheckpoint(user_id: string, lsn: string): Promise<bigint | null>;
-
+  /**
+   * Yields the latest user write checkpoint whenever the sync checkpoint updates.
+   */
   watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable<WriteCheckpoint>;
 
   /**
@@ -98,20 +106,22 @@ export interface BucketStorageFactory {
   getPowerSyncInstanceId(): Promise<string>;
 }
 
-export interface WriteCheckpoint {
-  base: ActiveCheckpoint;
-  writeCheckpoint: bigint | null;
-}
-
-export interface ActiveCheckpoint {
+export interface Checkpoint {
   readonly checkpoint: util.OpId;
   readonly lsn: string | null;
+}
 
+export interface ActiveCheckpoint extends Checkpoint {
   hasSyncRules(): boolean;
 
   getBucketStorage(): Promise<SyncRulesBucketStorage | null>;
 }
 
+export interface WriteCheckpoint {
+  base: ActiveCheckpoint;
+  writeCheckpoint: bigint | null;
+}
+
 export interface StorageMetrics {
   /**
    * Size of operations (bucket_data)
@@ -194,7 +204,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
   zeroLSN: string;
 }
 
-export interface SyncRulesBucketStorage {
+export interface SyncRulesBucketStorageListener extends DisposableListener {
+  batchStarted: (batch: BucketStorageBatch) => void;
+}
+
+export interface SyncRulesBucketStorage extends DisposableObserverClient<SyncRulesBucketStorageListener> {
   readonly group_id: number;
   readonly slot_name: string;
 
@@ -207,7 +221,7 @@ export interface SyncRulesBucketStorage {
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
-  getCheckpoint(): Promise<{ checkpoint: util.OpId }>;
+  getCheckpoint(): Promise<Checkpoint>;
 
   getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules;
 
@@ -293,7 +307,11 @@ export interface FlushedResult {
   flushed_op: string;
 }
 
-export interface BucketStorageBatch {
+export interface BucketBatchStorageListener extends DisposableListener {
+  replicationEvent: (payload: ReplicationEventPayload) => void;
+}
+
+export interface BucketStorageBatch extends DisposableObserverClient<BucketBatchStorageListener> {
   /**
    * Save an op, and potentially flush.
    *
@@ -345,6 +363,11 @@ export interface BucketStorageBatch {
   lastCheckpointLsn: string | null;
 
   markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
+
+  /**
+   * Queues the creation of a custom Write Checkpoint. This will be persisted after operations are flushed.
+   */
+  addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
 }
 
 export interface SaveParameterData {
@@ -362,10 +385,18 @@ export interface SaveBucketData {
   evaluated: EvaluatedRow[];
 }
 
+export type SaveOp = 'insert' | 'update' | 'delete';
+
 export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete;
 
+export enum SaveOperationTag {
+  INSERT = 'insert',
+  UPDATE = 'update',
+  DELETE = 'delete'
+}
+
 export interface SaveInsert {
-  tag: 'insert';
+  tag: SaveOperationTag.INSERT;
   sourceTable: SourceTable;
   before?: undefined;
   beforeReplicaId?: undefined;
@@ -374,7 +405,7 @@ export interface SaveInsert {
 }
 
 export interface SaveUpdate {
-  tag: 'update';
+  tag: SaveOperationTag.UPDATE;
   sourceTable: SourceTable;
 
   /**
@@ -393,7 +424,7 @@ export interface SaveUpdate {
 }
 
 export interface SaveDelete {
-  tag: 'delete';
+  tag: SaveOperationTag.DELETE;
   sourceTable: SourceTable;
   before?: SqliteRow;
   beforeReplicaId: ReplicaId;