@powersync/service-core 1.21.0 → 1.23.0

package/package.json

@@ -1,14 +1,13 @@
 {
   "name": "@powersync/service-core",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "types": "dist/index.d.ts",
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.21.0",
-  "main": "dist/index.js",
+  "version": "1.23.0",
   "license": "FSL-1.1-ALv2",
   "type": "module",
+  "exports": "./dist/index.js",
   "dependencies": {
     "@js-sdsl/ordered-set": "^4.4.2",
     "@opentelemetry/api": "^1.9.0",
@@ -33,11 +32,11 @@
     "uuid": "^14.0.0",
     "winston": "^3.13.0",
     "yaml": "^2.8.3",
-    "@powersync/lib-services-framework": "0.9.4",
+    "@powersync/lib-services-framework": "0.9.6",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-rsocket-router": "0.2.20",
-    "@powersync/service-sync-rules": "0.36.0",
-    "@powersync/service-types": "0.15.2"
+    "@powersync/service-rsocket-router": "0.2.22",
+    "@powersync/service-sync-rules": "0.38.0",
+    "@powersync/service-types": "0.16.0"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",

package/src/api/RouteAPI.ts

@@ -1,6 +1,6 @@
 import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
 import * as types from '@powersync/service-types';
-import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
+import { ParseSyncConfigOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
 
 export interface PatternResult {
   schema: string;
@@ -94,7 +94,7 @@ export interface RouteAPI {
   /**
    * Get the default schema (or database) when only a table name is specified in sync config.
    */
-  getParseSyncRulesOptions(): ParseSyncRulesOptions;
+  getParseSyncRulesOptions(): ParseSyncConfigOptions;
 }
 
 export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;

package/src/api/diagnostics.ts

@@ -1,5 +1,5 @@
-import { logger } from '@powersync/lib-services-framework';
-import { DEFAULT_TAG, SourceTableInterface, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { DEFAULT_TAG, SourceTableRef, SyncConfigWithErrors } from '@powersync/service-sync-rules';
 import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types';
 
 import * as storage from '../storage/storage-index.js';
@@ -19,6 +19,11 @@ export interface DiagnosticsOptions {
    */
   live_status: boolean;
 
+  /**
+   * Whether this is the active sync config or not.
+   */
+  active: boolean;
+
   /**
    * Check against the source postgres connection.
    */
@@ -28,12 +33,18 @@ export interface DiagnosticsOptions {
 export const DEFAULT_DATASOURCE_ID = 'default';
 
 export async function getSyncRulesStatus(
-  bucketStorage: storage.BucketStorageFactory,
   apiHandler: RouteAPI,
-  sync_rules: storage.PersistedSyncRulesContent | null,
-  options: DiagnosticsOptions
+  syncConfig: storage.PersistedSyncConfigContent | null,
+  options: DiagnosticsOptions,
+  /**
+   * Storage instance for the replication stream of this config.
+   *
+   * Required to populate live status (snapshot/checkpoint info). The content object
+   * itself is no longer a replication stream, so the caller must resolve this.
+   */
+  systemStorage?: storage.SyncRulesBucketStorage
 ): Promise<SyncRulesStatus | undefined> {
-  if (sync_rules == null) {
+  if (syncConfig == null) {
     return undefined;
   }
 
@@ -43,13 +54,18 @@ export async function getSyncRulesStatus(
   const now = new Date().toISOString();
 
   let parsed: SyncConfigWithErrors;
-  let persisted: storage.PersistedSyncRules;
+  let persisted: storage.ParsedSyncConfigSet;
   try {
-    persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
-    parsed = persisted.sync_rules;
+    persisted = syncConfig.parsed(apiHandler.getParseSyncRulesOptions());
+    // A content object represents a single sync config, so its parsed result has exactly one entry.
+    const [singleConfig] = persisted.syncConfigs;
+    if (singleConfig == null) {
+      throw new ServiceAssertionError('Expected one sync config');
+    }
+    parsed = singleConfig;
   } catch (e) {
     return {
-      content: include_content ? sync_rules.sync_rules_content : undefined,
+      content: include_content ? syncConfig.sync_rules_content : undefined,
       connections: [],
       errors: [{ level: 'fatal', message: e.message, ts: now }]
     };
@@ -60,8 +76,7 @@ export async function getSyncRulesStatus(
   // This method can run under some situations if no connection is configured yet.
   // It will return a default tag in such a case. This default tag is not module specific.
   const tag = sourceConfig.tag ?? DEFAULT_TAG;
-  const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
-  const status = await systemStorage?.getStatus();
+  const status = live_status ? await systemStorage?.getStatus() : undefined;
   let replication_lag_bytes: number | undefined = undefined;
   let slot_wal_budget: SlotWalBudgetInfo | undefined = undefined;
 
@@ -90,10 +105,10 @@ export async function getSyncRulesStatus(
         logger.warn(`Unable to get replication lag`, e);
       }
 
-      if (apiHandler.getSlotWalBudget && sync_rules.slot_name) {
+      if (apiHandler.getSlotWalBudget && syncConfig.replicationStreamName) {
         try {
           slot_wal_budget = await apiHandler.getSlotWalBudget({
-            slotName: sync_rules.slot_name
+            slotName: syncConfig.replicationStreamName
           });
         } catch (e) {
           logger.warn(`Unable to get WAL budget`, e);
@@ -116,7 +131,7 @@ export async function getSyncRulesStatus(
           errors: [{ level: 'fatal', message: 'connection failed', ts: now }]
         };
       } else {
-        const source: SourceTableInterface = {
+        const source: SourceTableRef = {
           connectionTag: tag,
           schema: pattern.schema,
           name: pattern.tablePattern
@@ -136,11 +151,13 @@ export async function getSyncRulesStatus(
   }
 
   const errors = tables_flat.flatMap((info) => info.errors);
-  if (sync_rules.last_fatal_error) {
+  const statusSource = await syncConfig.getSyncConfigStatus();
+
+  if (statusSource?.last_fatal_error) {
     errors.push({
       level: 'fatal',
-      message: sync_rules.last_fatal_error,
-      ts: sync_rules.last_fatal_error_ts?.toISOString()
+      message: statusSource.last_fatal_error,
+      ts: statusSource.last_fatal_error_ts?.toISOString()
     });
   }
   errors.push(...syncRuleErrors.map((error) => syncConfigYamlErrorToReplicationError(error, now)));
@@ -168,10 +185,10 @@ export async function getSyncRulesStatus(
     }
   }
 
-  if (live_status && status?.active) {
+  if (live_status && options.active) {
     // Check replication lag for active replication stream.
     // Right now we exclude mysql, since it we don't have consistent keepalives for it.
-    if (sync_rules.last_checkpoint_ts == null && sync_rules.last_keepalive_ts == null) {
+    if (statusSource?.last_checkpoint_ts == null && statusSource?.last_keepalive_ts == null) {
       errors.push({
         level: 'warning',
         message: 'No checkpoint found, cannot calculate replication lag',
@@ -179,8 +196,8 @@ export async function getSyncRulesStatus(
       });
     } else {
       const lastTime = Math.max(
-        sync_rules.last_checkpoint_ts?.getTime() ?? 0,
-        sync_rules.last_keepalive_ts?.getTime() ?? 0
+        statusSource.last_checkpoint_ts?.getTime() ?? 0,
+        statusSource.last_keepalive_ts?.getTime() ?? 0
       );
       const lagSeconds = Math.round((Date.now() - lastTime) / 1000);
       // On idle instances, keepalive messages are only persisted every 60 seconds.
@@ -204,17 +221,17 @@ export async function getSyncRulesStatus(
   }
 
   return {
-    content: include_content ? sync_rules.sync_rules_content : undefined,
+    content: include_content ? syncConfig.sync_rules_content : undefined,
     connections: [
       {
         id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
         tag: tag,
-        slot_name: sync_rules.slot_name,
-        initial_replication_done: status?.snapshot_done ?? false,
+        slot_name: syncConfig.replicationStreamName,
+        initial_replication_done: status?.snapshotDone ?? false,
         // TODO: Rename?
-        last_lsn: status?.checkpoint_lsn ?? undefined,
-        last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
-        last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
+        last_lsn: status?.resumeLsn ?? undefined,
+        last_checkpoint_ts: statusSource?.last_checkpoint_ts?.toISOString(),
+        last_keepalive_ts: statusSource?.last_keepalive_ts?.toISOString(),
         replication_lag_bytes: replication_lag_bytes,
         wal_status: slot_wal_budget?.wal_status,
         safe_wal_size: slot_wal_budget?.safe_wal_size,

package/src/auth/RemoteJWKSCollector.ts

@@ -5,6 +5,7 @@ import fetch from 'node-fetch';
 import {
   AuthorizationError,
   ErrorCode,
+  hostnameFromSocketAddress,
   LookupOptions,
   makeHostnameLookupFunction,
   ServiceAssertionError,
@@ -140,7 +141,8 @@ export class RemoteJWKSCollector implements KeyCollector {
    */
   resolveAgent(): http.Agent | https.Agent {
     const lookupOptions = this.options?.lookupOptions ?? { reject_ip_ranges: [] };
-    const lookup = makeHostnameLookupFunction(this.url.hostname, lookupOptions);
+    const hostname = hostnameFromSocketAddress(this.url.hostname);
+    const lookup = makeHostnameLookupFunction(hostname, lookupOptions);
 
     const options: http.AgentOptions = {
       lookup

package/src/entry/commands/compact-action.ts

@@ -83,7 +83,7 @@ export function registerCompactAction(program: Command) {
       await serviceContext.lifeCycleEngine.start();
       const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
 
-      const active = await bucketStorage.getActiveStorage();
+      const active = (await bucketStorage.getActiveSyncConfig())?.storage;
       if (active == null) {
         logger.info('No active instance to compact');
         return;

package/src/modules/AbstractModule.ts

@@ -1,13 +1,17 @@
 import { logger } from '@powersync/lib-services-framework';
 import winston from 'winston';
-import { PersistedSyncRulesContent } from '../storage/storage-index.js';
+import { PersistedReplicationStream } from '../storage/storage-index.js';
 import { ServiceContextContainer } from '../system/ServiceContext.js';
 
 export interface TearDownOptions {
   /**
    *  If required, tear down any configuration/state for the specific replication stream
    */
-  syncRules?: PersistedSyncRulesContent[];
+  replicationStreams?: PersistedReplicationStream[];
+  /**
+   * @deprecated Use replicationStreams.
+   */
+  syncRules?: PersistedReplicationStream[];
 }
 
 export interface AbstractModuleOptions {

package/src/replication/AbstractReplicator.ts

@@ -38,10 +38,10 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
   protected logger: winston.Logger;
   private lockAlerted: boolean = false;
   /**
-   *  Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
-   *  transitioning to a new replication stream.
+   *  Map of replication jobs by replication stream job id. Usually there is only one running job, but there could be two
+   *  when transitioning to a new replication stream.
    */
-  private replicationJobs = new Map<number, T>();
+  private replicationJobs = new Map<string, T>();
 
   /**
    * Map of replciation stream ids to promises that are clearing the replication stream.
@@ -188,48 +188,49 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
 
     let configuredLock = options?.configured_lock;
 
-    const existingJobs = new Map<number, T>(this.replicationJobs.entries());
-    const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
-    const newJobs = new Map<number, T>();
+    const existingJobs = new Map<string, T>(this.replicationJobs.entries());
+    const replicatingStreams = await this.storage.getReplicatingReplicationStreams();
+    const newJobs = new Map<string, T>();
     let activeJob: T | undefined = undefined;
-    for (let syncRules of replicatingSyncRules) {
-      const existingJob = existingJobs.get(syncRules.id);
-      if (syncRules.active && activeJob == null) {
+    for (let replicationStream of replicatingStreams) {
+      const jobId = replicationStream.replicationJobId;
+      const existingJob = existingJobs.get(jobId);
+      if (replicationStream.state == storage.SyncRuleState.ACTIVE && activeJob == null) {
         activeJob = existingJob;
       }
       if (existingJob && !existingJob.isStopped) {
         // No change
-        existingJobs.delete(syncRules.id);
-        newJobs.set(syncRules.id, existingJob);
+        existingJobs.delete(jobId);
+        newJobs.set(jobId, existingJob);
       } else if (existingJob && existingJob.isStopped) {
         // Stopped (e.g. fatal error).
         // Remove from the list. Next refresh call will restart the job.
-        existingJobs.delete(syncRules.id);
+        existingJobs.delete(jobId);
       } else {
         // New sync config was found (or resume after restart)
         try {
           let lock: storage.ReplicationLock;
-          if (configuredLock?.sync_rules_id == syncRules.id) {
+          if (configuredLock?.sync_rules_id == replicationStream.replicationStreamId) {
             lock = configuredLock;
           } else {
-            lock = await syncRules.lock();
+            lock = await replicationStream.lock();
           }
-          const storage = this.storage.getInstance(syncRules);
+          const syncRuleStorage = this.storage.getInstance(replicationStream);
           const newJob = this.createJob({
             lock: lock,
-            storage: storage
+            storage: syncRuleStorage
           });
 
-          newJobs.set(syncRules.id, newJob);
+          newJobs.set(jobId, newJob);
           newJob.start();
-          if (syncRules.active) {
+          if (replicationStream.state == storage.SyncRuleState.ACTIVE) {
             activeJob = newJob;
           }
           this.lockAlerted = false;
         } catch (e) {
           if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
             if (!this.lockAlerted) {
-              syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
+              replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
               this.lockAlerted = true;
             }
           } else {
@@ -237,7 +238,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
             // for example from stricter validation that was added.
             // This will be retried every couple of seconds.
             // When new (valid) sync config is deployed and processed, this one be disabled.
-            syncRules.logger.error('Failed to start replication for new sync config', e);
+            replicationStream.logger.error('Failed to start replication for new sync config', e);
           }
         }
       }
@@ -259,9 +260,9 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
     }
 
     // Replication stream stopped previously, including by a different process.
-    const stopped = await this.storage.getStoppedSyncRules();
-    for (let syncRules of stopped) {
-      if (this.clearingJobs.has(syncRules.id)) {
+    const stopped = await this.storage.getStoppedReplicationStreams();
+    for (let replicationStream of stopped) {
+      if (this.clearingJobs.has(replicationStream.replicationStreamId)) {
         // Already in progress
         continue;
       }
@@ -269,15 +270,15 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
       // We clear storage asynchronously.
       // It is important to be able to continue running the refresh loop, otherwise we cannot
       // retry locked replication stream, for example.
-      const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
+      const syncRuleStorage = this.storage.getInstance(replicationStream, { skipLifecycleHooks: true });
       const promise = this.terminateSyncRules(syncRuleStorage)
         .catch((e) => {
           syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
         })
         .finally(() => {
-          this.clearingJobs.delete(syncRules.id);
+          this.clearingJobs.delete(replicationStream.replicationStreamId);
         });
-      this.clearingJobs.set(syncRules.id, promise);
+      this.clearingJobs.set(replicationStream.replicationStreamId, promise);
     }
   }
 

package/src/replication/RelationCache.ts

@@ -1,24 +1,43 @@
 import { SourceTable } from '../storage/SourceTable.js';
 
 export class RelationCache<T> {
-  private cache = new Map<string | number, SourceTable>();
+  private cache = new Map<string | number, SourceTable[]>();
   private idFunction: (item: T | SourceTable) => string | number;
 
   constructor(idFunction: (item: T | SourceTable) => string | number) {
     this.idFunction = idFunction;
   }
 
+  /**
+   * Update a single table in-place - use when the snapshot status of the table has changed.
+   */
   update(table: SourceTable) {
     const id = this.idFunction(table);
-    this.cache.set(id, table);
+    const existing = this.cache.get(id) ?? [];
+    const replacementIndex = existing.findIndex((candidate) => candidate.id == table.id);
+    if (replacementIndex == -1) {
+      this.cache.set(id, [...existing, table]);
+    } else {
+      const next = [...existing];
+      next[replacementIndex] = table;
+      this.cache.set(id, next);
+    }
   }
 
-  get(source: T | SourceTable): SourceTable | undefined {
+  /**
+   * Set the full set of tables for a specific reference.
+   */
+  updateAll(ref: T, tables: SourceTable[]) {
+    const id = this.idFunction(ref);
+    this.cache.set(id, tables);
+  }
+
+  getAll(source: T | SourceTable): SourceTable[] | undefined {
     const id = this.idFunction(source);
     return this.cache.get(id);
   }
 
-  delete(source: T | SourceTable): boolean {
+  delete(source: T): boolean {
     const id = this.idFunction(source);
     return this.cache.delete(id);
   }

package/src/replication/ReplicationModule.ts

@@ -1,4 +1,3 @@
-import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
 import * as t from 'ts-codec';
 
 import { schema } from '@powersync/lib-services-framework';
@@ -18,7 +17,7 @@ export interface ConnectionTestResult {
 /**
  *  Provides a common interface for testing the connection to a DataSource.
  */
-export interface ConnectionTester<TConfig extends DataSourceConfig> {
+export interface ConnectionTester<TConfig extends types.configFile.DataSourceConfig> {
   /**
    *  Confirm if a connection can be established to the datasource for the provided datasource configuration
    *  @param config
@@ -35,7 +34,7 @@ export interface ReplicationModuleOptions extends modules.AbstractModuleOptions
  *  A replication module describes all the functionality that PowerSync requires to
  *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
  */
-export abstract class ReplicationModule<TConfig extends DataSourceConfig>
+export abstract class ReplicationModule<TConfig extends types.configFile.DataSourceConfig>
   extends modules.AbstractModule
   implements ConnectionTester<TConfig>
 {

package/src/routes/configure-fastify.ts

@@ -1,6 +1,10 @@
 import type fastify from 'fastify';
 
-import { registerFastifyNotFoundHandler, registerFastifyRoutes } from './route-register.js';
+import {
+  registerFastifyErrorHandler,
+  registerFastifyNotFoundHandler,
+  registerFastifyRoutes
+} from './route-register.js';
 
 import * as system from '../system/system-index.js';
 
@@ -66,6 +70,9 @@ export function configureFastifyServer(server: fastify.FastifyInstance, options:
     };
   };
 
+  // Set on the outer server so both child scopes inherit.
+  registerFastifyErrorHandler(server);
+
   /**
    * Fastify creates an encapsulated context for each `.register` call.
    * Creating a separate context here to separate the concurrency limits for Admin APIs