@powersync/service-core 1.20.4 → 1.21.0

package/src/routes/endpoints/sync-rules.ts

@@ -4,9 +4,9 @@ import type { FastifyPluginAsync } from 'fastify';
 import * as t from 'ts-codec';
 
 import { RouteAPI } from '../../api/RouteAPI.js';
+import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
 
 const DeploySyncRulesRequest = t.object({
   content: t.string
@@ -43,12 +43,12 @@ export const deploySyncRules = routeDefinition({
     const { storageEngine } = service_context;
 
     if (service_context.configuration.sync_rules.present) {
-      // If sync rules are configured via the config, disable deploy via the API.
+      // If sync config is configured via the service config, disable deploy via the API.
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4105,
-        description: 'Sync rules API disabled',
-        details: 'Use the management API to deploy sync rules'
+        description: 'Sync config API disabled',
+        details: 'Update sync config in the service configuration'
       });
     }
     const content = payload.params.content;
@@ -56,7 +56,7 @@ export const deploySyncRules = routeDefinition({
 
     try {
       const apiHandler = service_context.routerEngine.getAPI();
-      syncConfig = SqlSyncRules.fromYaml(payload.params.content, {
+      syncConfig = SqlSyncRules.fromYaml(content, {
         ...apiHandler.getParseSyncRulesOptions(),
         // We don't do any schema-level validation at this point
         schema: undefined
@@ -65,14 +65,12 @@ export const deploySyncRules = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_R0001,
-        description: 'Sync rules parsing failed',
+        description: 'Sync config parsing failed',
         details: e.message
       });
     }
 
-    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(
-      updateSyncRulesFromConfig(syncConfig.config)
-    );
+    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
 
     return {
       slot_name: sync_rules.slot_name
@@ -117,7 +115,7 @@ export const currentSyncRules = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
-        description: 'No active sync rules'
+        description: 'No active sync config'
       });
     }
 
@@ -164,13 +162,13 @@ export const reprocessSyncRules = routeDefinition({
       throw new errors.ServiceError({
         status: 422,
         code: ErrorCode.PSYNC_S4104,
-        description: 'No active sync rules'
+        description: 'No active sync config'
       });
     }
 
     const new_rules = await activeBucketStorage.updateSyncRules(
       updateSyncRulesFromYaml(sync_rules.sync_rules.config.content, {
-        // These sync rules already passed validation. But if the rules are not valid anymore due
+        // This sync config already passed validation. But if the rules are not valid anymore due
         // to a service change, we do want to report the error here.
         validate: true
       })

package/src/routes/endpoints/sync-stream.ts

@@ -68,7 +68,7 @@ export const syncStreamed = routeDefinition({
       throw new errors.ServiceError({
         status: 500,
         code: ErrorCode.PSYNC_S2302,
-        description: 'No sync rules available'
+        description: 'No sync config available'
       });
     }
 
@@ -121,7 +121,7 @@ export const syncStreamed = routeDefinition({
       });
 
       stream.on('end', () => {
-        // Auth failure or switch to new sync rules
+        // Auth failure or switch to new sync config
         closeReason ??= 'service closing stream';
       });
 

package/src/routes/hooks.ts

@@ -1,6 +1,6 @@
-import type fastify from 'fastify';
-import a from 'async';
 import { logger } from '@powersync/lib-services-framework';
+import a from 'async';
+import type fastify from 'fastify';
 
 export type CreateRequestQueueParams = {
   max_queue_depth: number;

package/src/routes/route-register.ts

@@ -1,17 +1,9 @@
 import type fastify from 'fastify';
 import * as uuid from 'uuid';
 
-import {
-  ErrorCode,
-  errors,
-  HTTPMethod,
-  logger,
-  RouteNotFound,
-  router,
-  ServiceError
-} from '@powersync/lib-services-framework';
-import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
+import { errors, HTTPMethod, logger, RouteNotFound, router, ServiceError } from '@powersync/lib-services-framework';
 import { FastifyReply } from 'fastify';
+import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {
   parse?: boolean;

package/src/runner/teardown.ts

@@ -35,13 +35,13 @@ export async function teardown(runnerConfig: utils.RunnerConfig) {
 }
 
 async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, moduleManager: modules.ModuleManager) {
-  logger.info(`Terminating sync rules...`);
+  logger.info(`Terminating replication stream...`);
   const start = Date.now();
   const locks: storage.ReplicationLock[] = [];
   while (Date.now() - start < 120_000) {
     let retry = false;
     const replicatingSyncRules = await storageFactory.getReplicatingSyncRules();
-    // Lock all the replicating sync rules
+    // Lock all the replicating replication streams
     for (const replicatingSyncRule of replicatingSyncRules) {
       const lock = await replicatingSyncRule.lock();
       locks.push(lock);
@@ -50,10 +50,10 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory,
     const stoppedSyncRules = await storageFactory.getStoppedSyncRules();
     const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules];
     try {
-      // Clean up any module specific configuration for the sync rules
+      // Clean up any module specific configuration for the replication stream
       await moduleManager.tearDown({ syncRules: combinedSyncRules });
 
-      // Mark the sync rules as terminated
+      // Mark the replication stream as terminated
       for (let syncRules of combinedSyncRules) {
         const syncRulesStorage = storageFactory.getInstance(syncRules);
         // The storage will be dropped at the end of the teardown, so we don't need to clear it here

package/src/storage/BucketStorage.ts

@@ -2,36 +2,36 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules';
 
 export enum SyncRuleState {
   /**
-   * New sync rules - needs to be processed (initial replication).
+   * New replication stream - needs to be processed (initial replication).
    *
-   * While multiple sets of sync rules _can_ be in PROCESSING,
+   * While multiple replication streams _can_ be in PROCESSING,
    * it's generally pointless, so we only keep one in that state.
    */
   PROCESSING = 'PROCESSING',
 
   /**
-   * Sync rule processing is done, and can be used for sync.
+   * Intial processing is done, and can be used for sync.
    *
-   * Only one set of sync rules should be in ACTIVE or ERRORED state.
+   * Only one replication stream should be in ACTIVE or ERRORED state.
    */
   ACTIVE = 'ACTIVE',
   /**
-   * This state is used when the sync rules has been replaced,
+   * This state is used when the replication stream has been replaced,
    * and replication is or should be stopped.
    */
   STOP = 'STOP',
   /**
-   * After sync rules have been stopped, the data needs to be
+   * After replication stream has been stopped, the data needs to be
    * deleted. Once deleted, the state is TERMINATED.
    */
   TERMINATED = 'TERMINATED',
 
   /**
-   * Sync rules has run into a permanent replication error. It
-   * is still the "active" sync rules for syncing to users,
+   * Replication stream has run into a permanent replication error. It
+   * is still the "active" replication stram for syncing to users,
    * but should not replicate anymore.
    *
-   * It will transition to STOP when a new sync rules is activated.
+   * It will transition to STOP when a new replication stream is activated.
    */
   ERRORED = 'ERRORED'
 }

package/src/storage/BucketStorageBatch.ts

@@ -1,10 +1,10 @@
 import { ObserverClient } from '@powersync/lib-services-framework';
 import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
 import { BSON } from 'bson';
+import { InternalOpId } from '../util/utils.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
-import { InternalOpId } from '../util/utils.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
   createEmptyCheckpoints: true,

package/src/storage/BucketStorageFactory.ts

@@ -1,31 +1,33 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
-import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
-import { ReplicationEventPayload } from './ReplicationEventPayload.js';
-import { ReplicationLock } from './ReplicationLock.js';
-import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
-import { ReportStorage } from './ReportStorage.js';
 import {
   PrecompiledSyncConfig,
   SerializedCompatibilityContext,
   serializeSyncPlan,
   SqlSyncRules,
-  SyncConfig
+  SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
+import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { ReportStorage } from './ReportStorage.js';
+import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
 
 /**
  * Represents a configured storage provider.
  *
- * The provider can handle multiple copies of sync rules concurrently, each with their own storage.
- * This is to handle replication of a new version of sync rules, while the old version is still active.
+ * The provider can handle multiple replication streams concurrently, each with their own storage.
+ * This is to handle replication of a new version of sync config, while the old replication stream is still active.
  *
- * Storage APIs for a specific copy of sync rules are provided by the `SyncRulesBucketStorage` instances.
+ * Storage APIs for a specific replication stream are provided by the `SyncRulesBucketStorage` instances.
  */
 export abstract class BucketStorageFactory
   extends BaseObserver<BucketStorageFactoryListener>
   implements AsyncDisposable
 {
   /**
-   * Update sync rules from configuration, if changed.
+   * Update sync config from configuration, if changed.
    */
   async configureSyncRules(
     options: UpdateSyncRulesOptions
@@ -34,42 +36,42 @@ export abstract class BucketStorageFactory
     const active = await this.getActiveSyncRulesContent();
 
     if (next?.sync_rules_content == options.config.yaml) {
-      logger.info('Sync rules from configuration unchanged');
+      logger.info('Sync config unchanged');
       return { updated: false };
     } else if (next == null && active?.sync_rules_content == options.config.yaml) {
-      logger.info('Sync rules from configuration unchanged');
+      logger.info('Sync config unchanged');
       return { updated: false };
     } else {
-      logger.info('Sync rules updated from configuration');
+      logger.info('Sync config updated');
       const persisted_sync_rules = await this.updateSyncRules(options);
       return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
     }
   }
 
   /**
-   * Get a storage instance to query sync data for specific sync rules.
+   * Get a storage instance to query sync data for specific sync config.
    */
   abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage;
 
   /**
-   * Deploy new sync rules.
+   * Deploy new sync config.
    */
   abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedSyncRulesContent>;
 
   /**
    * Indicate that a slot was removed, and we should re-sync by creating
-   * a new sync rules instance.
+   * a new replication stream.
    *
    * This is roughly the same as deploying a new version of the current sync
-   * rules, but also accounts for cases where the current sync rules are not
-   * the latest ones.
+   * config, but also accounts for cases where the current sync config is not
+   * the latest one.
    *
    * Replication should be restarted after this.
    */
   abstract restartReplication(sync_rules_group_id: number): Promise<void>;
 
   /**
-   * Get the sync rules used for querying.
+   * Get the sync config used for querying.
    */
   async getActiveSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getActiveSyncRulesContent();
@@ -77,12 +79,12 @@ export abstract class BucketStorageFactory
   }
 
   /**
-   * Get the sync rules used for querying.
+   * Get the sync config used for querying.
    */
   abstract getActiveSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
-   * Get the sync rules that will be active next once done with initial replicatino.
+   * Get the sync config that will be active next once done with initial replicatino.
    */
   async getNextSyncRules(options: ParseSyncRulesOptions): Promise<PersistedSyncRules | null> {
     const content = await this.getNextSyncRulesContent();
@@ -90,17 +92,17 @@ export abstract class BucketStorageFactory
   }
 
   /**
-   * Get the sync rules that will be active next once done with initial replicatino.
+   * Get the sync config that will be active next once done with initial replicatino.
    */
   abstract getNextSyncRulesContent(): Promise<PersistedSyncRulesContent | null>;
 
   /**
-   * Get all sync rules currently replicating. Typically this is the "active" and "next" sync rules.
+   * Get all sync config currently replicating. Typically this is the "active" and "next" sync config.
    */
   abstract getReplicatingSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
   /**
-   * Get all sync rules stopped but not terminated yet.
+   * Get all sync config stopped but not terminated yet.
    */
   abstract getStoppedSyncRules(): Promise<PersistedSyncRulesContent[]>;
 
@@ -110,12 +112,12 @@ export abstract class BucketStorageFactory
   abstract getActiveStorage(): Promise<SyncRulesBucketStorage | null>;
 
   /**
-   * Get storage size of active sync rules.
+   * Get storage size of active replication stream.
    */
   abstract getStorageMetrics(): Promise<StorageMetrics>;
 
   /**
-   * Get the unique identifier for this instance of Powersync
+   * Get the unique identifier for this instance of Powersync.
    */
   abstract getPowerSyncInstanceId(): Promise<string>;
 
@@ -159,6 +161,12 @@ export interface UpdateSyncRulesOptions {
      * compiler.
      */
     plan: SerializedSyncPlan | null;
+
+    /**
+     * Parsed sync config, primarily to generate a definition mapping.
+     * Not persisted, and the defaultSchema used for parsing is not relevant.
+     */
+    parsed: SyncConfigWithErrors;
   };
   lock?: boolean;
   storageVersion?: number;
@@ -178,13 +186,14 @@ export interface SerializedSyncPlan {
    * them.
    */
   eventDescriptors: Record<string, string[]>;
+  errors?: ReplicationError[];
 }
 
 export function updateSyncRulesFromYaml(
   content: string,
   options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
 ): UpdateSyncRulesOptions {
-  const { config } = SqlSyncRules.fromYaml(content, {
+  const config = SqlSyncRules.fromYaml(content, {
     // No schema-based validation at this point
     schema: undefined,
     defaultSchema: 'not_applicable', // Not needed for validation
@@ -195,24 +204,26 @@ export function updateSyncRulesFromYaml(
 }
 
 export function updateSyncRulesFromConfig(
-  parsed: SyncConfig,
+  parsed: SyncConfigWithErrors,
   options?: Omit<UpdateSyncRulesOptions, 'config'>
 ): UpdateSyncRulesOptions {
   let plan: SerializedSyncPlan | null = null;
-  if (parsed instanceof PrecompiledSyncConfig) {
+  const { config, errors } = parsed;
+  if (config instanceof PrecompiledSyncConfig) {
     const eventDescriptors: Record<string, string[]> = {};
-    for (const event of parsed.eventDescriptors) {
+    for (const event of config.eventDescriptors) {
       eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
     }
 
     plan = {
-      compatibility: parsed.compatibility.serialize(),
-      plan: serializeSyncPlan(parsed.plan),
-      eventDescriptors
+      compatibility: config.compatibility.serialize(),
+      plan: serializeSyncPlan(config.plan),
+      eventDescriptors,
+      errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
     };
   }
 
-  return { config: { yaml: parsed.content, plan }, ...options };
+  return { config: { yaml: config.content, plan, parsed }, ...options };
 }
 
 export interface GetIntanceOptions {

package/src/storage/ChecksumCache.ts

@@ -1,6 +1,6 @@
 import { OrderedSet } from '@js-sdsl/ordered-set';
-import { LRUCache } from 'lru-cache/min';
 import { BucketDataSource } from '@powersync/service-sync-rules';
+import { LRUCache } from 'lru-cache/min';
 import { BucketChecksum } from '../util/protocol-types.js';
 import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
 import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';

package/src/storage/PersistedSyncRulesContent.ts

@@ -1,9 +1,10 @@
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { logger as defaultLogger, ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
 import {
   CompatibilityContext,
   CompatibilityOption,
   DEFAULT_HYDRATION_STATE,
   deserializeSyncPlan,
+  ErrorLocation,
   HydratedSyncRules,
   HydrationState,
   javaScriptExpressionEngine,
@@ -11,7 +12,8 @@ import {
   SqlEventDescriptor,
   SqlSyncRules,
   SyncConfigWithErrors,
-  versionedHydrationState
+  versionedHydrationState,
+  YamlError
 } from '@powersync/service-sync-rules';
 import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
@@ -27,7 +29,7 @@ export interface PersistedSyncRulesContentData {
   readonly compiled_plan: SerializedSyncPlan | null;
   readonly slot_name: string;
   /**
-   * True if this is the "active" copy of the sync rules.
+   * True if this is the "active" copy of the sync config.
    */
   readonly active: boolean;
   readonly storageVersion: number;
@@ -47,6 +49,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
   readonly slot_name!: string;
   readonly active!: boolean;
   readonly storageVersion!: number;
+  readonly logger: Logger;
 
   readonly last_checkpoint_lsn!: string | null;
 
@@ -59,6 +62,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
 
   constructor(data: PersistedSyncRulesContentData) {
     Object.assign(this, data);
+    this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` });
   }
 
   /**
@@ -71,7 +75,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
     if (storageConfig == null) {
       throw new ServiceError(
         ErrorCode.PSYNC_S1005,
-        `Unsupported storage version ${this.storageVersion} for sync rules ${this.id}`
+        `Unsupported storage version ${this.storageVersion} for replication stream ${this.id}`
       );
     }
     return storageConfig;
@@ -101,7 +105,25 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
         sourceText: this.sync_rules_content
       });
 
-      config = { config: precompiled, errors: [] };
+      // Note: If the original content did not define a storage version, this will still set the storage version.
+      // This means asUpdateOptions will not change the storage version, even if the default changes.
+      precompiled.storageVersion = this.storageVersion;
+
+      const errors: YamlError[] = [];
+      if (this.compiled_plan.errors) {
+        for (const error of this.compiled_plan.errors) {
+          const location: ErrorLocation | undefined = error.location && {
+            start: error.location.start_offset,
+            end: error.location.end_offset
+          };
+          const asYamlError = new YamlError(new Error(error.message), location);
+          asYamlError.type = error.level;
+
+          errors.push(asYamlError);
+        }
+      }
+
+      config = { config: precompiled, errors };
     } else {
       config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
     }
@@ -128,8 +150,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
   }
 
   asUpdateOptions(options?: Omit<UpdateSyncRulesOptions, 'config'>): UpdateSyncRulesOptions {
+    // defaultSchema is not relevant for the parsed version here
+    const parsed = this.parsed({ defaultSchema: 'not_applicable' });
     return {
-      config: { yaml: this.sync_rules_content, plan: this.compiled_plan },
+      config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules },
       ...options
     };
   }

package/src/storage/ReplicationEventPayload.ts

@@ -1,6 +1,6 @@
 import * as sync_rules from '@powersync/service-sync-rules';
-import { SourceTable } from './SourceTable.js';
 import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
+import { SourceTable } from './SourceTable.js';
 
 export type EventData = {
   op: SaveOp;

package/src/storage/SourceTable.ts

@@ -1,7 +1,7 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
+import { bson } from '../index.js';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
-import { bson } from '../index.js';
 
 /**
  * Format of the id depends on the bucket storage module. It should be consistent within the module.
@@ -28,7 +28,7 @@ export class SourceTable implements SourceEntityDescriptor {
   static readonly DEFAULT_TAG = DEFAULT_TAG;
 
   /**
-   * True if the table is used in sync rules for data queries.
+   * True if the table is used in sync config for data queries.
    *
    * This value is resolved externally, and cached here.
    *
@@ -37,7 +37,7 @@ export class SourceTable implements SourceEntityDescriptor {
   public syncData = true;
 
   /**
-   * True if the table is used in sync rules for data queries.
+   * True if the table is used in sync config for data queries.
    *
    * This value is resolved externally, and cached here.
    *
@@ -46,7 +46,7 @@ export class SourceTable implements SourceEntityDescriptor {
   public syncParameters = true;
 
   /**
-   * True if the table is used in sync rules for events.
+   * True if the table is used in sync config for events.
    *
    * This value is resolved externally, and cached here.
    *

package/src/storage/StorageVersionConfig.ts

@@ -45,7 +45,7 @@ export const STORAGE_VERSION_3 = 3;
 export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
 
 /**
- * Default storage version for newly persisted sync rules.
+ * Default storage version for newly persisted replication streams.
  */
 export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;