@powersync/service-core 1.20.3 → 1.20.5

package/src/replication/AbstractReplicator.ts

@@ -107,15 +107,17 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
       }, 1000);
     });
     this.metrics.getObservableGauge(ReplicationMetric.REPLICATION_LAG_SECONDS).setValueProvider(async () => {
-      const lag = await this.getReplicationLagMillis().catch((e) => {
+      try {
+        const lag = this.getReplicationLagMillis();
+        if (lag == null) {
+          return undefined;
+        }
+        // ms to seconds
+        return Math.round(lag / 1000);
+      } catch (e) {
         this.logger.error('Failed to get replication lag', e);
         return undefined;
-      });
-      if (lag == null) {
-        return undefined;
       }
-      // ms to seconds
-      return Math.round(lag / 1000);
     });
   }
 
@@ -312,7 +314,7 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
    *
    * "processing" replication streams are not taken into account for this metric.
    */
-  async getReplicationLagMillis(): Promise<number | undefined> {
+  getReplicationLagMillis(): number | undefined {
     return this.activeReplicationJob?.getReplicationLagMillis();
   }
 }

package/src/replication/ReplicationLagTracker.ts

@@ -0,0 +1,86 @@
+import { RollingBucketMax } from '../metrics/RollingBucketMax.js';
+
+/**
+ * Tracks replication lag across the current in-flight transaction and a rolling
+ * max of recently observed lag values.
+ */
+export class ReplicationLagTracker {
+  private readonly rollingReplicationLag = new RollingBucketMax();
+  private _oldestUncommittedChange: Date | null = null;
+  private _isStartingReplication = true;
+
+  /**
+   * The oldest source timestamp still part of the current in-flight work.
+   */
+  get oldestUncommittedChange(): Date | null {
+    return this._oldestUncommittedChange;
+  }
+
+  /**
+   * True until replication has seen its first completed commit or equivalent keepalive.
+   */
+  get isStartingReplication(): boolean {
+    return this._isStartingReplication;
+  }
+
+  /**
+   * Registers the first source timestamp for the current in-flight work,
+   * for example the start of a transaction
+   */
+  trackUncommittedChange(timestamp: Date | null | undefined): void {
+    if (this._oldestUncommittedChange == null && timestamp != null) {
+      this._oldestUncommittedChange = timestamp;
+    }
+  }
+
+  /**
+   * Clears the current in-flight timestamp without changing startup state.
+   */
+  clearUncommittedChange(): void {
+    this._oldestUncommittedChange = null;
+  }
+
+  /**
+   * Marks replication as started even if no committed transaction lag was recorded.
+   */
+  markStarted(): void {
+    this._isStartingReplication = false;
+  }
+
+  /**
+   * Mark the current pending changes as "committed".
+   *
+   * Records the current in-flight lag into the rolling window and clears it.
+   * The current lag is calculated as the differnence between current time and the oldest change,
+   * as marked by trackUncommittedChange.
+   */
+  markCommitted(timestampMs = Date.now()): void {
+    if (this._oldestUncommittedChange != null) {
+      this.rollingReplicationLag.report(timestampMs - this._oldestUncommittedChange.getTime(), timestampMs);
+    }
+    this.clearUncommittedChange();
+    this.markStarted();
+  }
+
+  /**
+   * Returns the lag for the current in-flight work.
+   *
+   * 0 if idle (no pending changes to replicate).
+   *
+   * undefined when replication is still starting up.
+   */
+  getCurrentLagMillis(timestampMs = Date.now()): number | undefined {
+    if (this._oldestUncommittedChange == null) {
+      return this._isStartingReplication ? undefined : 0;
+    }
+    return timestampMs - this._oldestUncommittedChange.getTime();
+  }
+
+  /**
+   * Returns the rolling lag metric value, including the current in-flight lag when present.
+   */
+  getLagMillis(timestampMs = Date.now()): number | undefined {
+    this.rollingReplicationLag.report(this.getCurrentLagMillis(timestampMs), timestampMs);
+    return this.rollingReplicationLag.getRollingMax(timestampMs);
+  }
+}

package/src/replication/replication-index.ts

@@ -1,7 +1,8 @@
 export * from './AbstractReplicationJob.js';
 export * from './AbstractReplicator.js';
 export * from './ErrorRateLimiter.js';
+export * from './RelationCache.js';
+export * from './replication-metrics.js';
 export * from './ReplicationEngine.js';
+export * from './ReplicationLagTracker.js';
 export * from './ReplicationModule.js';
-export * from './replication-metrics.js';
-export * from './RelationCache.js';

package/src/replication/replication-metrics.ts

@@ -1,5 +1,5 @@
-import { MetricsEngine } from '../metrics/metrics-index.js';
 import { ReplicationMetric } from '@powersync/service-types';
+import { MetricsEngine } from '../metrics/metrics-index.js';
 
 /**
  *  Create and register the core replication metrics.

package/src/routes/endpoints/sync-rules.ts

@@ -4,9 +4,9 @@ import type { FastifyPluginAsync } from 'fastify';
 import * as t from 'ts-codec';
 
 import { RouteAPI } from '../../api/RouteAPI.js';
+import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
 import { authApi } from '../auth.js';
 import { routeDefinition } from '../router.js';
-import { updateSyncRulesFromConfig, updateSyncRulesFromYaml } from '../../storage/BucketStorageFactory.js';
 
 const DeploySyncRulesRequest = t.object({
   content: t.string
@@ -56,7 +56,7 @@ export const deploySyncRules = routeDefinition({
 
     try {
       const apiHandler = service_context.routerEngine.getAPI();
-      syncConfig = SqlSyncRules.fromYaml(payload.params.content, {
+      syncConfig = SqlSyncRules.fromYaml(content, {
         ...apiHandler.getParseSyncRulesOptions(),
         // We don't do any schema-level validation at this point
         schema: undefined
@@ -70,9 +70,7 @@ export const deploySyncRules = routeDefinition({
       });
     }
 
-    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(
-      updateSyncRulesFromConfig(syncConfig.config)
-    );
+    const sync_rules = await storageEngine.activeBucketStorage.updateSyncRules(updateSyncRulesFromConfig(syncConfig));
 
     return {
       slot_name: sync_rules.slot_name

package/src/routes/hooks.ts

@@ -1,6 +1,6 @@
-import type fastify from 'fastify';
-import a from 'async';
 import { logger } from '@powersync/lib-services-framework';
+import a from 'async';
+import type fastify from 'fastify';
 
 export type CreateRequestQueueParams = {
   max_queue_depth: number;

package/src/routes/route-register.ts

@@ -1,17 +1,9 @@
 import type fastify from 'fastify';
 import * as uuid from 'uuid';
 
-import {
-  ErrorCode,
-  errors,
-  HTTPMethod,
-  logger,
-  RouteNotFound,
-  router,
-  ServiceError
-} from '@powersync/lib-services-framework';
-import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
+import { errors, HTTPMethod, logger, RouteNotFound, router, ServiceError } from '@powersync/lib-services-framework';
 import { FastifyReply } from 'fastify';
+import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {
   parse?: boolean;

package/src/storage/BucketStorageBatch.ts

@@ -1,10 +1,10 @@
 import { ObserverClient } from '@powersync/lib-services-framework';
 import { EvaluatedParameters, EvaluatedRow, SqliteRow, ToastableSqliteRow } from '@powersync/service-sync-rules';
 import { BSON } from 'bson';
+import { InternalOpId } from '../util/utils.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceTable, TableSnapshotStatus } from './SourceTable.js';
 import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
-import { InternalOpId } from '../util/utils.js';
 
 export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptions = {
   createEmptyCheckpoints: true,

package/src/storage/BucketStorageFactory.ts

@@ -1,16 +1,18 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
-import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
-import { ReplicationEventPayload } from './ReplicationEventPayload.js';
-import { ReplicationLock } from './ReplicationLock.js';
-import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
-import { ReportStorage } from './ReportStorage.js';
 import {
   PrecompiledSyncConfig,
   SerializedCompatibilityContext,
   serializeSyncPlan,
   SqlSyncRules,
-  SyncConfig
+  SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
+import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { ReportStorage } from './ReportStorage.js';
+import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
 
 /**
  * Represents a configured storage provider.
@@ -178,13 +180,14 @@ export interface SerializedSyncPlan {
    * them.
    */
   eventDescriptors: Record<string, string[]>;
+  errors?: ReplicationError[];
 }
 
 export function updateSyncRulesFromYaml(
   content: string,
   options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
 ): UpdateSyncRulesOptions {
-  const { config } = SqlSyncRules.fromYaml(content, {
+  const config = SqlSyncRules.fromYaml(content, {
     // No schema-based validation at this point
     schema: undefined,
     defaultSchema: 'not_applicable', // Not needed for validation
@@ -195,24 +198,25 @@ export function updateSyncRulesFromYaml(
 }
 
 export function updateSyncRulesFromConfig(
-  parsed: SyncConfig,
+  { config, errors }: SyncConfigWithErrors,
   options?: Omit<UpdateSyncRulesOptions, 'config'>
 ): UpdateSyncRulesOptions {
   let plan: SerializedSyncPlan | null = null;
-  if (parsed instanceof PrecompiledSyncConfig) {
+  if (config instanceof PrecompiledSyncConfig) {
     const eventDescriptors: Record<string, string[]> = {};
-    for (const event of parsed.eventDescriptors) {
+    for (const event of config.eventDescriptors) {
       eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
     }
 
     plan = {
-      compatibility: parsed.compatibility.serialize(),
-      plan: serializeSyncPlan(parsed.plan),
-      eventDescriptors
+      compatibility: config.compatibility.serialize(),
+      plan: serializeSyncPlan(config.plan),
+      eventDescriptors,
+      errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
     };
   }
 
-  return { config: { yaml: parsed.content, plan }, ...options };
+  return { config: { yaml: config.content, plan }, ...options };
 }
 
 export interface GetIntanceOptions {

package/src/storage/ChecksumCache.ts

@@ -1,6 +1,6 @@
 import { OrderedSet } from '@js-sdsl/ordered-set';
-import { LRUCache } from 'lru-cache/min';
 import { BucketDataSource } from '@powersync/service-sync-rules';
+import { LRUCache } from 'lru-cache/min';
 import { BucketChecksum } from '../util/protocol-types.js';
 import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
 import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';

package/src/storage/PersistedSyncRulesContent.ts

@@ -4,6 +4,7 @@ import {
   CompatibilityOption,
   DEFAULT_HYDRATION_STATE,
   deserializeSyncPlan,
+  ErrorLocation,
   HydratedSyncRules,
   HydrationState,
   javaScriptExpressionEngine,
@@ -11,7 +12,8 @@ import {
   SqlEventDescriptor,
   SqlSyncRules,
   SyncConfigWithErrors,
-  versionedHydrationState
+  versionedHydrationState,
+  YamlError
 } from '@powersync/service-sync-rules';
 import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
@@ -101,7 +103,21 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
         sourceText: this.sync_rules_content
       });
 
-      config = { config: precompiled, errors: [] };
+      const errors: YamlError[] = [];
+      if (this.compiled_plan.errors) {
+        for (const error of this.compiled_plan.errors) {
+          const location: ErrorLocation | undefined = error.location && {
+            start: error.location.start_offset,
+            end: error.location.end_offset
+          };
+          const asYamlError = new YamlError(new Error(error.message), location);
+          asYamlError.type = error.level;
+
+          errors.push(asYamlError);
+        }
+      }
+
+      config = { config: precompiled, errors };
     } else {
       config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
     }

package/src/storage/ReplicationEventPayload.ts

@@ -1,6 +1,6 @@
 import * as sync_rules from '@powersync/service-sync-rules';
-import { SourceTable } from './SourceTable.js';
 import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
+import { SourceTable } from './SourceTable.js';
 
 export type EventData = {
   op: SaveOp;

package/src/storage/SourceTable.ts

@@ -1,7 +1,7 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
+import { bson } from '../index.js';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
-import { bson } from '../index.js';
 
 /**
  * Format of the id depends on the bucket storage module. It should be consistent within the module.

package/src/storage/storage-index.ts

@@ -1,17 +1,17 @@
 export * from './bson.js';
 export * from './BucketStorage.js';
+export * from './BucketStorageBatch.js';
+export * from './BucketStorageFactory.js';
 export * from './ChecksumCache.js';
+export * from './PersistedSyncRulesContent.js';
 export * from './ReplicationEventPayload.js';
+export * from './ReplicationLock.js';
+export * from './ReportStorage.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
+export * from './storage-metrics.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';
-export * from './storage-metrics.js';
-export * from './WriteCheckpointAPI.js';
-export * from './BucketStorageFactory.js';
-export * from './BucketStorageBatch.js';
-export * from './SyncRulesBucketStorage.js';
-export * from './PersistedSyncRulesContent.js';
-export * from './ReplicationLock.js';
-export * from './ReportStorage.js';
 export * from './StorageVersionConfig.js';
+export * from './SyncRulesBucketStorage.js';
+export * from './WriteCheckpointAPI.js';

package/src/storage/storage-metrics.ts

@@ -1,7 +1,7 @@
-import { MetricsEngine } from '../metrics/MetricsEngine.js';
 import { logger } from '@powersync/lib-services-framework';
-import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
 import { StorageMetric } from '@powersync/service-types';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
 
 export function createCoreStorageMetrics(engine: MetricsEngine): void {
   engine.createObservableGauge({

package/src/streams/streams-index.ts

@@ -1,3 +1,3 @@
-export * from './merge.js';
-export * from './LastValueSink.js';
 export * from './BroadcastIterable.js';
+export * from './LastValueSink.js';
+export * from './merge.js';

package/src/sync/BucketChecksumState.ts

@@ -1,14 +1,13 @@
 import {
-  BucketDescription,
   BucketParameterQuerier,
   BucketPriority,
   BucketSource,
   HydratedSyncRules,
+  mergeBuckets,
   QuerierError,
   RequestedStream,
   RequestParameters,
-  ResolvedBucket,
-  mergeBuckets
+  ResolvedBucket
 } from '@powersync/service-sync-rules';
 
 import * as storage from '../storage/storage-index.js';
@@ -445,6 +444,7 @@ export class BucketParameterState {
         const subscription = explicitStreamSubscriptions[i];
 
         const syncRuleStream: RequestedStream = {
+          priorityOverride: subscription.override_priority as BucketPriority | null,
           parameters: subscription.parameters ?? {},
           opaque_id: i
         };
@@ -480,25 +480,9 @@ export class BucketParameterState {
    * reference default buckets by their stream index instead of duplicating the name on wire.
    */
   translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map<string, number>): util.ClientBucketDescription {
-    // If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that
-    // priority.
-    let priorityOverride: BucketPriority | null = null;
-    for (const reason of description.inclusion_reasons) {
-      if (reason != 'default') {
-        const requestedPriority = this.explicitStreamSubscriptions[reason.subscription]?.override_priority;
-        if (requestedPriority != null) {
-          if (priorityOverride == null) {
-            priorityOverride = requestedPriority as BucketPriority;
-          } else {
-            priorityOverride = Math.min(requestedPriority, priorityOverride) as BucketPriority;
-          }
-        }
-      }
-    }
-
     return {
       bucket: description.bucket,
-      priority: priorityOverride ?? description.priority,
+      priority: description.priority,
       subscriptions: description.inclusion_reasons.map((reason) => {
         if (reason == 'default') {
           const stream = description.definition;
@@ -533,7 +517,7 @@ export class BucketParameterState {
 
       let errorMessage = error.message;
       const logData: any = {
-        checkpoint: checkpoint,
+        checkpoint: checkpoint.base.checkpoint,
         user_id: this.syncParams.userId,
         parameter_query_results: update.buckets.length
       };

package/src/sync/RequestTracker.ts

@@ -1,8 +1,8 @@
 import { MetricsEngine } from '../metrics/MetricsEngine.js';
 
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import { APIMetric } from '@powersync/service-types';
 import { SyncBucketData } from '../util/protocol-types.js';
-import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 /**
  * Record sync stats per request stream.

package/src/sync/sync-index.ts

@@ -1,6 +1,6 @@
+export * from './BucketChecksumState.js';
 export * from './RequestTracker.js';
 export * from './safeRace.js';
 export * from './sync.js';
-export * from './util.js';
-export * from './BucketChecksumState.js';
 export * from './SyncContext.js';
+export * from './util.js';

package/src/sync/sync.ts

@@ -1,11 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import {
-  BucketDescription,
-  BucketPriority,
-  HydratedSyncRules,
-  ResolvedBucket,
-  SqliteJsonValue
-} from '@powersync/service-sync-rules';
+import { BucketPriority, HydratedSyncRules, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 

package/src/sync/util.ts

@@ -1,9 +1,9 @@
 import * as timers from 'timers/promises';
 
 import { SemaphoreInterface } from 'async-mutex';
+import { serialize } from 'bson';
 import * as util from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
-import { serialize } from 'bson';
 
 export type TokenStreamOptions = {
   /**

package/src/system/ServiceContext.ts

@@ -1,5 +1,6 @@
 import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
 
+import { EventsEngine } from '../events/EventsEngine.js';
 import { framework } from '../index.js';
 import * as metrics from '../metrics/MetricsEngine.js';
 import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
@@ -8,7 +9,6 @@ import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
 import { SyncContext } from '../sync/SyncContext.js';
 import * as utils from '../util/util-index.js';
-import { EventsEngine } from '../events/EventsEngine.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;

package/src/util/config/collectors/impl/base64-config-collector.ts

@@ -1,5 +1,5 @@
-import { ConfigCollector } from '../config-collector.js';
 import { RunnerConfig } from '../../types.js';
+import { ConfigCollector } from '../config-collector.js';
 
 export class Base64ConfigCollector extends ConfigCollector {
   get name(): string {

package/src/util/config/collectors/impl/filesystem-config-collector.ts

@@ -1,9 +1,9 @@
 import * as fs from 'fs/promises';
 import * as path from 'path';
 
-import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
-import { RunnerConfig } from '../../types.js';
 import { logger } from '@powersync/lib-services-framework';
+import { RunnerConfig } from '../../types.js';
+import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
 
 export class FileSystemConfigCollector extends ConfigCollector {
   get name(): string {

package/src/util/config/sync-rules/sync-rules-provider.ts

@@ -1,5 +1,5 @@
-import { SyncRulesConfig } from '../types.js';
 import fs from 'fs/promises';
+import { SyncRulesConfig } from '../types.js';
 
 export interface SyncRulesProvider {
   get(): Promise<string | undefined>;

package/src/util/config.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs/promises';
 import winston from 'winston';
 
-import { container, logger, LogFormat, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT } from '@powersync/lib-services-framework';
+import { container, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, LogFormat, logger } from '@powersync/lib-services-framework';
 import { configFile } from '@powersync/service-types';
 import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
 import { CompoundConfigCollector } from './util-index.js';

package/src/util/errors.ts

@@ -0,0 +1,21 @@
+import { YamlError } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+
+export function syncConfigYamlErrorToReplicationError(
+  { type, message, location }: YamlError,
+  ts?: string
+): ReplicationError {
+  const error: ReplicationError = {
+    level: type,
+    message,
+    ts
+  };
+  if (location != null) {
+    error.location = {
+      start_offset: location.start,
+      end_offset: location.end
+    };
+  }
+
+  return error;
+}

package/src/util/util-index.ts

@@ -1,4 +1,5 @@
 export * from './alerting.js';
+export * from './checkpointing.js';
 export * from './env.js';
 export * from './lsn.js';
 export * from './memory-tracking.js';
@@ -6,7 +7,6 @@ export * from './Mutex.js';
 export * from './protocol-types.js';
 export * from './secs.js';
 export * from './utils.js';
-export * from './checkpointing.js';
 export * from './version.js';
 
 export * from './config.js';

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
+import { BucketChecksum, OplogEntry, ProtocolOpId } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';