@powersync/service-core 1.20.4 → 1.20.5

package/src/storage/BucketStorageFactory.ts

@@ -1,16 +1,18 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
-import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
-import { ReplicationEventPayload } from './ReplicationEventPayload.js';
-import { ReplicationLock } from './ReplicationLock.js';
-import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
-import { ReportStorage } from './ReportStorage.js';
 import {
   PrecompiledSyncConfig,
   SerializedCompatibilityContext,
   serializeSyncPlan,
   SqlSyncRules,
-  SyncConfig
+  SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
+import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js';
+import { ReplicationEventPayload } from './ReplicationEventPayload.js';
+import { ReplicationLock } from './ReplicationLock.js';
+import { ReportStorage } from './ReportStorage.js';
+import { SyncRulesBucketStorage } from './SyncRulesBucketStorage.js';
 
 /**
  * Represents a configured storage provider.
@@ -178,13 +180,14 @@ export interface SerializedSyncPlan {
    * them.
    */
   eventDescriptors: Record<string, string[]>;
+  errors?: ReplicationError[];
 }
 
 export function updateSyncRulesFromYaml(
   content: string,
   options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
 ): UpdateSyncRulesOptions {
-  const { config } = SqlSyncRules.fromYaml(content, {
+  const config = SqlSyncRules.fromYaml(content, {
     // No schema-based validation at this point
     schema: undefined,
     defaultSchema: 'not_applicable', // Not needed for validation
@@ -195,24 +198,25 @@ export function updateSyncRulesFromYaml(
 }
 
 export function updateSyncRulesFromConfig(
-  parsed: SyncConfig,
+  { config, errors }: SyncConfigWithErrors,
   options?: Omit<UpdateSyncRulesOptions, 'config'>
 ): UpdateSyncRulesOptions {
   let plan: SerializedSyncPlan | null = null;
-  if (parsed instanceof PrecompiledSyncConfig) {
+  if (config instanceof PrecompiledSyncConfig) {
     const eventDescriptors: Record<string, string[]> = {};
-    for (const event of parsed.eventDescriptors) {
+    for (const event of config.eventDescriptors) {
       eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql);
     }
 
     plan = {
-      compatibility: parsed.compatibility.serialize(),
-      plan: serializeSyncPlan(parsed.plan),
-      eventDescriptors
+      compatibility: config.compatibility.serialize(),
+      plan: serializeSyncPlan(config.plan),
+      eventDescriptors,
+      errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e))
     };
   }
 
-  return { config: { yaml: parsed.content, plan }, ...options };
+  return { config: { yaml: config.content, plan }, ...options };
 }
 
 export interface GetIntanceOptions {

package/src/storage/ChecksumCache.ts

@@ -1,6 +1,6 @@
 import { OrderedSet } from '@js-sdsl/ordered-set';
-import { LRUCache } from 'lru-cache/min';
 import { BucketDataSource } from '@powersync/service-sync-rules';
+import { LRUCache } from 'lru-cache/min';
 import { BucketChecksum } from '../util/protocol-types.js';
 import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
 import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';

package/src/storage/PersistedSyncRulesContent.ts

@@ -4,6 +4,7 @@ import {
   CompatibilityOption,
   DEFAULT_HYDRATION_STATE,
   deserializeSyncPlan,
+  ErrorLocation,
   HydratedSyncRules,
   HydrationState,
   javaScriptExpressionEngine,
@@ -11,7 +12,8 @@ import {
   SqlEventDescriptor,
   SqlSyncRules,
   SyncConfigWithErrors,
-  versionedHydrationState
+  versionedHydrationState,
+  YamlError
 } from '@powersync/service-sync-rules';
 import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
 import { ReplicationLock } from './ReplicationLock.js';
@@ -101,7 +103,21 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
         sourceText: this.sync_rules_content
       });
 
-      config = { config: precompiled, errors: [] };
+      const errors: YamlError[] = [];
+      if (this.compiled_plan.errors) {
+        for (const error of this.compiled_plan.errors) {
+          const location: ErrorLocation | undefined = error.location && {
+            start: error.location.start_offset,
+            end: error.location.end_offset
+          };
+          const asYamlError = new YamlError(new Error(error.message), location);
+          asYamlError.type = error.level;
+
+          errors.push(asYamlError);
+        }
+      }
+
+      config = { config: precompiled, errors };
     } else {
       config = SqlSyncRules.fromYaml(this.sync_rules_content, options);
     }

package/src/storage/ReplicationEventPayload.ts

@@ -1,6 +1,6 @@
 import * as sync_rules from '@powersync/service-sync-rules';
-import { SourceTable } from './SourceTable.js';
 import { BucketStorageBatch, SaveOp } from './BucketStorageBatch.js';
+import { SourceTable } from './SourceTable.js';
 
 export type EventData = {
   op: SaveOp;

package/src/storage/SourceTable.ts

@@ -1,7 +1,7 @@
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
+import { bson } from '../index.js';
 import * as util from '../util/util-index.js';
 import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
-import { bson } from '../index.js';
 
 /**
  * Format of the id depends on the bucket storage module. It should be consistent within the module.

package/src/storage/storage-index.ts

@@ -1,17 +1,17 @@
 export * from './bson.js';
 export * from './BucketStorage.js';
+export * from './BucketStorageBatch.js';
+export * from './BucketStorageFactory.js';
 export * from './ChecksumCache.js';
+export * from './PersistedSyncRulesContent.js';
 export * from './ReplicationEventPayload.js';
+export * from './ReplicationLock.js';
+export * from './ReportStorage.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
+export * from './storage-metrics.js';
 export * from './StorageEngine.js';
 export * from './StorageProvider.js';
-export * from './storage-metrics.js';
-export * from './WriteCheckpointAPI.js';
-export * from './BucketStorageFactory.js';
-export * from './BucketStorageBatch.js';
-export * from './SyncRulesBucketStorage.js';
-export * from './PersistedSyncRulesContent.js';
-export * from './ReplicationLock.js';
-export * from './ReportStorage.js';
 export * from './StorageVersionConfig.js';
+export * from './SyncRulesBucketStorage.js';
+export * from './WriteCheckpointAPI.js';

package/src/storage/storage-metrics.ts

@@ -1,7 +1,7 @@
-import { MetricsEngine } from '../metrics/MetricsEngine.js';
 import { logger } from '@powersync/lib-services-framework';
-import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
 import { StorageMetric } from '@powersync/service-types';
+import { MetricsEngine } from '../metrics/MetricsEngine.js';
+import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js';
 
 export function createCoreStorageMetrics(engine: MetricsEngine): void {
   engine.createObservableGauge({

package/src/streams/streams-index.ts

@@ -1,3 +1,3 @@
-export * from './merge.js';
-export * from './LastValueSink.js';
 export * from './BroadcastIterable.js';
+export * from './LastValueSink.js';
+export * from './merge.js';

package/src/sync/BucketChecksumState.ts

@@ -1,14 +1,13 @@
 import {
-  BucketDescription,
   BucketParameterQuerier,
   BucketPriority,
   BucketSource,
   HydratedSyncRules,
+  mergeBuckets,
   QuerierError,
   RequestedStream,
   RequestParameters,
-  ResolvedBucket,
-  mergeBuckets
+  ResolvedBucket
 } from '@powersync/service-sync-rules';
 
 import * as storage from '../storage/storage-index.js';
@@ -445,6 +444,7 @@ export class BucketParameterState {
         const subscription = explicitStreamSubscriptions[i];
 
         const syncRuleStream: RequestedStream = {
+          priorityOverride: subscription.override_priority as BucketPriority | null,
           parameters: subscription.parameters ?? {},
           opaque_id: i
         };
@@ -480,25 +480,9 @@ export class BucketParameterState {
    * reference default buckets by their stream index instead of duplicating the name on wire.
    */
   translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map<string, number>): util.ClientBucketDescription {
-    // If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that
-    // priority.
-    let priorityOverride: BucketPriority | null = null;
-    for (const reason of description.inclusion_reasons) {
-      if (reason != 'default') {
-        const requestedPriority = this.explicitStreamSubscriptions[reason.subscription]?.override_priority;
-        if (requestedPriority != null) {
-          if (priorityOverride == null) {
-            priorityOverride = requestedPriority as BucketPriority;
-          } else {
-            priorityOverride = Math.min(requestedPriority, priorityOverride) as BucketPriority;
-          }
-        }
-      }
-    }
-
     return {
       bucket: description.bucket,
-      priority: priorityOverride ?? description.priority,
+      priority: description.priority,
       subscriptions: description.inclusion_reasons.map((reason) => {
         if (reason == 'default') {
           const stream = description.definition;
@@ -533,7 +517,7 @@ export class BucketParameterState {
 
       let errorMessage = error.message;
       const logData: any = {
-        checkpoint: checkpoint,
+        checkpoint: checkpoint.base.checkpoint,
         user_id: this.syncParams.userId,
         parameter_query_results: update.buckets.length
       };

package/src/sync/RequestTracker.ts

@@ -1,8 +1,8 @@
 import { MetricsEngine } from '../metrics/MetricsEngine.js';
 
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import { APIMetric } from '@powersync/service-types';
 import { SyncBucketData } from '../util/protocol-types.js';
-import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 /**
  * Record sync stats per request stream.

package/src/sync/sync-index.ts

@@ -1,6 +1,6 @@
+export * from './BucketChecksumState.js';
 export * from './RequestTracker.js';
 export * from './safeRace.js';
 export * from './sync.js';
-export * from './util.js';
-export * from './BucketChecksumState.js';
 export * from './SyncContext.js';
+export * from './util.js';

package/src/sync/sync.ts

@@ -1,11 +1,5 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import {
-  BucketDescription,
-  BucketPriority,
-  HydratedSyncRules,
-  ResolvedBucket,
-  SqliteJsonValue
-} from '@powersync/service-sync-rules';
+import { BucketPriority, HydratedSyncRules, ResolvedBucket, SqliteJsonValue } from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 

package/src/sync/util.ts

@@ -1,9 +1,9 @@
 import * as timers from 'timers/promises';
 
 import { SemaphoreInterface } from 'async-mutex';
+import { serialize } from 'bson';
 import * as util from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
-import { serialize } from 'bson';
 
 export type TokenStreamOptions = {
   /**

package/src/system/ServiceContext.ts

@@ -1,5 +1,6 @@
 import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework';
 
+import { EventsEngine } from '../events/EventsEngine.js';
 import { framework } from '../index.js';
 import * as metrics from '../metrics/MetricsEngine.js';
 import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
@@ -8,7 +9,6 @@ import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
 import { SyncContext } from '../sync/SyncContext.js';
 import * as utils from '../util/util-index.js';
-import { EventsEngine } from '../events/EventsEngine.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;

package/src/util/config/collectors/impl/base64-config-collector.ts

@@ -1,5 +1,5 @@
-import { ConfigCollector } from '../config-collector.js';
 import { RunnerConfig } from '../../types.js';
+import { ConfigCollector } from '../config-collector.js';
 
 export class Base64ConfigCollector extends ConfigCollector {
   get name(): string {

package/src/util/config/collectors/impl/filesystem-config-collector.ts

@@ -1,9 +1,9 @@
 import * as fs from 'fs/promises';
 import * as path from 'path';
 
-import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
-import { RunnerConfig } from '../../types.js';
 import { logger } from '@powersync/lib-services-framework';
+import { RunnerConfig } from '../../types.js';
+import { ConfigCollector, ConfigFileFormat } from '../config-collector.js';
 
 export class FileSystemConfigCollector extends ConfigCollector {
   get name(): string {

package/src/util/config/sync-rules/sync-rules-provider.ts

@@ -1,5 +1,5 @@
-import { SyncRulesConfig } from '../types.js';
 import fs from 'fs/promises';
+import { SyncRulesConfig } from '../types.js';
 
 export interface SyncRulesProvider {
   get(): Promise<string | undefined>;

package/src/util/config.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs/promises';
 import winston from 'winston';
 
-import { container, logger, LogFormat, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT } from '@powersync/lib-services-framework';
+import { container, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL, LogFormat, logger } from '@powersync/lib-services-framework';
 import { configFile } from '@powersync/service-types';
 import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
 import { CompoundConfigCollector } from './util-index.js';

package/src/util/errors.ts

@@ -0,0 +1,21 @@
+import { YamlError } from '@powersync/service-sync-rules';
+import { ReplicationError } from '@powersync/service-types';
+
+export function syncConfigYamlErrorToReplicationError(
+  { type, message, location }: YamlError,
+  ts?: string
+): ReplicationError {
+  const error: ReplicationError = {
+    level: type,
+    message,
+    ts
+  };
+  if (location != null) {
+    error.location = {
+      start_offset: location.start,
+      end_offset: location.end
+    };
+  }
+
+  return error;
+}

package/src/util/util-index.ts

@@ -1,4 +1,5 @@
 export * from './alerting.js';
+export * from './checkpointing.js';
 export * from './env.js';
 export * from './lsn.js';
 export * from './memory-tracking.js';
@@ -6,7 +7,6 @@ export * from './Mutex.js';
 export * from './protocol-types.js';
 export * from './secs.js';
 export * from './utils.js';
-export * from './checkpointing.js';
 export * from './version.js';
 
 export * from './config.js';

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
+import { BucketChecksum, OplogEntry, ProtocolOpId } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 

package/test/src/auth.test.ts

@@ -1,13 +1,13 @@
-import { describe, expect, test } from 'vitest';
-import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import { StaticSupabaseKeyCollector } from '@/index.js';
 import * as jose from 'jose';
-import { KeyStore } from '../../src/auth/KeyStore.js';
+import { describe, expect, test } from 'vitest';
+import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
+import { KeyResult } from '../../src/auth/KeyCollector.js';
 import { KeySpec } from '../../src/auth/KeySpec.js';
+import { KeyStore } from '../../src/auth/KeyStore.js';
 import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
-import { KeyResult } from '../../src/auth/KeyCollector.js';
-import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
-import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
-import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
+import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import { getSupabaseJwksUrl } from '../../src/auth/utils.js';
 
 const publicKeyRSA: jose.JWK = {
   use: 'sig',

package/test/src/module-loader.test.ts

@@ -1,5 +1,5 @@
 import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js';
-import { describe, expect, it, vi } from 'vitest';
+import { describe, expect, it } from 'vitest';
 
 interface MockConfig {
   connections?: { type: string }[];

package/test/src/routes/mocks.ts

@@ -1,3 +1,4 @@
+import { EventsEngine } from '@/events/EventsEngine.js';
 import {
   BucketStorageFactory,
   createCoreAPIMetrics,
@@ -11,7 +12,6 @@ import {
   SyncRulesBucketStorage
 } from '@/index.js';
 import { MeterProvider } from '@opentelemetry/sdk-metrics';
-import { EventsEngine } from '@/events/EventsEngine.js';
 
 export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
   // This is very incomplete - just enough to get the current tests passing.

package/test/src/routes/stream.test.ts

@@ -1,6 +1,6 @@
 import { BasicRouterRequest, Context, JwtPayload, SyncRulesBucketStorage } from '@/index.js';
 import { RouterResponse, ServiceError, logger } from '@powersync/lib-services-framework';
-import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { DEFAULT_HYDRATION_STATE, SqlSyncRules } from '@powersync/service-sync-rules';
 import { Readable, Writable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { describe, expect, it } from 'vitest';
@@ -8,7 +8,6 @@ import winston from 'winston';
 import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
 import { DEFAULT_PARAM_LOGGING_FORMAT_OPTIONS, limitParamsForLogging } from '../../../src/util/param-logging.js';
 import { mockServiceContext } from './mocks.js';
-import { DEFAULT_HYDRATION_STATE } from '@powersync/service-sync-rules';
 
 describe('Stream Route', () => {
   describe('compressed stream', () => {

package/test/src/sync/BucketChecksumState.test.ts

@@ -15,13 +15,12 @@ import {
 import { JSONBig } from '@powersync/service-jsonbig';
 import {
   ParameterIndexLookupCreator,
-  RequestJwtPayload,
   ScopedParameterLookup,
+  SourceTableInterface,
   SqliteJsonRow,
   SqliteRow,
   SqlSyncRules,
   TablePattern,
-  SourceTableInterface,
   versionedHydrationState
 } from '@powersync/service-sync-rules';
 import { ParameterLookupScope } from '@powersync/service-sync-rules/src/HydrationState.js';
@@ -986,6 +985,7 @@ bucket_definitions:
       expect(errorMessages[0]).toContain('tasks: 20');
       expect(errorMessages[0]).toContain('comments: 10');
 
+      expect(errorData[0].checkpoint).toEqual(1n);
       expect(errorData[0].parameter_query_results).toBe(60);
       expect(errorData[0].parameter_query_results_by_definition).toEqual({
         projects: 30,

package/test/src/util/protocol_types.test.ts

@@ -1,6 +1,6 @@
 import { StreamingSyncRequest } from '@/index.js';
 import { schema } from '@powersync/lib-services-framework';
-import { describe, test, expect, it } from 'vitest';
+import { describe, expect, test } from 'vitest';
 
 describe('protocol types', () => {
   describe('StreamingSyncRequest', () => {