@powersync/service-core 1.27.1 → 1.28.0

package/src/routes/endpoints/sync-stream.ts

@@ -1,4 +1,4 @@
-import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework';
+import { router, schema } from '@powersync/lib-services-framework';
 import Negotiator from 'negotiator';
 import { Readable } from 'stream';
 
@@ -9,8 +9,15 @@ import { APIMetric, event_types } from '@powersync/service-types';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
 
+import {
+  recordSyncConnection,
+  SyncCloseReason,
+  syncConnectionCloseReasonLogText,
+  SyncTransport
+} from '../../metrics/connection-metrics.js';
 import { limitParamsForLogging } from '../../util/param-logging.js';
 import { maybeCompressResponseStream } from '../compression.js';
+import { resolveSyncConnectionSetup } from '../sync-connection.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -27,7 +34,7 @@ export const syncStreamed = routeDefinition({
   validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
   handler: async (payload) => {
     const { service_context, logger, token_payload } = payload.context;
-    const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context;
+    const { routerEngine, metricsEngine, syncContext } = service_context;
     const headers = payload.request.headers;
     const userAgent = headers['x-user-agent'] ?? headers['user-agent'];
     const clientId = payload.params.client_id;
@@ -54,25 +61,7 @@ export const syncStreamed = routeDefinition({
       connected_at: new Date(streamStart)
     };
 
-    if (routerEngine.closed) {
-      throw new errors.ServiceError({
-        status: 503,
-        code: ErrorCode.PSYNC_S2003,
-        description: 'Service temporarily unavailable'
-      });
-    }
-
-    const bucketStorage = (await storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
-
-    if (bucketStorage == null) {
-      throw new errors.ServiceError({
-        status: 500,
-        code: ErrorCode.PSYNC_S2302,
-        description: 'No sync config available'
-      });
-    }
-
-    const syncRules = bucketStorage.getParsedSyncRules(routerEngine.getAPI().getParseSyncRulesOptions());
+    const { bucketStorage, syncRules } = await resolveSyncConnectionSetup(service_context, SyncTransport.HttpStream);
 
     const controller = new AbortController();
     const tracker = new sync.RequestTracker(metricsEngine);
@@ -111,18 +100,19 @@ export const syncStreamed = routeDefinition({
       // Best effort guess on why the stream was closed.
       // We use the `??=` operator everywhere, so that we catch the first relevant
       // event, which is usually the most specific.
-      let closeReason: string | undefined = undefined;
+      let closeReason: SyncCloseReason | undefined = undefined;
+      let connectionError: unknown;
 
       const deregister = routerEngine.addStopHandler(() => {
         // This error is not currently propagated to the client
         controller.abort();
-        closeReason ??= 'process shutdown';
+        closeReason ??= SyncCloseReason.ProcessShutdown;
         stream.destroy(new Error('Shutting down system'));
       });
 
       stream.on('end', () => {
         // Auth failure or switch to new sync config
-        closeReason ??= 'service closing stream';
+        closeReason ??= SyncCloseReason.ServiceClosed;
       });
 
       stream.on('close', () => {
@@ -130,7 +120,10 @@ export const syncStreamed = routeDefinition({
       });
 
       stream.on('error', (error) => {
-        closeReason ??= 'stream error';
+        if (closeReason == null) {
+          closeReason = SyncCloseReason.StreamError;
+          connectionError = error;
+        }
         controller.abort();
         // Note: This appears as a 200 response in the logs.
         if (error.message != 'Shutting down system') {
@@ -151,11 +144,17 @@ export const syncStreamed = routeDefinition({
         },
         data: stream,
         afterSend: async (details) => {
+          // A client disconnect must not overwrite an earlier error or server close.
           if (details.clientClosed) {
-            closeReason ??= 'client closing stream';
+            closeReason ??= SyncCloseReason.ClientClosed;
           }
           controller.abort();
           metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
+          recordSyncConnection(metricsEngine, {
+            transport: SyncTransport.HttpStream,
+            closeReason: closeReason ?? SyncCloseReason.Unknown,
+            error: connectionError
+          });
           service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
             ...sdkData,
             disconnected_at: new Date()
@@ -164,13 +163,18 @@ export const syncStreamed = routeDefinition({
             ...tracker.getLogMeta(),
             app_metadata: formattedAppMetadata,
             stream_ms: Date.now() - streamStart,
-            close_reason: closeReason ?? 'unknown'
+            close_reason: syncConnectionCloseReasonLogText(closeReason)
           });
         }
       });
     } catch (ex) {
       controller.abort();
       metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1);
+      recordSyncConnection(metricsEngine, {
+        transport: SyncTransport.HttpStream,
+        closeReason: SyncCloseReason.StreamError,
+        error: ex
+      });
       service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, {
         ...sdkData,
         disconnected_at: new Date()

package/src/routes/hooks.ts

@@ -11,7 +11,10 @@ export type CreateRequestQueueParams = {
  * Creates a request queue which limits the amount of concurrent connections which
  * are active at any time.
  */
-export const createRequestQueueHook = (params: CreateRequestQueueParams): fastify.onRequestHookHandler => {
+export const createRequestQueueHook = (
+  params: CreateRequestQueueParams,
+  onRejected?: () => void
+): fastify.onRequestHookHandler => {
   const request_queue = a.queue<() => Promise<void>>((event, done) => {
     event().finally(done);
   }, params.concurrency);
@@ -27,6 +30,7 @@ export const createRequestQueueHook = (params: CreateRequestQueueParams): fastif
         path: request.url,
         queue_overflow: true
       });
+      onRejected?.();
       return reply.status(429).send();
     }
 

package/src/routes/sync-connection.ts

@@ -0,0 +1,39 @@
+import { ErrorCode, errors } from '@powersync/lib-services-framework';
+
+import { recordSyncConnection, SyncCloseReason, SyncTransport } from '../metrics/connection-metrics.js';
+import type { RouterServiceContext } from './router.js';
+
+/** Resolves stream resources, counting setup failures as rejections before rethrowing. */
+export async function resolveSyncConnectionSetup(serviceContext: RouterServiceContext, transport: SyncTransport) {
+  const { routerEngine, storageEngine, metricsEngine } = serviceContext;
+  let closeReason = SyncCloseReason.ServiceUnavailable;
+
+  try {
+    if (routerEngine.closed) {
+      throw new errors.ServiceError({
+        status: 503,
+        code: ErrorCode.PSYNC_S2003,
+        description: 'Service temporarily unavailable'
+      });
+    }
+
+    closeReason = SyncCloseReason.StorageError;
+    const bucketStorage = (await storageEngine.activeBucketStorage.getActiveSyncConfig())?.storage;
+
+    closeReason = SyncCloseReason.NoSyncConfig;
+    if (bucketStorage == null) {
+      throw new errors.ServiceError({
+        status: 500,
+        code: ErrorCode.PSYNC_S2302,
+        description: 'No sync config available'
+      });
+    }
+
+    closeReason = SyncCloseReason.SyncConfigError;
+    const syncRules = bucketStorage.getParsedSyncRules(routerEngine.getAPI().getParseSyncRulesOptions());
+    return { bucketStorage, syncRules };
+  } catch (error) {
+    recordSyncConnection(metricsEngine, { transport, closeReason, error });
+    throw error;
+  }
+}

package/src/storage/BucketStorageFactory.ts

@@ -1,13 +1,13 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
 import {
+  parseSyncConfig,
   PrecompiledSyncConfig,
   SerializedSyncPlan as RawSerializedSyncPlan,
   SerializedCompatibilityContext,
   serializeSyncPlan,
-  SqlSyncRules,
   SyncConfigWithErrors
 } from '@powersync/service-sync-rules';
-import { ReplicationError } from '@powersync/service-types';
+import { ReplicationError, ValidationDiagnostic } from '@powersync/service-types';
 import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
 import { PersistedReplicationStream } from './PersistedReplicationStream.js';
 import { PersistedSyncConfigContent } from './PersistedSyncConfigContent.js';
@@ -70,6 +70,11 @@ export abstract class BucketStorageFactory
    */
   abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise<PersistedReplicationStream>;
 
+  /**
+   * Stop pending processing atomically, preserving active configs. Used for rejected startup submissions.
+   */
+  abstract stopProcessingSyncConfigs(): Promise<void>;
+
   /**
    * Indicate that a slot was removed, and we should re-sync by creating
    * a new replication stream.
@@ -78,6 +83,8 @@ export abstract class BucketStorageFactory
    * config, but also accounts for cases where the current sync config is not
    * the latest one.
    *
+   * Recovery of the active config must atomically recheck pending work and create its replacement,
+   * preserving any newer pending deployment rather than superseding it with the active config.
    * Replication should be restarted after this.
    */
   abstract restartReplication(replicationStreamId: number): Promise<void>;
@@ -175,23 +182,29 @@ export interface StorageSyncConfigMetrics {
   attributed_object_storage_bytes: number;
 }
 
+/**
+ * Sync config content and its compiled representation, shared by startup validation and deployment.
+ * Validation errors are retained in parsed even when the config cannot be deployed.
+ */
+export interface CompiledSyncConfig {
+  yaml: string;
+  /**
+   * The serialized sync plan for the sync configuration, or `null` for configurations not using the Sync Streams
+   * compiler.
+   */
+  plan: SerializedSyncPlan | null;
+
+  /**
+   * Parsed sync config, primarily to generate a definition mapping.
+   * Not persisted, and the defaultSchema used for parsing is not relevant.
+   */
+  parsed: SyncConfigWithErrors;
+}
+
 export interface UpdateSyncRulesOptions {
   /** Optional operator-supplied label identifying this deployed sync config version. */
   version_label?: string;
-  config: {
-    yaml: string;
-    /**
-     * The serialized sync plan for the sync configuration, or `null` for configurations not using the Sync Streams
-     * compiler.
-     */
-    plan: SerializedSyncPlan | null;
-
-    /**
-     * Parsed sync config, primarily to generate a definition mapping.
-     * Not persisted, and the defaultSchema used for parsing is not relevant.
-     */
-    parsed: SyncConfigWithErrors;
-  };
+  config: CompiledSyncConfig;
   lock?: boolean;
   storageVersion?: number;
 
@@ -228,7 +241,7 @@ export function updateSyncRulesFromYaml(
   options?: Omit<UpdateSyncRulesOptions, 'config'> & { validate?: boolean }
 ): UpdateSyncRulesOptions {
   const { validate, ...updateOptions } = options ?? {};
-  const config = SqlSyncRules.fromYaml(content, {
+  const config = parseSyncConfig(content, {
     // No schema-based validation at this point
     schema: undefined,
     defaultSchema: options?.defaultSchema ?? 'not_applicable', // Not needed for validation
@@ -263,20 +276,23 @@ export function updateSyncRulesFromConfig(
 }
 
 /**
- * Logs the errors and warnings of a parsed sync config
- *
- * Call this where a sync config is loaded or deployed, so that operators see these in the service logs and not only
- * in the diagnostics and validate API responses.
- *
- * Fatal errors usually throw during parsing before this is reached. They can reach this when the sync config is parsed
- * without validation, such as at startup with `exit_on_error: false`.
+ * Log parser errors and warnings from a parsed config, or combined parsing and source-capability
+ * diagnostics from SyncConfigEngine. Use when loading or deploying a sync config so operators see
+ * diagnostics in service logs as well as API responses. Fatal diagnostics are logged at error level;
+ * warnings remain advisory. Uses the supplied logger or the service logger by default.
  */
-export function logSyncConfigErrors(parsed: SyncConfigWithErrors, log: typeof logger = logger) {
-  for (const error of parsed.errors) {
-    if (error.type == 'warning') {
-      log.warn(`Sync config warning: ${error.message}`);
+export function logSyncConfigErrors(
+  result: SyncConfigWithErrors | ValidationDiagnostic[],
+  log: typeof logger = logger
+) {
+  const diagnostics = Array.isArray(result)
+    ? result
+    : result.errors.map((error) => ({ level: error.type, message: error.message }));
+  for (const diagnostic of diagnostics) {
+    if (diagnostic.level == 'warning') {
+      log.warn(`Sync config warning: ${diagnostic.message}`);
     } else {
-      log.error(`Sync config error: ${error.message}`);
+      log.error(`Sync config error: ${diagnostic.message}`);
     }
   }
 }

package/src/storage/PersistedSyncConfigContent.ts

@@ -9,9 +9,9 @@ import {
   HydratedSyncConfig,
   HydrationState,
   nodeSqlite,
+  parseSyncConfig,
   PrecompiledSyncConfig,
   SourceConnectionMap,
-  SqlSyncRules,
   SyncConfigWithErrors,
   versionedHydrationState,
   YamlError
@@ -36,7 +36,7 @@ export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfi
 
   if (compiledPlan == null) {
     // Fallback: Only parse from YAML if no compiled plan is available.
-    return SqlSyncRules.fromYaml(content, parseOptions);
+    return parseSyncConfig(content, parseOptions);
   }
 
   const plan = deserializeSyncPlan(compiledPlan.plan);

package/src/storage/SyncRulesBucketStorage.ts

@@ -274,9 +274,25 @@ export interface CreateWriterOptions extends ParseSyncConfigOptions {
   skipExistingRows?: boolean;
 
   /**
-   * Callback called if we streamed an update to a record that we don't have yet.
+   * Callback called when an update cannot reconstruct a complete current row,
+   * including replay of an update whose incomplete row was already persisted.
    *
    * This is expected to happen in some initial replication edge cases, only if storeCurrentData = true.
+   *
+   * The callback must synchronously enqueue repair keys, without mutating the
+   * record, throwing, returning a promise, or re-entering storage. It can run
+   * during flush rather than save, including after the data transaction commits.
+   * It is not an indication that a source checkpoint has advanced. Notifications
+   * can repeat on source replay; consumers must tolerate repeated repair keys.
+   *
+   * Consumers must explicitly await flush(), complete its queued repairs, and
+   * only then commit/keepalive, persist a resume position, or acknowledge the
+   * source. Do not rely on commit() auto-flushing pending rows when supplying
+   * this callback. Repair failures must stop progress and restart replication;
+   * the in-memory repair queue is not durable. Keep input records stable until
+   * their flush completes so deferred callbacks observe the original key. The
+   * update's after record must contain the full repair key; the callback receives
+   * that original update, rather than the merged current row.
    */
   markRecordUnavailable?: BucketStorageMarkRecordUnavailable;
 
@@ -582,4 +598,9 @@ export const CHECKPOINT_INVALIDATE_ALL: CheckpointChanges = {
   invalidateParameterBuckets: true
 };
 
-export type BucketStorageMarkRecordUnavailable = (record: SaveUpdate) => void;
+/**
+ * Synchronously queues a repair; see CreateWriterOptions.markRecordUnavailable for ordering requirements.
+ *
+ * Requires an explicit undefined return type to avoid accidental async usage.
+ */
+export type BucketStorageMarkRecordUnavailable = (record: SaveUpdate) => undefined;

package/src/streams/BroadcastIterable.ts

@@ -1,4 +1,3 @@
-import { AbortError } from 'ix/aborterror.js';
 import { LastValueSink } from './LastValueSink.js';
 
 export type IterableSource<T> = (signal: AbortSignal) => AsyncIterable<T>;
@@ -47,8 +46,9 @@ export class BroadcastIterable<T> implements AsyncIterable<T> {
   private async loop(abortController: AbortController, sinks: Set<LastValueSink<T>>) {
     try {
       for await (let doc of this.source(abortController.signal)) {
-        if (abortController.signal.aborted || sinks.size == 0) {
-          throw new AbortError();
+        abortController.signal.throwIfAborted();
+        if (sinks.size == 0) {
+          throw new DOMException('No subscribers remain', 'AbortError');
         }
         this.last = doc;
         for (let sink of sinks) {
@@ -62,7 +62,7 @@ export class BroadcastIterable<T> implements AsyncIterable<T> {
       }
     } catch (e) {
       // Just in case the error is not from the source
-      abortController.abort();
+      abortController.abort(e);
 
       for (let listener of sinks) {
         listener.error(e);
@@ -77,12 +77,12 @@ export class BroadcastIterable<T> implements AsyncIterable<T> {
     }
   }
 
-  private removeSink(listener: LastValueSink<T>) {
+  private removeSink(listener: LastValueSink<T>, reason?: unknown) {
     this.subscribers?.delete(listener);
     if (this.subscribers?.size == 0) {
       // This is not immediate - there may be a delay until it is fully stopped,
       // depending on the underlying source.
-      this.abortController?.abort();
+      this.abortController?.abort(reason);
       this.last = undefined;
       this.subscribers = undefined;
       this.abortController = undefined;
@@ -98,12 +98,13 @@ export class BroadcastIterable<T> implements AsyncIterable<T> {
   }
 
   async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterableIterator<T> {
+    signal?.throwIfAborted();
     const sink = new LastValueSink(this.last);
     this.addSink(sink);
     try {
       yield* sink.withSignal(signal);
     } finally {
-      this.removeSink(sink);
+      this.removeSink(sink, signal?.reason);
     }
   }
 

package/src/streams/LastValueSink.ts

@@ -1,5 +1,3 @@
-import { AbortError } from 'ix/aborterror.js';
-
 /**
  * An AsyncIterable that allows pushing values.
  *
@@ -13,27 +11,25 @@ export class LastValueSink<T> implements AsyncIterable<T> {
 
   constructor(initial: T | undefined) {
     if (initial != null) {
-      this.buffer = { value: initial, done: false, error: undefined };
+      this.buffer = { value: initial, done: false };
     }
   }
 
   write(value: T) {
     this.push({
       value,
-      done: false,
-      error: undefined
+      done: false
     });
   }
 
   end() {
     this.push({
       value: undefined,
-      done: true,
-      error: undefined
+      done: true
     });
   }
 
-  error(e: any) {
+  error(e: unknown) {
     this.push({
       value: undefined,
       done: true,
@@ -55,19 +51,17 @@ export class LastValueSink<T> implements AsyncIterable<T> {
       return;
     }
 
-    if (signal?.aborted) {
-      throw new AbortError();
-    }
+    signal.throwIfAborted();
 
     const onAbort = () => {
-      this.error(new AbortError());
+      this.error(signal.reason);
     };
-    signal?.addEventListener('abort', onAbort);
+    signal.addEventListener('abort', onAbort, { once: true });
 
     try {
       yield* this;
     } finally {
-      signal?.removeEventListener('abort', onAbort);
+      signal.removeEventListener('abort', onAbort);
     }
   }
 
@@ -82,7 +76,7 @@ export class LastValueSink<T> implements AsyncIterable<T> {
       const n = this.buffer!;
       this.buffer = undefined;
 
-      if (n.error) {
+      if ('error' in n) {
         throw n.error;
       } else if (n.done) {
         return;
@@ -96,5 +90,5 @@ export class LastValueSink<T> implements AsyncIterable<T> {
 interface NextResult<T> {
   value: T | undefined;
   done: boolean;
-  error: any;
+  error?: unknown;
 }