@powersync/service-core 1.11.2 → 1.12.0

package/src/sync/sync.ts

@@ -7,12 +7,12 @@ import * as auth from '../auth/auth-index.js';
 import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
-import { logger } from '@powersync/lib-services-framework';
-import { BucketChecksumState } from './BucketChecksumState.js';
+import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
+import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
 import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
 import { SyncContext } from './SyncContext.js';
-import { RequestTracker } from './RequestTracker.js';
+import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
 
 export interface SyncStreamParameters {
   syncContext: SyncContext;
@@ -21,6 +21,7 @@ export interface SyncStreamParameters {
   params: util.StreamingSyncRequest;
   syncParams: RequestParameters;
   token: auth.JwtPayload;
+  logger?: Logger;
   /**
    * If this signal is aborted, the stream response ends as soon as possible, without error.
    */
@@ -35,6 +36,8 @@ export async function* streamResponse(
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
   const { syncContext, bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } =
     options;
+  const logger = options.logger ?? defaultLogger;
+
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -57,7 +60,8 @@ export async function* streamResponse(
     params,
     syncParams,
     tracker,
-    controller.signal
+    controller.signal,
+    logger
   );
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
@@ -77,11 +81,6 @@ export async function* streamResponse(
   }
 }
 
-export type BucketSyncState = {
-  description?: BucketDescription; // Undefined if the bucket has not yet been resolved by us.
-  start_op_id: util.InternalOpId;
-};
-
 async function* streamResponseInner(
   syncContext: SyncContext,
   bucketStorage: storage.SyncRulesBucketStorage,
@@ -89,7 +88,8 @@ async function* streamResponseInner(
   params: util.StreamingSyncRequest,
   syncParams: RequestParameters,
   tracker: RequestTracker,
-  signal: AbortSignal
+  signal: AbortSignal,
+  logger: Logger
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
   const { raw_data, binary_data } = params;
 
@@ -103,7 +103,8 @@ async function* streamResponseInner(
     initialBucketPositions: params.buckets?.map((bucket) => ({
       name: bucket.name,
       after: BigInt(bucket.after)
-    }))
+    })),
+    logger: logger
   });
   const stream = bucketStorage.watchCheckpointChanges({
     user_id: checkpointUserId,
@@ -111,16 +112,29 @@ async function* streamResponseInner(
   });
   const newCheckpoints = stream[Symbol.asyncIterator]();
 
+  type CheckpointAndLine = {
+    checkpoint: bigint;
+    line: CheckpointLine | null;
+  };
+
+  async function waitForNewCheckpointLine(): Promise<IteratorResult<CheckpointAndLine>> {
+    const next = await newCheckpoints.next();
+    if (next.done) {
+      return { done: true, value: undefined };
+    }
+
+    const line = await checksumState.buildNextCheckpointLine(next.value);
+    return { done: false, value: { checkpoint: next.value.base.checkpoint, line } };
+  }
+
   try {
-    let nextCheckpointPromise:
-      | Promise<PromiseSettledResult<IteratorResult<storage.StorageCheckpointUpdate>>>
-      | undefined;
+    let nextCheckpointPromise: Promise<PromiseSettledResult<IteratorResult<CheckpointAndLine>>> | undefined;
 
     do {
       if (!nextCheckpointPromise) {
         // Wrap in a settledPromise, so that abort errors after the parent stopped iterating
         // does not result in uncaught errors.
-        nextCheckpointPromise = settledPromise(newCheckpoints.next());
+        nextCheckpointPromise = settledPromise(waitForNewCheckpointLine());
       }
       const next = await nextCheckpointPromise;
       nextCheckpointPromise = undefined;
@@ -130,7 +144,7 @@ async function* streamResponseInner(
       if (next.value.done) {
         break;
       }
-      const line = await checksumState.buildNextCheckpointLine(next.value.value);
+      const line = next.value.value.line;
       if (line == null) {
         // No update to sync
         continue;
@@ -138,7 +152,10 @@ async function* streamResponseInner(
 
       const { checkpointLine, bucketsToFetch } = line;
 
+      // Since yielding can block, we update the state just before yielding the line.
+      line.advance();
       yield checkpointLine;
+
       // Start syncing data for buckets up to the checkpoint. As soon as we have completed at least one priority and
       // at least 1000 operations, we also start listening for new checkpoints concurrently. When a new checkpoint comes
       // in while we're still busy syncing data for lower priorities, interrupt the current operation and start syncing
@@ -164,30 +181,32 @@ async function* streamResponseInner(
       function maybeRaceForNewCheckpoint() {
         if (syncedOperations >= 1000 && nextCheckpointPromise === undefined) {
           nextCheckpointPromise = (async () => {
-            const next = await settledPromise(newCheckpoints.next());
-            if (next.status == 'rejected') {
-              abortCheckpointController.abort();
-            } else if (!next.value.done) {
-              // Stop the running bucketDataInBatches() iterations, making the main flow reach the new checkpoint.
-              abortCheckpointController.abort();
+            while (true) {
+              const next = await settledPromise(waitForNewCheckpointLine());
+              if (next.status == 'rejected') {
+                abortCheckpointController.abort();
+              } else if (!next.value.done) {
+                if (next.value.value.line == null) {
+                  // There's a new checkpoint that doesn't affect this sync stream. Keep listening, but don't
+                  // interrupt this batch.
+                  continue;
+                }
+
+                // A new sync line can be emitted. Stop running the bucketDataInBatches() iterations, making the
+                // main flow reach the new checkpoint.
+                abortCheckpointController.abort();
+              }
+
+              return next;
             }
-
-            return next;
           })();
         }
       }
 
-      function markOperationsSent(operations: number) {
-        syncedOperations += operations;
-        tracker.addOperationsSynced(operations);
-        // Disable interrupting checkpoints for now.
-        // There is a bug with interrupting checkpoints where:
-        // 1. User is in the middle of syncing a large batch of data (for example initial sync).
-        // 2. A new checkpoint comes in, which interrupts the batch.
-        // 3. However, the new checkpoint does not contain any new data for this connection, so nothing further is synced.
-        // This then causes the client to wait indefinitely for the remaining data or checkpoint_complete message. That is
-        // only resolved when a new checkpoint comes in that does have data for this connection, or the connection is restarted.
-        // maybeRaceForNewCheckpoint();
+      function markOperationsSent(stats: OperationsSentStats) {
+        syncedOperations += stats.total;
+        tracker.addOperationsSynced(stats);
+        maybeRaceForNewCheckpoint();
       }
 
       // This incrementally updates dataBuckets with each individual bucket position.
@@ -201,9 +220,9 @@ async function* streamResponseInner(
         yield* bucketDataInBatches({
           syncContext: syncContext,
           bucketStorage: bucketStorage,
-          checkpoint: next.value.value.base.checkpoint,
+          checkpoint: next.value.value.checkpoint,
           bucketsToFetch: buckets,
-          checksumState,
+          checkpointLine: line,
           raw_data,
           binary_data,
           onRowsSent: markOperationsSent,
@@ -212,7 +231,8 @@ async function* streamResponseInner(
           user_id: syncParams.user_id,
           // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial
           // sync complete message instead.
-          forPriority: !isLast ? priority : null
+          forPriority: !isLast ? priority : null,
+          logger
         });
       }
 
@@ -229,9 +249,10 @@ interface BucketDataRequest {
   syncContext: SyncContext;
   bucketStorage: storage.SyncRulesBucketStorage;
   checkpoint: util.InternalOpId;
+  /** Contains current bucket state. Modified by the request as data is sent. */
+  checkpointLine: CheckpointLine;
+  /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
   bucketsToFetch: BucketDescription[];
-  /** Contains current bucket state. Modified by the request as data is sent.  */
-  checksumState: BucketChecksumState;
   raw_data: boolean | undefined;
   binary_data: boolean | undefined;
   /** Signals that the connection was aborted and that streaming should stop ASAP. */
@@ -243,7 +264,8 @@ interface BucketDataRequest {
   abort_batch: AbortSignal;
   user_id?: string;
   forPriority: BucketPriority | null;
-  onRowsSent: (amount: number) => void;
+  onRowsSent: (stats: OperationsSentStats) => void;
+  logger: Logger;
 }
 
 async function* bucketDataInBatches(request: BucketDataRequest) {
@@ -292,12 +314,13 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     bucketStorage: storage,
     checkpoint,
     bucketsToFetch,
-    checksumState,
+    checkpointLine,
     raw_data,
     binary_data,
     abort_connection,
     abort_batch,
-    onRowsSent
+    onRowsSent,
+    logger
   } = request;
 
   let checkpointInvalidated = false;
@@ -322,12 +345,12 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     }
     // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
     // For the first batch, this will be all buckets.
-    const filteredBuckets = checksumState.getFilteredBucketPositions(bucketsToFetch);
+    const filteredBuckets = checkpointLine.getFilteredBucketPositions(bucketsToFetch);
     const dataBatches = storage.getBucketDataBatch(checkpoint, filteredBuckets);
 
     let has_more = false;
 
-    for await (let { batch: r, targetOp } of dataBatches) {
+    for await (let { chunkData: r, targetOp } of dataBatches) {
       // Abort in current batch if the connection is closed
       if (abort_connection.aborted) {
         return;
@@ -369,9 +392,9 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         // iterator memory in case if large data sent.
         yield { data: null, done: false };
       }
-      onRowsSent(r.data.length);
+      onRowsSent(statsForBatch(r));
 
-      checksumState.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
+      checkpointLine.updateBucketPosition({ bucket: r.bucket, nextAfter: BigInt(r.next_after), hasMore: r.has_more });
 
       // Check if syncing bucket data is supposed to stop before fetching more data
       // from storage.

package/src/system/ServiceContext.ts

@@ -6,18 +6,34 @@ import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManag
 import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
-import * as utils from '../util/util-index.js';
 import { SyncContext } from '../sync/SyncContext.js';
+import * as utils from '../util/util-index.js';
 
 export interface ServiceContext {
   configuration: utils.ResolvedPowerSyncConfig;
   lifeCycleEngine: LifeCycledSystem;
   metricsEngine: metrics.MetricsEngine;
   replicationEngine: replication.ReplicationEngine | null;
-  routerEngine: routes.RouterEngine | null;
+  routerEngine: routes.RouterEngine;
   storageEngine: storage.StorageEngine;
   migrations: PowerSyncMigrationManager;
   syncContext: SyncContext;
+  serviceMode: ServiceContextMode;
+}
+
+export enum ServiceContextMode {
+  API = utils.ServiceRunner.API,
+  SYNC = utils.ServiceRunner.SYNC,
+  UNIFIED = utils.ServiceRunner.UNIFIED,
+  COMPACT = 'compact',
+  MIGRATION = 'migration',
+  TEARDOWN = 'teardown',
+  TEST_CONNECTION = 'test-connection'
+}
+
+export interface ServiceContextOptions {
+  serviceMode: ServiceContextMode;
+  configuration: utils.ResolvedPowerSyncConfig;
 }
 
 /**
@@ -26,11 +42,18 @@ export interface ServiceContext {
  * This controls registering, initializing and the lifecycle of various services.
  */
 export class ServiceContextContainer implements ServiceContext {
+  configuration: utils.ResolvedPowerSyncConfig;
   lifeCycleEngine: LifeCycledSystem;
   storageEngine: storage.StorageEngine;
   syncContext: SyncContext;
+  routerEngine: routes.RouterEngine;
+  serviceMode: ServiceContextMode;
+
+  constructor(options: ServiceContextOptions) {
+    this.serviceMode = options.serviceMode;
+    const { configuration } = options;
+    this.configuration = configuration;
 
-  constructor(public configuration: utils.ResolvedPowerSyncConfig) {
     this.lifeCycleEngine = new LifeCycledSystem();
 
     this.storageEngine = new storage.StorageEngine({
@@ -42,6 +65,11 @@ export class ServiceContextContainer implements ServiceContext {
       stop: (storageEngine) => storageEngine.shutDown()
     });
 
+    this.routerEngine = new routes.RouterEngine();
+    this.lifeCycleEngine.withLifecycle(this.routerEngine, {
+      stop: (routerEngine) => routerEngine.shutDown()
+    });
+
     this.syncContext = new SyncContext({
       maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency,
       maxBuckets: configuration.api_parameters.max_buckets_per_connection,
@@ -55,21 +83,12 @@ export class ServiceContextContainer implements ServiceContext {
       // Migrations should be executed before the system starts
       start: () => migrationManager[Symbol.asyncDispose]()
     });
-
-    this.lifeCycleEngine.withLifecycle(this.storageEngine, {
-      start: (storageEngine) => storageEngine.start(),
-      stop: (storageEngine) => storageEngine.shutDown()
-    });
   }
 
   get replicationEngine(): replication.ReplicationEngine | null {
     return container.getOptional(replication.ReplicationEngine);
   }
 
-  get routerEngine(): routes.RouterEngine | null {
-    return container.getOptional(routes.RouterEngine);
-  }
-
   get metricsEngine(): metrics.MetricsEngine {
     return container.getImplementation(metrics.MetricsEngine);
   }

package/src/util/config/collectors/config-collector.ts

@@ -1,34 +1,18 @@
-import * as t from 'ts-codec';
 import * as yaml from 'yaml';
 
-import { configFile } from '@powersync/service-types';
 import { schema } from '@powersync/lib-services-framework';
+import { configFile } from '@powersync/service-types';
 
 import { RunnerConfig } from '../types.js';
+import { YamlEnvTag } from './impl/yaml-env.js';
 
 export enum ConfigFileFormat {
   YAML = 'yaml',
   JSON = 'json'
 }
 
-/**
- * Environment variables can be substituted into the YAML config
- * when parsing if the environment variable name starts with this prefix.
- * Attempting to substitute any other environment variable will throw an exception.
- *
- * Example of substitution:
- * storage:
- *    type: mongodb
- *    uri: !env PS_MONGO_URI
- */
-const YAML_ENV_PREFIX = 'PS_';
-
 // ts-codec itself doesn't give great validation errors, so we use json schema for that
-const configSchemaValidator = schema
-  .parseJSONSchema(
-    t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] })
-  )
-  .validator();
+const configSchemaValidator = schema.parseJSONSchema(configFile.PowerSyncConfigJSONSchema).validator();
 
 export abstract class ConfigCollector {
   abstract get name(): string;
@@ -105,27 +89,7 @@ export abstract class ConfigCollector {
       schema: 'core',
       keepSourceTokens: true,
       lineCounter,
-      customTags: [
-        {
-          tag: '!env',
-          resolve(envName: string, onError: (error: string) => void) {
-            if (!envName.startsWith(YAML_ENV_PREFIX)) {
-              onError(
-                `Attempting to substitute environment variable ${envName} is not allowed. Variables must start with "${YAML_ENV_PREFIX}"`
-              );
-              return envName;
-            }
-            const value = process.env[envName];
-            if (typeof value == 'undefined') {
-              onError(
-                `Attempted to substitute environment variable "${envName}" which is undefined. Set this variable on the environment.`
-              );
-              return envName;
-            }
-            return value;
-          }
-        }
-      ]
+      customTags: [YamlEnvTag]
     });
 
     if (parsed.errors.length) {

package/src/util/config/collectors/impl/yaml-env.ts

@@ -0,0 +1,67 @@
+import * as yaml from 'yaml';
+
+/**
+ * Environment variables can be substituted into the YAML config
+ * when parsing if the environment variable name starts with this prefix.
+ * Attempting to substitute any other environment variable will throw an exception.
+ *
+ * Example of substitution:
+ * storage:
+ *    type: mongodb
+ *    uri: !env PS_MONGO_URI
+ */
+const YAML_ENV_PREFIX = 'PS_';
+
+/**
+ * Custom YAML tag which performs string environment variable substitution
+ * Allows for type casting string environment variables to boolean or number
+ * by using the syntax !env PS_MONGO_PORT::number or !env PS_USE_SUPABASE::boolean
+ */
+export const YamlEnvTag: yaml.ScalarTag = {
+  tag: '!env',
+  resolve(envName: string, onError: (error: string) => void) {
+    if (!envName.startsWith(YAML_ENV_PREFIX)) {
+      onError(
+        `Attempting to substitute environment variable ${envName} is not allowed. Variables must start with "${YAML_ENV_PREFIX}"`
+      );
+      return envName;
+    }
+
+    // allow type casting if the envName contains a type suffix
+    // e.g. PS_MONGO_PORT::number or PS_USE_SUPABASE::boolean
+    const [name, type = 'string'] = envName.split('::');
+
+    let value = process.env[name];
+
+    if (typeof value == 'undefined') {
+      onError(
+        `Attempted to substitute environment variable "${envName}" which is undefined. Set this variable on the environment.`
+      );
+      return envName;
+    }
+
+    switch (type) {
+      case 'string':
+        return value;
+      case 'number':
+        const numberValue = Number(value);
+        if (Number.isNaN(numberValue)) {
+          onError(`Environment variable "${envName}" is not a valid number. Got: "${value}".`);
+          return envName;
+        }
+        return numberValue;
+      case 'boolean':
+        if (value?.toLowerCase() == 'true') {
+          return true;
+        } else if (value?.toLowerCase() == 'false') {
+          return false;
+        } else {
+          onError(`Environment variable "${envName}" is not a boolean. Expected "true" or "false", got "${value}".`);
+          return envName;
+        }
+      default:
+        onError(`Environment variable "${envName}" has an invalid type suffix "${type}".`);
+        return envName;
+    }
+  }
+};

package/src/util/config/compound-config-collector.ts

@@ -5,11 +5,6 @@ import { ConfigCollector } from './collectors/config-collector.js';
 import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
 import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
 import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
-import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-collector.js';
-import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
-import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
-import { SyncRulesCollector } from './sync-rules/sync-collector.js';
-import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
 import {
   DEFAULT_MAX_BUCKETS_PER_CONNECTION,
   DEFAULT_MAX_CONCURRENT_CONNECTIONS,
@@ -17,6 +12,11 @@ import {
   DEFAULT_MAX_PARAMETER_QUERY_RESULTS,
   DEFAULT_MAX_POOL_SIZE
 } from './defaults.js';
+import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-collector.js';
+import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
+import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
+import { SyncRulesCollector } from './sync-rules/sync-collector.js';
+import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
 
 export type CompoundConfigCollectorOptions = {
   /**
@@ -159,6 +159,23 @@ export class CompoundConfigCollector {
         internal_service_endpoint:
           baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
       },
+      healthcheck: {
+        /**
+         * Default to legacy mode if no probes config is provided.
+         * If users provide a config, all options require explicit opt-in.
+         */
+        probes: baseConfig.healthcheck?.probes
+          ? {
+              use_filesystem: baseConfig.healthcheck.probes.use_filesystem ?? false,
+              use_http: baseConfig.healthcheck.probes.use_http ?? false,
+              use_legacy: baseConfig.healthcheck.probes.use_legacy ?? false
+            }
+          : {
+              use_filesystem: false,
+              use_http: false,
+              use_legacy: true
+            }
+      },
       api_parameters: {
         max_buckets_per_connection:
           baseConfig.api?.parameters?.max_buckets_per_connection ?? DEFAULT_MAX_BUCKETS_PER_CONNECTION,

package/src/util/config/types.ts

@@ -69,5 +69,18 @@ export type ResolvedPowerSyncConfig = {
 
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
+
+  healthcheck: {
+    probes: {
+      use_filesystem: boolean;
+      use_http: boolean;
+      /**
+       * @deprecated This maintains backwards compatibility with the legacy default.
+       * Explicit probe configuration should be used instead.
+       */
+      use_legacy: boolean;
+    };
+  };
+
   parameters: Record<string, number | string | boolean | null>;
 };

package/src/util/protocol-types.ts

@@ -117,11 +117,11 @@ export interface SyncBucketData {
   bucket: string;
   data: OplogEntry[];
   /**
-   * True if the response does not contain all the data for this bucket, and another request must be made.
+   * True if there _could_ be more data for this bucket, and another request must be made.
    */
   has_more: boolean;
   /**
-   * The `after` specified in the request.
+   * The `after` specified in the request, or the next_after from the previous batch returned.
    */
   after: ProtocolOpId;
   /**