@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/src/storage/mongo/OperationBatch.ts

@@ -1,8 +1,9 @@
-import * as bson from 'bson';
 import { ToastableSqliteRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 
-import * as util from '../../util/util-index.js';
 import { SaveOptions } from '../BucketStorage.js';
+import { isUUID } from './util.js';
+import { ReplicaId } from './models.js';
 
 /**
  * Maximum number of operations in a batch.
@@ -63,18 +64,15 @@ export class OperationBatch {
 }
 
 export class RecordOperation {
-  public readonly afterId: bson.UUID | null;
-  public readonly beforeId: bson.UUID;
+  public readonly afterId: ReplicaId | null;
+  public readonly beforeId: ReplicaId;
   public readonly internalBeforeKey: string;
   public readonly internalAfterKey: string | null;
   public readonly estimatedSize: number;
 
   constructor(public readonly record: SaveOptions) {
-    const after = record.after;
-    const afterId = after ? util.getUuidReplicaIdentityBson(after, record.sourceTable.replicaIdColumns!) : null;
-    const beforeId = record.before
-      ? util.getUuidReplicaIdentityBson(record.before, record.sourceTable.replicaIdColumns!)
-      : afterId!;
+    const afterId = record.afterReplicaId ?? null;
+    const beforeId = record.beforeReplicaId ?? record.afterReplicaId;
     this.afterId = afterId;
     this.beforeId = beforeId;
     this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId);
@@ -84,8 +82,17 @@ export class RecordOperation {
   }
 }
 
-export function cacheKey(table: bson.ObjectId, id: bson.UUID) {
-  return `${table.toHexString()}.${id.toHexString()}`;
+/**
+ * In-memory cache key - must not be persisted.
+ */
+export function cacheKey(table: bson.ObjectId, id: ReplicaId) {
+  if (isUUID(id)) {
+    return `${table.toHexString()}.${id.toHexString()}`;
+  } else if (typeof id == 'string') {
+    return `${table.toHexString()}.${id}`;
+  } else {
+    return `${table.toHexString()}.${(bson.serialize({ id: id }) as Buffer).toString('base64')}`;
+  }
 }
 
 /**

package/src/storage/mongo/PersistedBatch.ts

@@ -13,9 +13,10 @@ import {
   BucketParameterDocument,
   CurrentBucket,
   CurrentDataDocument,
-  SourceKey
+  SourceKey,
+  ReplicaId
 } from './models.js';
-import { serializeLookup } from './util.js';
+import { replicaIdToSubkey, serializeLookup } from './util.js';
 import { logger } from '@powersync/lib-services-framework';
 
 /**
@@ -59,7 +60,7 @@ export class PersistedBatch {
 
   saveBucketData(options: {
     op_seq: MongoIdSequence;
-    sourceKey: bson.UUID;
+    sourceKey: ReplicaId;
     table: SourceTable;
     evaluated: EvaluatedRow[];
     before_buckets: CurrentBucket[];
@@ -70,7 +71,7 @@ export class PersistedBatch {
       remaining_buckets.set(key, b);
     }
 
-    const dchecksum = util.hashDelete(`${options.table.id}/${options.sourceKey}`);
+    const dchecksum = util.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey));
 
     for (let k of options.evaluated) {
       const key = currentBucketKey(k);
@@ -134,7 +135,7 @@ export class PersistedBatch {
 
   saveParameterData(data: {
     op_seq: MongoIdSequence;
-    sourceKey: bson.UUID;
+    sourceKey: ReplicaId;
     sourceTable: SourceTable;
     evaluated: EvaluatedParameters[];
     existing_lookups: bson.Binary[];

package/src/storage/mongo/models.ts

@@ -1,13 +1,22 @@
 import * as bson from 'bson';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = bson.UUID | bson.Document | any;
+
 export interface SourceKey {
   /** group_id */
   g: number;
   /** source table id */
   t: bson.ObjectId;
   /** source key */
-  k: bson.UUID;
+  k: ReplicaId;
 }
 
 export interface BucketDataKey {
@@ -42,12 +51,13 @@ export interface BucketParameterDocument {
 export interface BucketDataDocument {
   _id: BucketDataKey;
   op: OpType;
-  source_table: bson.ObjectId;
-  source_key: bson.UUID;
-  table: string;
-  row_id: string;
+  source_table?: bson.ObjectId;
+  source_key?: ReplicaId;
+  table?: string;
+  row_id?: string;
   checksum: number;
   data: string | null;
+  target_op?: bigint | null;
 }
 
 export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
@@ -56,11 +66,11 @@ export interface SourceTableDocument {
   _id: bson.ObjectId;
   group_id: number;
   connection_id: number;
-  relation_id: number | undefined;
+  relation_id: number | string | undefined;
   schema_name: string;
   table_name: string;
   replica_id_columns: string[] | null;
-  replica_id_columns2: { name: string; type_oid: number }[] | undefined;
+  replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
   snapshot_done: boolean | undefined;
 }
 

package/src/storage/mongo/util.ts

@@ -1,7 +1,11 @@
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import * as mongo from 'mongodb';
 import * as crypto from 'crypto';
+import * as mongo from 'mongodb';
+import * as uuid from 'uuid';
+import { OplogEntry } from '../../util/protocol-types.js';
+import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js';
+import { BucketDataDocument, ReplicaId } from './models.js';
 
 /**
  * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
@@ -86,3 +90,69 @@ export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
   // use bigint instead of Long
   useBigInt64: true
 };
+
+export function mapOpEntry(row: BucketDataDocument): OplogEntry {
+  if (row.op == 'PUT' || row.op == 'REMOVE') {
+    return {
+      op_id: timestampToOpId(row._id.o),
+      op: row.op,
+      object_type: row.table,
+      object_id: row.row_id,
+      checksum: Number(row.checksum),
+      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
+      data: row.data
+    };
+  } else {
+    // MOVE, CLEAR
+
+    return {
+      op_id: timestampToOpId(row._id.o),
+      op: row.op,
+      checksum: Number(row.checksum)
+    };
+  }
+}
+
+/**
+ * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
+ */
+export function replicaIdEquals(a: ReplicaId, b: ReplicaId) {
+  if (a === b) {
+    return true;
+  } else if (typeof a == 'string' && typeof b == 'string') {
+    return a == b;
+  } else if (isUUID(a) && isUUID(b)) {
+    return a.equals(b);
+  } else if (a == null && b == null) {
+    return true;
+  } else if (a != null || b != null) {
+    return false;
+  } else {
+    // There are many possible primitive values, this covers them all
+    return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
+  }
+}
+
+export function replicaIdToSubkey(table: bson.ObjectId, id: ReplicaId): string {
+  if (isUUID(id)) {
+    // Special case for UUID for backwards-compatiblity
+    return `${table.toHexString()}/${id.toHexString()}`;
+  } else {
+    // Hashed UUID from the table and id
+    const repr = bson.serialize({ table, id });
+    return uuid.v5(repr, ID_NAMESPACE);
+  }
+}
+
+/**
+ * True if this is a bson.UUID.
+ *
+ * Works even with multiple copies of the bson package.
+ */
+export function isUUID(value: any): value is bson.UUID {
+  if (value == null || typeof value != 'object') {
+    return false;
+  }
+  const uuid = value as bson.UUID;
+  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
+}

package/src/storage/storage-index.ts

@@ -1,6 +1,8 @@
-export * from './SourceTable.js';
-export * from './MongoBucketStorage.js';
 export * from './BucketStorage.js';
+export * from './MongoBucketStorage.js';
+export * from './SourceEntity.js';
+export * from './SourceTable.js';
+export * from './StorageEngine.js';
 
 export * from './mongo/db.js';
 export * from './mongo/models.js';
@@ -8,6 +10,7 @@ export * from './mongo/MongoBucketBatch.js';
 export * from './mongo/MongoIdSequence.js';
 export * from './mongo/MongoPersistedSyncRules.js';
 export * from './mongo/MongoPersistedSyncRulesContent.js';
+export * from './mongo/MongoStorageProvider.js';
 export * from './mongo/MongoSyncBucketStorage.js';
 export * from './mongo/MongoSyncRulesLock.js';
 export * from './mongo/OperationBatch.js';

package/src/sync/RequestTracker.ts

@@ -1,4 +1,3 @@
-import { container } from '@powersync/lib-services-framework';
 import { Metrics } from '../metrics/Metrics.js';
 
 /**
@@ -10,12 +9,13 @@ export class RequestTracker {
 
   addOperationsSynced(operations: number) {
     this.operationsSynced += operations;
-    container.getImplementation(Metrics).operations_synced_total.add(operations);
+
+    Metrics.getInstance().operations_synced_total.add(operations);
   }
 
   addDataSynced(bytes: number) {
     this.dataSyncedBytes += bytes;
 
-    container.getImplementation(Metrics).data_synced_bytes.add(bytes);
+    Metrics.getInstance().data_synced_bytes.add(bytes);
   }
 }

package/src/sync/sync-index.ts

@@ -1,6 +1,7 @@
 export * from './BroadcastIterable.js';
 export * from './LastValueSink.js';
 export * from './merge.js';
+export * from './RequestTracker.js';
 export * from './safeRace.js';
 export * from './sync.js';
 export * from './util.js';

package/src/sync/sync.ts

@@ -1,6 +1,7 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
-import { Semaphore } from 'async-mutex';
+import { Semaphore, withTimeout } from 'async-mutex';
+
 import { AbortError } from 'ix/aborterror.js';
 
 import * as auth from '../auth/auth-index.js';
@@ -9,20 +10,33 @@ import * as util from '../util/util-index.js';
 
 import { logger } from '@powersync/lib-services-framework';
 import { mergeAsyncIterables } from './merge.js';
-import { TokenStreamOptions, tokenStream } from './util.js';
 import { RequestTracker } from './RequestTracker.js';
+import { TokenStreamOptions, tokenStream } from './util.js';
 
 /**
  * Maximum number of connections actively fetching data.
  */
 const MAX_ACTIVE_CONNECTIONS = 10;
-const syncSemaphore = new Semaphore(MAX_ACTIVE_CONNECTIONS);
+
+/**
+ * Maximum duration to wait for the mutex to become available.
+ *
+ * This gives an explicit error if there are mutex issues, rather than just hanging.
+ */
+const MUTEX_ACQUIRE_TIMEOUT = 30_000;
+
+const syncSemaphore = withTimeout(
+  new Semaphore(MAX_ACTIVE_CONNECTIONS),
+  MUTEX_ACQUIRE_TIMEOUT,
+  new Error(`Timeout while waiting for data`)
+);
 
 export interface SyncStreamParameters {
   storage: storage.BucketStorageFactory;
   params: util.StreamingSyncRequest;
   syncParams: RequestParameters;
   token: auth.JwtPayload;
+  parseOptions: storage.ParseSyncRulesOptions;
   /**
    * If this signal is aborted, the stream response ends as soon as possible, without error.
    */
@@ -35,7 +49,7 @@ export interface SyncStreamParameters {
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
+  const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal, parseOptions } = options;
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -51,7 +65,7 @@ export async function* streamResponse(
     }
   }
   const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-  const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
+  const stream = streamResponseInner(storage, params, syncParams, tracker, parseOptions, controller.signal);
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
 
@@ -75,6 +89,7 @@ async function* streamResponseInner(
   params: util.StreamingSyncRequest,
   syncParams: RequestParameters,
   tracker: RequestTracker,
+  parseOptions: storage.ParseSyncRulesOptions,
   signal: AbortSignal
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
   // Bucket state of bucket id -> op_id.
@@ -92,7 +107,8 @@ async function* streamResponseInner(
     }
   }
 
-  const stream = storage.watchWriteCheckpoint(syncParams.token_parameters.user_id as string, signal);
+  const checkpointUserId = util.checkpointUserId(syncParams.token_parameters.user_id as string, params.client_id);
+  const stream = storage.watchWriteCheckpoint(checkpointUserId, signal);
   for await (const next of stream) {
     const { base, writeCheckpoint } = next;
     const checkpoint = base.checkpoint;
@@ -102,9 +118,9 @@ async function* streamResponseInner(
       // Sync rules deleted in the meantime - try again with the next checkpoint.
       continue;
     }
-    const sync_rules = storage.sync_rules;
+    const syncRules = storage.getParsedSyncRules(parseOptions);
 
-    const allBuckets = await sync_rules.queryBucketIds({
+    const allBuckets = await syncRules.queryBucketIds({
       getParameterSets(lookups) {
         return storage.getParameterSets(checkpoint, lookups);
       },
@@ -195,7 +211,8 @@ async function* streamResponseInner(
       raw_data,
       binary_data,
       signal,
-      tracker
+      tracker,
+      user_id: syncParams.user_id
     });
 
     await new Promise((resolve) => setTimeout(resolve, 10));
@@ -212,6 +229,7 @@ interface BucketDataRequest {
   binary_data: boolean | undefined;
   tracker: RequestTracker;
   signal: AbortSignal;
+  user_id?: string;
 }
 
 async function* bucketDataInBatches(request: BucketDataRequest) {
@@ -257,8 +275,22 @@ interface BucketDataBatchResult {
 async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
   const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
 
-  const [_, release] = await syncSemaphore.acquire();
+  const checkpointOp = BigInt(checkpoint);
+  let checkpointInvalidated = false;
+
+  if (syncSemaphore.isLocked()) {
+    logger.info('Sync concurrency limit reached, waiting for lock', { user_id: request.user_id });
+  }
+  const [value, release] = await syncSemaphore.acquire();
   try {
+    if (value <= 3) {
+      // This can be noisy, so we only log when we get close to the
+      // concurrency limit.
+      logger.info(`Got sync lock. Slots available: ${value - 1}`, {
+        user_id: request.user_id,
+        sync_data_slots: value - 1
+      });
+    }
     // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
     // For the first batch, this will be all buckets.
     const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)!]));
@@ -266,13 +298,16 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
 
     let has_more = false;
 
-    for await (let r of data) {
+    for await (let { batch: r, targetOp } of data) {
       if (signal.aborted) {
         return;
       }
       if (r.has_more) {
         has_more = true;
       }
+      if (targetOp != null && targetOp > checkpointOp) {
+        checkpointInvalidated = true;
+      }
       if (r.data.length == 0) {
         continue;
       }
@@ -308,14 +343,28 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
     }
 
     if (!has_more) {
-      const line: util.StreamingSyncCheckpointComplete = {
-        checkpoint_complete: {
-          last_op_id: checkpoint
-        }
-      };
-      yield { data: line, done: true };
+      if (checkpointInvalidated) {
+        // Checkpoint invalidated by a CLEAR or MOVE op.
+        // Don't send the checkpoint_complete line in this case.
+        // More data should be available immediately for a new checkpoint.
+        yield { data: null, done: true };
+      } else {
+        const line: util.StreamingSyncCheckpointComplete = {
+          checkpoint_complete: {
+            last_op_id: checkpoint
+          }
+        };
+        yield { data: line, done: true };
+      }
     }
   } finally {
+    if (value <= 3) {
+      // This can be noisy, so we only log when we get close to the
+      // concurrency limit.
+      logger.info(`Releasing sync lock`, {
+        user_id: request.user_id
+      });
+    }
     release();
   }
 }

package/src/system/ServiceContext.ts

@@ -0,0 +1,68 @@
+import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
+
+import * as metrics from '../metrics/Metrics.js';
+import * as replication from '../replication/replication-index.js';
+import * as routes from '../routes/routes-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as utils from '../util/util-index.js';
+
+export interface ServiceContext {
+  configuration: utils.ResolvedPowerSyncConfig;
+  lifeCycleEngine: LifeCycledSystem;
+  metrics: metrics.Metrics | null;
+  replicationEngine: replication.ReplicationEngine | null;
+  routerEngine: routes.RouterEngine | null;
+  storageEngine: storage.StorageEngine;
+}
+
+/**
+ * Context which allows for registering and getting implementations
+ * of various service engines.
+ * This controls registering, initializing and the lifecycle of various services.
+ */
+export class ServiceContextContainer implements ServiceContext {
+  lifeCycleEngine: LifeCycledSystem;
+  storageEngine: storage.StorageEngine;
+
+  constructor(public configuration: utils.ResolvedPowerSyncConfig) {
+    this.lifeCycleEngine = new LifeCycledSystem();
+
+    this.storageEngine = new storage.StorageEngine({
+      configuration
+    });
+    this.lifeCycleEngine.withLifecycle(this.storageEngine, {
+      start: (storageEngine) => storageEngine.start(),
+      stop: (storageEngine) => storageEngine.shutDown()
+    });
+
+    // Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
+    this.storageEngine.registerProvider(new storage.MongoStorageProvider());
+  }
+
+  get replicationEngine(): replication.ReplicationEngine | null {
+    return container.getOptional(replication.ReplicationEngine);
+  }
+
+  get routerEngine(): routes.RouterEngine | null {
+    return container.getOptional(routes.RouterEngine);
+  }
+
+  get metrics(): metrics.Metrics | null {
+    return container.getOptional(metrics.Metrics);
+  }
+
+  /**
+   * Allows for registering core and generic implementations of services/helpers.
+   * This uses the framework container under the hood.
+   */
+  register<T>(identifier: ServiceIdentifier<T>, implementation: T) {
+    container.register(identifier, implementation);
+  }
+
+  /**
+   * Gets the implementation of an identifiable service.
+   */
+  get<T>(identifier: ServiceIdentifier<T>) {
+    return container.getImplementation(identifier);
+  }
+}

package/src/system/system-index.ts

@@ -1 +1 @@
-export * from './CorePowerSyncSystem.js';
+export * from './ServiceContext.js';

package/src/util/config/collectors/config-collector.ts

@@ -1,6 +1,8 @@
+import * as t from 'ts-codec';
 import * as yaml from 'yaml';
 
 import { configFile } from '@powersync/service-types';
+import { schema } from '@powersync/lib-services-framework';
 
 import { RunnerConfig } from '../types.js';
 
@@ -21,6 +23,13 @@ export enum ConfigFileFormat {
  */
 const YAML_ENV_PREFIX = 'PS_';
 
+// ts-codec itself doesn't give great validation errors, so we use json schema for that
+const configSchemaValidator = schema
+  .parseJSONSchema(
+    t.generateJSONSchema(configFile.powerSyncConfig, { allowAdditional: true, parsers: [configFile.portParser] })
+  )
+  .validator();
+
 export abstract class ConfigCollector {
   abstract get name(): string;
 
@@ -30,6 +39,45 @@ export abstract class ConfigCollector {
    */
   abstract collectSerialized(runnerConfig: RunnerConfig): Promise<configFile.SerializedPowerSyncConfig | null>;
 
+  /**
+   * Collects the PowerSyncConfig settings.
+   * Validates and decodes the config.
+   * @returns null if this collector cannot provide a config
+   */
+  async collect(runner_config: RunnerConfig): Promise<configFile.PowerSyncConfig | null> {
+    const serialized = await this.collectSerialized(runner_config);
+    if (!serialized) {
+      return null;
+    }
+
+    /**
+     * After this point a serialized config has been found. Any failures to decode or validate
+     * will result in a hard stop.
+     */
+    const decoded = this.decode(serialized);
+    this.validate(decoded);
+    return decoded;
+  }
+
+  /**
+   * Validates input config
+   * ts-codec itself doesn't give great validation errors, so we use json schema for that
+   */
+  validate(config: configFile.PowerSyncConfig) {
+    const valid = configSchemaValidator.validate(config);
+    if (!valid.valid) {
+      throw new Error(`Failed to validate PowerSync config: ${valid.errors.join(', ')}`);
+    }
+  }
+
+  decode(encoded: configFile.SerializedPowerSyncConfig): configFile.PowerSyncConfig {
+    try {
+      return configFile.powerSyncConfig.decode(encoded);
+    } catch (ex) {
+      throw new Error(`Failed to decode PowerSync config: ${ex}`);
+    }
+  }
+
   protected parseContent(content: string, contentType?: ConfigFileFormat) {
     switch (contentType) {
       case ConfigFileFormat.YAML: