@powersync/service-module-mongodb-storage 0.18.3 → 0.20.0

package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts

@@ -0,0 +1,343 @@
+import {
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  ListObjectsV2CommandOutput,
+  PutObjectCommand,
+  S3Client
+} from '@aws-sdk/client-s3';
+import { acquireSemaphoreAbortable, isAbortError } from '@powersync/service-core';
+import { isThrottlingError, isTransientError } from '@smithy/core/retry';
+import { Semaphore, SemaphoreInterface } from 'async-mutex';
+import { ObjectStorageError, type ObjectStorage, type ObjectStoragePutMetadata } from './ObjectStorage.js';
+
+const DEFAULT_S3_OPERATION_CONCURRENCY = 16;
+const S3_DELETE_PREFIX_BATCH_SIZE = 1000;
+/**
+ * Slightly smaller than DEFAULT_S3_OPERATION_CONCURRENCY.
+ */
+const S3_DELETE_PREFIX_CONCURRENCY = 12;
+const MAX_S3_PREFIX_BYTES = 256;
+const SAFE_S3_KEY_BYTES = 896;
+
+export interface S3ObjectStorageOptions {
+  bucket: string;
+  region?: string;
+  prefix?: string;
+  endpoint?: string;
+  forcePathStyle?: boolean;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  concurrencyLimit?: number;
+}
+
+export class S3ObjectStorage implements ObjectStorage {
+  /**
+   * Public for tests only.
+   */
+  public readonly client: S3Client;
+  private bucket: string;
+  private prefix: string;
+  private readonly operationSemaphore: SemaphoreInterface;
+
+  constructor(options: S3ObjectStorageOptions) {
+    const concurrencyLimit = options.concurrencyLimit ?? DEFAULT_S3_OPERATION_CONCURRENCY;
+    if (!Number.isInteger(concurrencyLimit) || concurrencyLimit <= 0) {
+      throw new Error('S3 object storage concurrencyLimit must be a positive integer');
+    }
+    if ((options.accessKeyId == null) !== (options.secretAccessKey == null)) {
+      throw new Error('S3 object storage accessKeyId and secretAccessKey must be configured together');
+    }
+
+    const prefix = options.prefix ?? '';
+    if (Buffer.byteLength(prefix, 'utf8') > MAX_S3_PREFIX_BYTES) {
+      throw new Error(`S3 object storage prefix must be at most ${MAX_S3_PREFIX_BYTES} UTF-8 bytes`);
+    }
+    if (prefix.endsWith('/')) {
+      throw new Error('S3 object storage prefix must not end with "/"');
+    }
+
+    this.bucket = options.bucket;
+    this.prefix = prefix;
+    this.operationSemaphore = new Semaphore(concurrencyLimit);
+    this.client = new S3Client({
+      region: options.region,
+      endpoint: options.endpoint,
+      forcePathStyle: options.forcePathStyle,
+      credentials:
+        options.accessKeyId && options.secretAccessKey
+          ? { accessKeyId: options.accessKeyId, secretAccessKey: options.secretAccessKey }
+          : undefined
+    });
+  }
+
+  async put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void> {
+    const fullPath = this.fullPath(path);
+    await using _ = await this.withOperation();
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: fullPath,
+          Body: data,
+          ContentType: metadata.contentType,
+          ContentEncoding: metadata.contentEncoding ?? undefined
+        })
+      );
+    } catch (error) {
+      throw s3OperationError('upload', fullPath, error);
+    }
+  }
+
+  async get(
+    path: string,
+    options?: { signal?: AbortSignal }
+  ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
+    const fullPath = this.fullPath(path);
+    const signal = options?.signal;
+    await using _ = await this.withOperation(signal);
+    try {
+      const response = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: fullPath
+        }),
+        { abortSignal: signal }
+      );
+      const contentLength = response.ContentLength;
+      if (contentLength == null) {
+        throw new Error(`S3 download response for ${fullPath} is missing ContentLength`);
+      }
+      if (!Number.isSafeInteger(contentLength) || contentLength < 0) {
+        throw new Error(`S3 download response for ${fullPath} has invalid ContentLength ${contentLength}`);
+      }
+
+      const data = new Uint8Array(contentLength);
+      let offset = 0;
+      const stream = response.Body as AsyncIterable<Uint8Array>;
+      for await (const chunk of stream) {
+        signal?.throwIfAborted();
+        const nextOffset = offset + chunk.byteLength;
+        if (nextOffset > contentLength) {
+          throw new Error(
+            `S3 download ContentLength mismatch for ${fullPath}: expected ${contentLength} bytes, received at least ${nextOffset}`
+          );
+        }
+        data.set(chunk, offset);
+        offset = nextOffset;
+      }
+      if (offset !== contentLength) {
+        throw new Error(
+          `S3 download ContentLength mismatch for ${fullPath}: expected ${contentLength} bytes, received ${offset}`
+        );
+      }
+
+      return {
+        data,
+        metadata: {
+          contentType: response.ContentType ?? 'application/octet-stream',
+          contentEncoding: response.ContentEncoding ?? null
+        }
+      };
+    } catch (err: any) {
+      if (err.name === 'NoSuchKey' || err.Code === 'NoSuchKey') {
+        throw new ObjectStorageError(`S3 object not found: ${fullPath}`, { cause: err, retryable: false });
+      }
+      throw s3OperationError('download', fullPath, err);
+    }
+  }
+
+  async *list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string> {
+    const fullPrefix = this.fullPath(prefix);
+    let continuationToken: string | undefined;
+    const signal = options?.signal;
+
+    do {
+      signal?.throwIfAborted();
+      const response = await this.listPage(fullPrefix, continuationToken, signal);
+      for (const object of response.Contents ?? []) {
+        if (object.Key == null) {
+          continue;
+        }
+        yield this.prefix ? object.Key.slice(this.prefix.length + 1) : object.Key;
+      }
+      continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
+      if (response.IsTruncated && continuationToken == null) {
+        throw new Error(`S3 listing for ${fullPrefix} was truncated without a continuation token`);
+      }
+    } while (continuationToken != null);
+  }
+
+  async delete(paths: string[]): Promise<void> {
+    const fullPaths = paths.map((path) => ({ Key: this.fullPath(path) }));
+    await this.deleteFullPaths(fullPaths);
+  }
+
+  async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
+    const fullPrefix = this.fullPath(prefix);
+    const signal = options?.signal;
+    let continuationToken: string | undefined;
+    let objectCount = 0;
+    const deleting = new Set<Promise<void>>();
+    let deleteError: { error: unknown } | undefined;
+
+    const scheduleDelete = (fullPaths: { Key: string }[]) => {
+      let tracked: Promise<void>;
+      tracked = this.deleteFullPaths(fullPaths, signal)
+        .catch((error) => {
+          deleteError ??= { error };
+        })
+        .finally(() => deleting.delete(tracked));
+      deleting.add(tracked);
+    };
+    const throwDeleteError = () => {
+      if (deleteError != null) {
+        throw deleteError.error;
+      }
+    };
+
+    let operationError: { error: unknown } | undefined;
+    try {
+      do {
+        signal?.throwIfAborted();
+        const response = await this.listPage(fullPrefix, continuationToken, signal);
+        throwDeleteError();
+        const fullPaths: { Key: string }[] = [];
+        for (const object of response.Contents ?? []) {
+          if (object.Key != null) {
+            fullPaths.push({ Key: object.Key });
+          }
+        }
+        objectCount += fullPaths.length;
+        if (fullPaths.length !== 0) {
+          while (deleting.size >= S3_DELETE_PREFIX_CONCURRENCY) {
+            await Promise.race(deleting);
+            throwDeleteError();
+          }
+          scheduleDelete(fullPaths);
+        }
+        continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
+        if (response.IsTruncated && continuationToken == null) {
+          throw new Error(`S3 listing for ${fullPrefix} was truncated without a continuation token`);
+        }
+      } while (continuationToken != null);
+    } catch (error) {
+      operationError = { error };
+    }
+
+    await Promise.all(deleting);
+    if (operationError != null) {
+      throw operationError.error;
+    }
+    throwDeleteError();
+    return { objectCount };
+  }
+
+  private async listPage(
+    fullPrefix: string,
+    continuationToken: string | undefined,
+    signal?: AbortSignal
+  ): Promise<ListObjectsV2CommandOutput> {
+    try {
+      await using _ = await this.withOperation(signal);
+      return await this.client.send(
+        new ListObjectsV2Command({
+          Bucket: this.bucket,
+          Prefix: fullPrefix,
+          ContinuationToken: continuationToken,
+          MaxKeys: S3_DELETE_PREFIX_BATCH_SIZE
+        }),
+        { abortSignal: signal }
+      );
+    } catch (error) {
+      throw s3OperationError('list', fullPrefix, error);
+    }
+  }
+
+  private async deleteFullPaths(fullPaths: { Key: string }[], signal?: AbortSignal): Promise<void> {
+    if (fullPaths.length === 0) return;
+    await using _ = await this.withOperation(signal);
+    let response;
+    try {
+      response = await this.client.send(
+        new DeleteObjectsCommand({
+          Bucket: this.bucket,
+          Delete: { Objects: fullPaths, Quiet: true }
+        }),
+        { abortSignal: signal }
+      );
+    } catch (error) {
+      throw s3OperationError('delete', `${fullPaths.length} objects`, error);
+    }
+    if (response.Errors?.length) {
+      const errors = response.Errors.map((error) => `${error.Key}: ${error.Code}`).join(', ');
+      throw new ObjectStorageError(`Failed to delete S3 objects: ${errors}`, {
+        cause: response.Errors,
+        retryable: response.Errors.some((error) => isTransientS3Error({ name: error.Code }))
+      });
+    }
+  }
+
+  private fullPath(path: string): string {
+    const fullPath = this.prefix ? `${this.prefix}/${path}` : path;
+    const size = Buffer.byteLength(fullPath, 'utf8');
+    if (size > SAFE_S3_KEY_BYTES) {
+      throw new Error(`S3 object key is ${size} bytes, exceeding the safe limit of ${SAFE_S3_KEY_BYTES} bytes`);
+    }
+    return fullPath;
+  }
+
+  private async withOperation(signal?: AbortSignal): Promise<AsyncDisposable> {
+    signal?.throwIfAborted();
+    const acquired = signal
+      ? await acquireSemaphoreAbortable(this.operationSemaphore, signal)
+      : await this.operationSemaphore.acquire();
+    if (acquired === 'aborted') {
+      signal!.throwIfAborted();
+      throw new Error('S3 operation aborted while waiting for a concurrency slot');
+    }
+
+    const [, release] = acquired;
+    if (signal?.aborted) {
+      release();
+      signal.throwIfAborted();
+    }
+    return {
+      [Symbol.asyncDispose]: async () => release()
+    };
+  }
+}
+
+function s3OperationError(operation: string, target: string, error: unknown): Error {
+  if (error instanceof ObjectStorageError) {
+    return error;
+  }
+  // Preserve abort errors so callers can distinguish cancellation from an S3
+  // failure and avoid retrying work after shutdown.
+  if (isAbortError(error)) {
+    return error as Error;
+  }
+  const detail = error instanceof Error ? `: ${error.message}` : '';
+  return new ObjectStorageError(`S3 ${operation} failed for ${target}${detail}`, {
+    cause: error,
+    retryable: isTransientS3Error(error)
+  });
+}
+
+function isTransientS3Error(error: unknown): boolean {
+  if (typeof error != 'object' || error == null) {
+    return false;
+  }
+  if (isAbortError(error)) {
+    return false;
+  }
+
+  const sdkError = error as Parameters<typeof isTransientError>[0];
+  // Use the same classification as the AWS SDK retry middleware. The SDK
+  // returns the final error after its own attempt/quota limits are exhausted;
+  // this tells the compactor whether restarting the larger bucket operation is
+  // still appropriate.
+  // One exception is clock skew errors: We don't treat that as retryable by us,
+  // although the SDK may internally retry them.
+  return isThrottlingError(sdkError) || isTransientError(sdkError);
+}

package/src/storage/implementation/v3/source-table-utils.ts

@@ -1,5 +1,5 @@
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { BucketDefinitionMapping, ColumnDescriptor, storage } from '@powersync/service-core';
+import { BucketDefinitionMapping, ColumnDescriptor, JsonValue, storage } from '@powersync/service-core';
 import {
   BucketDataSource,
   BucketDefinitionId,
@@ -38,6 +38,14 @@ export interface SourceTableReconciliationContext {
   syncConfig: HydratedSyncConfig;
   mapping: BucketDefinitionMapping;
   desired: SourceTableDesiredResolution;
+  /**
+   * Candidates the source connector considers safe to reuse.
+   */
+  sourceCompatibleTables: readonly storage.SourceTableCandidate[];
+  /**
+   * Source metadata for records created by this resolution.
+   */
+  newTableSourceMetadata: JsonValue;
 }
 
 export interface SourceTableReconciliationPlan {
@@ -134,9 +142,8 @@ class SourceTableReconciliationPlanner {
   constructor(private readonly context: SourceTableReconciliationContext) {}
 
   plan(candidateDocs: SourceTableDocumentV3[]): SourceTableReconciliationPlan {
-    const { identity } = this.context;
     for (const doc of candidateDocs) {
-      if (matchingSourceTableIdentity(doc, identity)) {
+      if (this.isCompatible(doc)) {
         this.retainDoc(doc);
       }
     }
@@ -145,14 +152,19 @@ class SourceTableReconciliationPlanner {
       tables: this.tables,
       narrowingUpdates: this.narrowingUpdates,
       newTableMemberships: this.newTableMemberships(),
-      // Non-retained overlapping docs represent renames / relation-id changes /
-      // replica-identity changes. Same-identity stale docs are retained in production.
-      dropDocs: candidateDocs.filter(
-        (doc) => !this.retainedDocIds.has(doc._id.toHexString()) && !matchingSourceTableIdentity(doc, identity)
-      )
+      // Compatible records remain available for reuse even when not needed by this config.
+      dropDocs: candidateDocs.filter((doc) => !this.isCompatible(doc))
     };
   }
 
+  private isCompatible(doc: SourceTableDocumentV3) {
+    return this.compatibleTableFor(doc) != null;
+  }
+
+  private compatibleTableFor(doc: SourceTableDocumentV3): storage.SourceTableCandidate | undefined {
+    return this.context.sourceCompatibleTables.find((table) => storage.sourceTableIdEquals(table.id, doc._id));
+  }
+
   private retainDoc(doc: SourceTableDocumentV3) {
     const memberships = intersectMembershipIds(doc, this.context.desired);
     const coversDesiredMembership = hasMembershipIds(memberships);
@@ -237,7 +249,7 @@ class SourceTableReconciliationPlanner {
 
   private sourceTableFor(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds): storage.SourceTable {
     const { connectionTag, syncConfig, mapping, desired, storeCurrentData } = this.context;
-    const table = sourceTableFromDocument(
+    const built = sourceTableFromDocument(
       doc,
       connectionTag,
       syncConfig,
@@ -245,31 +257,14 @@ class SourceTableReconciliationPlanner {
       matchingSourcesFor(desired, memberships),
       memberships
     );
+    // Use any metadata update returned by the reconciler.
+    const resolved = this.compatibleTableFor(doc);
+    const table = resolved == null ? built : built.withSourceMetadata(resolved.sourceMetadata);
     table.storeCurrentData = storeCurrentData;
     return table;
   }
 }
 
-function sameReplicaIdColumns(left: ReplicaIdColumn[] | undefined, right: ReplicaIdColumn[]) {
-  return (
-    left != null &&
-    left.length == right.length &&
-    left.every(
-      (column, index) =>
-        column.name == right[index].name && column.type == right[index].type && column.type_oid == right[index].type_oid
-    )
-  );
-}
-
-function matchingSourceTableIdentity(doc: SourceTableDocumentV3, identity: SourceTableIdentity) {
-  return (
-    doc.schema_name == identity.schema &&
-    doc.table_name == identity.name &&
-    (identity.objectId == null || doc.relation_id == identity.objectId) &&
-    sameReplicaIdColumns(doc.replica_id_columns, identity.replicaIdColumns)
-  );
-}
-
 export function overlappingSourceTableFilter(
   connectionId: number,
   identity: Pick<SourceTableIdentity, 'schema' | 'name' | 'objectId'>
@@ -290,7 +285,16 @@ export function createNewSourceTable(
   memberships: SourceTableMembershipIds,
   context: SourceTableReconciliationContext
 ): NewSourceTable {
-  const { connectionId, connectionTag, identity, syncConfig, mapping, desired, storeCurrentData } = context;
+  const {
+    connectionId,
+    connectionTag,
+    identity,
+    syncConfig,
+    mapping,
+    desired,
+    storeCurrentData,
+    newTableSourceMetadata
+  } = context;
   const doc: SourceTableDocumentV3 = {
     _id: id,
     connection_id: connectionId,
@@ -301,7 +305,9 @@ export function createNewSourceTable(
     snapshot_done: false,
     snapshot_status: undefined,
     bucket_data_source_ids: memberships.bucketDataSourceIds,
-    parameter_lookup_source_ids: memberships.parameterLookupSourceIds
+    parameter_lookup_source_ids: memberships.parameterLookupSourceIds,
+    // Records created together share the same source metadata.
+    source_metadata: newTableSourceMetadata
   };
   const table = sourceTableFromDocument(
     doc,
@@ -368,14 +374,16 @@ export function sourceTableFromDocument(
       name: doc.table_name
     },
     objectId: doc.relation_id,
-    replicaIdColumns: doc.replica_id_columns!.map(
-      (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
-    ),
+    replicaIdColumns:
+      doc.replica_id_columns?.map(
+        (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
+      ) ?? [],
     snapshotComplete: doc.snapshot_done,
     bucketDataSources: resolvedMemberships.bucketDataSources,
     parameterLookupSources: resolvedMemberships.parameterLookupSources,
     bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
-    parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds)
+    parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds),
+    sourceMetadata: doc.source_metadata ?? null
   });
   table.syncData = table.bucketDataSources.length > 0;
   table.syncParameters = table.parameterLookupSources.length > 0;

package/src/types/types.ts

@@ -1,7 +1,23 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
 
+const S3ObjectStorageConfig = t.object({
+  type: t.literal('s3'),
+  bucket: t.string,
+  region: t.string.optional(),
+  prefix: t.string.optional(),
+  endpoint: t.string.optional(),
+  force_path_style: t.boolean.optional(),
+  access_key_id: t.string.optional(),
+  secret_access_key: t.string.optional(),
+  concurrency_limit: t.number.optional(),
+  // Chunks whose BSON-serialized size falls below this byte threshold
+  // stay inline in MongoDB instead of being offloaded to S3. Default 1024.
+  inline_threshold_bytes: t.number.optional()
+});
+
 export const MongoStorageReadPreference = t
   .literal('primary')
   .or(t.literal('primaryPreferred'))
@@ -20,13 +36,38 @@ export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
      *
      * This is an experimental option, and may be removed in a future release.
      */
-    bulk_read_preference: MongoStorageReadPreference.optional()
+    bulk_read_preference: MongoStorageReadPreference.optional(),
+    /**
+     * Throttle clear operations after sync config deploys, by pausing between batches.
+     * The rate produces a pause proportional to the previous batch duration.
+     *
+     * Increase this to reduce impact of clear operations on the storage cluster.
+     * Use 0 to clear as fast as possible.
+     *
+     * Defaults to 0.2.
+     */
+    clear_batch_throttle_rate: t.number.optional(),
+
+    object_storage: S3ObjectStorageConfig.optional()
   })
 );
 
 export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
 export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
 
+export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+
+export function normalizeClearBatchThrottleRate(value: number | undefined): number {
+  const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
+  if (!Number.isFinite(rate) || rate < 0 || rate > 20) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S3201,
+      'storage.clear_batch_throttle_rate must be a finite number between 0 and 20'
+    );
+  }
+  return rate;
+}
+
 export function isMongoStorageConfig(
   config: service_types.configFile.GenericStorageConfig
 ): config is MongoStorageConfig {

package/src/utils/test-utils.ts

@@ -29,7 +29,9 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
       return new MongoBucketStorage(db, {
         replicationStreamNamePrefix: 'test_',
         checksumOptions: factoryOptions.checksumOptions,
-        supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs
+        supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
+        objectStorage: factoryOptions.objectStorage,
+        inlineThresholdBytes: factoryOptions.inlineThresholdBytes
       });
     },
     tableIdStrings: false