@powersync/service-module-mongodb-storage 0.18.3 → 0.19.0

package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts

@@ -0,0 +1,343 @@
+import {
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  ListObjectsV2CommandOutput,
+  PutObjectCommand,
+  S3Client
+} from '@aws-sdk/client-s3';
+import { acquireSemaphoreAbortable, isAbortError } from '@powersync/service-core';
+import { isThrottlingError, isTransientError } from '@smithy/core/retry';
+import { Semaphore, SemaphoreInterface } from 'async-mutex';
+import { ObjectStorageError, type ObjectStorage, type ObjectStoragePutMetadata } from './ObjectStorage.js';
+
+const DEFAULT_S3_OPERATION_CONCURRENCY = 16;
+const S3_DELETE_PREFIX_BATCH_SIZE = 1000;
+/**
+ * Slightly smaller than DEFAULT_S3_OPERATION_CONCURRENCY.
+ */
+const S3_DELETE_PREFIX_CONCURRENCY = 12;
+const MAX_S3_PREFIX_BYTES = 256;
+const SAFE_S3_KEY_BYTES = 896;
+
+export interface S3ObjectStorageOptions {
+  bucket: string;
+  region?: string;
+  prefix?: string;
+  endpoint?: string;
+  forcePathStyle?: boolean;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  concurrencyLimit?: number;
+}
+
+export class S3ObjectStorage implements ObjectStorage {
+  /**
+   * Public for tests only.
+   */
+  public readonly client: S3Client;
+  private bucket: string;
+  private prefix: string;
+  private readonly operationSemaphore: SemaphoreInterface;
+
+  constructor(options: S3ObjectStorageOptions) {
+    const concurrencyLimit = options.concurrencyLimit ?? DEFAULT_S3_OPERATION_CONCURRENCY;
+    if (!Number.isInteger(concurrencyLimit) || concurrencyLimit <= 0) {
+      throw new Error('S3 object storage concurrencyLimit must be a positive integer');
+    }
+    if ((options.accessKeyId == null) !== (options.secretAccessKey == null)) {
+      throw new Error('S3 object storage accessKeyId and secretAccessKey must be configured together');
+    }
+
+    const prefix = options.prefix ?? '';
+    if (Buffer.byteLength(prefix, 'utf8') > MAX_S3_PREFIX_BYTES) {
+      throw new Error(`S3 object storage prefix must be at most ${MAX_S3_PREFIX_BYTES} UTF-8 bytes`);
+    }
+    if (prefix.endsWith('/')) {
+      throw new Error('S3 object storage prefix must not end with "/"');
+    }
+
+    this.bucket = options.bucket;
+    this.prefix = prefix;
+    this.operationSemaphore = new Semaphore(concurrencyLimit);
+    this.client = new S3Client({
+      region: options.region,
+      endpoint: options.endpoint,
+      forcePathStyle: options.forcePathStyle,
+      credentials:
+        options.accessKeyId && options.secretAccessKey
+          ? { accessKeyId: options.accessKeyId, secretAccessKey: options.secretAccessKey }
+          : undefined
+    });
+  }
+
+  async put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void> {
+    const fullPath = this.fullPath(path);
+    await using _ = await this.withOperation();
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: fullPath,
+          Body: data,
+          ContentType: metadata.contentType,
+          ContentEncoding: metadata.contentEncoding ?? undefined
+        })
+      );
+    } catch (error) {
+      throw s3OperationError('upload', fullPath, error);
+    }
+  }
+
+  async get(
+    path: string,
+    options?: { signal?: AbortSignal }
+  ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }> {
+    const fullPath = this.fullPath(path);
+    const signal = options?.signal;
+    await using _ = await this.withOperation(signal);
+    try {
+      const response = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: fullPath
+        }),
+        { abortSignal: signal }
+      );
+      const contentLength = response.ContentLength;
+      if (contentLength == null) {
+        throw new Error(`S3 download response for ${fullPath} is missing ContentLength`);
+      }
+      if (!Number.isSafeInteger(contentLength) || contentLength < 0) {
+        throw new Error(`S3 download response for ${fullPath} has invalid ContentLength ${contentLength}`);
+      }
+
+      const data = new Uint8Array(contentLength);
+      let offset = 0;
+      const stream = response.Body as AsyncIterable<Uint8Array>;
+      for await (const chunk of stream) {
+        signal?.throwIfAborted();
+        const nextOffset = offset + chunk.byteLength;
+        if (nextOffset > contentLength) {
+          throw new Error(
+            `S3 download ContentLength mismatch for ${fullPath}: expected ${contentLength} bytes, received at least ${nextOffset}`
+          );
+        }
+        data.set(chunk, offset);
+        offset = nextOffset;
+      }
+      if (offset !== contentLength) {
+        throw new Error(
+          `S3 download ContentLength mismatch for ${fullPath}: expected ${contentLength} bytes, received ${offset}`
+        );
+      }
+
+      return {
+        data,
+        metadata: {
+          contentType: response.ContentType ?? 'application/octet-stream',
+          contentEncoding: response.ContentEncoding ?? null
+        }
+      };
+    } catch (err: any) {
+      if (err.name === 'NoSuchKey' || err.Code === 'NoSuchKey') {
+        throw new ObjectStorageError(`S3 object not found: ${fullPath}`, { cause: err, retryable: false });
+      }
+      throw s3OperationError('download', fullPath, err);
+    }
+  }
+
+  async *list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string> {
+    const fullPrefix = this.fullPath(prefix);
+    let continuationToken: string | undefined;
+    const signal = options?.signal;
+
+    do {
+      signal?.throwIfAborted();
+      const response = await this.listPage(fullPrefix, continuationToken, signal);
+      for (const object of response.Contents ?? []) {
+        if (object.Key == null) {
+          continue;
+        }
+        yield this.prefix ? object.Key.slice(this.prefix.length + 1) : object.Key;
+      }
+      continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
+      if (response.IsTruncated && continuationToken == null) {
+        throw new Error(`S3 listing for ${fullPrefix} was truncated without a continuation token`);
+      }
+    } while (continuationToken != null);
+  }
+
+  async delete(paths: string[]): Promise<void> {
+    const fullPaths = paths.map((path) => ({ Key: this.fullPath(path) }));
+    await this.deleteFullPaths(fullPaths);
+  }
+
+  async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
+    const fullPrefix = this.fullPath(prefix);
+    const signal = options?.signal;
+    let continuationToken: string | undefined;
+    let objectCount = 0;
+    const deleting = new Set<Promise<void>>();
+    let deleteError: { error: unknown } | undefined;
+
+    const scheduleDelete = (fullPaths: { Key: string }[]) => {
+      let tracked: Promise<void>;
+      tracked = this.deleteFullPaths(fullPaths, signal)
+        .catch((error) => {
+          deleteError ??= { error };
+        })
+        .finally(() => deleting.delete(tracked));
+      deleting.add(tracked);
+    };
+    const throwDeleteError = () => {
+      if (deleteError != null) {
+        throw deleteError.error;
+      }
+    };
+
+    let operationError: { error: unknown } | undefined;
+    try {
+      do {
+        signal?.throwIfAborted();
+        const response = await this.listPage(fullPrefix, continuationToken, signal);
+        throwDeleteError();
+        const fullPaths: { Key: string }[] = [];
+        for (const object of response.Contents ?? []) {
+          if (object.Key != null) {
+            fullPaths.push({ Key: object.Key });
+          }
+        }
+        objectCount += fullPaths.length;
+        if (fullPaths.length !== 0) {
+          while (deleting.size >= S3_DELETE_PREFIX_CONCURRENCY) {
+            await Promise.race(deleting);
+            throwDeleteError();
+          }
+          scheduleDelete(fullPaths);
+        }
+        continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
+        if (response.IsTruncated && continuationToken == null) {
+          throw new Error(`S3 listing for ${fullPrefix} was truncated without a continuation token`);
+        }
+      } while (continuationToken != null);
+    } catch (error) {
+      operationError = { error };
+    }
+
+    await Promise.all(deleting);
+    if (operationError != null) {
+      throw operationError.error;
+    }
+    throwDeleteError();
+    return { objectCount };
+  }
+
+  private async listPage(
+    fullPrefix: string,
+    continuationToken: string | undefined,
+    signal?: AbortSignal
+  ): Promise<ListObjectsV2CommandOutput> {
+    try {
+      await using _ = await this.withOperation(signal);
+      return await this.client.send(
+        new ListObjectsV2Command({
+          Bucket: this.bucket,
+          Prefix: fullPrefix,
+          ContinuationToken: continuationToken,
+          MaxKeys: S3_DELETE_PREFIX_BATCH_SIZE
+        }),
+        { abortSignal: signal }
+      );
+    } catch (error) {
+      throw s3OperationError('list', fullPrefix, error);
+    }
+  }
+
+  private async deleteFullPaths(fullPaths: { Key: string }[], signal?: AbortSignal): Promise<void> {
+    if (fullPaths.length === 0) return;
+    await using _ = await this.withOperation(signal);
+    let response;
+    try {
+      response = await this.client.send(
+        new DeleteObjectsCommand({
+          Bucket: this.bucket,
+          Delete: { Objects: fullPaths, Quiet: true }
+        }),
+        { abortSignal: signal }
+      );
+    } catch (error) {
+      throw s3OperationError('delete', `${fullPaths.length} objects`, error);
+    }
+    if (response.Errors?.length) {
+      const errors = response.Errors.map((error) => `${error.Key}: ${error.Code}`).join(', ');
+      throw new ObjectStorageError(`Failed to delete S3 objects: ${errors}`, {
+        cause: response.Errors,
+        retryable: response.Errors.some((error) => isTransientS3Error({ name: error.Code }))
+      });
+    }
+  }
+
+  private fullPath(path: string): string {
+    const fullPath = this.prefix ? `${this.prefix}/${path}` : path;
+    const size = Buffer.byteLength(fullPath, 'utf8');
+    if (size > SAFE_S3_KEY_BYTES) {
+      throw new Error(`S3 object key is ${size} bytes, exceeding the safe limit of ${SAFE_S3_KEY_BYTES} bytes`);
+    }
+    return fullPath;
+  }
+
+  private async withOperation(signal?: AbortSignal): Promise<AsyncDisposable> {
+    signal?.throwIfAborted();
+    const acquired = signal
+      ? await acquireSemaphoreAbortable(this.operationSemaphore, signal)
+      : await this.operationSemaphore.acquire();
+    if (acquired === 'aborted') {
+      signal!.throwIfAborted();
+      throw new Error('S3 operation aborted while waiting for a concurrency slot');
+    }
+
+    const [, release] = acquired;
+    if (signal?.aborted) {
+      release();
+      signal.throwIfAborted();
+    }
+    return {
+      [Symbol.asyncDispose]: async () => release()
+    };
+  }
+}
+
+function s3OperationError(operation: string, target: string, error: unknown): Error {
+  if (error instanceof ObjectStorageError) {
+    return error;
+  }
+  // Preserve abort errors so callers can distinguish cancellation from an S3
+  // failure and avoid retrying work after shutdown.
+  if (isAbortError(error)) {
+    return error as Error;
+  }
+  const detail = error instanceof Error ? `: ${error.message}` : '';
+  return new ObjectStorageError(`S3 ${operation} failed for ${target}${detail}`, {
+    cause: error,
+    retryable: isTransientS3Error(error)
+  });
+}
+
+function isTransientS3Error(error: unknown): boolean {
+  if (typeof error != 'object' || error == null) {
+    return false;
+  }
+  if (isAbortError(error)) {
+    return false;
+  }
+
+  const sdkError = error as Parameters<typeof isTransientError>[0];
+  // Use the same classification as the AWS SDK retry middleware. The SDK
+  // returns the final error after its own attempt/quota limits are exhausted;
+  // this tells the compactor whether restarting the larger bucket operation is
+  // still appropriate.
+  // One exception is clock skew errors: We don't treat that as retryable by us,
+  // although the SDK may internally retry them.
+  return isThrottlingError(sdkError) || isTransientError(sdkError);
+}

package/src/types/types.ts

@@ -1,7 +1,23 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
 
+const S3ObjectStorageConfig = t.object({
+  type: t.literal('s3'),
+  bucket: t.string,
+  region: t.string.optional(),
+  prefix: t.string.optional(),
+  endpoint: t.string.optional(),
+  force_path_style: t.boolean.optional(),
+  access_key_id: t.string.optional(),
+  secret_access_key: t.string.optional(),
+  concurrency_limit: t.number.optional(),
+  // Chunks whose BSON-serialized size falls below this byte threshold
+  // stay inline in MongoDB instead of being offloaded to S3. Default 1024.
+  inline_threshold_bytes: t.number.optional()
+});
+
 export const MongoStorageReadPreference = t
   .literal('primary')
   .or(t.literal('primaryPreferred'))
@@ -20,13 +36,38 @@ export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
      *
      * This is an experimental option, and may be removed in a future release.
      */
-    bulk_read_preference: MongoStorageReadPreference.optional()
+    bulk_read_preference: MongoStorageReadPreference.optional(),
+    /**
+     * Throttle clear operations after sync config deploys, by pausing between batches.
+     * The rate produces a pause proportional to the previous batch duration.
+     *
+     * Increase this to reduce impact of clear operations on the storage cluster.
+     * Use 0 to clear as fast as possible.
+     *
+     * Defaults to 0.2.
+     */
+    clear_batch_throttle_rate: t.number.optional(),
+
+    object_storage: S3ObjectStorageConfig.optional()
   })
 );
 
 export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
 export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
 
+export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+
+export function normalizeClearBatchThrottleRate(value: number | undefined): number {
+  const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
+  if (!Number.isFinite(rate) || rate < 0 || rate > 20) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S3201,
+      'storage.clear_batch_throttle_rate must be a finite number between 0 and 20'
+    );
+  }
+  return rate;
+}
+
 export function isMongoStorageConfig(
   config: service_types.configFile.GenericStorageConfig
 ): config is MongoStorageConfig {

package/src/utils/test-utils.ts

@@ -29,7 +29,9 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
       return new MongoBucketStorage(db, {
         replicationStreamNamePrefix: 'test_',
         checksumOptions: factoryOptions.checksumOptions,
-        supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs
+        supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
+        objectStorage: factoryOptions.objectStorage,
+        inlineThresholdBytes: factoryOptions.inlineThresholdBytes
       });
     },
     tableIdStrings: false

package/src/utils/util.ts

@@ -1,14 +1,56 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { Logger, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import * as bson from 'bson';
 import * as crypto from 'crypto';
 import * as timers from 'node:timers/promises';
 import * as uuid from 'uuid';
 import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
+import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
 
-export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
+/**
+ * Default and max batch size for clearing.
+ */
+const CLEAR_BATCH_SIZE = 10_000;
+
+/**
+ * Minimum batch size for clearing when a batch takes too long.
+ */
+const CLEAR_MIN_BATCH_SIZE = 100;
+
+/**
+ * We target batch duration to stay between the min and max. If the druation is outside this range,
+ * we increase or decrease the batch size.
+ *
+ * MONGO_CLEAR_OPERATION_TIMEOUT_MS controls the absolute max of individual find and delete operations,
+ * but we try to avoid hitting that.
+ */
+const CLEAR_BATCH_MIN_DURATION_MS = 500;
+const CLEAR_BATCH_MAX_DURATION_MS = 2000;
+
+type ClearCollectionFilter<T extends mongo.Document> = mongo.Filter<T> & {
+  _id: mongo.FilterOperators<mongo.InferIdType<T>>;
+};
+
+function throwIfClearAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
+  }
+}
+
+async function waitWithSignal(delayMs: number, signal: AbortSignal | undefined, abortMessage: string): Promise<void> {
+  try {
+    await timers.setTimeout(delayMs, undefined, { signal });
+  } catch (error) {
+    if (signal?.aborted) {
+      throw new ReplicationAbortedError(abortMessage, signal.reason);
+    }
+    throw error;
+  }
+}
+
+export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.FilterOperators<T> {
   let filter = {
     $gte: {
       ...prefix
@@ -149,16 +191,152 @@ export async function retryOnMongoMaxTimeMSExpired<T>(
     try {
       return await operation();
     } catch (e) {
-      if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
+      if (!lib_mongo.isMaxTimeMSExpiredError(e)) {
         throw e;
       }
       retryCount += 1;
       options.onRetry?.(retryCount);
-      await timers.setTimeout(options.retryDelayMs);
+      await waitWithSignal(options.retryDelayMs, options.signal, options.abortMessage ?? 'Aborted MongoDB operation');
+    }
+  }
+}
+
+async function clearCollectionInBatches(
+  logger: Logger,
+  label: string,
+  findAndDeleteBatch: (batchSize: number) => Promise<number>,
+  signal?: AbortSignal,
+  throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
+): Promise<number> {
+  let batchSize = CLEAR_BATCH_SIZE;
+  let deletedCount = 0;
+  while (true) {
+    throwIfClearAborted(signal);
+    try {
+      const batchStartedAt = performance.now();
+      const foundBatchSize = await findAndDeleteBatch(batchSize);
+      const batchDurationMs = performance.now() - batchStartedAt;
+      deletedCount += foundBatchSize;
+      if (foundBatchSize > 0) {
+        logger.info(
+          `Cleared batch of ${label} (${foundBatchSize} documents) in ${Math.round(batchDurationMs)}ms, continuing...`
+        );
+      }
+      if (foundBatchSize < batchSize) {
+        return deletedCount;
+      }
+      if (batchDurationMs > CLEAR_BATCH_MAX_DURATION_MS) {
+        batchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
+      } else if (batchDurationMs < CLEAR_BATCH_MIN_DURATION_MS) {
+        batchSize = Math.min(CLEAR_BATCH_SIZE, batchSize * 2);
+      }
+      await waitWithSignal(batchDurationMs * throttleRate, signal, 'Aborted clearing data');
+    } catch (error) {
+      if (!lib_mongo.isTimeoutError(error) || batchSize == CLEAR_MIN_BATCH_SIZE) {
+        throw error;
+      }
+      const nextBatchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
+      logger.info(
+        `Clearing batch of ${label} timed out with batch size ${batchSize}, retrying find and delete with ${nextBatchSize}...`
+      );
+      batchSize = nextBatchSize;
+
+      // Pause for at least the equivalent of one query timeout before retrying, since the cause may be high database load.
+      const pauseDuration = Math.max(throttleRate, 1.0) * lib_mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS;
+      await waitWithSignal(pauseDuration, signal, 'Aborted clearing data');
     }
   }
 }
 
+export async function clearCollectionInIdRanges<T extends mongo.Document>(
+  logger: Logger,
+  label: string,
+  collection: mongo.Collection<T>,
+  filter: ClearCollectionFilter<T>,
+  signal?: AbortSignal,
+  throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
+): Promise<number> {
+  return clearCollectionInBatches(
+    logger,
+    label,
+    async (batchSize) => {
+      const queryOptions: mongo.FindOptions<T> = {
+        maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+        projection: { _id: 1 }
+      };
+      const [batchEnd] = await collection
+        .find(filter, queryOptions)
+        .sort({ _id: 1 })
+        .skip(batchSize)
+        .limit(1)
+        .toArray();
+      throwIfClearAborted(signal);
+
+      if (batchEnd == null) {
+        // We're on the last batch
+        const result = await collection.deleteMany(filter, {
+          maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
+        });
+        return result.deletedCount;
+      }
+
+      const idRange: mongo.FilterOperators<mongo.InferIdType<T>> = {
+        ...filter._id,
+        $lt: batchEnd._id
+      };
+      await collection.deleteMany(
+        {
+          ...filter,
+          _id: idRange
+        } as mongo.Filter<T>,
+        { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
+      );
+      return batchSize;
+    },
+    signal,
+    throttleRate
+  );
+}
+
+export async function clearCollectionInIdBatches<T extends mongo.Document>(
+  logger: Logger,
+  label: string,
+  collection: mongo.Collection<T>,
+  filter: mongo.Filter<T>,
+  signal?: AbortSignal,
+  throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE
+): Promise<number> {
+  return clearCollectionInBatches(
+    logger,
+    label,
+    async (batchSize) => {
+      const documents = await collection
+        .find(filter, {
+          maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+          projection: { _id: 1 }
+        })
+        .limit(batchSize)
+        .toArray();
+      throwIfClearAborted(signal);
+
+      if (documents.length === 0) {
+        return 0;
+      }
+      await collection.deleteMany(
+        {
+          _id: {
+            $in: documents.map((document) => document._id)
+          }
+        } as mongo.Filter<T>,
+        { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
+      );
+      return documents.length;
+    },
+    signal,
+    throttleRate
+  );
+}
+
 export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
   query: mongo.Filter<T>,
   collection: mongo.Collection<T>,