@powersync/service-module-mongodb-storage 0.18.2 → 0.19.0

package/src/storage/implementation/v3/bucket-format.ts

@@ -1,20 +1,30 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { bson, InternalOpId } from '@powersync/service-core';
 import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
 import { BucketDataDocumentV3, BucketOperation } from './models.js';
 
-export function serializeBucketData(bucket: string, operations: BucketDataDoc[]): BucketDataDocumentV3 {
+/**
+ * Serialize a non-empty list of operations in strictly ascending `o` order.
+ * Preserving that order establishes the {@link BucketDataDocumentV3} range
+ * invariants used by metadata-only queries.
+ */
+export function serializeBucketData(
+  bucket: string,
+  operations: BucketDataDoc[],
+  options?: { targetOp?: InternalOpId | null }
+): BucketDataDocumentV3 {
   const minOp = operations[0].o;
   const maxOp = operations[operations.length - 1].o;
 
   let totalChecksum = 0n;
-  let totalSize = 0;
-  let maxTargetOp: bigint | null = null;
+  let maxTargetOp: InternalOpId | null = options?.targetOp ?? null;
+  let hasClearOp = false;
 
   const ops: BucketOperation[] = operations.map((op) => {
     totalChecksum += op.checksum;
-    totalSize += op.data?.length ?? 0;
 
-    if (op.target_op != null && (maxTargetOp == null || op.target_op > maxTargetOp)) {
-      maxTargetOp = op.target_op;
+    if (op.op == 'CLEAR') {
+      hasClearOp = true;
     }
 
     return {
@@ -29,6 +39,8 @@ export function serializeBucketData(bucket: string, operations: BucketDataDoc[])
     };
   });
 
+  const size = bson.calculateObjectSize(ops);
+
   return {
     _id: {
       b: bucket,
@@ -37,8 +49,9 @@ export function serializeBucketData(bucket: string, operations: BucketDataDoc[])
     min_op: minOp,
     checksum: totalChecksum,
     count: operations.length,
-    size: totalSize,
+    size,
     target_op: maxTargetOp,
+    ...(hasClearOp ? { has_clear_op: true as const } : {}),
     ops
   };
 }
@@ -48,6 +61,11 @@ export function* loadBucketDataDocument(
   doc: BucketDataDocumentV3
 ): Generator<BucketDataDoc> {
   const { _id, ops } = doc;
+  if (!ops) {
+    throw new ServiceAssertionError(
+      `Missing ops array on BucketDataDocumentV3 at _id.o=${_id.o}. Callers must patch doc.ops from S3 before calling this function.`
+    );
+  }
   const bucketKey = {
     ...context,
     bucket: _id.b
@@ -63,8 +81,17 @@ export function* loadBucketDataDocument(
       table: op.table,
       row_id: op.row_id,
       checksum: op.checksum,
-      data: op.data,
-      target_op: doc.target_op ?? null
+      data: op.data
     };
   }
 }
+
+export function maxOpId(a: InternalOpId | null | undefined, b: InternalOpId | null | undefined): InternalOpId | null {
+  if (a == null) {
+    return b ?? null;
+  }
+  if (b == null) {
+    return a ?? null;
+  }
+  return a > b ? a : b;
+}

package/src/storage/implementation/v3/chunking.ts

@@ -1,3 +1,4 @@
+import { bson } from '@powersync/service-core';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
 
 export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
@@ -18,7 +19,8 @@ export function chunkBucketData(
   let currentSize = 0;
 
   for (const op of operations) {
-    const opSize = op.data?.length ?? 0;
+    // Add 5 to account for array index up to 4 digits (9999) plus null byte.
+    const opSize = bson.calculateObjectSize(op) + 5;
 
     if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
       chunks.push(currentChunk);

package/src/storage/implementation/v3/models.ts

@@ -176,14 +176,53 @@ export interface BucketOperation {
   data: string | null;
 }
 
+export interface StorageRef {
+  path: string;
+  file_size: number;
+}
+
+/** An S3 object that may be deleted once its grace period has elapsed. */
+export interface ObjectStorageDeletionMarker {
+  _id: bson.ObjectId;
+  path: string;
+  delete_after: Date;
+}
+
+/**
+ * A non-empty chunk of a bucket's ordered operation stream.
+ *
+ * All writers maintain these invariants:
+ *
+ * - `ops` is non-empty and strictly ordered by `o`.
+ * - `min_op` is `ops[0].o` and `_id.o` is `ops[ops.length - 1].o`.
+ * - Documents for the same bucket have ordered, disjoint operation ranges,
+ *   although gaps between ranges are allowed.
+ * - `checksum` is the sum of every operation checksum and `count` is
+ *   `ops.length`.
+ *
+ * Metadata-only checksum and data queries rely on these invariants to identify
+ * boundary-straddling documents without reading `ops`.
+ */
 export interface BucketDataDocumentV3 {
   _id: BucketDataKey;
   min_op: bigint;
   checksum: bigint;
   count: number;
   size: number;
+  /**
+   * The greatest operation boundary that influenced this document's contents.
+   * Bucket-data reads propagate it so serving an earlier checkpoint can be
+   * invalidated when compaction changed the operations being returned.
+   */
   target_op?: bigint | null;
-  ops: BucketOperation[];
+  ops?: BucketOperation[];
+  storage_ref?: StorageRef;
+  /**
+   * Present (and always true) when this document contains a CLEAR operation.
+   * In that case, this is the first document in the bucket: all preceding
+   * documents have been removed.
+   */
+  has_clear_op?: true;
 }
 
 export function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary {

package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts

@@ -0,0 +1,54 @@
+import { storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { BucketDataDocumentV3, BucketOperation } from '../models.js';
+import { ObjectStorage } from './ObjectStorage.js';
+
+export class BucketDataObjectStorage {
+  constructor(private readonly storage: ObjectStorage) {}
+
+  async store(path: string, ops: BucketOperation[]): Promise<{ fileSize: number }> {
+    const bsonBuffer = bson.serialize({ ops });
+    await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
+    return { fileSize: bsonBuffer.byteLength };
+  }
+
+  async retrieve(path: string, options: { signal?: AbortSignal }): Promise<BucketOperation[]> {
+    const { data, metadata } = await this.storage.get(path, { signal: options.signal });
+    if (metadata.contentEncoding != null) {
+      throw new Error(`Unexpected content encoding: ${metadata.contentEncoding}`);
+    }
+    if (metadata.contentType !== 'application/bson') {
+      throw new Error(`Unexpected content type: ${metadata.contentType}`);
+    }
+    const wrapper = bson.deserialize(data, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
+    return wrapper.ops;
+  }
+
+  async delete(paths: string[]): Promise<void> {
+    return this.storage.delete(paths);
+  }
+}
+
+/**
+ * Load offloaded operations and patch them onto their MongoDB metadata documents.
+ * The object-storage implementation is responsible for limiting request
+ * concurrency across all callers.
+ */
+export async function hydrateBucketDataDocuments(
+  documents: BucketDataDocumentV3[],
+  objectStorage: ObjectStorage | undefined,
+  options: { signal?: AbortSignal }
+): Promise<void> {
+  if (!objectStorage) {
+    return;
+  }
+
+  options.signal?.throwIfAborted();
+  const store = new BucketDataObjectStorage(objectStorage);
+  const storedDocuments = documents.filter((document) => document.storage_ref);
+  await Promise.all(
+    storedDocuments.map(async (document) => {
+      document.ops = await store.retrieve(document.storage_ref!.path, { signal: options.signal });
+    })
+  );
+}

package/src/storage/implementation/v3/object-storage/ObjectStorage.ts

@@ -0,0 +1,29 @@
+export interface ObjectStoragePutMetadata {
+  contentType: string;
+  contentEncoding: string | null;
+}
+
+export class ObjectStorageError extends Error {
+  readonly retryable: boolean;
+
+  constructor(message: string, options: { cause: unknown; retryable: boolean }) {
+    super(message, { cause: options.cause });
+    this.name = 'ObjectStorageError';
+    this.retryable = options.retryable;
+  }
+}
+
+export function isRetryableObjectStorageError(error: unknown): error is ObjectStorageError {
+  return error instanceof ObjectStorageError && error.retryable;
+}
+
+export interface ObjectStorage {
+  put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
+  get(
+    path: string,
+    options?: { signal?: AbortSignal }
+  ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }>;
+  list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string>;
+  delete(paths: string[]): Promise<void>;
+  deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }>;
+}

package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts

@@ -0,0 +1,171 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
+import * as bson from 'bson';
+import { createHash, randomUUID } from 'node:crypto';
+import { ObjectStorageDeletionMarker } from '../models.js';
+import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
+import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
+import { ObjectStorage } from './ObjectStorage.js';
+
+export const OBJECT_STORAGE_UPLOAD_LEASE_MS = 60 * 60 * 1000;
+export const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS = 2 * 60 * 1000;
+export const OBJECT_STORAGE_REFERENCE_GRACE_MS = 15 * 60 * 1000;
+/**
+ * Protect against arbitrary-long bucket names and parameters exceeding the S3 key length limits.
+ *
+ * Any bucket name over this length is hashed.
+ */
+const MAX_READABLE_BUCKET_SEGMENT_BYTES = 128;
+
+export interface PreparedObjectStorageUpload {
+  markerId: bson.ObjectId;
+  path: string;
+  deleteAfter: Date;
+}
+
+/**
+ * Coordinates the MongoDB outbox used to make object-storage changes recoverable.
+ *
+ * A MongoDB collection is used to track files that will be uploaded, or should be deleted.
+ * By using a MongoDB collection, we can use the same transaction that creates or removes
+ * references to these files, making the operations atomic.
+ *
+ * These markers indicate which files should be deleted - either by being orphaned during create,
+ * or after references to them are removed.
+ */
+export class ObjectStorageLifecycle {
+  readonly bucketData: BucketDataObjectStorage;
+
+  constructor(
+    private readonly db: VersionedPowerSyncMongoV3,
+    private readonly replicationStreamId: number,
+    private readonly objectStorage: ObjectStorage
+  ) {
+    this.bucketData = new BucketDataObjectStorage(objectStorage);
+  }
+
+  streamPrefix(): string {
+    return `bucket-data/${this.replicationStreamId}/`;
+  }
+
+  definitionPrefix(definitionId: string): string {
+    return `${this.streamPrefix()}${definitionId}/`;
+  }
+
+  allocatePath(definitionId: string, bucket: string, minOp: bigint, maxOp: bigint): string {
+    // Preserve readable paths for normal buckets, but bound user-controlled
+    // parameter values so they cannot consume the S3 key-length budget.
+    const escapedBucket = encodeURIComponent(bucket);
+    const bucketSegment =
+      Buffer.byteLength(escapedBucket, 'utf8') <= MAX_READABLE_BUCKET_SEGMENT_BYTES
+        ? escapedBucket
+        : `sha256-${createHash('sha256').update(bucket).digest('base64url')}`;
+    return `${this.definitionPrefix(definitionId)}${bucketSegment}/${minOp}-${maxOp}-${randomUUID()}.bson`;
+  }
+
+  /**
+   * Indicate that files will be uploaded to S3.
+   *
+   * These markers are expected to be removed using publishUploads(). If that does not happen within
+   * the grace period, the files will be deleted.
+   */
+  async prepareUploads(paths: string[], now = new Date()): Promise<PreparedObjectStorageUpload[]> {
+    const deleteAfter = new Date(now.getTime() + OBJECT_STORAGE_UPLOAD_LEASE_MS);
+    const uploads = paths.map((path) => ({ markerId: new bson.ObjectId(), path, deleteAfter }));
+    if (uploads.length) {
+      await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(
+        uploads.map((upload) => ({
+          _id: upload.markerId,
+          path: upload.path,
+          delete_after: upload.deleteAfter
+        }))
+      );
+    }
+    return uploads;
+  }
+
+  canPublish(upload: PreparedObjectStorageUpload, now = new Date()): boolean {
+    return now.getTime() < upload.deleteAfter.getTime() - OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
+  }
+
+  /**
+   * Remove markers for files that have finished uploading, and are now referenced from other documents.
+   *
+   * Call this in the same transaction as the one creating the references.
+   */
+  async publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void> {
+    for (const upload of uploads) {
+      if (!this.canPublish(upload)) {
+        throw new Error(`Object storage publication lease expired for ${upload.path}`);
+      }
+    }
+    if (uploads.length === 0) {
+      return;
+    }
+
+    const result = await this.db
+      .pendingObjectStorageDeletes(this.replicationStreamId)
+      .deleteMany({ _id: { $in: uploads.map((upload) => upload.markerId) } }, { session });
+    if (result.deletedCount !== uploads.length) {
+      throw new Error(`Missing object storage publication markers`);
+    }
+  }
+
+  /**
+   * Mark old paths for deletion.
+   *
+   * By marking for deletion instead of immediately deleting:
+   * 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
+   *    be persisted transactionally, while the deletes happen asynchronously and can be retried.
+   * 2. The deletes are delayed, giving clients a grace period to continue using the files for
+   *    in-progress requests.
+   *
+   * Call this in the same transaction as the one removing the references to these files.
+   */
+  async retire(paths: Iterable<string>, session: mongo.ClientSession, now = new Date()): Promise<void> {
+    const markers: ObjectStorageDeletionMarker[] = Array.from(paths, (path) => ({
+      _id: new bson.ObjectId(),
+      path,
+      delete_after: new Date(now.getTime() + OBJECT_STORAGE_REFERENCE_GRACE_MS)
+    }));
+    if (markers.length) {
+      await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(markers, { session });
+    }
+  }
+
+  /**
+   * Discover and delete objects by their storage prefix
+   * without touching the database.
+   */
+  async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
+    return this.objectStorage.deletePrefix(prefix, { signal: options?.signal });
+  }
+
+  async cleanup(logger: Logger): Promise<void> {
+    const markers = this.db.pendingObjectStorageDeletes(this.replicationStreamId);
+    const cursor = markers.find({ delete_after: { $lte: new Date() } });
+    let batch: ObjectStorageDeletionMarker[] = [];
+
+    const deleteBatch = async () => {
+      if (batch.length === 0) {
+        return;
+      }
+      const deleting = batch;
+      batch = [];
+      try {
+        await this.bucketData.delete(deleting.map((marker) => marker.path));
+        await markers.deleteMany({ _id: { $in: deleting.map((marker) => marker._id) } });
+      } catch (error) {
+        logger.warn('Failed to clean up object storage deletion markers; will retry during the next compaction', error);
+      }
+    };
+
+    for await (const marker of cursor) {
+      batch.push(marker);
+      if (batch.length === 500) {
+        await deleteBatch();
+      }
+    }
+    await deleteBatch();
+  }
+}