@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

package/src/storage/implementation/v3/compact-utils.ts

@@ -0,0 +1,329 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { addChecksums, InternalOpId } from '@powersync/service-core';
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
+import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js';
+import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js';
+
+/**
+ * Perform chunk compaction if at least this many chunks have been added since
+ * the latest compaction.
+ *
+ * A lower value increases compaction frequency and can increase chunk rewrites
+ * and object-storage operations. A higher value leaves more small chunks for
+ * longer, which can hurt sync performance.
+ */
+const MERGE_CHUNKS_THRESHOLD = 8;
+
+export const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000;
+
+export interface PendingCompactionGroup {
+  /**
+   * Input documents are ordered from oldest to newest, matching `ops`.
+   * Keeping the inputs intact lets unchanged singletons retain their object.
+   */
+  inputs: BucketDataDocumentV3[];
+  ops: BucketDataDoc[];
+  changed: boolean;
+  targetOp: InternalOpId | null;
+}
+
+export interface CompactIntervalConfig {
+  readonly minCompactChunkIntervalMs: number;
+  readonly minCompactFullIntervalMs: number;
+  readonly maxCompactFullIntervalMs: number;
+}
+export interface CompactTargetConfig {
+  readonly maxOpIdCap: InternalOpId | undefined;
+  readonly forceChunkCompaction?: boolean;
+}
+
+export enum CompactionKind {
+  Full = 'full',
+  Chunks = 'chunks'
+}
+
+export interface CompactionDecision {
+  kind: CompactionKind | null;
+  nextCompactCheck: mongo.Document;
+}
+
+export interface ScheduledCompactionOptions {
+  /** Process checks scheduled this far after the captured job start. */
+  dueAheadMs?: number;
+  /** Used by initial replication, which must not run a full compact. */
+  requestedKind?: CompactionKind;
+}
+
+export class CompactionContext {
+  constructor(
+    readonly lease: CompactionLease,
+    readonly kind: CompactionKind,
+    readonly decision: CompactionDecision,
+    readonly rescheduleNotBefore: Date | undefined,
+    readonly targetOp: InternalOpId
+  ) {}
+
+  get state() {
+    return this.lease.state;
+  }
+
+  get startedAt() {
+    return this.lease.startedAt;
+  }
+
+  get lastOp() {
+    return this.lease.lastOp;
+  }
+}
+
+export interface BucketStats {
+  count: number;
+  bytes: bigint;
+  chunks: number;
+}
+
+/** Bucket stats read from bucket-data documents, including their checksum. */
+export interface BucketStatsWithChecksum extends BucketStats {
+  checksum: number;
+}
+
+export interface CompactionResult {
+  /** Metadata cached at compacted_state.op_id. */
+  compactedState: BucketStatsWithChecksum;
+  /** Complete bucket metadata through the op head captured at claim time. */
+  bucketStats: BucketStats;
+}
+
+export function bucketStats(state: BucketStateDocumentV3): BucketStats {
+  return {
+    count: state.bucket_stats.count,
+    bytes: state.bucket_stats.bytes,
+    chunks: state.bucket_stats.chunks
+  };
+}
+
+export function emptyBucketStats(): BucketStatsWithChecksum {
+  return { count: 0, bytes: 0n, chunks: 0, checksum: 0 };
+}
+
+export function statsForDocument(
+  document: Pick<BucketDataDocumentV3, 'count' | 'size' | 'checksum'>
+): BucketStatsWithChecksum {
+  return {
+    count: document.count,
+    bytes: BigInt(document.size),
+    chunks: 1,
+    checksum: addChecksums(0, Number(document.checksum))
+  };
+}
+
+export function statsForDocuments(
+  documents: Iterable<Pick<BucketDataDocumentV3, 'count' | 'size' | 'checksum'>>
+): BucketStatsWithChecksum {
+  let result = emptyBucketStats();
+  for (const document of documents) {
+    result = combineAdjacentStats(result, statsForDocument(document));
+  }
+  return result;
+}
+
+/** A scheduled bucket always has writes awaiting a full compact. */
+export function firstUncompactedWrite(state: BucketStateDocumentV3): Date {
+  if (state.first_uncompacted_write == null) {
+    throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`);
+  }
+  return state.first_uncompacted_write;
+}
+
+export function combineChunkStats(
+  previous: NonNullable<BucketStateDocumentV3['compacted_state']>,
+  compactedTail: BucketStatsWithChecksum,
+  overlappingCompactedChunk: BucketStatsWithChecksum
+): BucketStatsWithChecksum {
+  return {
+    count: previous.count - overlappingCompactedChunk.count + compactedTail.count,
+    bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes,
+    chunks: previous.chunks - 1 + compactedTail.chunks,
+    checksum: addChecksums(
+      addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum),
+      compactedTail.checksum
+    )
+  };
+}
+
+export function combineAdjacentStats(
+  first: BucketStatsWithChecksum,
+  second: BucketStatsWithChecksum
+): BucketStatsWithChecksum {
+  return {
+    count: first.count + second.count,
+    bytes: first.bytes + second.bytes,
+    chunks: first.chunks + second.chunks,
+    checksum: addChecksums(first.checksum, second.checksum)
+  };
+}
+
+export function applyCompactionDelta(
+  total: BucketStats,
+  before: BucketStatsWithChecksum,
+  after: BucketStatsWithChecksum
+): BucketStats {
+  return {
+    count: total.count - before.count + after.count,
+    bytes: total.bytes - before.bytes + after.bytes,
+    chunks: total.chunks - before.chunks + after.chunks
+  };
+}
+
+/** Replace one stored range in an accumulated statistics snapshot. */
+export function applyStatsReplacement(
+  total: BucketStatsWithChecksum,
+  before: BucketStatsWithChecksum,
+  after: BucketStatsWithChecksum
+): BucketStatsWithChecksum {
+  return {
+    ...applyCompactionDelta(total, before, after),
+    checksum: addChecksums(addChecksums(total.checksum, -before.checksum), after.checksum)
+  };
+}
+
+/**
+ * This checks that the bucket state hasn't changed by a concurrent write since we checked.
+ *
+ * If it has changed, we'll re-check in the next batch.
+ */
+export function unclaimedSnapshotFilter(state: BucketStateDocumentV3): mongo.Filter<BucketStateDocumentV3> {
+  return {
+    _id: state._id,
+    last_op: state.last_op,
+    next_compact_check: state.next_compact_check,
+    first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false },
+    bucket_stats: state.bucket_stats,
+    compacted_state: state.compacted_state ?? { $exists: false },
+    last_full_compact: state.last_full_compact ?? { $exists: false },
+    ...AVAILABLE_LEASE_EXPR
+  };
+}
+
+/**
+ * Calculate the earliest full compaction time from the first uncompacted
+ * write, bounded by the maximum retention interval.
+ */
+export function fullCompactionCheckAt(state: BucketStateDocumentV3, config: CompactIntervalConfig): Date {
+  const firstWrite = firstUncompactedWrite(state);
+  const stats = bucketStats(state);
+  const lastFull = state.last_full_compact;
+
+  // The number of operations since the last full compact.
+  // We may make this more specific in the future, to track new updates and deletes only, ignoring
+  // full new inserts, but that requires more granular tracking when replicating.
+  const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count);
+  const compactedRows = lastFull?.puts ?? 0;
+
+  // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs.
+  // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2.
+  // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11.
+  // If every row has been updated multiple times, the ratio tends closer to 1 again.
+  const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount);
+  const fullIntervalMs = ratio == 0 ? config.maxCompactFullIntervalMs : config.minCompactFullIntervalMs / ratio;
+  return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, config.maxCompactFullIntervalMs));
+}
+
+export function chooseCompactionKind(
+  state: BucketStateDocumentV3,
+  now: Date,
+  config: CompactIntervalConfig
+): CompactionDecision {
+  // For chunk compaction, we consider the number of chunks added.
+  // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added.
+  // A future policy could also use bytes per chunk or records per chunk to
+  // decide when to compact.
+  const fullCheckAt = fullCompactionCheckAt(state, config);
+  // Schedule a little late so a worker using a slightly earlier clock does
+  // not wake before the exact full-compaction condition is true.
+  const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS);
+  const compacted = state.compacted_state;
+  const shouldCompactChunks = hasEnoughChunksToMerge(state);
+  const canCheckChunks =
+    compacted == null || now.getTime() - compacted.at.getTime() >= config.minCompactChunkIntervalMs;
+  // Too few new chunks cannot make chunk compaction eligible. Do not poll this
+  // bucket at the chunk-compaction interval; only wake it for its full-compact check.
+  const nextCompactCheck: mongo.Document = !shouldCompactChunks
+    ? fullCheckWithMargin
+    : {
+        $min: [
+          fullCheckWithMargin,
+          { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: config.minCompactChunkIntervalMs } }
+        ]
+      };
+  let kind: CompactionKind | null = null;
+  if (now >= fullCheckAt) {
+    kind = CompactionKind.Full;
+  } else if (canCheckChunks && shouldCompactChunks) {
+    kind = CompactionKind.Chunks;
+  }
+  return {
+    kind,
+    nextCompactCheck
+  };
+}
+
+export function chooseRequestedCompactionKind(
+  state: BucketStateDocumentV3,
+  requestedKind: CompactionKind | undefined,
+  config: CompactTargetConfig
+): CompactionKind | null {
+  if (requestedKind == null) {
+    return null;
+  }
+  // Initial replication bypasses the interval and never runs full compaction,
+  // but small buckets should still use the normal bulk-rescheduling path.
+  if (requestedKind == CompactionKind.Chunks && !config.forceChunkCompaction && !hasEnoughChunksToMerge(state)) {
+    return null;
+  }
+  const maxOpId = config.maxOpIdCap == null || state.last_op < config.maxOpIdCap ? state.last_op : config.maxOpIdCap;
+  return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? requestedKind : null;
+}
+
+function hasEnoughChunksToMerge(state: BucketStateDocumentV3): boolean {
+  return state.bucket_stats.chunks - (state.compacted_state?.chunks ?? 0) >= MERGE_CHUNKS_THRESHOLD;
+}
+
+/**
+ * Read one bounded prefix from a compaction cursor.
+ *
+ * The document that would cross the byte limit is deliberately not returned:
+ * pagination resumes past the last returned `_id`, so that document remains
+ * eligible for the next query. The first document is always accepted to
+ * ensure progress when a single document exceeds the configured byte limit.
+ *
+ * `hasMore` is conservative when the document limit is reached. An extra empty
+ * query is preferable to exhausting the cursor just to determine whether the
+ * limited MongoDB query contained another document.
+ */
+export async function readCompactionBatch(
+  cursor: mongo.AggregationCursor<BucketDataDocumentV3>,
+  options: { byteLimit: number; documentLimit: number }
+): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> {
+  const documents: BucketDataDocumentV3[] = [];
+  let cumulativeBytes = 0;
+
+  try {
+    for await (const document of cursor) {
+      if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) {
+        return { documents, hasMore: true };
+      }
+
+      documents.push(document);
+      cumulativeBytes += document.size;
+
+      if (documents.length >= options.documentLimit) {
+        return { documents, hasMore: true };
+      }
+    }
+    return { documents, hasMore: false };
+  } finally {
+    await cursor.close();
+  }
+}

package/src/storage/implementation/v3/compaction-constants.ts

@@ -0,0 +1,5 @@
+/**
+ * Writers use this delay before scheduling their first chunk compaction.
+ * Initial replication includes the same delay when it picks up that work.
+ */
+export const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000;

package/src/storage/implementation/v3/models.ts

@@ -1,5 +1,6 @@
 import {
   InternalOpId,
+  JsonValue,
   PersistedDefinitionMapping,
   SerializedSyncPlan,
   SyncRuleState,
@@ -7,6 +8,7 @@ import {
 } from '@powersync/service-core';
 import {
   BucketDefinitionId,
+  EventDefinitionId,
   ParameterIndexId,
   ScopedParameterLookup,
   SqliteJsonValue
@@ -15,7 +17,6 @@ import * as bson from 'bson';
 import {
   BucketDataKey,
   BucketParameterDocumentBase,
-  BucketStateDocumentBase,
   CurrentBucket,
   OpType,
   ReplicaId,
@@ -94,7 +95,7 @@ export interface SyncConfigDefinition {
   created_at: Date;
   storage_version: number;
   /**
-   * The related SyncRuleDocumentV3.
+   * The related ReplicationStreamDocumentV3.
    *
    * Note that a specific sync config definition never moves between replication streams. Instead, we can create a new copy for the new replication stream.
    *
@@ -103,6 +104,8 @@ export interface SyncConfigDefinition {
   replication_stream_id: number;
 
   content: string;
+  /** Optional operator-supplied label identifying this sync config version. */
+  version_label?: string;
   serialized_plan?: SerializedSyncPlan | null;
 
   rule_mapping: PersistedDefinitionMapping;
@@ -156,13 +159,68 @@ export interface SourceTableDocumentV3 {
   snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
   bucket_data_source_ids: BucketDefinitionId[];
   parameter_lookup_source_ids: ParameterIndexId[];
+  /**
+   * Stream-local ids of the compiled event definitions evaluated by this source table.
+   */
+  event_definition_ids: EventDefinitionId[];
   latest_pending_delete?: InternalOpId | undefined;
+  /**
+   * Source-specific metadata. Absent for legacy records.
+   */
+  source_metadata?: JsonValue;
 }
 
-export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
-  _id: BucketStateDocumentBase['_id'] & {
+export interface BucketStateDocumentV3 {
+  _id: {
+    b: string;
     d: BucketDefinitionId;
   };
+
+  /** Must always identify an actual operation in this logical stream. */
+  last_op: bigint;
+
+  /** The next time a compact worker should inspect this bucket. */
+  next_compact_check: Date | undefined;
+  /**
+   * Scheduling epoch for work not covered by a full compact. Writers set the
+   * first actual write time; a partial full compact advances it to completion.
+   */
+  first_uncompacted_write: Date | undefined;
+
+  /**
+   * A checksum cache and the statistics captured by the latest compact (full
+   * or chunk). Keeping these separate from bucket_stats lets writers only
+   * update one set of counters.
+   */
+  compacted_state?: {
+    op_id: InternalOpId;
+    checksum: bigint;
+    count: number;
+    bytes: bigint;
+    at: Date;
+    chunks: number;
+  };
+
+  /** Statistics for the prefix covered by the most recent full compact. */
+  last_full_compact?: {
+    op_id: InternalOpId;
+    count: number;
+    at: Date;
+    puts: number;
+  };
+
+  /** Current aggregate bucket statistics, maintained by writers and compactors. */
+  bucket_stats: {
+    count: number;
+    bytes: bigint;
+    chunks: number;
+  };
+
+  /** A short-lived ownership marker used to distribute bucket compaction. */
+  compact_lease?: {
+    expires_at: Date;
+    id: unknown;
+  };
 }
 
 export interface BucketOperation {
@@ -170,6 +228,7 @@ export interface BucketOperation {
   op: OpType;
   source_table?: bson.ObjectId;
   source_key?: ReplicaId;
+  subkey?: string;
   table?: string;
   row_id?: string;
   checksum: bigint;
@@ -181,6 +240,15 @@ export interface StorageRef {
   file_size: number;
 }
 
+export interface ObjectStorageUsageDocument {
+  _id: {
+    g: number;
+    w: string;
+  };
+  updated_at: Date;
+  definitions: Record<BucketDefinitionId, bigint>;
+}
+
 /** An S3 object that may be deleted once its grace period has elapsed. */
 export interface ObjectStorageDeletionMarker {
   _id: bson.ObjectId;
@@ -225,6 +293,31 @@ export interface BucketDataDocumentV3 {
   has_clear_op?: true;
 }
 
+/**
+ * v3 Custom checkpoints (new name for legacy write checkpoints) are related to
+ * a replication stream and a specific event (in a sync config) which created them.
+ * Unchanged events, between sync configs, can share the documents.
+ * For v3, we store each replication stream's custom checkpoint requests in their own collection:
+ * `custom_checkpoint_requests_${replicationStreamId}_${eventId}`
+ */
+export interface CustomCheckpointRequestDocumentV3 {
+  _id: bson.ObjectId;
+  user_id: string;
+  checkpoint: bigint;
+  /**
+   * Unlike managed write checkpoints, custom write checkpoints are flushed together with
+   * normal ops. This means we can assign an op_id for ordering / correlating with read checkpoints.
+   *
+   * This is not unique - multiple write checkpoints can have the same op_id.
+   */
+  op_id?: InternalOpId;
+  /**
+   * Set when this checkpoint was created from a client-supplied checkpoint
+   * request rather than a persistent custom write checkpoint.
+   */
+  checkpoint_requested_at?: Date | null;
+}
+
 export function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary {
   return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
 }

package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts

@@ -1,18 +1,22 @@
-import { storage } from '@powersync/service-core';
+import { PerformanceTracer, storage } from '@powersync/service-core';
 import * as bson from 'bson';
 import { BucketDataDocumentV3, BucketOperation } from '../models.js';
-import { ObjectStorage } from './ObjectStorage.js';
+import { ObjectStorage, ObjectStorageOperationOptions } from './ObjectStorage.js';
 
 export class BucketDataObjectStorage {
   constructor(private readonly storage: ObjectStorage) {}
 
-  async store(path: string, ops: BucketOperation[]): Promise<{ fileSize: number }> {
+  async store(
+    path: string,
+    ops: BucketOperation[],
+    options?: ObjectStorageOperationOptions
+  ): Promise<{ fileSize: number }> {
     const bsonBuffer = bson.serialize({ ops });
-    await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
+    await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null }, options);
     return { fileSize: bsonBuffer.byteLength };
   }
 
-  async retrieve(path: string, options: { signal?: AbortSignal }): Promise<BucketOperation[]> {
+  async retrieve(path: string, options: ObjectStorageOperationOptions): Promise<BucketOperation[]> {
     const { data, metadata } = await this.storage.get(path, { signal: options.signal });
     if (metadata.contentEncoding != null) {
       throw new Error(`Unexpected content encoding: ${metadata.contentEncoding}`);
@@ -24,8 +28,8 @@ export class BucketDataObjectStorage {
     return wrapper.ops;
   }
 
-  async delete(paths: string[]): Promise<void> {
-    return this.storage.delete(paths);
+  async delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void> {
+    return this.storage.delete(paths, options);
   }
 }
 
@@ -37,7 +41,7 @@ export class BucketDataObjectStorage {
 export async function hydrateBucketDataDocuments(
   documents: BucketDataDocumentV3[],
   objectStorage: ObjectStorage | undefined,
-  options: { signal?: AbortSignal }
+  options: ObjectStorageOperationOptions & { tracer?: PerformanceTracer<string> }
 ): Promise<void> {
   if (!objectStorage) {
     return;
@@ -46,9 +50,20 @@ export async function hydrateBucketDataDocuments(
   options.signal?.throwIfAborted();
   const store = new BucketDataObjectStorage(objectStorage);
   const storedDocuments = documents.filter((document) => document.storage_ref);
-  await Promise.all(
+  if (storedDocuments.length == 0) {
+    return;
+  }
+  using _ = options?.tracer?.span('s3', 'read');
+  const results = await Promise.allSettled(
     storedDocuments.map(async (document) => {
       document.ops = await store.retrieve(document.storage_ref!.path, { signal: options.signal });
     })
   );
+  // Drain every download before propagating failure so callers can release their
+  // worker slot and payloads without a sibling download hydrating a document later.
+  for (const result of results) {
+    if (result.status === 'rejected') {
+      throw result.reason;
+    }
+  }
 }

package/src/storage/implementation/v3/object-storage/ObjectStorage.ts

@@ -17,13 +17,28 @@ export function isRetryableObjectStorageError(error: unknown): error is ObjectSt
   return error instanceof ObjectStorageError && error.retryable;
 }
 
+export interface ObjectStorageOperationOptions {
+  /**
+   * Cancels the operation, including while it waits for a concurrency slot.
+   *
+   * Operations are bounded by their own timeouts regardless, but a signal releases the slot as
+   * soon as the caller no longer needs the result.
+   */
+  signal?: AbortSignal;
+}
+
 export interface ObjectStorage {
-  put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
+  put(
+    path: string,
+    data: Uint8Array,
+    metadata: ObjectStoragePutMetadata,
+    options?: ObjectStorageOperationOptions
+  ): Promise<void>;
   get(
     path: string,
-    options?: { signal?: AbortSignal }
+    options?: ObjectStorageOperationOptions
   ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }>;
-  list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string>;
-  delete(paths: string[]): Promise<void>;
-  deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }>;
+  list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string>;
+  delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void>;
+  deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{ objectCount: number }>;
 }