@powersync/service-module-mongodb 0.15.4 → 0.17.0

package/src/replication/ChangeStreamReplicationJob.ts

@@ -1,4 +1,4 @@
-import { container, logger as defaultLogger } from '@powersync/lib-services-framework';
+import { container } from '@powersync/lib-services-framework';
 import { replication } from '@powersync/service-core';
 
 import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js';
@@ -16,7 +16,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
     super(options);
     this.connectionFactory = options.connectionFactory;
     // We use a custom formatter to process the prefix
-    this.logger = defaultLogger.child({ prefix: `[powersync_${this.storage.group_id}] ` });
+    this.logger = options.storage.logger;
   }
 
   async cleanUp(): Promise<void> {

package/src/replication/JsonBufferWriter.ts

@@ -0,0 +1,390 @@
+const JSON_BUFFER_INITIAL_CAPACITY = 1024 * 1024;
+const HEX_LOWER_BYTES = Buffer.from('0123456789abcdef', 'ascii');
+export const BYTE_DQUOTE = 0x22; // "
+export const BYTE_BACKSLASH = 0x5c; // \
+export const BYTE_BACKSPACE = 0x08; // \b
+export const BYTE_TAB = 0x09; // \t
+export const BYTE_NEWLINE = 0x0a; // \n
+export const BYTE_FORM_FEED = 0x0c; // \f
+export const BYTE_CARRIAGE_RETURN = 0x0d; // \r
+export const BYTE_LBRACE = 0x7b; // {
+export const BYTE_RBRACE = 0x7d; // }
+export const BYTE_LBRACKET = 0x5b; // [
+export const BYTE_RBRACKET = 0x5d; // ]
+export const BYTE_COMMA = 0x2c; // ,
+export const BYTE_COLON = 0x3a; // :
+export const BYTE_ZERO = 0x30; // 0
+export const BYTE_ONE = 0x31; // 1
+export const BYTE_T = 0x54; // T
+export const BYTE_SPACE = 0x20; // ' '
+const BYTE_DASH = 0x2d; // -
+const BYTE_DOT = 0x2e; // .
+const BYTE_Z = 0x5a; // Z
+const BYTE_B = 0x62; // b
+const BYTE_T_LOWER = 0x74; // t
+const BYTE_N = 0x6e; // n
+const BYTE_F = 0x66; // f
+const BYTE_R = 0x72; // r
+const BYTE_U = 0x75; // u
+
+/**
+ * Low-level class to generate JSON.
+ *
+ * This writes to a Buffer, which can then be decoded as a string.
+ */
+export class JsonBufferWriter {
+  /**
+   * The raw buffer. Capacity can increase, but is never decreased.
+   */
+  private buffer: Buffer;
+
+  /**
+   * length of data written to the buffer.
+   */
+  private length = 0;
+
+  constructor(capacity = JSON_BUFFER_INITIAL_CAPACITY) {
+    // In theory allocUnsafe could be fine. But in case there are any bugs where this.length
+    // is not updated correctly, that could lead to leaking data from memory.
+    // Buffer.alloc makes sure no data is leaked if a bug like that is hit.
+    // The same applies to ensureCapacity, and for the same reason we zero out changes in truncate().
+    this.buffer = Buffer.alloc(capacity);
+  }
+
+  /**
+   * Resets the length, equivalent to truncate(0).
+   */
+  reset() {
+    this.truncate(0);
+  }
+
+  toString() {
+    return this.buffer.toString('utf8', 0, this.length);
+  }
+
+  getLength() {
+    return this.length;
+  }
+
+  truncate(length: number) {
+    // Safely reset data
+    this.buffer.fill(0, length, this.length);
+    this.length = length;
+  }
+
+  /**
+   * Write a single raw byte.
+   *
+   * Caller is responsible for ensuring this produces valid JSON.
+   */
+  writeByte(value: number) {
+    this.ensureCapacity(1);
+    this.buffer[this.length++] = value;
+  }
+
+  /**
+   * Write raw ascii string - one byte per character. No quoting or escaping is performed.
+   *
+   * Caller is responsible for ensuring this produces valid JSON.
+   */
+  writeAscii(text: string) {
+    const length = text.length;
+    this.ensureCapacity(length);
+    this.buffer.write(text, this.length, length, 'ascii');
+    this.length += length;
+  }
+
+  /**
+   * Write UTF-8 string. No quoting or escaping is performed.
+   *
+   * Caller is responsible for ensuring this produces valid JSON.
+   */
+  writeUtf8(text: string) {
+    const length = Buffer.byteLength(text);
+    this.ensureCapacity(length);
+    this.buffer.write(text, this.length, length, 'utf8');
+    this.length += length;
+  }
+
+  /**
+   * Quote and write a string, escaping characters as needed.
+   */
+  writeQuotedJsonString(text: string) {
+    this.writeByte(BYTE_DQUOTE);
+    let start = 0;
+
+    for (let i = 0; i < text.length; i++) {
+      const ch = text.charCodeAt(i);
+      let escaped: string | undefined;
+
+      switch (ch) {
+        case BYTE_DQUOTE:
+          escaped = '\\"';
+          break;
+        case BYTE_BACKSLASH:
+          escaped = '\\\\';
+          break;
+        case BYTE_BACKSPACE:
+          escaped = '\\b';
+          break;
+        case BYTE_TAB:
+          escaped = '\\t';
+          break;
+        case BYTE_NEWLINE:
+          escaped = '\\n';
+          break;
+        case BYTE_FORM_FEED:
+          escaped = '\\f';
+          break;
+        case BYTE_CARRIAGE_RETURN:
+          escaped = '\\r';
+          break;
+        default:
+          if (ch < 0x20) {
+            escaped = `\\u${ch.toString(16).padStart(4, '0')}`;
+          }
+      }
+
+      if (escaped == null) {
+        continue;
+      }
+
+      if (start < i) {
+        this.writeUtf8(text.slice(start, i));
+      }
+      this.writeAscii(escaped);
+      start = i + 1;
+    }
+
+    if (start < text.length) {
+      this.writeUtf8(text.slice(start));
+    }
+
+    this.writeByte(BYTE_DQUOTE);
+  }
+
+  /**
+   * Quotes and write an UTF-8 string from a source buffer, escaping characters as needed.
+   *
+   * @param bytes source buffer
+   * @param start start offset, inclusive
+   * @param end end offset, exclusive
+   */
+  writeQuotedUtf8Slice(bytes: Buffer, start: number, end: number): void {
+    let firstEscape = -1;
+    for (let index = start; index < end; index++) {
+      const value = bytes[index];
+      if (value < 0x20 || value === BYTE_DQUOTE || value === BYTE_BACKSLASH) {
+        firstEscape = index;
+        break;
+      }
+    }
+
+    const rawLength = end - start;
+    let length = this.length;
+    this.ensureCapacity(rawLength + 2);
+    let buffer = this.buffer;
+
+    buffer[length++] = BYTE_DQUOTE;
+
+    if (firstEscape < 0) {
+      bytes.copy(buffer, length, start, end);
+      length += rawLength;
+      buffer[length++] = BYTE_DQUOTE;
+      this.length = length;
+      return;
+    }
+
+    if (firstEscape > start) {
+      bytes.copy(buffer, length, start, firstEscape);
+      length += firstEscape - start;
+    }
+
+    let chunkStart = firstEscape;
+    for (let index = firstEscape; index < end; index++) {
+      const value = bytes[index];
+      if (value >= 0x20 && value !== BYTE_DQUOTE && value !== BYTE_BACKSLASH) {
+        continue;
+      }
+
+      if (chunkStart < index) {
+        const chunkLength = index - chunkStart;
+        this.length = length;
+        this.ensureCapacity(chunkLength + 6);
+        buffer = this.buffer;
+        bytes.copy(buffer, length, chunkStart, index);
+        length += chunkLength;
+      } else {
+        this.length = length;
+        this.ensureCapacity(6);
+        buffer = this.buffer;
+      }
+
+      switch (value) {
+        case BYTE_DQUOTE:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_DQUOTE;
+          break;
+        case BYTE_BACKSLASH:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_BACKSLASH;
+          break;
+        case BYTE_BACKSPACE:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_B;
+          break;
+        case BYTE_TAB:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_T_LOWER;
+          break;
+        case BYTE_NEWLINE:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_N;
+          break;
+        case BYTE_FORM_FEED:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_F;
+          break;
+        case BYTE_CARRIAGE_RETURN:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_R;
+          break;
+        default:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_U;
+          buffer[length++] = BYTE_ZERO;
+          buffer[length++] = BYTE_ZERO;
+          buffer[length++] = HEX_LOWER_BYTES[value >> 4];
+          buffer[length++] = HEX_LOWER_BYTES[value & 0x0f];
+          break;
+      }
+
+      chunkStart = index + 1;
+    }
+
+    if (chunkStart < end) {
+      const chunkLength = end - chunkStart;
+      this.length = length;
+      this.ensureCapacity(chunkLength + 1);
+      buffer = this.buffer;
+      bytes.copy(buffer, length, chunkStart, end);
+      length += chunkLength;
+    } else {
+      this.length = length;
+      this.ensureCapacity(1);
+      buffer = this.buffer;
+    }
+
+    buffer[length++] = BYTE_DQUOTE;
+    this.length = length;
+  }
+
+  /**
+   * Quote and write bytes as hex.
+   */
+  writeQuotedHexLower(bytes: Buffer, start: number, length: number) {
+    this.ensureCapacity(length * 2 + 2);
+    this.buffer[this.length++] = BYTE_DQUOTE;
+    for (let index = start; index < start + length; index++) {
+      const value = bytes[index];
+      this.buffer[this.length++] = HEX_LOWER_BYTES[value >> 4];
+      this.buffer[this.length++] = HEX_LOWER_BYTES[value & 0x0f];
+    }
+    this.buffer[this.length++] = BYTE_DQUOTE;
+  }
+
+  /**
+   * Quote and write 16 UUID bytes in canonical lower-case form.
+   */
+  writeQuotedUuid(bytes: Buffer, start: number) {
+    this.ensureCapacity(38);
+    const buffer = this.buffer;
+    let length = this.length;
+    buffer[length++] = BYTE_DQUOTE;
+    for (let index = 0; index < 16; index++) {
+      if (index === 4 || index === 6 || index === 8 || index === 10) {
+        buffer[length++] = BYTE_DASH;
+      }
+      const value = bytes[start + index];
+      buffer[length++] = HEX_LOWER_BYTES[value >> 4];
+      buffer[length++] = HEX_LOWER_BYTES[value & 0x0f];
+    }
+    buffer[length++] = BYTE_DQUOTE;
+    this.length = length;
+  }
+
+  private ensureCapacity(extra: number) {
+    const required = this.length + extra;
+    if (required <= this.buffer.length) {
+      return;
+    }
+
+    let nextLength = this.buffer.length;
+    while (nextLength < required) {
+      nextLength *= 2;
+    }
+
+    const next = Buffer.alloc(nextLength);
+    this.buffer.copy(next, 0, 0, this.length);
+    this.buffer = next;
+  }
+
+  writeDateTime(
+    year: number,
+    month: number,
+    day: number,
+    hour: number,
+    minute: number,
+    second: number,
+    millisecond: number,
+    quoted: boolean,
+    separator: number,
+    includeMilliseconds: boolean
+  ) {
+    // A more specific value would be this:
+    //   (quoted ? 2 : 0) + 20 + (includeMilliseconds ? 4 : 1)
+    // But there is no harm in over-allocating by 6 bytes.
+    this.ensureCapacity(26);
+    const buffer = this.buffer;
+    let offset = this.length;
+
+    if (quoted) {
+      buffer[offset++] = BYTE_DQUOTE;
+    }
+
+    buffer[offset++] = BYTE_ZERO + ((year / 1000) | 0);
+    buffer[offset++] = BYTE_ZERO + (((year / 100) | 0) % 10);
+    buffer[offset++] = BYTE_ZERO + (((year / 10) | 0) % 10);
+    buffer[offset++] = BYTE_ZERO + (year % 10);
+    buffer[offset++] = BYTE_DASH;
+    buffer[offset++] = BYTE_ZERO + ((month / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (month % 10);
+    buffer[offset++] = BYTE_DASH;
+    buffer[offset++] = BYTE_ZERO + ((day / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (day % 10);
+    buffer[offset++] = separator;
+    buffer[offset++] = BYTE_ZERO + ((hour / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (hour % 10);
+    buffer[offset++] = BYTE_COLON;
+    buffer[offset++] = BYTE_ZERO + ((minute / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (minute % 10);
+    buffer[offset++] = BYTE_COLON;
+    buffer[offset++] = BYTE_ZERO + ((second / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (second % 10);
+
+    if (includeMilliseconds) {
+      buffer[offset++] = BYTE_DOT;
+      buffer[offset++] = BYTE_ZERO + ((millisecond / 100) | 0);
+      buffer[offset++] = BYTE_ZERO + (((millisecond / 10) | 0) % 10);
+      buffer[offset++] = BYTE_ZERO + (millisecond % 10);
+    }
+
+    buffer[offset++] = BYTE_Z;
+
+    if (quoted) {
+      buffer[offset++] = BYTE_DQUOTE;
+    }
+
+    this.length = offset;
+  }
+}

package/src/replication/MongoRelation.ts

@@ -13,12 +13,15 @@ import {
   TimeValuePrecision
 } from '@powersync/service-sync-rules';
 
-import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
 import { MongoLSN } from '../common/MongoLSN.js';
-import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
 
-export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor {
+export function getMongoRelation(
+  source: mongo.ChangeStreamNameSpace,
+  connectionTag: string
+): storage.SourceEntityDescriptor {
   return {
+    connectionTag,
     name: source.coll,
     schema: source.db,
     // Not relevant for MongoDB - we use db + coll name as the identifier
@@ -115,6 +118,9 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
   } else if (typeof data == 'number') {
     if (autoBigNum && Number.isInteger(data)) {
       return BigInt(data);
+    } else if (!Number.isFinite(data)) {
+      // Only finite numbers can be represented in JSON.
+      return null;
     } else {
       return data;
     }
@@ -172,30 +178,53 @@ export async function createCheckpoint(
   db: mongo.Db,
   id: mongo.ObjectId | string
 ): Promise<string> {
-  const session = client.startSession();
-  try {
-    // We use an unique id per process, and clear documents on startup.
-    // This is so that we can filter events for our own process only, and ignore
-    // events from other processes.
-    await db.collection(CHECKPOINTS_COLLECTION).findOneAndUpdate(
-      {
-        _id: id as any
-      },
-      {
-        $inc: { i: 1 }
-      },
-      {
-        upsert: true,
-        returnDocument: 'after',
-        session
+  const TRIES = 2;
+  for (let i = 0; i < TRIES; i++) {
+    try {
+      return await createCheckpointInner(client, db, id);
+    } catch (e) {
+      if (i < TRIES - 1) {
+        logger.warn(`Failed to create checkpoint on attempt ${i + 1}`, e);
+      } else {
+        throw e;
       }
-    );
-    const time = session.operationTime!;
-    // TODO: Use the above when we support custom write checkpoints
-    return new MongoLSN({ timestamp: time }).comparable;
-  } finally {
-    await session.endSession();
+    }
+  }
+  throw new ServiceAssertionError(`Unreachable code`);
+}
+
+async function createCheckpointInner(
+  client: mongo.MongoClient,
+  db: mongo.Db,
+  id: mongo.ObjectId | string
+): Promise<string> {
+  // We use an unique id per process, and clear documents on startup.
+  // This is so that we can filter events for our own process only, and ignore
+  // events from other processes.
+
+  // We use a command instead of a regular update to avoid auto retries on writes.
+  // An auto retry on the write can trigger a weird edge case where the change stream event
+  // has the clusterTime of the first write, while the returned operation time is for the second no-op write.
+  // Instead, we do manual retries, which does not have the same write de-duplication logic.
+  // A sentinal-based approach would be better here, but that is a much bigger change.
+
+  const response = await db.command({
+    findAndModify: '_powersync_checkpoints',
+    query: {
+      _id: id as any
+    },
+    new: true,
+    upsert: true,
+    update: {
+      $inc: { i: 1 }
+    }
+  });
+
+  const time = response.operationTime as mongo.Timestamp | undefined;
+  if (time == null) {
+    throw new ServiceError(ErrorCode.PSYNC_S1004, `clusterTime not available for checkpoint`);
   }
+  return new MongoLSN({ timestamp: time }).comparable;
 }
 
 const mongoTimeOptions: DateTimeSourceOptions = {

package/src/replication/MongoSnapshotQuery.ts

@@ -1,6 +1,7 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { bson } from '@powersync/service-core';
+import { parseDocumentId } from './bufferToSqlite.js';
 import { getCursorBatchBytes } from './internal-mongodb-utils.js';
 
 /**
@@ -23,7 +24,7 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
   }
 
   async nextChunk(): Promise<
-    { docs: mongo.Document[]; lastKey: Uint8Array; bytes: number } | { docs: []; lastKey: null; bytes: 0 }
+    { docs: Buffer[]; lastKey: Uint8Array; bytes: number } | { docs: []; lastKey: null; bytes: 0 }
   > {
     let cursor = this.lastCursor;
     let newCursor = false;
@@ -46,7 +47,8 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
         // batchSize is 1 more than limit to auto-close the cursor.
         // See https://github.com/mongodb/node-mongodb-native/pull/4580
         batchSize: this.batchSize + 1,
-        sort: { _id: 1 }
+        sort: { _id: 1 },
+        raw: true
       });
       newCursor = true;
     }
@@ -62,14 +64,15 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
       }
     }
     const bytes = getCursorBatchBytes(cursor);
-    const docBatch = cursor.readBufferedDocuments();
+    const docBatch = cursor.readBufferedDocuments() as Buffer[];
     this.lastCursor = cursor;
     if (docBatch.length == 0) {
       throw new ReplicationAssertionError(`MongoDB snapshot query returned an empty batch, but hasNext() was true.`);
     }
-    const lastKey = docBatch[docBatch.length - 1]._id;
+    const lastDoc = docBatch[docBatch.length - 1];
+    const { id: lastKey, idBuffer } = parseDocumentId(lastDoc);
     this.lastKey = lastKey;
-    return { docs: docBatch, lastKey: bson.serialize({ _id: lastKey }), bytes };
+    return { docs: docBatch, lastKey: idBuffer, bytes };
   }
 
   async [Symbol.asyncDispose](): Promise<void> {