@powersync/service-module-mongodb 0.15.3 → 0.16.0

package/src/replication/ChangeStreamReplicationJob.ts

@@ -1,4 +1,4 @@
-import { container, logger as defaultLogger } from '@powersync/lib-services-framework';
+import { container } from '@powersync/lib-services-framework';
 import { replication } from '@powersync/service-core';
 
 import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js';
@@ -16,7 +16,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ
     super(options);
     this.connectionFactory = options.connectionFactory;
     // We use a custom formatter to process the prefix
-    this.logger = defaultLogger.child({ prefix: `[powersync_${this.storage.group_id}] ` });
+    this.logger = options.storage.logger;
   }
 
   async cleanUp(): Promise<void> {

package/src/replication/ChangeStreamReplicator.ts

@@ -1,10 +1,7 @@
-import { storage, replication } from '@powersync/service-core';
+import { replication, storage } from '@powersync/service-core';
+import { MongoModule } from '../module/MongoModule.js';
 import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js';
 import { ConnectionManagerFactory } from './ConnectionManagerFactory.js';
-import { MongoErrorRateLimiter } from './MongoErrorRateLimiter.js';
-import { MongoModule } from '../module/MongoModule.js';
-import { MongoLSN } from '../common/MongoLSN.js';
-import { timestampToDate } from './replication-utils.js';
 
 export interface ChangeStreamReplicatorOptions extends replication.AbstractReplicatorOptions {
   connectionFactory: ConnectionManagerFactory;

package/src/replication/JsonBufferWriter.ts

@@ -0,0 +1,390 @@
+const JSON_BUFFER_INITIAL_CAPACITY = 1024 * 1024;
+const HEX_LOWER_BYTES = Buffer.from('0123456789abcdef', 'ascii');
+export const BYTE_DQUOTE = 0x22; // "
+export const BYTE_BACKSLASH = 0x5c; // \
+export const BYTE_BACKSPACE = 0x08; // \b
+export const BYTE_TAB = 0x09; // \t
+export const BYTE_NEWLINE = 0x0a; // \n
+export const BYTE_FORM_FEED = 0x0c; // \f
+export const BYTE_CARRIAGE_RETURN = 0x0d; // \r
+export const BYTE_LBRACE = 0x7b; // {
+export const BYTE_RBRACE = 0x7d; // }
+export const BYTE_LBRACKET = 0x5b; // [
+export const BYTE_RBRACKET = 0x5d; // ]
+export const BYTE_COMMA = 0x2c; // ,
+export const BYTE_COLON = 0x3a; // :
+export const BYTE_ZERO = 0x30; // 0
+export const BYTE_ONE = 0x31; // 1
+export const BYTE_T = 0x54; // T
+export const BYTE_SPACE = 0x20; // ' '
+const BYTE_DASH = 0x2d; // -
+const BYTE_DOT = 0x2e; // .
+const BYTE_Z = 0x5a; // Z
+const BYTE_B = 0x62; // b
+const BYTE_T_LOWER = 0x74; // t
+const BYTE_N = 0x6e; // n
+const BYTE_F = 0x66; // f
+const BYTE_R = 0x72; // r
+const BYTE_U = 0x75; // u
+
+/**
+ * Low-level class to generate JSON.
+ *
+ * This writes to a Buffer, which can then be decoded as a string.
+ */
+export class JsonBufferWriter {
+  /**
+   * The raw buffer. Capacity can increase, but is never decreased.
+   */
+  private buffer: Buffer;
+
+  /**
+   * length of data written to the buffer.
+   */
+  private length = 0;
+
+  constructor(capacity = JSON_BUFFER_INITIAL_CAPACITY) {
+    // In theory allocUnsafe could be fine. But in case there are any bugs where this.length
+    // is not updated correctly, that could lead to leaking data from memory.
+    // Buffer.alloc makes sure no data is leaked if a bug like that is hit.
+    // The same applies to ensureCapacity, and for the same reason we zero out changes in truncate().
+    this.buffer = Buffer.alloc(capacity);
+  }
+
+  /**
+   * Resets the length, equivalent to truncate(0).
+   */
+  reset() {
+    this.truncate(0);
+  }
+
+  toString() {
+    return this.buffer.toString('utf8', 0, this.length);
+  }
+
+  getLength() {
+    return this.length;
+  }
+
+  truncate(length: number) {
+    // Safely reset data
+    this.buffer.fill(0, length, this.length);
+    this.length = length;
+  }
+
+  /**
+   * Write a single raw byte.
+   *
+   * Caller is responsible for ensuring this produces valid JSON.
+   */
+  writeByte(value: number) {
+    this.ensureCapacity(1);
+    this.buffer[this.length++] = value;
+  }
+
+  /**
+   * Write raw ascii string - one byte per character. No quoting or escaping is performed.
+   *
+   * Caller is responsible for ensuring this produces valid JSON.
+   */
+  writeAscii(text: string) {
+    const length = text.length;
+    this.ensureCapacity(length);
+    this.buffer.write(text, this.length, length, 'ascii');
+    this.length += length;
+  }
+
+  /**
+   * Write UTF-8 string. No quoting or escaping is performed.
+   *
+   * Caller is responsible for ensuring this produces valid JSON.
+   */
+  writeUtf8(text: string) {
+    const length = Buffer.byteLength(text);
+    this.ensureCapacity(length);
+    this.buffer.write(text, this.length, length, 'utf8');
+    this.length += length;
+  }
+
+  /**
+   * Quote and write a string, escaping characters as needed.
+   */
+  writeQuotedJsonString(text: string) {
+    this.writeByte(BYTE_DQUOTE);
+    let start = 0;
+
+    for (let i = 0; i < text.length; i++) {
+      const ch = text.charCodeAt(i);
+      let escaped: string | undefined;
+
+      switch (ch) {
+        case BYTE_DQUOTE:
+          escaped = '\\"';
+          break;
+        case BYTE_BACKSLASH:
+          escaped = '\\\\';
+          break;
+        case BYTE_BACKSPACE:
+          escaped = '\\b';
+          break;
+        case BYTE_TAB:
+          escaped = '\\t';
+          break;
+        case BYTE_NEWLINE:
+          escaped = '\\n';
+          break;
+        case BYTE_FORM_FEED:
+          escaped = '\\f';
+          break;
+        case BYTE_CARRIAGE_RETURN:
+          escaped = '\\r';
+          break;
+        default:
+          if (ch < 0x20) {
+            escaped = `\\u${ch.toString(16).padStart(4, '0')}`;
+          }
+      }
+
+      if (escaped == null) {
+        continue;
+      }
+
+      if (start < i) {
+        this.writeUtf8(text.slice(start, i));
+      }
+      this.writeAscii(escaped);
+      start = i + 1;
+    }
+
+    if (start < text.length) {
+      this.writeUtf8(text.slice(start));
+    }
+
+    this.writeByte(BYTE_DQUOTE);
+  }
+
+  /**
+   * Quotes and write an UTF-8 string from a source buffer, escaping characters as needed.
+   *
+   * @param bytes source buffer
+   * @param start start offset, inclusive
+   * @param end end offset, exclusive
+   */
+  writeQuotedUtf8Slice(bytes: Buffer, start: number, end: number): void {
+    let firstEscape = -1;
+    for (let index = start; index < end; index++) {
+      const value = bytes[index];
+      if (value < 0x20 || value === BYTE_DQUOTE || value === BYTE_BACKSLASH) {
+        firstEscape = index;
+        break;
+      }
+    }
+
+    const rawLength = end - start;
+    let length = this.length;
+    this.ensureCapacity(rawLength + 2);
+    let buffer = this.buffer;
+
+    buffer[length++] = BYTE_DQUOTE;
+
+    if (firstEscape < 0) {
+      bytes.copy(buffer, length, start, end);
+      length += rawLength;
+      buffer[length++] = BYTE_DQUOTE;
+      this.length = length;
+      return;
+    }
+
+    if (firstEscape > start) {
+      bytes.copy(buffer, length, start, firstEscape);
+      length += firstEscape - start;
+    }
+
+    let chunkStart = firstEscape;
+    for (let index = firstEscape; index < end; index++) {
+      const value = bytes[index];
+      if (value >= 0x20 && value !== BYTE_DQUOTE && value !== BYTE_BACKSLASH) {
+        continue;
+      }
+
+      if (chunkStart < index) {
+        const chunkLength = index - chunkStart;
+        this.length = length;
+        this.ensureCapacity(chunkLength + 6);
+        buffer = this.buffer;
+        bytes.copy(buffer, length, chunkStart, index);
+        length += chunkLength;
+      } else {
+        this.length = length;
+        this.ensureCapacity(6);
+        buffer = this.buffer;
+      }
+
+      switch (value) {
+        case BYTE_DQUOTE:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_DQUOTE;
+          break;
+        case BYTE_BACKSLASH:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_BACKSLASH;
+          break;
+        case BYTE_BACKSPACE:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_B;
+          break;
+        case BYTE_TAB:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_T_LOWER;
+          break;
+        case BYTE_NEWLINE:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_N;
+          break;
+        case BYTE_FORM_FEED:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_F;
+          break;
+        case BYTE_CARRIAGE_RETURN:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_R;
+          break;
+        default:
+          buffer[length++] = BYTE_BACKSLASH;
+          buffer[length++] = BYTE_U;
+          buffer[length++] = BYTE_ZERO;
+          buffer[length++] = BYTE_ZERO;
+          buffer[length++] = HEX_LOWER_BYTES[value >> 4];
+          buffer[length++] = HEX_LOWER_BYTES[value & 0x0f];
+          break;
+      }
+
+      chunkStart = index + 1;
+    }
+
+    if (chunkStart < end) {
+      const chunkLength = end - chunkStart;
+      this.length = length;
+      this.ensureCapacity(chunkLength + 1);
+      buffer = this.buffer;
+      bytes.copy(buffer, length, chunkStart, end);
+      length += chunkLength;
+    } else {
+      this.length = length;
+      this.ensureCapacity(1);
+      buffer = this.buffer;
+    }
+
+    buffer[length++] = BYTE_DQUOTE;
+    this.length = length;
+  }
+
+  /**
+   * Quote and write bytes as hex.
+   */
+  writeQuotedHexLower(bytes: Buffer, start: number, length: number) {
+    this.ensureCapacity(length * 2 + 2);
+    this.buffer[this.length++] = BYTE_DQUOTE;
+    for (let index = start; index < start + length; index++) {
+      const value = bytes[index];
+      this.buffer[this.length++] = HEX_LOWER_BYTES[value >> 4];
+      this.buffer[this.length++] = HEX_LOWER_BYTES[value & 0x0f];
+    }
+    this.buffer[this.length++] = BYTE_DQUOTE;
+  }
+
+  /**
+   * Quote and write 16 UUID bytes in canonical lower-case form.
+   */
+  writeQuotedUuid(bytes: Buffer, start: number) {
+    this.ensureCapacity(38);
+    const buffer = this.buffer;
+    let length = this.length;
+    buffer[length++] = BYTE_DQUOTE;
+    for (let index = 0; index < 16; index++) {
+      if (index === 4 || index === 6 || index === 8 || index === 10) {
+        buffer[length++] = BYTE_DASH;
+      }
+      const value = bytes[start + index];
+      buffer[length++] = HEX_LOWER_BYTES[value >> 4];
+      buffer[length++] = HEX_LOWER_BYTES[value & 0x0f];
+    }
+    buffer[length++] = BYTE_DQUOTE;
+    this.length = length;
+  }
+
+  private ensureCapacity(extra: number) {
+    const required = this.length + extra;
+    if (required <= this.buffer.length) {
+      return;
+    }
+
+    let nextLength = this.buffer.length;
+    while (nextLength < required) {
+      nextLength *= 2;
+    }
+
+    const next = Buffer.alloc(nextLength);
+    this.buffer.copy(next, 0, 0, this.length);
+    this.buffer = next;
+  }
+
+  writeDateTime(
+    year: number,
+    month: number,
+    day: number,
+    hour: number,
+    minute: number,
+    second: number,
+    millisecond: number,
+    quoted: boolean,
+    separator: number,
+    includeMilliseconds: boolean
+  ) {
+    // A more specific value would be this:
+    //   (quoted ? 2 : 0) + 20 + (includeMilliseconds ? 4 : 1)
+    // But there is no harm in over-allocating by 6 bytes.
+    this.ensureCapacity(26);
+    const buffer = this.buffer;
+    let offset = this.length;
+
+    if (quoted) {
+      buffer[offset++] = BYTE_DQUOTE;
+    }
+
+    buffer[offset++] = BYTE_ZERO + ((year / 1000) | 0);
+    buffer[offset++] = BYTE_ZERO + (((year / 100) | 0) % 10);
+    buffer[offset++] = BYTE_ZERO + (((year / 10) | 0) % 10);
+    buffer[offset++] = BYTE_ZERO + (year % 10);
+    buffer[offset++] = BYTE_DASH;
+    buffer[offset++] = BYTE_ZERO + ((month / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (month % 10);
+    buffer[offset++] = BYTE_DASH;
+    buffer[offset++] = BYTE_ZERO + ((day / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (day % 10);
+    buffer[offset++] = separator;
+    buffer[offset++] = BYTE_ZERO + ((hour / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (hour % 10);
+    buffer[offset++] = BYTE_COLON;
+    buffer[offset++] = BYTE_ZERO + ((minute / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (minute % 10);
+    buffer[offset++] = BYTE_COLON;
+    buffer[offset++] = BYTE_ZERO + ((second / 10) | 0);
+    buffer[offset++] = BYTE_ZERO + (second % 10);
+
+    if (includeMilliseconds) {
+      buffer[offset++] = BYTE_DOT;
+      buffer[offset++] = BYTE_ZERO + ((millisecond / 100) | 0);
+      buffer[offset++] = BYTE_ZERO + (((millisecond / 10) | 0) % 10);
+      buffer[offset++] = BYTE_ZERO + (millisecond % 10);
+    }
+
+    buffer[offset++] = BYTE_Z;
+
+    if (quoted) {
+      buffer[offset++] = BYTE_DQUOTE;
+    }
+
+    this.length = offset;
+  }
+}

package/src/replication/MongoManager.ts

@@ -1,8 +1,8 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 
-import { NormalizedMongoConnectionConfig } from '../types/types.js';
-import { BSON_DESERIALIZE_DATA_OPTIONS, POWERSYNC_VERSION } from '@powersync/service-core';
 import { BaseObserver } from '@powersync/lib-services-framework';
+import { BSON_DESERIALIZE_DATA_OPTIONS, POWERSYNC_VERSION } from '@powersync/service-core';
+import { NormalizedMongoConnectionConfig } from '../types/types.js';
 
 export interface MongoManagerListener {
   onEnded(): void;

package/src/replication/MongoRelation.ts

@@ -6,10 +6,10 @@ import {
   CustomArray,
   CustomObject,
   CustomSqliteValue,
+  DateTimeSourceOptions,
+  DateTimeValue,
   SqliteInputRow,
   SqliteInputValue,
-  DateTimeValue,
-  DateTimeSourceOptions,
   TimeValuePrecision
 } from '@powersync/service-sync-rules';
 
@@ -115,6 +115,9 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
   } else if (typeof data == 'number') {
     if (autoBigNum && Number.isInteger(data)) {
       return BigInt(data);
+    } else if (!Number.isFinite(data)) {
+      // Only finite numbers can be represented in JSON.
+      return null;
     } else {
       return data;
     }

package/src/replication/MongoSnapshotQuery.ts

@@ -1,6 +1,7 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { bson } from '@powersync/service-core';
+import { parseDocumentId } from './bufferToSqlite.js';
 import { getCursorBatchBytes } from './internal-mongodb-utils.js';
 
 /**
@@ -23,7 +24,7 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
   }
 
   async nextChunk(): Promise<
-    { docs: mongo.Document[]; lastKey: Uint8Array; bytes: number } | { docs: []; lastKey: null; bytes: 0 }
+    { docs: Buffer[]; lastKey: Uint8Array; bytes: number } | { docs: []; lastKey: null; bytes: 0 }
   > {
     let cursor = this.lastCursor;
     let newCursor = false;
@@ -46,7 +47,8 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
         // batchSize is 1 more than limit to auto-close the cursor.
         // See https://github.com/mongodb/node-mongodb-native/pull/4580
         batchSize: this.batchSize + 1,
-        sort: { _id: 1 }
+        sort: { _id: 1 },
+        raw: true
       });
       newCursor = true;
     }
@@ -62,14 +64,15 @@ export class ChunkedSnapshotQuery implements AsyncDisposable {
       }
     }
     const bytes = getCursorBatchBytes(cursor);
-    const docBatch = cursor.readBufferedDocuments();
+    const docBatch = cursor.readBufferedDocuments() as Buffer[];
     this.lastCursor = cursor;
     if (docBatch.length == 0) {
       throw new ReplicationAssertionError(`MongoDB snapshot query returned an empty batch, but hasNext() was true.`);
     }
-    const lastKey = docBatch[docBatch.length - 1]._id;
+    const lastDoc = docBatch[docBatch.length - 1];
+    const { id: lastKey, idBuffer } = parseDocumentId(lastDoc);
     this.lastKey = lastKey;
-    return { docs: docBatch, lastKey: bson.serialize({ _id: lastKey }), bytes };
+    return { docs: docBatch, lastKey: idBuffer, bytes };
   }
 
   async [Symbol.asyncDispose](): Promise<void> {