@depup/mongodb 7.1.0-depup.0

package/src/cmap/wire_protocol/compression.ts

@@ -0,0 +1,211 @@
+import * as zlib from 'zlib';
+
+import { LEGACY_HELLO_COMMAND } from '../../constants';
+import { getSnappy, getZstdLibrary, type SnappyLib, type ZStandard } from '../../deps';
+import { MongoDecompressionError, MongoInvalidArgumentError } from '../../error';
+import {
+  type MessageHeader,
+  OpCompressedRequest,
+  type OpCompressesRequestOptions,
+  OpMsgResponse,
+  OpReply,
+  type WriteProtocolMessageType
+} from '../commands';
+import { OP_COMPRESSED, OP_MSG } from './constants';
+
+/** @public */
+export const Compressor = Object.freeze({
+  none: 0,
+  snappy: 1,
+  zlib: 2,
+  zstd: 3
+} as const);
+
+/** @public */
+export type Compressor = (typeof Compressor)[CompressorName];
+
+/** @public */
+export type CompressorName = keyof typeof Compressor;
+
+export const uncompressibleCommands = new Set([
+  LEGACY_HELLO_COMMAND,
+  'saslStart',
+  'saslContinue',
+  'getnonce',
+  'authenticate',
+  'createUser',
+  'updateUser',
+  'copydbSaslStart',
+  'copydbgetnonce',
+  'copydb'
+]);
+
+const ZSTD_COMPRESSION_LEVEL = 3;
+
+const zlibInflate = (buf: zlib.InputType) => {
+  return new Promise<Buffer>((resolve, reject) => {
+    zlib.inflate(buf, (error, result) => {
+      if (error) return reject(error);
+      resolve(result);
+    });
+  });
+};
+
+const zlibDeflate = (buf: zlib.InputType, options: zlib.ZlibOptions) => {
+  return new Promise<Buffer>((resolve, reject) => {
+    zlib.deflate(buf, options, (error, result) => {
+      if (error) return reject(error);
+      resolve(result);
+    });
+  });
+};
+
+let zstd: ZStandard;
+let Snappy: SnappyLib | null = null;
+function loadSnappy() {
+  if (Snappy == null) {
+    const snappyImport = getSnappy();
+    if ('kModuleError' in snappyImport) {
+      throw snappyImport.kModuleError;
+    }
+    Snappy = snappyImport;
+  }
+  return Snappy;
+}
+
+// Facilitate compressing a message using an agreed compressor
+export async function compress(
+  options: OpCompressesRequestOptions,
+  dataToBeCompressed: Buffer
+): Promise<Buffer> {
+  const zlibOptions = {} as zlib.ZlibOptions;
+  switch (options.agreedCompressor) {
+    case 'snappy': {
+      Snappy ??= loadSnappy();
+      return await Snappy.compress(dataToBeCompressed);
+    }
+    case 'zstd': {
+      loadZstd();
+      if ('kModuleError' in zstd) {
+        throw zstd['kModuleError'];
+      }
+      return await zstd.compress(dataToBeCompressed, ZSTD_COMPRESSION_LEVEL);
+    }
+    case 'zlib': {
+      if (options.zlibCompressionLevel) {
+        zlibOptions.level = options.zlibCompressionLevel;
+      }
+      return await zlibDeflate(dataToBeCompressed, zlibOptions);
+    }
+    default: {
+      throw new MongoInvalidArgumentError(
+        `Unknown compressor ${options.agreedCompressor} failed to compress`
+      );
+    }
+  }
+}
+
+// Decompress a message using the given compressor
+export async function decompress(compressorID: number, compressedData: Buffer): Promise<Buffer> {
+  if (
+    compressorID !== Compressor.snappy &&
+    compressorID !== Compressor.zstd &&
+    compressorID !== Compressor.zlib &&
+    compressorID !== Compressor.none
+  ) {
+    throw new MongoDecompressionError(
+      `Server sent message compressed using an unsupported compressor. (Received compressor ID ${compressorID})`
+    );
+  }
+
+  switch (compressorID) {
+    case Compressor.snappy: {
+      Snappy ??= loadSnappy();
+      return await Snappy.uncompress(compressedData, { asBuffer: true });
+    }
+    case Compressor.zstd: {
+      loadZstd();
+      if ('kModuleError' in zstd) {
+        throw zstd['kModuleError'];
+      }
+      return await zstd.decompress(compressedData);
+    }
+    case Compressor.zlib: {
+      return await zlibInflate(compressedData);
+    }
+    default: {
+      return compressedData;
+    }
+  }
+}
+
+/**
+ * Load ZStandard if it is not already set.
+ */
+function loadZstd() {
+  if (!zstd) {
+    zstd = getZstdLibrary();
+  }
+}
+
+const MESSAGE_HEADER_SIZE = 16;
+
+/**
+ * @internal
+ *
+ * Compresses an OP_MSG or OP_QUERY message, if compression is configured.  This method
+ * also serializes the command to BSON.
+ */
+export async function compressCommand(
+  command: WriteProtocolMessageType,
+  description: { agreedCompressor?: CompressorName; zlibCompressionLevel?: number }
+): Promise<Buffer> {
+  const finalCommand =
+    description.agreedCompressor === 'none' || !OpCompressedRequest.canCompress(command)
+      ? command
+      : new OpCompressedRequest(command, {
+          agreedCompressor: description.agreedCompressor ?? 'none',
+          zlibCompressionLevel: description.zlibCompressionLevel ?? 0
+        });
+  const data = await finalCommand.toBin();
+  return Buffer.concat(data);
+}
+
+/**
+ * @internal
+ *
+ * Decompresses an OP_MSG or OP_QUERY response from the server, if compression is configured.
+ *
+ * This method does not parse the response's BSON.
+ */
+export async function decompressResponse(message: Buffer): Promise<OpMsgResponse | OpReply> {
+  const messageHeader: MessageHeader = {
+    length: message.readInt32LE(0),
+    requestId: message.readInt32LE(4),
+    responseTo: message.readInt32LE(8),
+    opCode: message.readInt32LE(12)
+  };
+
+  if (messageHeader.opCode !== OP_COMPRESSED) {
+    const ResponseType = messageHeader.opCode === OP_MSG ? OpMsgResponse : OpReply;
+    const messageBody = message.subarray(MESSAGE_HEADER_SIZE);
+    return new ResponseType(message, messageHeader, messageBody);
+  }
+
+  const header: MessageHeader = {
+    ...messageHeader,
+    fromCompressed: true,
+    opCode: message.readInt32LE(MESSAGE_HEADER_SIZE),
+    length: message.readInt32LE(MESSAGE_HEADER_SIZE + 4)
+  };
+  const compressorID = message[MESSAGE_HEADER_SIZE + 8];
+  const compressedBuffer = message.slice(MESSAGE_HEADER_SIZE + 9);
+
+  // recalculate based on wrapped opcode
+  const ResponseType = header.opCode === OP_MSG ? OpMsgResponse : OpReply;
+  const messageBody = await decompress(compressorID, compressedBuffer);
+  if (messageBody.length !== header.length) {
+    throw new MongoDecompressionError('Message body and message header must be the same length');
+  }
+  return new ResponseType(message, header, messageBody);
+}

package/src/cmap/wire_protocol/constants.ts

@@ -0,0 +1,17 @@
+export const MIN_SUPPORTED_SERVER_VERSION = '4.2';
+export const MAX_SUPPORTED_SERVER_VERSION = '8.2';
+export const MIN_SUPPORTED_SNAPSHOT_READS_WIRE_VERSION = 13;
+export const MIN_SUPPORTED_SNAPSHOT_READS_SERVER_VERSION = '5.0';
+export const MIN_SUPPORTED_WIRE_VERSION = 8;
+export const MAX_SUPPORTED_WIRE_VERSION = 27;
+export const MIN_SUPPORTED_QE_WIRE_VERSION = 21;
+export const MIN_SUPPORTED_QE_SERVER_VERSION = '7.0';
+export const MIN_SUPPORTED_RAW_DATA_WIRE_VERSION = 27;
+export const MIN_SUPPORTED_RAW_DATA_SERVER_VERSION = '8.2';
+export const OP_REPLY = 1;
+export const OP_UPDATE = 2001;
+export const OP_INSERT = 2002;
+export const OP_QUERY = 2004;
+export const OP_DELETE = 2006;
+export const OP_COMPRESSED = 2012;
+export const OP_MSG = 2013;

package/src/cmap/wire_protocol/on_data.ts

@@ -0,0 +1,139 @@
+import { type EventEmitter } from 'events';
+
+import { type Abortable } from '../../mongo_types';
+import { type TimeoutContext } from '../../timeout';
+import { addAbortListener, kDispose, List, promiseWithResolvers } from '../../utils';
+
+/**
+ * @internal
+ * An object holding references to a promise's resolve and reject functions.
+ */
+type PendingPromises = Omit<
+  ReturnType<typeof promiseWithResolvers<IteratorResult<Buffer>>>,
+  'promise'
+>;
+
+/**
+ * onData is adapted from Node.js' events.on helper
+ * https://nodejs.org/api/events.html#eventsonemitter-eventname-options
+ *
+ * Returns an AsyncIterator that iterates each 'data' event emitted from emitter.
+ * It will reject upon an error event.
+ */
+export function onData(
+  emitter: EventEmitter,
+  { timeoutContext, signal }: { timeoutContext?: TimeoutContext } & Abortable
+) {
+  signal?.throwIfAborted();
+
+  // Setup pending events and pending promise lists
+  /**
+   * When the caller has not yet called .next(), we store the
+   * value from the event in this list. Next time they call .next()
+   * we pull the first value out of this list and resolve a promise with it.
+   */
+  const unconsumedEvents = new List<Buffer>();
+  /**
+   * When there has not yet been an event, a new promise will be created
+   * and implicitly stored in this list. When an event occurs we take the first
+   * promise in this list and resolve it.
+   */
+  const unconsumedPromises = new List<PendingPromises>();
+
+  /**
+   * Stored an error created by an error event.
+   * This error will turn into a rejection for the subsequent .next() call
+   */
+  let error: Error | null = null;
+
+  /** Set to true only after event listeners have been removed. */
+  let finished = false;
+
+  const iterator: AsyncGenerator<Buffer> & AsyncDisposable = {
+    next() {
+      // First, we consume all unread events
+      const value = unconsumedEvents.shift();
+      if (value != null) {
+        return Promise.resolve({ value, done: false });
+      }
+
+      // Then we error, if an error happened
+      // This happens one time if at all, because after 'error'
+      // we stop listening
+      if (error != null) {
+        const p = Promise.reject(error);
+        // Only the first element errors
+        error = null;
+        return p;
+      }
+
+      // If the iterator is finished, resolve to done
+      if (finished) return closeHandler();
+
+      // Wait until an event happens
+      const { promise, resolve, reject } = promiseWithResolvers<IteratorResult<Buffer>>();
+      unconsumedPromises.push({ resolve, reject });
+      return promise;
+    },
+
+    return() {
+      return closeHandler();
+    },
+
+    throw(err: Error) {
+      errorHandler(err);
+      return Promise.resolve({ value: undefined, done: true });
+    },
+
+    [Symbol.asyncIterator]() {
+      return this;
+    },
+
+    async [Symbol.asyncDispose]() {
+      await closeHandler();
+    }
+  };
+
+  // Adding event handlers
+  emitter.on('data', eventHandler);
+  emitter.on('error', errorHandler);
+  const abortListener = addAbortListener(signal, function () {
+    errorHandler(this.reason);
+  });
+
+  const timeoutForSocketRead = timeoutContext?.timeoutForSocketRead;
+  timeoutForSocketRead?.throwIfExpired();
+  timeoutForSocketRead?.then(undefined, errorHandler);
+
+  return iterator;
+
+  function eventHandler(value: Buffer) {
+    const promise = unconsumedPromises.shift();
+    if (promise != null) promise.resolve({ value, done: false });
+    else unconsumedEvents.push(value);
+  }
+
+  function errorHandler(err: Error) {
+    const promise = unconsumedPromises.shift();
+
+    if (promise != null) promise.reject(err);
+    else error = err;
+    void closeHandler();
+  }
+
+  function closeHandler() {
+    // Adding event handlers
+    emitter.off('data', eventHandler);
+    emitter.off('error', errorHandler);
+    abortListener?.[kDispose]();
+    finished = true;
+    timeoutForSocketRead?.clear();
+    const doneResult = { value: undefined, done: finished } as const;
+
+    for (const promise of unconsumedPromises) {
+      promise.resolve(doneResult);
+    }
+
+    return Promise.resolve(doneResult);
+  }
+}