@fluidframework/container-runtime 2.0.0-dev.1.4.5.105745 → 2.0.0-dev.2.2.0.111723

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.0.0-dev.1.4.5.105745",
+  "version": "2.0.0-dev.2.2.0.111723",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -28,17 +28,19 @@
     "clean": "rimraf dist lib *.tsbuildinfo *.build.log",
     "eslint": "eslint --format stylish src",
     "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
+    "format": "npm run prettier:fix",
     "lint": "npm run eslint",
     "lint:fix": "npm run eslint:fix",
+    "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
+    "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
     "test": "npm run test:mocha",
     "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
     "test:mocha": "mocha --ignore 'dist/test/types/*' --recursive dist/test -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
     "tsc": "tsc",
     "tsc:watch": "tsc --watch",
-    "tsfmt": "tsfmt --verify",
-    "tsfmt:fix": "tsfmt --replace",
-    "typetests:gen": "fluid-type-validator -g -d ."
+    "typetests:gen": "flub generate typetests --generate --dir .",
+    "typetests:prepare": "flub generate typetests --prepare --dir . --pin"
   },
   "nyc": {
     "all": true,
@@ -63,29 +65,31 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^1.0.0",
-    "@fluidframework/container-definitions": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/container-runtime-definitions": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/container-utils": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/core-interfaces": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/datastore": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/driver-definitions": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/driver-utils": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/garbage-collector": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/protocol-base": "^0.1037.2001",
-    "@fluidframework/protocol-definitions": "^1.0.0",
-    "@fluidframework/runtime-definitions": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/runtime-utils": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/telemetry-utils": "2.0.0-dev.1.4.5.105745",
+    "@fluidframework/container-definitions": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/container-runtime-definitions": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/container-utils": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/core-interfaces": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/datastore": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/driver-definitions": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/driver-utils": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/garbage-collector": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/protocol-base": "^0.1038.2000",
+    "@fluidframework/protocol-definitions": "^1.1.0",
+    "@fluidframework/runtime-definitions": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/runtime-utils": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/telemetry-utils": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
     "double-ended-queue": "^2.1.0-0",
+    "lz4js": "^0.2.0",
     "uuid": "^8.3.1"
   },
   "devDependencies": {
-    "@fluidframework/build-common": "^1.0.0",
-    "@fluidframework/build-tools": "^0.4.4000",
-    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@^1.0.0",
-    "@fluidframework/eslint-config-fluid": "^1.0.0",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev.1.4.5.105745",
-    "@fluidframework/test-runtime-utils": "2.0.0-dev.1.4.5.105745",
+    "@fluid-tools/build-cli": "^0.6.0-109663",
+    "@fluidframework/build-common": "^1.1.0",
+    "@fluidframework/build-tools": "^0.6.0-109663",
+    "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.2.1.0",
+    "@fluidframework/eslint-config-fluid": "^1.2.0",
+    "@fluidframework/mocha-test-setup": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
+    "@fluidframework/test-runtime-utils": ">=2.0.0-dev.2.2.0.111723 <2.0.0-dev.3.0.0",
     "@microsoft/api-extractor": "^7.22.2",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/double-ended-queue": "^2.1.0",
@@ -101,54 +105,24 @@
     "nyc": "^15.0.0",
     "rimraf": "^2.6.2",
     "sinon": "^7.4.2",
-    "typescript": "~4.5.5",
-    "typescript-formatter": "7.1.0"
+    "typescript": "~4.5.5"
   },
   "typeValidation": {
-    "version": "2.0.0",
+    "version": "2.0.0-internal.2.2.0",
+    "baselineRange": ">=2.0.0-internal.2.1.0 <2.0.0-internal.2.2.0",
+    "baselineVersion": "2.0.0-internal.2.1.0",
     "broken": {
-      "VariableDeclaration_DefaultSummaryConfiguration": {
-        "forwardCompat": false,
-        "backCompat": false
-      },
-      "TypeAliasDeclaration_ISummaryConfiguration": {
-        "forwardCompat": false,
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISummaryConfigurationHeuristics": {
-        "forwardCompat": false,
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IGeneratedSummaryStats": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IGenerateSummaryTreeResult": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISubmitSummaryOpResult": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IUploadSummaryResult": {
-        "backCompat": false
+      "InterfaceDeclaration_ICompressionRuntimeOptions": {
+        "backCompat": false,
+        "forwardCompat": false
       },
-      "TypeAliasDeclaration_SubmitSummaryResult": {
-        "backCompat": false
+      "InterfaceDeclaration_IContainerRuntimeOptions": {
+        "backCompat": false,
+        "forwardCompat": false
       },
       "ClassDeclaration_ContainerRuntime": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_IConnectableRuntime": {
-        "backCompat": false
-      },
-      "InterfaceDeclaration_ISummarizerRuntime": {
-        "backCompat": false
-      },
-      "RemovedInterfaceDeclaration_IPendingRuntimeState": {
         "forwardCompat": false,
         "backCompat": false
-      },
-      "ClassDeclaration_ScheduleManager": {
-        "forwardCompat": false
       }
     }
   }

package/prettier.config.cjs

@@ -0,0 +1,8 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+module.exports = {
+    ...require("@fluidframework/build-common/prettier.config.cjs"),
+};

package/src/batchManager.ts

@@ -3,8 +3,10 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ContainerRuntimeMessage } from "./containerRuntime";
+import { ContainerRuntimeMessage, ICompressionRuntimeOptions } from "./containerRuntime";
+import { OpCompressor } from "./opCompressor";
 
 /**
  * Message type used by BatchManager
@@ -15,27 +17,28 @@ export type BatchMessage = IBatchMessage & {
     referenceSequenceNumber: number;
 };
 
+export interface IBatchManagerOptions {
+    readonly hardLimit: number;
+    readonly softLimit?: number;
+    readonly compressionOptions?: ICompressionRuntimeOptions;
+}
+
 /**
  * Helper class that manages partial batch & rollback.
  */
 export class BatchManager {
+    private readonly opCompressor: OpCompressor;
     private pendingBatch: BatchMessage [] = [];
     private batchContentSize = 0;
 
-    // The actual limit is 1Mb (socket.io and Kafka limits)
-    // We can't estimate it fully, as we
-    // - do not know what properties relay service will add
-    // - we do not stringify final op, thus we do not know how much escaping will be added.
-    private static readonly hardLimit = 950 * 1024;
-
     public get length() { return this.pendingBatch.length; }
-    public get limit() { return BatchManager.hardLimit; }
-    public static get limit() { return BatchManager.hardLimit; }
 
-    constructor(public readonly softLimit?: number) {}
+    constructor(public readonly logger: ITelemetryLogger, public readonly options: IBatchManagerOptions) {
+        this.opCompressor = new OpCompressor(logger);
+    }
 
     public push(message: BatchMessage): boolean {
-        const contentSize = this.batchContentSize + message.contents.length;
+        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
         const opCount = this.pendingBatch.length;
 
         // Attempt to estimate batch size, aka socket message size.
@@ -48,11 +51,17 @@ export class BatchManager {
         // If we were provided soft limit, check for exceeding it.
         // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
         // and start over. That's not an option if we have no ops.
-        if (this.softLimit !== undefined && this.length > 0 && socketMessageSize >= this.softLimit) {
+        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
+        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
+        if (this.options.softLimit !== undefined
+            && this.length > 0
+            && socketMessageSize >= this.options.softLimit
+            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {
             return false;
         }
 
-        if (socketMessageSize >= this.limit) {
+        if (socketMessageSize >= this.options.hardLimit
+            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {
             return false;
         }
 
@@ -65,22 +74,30 @@ export class BatchManager {
 
     public popBatch() {
         const batch = this.pendingBatch;
+        const size = this.batchContentSize;
         this.pendingBatch = [];
         this.batchContentSize = 0;
+
+        if (batch.length > 0
+            && this.options.compressionOptions !== undefined
+            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {
+            return this.opCompressor.compressBatch(batch, size);
+        }
+
         return batch;
     }
 
     /**
      * Capture the pending state at this point
      */
-     public checkpoint() {
+    public checkpoint() {
         const startPoint = this.pendingBatch.length;
         return {
             rollback: (process: (message: BatchMessage) => void) => {
                 for (let i = this.pendingBatch.length; i > startPoint;) {
                     i--;
                     const message = this.pendingBatch[i];
-                    this.batchContentSize -= message.contents.length;
+                    this.batchContentSize -= message.contents?.length ?? 0;
                     process(message);
                 }