@fluid-experimental/property-dds 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluid-experimental/property-dds",
-  "version": "2.0.0-dev.5.2.0.169897",
+  "version": "2.0.0-dev.6.4.0.191258",
   "description": "definition of the property distributed data store",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -15,16 +15,16 @@
   "module": "lib/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
-    "@fluid-experimental/property-changeset": "2.0.0-dev.5.2.0.169897",
-    "@fluid-experimental/property-properties": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/common-utils": "^1.1.1",
-    "@fluidframework/container-definitions": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/core-interfaces": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/datastore-definitions": "2.0.0-dev.5.2.0.169897",
+    "@fluid-experimental/property-changeset": "2.0.0-dev.6.4.0.191258",
+    "@fluid-experimental/property-properties": "2.0.0-dev.6.4.0.191258",
+    "@fluid-internal/client-utils": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/container-definitions": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/core-interfaces": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/datastore-definitions": "2.0.0-dev.6.4.0.191258",
     "@fluidframework/protocol-definitions": "^1.1.0",
-    "@fluidframework/runtime-definitions": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/runtime-utils": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/shared-object-base": "2.0.0-dev.5.2.0.169897",
+    "@fluidframework/runtime-definitions": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/runtime-utils": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/shared-object-base": "2.0.0-dev.6.4.0.191258",
     "axios": "^0.26.0",
     "buffer": "^6.0.3",
     "fastest-json-copy": "^1.0.1",
@@ -32,27 +32,27 @@
     "lz4js": "^0.2.0",
     "msgpackr": "^1.4.7",
     "pako": "^2.0.4",
-    "uuid": "^8.3.1"
+    "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "@fluid-experimental/property-common": "2.0.0-dev.5.2.0.169897",
-    "@fluid-internal/test-drivers": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/build-common": "^1.2.0",
-    "@fluidframework/container-loader": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/driver-definitions": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/eslint-config-fluid": "^2.0.0",
-    "@fluidframework/local-driver": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/mocha-test-setup": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/sequence": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/server-local-server": "^0.1039.1000",
-    "@fluidframework/test-runtime-utils": "2.0.0-dev.5.2.0.169897",
-    "@fluidframework/test-utils": "2.0.0-dev.5.2.0.169897",
+    "@fluid-experimental/property-common": "2.0.0-dev.6.4.0.191258",
+    "@fluid-internal/test-drivers": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/build-common": "^2.0.0",
+    "@fluidframework/build-tools": "^0.22.0",
+    "@fluidframework/container-loader": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/driver-definitions": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/eslint-config-fluid": "^2.1.0",
+    "@fluidframework/local-driver": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/mocha-test-setup": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/sequence": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/server-local-server": "^1.0.1",
+    "@fluidframework/test-runtime-utils": "2.0.0-dev.6.4.0.191258",
+    "@fluidframework/test-utils": "2.0.0-dev.6.4.0.191258",
     "@microsoft/api-extractor": "^7.34.4",
     "@types/lodash": "^4.14.118",
     "@types/mocha": "^9.1.1",
-    "@types/node": "^14.18.38",
+    "@types/node": "^16.18.38",
     "chai": "^4.2.0",
-    "concurrently": "^7.6.0",
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.3",
     "easy-table": "^1.1.1",
@@ -62,7 +62,6 @@
     "mocha-json-output-reporter": "^2.0.1",
     "mocha-multi-reporters": "^1.5.1",
     "moment": "^2.21.0",
-    "nyc": "^15.1.0",
     "prettier": "~2.6.2",
     "rimraf": "^4.4.0",
     "typescript": "~4.5.5"
@@ -78,7 +77,7 @@
     "build:esnext": "tsc --project ./tsconfig.esnext.json",
     "build:test": "tsc --project ./src/test/tsconfig.json",
     "ci:build:docs": "api-extractor run --typescript-compiler-folder ../../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../../_api-extractor-temp/",
-    "clean": "rimraf dist lib *.tsbuildinfo *.build.log",
+    "clean": "rimraf --glob 'dist' 'lib' '*.tsbuildinfo' '*.build.log' '_api-extractor-temp' 'nyc'",
     "eslint": "eslint src",
     "eslint:fix": "eslint src --fix",
     "format": "npm run prettier:fix",
@@ -87,9 +86,9 @@
     "prettier": "prettier --check . --ignore-path ../../../../.prettierignore",
     "prettier:fix": "prettier --write . --ignore-path ../../../../.prettierignore",
     "test": "npm run test:mocha",
-    "test:mocha": "mocha \"dist/**/*.spec.js\" --exit -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
-    "test:mocha-ts": "cross-env FLUID_TEST_VERBOSE=1 TS_NODE_PROJECT=\"./src/test/tsconfig.json\" mocha --require ts-node/register --extensions ts,tsx  \"src/test/**/*.spec.ts\" --exit -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict --timeout 1500000",
-    "test:mocha:multireport": "cross-env FLUID_TEST_MULTIREPORT=1 npm run test:mocha",
+    "test:mocha": "mocha \"dist/**/*.spec.js\" --exit -r node_modules/@fluidframework/mocha-test-setup",
+    "test:mocha-ts": "cross-env FLUID_TEST_VERBOSE=1 TS_NODE_PROJECT=\"./src/test/tsconfig.json\" mocha --require ts-node/register --extensions ts,tsx  \"src/test/**/*.spec.ts\" --exit -r node_modules/@fluidframework/mocha-test-setup --timeout 1500000",
+    "test:mocha-ts-inspect": "cross-env FLUID_TEST_VERBOSE=1 TS_NODE_PROJECT=\"./src/test/tsconfig.json\" mocha --inspect-brk --require ts-node/register --extensions ts,tsx  \"src/test/**/*.spec.ts\" --exit -r node_modules/@fluidframework/mocha-test-setup --timeout 1500000",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
     "tsc": "tsc"
   }

package/src/propertyTree.ts

@@ -6,6 +6,8 @@
 /* eslint-disable import/no-internal-modules */
 import isEmpty from "lodash/isEmpty";
 import findIndex from "lodash/findIndex";
+import find from "lodash/find";
+import isEqual from "lodash/isEqual";
 import range from "lodash/range";
 import { copy as cloneDeep } from "fastest-json-copy";
 import { Packr } from "msgpackr";
@@ -19,7 +21,7 @@ import {
 	IChannelFactory,
 } from "@fluidframework/datastore-definitions";
 
-import { bufferToString, stringToBuffer } from "@fluidframework/common-utils";
+import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
 import { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 import { IFluidSerializer, SharedObject } from "@fluidframework/shared-object-base";
 import { SummaryTreeBuilder } from "@fluidframework/runtime-utils";
@@ -84,6 +86,7 @@ export interface SharedPropertyTreeOptions {
 	paths?: string[];
 	clientFiltering?: boolean;
 	useMH?: boolean;
+	disablePartialCheckout?: boolean;
 }
 
 export interface ISharedPropertyTreeEncDec {
@@ -195,12 +198,14 @@ export class SharedPropertyTree extends SharedObject {
 	private scopeFutureDeltasToPaths(paths?: string[]) {
 		// Backdoor to emit "partial_checkout" events on the socket. The delta manager at container runtime layer is
 		// a proxy and the delta manager at the container context layer is yet another proxy, so account for that.
-		let dm = (this.runtime.deltaManager as any).deltaManager;
-		if (dm.deltaManager !== undefined) {
-			dm = dm.deltaManager;
+		if (!this.options.disablePartialCheckout) {
+			let dm = (this.runtime.deltaManager as any).deltaManager;
+			if (dm.deltaManager !== undefined) {
+				dm = dm.deltaManager;
+			}
+			const socket = dm.connectionManager.connection.socket;
+			socket.emit("partial_checkout", { paths });
 		}
-		const socket = dm.connectionManager.connection.socket;
-		socket.emit("partial_checkout", { paths });
 	}
 
 	public _reportDirtinessToView() {
@@ -678,6 +683,7 @@ export class SharedPropertyTree extends SharedObject {
 				const lastDelta = commitMetadata.sequenceNumber;
 
 				const dm = (this.runtime.deltaManager as any).deltaManager;
+				// TODO: This is accessing a private member of the delta manager, and should not be.
 				await dm.getDeltas(
 					"DocumentOpen",
 					firstDelta,
@@ -690,8 +696,11 @@ export class SharedPropertyTree extends SharedObject {
 				// eslint-disable-next-line @typescript-eslint/prefer-for-of
 				for (let i = 0; i < missingDeltas.length; i++) {
 					if (missingDeltas[i].sequenceNumber < commitMetadata.sequenceNumber) {
+						// TODO: Don't spy on the DeltaManager's private internals.
+						// This is trying to mimic what DeltaManager does in processInboundMessage, but there's no guarantee that
+						// private implementation won't change.
 						const remoteChange: IPropertyTreeMessage = JSON.parse(
-							missingDeltas[i].contents,
+							missingDeltas[i].contents as string,
 						).contents.contents.content.contents;
 						const { changeSet } = (
 							await axios.get(
@@ -791,6 +800,10 @@ export class SharedPropertyTree extends SharedObject {
 
 	getRebasedChanges(startGuid: string, endGuid?: string) {
 		const startIndex = findIndex(this.remoteChanges, (c) => c.guid === startGuid);
+		if (startIndex === -1 && startGuid !== "") {
+			// TODO: Consider throwing an error once clients have picked up PR #16277.
+			console.error("Unknown start GUID specified.");
+		}
 		if (endGuid !== undefined) {
 			const endIndex = findIndex(this.remoteChanges, (c) => c.guid === endGuid);
 			return this.remoteChanges.slice(startIndex + 1, endIndex + 1);
@@ -803,7 +816,7 @@ export class SharedPropertyTree extends SharedObject {
 		pendingChanges: SerializedChangeSet,
 		newTipDelta: SerializedChangeSet,
 	): boolean {
-		let rebaseBaseChangeSet = cloneDeep(change.changeSet);
+		let rebaseBaseChangeSet;
 
 		const accumulatedChanges: SerializedChangeSet = {};
 		const conflicts = [] as any[];
@@ -813,15 +826,29 @@ export class SharedPropertyTree extends SharedObject {
 			// assert(JSON.stringify(this.localChanges[0].changeSet) === JSON.stringify(change.changeSet),
 			//        "Local change different than rebased remote change.");
 
-			// If we got a confirmation of the commit on the tip of the localChanges array,
-			// there will be no update of the tip view at all. We just move it from local changes
-			// to remote changes
-			this.localChanges.shift();
+			if (isEqual(this.localChanges[0].changeSet, change.changeSet)) {
+				// If we got a confirmation of the commit on the tip of the localChanges array,
+				// there will be no update of the tip view at all. We just move it from local changes
+				// to remote changes
+				this.localChanges.shift();
 
-			return false;
+				return false;
+			} else {
+				// There is a case where the localChanges that were created by incrementally rebasing with respect
+				// to every incoming change do no exactly agree with the rebased remote change (this happens
+				// when there are changes that cancel out with each other that have happened in the meantime).
+				// In that case, we must make sure, we correctly update the local view to take this difference into
+				// account by rebasing with respect to the changeset that is obtained by combining the inverse of the
+				// local change with the incoming remote change.
+
+				rebaseBaseChangeSet = new ChangeSet(this.localChanges.shift()?.changeSet);
+				rebaseBaseChangeSet.toInverseChangeSet();
+				rebaseBaseChangeSet.applyChangeSet(change.changeSet);
+			}
+		} else {
+			rebaseBaseChangeSet = cloneDeep(change.changeSet);
 		}
 
-		// eslint-disable-next-line @typescript-eslint/prefer-for-of
 		for (let i = 0; i < this.localChanges.length; i++) {
 			// Make sure we never receive changes out of order
 			console.assert(this.localChanges[i].guid !== change.guid);
@@ -845,6 +872,12 @@ export class SharedPropertyTree extends SharedObject {
 			rebaseBaseChangeSet = copiedChangeSet.getSerializedChangeSet();
 
 			new ChangeSet(accumulatedChanges).applyChangeSet(this.localChanges[i].changeSet);
+
+			// Update the reference and head guids
+			this.localChanges[i].remoteHeadGuid = change.guid;
+			if (i === 0) {
+				this.localChanges[i].referenceGuid = change.guid;
+			}
 		}
 
 		// Compute the inverse of the pending changes and store the result in newTipDelta
@@ -874,6 +907,26 @@ export class SharedPropertyTree extends SharedObject {
 		return true;
 	}
 
+	protected reSubmitCore(content: any, localOpMetadata: unknown) {
+		// We have to provide our own implementation of the resubmit core function, to
+		// handle the case where an operation is no longer referencing a commit within
+		// the collaboration window as its referenceGuid. Other clients would not be
+		// able to perform the rebase for such an operation. To handle this problem
+		// we have to resubmit a version of the operations which has been rebased to
+		// the current remote tip. We already have these rebased versions of the operations
+		// in our localChanges, because we continuously update those to follow the tip.
+		// Therefore our reSubmitCore function searches for the rebased operation in the
+		// localChanges array and submits this up-to-date version instead of the old operation.
+		const rebasedOperation = find(this.localChanges, (op) => op.guid === content.guid);
+
+		if (rebasedOperation) {
+			this.submitLocalMessage(cloneDeep(rebasedOperation), localOpMetadata);
+		} else {
+			// Could this happen or is there a guard that we will never resubmit an already submitted op?
+			console.warn("Resubmitting operation which has already been received back.");
+		}
+	}
+
 	protected applyStashedOp() {
 		throw new Error("not implemented");
 	}

package/src/propertyTreeExtFactories.ts

@@ -5,7 +5,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 import { deflate, inflate } from "pako";
 import { compress, decompress } from "lz4js";
-import { bufferToString, stringToBuffer } from "@fluidframework/common-utils";
+import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
 import {
 	IChannelAttributes,
 	IFluidDataStoreRuntime,