@fluidframework/odsp-driver 1.2.7 → 2.0.0-dev.1.3.0.96595

package/src/WriteBufferUtils.ts

@@ -3,8 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert, unreachableCase } from "@fluidframework/common-utils";
-import { ReadBuffer } from "./ReadBufferUtils";
+import { assert, IsoBuffer } from "@fluidframework/common-utils";
 import {
     BlobCore,
     codeToBytesMap,
@@ -13,7 +12,6 @@ import {
     MarkerCodesEnd,
     MarkerCodesStart,
     NodeCore,
-    TreeBuilder,
 } from "./zipItDataRepresentationUtils";
 
 /**
@@ -52,11 +50,11 @@ export class WriteBuffer {
         assert(code === 0, 0x226 /* Should write complete data */);
     }
 
-    public done(): ReadBuffer {
+    public done(): Uint8Array {
         assert(this.data !== undefined, 0x227 /* "Data should be there" */);
         // We can slice it to have smaller memory representation.
         // But it will be way more expensive in terms of CPU cycles!
-        const buffer = new ReadBuffer(this.data.subarray(0, this.index));
+        const buffer = this.data.subarray(0, this.index);
         this.data = undefined;
         return buffer;
     }
@@ -143,56 +141,66 @@ const boolToCodeMap = [
 /**
  * Implementation of serialization of blobs in buffer with Marker Codes etc.
  * @param buffer - Buffer to serialize to.
- * @param blob - blob to be serialized.
+ * @param content - string to be serialized.
  * @param dictionary - Const strings dictionary to be used while serializing.
  */
-function serializeBlob(buffer: WriteBuffer, blob: BlobCore, dictionary: Map<string, number>) {
-    const data = blob.buffer;
-    const lengthOfDataLen = calcLength(data.length);
-    if (blob.constString) {
-        const content = blob.toString();
-        let id = dictionary.get(content);
-        let idLength: number;
-        if (id === undefined) {
-            id = dictionary.size + 1;
-            idLength = calcLength(id);
-            dictionary.set(content, id);
-            let code: number;
-            if (lengthOfDataLen > 1 || idLength > 1) {
-                code = MarkerCodes.ConstStringDeclareBig;
-            } else {
-                code = MarkerCodes.ConstStringDeclare;
-            }
-            // Write marker code for const string.
-            buffer.write(code);
-            const bytes = getValueSafely(codeToBytesMap, code);
-            assert(bytes >= lengthOfDataLen, 0x283 /* "Length of data len should fit in the bytes from the map" */);
-            assert(bytes >= idLength, 0x284 /* "Length of id should fit in the bytes from the map" */);
-            // Assign and write id for const string.
-            buffer.write(id, bytes);
-            // Write length of const string.
-            buffer.write(data.length, bytes);
-            // Write const string data.
-            for (const element of data) {
-                buffer.write(element);
-            }
-        } else {
-            idLength = calcLength(id);
+ function serializeDictionaryString(buffer: WriteBuffer, content: string, dictionary: Map<string, number>) {
+    let id = dictionary.get(content);
+    let idLength: number;
+    if (id === undefined) {
+        const data = IsoBuffer.from(content, "utf8");
+        const lengthOfDataLen = calcLength(data.length);
+
+        id = dictionary.size + 1;
+        idLength = calcLength(id);
+        dictionary.set(content, id);
+        const code = lengthOfDataLen > 1 || idLength > 1
+            ? MarkerCodes.ConstStringDeclareBig
+            : MarkerCodes.ConstStringDeclare;
+        // Write marker code for const string.
+        buffer.write(code);
+        const bytes = getValueSafely(codeToBytesMap, code);
+        assert(bytes >= lengthOfDataLen, 0x283 /* "Length of data len should fit in the bytes from the map" */);
+        assert(bytes >= idLength, 0x284 /* "Length of id should fit in the bytes from the map" */);
+        // Assign and write id for const string.
+        buffer.write(id, bytes);
+        // Write length of const string.
+        buffer.write(data.length, bytes);
+        // Write const string data.
+        for (const element of data) {
+            buffer.write(element);
         }
-        // Write Marker Code
-        buffer.write(getValueSafely(constStringBytesToCodeMap, idLength));
-        // Write id of const string
-        buffer.write(id, idLength);
     } else {
-        // Write Marker code.
-        buffer.write(blob.useUtf8Code ? getValueSafely(utf8StringBytesToCodeMap, lengthOfDataLen)
-            : getValueSafely(binaryBytesToCodeMap, lengthOfDataLen));
-        // Write actual data if length greater than 0, otherwise Marker Code is enough.
-        if (lengthOfDataLen > 0) {
-            buffer.write(data.length, lengthOfDataLen);
-            for (const element of data) {
-                buffer.write(element);
-            }
+        idLength = calcLength(id);
+    }
+    // Write Marker Code
+    buffer.write(getValueSafely(constStringBytesToCodeMap, idLength));
+    // Write id of const string
+    buffer.write(id, idLength);
+}
+
+function serializeString(buffer: WriteBuffer, content: string, codeMap = binaryBytesToCodeMap) {
+    serializeBlob(
+        buffer,
+        IsoBuffer.from(content, "utf8"),
+        utf8StringBytesToCodeMap);
+}
+
+/**
+ * Implementation of serialization of blobs in buffer with Marker Codes etc.
+ * @param buffer - Buffer to serialize to.
+ * @param blob - blob to be serialized.
+ * @param dictionary - Const strings dictionary to be used while serializing.
+ */
+function serializeBlob(buffer: WriteBuffer, data: Uint8Array, codeMap: Record<number, number> = binaryBytesToCodeMap) {
+    const lengthOfDataLen = calcLength(data.length);
+    // Write Marker code.
+    buffer.write(getValueSafely(codeMap, lengthOfDataLen));
+    // Write actual data if length greater than 0, otherwise Marker Code is enough.
+    if (lengthOfDataLen > 0) {
+        buffer.write(data.length, lengthOfDataLen);
+        for (const element of data) {
+            buffer.write(element);
         }
     }
 }
@@ -215,7 +223,7 @@ function serializeNodeCore(buffer: WriteBuffer, nodeCore: NodeCore, dictionary:
             serializeNodeCore(buffer, child, dictionary);
             buffer.write(endCode);
         } else if (child instanceof BlobCore) {
-            serializeBlob(buffer, child, dictionary);
+            serializeBlob(buffer, child.buffer);
         } else if (typeof child === "number") {
             // Calculate length in which integer will be stored
             const len = calcLength(child);
@@ -228,7 +236,12 @@ function serializeNodeCore(buffer: WriteBuffer, nodeCore: NodeCore, dictionary:
         } else if (typeof child === "boolean") {
             buffer.write(boolToCodeMap[child ? 1 : 0]);
         } else {
-            unreachableCase(child, "Unsupported node type");
+            assert(child._stringElement, 0x3dd /* Unsupported node type */);
+            if (child.dictionary) {
+                serializeDictionaryString(buffer, child.content, dictionary);
+            } else {
+                serializeString(buffer, child.content);
+            }
         }
     }
 }
@@ -248,11 +261,7 @@ export class TreeBuilderSerializer extends NodeCoreSerializer {
         super();
     }
 
-    static load(buffer: ReadBuffer): TreeBuilder {
-        return TreeBuilder.load(buffer);
-    }
-
-    public serialize(): ReadBuffer {
+    public serialize(): Uint8Array {
         const buffer = new WriteBuffer();
         super.serialize(buffer);
         return buffer.done();

package/src/compactSnapshotParser.ts

@@ -5,15 +5,16 @@
 
 import { assert } from "@fluidframework/common-utils";
 import { ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions";
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { ISnapshotContents } from "./odspPublicUtils";
 import { ReadBuffer } from "./ReadBufferUtils";
-import { ISnapshotTreeEx } from "./contracts";
 import {
     assertBlobCoreInstance,
+    getStringInstance,
     assertBoolInstance,
     assertNodeCoreInstance,
     assertNumberInstance,
-    getAndValidateNodeProps,
+    getNodeProps,
     NodeCore,
     NodeTypes,
     TreeBuilder,
@@ -34,12 +35,28 @@ interface ISnapshotSection {
 function readBlobSection(node: NodeTypes) {
     assertNodeCoreInstance(node, "TreeBlobs should be of type NodeCore");
     const blobs: Map<string, ArrayBuffer> = new Map();
-    for (let count = 0; count < node.length; ++count) {
-        const blob = node.getNode(count);
-        const records = getAndValidateNodeProps(blob, ["id", "data"]);
+    for (const blob of node) {
+        assertNodeCoreInstance(blob, "blob should be node");
+
+        /**
+         * Perf optimization - the most common cases!
+         * This is essentially unrolling code below for faster processing
+         * It speeds up tree parsing by 2-3x times!
+         */
+        if (blob.length === 4 && blob.getMaybeString(0) === "id" && blob.getMaybeString(2) === "data") {
+            // "id": <node name>
+            // "data": <blob>
+            blobs.set(blob.getString(1), blob.getBlob(3).arrayBuffer);
+            continue;
+        }
+
+        /**
+         * More generalized workflow
+         */
+        const records = getNodeProps(blob);
         assertBlobCoreInstance(records.data, "data should be of BlobCore type");
-        assertBlobCoreInstance(records.id, "blob id should be of BlobCore type");
-        blobs.set(records.id.toString(), records.data.arrayBuffer);
+        const id = getStringInstance(records.id, "blob id should be string");
+        blobs.set(id, records.data.arrayBuffer);
     }
     return blobs;
 }
@@ -51,7 +68,7 @@ function readBlobSection(node: NodeTypes) {
 function readOpsSection(node: NodeTypes) {
     assertNodeCoreInstance(node, "Deltas should be of type NodeCore");
     const ops: ISequencedDocumentMessage[] = [];
-    const records = getAndValidateNodeProps(node, ["firstSequenceNumber", "deltas"]);
+    const records = getNodeProps(node);
     assertNumberInstance(records.firstSequenceNumber, "Seq number should be a number");
     assertNodeCoreInstance(records.deltas, "Deltas should be a Node");
     for (let i = 0; i < records.deltas.length; ++i) {
@@ -67,31 +84,79 @@ function readOpsSection(node: NodeTypes) {
  * @param node - tree node to de-serialize from
  */
 function readTreeSection(node: NodeCore) {
-    const snapshotTree: ISnapshotTreeEx = {
+    const trees = {};
+    const snapshotTree: ISnapshotTree = {
         blobs: {},
-        commits: {},
-        trees: {},
+        trees,
     };
-    for (let count = 0; count < node.length; count++) {
-        const treeNode = node.getNode(count);
-        const records = getAndValidateNodeProps(treeNode,
-            ["name", "value", "children", "unreferenced"], false);
-        assertBlobCoreInstance(records.name, "Path should be of BlobCore");
-        const path = records.name.toString();
+    for (const treeNode of node) {
+        assertNodeCoreInstance(treeNode, "tree nodes should be nodes");
+
+        /**
+         * Perf optimization - the most common cases!
+         * This is essentially unrolling code below for faster processing
+         * It speeds up tree parsing by 2-3x times!
+         */
+         const length = treeNode.length;
+         if (length > 0 && treeNode.getMaybeString(0) === "name") {
+            if (length === 4) {
+                const content = treeNode.getMaybeString(2);
+                // "name": <node name>
+                // "children": <blob id>
+                if (content === "children") {
+                    trees[treeNode.getString(1)] = readTreeSection(treeNode.getNode(3));
+                    continue;
+                }
+                // "name": <node name>
+                // "value": <blob id>
+                if (content === "value") {
+                    snapshotTree.blobs[treeNode.getString(1)] = treeNode.getString(3);
+                    continue;
+                }
+            }
+
+            // "name": <node name>
+            // "nodeType": 3
+            // "value": <blob id>
+            if (length === 6 &&
+                    treeNode.getMaybeString(2) === "nodeType" &&
+                    treeNode.getMaybeString(4) === "value") {
+                snapshotTree.blobs[treeNode.getString(1)] = treeNode.getString(5);
+                continue;
+            }
+
+            // "name": <node name>
+            // "unreferenced": true
+            // "children": <blob id>
+            if (length === 6 &&
+                    treeNode.getMaybeString(2) === "unreferenced" &&
+                    treeNode.getMaybeString(4) === "children") {
+                trees[treeNode.getString(1)] = readTreeSection(treeNode.getNode(5));
+                assert(treeNode.getBool(3), 0x3db /* Unreferenced if present should be true */);
+                snapshotTree.unreferenced = true;
+                continue;
+            }
+        }
+
+        /**
+         * More generalized workflow
+         */
+        const records = getNodeProps(treeNode);
+
+        if (records.unreferenced !== undefined) {
+            assertBoolInstance(records.unreferenced, "Unreferenced flag should be bool");
+            assert(records.unreferenced, 0x281 /* "Unreferenced if present should be true" */);
+            snapshotTree.unreferenced = true;
+        }
+
+        const path = getStringInstance(records.name, "Path name should be string");
         if (records.value !== undefined) {
-            assertBlobCoreInstance(records.value, "Blob value should be BlobCore");
-            snapshotTree.blobs[path] = records.value.toString();
+            snapshotTree.blobs[path] = getStringInstance(records.value, "Blob value should be string");
         } else if (records.children !== undefined) {
             assertNodeCoreInstance(records.children, "Trees should be of type NodeCore");
-            snapshotTree.trees[path] = readTreeSection(records.children);
+            trees[path] = readTreeSection(records.children);
         } else {
-            snapshotTree.trees[path] = { blobs: {}, commits: {}, trees: {} };
-        }
-        if (records.unreferenced !== undefined) {
-            assertBoolInstance(records.unreferenced, "Unreferenced flag should be bool");
-            const unreferenced = records.unreferenced.valueOf();
-            assert(unreferenced, 0x281 /* "Unreferenced if present should be true" */);
-            snapshotTree.unreferenced = unreferenced;
+            trees[path] = { blobs: {}, trees: {} };
         }
     }
     return snapshotTree;
@@ -103,14 +168,12 @@ function readTreeSection(node: NodeCore) {
  */
 function readSnapshotSection(node: NodeTypes): ISnapshotSection {
     assertNodeCoreInstance(node, "Snapshot should be of type NodeCore");
-    const records = getAndValidateNodeProps(node,
-        ["id", "sequenceNumber", "treeNodes"]);
+    const records = getNodeProps(node);
 
     assertNodeCoreInstance(records.treeNodes, "TreeNodes should be of type NodeCore");
     assertNumberInstance(records.sequenceNumber, "sequenceNumber should be of type number");
-    assertBlobCoreInstance(records.id, "snapshotId should be BlobCore");
     const snapshotTree: ISnapshotTree = readTreeSection(records.treeNodes);
-    snapshotTree.id = records.id.toString();
+    snapshotTree.id = getStringInstance(records.id, "snapshotId should be string");
     const sequenceNumber = records.sequenceNumber.valueOf();
     return {
         sequenceNumber,
@@ -123,25 +186,24 @@ function readSnapshotSection(node: NodeTypes): ISnapshotSection {
  * @param buffer - Compact snapshot to be parsed into tree/blobs/ops.
  * @returns - tree, blobs and ops from the snapshot.
  */
-export function parseCompactSnapshotResponse(buffer: ReadBuffer): ISnapshotContents {
-    const builder = TreeBuilder.load(buffer);
+export function parseCompactSnapshotResponse(buffer: Uint8Array, logger: ITelemetryLogger): ISnapshotContents {
+    const builder = TreeBuilder.load(new ReadBuffer(buffer), logger);
     assert(builder.length === 1, 0x219 /* "1 root should be there" */);
     const root = builder.getNode(0);
 
-    const records = getAndValidateNodeProps(root,
-        ["mrv", "cv", "lsn", "snapshot", "blobs", "deltas"], false);
+    const records = getNodeProps(root);
 
-    assertBlobCoreInstance(records.mrv, "minReadVersion should be of BlobCore type");
-    assertBlobCoreInstance(records.cv, "createVersion should be of BlobCore type");
+    const mrv = getStringInstance(records.mrv, "minReadVersion should be string");
+    const cv = getStringInstance(records.cv, "createVersion should be string");
     if (records.lsn !== undefined) {
         assertNumberInstance(records.lsn, "lsn should be a number");
     }
 
-    assert(parseFloat(snapshotMinReadVersion) >= parseFloat(records.mrv.toString()),
+    assert(parseFloat(snapshotMinReadVersion) >= parseFloat(mrv),
         0x20f /* "Driver min read version should >= to server minReadVersion" */);
-    assert(parseFloat(records.cv.toString()) >= parseFloat(snapshotMinReadVersion),
+    assert(parseFloat(cv) >= parseFloat(snapshotMinReadVersion),
         0x210 /* "Snapshot should be created with minReadVersion or above" */);
-    assert(currentReadVersion === records.cv.toString(),
+    assert(currentReadVersion === cv,
         0x2c2 /* "Create Version should be equal to currentReadVersion" */);
 
     return {

package/src/compactSnapshotWriter.ts

@@ -7,9 +7,14 @@ import { assert, stringToBuffer } from "@fluidframework/common-utils";
 import { IBlob, ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions";
 import { snapshotMinReadVersion } from "./compactSnapshotParser";
 import { ISnapshotContents } from "./odspPublicUtils";
-import { ReadBuffer } from "./ReadBufferUtils";
 import { TreeBuilderSerializer } from "./WriteBufferUtils";
-import { addBoolProperty, addNumberProperty, addStringProperty, NodeCore } from "./zipItDataRepresentationUtils";
+import {
+    addBoolProperty,
+    addNumberProperty,
+    addStringProperty,
+    addDictionaryStringProperty,
+    NodeCore,
+} from "./zipItDataRepresentationUtils";
 
 /**
  * Writes header section of the snapshot.
@@ -28,16 +33,16 @@ function writeSnapshotProps(node: NodeCore, latestSequenceNumber: number) {
  * @param blobs - blobs that is being serialized
 */
 function writeBlobsSection(snapshotNode: NodeCore, blobs: Map<string, IBlob | ArrayBuffer>) {
-    snapshotNode.addString("blobs", true);
+    snapshotNode.addDictionaryString("blobs");
     const blobsNode = snapshotNode.addNode("list");
     for (const [storageBlobId, blob] of blobs) {
         const blobNode = blobsNode.addNode();
-        addStringProperty(blobNode, "id", storageBlobId, true);
-        blobNode.addString("data", true);
+        addDictionaryStringProperty(blobNode, "id", storageBlobId);
+        blobNode.addString("data");
         if (blob instanceof ArrayBuffer) {
-            blobNode.addBlob(new Uint8Array(blob), false);
+            blobNode.addBlob(new Uint8Array(blob));
         } else {
-            blobNode.addBlob(new Uint8Array(stringToBuffer(blob.contents, blob.encoding ?? "utf-8")), false);
+            blobNode.addBlob(new Uint8Array(stringToBuffer(blob.contents, blob.encoding ?? "utf-8")));
         }
     }
 }
@@ -48,7 +53,7 @@ function writeBlobsSection(snapshotNode: NodeCore, blobs: Map<string, IBlob | Ar
  * @param snapshotTree - snapshot tree that is being serialized
 */
 function writeTreeSection(snapshotNode: NodeCore, snapshotTree: ISnapshotTree) {
-    snapshotNode.addString("treeNodes", true);
+    snapshotNode.addDictionaryString("treeNodes");
     const treesNode = snapshotNode.addNode("list");
     writeTreeSectionCore(treesNode, snapshotTree);
 }
@@ -56,13 +61,16 @@ function writeTreeSection(snapshotNode: NodeCore, snapshotTree: ISnapshotTree) {
 function writeTreeSectionCore(treesNode: NodeCore, snapshotTree: ISnapshotTree) {
     for (const [path, value] of Object.entries(snapshotTree.trees)) {
         const treeNode = treesNode.addNode();
-        addStringProperty(treeNode, "name", path);
+        // Many leaf nodes in the tree have same names like "content", "body", "header"
+        // We could be smarter here and not use dictionary where we are sure reuse is unlikely, but
+        // it does not feel like it's worth it.
+        addDictionaryStringProperty(treeNode, "name", path);
         if (snapshotTree.unreferenced) {
             addBoolProperty(treeNode, "unreferenced", snapshotTree.unreferenced);
         }
         // Only write children prop if either blobs or trees are present.
         if (Object.keys(value.blobs).length > 0 || Object.keys(value.trees).length > 0) {
-            treeNode.addString("children", true);
+            treeNode.addDictionaryString("children");
             const childNode = treeNode.addNode("list");
             writeTreeSectionCore(childNode, value);
         }
@@ -71,8 +79,8 @@ function writeTreeSectionCore(treesNode: NodeCore, snapshotTree: ISnapshotTree)
     if (snapshotTree.blobs) {
         for (const [path, id] of Object.entries(snapshotTree.blobs)) {
             const blobNode = treesNode.addNode();
-            addStringProperty(blobNode, "name", path);
-            addStringProperty(blobNode, "value", id, true);
+            addDictionaryStringProperty(blobNode, "name", path);
+            addDictionaryStringProperty(blobNode, "value", id);
         }
     }
 }
@@ -89,7 +97,7 @@ function writeSnapshotSection(
     snapshotTree: ISnapshotTree,
     snapshotSequenceNumber: number,
 ) {
-    rootNode.addString("snapshot", true);
+    rootNode.addDictionaryString("snapshot");
     const snapshotNode = rootNode.addNode();
 
     const snapshotId = snapshotTree.id;
@@ -113,13 +121,13 @@ function writeOpsSection(rootNode: NodeCore, ops: ISequencedDocumentMessage[]) {
         firstSequenceNumber = ops[0].sequenceNumber;
     }
     if (firstSequenceNumber !== undefined) {
-        rootNode.addString("deltas", true);
+        rootNode.addDictionaryString("deltas");
         const opsNode = rootNode.addNode();
         addNumberProperty(opsNode, "firstSequenceNumber", firstSequenceNumber);
-        opsNode.addString("deltas", true);
+        opsNode.addDictionaryString("deltas");
         const deltaNode = opsNode.addNode("list");
         ops.forEach((op) => {
-            deltaNode.addString(JSON.stringify(op), false);
+            deltaNode.addString(JSON.stringify(op));
         });
     }
 }
@@ -129,7 +137,7 @@ function writeOpsSection(rootNode: NodeCore, ops: ISequencedDocumentMessage[]) {
  * @param snapshotContents - snapshot tree contents to serialize
  * @returns - ReadBuffer - binary representation of the data.
  */
-export function convertToCompactSnapshot(snapshotContents: ISnapshotContents): ReadBuffer {
+export function convertToCompactSnapshot(snapshotContents: ISnapshotContents): Uint8Array {
     const builder = new TreeBuilderSerializer();
     // Create the root node.
     const rootNode = builder.addNode();

package/src/contracts.ts

@@ -7,20 +7,6 @@ import * as api from "@fluidframework/protocol-definitions";
 import { HostStoragePolicy } from "@fluidframework/odsp-driver-definitions";
 import { ISnapshotContents } from "./odspPublicUtils";
 
-/** https://portal.microsofticm.com/imp/v3/incidents/details/308931013/home
- * The commits property was removed from protocol-definitions but in order to support back compat, we will
- * temporarily add back in this local structure in order to upload the snapshot to support rolling back to 0.58.
- * Notice this entire interface will be removed once the backward compatibility is not required anymore.
-*/
-export interface ISnapshotTreeEx {
-    id?: string;
-    blobs: { [path: string]: string; };
-    commits: { [path: string]: string; };
-    trees: { [path: string]: ISnapshotTreeEx; };
-    // Indicates that this tree is unreferenced. If this is not present, the tree is considered referenced.
-    unreferenced?: true;
-}
-
 /**
  * Socket storage discovery api response
  */
@@ -216,6 +202,8 @@ export interface ICreateFileResponse {
     itemId: string;
     itemUrl: string;
     sequenceNumber: number;
+    // sharing object contains shareId, sharingLink data or error in the response
+    sharing?: any;
     sharingLink?: string;
     sharingLinkErrorReason?: string;
 }