@fluidframework/tree 2.0.0-dev-rc.1.0.0.224419

package/lib/feature-libraries/chunked-forest/codec/chunkDecoding.d.ts

@@ -0,0 +1,61 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { Value } from "../../../core";
+import { TreeChunk } from "../chunk";
+import { EncodedChunkShape, EncodedFieldBatch, EncodedInlineArray, EncodedNestedArray, EncodedTreeShape, EncodedValueShape } from "./format";
+import { ChunkDecoder, StreamCursor } from "./chunkCodecUtilities";
+import { DecoderContext } from "./chunkDecodingGeneric";
+/**
+ * Decode `chunk` into a TreeChunk.
+ */
+export declare function decode(chunk: EncodedFieldBatch): TreeChunk[];
+/**
+ * Decode a node's value from `stream` using its shape.
+ */
+export declare function readValue(stream: StreamCursor, shape: EncodedValueShape): Value;
+/**
+ * Normalize a {@link TreeChunk} into an array.
+ *
+ * Unwraps {@link SequenceChunk}s, and wraps other chunks.
+ */
+export declare function deaggregateChunks(chunk: TreeChunk): TreeChunk[];
+/**
+ * Normalize a {@link TreeChunk}[] into a single TreeChunk.
+ *
+ * Avoids creating nested or less than 2 child {@link SequenceChunk}s.
+ */
+export declare function aggregateChunks(input: TreeChunk[]): TreeChunk;
+/**
+ * Decoder for {@link EncodedNestedArray}s.
+ */
+export declare class NestedArrayDecoder implements ChunkDecoder {
+    private readonly shape;
+    constructor(shape: EncodedNestedArray);
+    decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk;
+}
+/**
+ * Decoder for {@link EncodedInlineArray}s.
+ */
+export declare class InlineArrayDecoder implements ChunkDecoder {
+    private readonly shape;
+    constructor(shape: EncodedInlineArray);
+    decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk;
+}
+/**
+ * Decoder for {@link EncodedAnyShape}s.
+ */
+export declare const anyDecoder: ChunkDecoder;
+/**
+ * Decoder for {@link EncodedTreeShape}s.
+ */
+export declare class TreeDecoder implements ChunkDecoder {
+    private readonly shape;
+    private readonly cache;
+    private readonly type?;
+    private readonly fieldDecoders;
+    constructor(shape: EncodedTreeShape, cache: DecoderContext<EncodedChunkShape>);
+    decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk;
+}
+//# sourceMappingURL=chunkDecoding.d.ts.map

package/lib/feature-libraries/chunked-forest/codec/chunkDecoding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkDecoding.d.ts","sourceRoot":"","sources":["../../../../src/feature-libraries/chunked-forest/codec/chunkDecoding.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAsC,KAAK,EAAE,MAAM,eAAe,CAAC;AAE1E,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAIrC,OAAO,EAEN,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,UAAU,CAAC;AAClB,OAAO,EACN,YAAY,EACZ,YAAY,EAOZ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACN,cAAc,EAGd,MAAM,wBAAwB,CAAC;AAEhC;;GAEG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,SAAS,EAAE,CAO5D;AAqBD;;GAEG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,iBAAiB,GAAG,KAAK,CAgB/E;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAyB/D;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,SAAS,CAU7D;AAED;;GAEG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK;gBAAL,KAAK,EAAE,kBAAkB;IACtD,MAAM,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS;CA0BjF;AAED;;GAEG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK;gBAAL,KAAK,EAAE,kBAAkB;IACtD,MAAM,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS;CASjF;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,YAMxB,CAAC;AAsBF;;GAEG;AACH,qBAAa,WAAY,YAAW,YAAY;IAI9C,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJvB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAA2B;IACjD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+B;gBAE3C,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,cAAc,CAAC,iBAAiB,CAAC;IAWnD,MAAM,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS;CAmCjF"}

package/lib/feature-libraries/chunked-forest/codec/chunkDecoding.js

@@ -0,0 +1,209 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { assert, unreachableCase } from "@fluidframework/core-utils";
+import { assertValidIndex } from "../../../util";
+import { DiscriminatedUnionDispatcher } from "../../../codec";
+import { BasicChunk } from "../basicChunk";
+import { SequenceChunk } from "../sequenceChunk";
+import { emptyChunk } from "../emptyChunk";
+import { getChecked, readStream, readStreamBoolean, readStreamNumber, readStreamStream, readStreamValue, } from "./chunkCodecUtilities";
+import { DecoderContext, decode as genericDecode, readStreamIdentifier, } from "./chunkDecodingGeneric";
+/**
+ * Decode `chunk` into a TreeChunk.
+ */
+export function decode(chunk) {
+    return genericDecode(decoderLibrary, new DecoderContext(chunk.identifiers, chunk.shapes), chunk, anyDecoder);
+}
+const decoderLibrary = new DiscriminatedUnionDispatcher({
+    a(shape, cache) {
+        return new NestedArrayDecoder(shape);
+    },
+    b(shape, cache) {
+        return new InlineArrayDecoder(shape);
+    },
+    c(shape, cache) {
+        return new TreeDecoder(shape, cache);
+    },
+    d(shape) {
+        return anyDecoder;
+    },
+});
+/**
+ * Decode a node's value from `stream` using its shape.
+ */
+export function readValue(stream, shape) {
+    if (shape === undefined) {
+        return readStreamBoolean(stream) ? readStreamValue(stream) : undefined;
+    }
+    else {
+        if (shape === true) {
+            return readStreamValue(stream);
+        }
+        else if (shape === false) {
+            return undefined;
+        }
+        else if (Array.isArray(shape)) {
+            assert(shape.length === 1, 0x734 /* expected a single constant for value */);
+            return shape[0];
+        }
+        else {
+            // EncodedCounter case:
+            unreachableCase(shape, "decoding values as deltas is not yet supported");
+        }
+    }
+}
+/**
+ * Normalize a {@link TreeChunk} into an array.
+ *
+ * Unwraps {@link SequenceChunk}s, and wraps other chunks.
+ */
+export function deaggregateChunks(chunk) {
+    if (chunk === emptyChunk) {
+        return [];
+    }
+    // TODO: when handling of SequenceChunks has better performance (for example in cursors),
+    // consider keeping SequenceChunks here if they are longer than some threshold.
+    if (chunk instanceof SequenceChunk) {
+        // Could return [] here, however the logic in this file is designed to never produce an empty SequenceChunk, so its better to throw an error here to detect bugs.
+        assert(chunk.subChunks.length > 0, 0x735 /* Unexpected empty sequence */);
+        // Logic in this file is designed to never produce an unneeded (single item) SequenceChunks, so its better to throw an error here to detect bugs.
+        assert(chunk.subChunks.length > 1, 0x736 /* Unexpected single item sequence */);
+        for (const sub of chunk.subChunks) {
+            // The logic in this file is designed to never produce an nested SequenceChunks or emptyChunk, so its better to throw an error here to detect bugs.
+            assert(!(sub instanceof SequenceChunk), 0x737 /* unexpected nested sequence */);
+            assert(sub !== emptyChunk, 0x738 /* unexpected empty chunk */);
+            sub.referenceAdded();
+        }
+        chunk.referenceRemoved();
+        return chunk.subChunks;
+    }
+    else {
+        return [chunk];
+    }
+}
+/**
+ * Normalize a {@link TreeChunk}[] into a single TreeChunk.
+ *
+ * Avoids creating nested or less than 2 child {@link SequenceChunk}s.
+ */
+export function aggregateChunks(input) {
+    const chunks = input.flatMap(deaggregateChunks);
+    switch (chunks.length) {
+        case 0:
+            return emptyChunk;
+        case 1:
+            return chunks[0];
+        default:
+            return new SequenceChunk(chunks);
+    }
+}
+/**
+ * Decoder for {@link EncodedNestedArray}s.
+ */
+export class NestedArrayDecoder {
+    constructor(shape) {
+        this.shape = shape;
+    }
+    decode(decoders, stream) {
+        const decoder = decoders[this.shape];
+        // TODO: uniform chunk fast path
+        const chunks = [];
+        const data = readStream(stream);
+        if (typeof data === "number") {
+            // This case means that the array contained only 0-sized items, and was thus encoded as the length of the array.
+            const inner = { data: [], offset: 0 };
+            for (let index = 0; index < data; index++) {
+                chunks.push(decoder.decode(decoders, inner));
+            }
+        }
+        else {
+            assert(Array.isArray(data), 0x739 /* expected number of array for encoding of nested array */);
+            const inner = { data, offset: 0 };
+            while (inner.offset !== inner.data.length) {
+                chunks.push(decoder.decode(decoders, inner));
+            }
+        }
+        return aggregateChunks(chunks);
+    }
+}
+/**
+ * Decoder for {@link EncodedInlineArray}s.
+ */
+export class InlineArrayDecoder {
+    constructor(shape) {
+        this.shape = shape;
+    }
+    decode(decoders, stream) {
+        const length = this.shape.length;
+        const decoder = decoders[this.shape.shape];
+        const chunks = [];
+        for (let index = 0; index < length; index++) {
+            chunks.push(decoder.decode(decoders, stream));
+        }
+        return aggregateChunks(chunks);
+    }
+}
+/**
+ * Decoder for {@link EncodedAnyShape}s.
+ */
+export const anyDecoder = {
+    decode(decoders, stream) {
+        const shapeIndex = readStreamNumber(stream);
+        const decoder = getChecked(decoders, shapeIndex);
+        return decoder.decode(decoders, stream);
+    },
+};
+/**
+ * Get a decoder for fields of a provided (via `shape` and `cache`) {@link EncodedChunkShape}.
+ */
+function fieldDecoder(cache, key, shape) {
+    assertValidIndex(shape, cache.shapes);
+    return (decoders, stream) => [key, decoders[shape].decode(decoders, stream)];
+}
+/**
+ * Decoder for {@link EncodedTreeShape}s.
+ */
+export class TreeDecoder {
+    constructor(shape, cache) {
+        this.shape = shape;
+        this.cache = cache;
+        this.type = shape.type === undefined ? undefined : cache.identifier(shape.type);
+        const fieldDecoders = [];
+        for (const field of shape.fields) {
+            const key = cache.identifier(field.key);
+            fieldDecoders.push(fieldDecoder(cache, key, field.shape));
+        }
+        this.fieldDecoders = fieldDecoders;
+    }
+    decode(decoders, stream) {
+        const type = this.type ?? readStreamIdentifier(stream, this.cache);
+        // TODO: Consider typechecking against stored schema in here somewhere.
+        const value = readValue(stream, this.shape.value);
+        const fields = new Map();
+        // Helper to add fields, but with unneeded array chunks removed.
+        function addField(key, data) {
+            // TODO: when handling of ArrayChunks has better performance (for example in cursors),
+            // consider keeping array chunks here if they are longer than some threshold.
+            const chunks = deaggregateChunks(data);
+            if (chunks.length !== 0) {
+                fields.set(key, chunks);
+            }
+        }
+        for (const field of this.fieldDecoders) {
+            const [key, content] = field(decoders, stream);
+            addField(key, content);
+        }
+        if (this.shape.extraFields !== undefined) {
+            const decoder = decoders[this.shape.extraFields];
+            const inner = readStreamStream(stream);
+            while (inner.offset !== inner.data.length) {
+                const key = readStreamIdentifier(inner, this.cache);
+                addField(key, decoder.decode(decoders, inner));
+            }
+        }
+        return new BasicChunk(type, fields, value);
+    }
+}
+//# sourceMappingURL=chunkDecoding.js.map

package/lib/feature-libraries/chunked-forest/codec/chunkDecoding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkDecoding.js","sourceRoot":"","sources":["../../../../src/feature-libraries/chunked-forest/codec/chunkDecoding.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjD,OAAO,EAAE,4BAA4B,EAAE,MAAM,gBAAgB,CAAC;AAE9D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAU3C,OAAO,EAGN,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,GACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACN,cAAc,EACd,MAAM,IAAI,aAAa,EACvB,oBAAoB,GACpB,MAAM,wBAAwB,CAAC;AAEhC;;GAEG;AACH,MAAM,UAAU,MAAM,CAAC,KAAwB;IAC9C,OAAO,aAAa,CACnB,cAAc,EACd,IAAI,cAAc,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EACnD,KAAK,EACL,UAAU,CACV,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,IAAI,4BAA4B,CAIrD;IACD,CAAC,CAAC,KAAyB,EAAE,KAAK;QACjC,OAAO,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IACD,CAAC,CAAC,KAAyB,EAAE,KAAK;QACjC,OAAO,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IACD,CAAC,CAAC,KAAuB,EAAE,KAAK;QAC/B,OAAO,IAAI,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC;IACD,CAAC,CAAC,KAAsB;QACvB,OAAO,UAAU,CAAC;IACnB,CAAC;CACD,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,MAAoB,EAAE,KAAwB;IACvE,IAAI,KAAK,KAAK,SAAS,EAAE;QACxB,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;KACvE;SAAM;QACN,IAAI,KAAK,KAAK,IAAI,EAAE;YACnB,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;SAC/B;aAAM,IAAI,KAAK,KAAK,KAAK,EAAE;YAC3B,OAAO,SAAS,CAAC;SACjB;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YAChC,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC7E,OAAO,KAAK,CAAC,CAAC,CAAU,CAAC;SACzB;aAAM;YACN,uBAAuB;YACvB,eAAe,CAAC,KAAK,EAAE,gDAAgD,CAAC,CAAC;SACzE;KACD;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAgB;IACjD,IAAI,KAAK,KAAK,UAAU,EAAE;QACzB,OAAO,EAAE,CAAC;KACV;IACD,yFAAyF;IACzF,+EAA+E;IAC/E,IAAI,KAAK,YAAY,aAAa,EAAE;QACnC,iKAAiK;QACjK,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC1E,iJAAiJ;QACjJ,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAEhF,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE;YAClC,mJAAmJ;YACnJ,MAAM,CAAC,CAAC,CAAC,GAAG,YAAY,aAAa,CAAC,EAAE,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAChF,MAAM,CAAC,GAAG,KAAK,UAAU,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAE/D,GAAG,CAAC,cAAc,EAAE,CAAC;SACrB;QAED,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,SAAS,CAAC;KACvB;SAAM;QACN,OAAO,CAAC,KAAK,CAAC,CAAC;KACf;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAkB;IACjD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAChD,QAAQ,MAAM,CAAC,MAAM,EAAE;QACtB,KAAK,CAAC;YACL,OAAO,UAAU,CAAC;QACnB,KAAK,CAAC;YACL,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;QAClB;YACC,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;KAClC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,kBAAkB;IAC9B,YAAoC,KAAyB;QAAzB,UAAK,GAAL,KAAK,CAAoB;IAAG,CAAC;IAC1D,MAAM,CAAC,QAAiC,EAAE,MAAoB;QACpE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAErC,gCAAgC;QAChC,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC7B,gHAAgH;YAChH,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE;gBAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;aAC7C;SACD;aAAM;YACN,MAAM,CACL,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EACnB,KAAK,CAAC,2DAA2D,CACjE,CAAC;YACF,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE;gBAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;aAC7C;SACD;QAED,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;CACD;AAED;;GAEG;AACH,MAAM,OAAO,kBAAkB;IAC9B,YAAoC,KAAyB;QAAzB,UAAK,GAAL,KAAK,CAAoB;IAAG,CAAC;IAC1D,MAAM,CAAC,QAAiC,EAAE,MAAoB;QACpE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QACjC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE;YAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;SAC9C;QACD,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;CACD;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAiB;IACvC,MAAM,CAAC,QAAiC,EAAE,MAAoB;QAC7D,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;CACD,CAAC;AAUF;;GAEG;AACH,SAAS,YAAY,CACpB,KAAwC,EACxC,GAAa,EACb,KAAa;IAEb,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACtC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,WAAW;IAGvB,YACkB,KAAuB,EACvB,KAAwC;QADxC,UAAK,GAAL,KAAK,CAAkB;QACvB,UAAK,GAAL,KAAK,CAAmC;QAEzD,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEhF,MAAM,aAAa,GAAwB,EAAE,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE;YACjC,MAAM,GAAG,GAAa,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClD,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;SAC1D;QACD,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACpC,CAAC;IACM,MAAM,CAAC,QAAiC,EAAE,MAAoB;QACpE,MAAM,IAAI,GACT,IAAI,CAAC,IAAI,IAAI,oBAAoB,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACvD,uEAAuE;QAEvE,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,MAAM,GAA+B,IAAI,GAAG,EAAE,CAAC;QAErD,gEAAgE;QAChE,SAAS,QAAQ,CAAC,GAAa,EAAE,IAAe;YAC/C,sFAAsF;YACtF,6EAA6E;YAC7E,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAEvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBACxB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;aACxB;QACF,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE;YACvC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC/C,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SACvB;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE;YACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACvC,OAAO,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE;gBAC1C,MAAM,GAAG,GAAa,oBAAoB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9D,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;aAC/C;SACD;QAED,OAAO,IAAI,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { assert, unreachableCase } from \"@fluidframework/core-utils\";\nimport { assertValidIndex } from \"../../../util\";\nimport { FieldKey, TreeNodeSchemaIdentifier, Value } from \"../../../core\";\nimport { DiscriminatedUnionDispatcher } from \"../../../codec\";\nimport { TreeChunk } from \"../chunk\";\nimport { BasicChunk } from \"../basicChunk\";\nimport { SequenceChunk } from \"../sequenceChunk\";\nimport { emptyChunk } from \"../emptyChunk\";\nimport {\n\tEncodedAnyShape,\n\tEncodedChunkShape,\n\tEncodedFieldBatch,\n\tEncodedInlineArray,\n\tEncodedNestedArray,\n\tEncodedTreeShape,\n\tEncodedValueShape,\n} from \"./format\";\nimport {\n\tChunkDecoder,\n\tStreamCursor,\n\tgetChecked,\n\treadStream,\n\treadStreamBoolean,\n\treadStreamNumber,\n\treadStreamStream,\n\treadStreamValue,\n} from \"./chunkCodecUtilities\";\nimport {\n\tDecoderContext,\n\tdecode as genericDecode,\n\treadStreamIdentifier,\n} from \"./chunkDecodingGeneric\";\n\n/**\n * Decode `chunk` into a TreeChunk.\n */\nexport function decode(chunk: EncodedFieldBatch): TreeChunk[] {\n\treturn genericDecode(\n\t\tdecoderLibrary,\n\t\tnew DecoderContext(chunk.identifiers, chunk.shapes),\n\t\tchunk,\n\t\tanyDecoder,\n\t);\n}\n\nconst decoderLibrary = new DiscriminatedUnionDispatcher<\n\tEncodedChunkShape,\n\t[cache: DecoderContext<EncodedChunkShape>],\n\tChunkDecoder\n>({\n\ta(shape: EncodedNestedArray, cache): ChunkDecoder {\n\t\treturn new NestedArrayDecoder(shape);\n\t},\n\tb(shape: EncodedInlineArray, cache): ChunkDecoder {\n\t\treturn new InlineArrayDecoder(shape);\n\t},\n\tc(shape: EncodedTreeShape, cache): ChunkDecoder {\n\t\treturn new TreeDecoder(shape, cache);\n\t},\n\td(shape: EncodedAnyShape): ChunkDecoder {\n\t\treturn anyDecoder;\n\t},\n});\n\n/**\n * Decode a node's value from `stream` using its shape.\n */\nexport function readValue(stream: StreamCursor, shape: EncodedValueShape): Value {\n\tif (shape === undefined) {\n\t\treturn readStreamBoolean(stream) ? readStreamValue(stream) : undefined;\n\t} else {\n\t\tif (shape === true) {\n\t\t\treturn readStreamValue(stream);\n\t\t} else if (shape === false) {\n\t\t\treturn undefined;\n\t\t} else if (Array.isArray(shape)) {\n\t\t\tassert(shape.length === 1, 0x734 /* expected a single constant for value */);\n\t\t\treturn shape[0] as Value;\n\t\t} else {\n\t\t\t// EncodedCounter case:\n\t\t\tunreachableCase(shape, \"decoding values as deltas is not yet supported\");\n\t\t}\n\t}\n}\n\n/**\n * Normalize a {@link TreeChunk} into an array.\n *\n * Unwraps {@link SequenceChunk}s, and wraps other chunks.\n */\nexport function deaggregateChunks(chunk: TreeChunk): TreeChunk[] {\n\tif (chunk === emptyChunk) {\n\t\treturn [];\n\t}\n\t// TODO: when handling of SequenceChunks has better performance (for example in cursors),\n\t// consider keeping SequenceChunks here if they are longer than some threshold.\n\tif (chunk instanceof SequenceChunk) {\n\t\t// Could return [] here, however the logic in this file is designed to never produce an empty SequenceChunk, so its better to throw an error here to detect bugs.\n\t\tassert(chunk.subChunks.length > 0, 0x735 /* Unexpected empty sequence */);\n\t\t// Logic in this file is designed to never produce an unneeded (single item) SequenceChunks, so its better to throw an error here to detect bugs.\n\t\tassert(chunk.subChunks.length > 1, 0x736 /* Unexpected single item sequence */);\n\n\t\tfor (const sub of chunk.subChunks) {\n\t\t\t// The logic in this file is designed to never produce an nested SequenceChunks or emptyChunk, so its better to throw an error here to detect bugs.\n\t\t\tassert(!(sub instanceof SequenceChunk), 0x737 /* unexpected nested sequence */);\n\t\t\tassert(sub !== emptyChunk, 0x738 /* unexpected empty chunk */);\n\n\t\t\tsub.referenceAdded();\n\t\t}\n\n\t\tchunk.referenceRemoved();\n\t\treturn chunk.subChunks;\n\t} else {\n\t\treturn [chunk];\n\t}\n}\n\n/**\n * Normalize a {@link TreeChunk}[] into a single TreeChunk.\n *\n * Avoids creating nested or less than 2 child {@link SequenceChunk}s.\n */\nexport function aggregateChunks(input: TreeChunk[]): TreeChunk {\n\tconst chunks = input.flatMap(deaggregateChunks);\n\tswitch (chunks.length) {\n\t\tcase 0:\n\t\t\treturn emptyChunk;\n\t\tcase 1:\n\t\t\treturn chunks[0];\n\t\tdefault:\n\t\t\treturn new SequenceChunk(chunks);\n\t}\n}\n\n/**\n * Decoder for {@link EncodedNestedArray}s.\n */\nexport class NestedArrayDecoder implements ChunkDecoder {\n\tpublic constructor(private readonly shape: EncodedNestedArray) {}\n\tpublic decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {\n\t\tconst decoder = decoders[this.shape];\n\n\t\t// TODO: uniform chunk fast path\n\t\tconst chunks: TreeChunk[] = [];\n\n\t\tconst data = readStream(stream);\n\t\tif (typeof data === \"number\") {\n\t\t\t// This case means that the array contained only 0-sized items, and was thus encoded as the length of the array.\n\t\t\tconst inner = { data: [], offset: 0 };\n\t\t\tfor (let index = 0; index < data; index++) {\n\t\t\t\tchunks.push(decoder.decode(decoders, inner));\n\t\t\t}\n\t\t} else {\n\t\t\tassert(\n\t\t\t\tArray.isArray(data),\n\t\t\t\t0x739 /* expected number of array for encoding of nested array */,\n\t\t\t);\n\t\t\tconst inner = { data, offset: 0 };\n\t\t\twhile (inner.offset !== inner.data.length) {\n\t\t\t\tchunks.push(decoder.decode(decoders, inner));\n\t\t\t}\n\t\t}\n\n\t\treturn aggregateChunks(chunks);\n\t}\n}\n\n/**\n * Decoder for {@link EncodedInlineArray}s.\n */\nexport class InlineArrayDecoder implements ChunkDecoder {\n\tpublic constructor(private readonly shape: EncodedInlineArray) {}\n\tpublic decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {\n\t\tconst length = this.shape.length;\n\t\tconst decoder = decoders[this.shape.shape];\n\t\tconst chunks: TreeChunk[] = [];\n\t\tfor (let index = 0; index < length; index++) {\n\t\t\tchunks.push(decoder.decode(decoders, stream));\n\t\t}\n\t\treturn aggregateChunks(chunks);\n\t}\n}\n\n/**\n * Decoder for {@link EncodedAnyShape}s.\n */\nexport const anyDecoder: ChunkDecoder = {\n\tdecode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {\n\t\tconst shapeIndex = readStreamNumber(stream);\n\t\tconst decoder = getChecked(decoders, shapeIndex);\n\t\treturn decoder.decode(decoders, stream);\n\t},\n};\n\n/**\n * Decoder for field.\n */\ntype BasicFieldDecoder = (\n\tdecoders: readonly ChunkDecoder[],\n\tstream: StreamCursor,\n) => [FieldKey, TreeChunk];\n\n/**\n * Get a decoder for fields of a provided (via `shape` and `cache`) {@link EncodedChunkShape}.\n */\nfunction fieldDecoder(\n\tcache: DecoderContext<EncodedChunkShape>,\n\tkey: FieldKey,\n\tshape: number,\n): BasicFieldDecoder {\n\tassertValidIndex(shape, cache.shapes);\n\treturn (decoders, stream) => [key, decoders[shape].decode(decoders, stream)];\n}\n\n/**\n * Decoder for {@link EncodedTreeShape}s.\n */\nexport class TreeDecoder implements ChunkDecoder {\n\tprivate readonly type?: TreeNodeSchemaIdentifier;\n\tprivate readonly fieldDecoders: readonly BasicFieldDecoder[];\n\tpublic constructor(\n\t\tprivate readonly shape: EncodedTreeShape,\n\t\tprivate readonly cache: DecoderContext<EncodedChunkShape>,\n\t) {\n\t\tthis.type = shape.type === undefined ? undefined : cache.identifier(shape.type);\n\n\t\tconst fieldDecoders: BasicFieldDecoder[] = [];\n\t\tfor (const field of shape.fields) {\n\t\t\tconst key: FieldKey = cache.identifier(field.key);\n\t\t\tfieldDecoders.push(fieldDecoder(cache, key, field.shape));\n\t\t}\n\t\tthis.fieldDecoders = fieldDecoders;\n\t}\n\tpublic decode(decoders: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {\n\t\tconst type: TreeNodeSchemaIdentifier =\n\t\t\tthis.type ?? readStreamIdentifier(stream, this.cache);\n\t\t// TODO: Consider typechecking against stored schema in here somewhere.\n\n\t\tconst value = readValue(stream, this.shape.value);\n\t\tconst fields: Map<FieldKey, TreeChunk[]> = new Map();\n\n\t\t// Helper to add fields, but with unneeded array chunks removed.\n\t\tfunction addField(key: FieldKey, data: TreeChunk): void {\n\t\t\t// TODO: when handling of ArrayChunks has better performance (for example in cursors),\n\t\t\t// consider keeping array chunks here if they are longer than some threshold.\n\t\t\tconst chunks = deaggregateChunks(data);\n\n\t\t\tif (chunks.length !== 0) {\n\t\t\t\tfields.set(key, chunks);\n\t\t\t}\n\t\t}\n\n\t\tfor (const field of this.fieldDecoders) {\n\t\t\tconst [key, content] = field(decoders, stream);\n\t\t\taddField(key, content);\n\t\t}\n\n\t\tif (this.shape.extraFields !== undefined) {\n\t\t\tconst decoder = decoders[this.shape.extraFields];\n\t\t\tconst inner = readStreamStream(stream);\n\t\t\twhile (inner.offset !== inner.data.length) {\n\t\t\t\tconst key: FieldKey = readStreamIdentifier(inner, this.cache);\n\t\t\t\taddField(key, decoder.decode(decoders, inner));\n\t\t\t}\n\t\t}\n\n\t\treturn new BasicChunk(type, fields, value);\n\t}\n}\n"]}

package/lib/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.d.ts

@@ -0,0 +1,30 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { BrandedType } from "../../../util";
+import { DiscriminatedUnionDispatcher } from "../../../codec";
+import { TreeChunk } from "../chunk";
+import { EncodedFieldBatchGeneric, IdentifierOrIndex } from "./formatGeneric";
+import { ChunkDecoder, StreamCursor } from "./chunkCodecUtilities";
+/**
+ * General purpose shape based tree decoder which gets its support for specific shapes from the caller.
+ */
+export declare function decode<TEncodedShape extends object, TCache>(decoderLibrary: DiscriminatedUnionDispatcher<TEncodedShape, [cache: TCache], ChunkDecoder>, cache: TCache, batch: EncodedFieldBatchGeneric<TEncodedShape>, rootDecoder: ChunkDecoder): TreeChunk[];
+/**
+ * Shared data for use in constructing decoders.
+ */
+export declare class DecoderContext<TEncodedShape = unknown> {
+    readonly identifiers: readonly string[];
+    readonly shapes: readonly TEncodedShape[];
+    /**
+     * @param identifiers - identifier substitution table (use to replace numeric identifier indexes with the actual identifiers from this table).
+     */
+    constructor(identifiers: readonly string[], shapes: readonly TEncodedShape[]);
+    identifier<T extends string & BrandedType<string, string>>(encoded: IdentifierOrIndex): T;
+}
+/**
+ * Read one identifier from the stream, advancing the stream offset.
+ */
+export declare function readStreamIdentifier<T extends string & BrandedType<string, string>>(stream: StreamCursor, cache: DecoderContext): T;
+//# sourceMappingURL=chunkDecodingGeneric.d.ts.map

package/lib/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkDecodingGeneric.d.ts","sourceRoot":"","sources":["../../../../src/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,YAAY,EAAE,YAAY,EAA0B,MAAM,uBAAuB,CAAC;AAE3F;;GAEG;AACH,wBAAgB,MAAM,CAAC,aAAa,SAAS,MAAM,EAAE,MAAM,EAC1D,cAAc,EAAE,4BAA4B,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,YAAY,CAAC,EAC1F,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,wBAAwB,CAAC,aAAa,CAAC,EAC9C,WAAW,EAAE,YAAY,GACvB,SAAS,EAAE,CAcb;AAED;;GAEG;AACH,qBAAa,cAAc,CAAC,aAAa,GAAG,OAAO;aAKjC,WAAW,EAAE,SAAS,MAAM,EAAE;aAC9B,MAAM,EAAE,SAAS,aAAa,EAAE;IALjD;;OAEG;gBAEc,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,MAAM,EAAE,SAAS,aAAa,EAAE;IAG1C,UAAU,CAAC,CAAC,SAAS,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/D,OAAO,EAAE,iBAAiB,GACxB,CAAC;CAMJ;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EAClF,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,cAAc,GACnB,CAAC,CAOH"}

package/lib/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.js

@@ -0,0 +1,47 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { assert } from "@fluidframework/core-utils";
+import { getChecked, readStream } from "./chunkCodecUtilities";
+/**
+ * General purpose shape based tree decoder which gets its support for specific shapes from the caller.
+ */
+export function decode(decoderLibrary, cache, batch, rootDecoder) {
+    const decoders = batch.shapes.map((shape) => decoderLibrary.dispatch(shape, cache));
+    const chunks = [];
+    for (const field of batch.data) {
+        const stream = { data: field, offset: 0 };
+        const result = rootDecoder.decode(decoders, stream);
+        assert(stream.offset === stream.data.length, 0x73a /* expected decode to consume full stream */);
+        chunks.push(result);
+    }
+    return chunks;
+}
+/**
+ * Shared data for use in constructing decoders.
+ */
+export class DecoderContext {
+    /**
+     * @param identifiers - identifier substitution table (use to replace numeric identifier indexes with the actual identifiers from this table).
+     */
+    constructor(identifiers, shapes) {
+        this.identifiers = identifiers;
+        this.shapes = shapes;
+    }
+    identifier(encoded) {
+        if (typeof encoded === "string") {
+            return encoded;
+        }
+        return getChecked(this.identifiers, encoded);
+    }
+}
+/**
+ * Read one identifier from the stream, advancing the stream offset.
+ */
+export function readStreamIdentifier(stream, cache) {
+    const content = readStream(stream);
+    assert(typeof content === "number" || typeof content === "string", 0x73b /* content to be a number or string */);
+    return cache.identifier(content);
+}
+//# sourceMappingURL=chunkDecodingGeneric.js.map

package/lib/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkDecodingGeneric.js","sourceRoot":"","sources":["../../../../src/feature-libraries/chunked-forest/codec/chunkDecodingGeneric.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAKpD,OAAO,EAA8B,UAAU,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAE3F;;GAEG;AACH,MAAM,UAAU,MAAM,CACrB,cAA0F,EAC1F,KAAa,EACb,KAA8C,EAC9C,WAAyB;IAEzB,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IACpF,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE;QAC/B,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,CACL,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EACpC,KAAK,CAAC,4CAA4C,CAClD,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KACpB;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,cAAc;IAC1B;;OAEG;IACH,YACiB,WAA8B,EAC9B,MAAgC;QADhC,gBAAW,GAAX,WAAW,CAAmB;QAC9B,WAAM,GAAN,MAAM,CAA0B;IAC9C,CAAC;IAEG,UAAU,CAChB,OAA0B;QAE1B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;YAChC,OAAO,OAAY,CAAC;SACpB;QACD,OAAO,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAM,CAAC;IACnD,CAAC;CACD;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CACnC,MAAoB,EACpB,KAAqB;IAErB,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,CACL,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,OAAO,KAAK,QAAQ,EAC1D,KAAK,CAAC,sCAAsC,CAC5C,CAAC;IACF,OAAO,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { assert } from \"@fluidframework/core-utils\";\nimport { BrandedType } from \"../../../util\";\nimport { DiscriminatedUnionDispatcher } from \"../../../codec\";\nimport { TreeChunk } from \"../chunk\";\nimport { EncodedFieldBatchGeneric, IdentifierOrIndex } from \"./formatGeneric\";\nimport { ChunkDecoder, StreamCursor, getChecked, readStream } from \"./chunkCodecUtilities\";\n\n/**\n * General purpose shape based tree decoder which gets its support for specific shapes from the caller.\n */\nexport function decode<TEncodedShape extends object, TCache>(\n\tdecoderLibrary: DiscriminatedUnionDispatcher<TEncodedShape, [cache: TCache], ChunkDecoder>,\n\tcache: TCache,\n\tbatch: EncodedFieldBatchGeneric<TEncodedShape>,\n\trootDecoder: ChunkDecoder,\n): TreeChunk[] {\n\tconst decoders = batch.shapes.map((shape) => decoderLibrary.dispatch(shape, cache));\n\tconst chunks: TreeChunk[] = [];\n\tfor (const field of batch.data) {\n\t\tconst stream = { data: field, offset: 0 };\n\t\tconst result = rootDecoder.decode(decoders, stream);\n\t\tassert(\n\t\t\tstream.offset === stream.data.length,\n\t\t\t0x73a /* expected decode to consume full stream */,\n\t\t);\n\t\tchunks.push(result);\n\t}\n\n\treturn chunks;\n}\n\n/**\n * Shared data for use in constructing decoders.\n */\nexport class DecoderContext<TEncodedShape = unknown> {\n\t/**\n\t * @param identifiers - identifier substitution table (use to replace numeric identifier indexes with the actual identifiers from this table).\n\t */\n\tpublic constructor(\n\t\tpublic readonly identifiers: readonly string[],\n\t\tpublic readonly shapes: readonly TEncodedShape[],\n\t) {}\n\n\tpublic identifier<T extends string & BrandedType<string, string>>(\n\t\tencoded: IdentifierOrIndex,\n\t): T {\n\t\tif (typeof encoded === \"string\") {\n\t\t\treturn encoded as T;\n\t\t}\n\t\treturn getChecked(this.identifiers, encoded) as T;\n\t}\n}\n\n/**\n * Read one identifier from the stream, advancing the stream offset.\n */\nexport function readStreamIdentifier<T extends string & BrandedType<string, string>>(\n\tstream: StreamCursor,\n\tcache: DecoderContext,\n): T {\n\tconst content = readStream(stream);\n\tassert(\n\t\ttypeof content === \"number\" || typeof content === \"string\",\n\t\t0x73b /* content to be a number or string */,\n\t);\n\treturn cache.identifier(content);\n}\n"]}

package/lib/feature-libraries/chunked-forest/codec/chunkEncodingGeneric.d.ts

@@ -0,0 +1,81 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { FluidSerializableReadOnly } from "../../valueUtilities";
+import { EncodedFieldBatchGeneric } from "./formatGeneric";
+import { Counter, CounterFilter, DeduplicationTable } from "./chunkCodecUtilities";
+/**
+ * An identifier which can be compressed using {@link Counter}.
+ *
+ * @remarks
+ * Compression of Identifiers is done after the output is otherwise generated to enable counting all the usages.
+ * To avoid having to decode the decode the data array to determine which data is an identifier and which is some other string,
+ * some recognizable representation is required.
+ * Using a class and checking its prototype works for this, and is why IdentifierToken is a class.
+ */
+export declare class IdentifierToken {
+    readonly identifier: string;
+    constructor(identifier: string);
+}
+/**
+ * Format for data emitted during encoding, before dictionary compression of identifiers.
+ *
+ * @remarks
+ * These buffers are mutated after construction if they contain identifiers or shapes.
+ */
+export type BufferFormat<TEncodedShape> = (FluidSerializableReadOnly | Shape<TEncodedShape> | IdentifierToken | BufferFormat<TEncodedShape>)[];
+/**
+ * Replace shapes and identifiers in buffer and any nested arrays.
+ *
+ * This looks inside nested arrays (including transitively) but not inside objects.
+ *
+ * Note that this modifies `buffer` to avoid having to copy it.
+ */
+export declare function handleShapesAndIdentifiers<TEncodedShape>(version: number, buffer: BufferFormat<TEncodedShape>[], identifierFilter?: CounterFilter<string>): EncodedFieldBatchGeneric<TEncodedShape>;
+/**
+ * A tree shape.
+ * This similar to a schema for a tree, though it may be more or less specific than the actual schema system used.
+ * Can be encoded into a `TEncodedShape`: when doing so encodes references to shapes (if needed) using {@link Counter}:
+ * this can include recursive references (direct or indirect).
+ *
+ * @remarks
+ * Deduplication of shapes is done after the output is otherwise generated (including identifier dictionary encoding).
+ * To avoid having to decode the data array to determine which data is a shape and which is some other object,
+ * some recognizable representation is required.
+ * Using a class and checking its prototype works for this, and is why Shape is a class.
+ *
+ * Note that deduplication compares shapes by object identity not by content, so encoders must ensure shapes are not duplicated to achieve efficient encoding.
+ * Comparison by content would be difficult due to shape containing references to other shapes.
+ *
+ * @privateRemarks
+ * Unlike with identifiers, conversion from the initial form (this class / IdentifierToken) is done by the `encodeShape` method, not by general purpose logic in `handleShapesAndIdentifiers`.
+ * For `handleShapesAndIdentifiers` to do the conversion without help from `encodeShape`,
+ * instances of this Shape class would have to either be or output an object that is identical to the `TEncodedShape` format except with all shape references as object references instead of indexes.
+ * Those objects would have to be deeply traversed looking for shape objects to replace with reference indexes.
+ * This is possible, but making it type safe would involve generating derived types from the `TEncodedShape` deeply replacing any shape references, as well as requiring deep traversal of all objects in the encoded output.
+ * Such an approach seemed less maintainable and readable than the design taken here which avoids the need for those derived types.
+ */
+export declare abstract class Shape<TEncodedShape> {
+    /**
+     * Count this shape's contents.
+     *
+     * Used to discover referenced shapes (to ensure they are included in the `shapes` passed to `encodeShape`),
+     * as well as count usages of shapes and identifiers for more efficient dictionary encoding. See {@link Counter}.
+     *
+     * @param shapes - must be invoked with each directly referenced shape (which must provided to `encodeShape`).
+     * Can be invoked multiple times if a shape is referenced more than once for more efficient dictionary encoding.
+     * Should not be invoked with `this` unless this shape references itself.
+     */
+    abstract count(identifiers: Counter<string>, shapes: (shape: Shape<TEncodedShape>) => void): void;
+    /**
+     * Produce the final encoded format for this shape.
+     *
+     * @param identifiers - a subset of identifiers used in this tree, and their corresponding indexes to use for the dictionary encoding.
+     * Can be used to encode identifiers referenced by this shape.
+     * @param shapes - includes a superset of shapes reported by count.
+     * Used to encode references to shapes as numbers.
+     */
+    abstract encodeShape(identifiers: DeduplicationTable<string>, shapes: DeduplicationTable<Shape<TEncodedShape>>): TEncodedShape;
+}
+//# sourceMappingURL=chunkEncodingGeneric.d.ts.map

package/lib/feature-libraries/chunked-forest/codec/chunkEncodingGeneric.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkEncodingGeneric.d.ts","sourceRoot":"","sources":["../../../../src/feature-libraries/chunked-forest/codec/chunkEncodingGeneric.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EACN,OAAO,EACP,aAAa,EACb,kBAAkB,EAElB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;;;GAQG;AACH,qBAAa,eAAe;aACQ,UAAU,EAAE,MAAM;gBAAlB,UAAU,EAAE,MAAM;CACrD;AAED;;;;;GAKG;AACH,MAAM,MAAM,YAAY,CAAC,aAAa,IAAI,CACvC,yBAAyB,GACzB,KAAK,CAAC,aAAa,CAAC,GACpB,eAAe,GACf,YAAY,CAAC,aAAa,CAAC,CAC7B,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,aAAa,EACvD,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,YAAY,CAAC,aAAa,CAAC,EAAE,EACrC,gBAAgB,GAAE,aAAa,CAAC,MAAM,CAAwB,GAC5D,wBAAwB,CAAC,aAAa,CAAC,CAwEzC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,8BAAsB,KAAK,CAAC,aAAa;IACxC;;;;;;;;;OASG;aACa,KAAK,CACpB,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAC5B,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,CAAC,KAAK,IAAI,GAC3C,IAAI;IAEP;;;;;;;OAOG;aACa,WAAW,CAC1B,WAAW,EAAE,kBAAkB,CAAC,MAAM,CAAC,EACvC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAC9C,aAAa;CAChB"}

package/lib/feature-libraries/chunked-forest/codec/chunkEncodingGeneric.js

@@ -0,0 +1,120 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { fail } from "../../../util";
+import { Counter, jsonMinimizingFilter, } from "./chunkCodecUtilities";
+/**
+ * An identifier which can be compressed using {@link Counter}.
+ *
+ * @remarks
+ * Compression of Identifiers is done after the output is otherwise generated to enable counting all the usages.
+ * To avoid having to decode the decode the data array to determine which data is an identifier and which is some other string,
+ * some recognizable representation is required.
+ * Using a class and checking its prototype works for this, and is why IdentifierToken is a class.
+ */
+export class IdentifierToken {
+    constructor(identifier) {
+        this.identifier = identifier;
+    }
+}
+/**
+ * Replace shapes and identifiers in buffer and any nested arrays.
+ *
+ * This looks inside nested arrays (including transitively) but not inside objects.
+ *
+ * Note that this modifies `buffer` to avoid having to copy it.
+ */
+export function handleShapesAndIdentifiers(version, buffer, identifierFilter = jsonMinimizingFilter) {
+    const identifiers = new Counter();
+    const shapes = new Counter();
+    // Shapes can reference other shapes (and identifiers), so we need to traverse the shape graph.
+    // These collections enable that.
+    const shapesSeen = new Set();
+    const shapeToCount = [];
+    const shapeDiscovered = (shape) => {
+        shapes.add(shape);
+        if (!shapesSeen.has(shape)) {
+            shapesSeen.add(shape);
+            shapeToCount.push(shape);
+        }
+    };
+    const arrays = [buffer];
+    for (const array of arrays) {
+        for (const item of array) {
+            if (item instanceof IdentifierToken) {
+                identifiers.add(item.identifier);
+            }
+            else if (item instanceof Shape) {
+                shapeDiscovered(item);
+            }
+            else if (Array.isArray(item)) {
+                // In JS it is legal to push items to an array which is being iterated,
+                // and they will be visited in order.
+                arrays.push(item);
+            }
+            else if (item !== null &&
+                typeof item === "object" &&
+                item.shape instanceof Shape) {
+                // because "serializable" is allowed in buffer and it has type `any`, its very easy to mess up including of shapes in the buffer.
+                // This catches the easiest way to get it wrong.
+                fail("encoder interface instead of shape written to stream");
+            }
+        }
+    }
+    // Traverse shape graph, discovering and counting all shape to shape and shape to identifier references.
+    {
+        let item;
+        while ((item = shapeToCount.pop()) !== undefined) {
+            item.count(identifiers, shapeDiscovered);
+        }
+    }
+    // Determine substitutions for identifiers and shapes:
+    const identifierTable = identifiers.buildTable(identifierFilter);
+    const shapeTable = shapes.buildTable();
+    for (const array of arrays) {
+        for (let index = 0; index < array.length; index++) {
+            const item = array[index];
+            if (item instanceof IdentifierToken) {
+                array[index] = identifierTable.valueToIndex.get(item.identifier) ?? item.identifier;
+            }
+            else if (item instanceof Shape) {
+                array[index] = shapeTable.valueToIndex.get(item) ?? fail("missing shape");
+            }
+        }
+    }
+    const encodedShapes = shapeTable.indexToValue.map((shape) => shape.encodeShape(identifierTable, shapeTable));
+    return {
+        version,
+        // TODO: fix readonly typing issues to remove this cast.
+        identifiers: identifierTable.indexToValue,
+        shapes: encodedShapes,
+        data: buffer,
+    };
+}
+/**
+ * A tree shape.
+ * This similar to a schema for a tree, though it may be more or less specific than the actual schema system used.
+ * Can be encoded into a `TEncodedShape`: when doing so encodes references to shapes (if needed) using {@link Counter}:
+ * this can include recursive references (direct or indirect).
+ *
+ * @remarks
+ * Deduplication of shapes is done after the output is otherwise generated (including identifier dictionary encoding).
+ * To avoid having to decode the data array to determine which data is a shape and which is some other object,
+ * some recognizable representation is required.
+ * Using a class and checking its prototype works for this, and is why Shape is a class.
+ *
+ * Note that deduplication compares shapes by object identity not by content, so encoders must ensure shapes are not duplicated to achieve efficient encoding.
+ * Comparison by content would be difficult due to shape containing references to other shapes.
+ *
+ * @privateRemarks
+ * Unlike with identifiers, conversion from the initial form (this class / IdentifierToken) is done by the `encodeShape` method, not by general purpose logic in `handleShapesAndIdentifiers`.
+ * For `handleShapesAndIdentifiers` to do the conversion without help from `encodeShape`,
+ * instances of this Shape class would have to either be or output an object that is identical to the `TEncodedShape` format except with all shape references as object references instead of indexes.
+ * Those objects would have to be deeply traversed looking for shape objects to replace with reference indexes.
+ * This is possible, but making it type safe would involve generating derived types from the `TEncodedShape` deeply replacing any shape references, as well as requiring deep traversal of all objects in the encoded output.
+ * Such an approach seemed less maintainable and readable than the design taken here which avoids the need for those derived types.
+ */
+export class Shape {
+}
+//# sourceMappingURL=chunkEncodingGeneric.js.map

package/lib/feature-libraries/chunked-forest/codec/chunkEncodingGeneric.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkEncodingGeneric.js","sourceRoot":"","sources":["../../../../src/feature-libraries/chunked-forest/codec/chunkEncodingGeneric.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAGrC,OAAO,EACN,OAAO,EAGP,oBAAoB,GACpB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;;;GAQG;AACH,MAAM,OAAO,eAAe;IAC3B,YAAmC,UAAkB;QAAlB,eAAU,GAAV,UAAU,CAAQ;IAAG,CAAC;CACzD;AAeD;;;;;;GAMG;AACH,MAAM,UAAU,0BAA0B,CACzC,OAAe,EACf,MAAqC,EACrC,mBAA0C,oBAAoB;IAE9D,MAAM,WAAW,GAAG,IAAI,OAAO,EAAU,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,OAAO,EAAwB,CAAC;IACnD,+FAA+F;IAC/F,iCAAiC;IACjC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IACnD,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,eAAe,GAAG,CAAC,KAA2B,EAAE,EAAE;QACvD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YAC3B,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtB,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACzB;IACF,CAAC,CAAC;IAEF,MAAM,MAAM,GAAkC,CAAC,MAAM,CAAC,CAAC;IACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACzB,IAAI,IAAI,YAAY,eAAe,EAAE;gBACpC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;aACjC;iBAAM,IAAI,IAAI,YAAY,KAAK,EAAE;gBACjC,eAAe,CAAC,IAAI,CAAC,CAAC;aACtB;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;gBAC/B,uEAAuE;gBACvE,qCAAqC;gBACrC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAClB;iBAAM,IACN,IAAI,KAAK,IAAI;gBACb,OAAO,IAAI,KAAK,QAAQ;gBACvB,IAAY,CAAC,KAAK,YAAY,KAAK,EACnC;gBACD,iIAAiI;gBACjI,gDAAgD;gBAChD,IAAI,CAAC,sDAAsD,CAAC,CAAC;aAC7D;SACD;KACD;IAED,wGAAwG;IACxG;QACC,IAAI,IAAsC,CAAC;QAC3C,OAAO,CAAC,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,CAAC,KAAK,SAAS,EAAE;YACjD,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;SACzC;KACD;IAED,sDAAsD;IACtD,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IAEvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,IAAI,YAAY,eAAe,EAAE;gBACpC,KAAK,CAAC,KAAK,CAAC,GAAG,eAAe,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC;aACpF;iBAAM,IAAI,IAAI,YAAY,KAAK,EAAE;gBACjC,KAAK,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC;aAC1E;SACD;KACD;IAED,MAAM,aAAa,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC3D,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,UAAU,CAAC,CAC9C,CAAC;IAEF,OAAO;QACN,OAAO;QACP,wDAAwD;QACxD,WAAW,EAAE,eAAe,CAAC,YAAwB;QACrD,MAAM,EAAE,aAAa;QACrB,IAAI,EAAE,MAAuB;KAC7B,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,OAAgB,KAAK;CA4B1B","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { TreeValue } from \"../../../core\";\nimport { fail } from \"../../../util\";\nimport { FluidSerializableReadOnly } from \"../../valueUtilities\";\nimport { EncodedFieldBatchGeneric } from \"./formatGeneric\";\nimport {\n\tCounter,\n\tCounterFilter,\n\tDeduplicationTable,\n\tjsonMinimizingFilter,\n} from \"./chunkCodecUtilities\";\n\n/**\n * An identifier which can be compressed using {@link Counter}.\n *\n * @remarks\n * Compression of Identifiers is done after the output is otherwise generated to enable counting all the usages.\n * To avoid having to decode the decode the data array to determine which data is an identifier and which is some other string,\n * some recognizable representation is required.\n * Using a class and checking its prototype works for this, and is why IdentifierToken is a class.\n */\nexport class IdentifierToken {\n\tpublic constructor(public readonly identifier: string) {}\n}\n\n/**\n * Format for data emitted during encoding, before dictionary compression of identifiers.\n *\n * @remarks\n * These buffers are mutated after construction if they contain identifiers or shapes.\n */\nexport type BufferFormat<TEncodedShape> = (\n\t| FluidSerializableReadOnly\n\t| Shape<TEncodedShape>\n\t| IdentifierToken\n\t| BufferFormat<TEncodedShape>\n)[];\n\n/**\n * Replace shapes and identifiers in buffer and any nested arrays.\n *\n * This looks inside nested arrays (including transitively) but not inside objects.\n *\n * Note that this modifies `buffer` to avoid having to copy it.\n */\nexport function handleShapesAndIdentifiers<TEncodedShape>(\n\tversion: number,\n\tbuffer: BufferFormat<TEncodedShape>[],\n\tidentifierFilter: CounterFilter<string> = jsonMinimizingFilter,\n): EncodedFieldBatchGeneric<TEncodedShape> {\n\tconst identifiers = new Counter<string>();\n\tconst shapes = new Counter<Shape<TEncodedShape>>();\n\t// Shapes can reference other shapes (and identifiers), so we need to traverse the shape graph.\n\t// These collections enable that.\n\tconst shapesSeen = new Set<Shape<TEncodedShape>>();\n\tconst shapeToCount: Shape<TEncodedShape>[] = [];\n\tconst shapeDiscovered = (shape: Shape<TEncodedShape>) => {\n\t\tshapes.add(shape);\n\t\tif (!shapesSeen.has(shape)) {\n\t\t\tshapesSeen.add(shape);\n\t\t\tshapeToCount.push(shape);\n\t\t}\n\t};\n\n\tconst arrays: BufferFormat<TEncodedShape>[] = [buffer];\n\tfor (const array of arrays) {\n\t\tfor (const item of array) {\n\t\t\tif (item instanceof IdentifierToken) {\n\t\t\t\tidentifiers.add(item.identifier);\n\t\t\t} else if (item instanceof Shape) {\n\t\t\t\tshapeDiscovered(item);\n\t\t\t} else if (Array.isArray(item)) {\n\t\t\t\t// In JS it is legal to push items to an array which is being iterated,\n\t\t\t\t// and they will be visited in order.\n\t\t\t\tarrays.push(item);\n\t\t\t} else if (\n\t\t\t\titem !== null &&\n\t\t\t\ttypeof item === \"object\" &&\n\t\t\t\t(item as any).shape instanceof Shape\n\t\t\t) {\n\t\t\t\t// because \"serializable\" is allowed in buffer and it has type `any`, its very easy to mess up including of shapes in the buffer.\n\t\t\t\t// This catches the easiest way to get it wrong.\n\t\t\t\tfail(\"encoder interface instead of shape written to stream\");\n\t\t\t}\n\t\t}\n\t}\n\n\t// Traverse shape graph, discovering and counting all shape to shape and shape to identifier references.\n\t{\n\t\tlet item: Shape<TEncodedShape> | undefined;\n\t\twhile ((item = shapeToCount.pop()) !== undefined) {\n\t\t\titem.count(identifiers, shapeDiscovered);\n\t\t}\n\t}\n\n\t// Determine substitutions for identifiers and shapes:\n\tconst identifierTable = identifiers.buildTable(identifierFilter);\n\tconst shapeTable = shapes.buildTable();\n\n\tfor (const array of arrays) {\n\t\tfor (let index = 0; index < array.length; index++) {\n\t\t\tconst item = array[index];\n\t\t\tif (item instanceof IdentifierToken) {\n\t\t\t\tarray[index] = identifierTable.valueToIndex.get(item.identifier) ?? item.identifier;\n\t\t\t} else if (item instanceof Shape) {\n\t\t\t\tarray[index] = shapeTable.valueToIndex.get(item) ?? fail(\"missing shape\");\n\t\t\t}\n\t\t}\n\t}\n\n\tconst encodedShapes = shapeTable.indexToValue.map((shape) =>\n\t\tshape.encodeShape(identifierTable, shapeTable),\n\t);\n\n\treturn {\n\t\tversion,\n\t\t// TODO: fix readonly typing issues to remove this cast.\n\t\tidentifiers: identifierTable.indexToValue as string[],\n\t\tshapes: encodedShapes,\n\t\tdata: buffer as TreeValue[][],\n\t};\n}\n\n/**\n * A tree shape.\n * This similar to a schema for a tree, though it may be more or less specific than the actual schema system used.\n * Can be encoded into a `TEncodedShape`: when doing so encodes references to shapes (if needed) using {@link Counter}:\n * this can include recursive references (direct or indirect).\n *\n * @remarks\n * Deduplication of shapes is done after the output is otherwise generated (including identifier dictionary encoding).\n * To avoid having to decode the data array to determine which data is a shape and which is some other object,\n * some recognizable representation is required.\n * Using a class and checking its prototype works for this, and is why Shape is a class.\n *\n * Note that deduplication compares shapes by object identity not by content, so encoders must ensure shapes are not duplicated to achieve efficient encoding.\n * Comparison by content would be difficult due to shape containing references to other shapes.\n *\n * @privateRemarks\n * Unlike with identifiers, conversion from the initial form (this class / IdentifierToken) is done by the `encodeShape` method, not by general purpose logic in `handleShapesAndIdentifiers`.\n * For `handleShapesAndIdentifiers` to do the conversion without help from `encodeShape`,\n * instances of this Shape class would have to either be or output an object that is identical to the `TEncodedShape` format except with all shape references as object references instead of indexes.\n * Those objects would have to be deeply traversed looking for shape objects to replace with reference indexes.\n * This is possible, but making it type safe would involve generating derived types from the `TEncodedShape` deeply replacing any shape references, as well as requiring deep traversal of all objects in the encoded output.\n * Such an approach seemed less maintainable and readable than the design taken here which avoids the need for those derived types.\n */\nexport abstract class Shape<TEncodedShape> {\n\t/**\n\t * Count this shape's contents.\n\t *\n\t * Used to discover referenced shapes (to ensure they are included in the `shapes` passed to `encodeShape`),\n\t * as well as count usages of shapes and identifiers for more efficient dictionary encoding. See {@link Counter}.\n\t *\n\t * @param shapes - must be invoked with each directly referenced shape (which must provided to `encodeShape`).\n\t * Can be invoked multiple times if a shape is referenced more than once for more efficient dictionary encoding.\n\t * Should not be invoked with `this` unless this shape references itself.\n\t */\n\tpublic abstract count(\n\t\tidentifiers: Counter<string>,\n\t\tshapes: (shape: Shape<TEncodedShape>) => void,\n\t): void;\n\n\t/**\n\t * Produce the final encoded format for this shape.\n\t *\n\t * @param identifiers - a subset of identifiers used in this tree, and their corresponding indexes to use for the dictionary encoding.\n\t * Can be used to encode identifiers referenced by this shape.\n\t * @param shapes - includes a superset of shapes reported by count.\n\t * Used to encode references to shapes as numbers.\n\t */\n\tpublic abstract encodeShape(\n\t\tidentifiers: DeduplicationTable<string>,\n\t\tshapes: DeduplicationTable<Shape<TEncodedShape>>,\n\t): TEncodedShape;\n}\n"]}