@fluidframework/merge-tree 0.58.2002 → 0.59.1000

package/lib/snapshotV1.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotV1.js","sourceRoot":"","sources":["../src/snapshotV1.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAKvD,OAAO,EACH,eAAe,GAElB,MAAM,cAAc,CAAC;AACtB,OAAO,EAKH,eAAe,EACf,8BAA8B,GACjC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,OAAO,UAAU;IAenB,YACW,SAAoB,EAC3B,MAAwB,EACP,eAAuC,EACjD,QAAiB,EACjB,YAAyB;;QAJzB,cAAS,GAAT,SAAS,CAAW;QAEV,oBAAe,GAAf,eAAe,CAAwB;QACjD,aAAQ,GAAR,QAAQ,CAAS;QACjB,iBAAY,GAAZ,YAAY,CAAa;QAEhC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,eAAG,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,0CAAE,0BAA0B,mCAAI,UAAU,CAAC,SAAS,CAAC;QAExF,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,eAAe,EAAE,CAAC;QAC3D,IAAI,CAAC,MAAM,GAAG;YACV,iBAAiB,EAAE,MAAM;YACzB,cAAc,EAAE,UAAU;YAC1B,oBAAoB,EAAE,EAAE;YACxB,WAAW,EAAE,CAAC;YACd,iBAAiB,EAAE,CAAC;SACvB,CAAC;QAEF,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC7B,CAAC;IAEO,gBAAgB,CACpB,WAA+B,EAC/B,UAAoB,EACpB,oBAA4B,EAC5B,UAAU,GAAG,CAAC;QACd,MAAM,QAAQ,GAAuB,EAAE,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,OAAO,CAAC,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE;YAC1F,MAAM,IAAI,GAAG,WAAW,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC;YACpD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,MAAM,IAAI,UAAU,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC;YAChD,YAAY,EAAE,CAAC;SAClB;QACD,OAAO;YACH,OAAO,EAAE,GAAG;YACZ,YAAY;YACZ,MAAM;YACN,QAAQ;YACR,UAAU;YACV,cAAc,EAAE,SAAS;SAC5B,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,IAAI,CACA,UAA4B,EAC5B,IAAkB;QAElB,MAAM,MAAM,GAAuB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QAC5B,GAAG;YACC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAC/B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,KAAK,CAAC,YAAY,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;SAC3C,QAAQ,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;QAE/D,yDAAyD;QACzD,oEAAoE;QACpE,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAG,CAAC;QACpC,WAAW,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QACzC,WAAW,CAAC,cAAc,CAAC,oBAAoB,GAAG,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,MAAM,KAAK,GAAqC,EAAE,CAAC;QACnD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YAC5B,MAAM,EAAE,GAAG,GAAG,cAAc,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,8BAA8B,CAC1C,EAAE,EACF,KAAK,EACL,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,CAAC,OAAO,EACtB,UAAU,EACV,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,8BAA8B,CACjE,cAAc,CAAC,MAAM,EACrB,WAAW,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,CAAC,OAAO,EACtB,UAAU,EACV,IAAI,CAAC,CAAC,CAAC;QACX,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACpB,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED,WAAW;QACP,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAE7C,yEAAyE;QACzE,MAAM,UAAU,GAAG,CAAC,IAAsB,EAAE,MAAc,EAAE,EAAE;YAC1D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,iGAAiG;QACjG,MAAM,OAAO,GAAG,CAAC,OAAkB,EAAE,EAAE;YACnC,IAAI,OAAO,EAAE;gBAAE,UAAU,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;aAAE;QAC9E,CAAC,CAAC;QAEF,IAAI,IAA0B,CAAC;QAC/B,MAAM,cAAc,GAAG,CAAC,OAAiB,EAAE,EAAE;YACzC,8FAA8F;YAC9F,4BAA4B;YAC5B,gGAAgG;YAChG,mFAAmF;YACnF,8FAA8F;YAC9F,iDAAiD;YACjD,oEAAoE;YACpE,IAAI,OAAO,CAAC,GAAG,KAAK,wBAAwB,IAAI,OAAO,CAAC,UAAW,IAAI,MAAM,EAAE;gBAC3E,OAAO,IAAI,CAAC;aACf;YAED,gGAAgG;YAChG,oGAAoG;YACpG,0BAA0B;YAC1B,oEAAoE;YACpE,IAAI,CAAC,OAAO,CAAC,GAAI,IAAI,MAAM,CAAC,CAAmC,mCAAmC;mBAC3F,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAsB,yCAAyC;uBAC5F,OAAO,CAAC,UAAU,KAAK,wBAAwB,CAAC,CAAG,6CAA6C;cACzG;gBACE,gGAAgG;gBAChG,uDAAuD;gBACvD,IAAI,CAAC,IAAI,EAAE;oBACP,mGAAmG;oBACnG,IAAI,GAAG,OAAO,CAAC;iBAClB;qBAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE;oBACxF,yFAAyF;oBACzF,8DAA8D;oBAC9D,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;iBAChC;qBAAM;oBACH,sFAAsF;oBACtF,iEAAiE;oBACjE,OAAO,CAAC,IAAI,CAAC,CAAC;oBACd,IAAI,GAAG,OAAO,CAAC;iBAClB;aACJ;iBAAM;gBACH,uGAAuG;gBACvG,2DAA2D;gBAC3D,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,IAAI,GAAG,SAAS,CAAC;gBAEjB,MAAM,GAAG,GAA8B,EAAE,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;gBACxE,8EAA8E;gBAC9E,oEAAoE;gBACpE,IAAI,OAAO,CAAC,GAAI,GAAG,MAAM,EAAE;oBACvB,GAAG,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;oBACtB,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;iBACvD;gBACD,qGAAqG;gBACrG,qEAAqE;gBACrE,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE;oBAClC,MAAM,CAAC,OAAO,CAAC,UAAU,KAAK,wBAAwB,IAAI,OAAO,CAAC,UAAU,GAAG,MAAM,EACjF,KAAK,CAAC,kFAAkF,CAAC,CAAC;oBAC9F,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;oBACpC,oEAAoE;oBACpE,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,eAAgB,CAAC,CAAC;iBACtE;gBAEL,2FAA2F;gBACvF,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;uBACjD,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,EAClE,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBAE/D,kDAAkD;gBAClD,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;aACzC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAEF,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;QAEhE,gEAAgE;QAChE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEd,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,SAAS,CACzB,OAA+B,EAC/B,IAAY,EACZ,MAAwB,EACxB,OAAgC,EAChC,UAA6B;QAE7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACrF,CAAC;IAEM,MAAM,CAAC,YAAY,CACtB,IAAY,EACZ,KAAa,EACb,MAAwB,EACxB,OAAgC,EAChC,UAA6B;QAE7B,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1E,OAAO,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5D,CAAC;;AAvOD,2GAA2G;AAC3G,gHAAgH;AAChH,wFAAwF;AACxF,8FAA8F;AAC9F,wEAAwE;AACxE,sEAAsE;AAC/C,oBAAS,GAAW,KAAK,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IFluidHandle } from \"@fluidframework/core-interfaces\";\nimport { IFluidSerializer } from \"@fluidframework/shared-object-base\";\nimport { assert, bufferToString } from \"@fluidframework/common-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { IChannelStorageService } from \"@fluidframework/datastore-definitions\";\nimport { ISummaryTreeWithStats } from \"@fluidframework/runtime-definitions\";\nimport { SummaryTreeBuilder } from \"@fluidframework/runtime-utils\";\nimport { UnassignedSequenceNumber } from \"./constants\";\nimport {\n    ISegment,\n    MergeTree,\n} from \"./mergeTree\";\nimport {\n    matchProperties,\n    PropertySet,\n} from \"./properties\";\nimport {\n    IJSONSegmentWithMergeInfo,\n    JsonSegmentSpecs,\n    MergeTreeHeaderMetadata,\n    MergeTreeChunkV1,\n    toLatestVersion,\n    serializeAsMaxSupportedVersion,\n} from \"./snapshotChunks\";\nimport { SnapshotLegacy } from \"./snapshotlegacy\";\n\nexport class SnapshotV1 {\n    // Split snapshot into two entries - headers (small) and body (overflow) for faster loading initial content\n    // Please note that this number has no direct relationship to anything other than size of raw text (characters).\n    // As we produce json for the blob (and then send over the wire compressed), this number\n    // is really hard to correlate with any actual metric that matters (like bytes over the wire).\n    // For test with small number of chunks it would be closer to blob size,\n    // for very chunky text, blob size can easily be 4x-8x of that number.\n    public static readonly chunkSize: number = 10000;\n\n    private readonly header: MergeTreeHeaderMetadata;\n    private readonly segments: JsonSegmentSpecs[];\n    private readonly segmentLengths: number[];\n    private readonly logger: ITelemetryLogger;\n    private readonly chunkSize: number;\n\n    constructor(\n        public mergeTree: MergeTree,\n        logger: ITelemetryLogger,\n        private readonly getLongClientId: (id: number) => string,\n        public filename?: string,\n        public onCompletion?: () => void,\n    ) {\n        this.logger = ChildLogger.create(logger, \"Snapshot\");\n        this.chunkSize = mergeTree?.options?.mergeTreeSnapshotChunkSize ?? SnapshotV1.chunkSize;\n\n        const { currentSeq, minSeq } = mergeTree.getCollabWindow();\n        this.header = {\n            minSequenceNumber: minSeq,\n            sequenceNumber: currentSeq,\n            orderedChunkMetadata: [],\n            totalLength: 0,\n            totalSegmentCount: 0,\n        };\n\n        this.segments = [];\n        this.segmentLengths = [];\n    }\n\n    private getSeqLengthSegs(\n        allSegments: JsonSegmentSpecs[],\n        allLengths: number[],\n        approxSequenceLength: number,\n        startIndex = 0): MergeTreeChunkV1 {\n        const segments: JsonSegmentSpecs[] = [];\n        let length = 0;\n        let segmentCount = 0;\n        while ((length < approxSequenceLength) && ((startIndex + segmentCount) < allSegments.length)) {\n            const pseg = allSegments[startIndex + segmentCount];\n            segments.push(pseg);\n            length += allLengths[startIndex + segmentCount];\n            segmentCount++;\n        }\n        return {\n            version: \"1\",\n            segmentCount,\n            length,\n            segments,\n            startIndex,\n            headerMetadata: undefined,\n        };\n    }\n\n    /**\n     * Emits the snapshot to an ISummarizeResult. If provided the optional IFluidSerializer will be used when\n     * serializing the summary data rather than JSON.stringify.\n     */\n    emit(\n        serializer: IFluidSerializer,\n        bind: IFluidHandle,\n    ): ISummaryTreeWithStats {\n        const chunks: MergeTreeChunkV1[] = [];\n        this.header.totalSegmentCount = 0;\n        this.header.totalLength = 0;\n        do {\n            const chunk = this.getSeqLengthSegs(\n                this.segments,\n                this.segmentLengths,\n                this.chunkSize,\n                this.header.totalSegmentCount);\n            chunks.push(chunk);\n            this.header.totalSegmentCount += chunk.segmentCount;\n            this.header.totalLength += chunk.length;\n        } while (this.header.totalSegmentCount < this.segments.length);\n\n        // The do while loop should have added at least one chunk\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        const headerChunk = chunks.shift()!;\n        headerChunk.headerMetadata = this.header;\n        headerChunk.headerMetadata.orderedChunkMetadata = [{ id: SnapshotLegacy.header }];\n        const blobs: [key: string, content: string][] = [];\n        chunks.forEach((chunk, index) => {\n            const id = `${SnapshotLegacy.body}_${index}`;\n            this.header.orderedChunkMetadata.push({ id });\n            blobs.push([id, serializeAsMaxSupportedVersion(\n                id,\n                chunk,\n                this.logger,\n                this.mergeTree.options,\n                serializer,\n                bind)]);\n        });\n\n        const builder = new SummaryTreeBuilder();\n        builder.addBlob(SnapshotLegacy.header, serializeAsMaxSupportedVersion(\n            SnapshotLegacy.header,\n            headerChunk,\n            this.logger,\n            this.mergeTree.options,\n            serializer,\n            bind));\n        blobs.forEach((value) => {\n            builder.addBlob(value[0], value[1]);\n        });\n\n        return builder.getSummaryTree();\n    }\n\n    extractSync() {\n        const mergeTree = this.mergeTree;\n        const minSeq = this.header.minSequenceNumber;\n\n        // Helper to add the given `MergeTreeChunkV0SegmentSpec` to the snapshot.\n        const pushSegRaw = (json: JsonSegmentSpecs, length: number) => {\n            this.segments.push(json);\n            this.segmentLengths.push(length);\n        };\n\n        // Helper to serialize the given `segment` and add it to the snapshot (if a segment is provided).\n        const pushSeg = (segment?: ISegment) => {\n            if (segment) { pushSegRaw(segment.toJSONObject(), segment.cachedLength); }\n        };\n\n        let prev: ISegment | undefined;\n        const extractSegment = (segment: ISegment) => {\n            // Elide segments that do not need to be included in the snapshot.  A segment may be elided if\n            // either condition is true:\n            //   a) The segment has not yet been ACKed.  We do not need to snapshot unACKed segments because\n            //      there is a pending insert op that will deliver the segment on reconnection.\n            //   b) The segment was removed at or below the MSN.  Pending ops can no longer reference this\n            //      segment, and therefore we can discard it.\n            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n            if (segment.seq === UnassignedSequenceNumber || segment.removedSeq! <= minSeq) {\n                return true;\n            }\n\n            // Next determine if the snapshot needs to preserve information required for merging the segment\n            // (seq, client, etc.)  This information is only needed if the segment is above the MSN (and doesn't\n            // have a pending remove.)\n            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n            if ((segment.seq! <= minSeq)                                   // Segment is below the MSN, and...\n                && (segment.removedSeq === undefined                      // .. Segment has not been removed, or...\n                    || segment.removedSeq === UnassignedSequenceNumber)   // .. Removal op to be delivered on reconnect\n            ) {\n                // This segment is below the MSN, which means that future ops will not reference it.  Attempt to\n                // coalesce the new segment with the previous (if any).\n                if (!prev) {\n                    // We do not have a previous candidate for coalescing.  Make the current segment the new candidate.\n                    prev = segment;\n                } else if (prev.canAppend(segment) && matchProperties(prev.properties, segment.properties)) {\n                    // We have a compatible pair.  Replace `prev` with the coalesced segment.  Clone to avoid\n                    // modifying the segment instances currently in the MergeTree.\n                    prev = prev.clone();\n                    prev.append(segment.clone());\n                } else {\n                    // The segment pair could not be coalesced.  Record the `prev` segment in the snapshot\n                    // and make the current segment the new candidate for coalescing.\n                    pushSeg(prev);\n                    prev = segment;\n                }\n            } else {\n                // This segment needs to preserve it's metadata as it may be referenced by future ops.  It's ineligible\n                // for coalescing, so emit the 'prev' segment now (if any).\n                pushSeg(prev);\n                prev = undefined;\n\n                const raw: IJSONSegmentWithMergeInfo = { json: segment.toJSONObject() };\n                // If the segment insertion is above the MSN, record the insertion merge info.\n                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n                if (segment.seq! > minSeq) {\n                    raw.seq = segment.seq;\n                    raw.client = this.getLongClientId(segment.clientId);\n                }\n                // We have already dispensed with removed segments below the MSN and removed segments with unassigned\n                // sequence numbers.  Any remaining removal info should be preserved.\n                if (segment.removedSeq !== undefined) {\n                    assert(segment.removedSeq !== UnassignedSequenceNumber && segment.removedSeq > minSeq,\n                        0x065 /* \"On removal info preservation, segment has invalid removed sequence number!\" */);\n                    raw.removedSeq = segment.removedSeq;\n                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n                    raw.removedClient = this.getLongClientId(segment.removedClientId!);\n                }\n\n            // Sanity check that we are preserving either the seq < minSeq or a removed segment's info.\n                assert(raw.seq !== undefined && raw.client !== undefined\n                    || raw.removedSeq !== undefined && raw.removedClient !== undefined,\n                    0x066 /* \"Corrupted preservation of segment metadata!\" */);\n\n                // Record the segment with it's required metadata.\n                pushSegRaw(raw, segment.cachedLength);\n            }\n            return true;\n        };\n\n        mergeTree.walkAllSegments(mergeTree.root, extractSegment, this);\n\n        // If the last segment in the walk was coalescable, push it now.\n        pushSeg(prev);\n\n        return this.segments;\n    }\n\n    public static async loadChunk(\n        storage: IChannelStorageService,\n        path: string,\n        logger: ITelemetryLogger,\n        options: PropertySet | undefined,\n        serializer?: IFluidSerializer,\n    ): Promise<MergeTreeChunkV1> {\n        const blob = await storage.readBlob(path);\n        const chunkAsString = bufferToString(blob, \"utf8\");\n        return SnapshotV1.processChunk(path, chunkAsString, logger, options, serializer);\n    }\n\n    public static processChunk(\n        path: string,\n        chunk: string,\n        logger: ITelemetryLogger,\n        options: PropertySet | undefined,\n        serializer?: IFluidSerializer,\n    ): MergeTreeChunkV1 {\n        const chunkObj = serializer ? serializer.parse(chunk) : JSON.parse(chunk);\n        return toLatestVersion(path, chunkObj, logger, options);\n    }\n}\n"]}
+{"version":3,"file":"snapshotV1.js","sourceRoot":"","sources":["../src/snapshotV1.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAKvD,OAAO,EACH,eAAe,GAElB,MAAM,cAAc,CAAC;AACtB,OAAO,EAKH,eAAe,EACf,8BAA8B,GACjC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,OAAO,UAAU;IAenB,YACW,SAAoB,EAC3B,MAAwB,EACP,eAAuC,EACjD,QAAiB,EACjB,YAAyB;;QAJzB,cAAS,GAAT,SAAS,CAAW;QAEV,oBAAe,GAAf,eAAe,CAAwB;QACjD,aAAQ,GAAR,QAAQ,CAAS;QACjB,iBAAY,GAAZ,YAAY,CAAa;QAEhC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,eAAG,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,OAAO,0CAAE,0BAA0B,mCAAI,UAAU,CAAC,SAAS,CAAC;QAExF,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC,eAAe,EAAE,CAAC;QAC3D,IAAI,CAAC,MAAM,GAAG;YACV,iBAAiB,EAAE,MAAM;YACzB,cAAc,EAAE,UAAU;YAC1B,oBAAoB,EAAE,EAAE;YACxB,WAAW,EAAE,CAAC;YACd,iBAAiB,EAAE,CAAC;SACvB,CAAC;QAEF,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC7B,CAAC;IAEO,gBAAgB,CACpB,WAA+B,EAC/B,UAAoB,EACpB,oBAA4B,EAC5B,UAAU,GAAG,CAAC;QACd,MAAM,QAAQ,GAAuB,EAAE,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,OAAO,CAAC,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE;YAC1F,MAAM,IAAI,GAAG,WAAW,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC;YACpD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,MAAM,IAAI,UAAU,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC;YAChD,YAAY,EAAE,CAAC;SAClB;QACD,OAAO;YACH,OAAO,EAAE,GAAG;YACZ,YAAY;YACZ,MAAM;YACN,QAAQ;YACR,UAAU;YACV,cAAc,EAAE,SAAS;SAC5B,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,IAAI,CACA,UAA4B,EAC5B,IAAkB;QAElB,MAAM,MAAM,GAAuB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QAC5B,GAAG;YACC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAC/B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,KAAK,CAAC,YAAY,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;SAC3C,QAAQ,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;QAE/D,yDAAyD;QACzD,oEAAoE;QACpE,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAG,CAAC;QACpC,WAAW,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QACzC,WAAW,CAAC,cAAc,CAAC,oBAAoB,GAAG,CAAC,EAAE,EAAE,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,MAAM,KAAK,GAAqC,EAAE,CAAC;QACnD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YAC5B,MAAM,EAAE,GAAG,GAAG,cAAc,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,8BAA8B,CAC1C,EAAE,EACF,KAAK,EACL,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,CAAC,OAAO,EACtB,UAAU,EACV,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACzC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,8BAA8B,CACjE,cAAc,CAAC,MAAM,EACrB,WAAW,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,CAAC,OAAO,EACtB,UAAU,EACV,IAAI,CAAC,CAAC,CAAC;QACX,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACpB,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED,WAAW;QACP,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAE7C,yEAAyE;QACzE,MAAM,UAAU,GAAG,CAAC,IAAsB,EAAE,MAAc,EAAE,EAAE;YAC1D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,iGAAiG;QACjG,MAAM,OAAO,GAAG,CAAC,OAAkB,EAAE,EAAE;YACnC,IAAI,OAAO,EAAE;gBAAE,UAAU,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;aAAE;QAC9E,CAAC,CAAC;QAEF,IAAI,IAA0B,CAAC;QAC/B,MAAM,cAAc,GAAG,CAAC,OAAiB,EAAE,EAAE;;YACzC,8FAA8F;YAC9F,4BAA4B;YAC5B,gGAAgG;YAChG,mFAAmF;YACnF,8FAA8F;YAC9F,iDAAiD;YACjD,oEAAoE;YACpE,IAAI,OAAO,CAAC,GAAG,KAAK,wBAAwB,IAAI,OAAO,CAAC,UAAW,IAAI,MAAM,EAAE;gBAC3E,OAAO,IAAI,CAAC;aACf;YAED,gGAAgG;YAChG,oGAAoG;YACpG,0BAA0B;YAC1B,oEAAoE;YACpE,IAAI,CAAC,OAAO,CAAC,GAAI,IAAI,MAAM,CAAC,CAAmC,mCAAmC;mBAC3F,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAsB,yCAAyC;uBAC5F,OAAO,CAAC,UAAU,KAAK,wBAAwB,CAAC,CAAG,6CAA6C;cACzG;gBACE,gGAAgG;gBAChG,uDAAuD;gBACvD,IAAI,CAAC,IAAI,EAAE;oBACP,mGAAmG;oBACnG,IAAI,GAAG,OAAO,CAAC;iBAClB;qBAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE;oBACxF,yFAAyF;oBACzF,8DAA8D;oBAC9D,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;oBACpB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;iBAChC;qBAAM;oBACH,sFAAsF;oBACtF,iEAAiE;oBACjE,OAAO,CAAC,IAAI,CAAC,CAAC;oBACd,IAAI,GAAG,OAAO,CAAC;iBAClB;aACJ;iBAAM;gBACH,uGAAuG;gBACvG,2DAA2D;gBAC3D,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,IAAI,GAAG,SAAS,CAAC;gBAEjB,MAAM,GAAG,GAA8B,EAAE,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;gBACxE,8EAA8E;gBAC9E,oEAAoE;gBACpE,IAAI,OAAO,CAAC,GAAI,GAAG,MAAM,EAAE;oBACvB,GAAG,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;oBACtB,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;iBACvD;gBACD,qGAAqG;gBACrG,qEAAqE;gBACrE,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE;oBAClC,MAAM,CAAC,OAAO,CAAC,UAAU,KAAK,wBAAwB,IAAI,OAAO,CAAC,UAAU,GAAG,MAAM,EACjF,KAAK,CAAC,kFAAkF,CAAC,CAAC;oBAC9F,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;oBAEpC,2DAA2D;oBAC3D,GAAG,CAAC,aAAa;wBACb,OAAO,CAAC,gBAAgB,KAAK,SAAS;4BAClC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;4BACnD,CAAC,CAAC,SAAS,CAAC;oBAEpB,GAAG,CAAC,gBAAgB,SAAG,OAAO,CAAC,gBAAgB,0CAAE,GAAG,CAAC,CAAC,EAAE,EAAC,EAAE,CAAA,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC;iBACxF;gBAEL,2FAA2F;gBACvF,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;uBACjD,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,EAClE,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBAE/D,kDAAkD;gBAClD,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;aACzC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAEF,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;QAEhE,gEAAgE;QAChE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEd,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,SAAS,CACzB,OAA+B,EAC/B,IAAY,EACZ,MAAwB,EACxB,OAAgC,EAChC,UAA6B;QAE7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACrF,CAAC;IAEM,MAAM,CAAC,YAAY,CACtB,IAAY,EACZ,KAAa,EACb,MAAwB,EACxB,OAAgC,EAChC,UAA6B;QAE7B,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1E,OAAO,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5D,CAAC;;AA7OD,2GAA2G;AAC3G,gHAAgH;AAChH,wFAAwF;AACxF,8FAA8F;AAC9F,wEAAwE;AACxE,sEAAsE;AAC/C,oBAAS,GAAW,KAAK,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IFluidHandle } from \"@fluidframework/core-interfaces\";\nimport { IFluidSerializer } from \"@fluidframework/shared-object-base\";\nimport { assert, bufferToString } from \"@fluidframework/common-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { IChannelStorageService } from \"@fluidframework/datastore-definitions\";\nimport { ISummaryTreeWithStats } from \"@fluidframework/runtime-definitions\";\nimport { SummaryTreeBuilder } from \"@fluidframework/runtime-utils\";\nimport { UnassignedSequenceNumber } from \"./constants\";\nimport {\n    ISegment,\n    MergeTree,\n} from \"./mergeTree\";\nimport {\n    matchProperties,\n    PropertySet,\n} from \"./properties\";\nimport {\n    IJSONSegmentWithMergeInfo,\n    JsonSegmentSpecs,\n    MergeTreeHeaderMetadata,\n    MergeTreeChunkV1,\n    toLatestVersion,\n    serializeAsMaxSupportedVersion,\n} from \"./snapshotChunks\";\nimport { SnapshotLegacy } from \"./snapshotlegacy\";\n\nexport class SnapshotV1 {\n    // Split snapshot into two entries - headers (small) and body (overflow) for faster loading initial content\n    // Please note that this number has no direct relationship to anything other than size of raw text (characters).\n    // As we produce json for the blob (and then send over the wire compressed), this number\n    // is really hard to correlate with any actual metric that matters (like bytes over the wire).\n    // For test with small number of chunks it would be closer to blob size,\n    // for very chunky text, blob size can easily be 4x-8x of that number.\n    public static readonly chunkSize: number = 10000;\n\n    private readonly header: MergeTreeHeaderMetadata;\n    private readonly segments: JsonSegmentSpecs[];\n    private readonly segmentLengths: number[];\n    private readonly logger: ITelemetryLogger;\n    private readonly chunkSize: number;\n\n    constructor(\n        public mergeTree: MergeTree,\n        logger: ITelemetryLogger,\n        private readonly getLongClientId: (id: number) => string,\n        public filename?: string,\n        public onCompletion?: () => void,\n    ) {\n        this.logger = ChildLogger.create(logger, \"Snapshot\");\n        this.chunkSize = mergeTree?.options?.mergeTreeSnapshotChunkSize ?? SnapshotV1.chunkSize;\n\n        const { currentSeq, minSeq } = mergeTree.getCollabWindow();\n        this.header = {\n            minSequenceNumber: minSeq,\n            sequenceNumber: currentSeq,\n            orderedChunkMetadata: [],\n            totalLength: 0,\n            totalSegmentCount: 0,\n        };\n\n        this.segments = [];\n        this.segmentLengths = [];\n    }\n\n    private getSeqLengthSegs(\n        allSegments: JsonSegmentSpecs[],\n        allLengths: number[],\n        approxSequenceLength: number,\n        startIndex = 0): MergeTreeChunkV1 {\n        const segments: JsonSegmentSpecs[] = [];\n        let length = 0;\n        let segmentCount = 0;\n        while ((length < approxSequenceLength) && ((startIndex + segmentCount) < allSegments.length)) {\n            const pseg = allSegments[startIndex + segmentCount];\n            segments.push(pseg);\n            length += allLengths[startIndex + segmentCount];\n            segmentCount++;\n        }\n        return {\n            version: \"1\",\n            segmentCount,\n            length,\n            segments,\n            startIndex,\n            headerMetadata: undefined,\n        };\n    }\n\n    /**\n     * Emits the snapshot to an ISummarizeResult. If provided the optional IFluidSerializer will be used when\n     * serializing the summary data rather than JSON.stringify.\n     */\n    emit(\n        serializer: IFluidSerializer,\n        bind: IFluidHandle,\n    ): ISummaryTreeWithStats {\n        const chunks: MergeTreeChunkV1[] = [];\n        this.header.totalSegmentCount = 0;\n        this.header.totalLength = 0;\n        do {\n            const chunk = this.getSeqLengthSegs(\n                this.segments,\n                this.segmentLengths,\n                this.chunkSize,\n                this.header.totalSegmentCount);\n            chunks.push(chunk);\n            this.header.totalSegmentCount += chunk.segmentCount;\n            this.header.totalLength += chunk.length;\n        } while (this.header.totalSegmentCount < this.segments.length);\n\n        // The do while loop should have added at least one chunk\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        const headerChunk = chunks.shift()!;\n        headerChunk.headerMetadata = this.header;\n        headerChunk.headerMetadata.orderedChunkMetadata = [{ id: SnapshotLegacy.header }];\n        const blobs: [key: string, content: string][] = [];\n        chunks.forEach((chunk, index) => {\n            const id = `${SnapshotLegacy.body}_${index}`;\n            this.header.orderedChunkMetadata.push({ id });\n            blobs.push([id, serializeAsMaxSupportedVersion(\n                id,\n                chunk,\n                this.logger,\n                this.mergeTree.options,\n                serializer,\n                bind)]);\n        });\n\n        const builder = new SummaryTreeBuilder();\n        builder.addBlob(SnapshotLegacy.header, serializeAsMaxSupportedVersion(\n            SnapshotLegacy.header,\n            headerChunk,\n            this.logger,\n            this.mergeTree.options,\n            serializer,\n            bind));\n        blobs.forEach((value) => {\n            builder.addBlob(value[0], value[1]);\n        });\n\n        return builder.getSummaryTree();\n    }\n\n    extractSync() {\n        const mergeTree = this.mergeTree;\n        const minSeq = this.header.minSequenceNumber;\n\n        // Helper to add the given `MergeTreeChunkV0SegmentSpec` to the snapshot.\n        const pushSegRaw = (json: JsonSegmentSpecs, length: number) => {\n            this.segments.push(json);\n            this.segmentLengths.push(length);\n        };\n\n        // Helper to serialize the given `segment` and add it to the snapshot (if a segment is provided).\n        const pushSeg = (segment?: ISegment) => {\n            if (segment) { pushSegRaw(segment.toJSONObject(), segment.cachedLength); }\n        };\n\n        let prev: ISegment | undefined;\n        const extractSegment = (segment: ISegment) => {\n            // Elide segments that do not need to be included in the snapshot.  A segment may be elided if\n            // either condition is true:\n            //   a) The segment has not yet been ACKed.  We do not need to snapshot unACKed segments because\n            //      there is a pending insert op that will deliver the segment on reconnection.\n            //   b) The segment was removed at or below the MSN.  Pending ops can no longer reference this\n            //      segment, and therefore we can discard it.\n            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n            if (segment.seq === UnassignedSequenceNumber || segment.removedSeq! <= minSeq) {\n                return true;\n            }\n\n            // Next determine if the snapshot needs to preserve information required for merging the segment\n            // (seq, client, etc.)  This information is only needed if the segment is above the MSN (and doesn't\n            // have a pending remove.)\n            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n            if ((segment.seq! <= minSeq)                                   // Segment is below the MSN, and...\n                && (segment.removedSeq === undefined                      // .. Segment has not been removed, or...\n                    || segment.removedSeq === UnassignedSequenceNumber)   // .. Removal op to be delivered on reconnect\n            ) {\n                // This segment is below the MSN, which means that future ops will not reference it.  Attempt to\n                // coalesce the new segment with the previous (if any).\n                if (!prev) {\n                    // We do not have a previous candidate for coalescing.  Make the current segment the new candidate.\n                    prev = segment;\n                } else if (prev.canAppend(segment) && matchProperties(prev.properties, segment.properties)) {\n                    // We have a compatible pair.  Replace `prev` with the coalesced segment.  Clone to avoid\n                    // modifying the segment instances currently in the MergeTree.\n                    prev = prev.clone();\n                    prev.append(segment.clone());\n                } else {\n                    // The segment pair could not be coalesced.  Record the `prev` segment in the snapshot\n                    // and make the current segment the new candidate for coalescing.\n                    pushSeg(prev);\n                    prev = segment;\n                }\n            } else {\n                // This segment needs to preserve it's metadata as it may be referenced by future ops.  It's ineligible\n                // for coalescing, so emit the 'prev' segment now (if any).\n                pushSeg(prev);\n                prev = undefined;\n\n                const raw: IJSONSegmentWithMergeInfo = { json: segment.toJSONObject() };\n                // If the segment insertion is above the MSN, record the insertion merge info.\n                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n                if (segment.seq! > minSeq) {\n                    raw.seq = segment.seq;\n                    raw.client = this.getLongClientId(segment.clientId);\n                }\n                // We have already dispensed with removed segments below the MSN and removed segments with unassigned\n                // sequence numbers.  Any remaining removal info should be preserved.\n                if (segment.removedSeq !== undefined) {\n                    assert(segment.removedSeq !== UnassignedSequenceNumber && segment.removedSeq > minSeq,\n                        0x065 /* \"On removal info preservation, segment has invalid removed sequence number!\" */);\n                    raw.removedSeq = segment.removedSeq;\n\n                    // back compat for when we split overlap and removed client\n                    raw.removedClient =\n                        segment.removedClientIds !== undefined\n                            ? this.getLongClientId(segment.removedClientIds[0])\n                            : undefined;\n\n                    raw.removedClientIds = segment.removedClientIds?.map((id)=>this.getLongClientId(id));\n                }\n\n            // Sanity check that we are preserving either the seq < minSeq or a removed segment's info.\n                assert(raw.seq !== undefined && raw.client !== undefined\n                    || raw.removedSeq !== undefined && raw.removedClient !== undefined,\n                    0x066 /* \"Corrupted preservation of segment metadata!\" */);\n\n                // Record the segment with it's required metadata.\n                pushSegRaw(raw, segment.cachedLength);\n            }\n            return true;\n        };\n\n        mergeTree.walkAllSegments(mergeTree.root, extractSegment, this);\n\n        // If the last segment in the walk was coalescable, push it now.\n        pushSeg(prev);\n\n        return this.segments;\n    }\n\n    public static async loadChunk(\n        storage: IChannelStorageService,\n        path: string,\n        logger: ITelemetryLogger,\n        options: PropertySet | undefined,\n        serializer?: IFluidSerializer,\n    ): Promise<MergeTreeChunkV1> {\n        const blob = await storage.readBlob(path);\n        const chunkAsString = bufferToString(blob, \"utf8\");\n        return SnapshotV1.processChunk(path, chunkAsString, logger, options, serializer);\n    }\n\n    public static processChunk(\n        path: string,\n        chunk: string,\n        logger: ITelemetryLogger,\n        options: PropertySet | undefined,\n        serializer?: IFluidSerializer,\n    ): MergeTreeChunkV1 {\n        const chunkObj = serializer ? serializer.parse(chunk) : JSON.parse(chunk);\n        return toLatestVersion(path, chunkObj, logger, options);\n    }\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/merge-tree",
-  "version": "0.58.2002",
+  "version": "0.59.1000",
   "description": "Merge tree",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -31,7 +31,7 @@
     "lint:fix": "npm run eslint:fix",
     "test": "npm run test:mocha",
     "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
-    "test:mocha": "mocha --recursive dist/test --exit -r node_modules/@fluidframework/mocha-test-setup -r source-map-support/register --unhandled-rejections=strict",
+    "test:mocha": "mocha --ignore 'dist/test/types/*' --recursive dist/test --exit -r node_modules/@fluidframework/mocha-test-setup -r source-map-support/register --unhandled-rejections=strict",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
     "tsc": "tsc",
     "tsfmt": "tsfmt --verify",
@@ -60,20 +60,21 @@
   "dependencies": {
     "@fluidframework/common-definitions": "^0.20.1",
     "@fluidframework/common-utils": "^0.32.1",
-    "@fluidframework/container-definitions": "^0.47.1000",
-    "@fluidframework/core-interfaces": "^0.42.0",
-    "@fluidframework/datastore-definitions": "^0.58.2002",
-    "@fluidframework/protocol-definitions": "^0.1027.1000",
-    "@fluidframework/runtime-definitions": "^0.58.2002",
-    "@fluidframework/runtime-utils": "^0.58.2002",
-    "@fluidframework/shared-object-base": "^0.58.2002",
-    "@fluidframework/telemetry-utils": "^0.58.2002"
+    "@fluidframework/container-definitions": "^0.48.1000",
+    "@fluidframework/core-interfaces": "^0.43.1000",
+    "@fluidframework/datastore-definitions": "^0.59.1000",
+    "@fluidframework/protocol-definitions": "^0.1028.1000",
+    "@fluidframework/runtime-definitions": "^0.59.1000",
+    "@fluidframework/runtime-utils": "^0.59.1000",
+    "@fluidframework/shared-object-base": "^0.59.1000",
+    "@fluidframework/telemetry-utils": "^0.59.1000"
   },
   "devDependencies": {
     "@fluidframework/build-common": "^0.23.0",
-    "@fluidframework/eslint-config-fluid": "^0.27.0",
-    "@fluidframework/mocha-test-setup": "^0.58.2002",
-    "@fluidframework/test-runtime-utils": "^0.58.2002",
+    "@fluidframework/eslint-config-fluid": "^0.28.1000",
+    "@fluidframework/merge-tree-previous": "npm:@fluidframework/merge-tree@^0.58.0",
+    "@fluidframework/mocha-test-setup": "^0.59.1000",
+    "@fluidframework/test-runtime-utils": "^0.59.1000",
     "@microsoft/api-extractor": "^7.16.1",
     "@rushstack/eslint-config": "^2.5.1",
     "@types/diff": "^3.5.1",
@@ -100,5 +101,10 @@
     "source-map-support": "^0.5.16",
     "typescript": "~4.1.3",
     "typescript-formatter": "7.1.0"
+  },
+  "typeValidation": {
+    "version": "0.59.1000",
+    "broken": {},
+    "disabled": true
   }
 }

package/src/client.ts

@@ -545,6 +545,8 @@ export class Client {
             }
         }
 
+        // start and end are guaranteed to be non-null here, otherwise we throw above.
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
         return { start, end } as IIntegerRange;
     }
 

package/src/collections.ts

@@ -190,6 +190,7 @@ export class Heap<T> {
         this.fixup(this.count());
     }
 
+    /* eslint-disable no-bitwise */
     private fixup(k: number) {
         let _k = k;
         while (_k > 1 && (this.comp.compare(this.L[_k >> 1], this.L[_k]) > 0)) {
@@ -216,6 +217,7 @@ export class Heap<T> {
             _k = j;
         }
     }
+    /* eslint-enable no-bitwise */
 }
 
 export const enum RBColor {

package/src/mergeTree.ts

@@ -104,15 +104,21 @@ export interface IHierBlock extends IMergeBlock {
 }
 
 export interface IRemovalInfo {
-    removedSeq?: number;
-    removedClientId?: number;
-    removedClientOverlap?: number[];
+    removedSeq: number;
+    removedClientIds: number[];
+}
+export function toRemovalInfo(maybe: Partial<IRemovalInfo> | undefined): IRemovalInfo | undefined {
+    if (maybe?.removedClientIds !== undefined && maybe?.removedSeq !== undefined) {
+        return maybe as IRemovalInfo;
+    }
+    assert(maybe?.removedClientIds === undefined && maybe?.removedSeq === undefined,
+        0x2bf /* "both removedClientIds and removedSeq should be set or not set" */);
 }
 
 /**
  * A segment representing a portion of the merge tree.
  */
-export interface ISegment extends IMergeNodeCommon, IRemovalInfo {
+export interface ISegment extends IMergeNodeCommon, Partial<IRemovalInfo> {
     readonly type: string;
     readonly segmentGroups: SegmentGroupCollection;
     readonly trackingCollection: TrackingGroupCollection;
@@ -482,8 +488,7 @@ export abstract class BaseSegment extends MergeNode implements ISegment {
     public clientId: number = LocalClientId;
     public seq: number = UniversalSequenceNumber;
     public removedSeq?: number;
-    public removedClientId?: number;
-    public removedClientOverlap?: number[];
+    public removedClientIds?: number[];
     public readonly segmentGroups: SegmentGroupCollection = new SegmentGroupCollection(this);
     public readonly trackingCollection: TrackingGroupCollection = new TrackingGroupCollection(this);
     public propertyManager?: PropertiesManager;
@@ -521,7 +526,7 @@ export abstract class BaseSegment extends MergeNode implements ISegment {
         b.clientId = this.clientId;
         // TODO: deep clone properties
         b.properties = clone(this.properties);
-        b.removedClientId = this.removedClientId;
+        b.removedClientIds = this.removedClientIds?.slice();
         // TODO: copy removed client overlap and branch removal info
         b.removedSeq = this.removedSeq;
         b.seq = this.seq;
@@ -555,10 +560,9 @@ export abstract class BaseSegment extends MergeNode implements ISegment {
                 return true;
 
             case MergeTreeDeltaType.REMOVE:
-                // eslint-disable-next-line @typescript-eslint/no-this-alias
-                const removalInfo: IRemovalInfo = this;
-                assert(!!removalInfo, 0x046 /* "On remove ack, missing removal info!" */);
-                assert(!!removalInfo.removedSeq, 0x047 /* "On remove ack, missing removed sequence number!" */);
+
+                const removalInfo: IRemovalInfo | undefined = toRemovalInfo(this);
+                assert(removalInfo !== undefined, 0x046 /* "On remove ack, missing removal info!" */);
                 this.localRemovedSeq = undefined;
                 if (removalInfo.removedSeq === UnassignedSequenceNumber) {
                     removalInfo.removedSeq = opArgs.sequencedMessage!.sequenceNumber;
@@ -584,15 +588,12 @@ export abstract class BaseSegment extends MergeNode implements ISegment {
                 // but this ordinal meets all the necessary invariants for now.
                 leafSegment.ordinal = this.ordinal + String.fromCharCode(0);
 
-                leafSegment.removedClientId = this.removedClientId;
+                leafSegment.removedClientIds = this.removedClientIds?.slice();
                 leafSegment.removedSeq = this.removedSeq;
                 leafSegment.localRemovedSeq = this.localRemovedSeq;
                 leafSegment.seq = this.seq;
                 leafSegment.localSeq = this.localSeq;
                 leafSegment.clientId = this.clientId;
-                if (this.removedClientOverlap) {
-                    leafSegment.removedClientOverlap = [...this.removedClientOverlap];
-                }
                 this.segmentGroups.copyTo(leafSegment);
                 this.trackingCollection.copyTo(leafSegment);
                 if (this.localRefs) {
@@ -1035,10 +1036,6 @@ export class MergeTree {
 
     private static readonly initBlockUpdateActions: BlockUpdateActions;
     private static readonly theUnfinishedNode = <IMergeBlock>{ childCount: -1 };
-    // WARNING:
-    // Setting blockUpdateMarkers to false will result in eventual consistency issues
-    // for property updates on markers when loading from snapshots
-    private static readonly blockUpdateMarkers = true;
 
     root: IMergeBlock;
     private readonly blockUpdateActions: BlockUpdateActions = MergeTree.initBlockUpdateActions;
@@ -1059,12 +1056,7 @@ export class MergeTree {
     }
 
     private makeBlock(childCount: number) {
-        let block: MergeBlock;
-        if (MergeTree.blockUpdateMarkers) {
-            block = new HierMergeBlock(childCount);
-        } else {
-            block = new MergeBlock(childCount);
-        }
+        const block: MergeBlock = new HierMergeBlock(childCount);
         block.ordinal = "";
         return block;
     }
@@ -1100,8 +1092,8 @@ export class MergeTree {
     }
 
     public localNetLength(segment: ISegment) {
-        const removalInfo: IRemovalInfo = segment;
-        if (removalInfo.removedSeq !== undefined) {
+        const removalInfo = toRemovalInfo(segment);
+        if (removalInfo !== undefined) {
             return 0;
         } else {
             return segment.cachedLength;
@@ -1467,9 +1459,9 @@ export class MergeTree {
                 return node.partialLengths!.getPartialLength(refSeq, clientId);
             } else {
                 const segment = node;
-                const removalInfo: IRemovalInfo = segment;
+                const removalInfo = toRemovalInfo(segment);
 
-                if(removalInfo.removedSeq !== undefined
+                if(removalInfo !== undefined
                     && removalInfo.removedSeq !== UnassignedSequenceNumber
                     && removalInfo.removedSeq <= refSeq) {
                     // this segment is a tombstone eligible for zamboni
@@ -1480,12 +1472,8 @@ export class MergeTree {
                 if (((segment.clientId === clientId) ||
                     ((segment.seq !== UnassignedSequenceNumber) && (segment.seq! <= refSeq)))) {
                     // Segment happened by reference sequence number or segment from requesting client
-                    if (removalInfo.removedSeq !== undefined) {
-                        if (
-                            removalInfo.removedClientId === clientId
-                            || (removalInfo.removedClientOverlap
-                                && removalInfo.removedClientOverlap.includes(clientId))
-                        ) {
+                    if (removalInfo !== undefined) {
+                        if (removalInfo.removedClientIds.includes(clientId)) {
                             return 0;
                         } else {
                             return segment.cachedLength;
@@ -1497,7 +1485,7 @@ export class MergeTree {
                     // the segment was inserted and removed before the
                     // this context, so it will never exist for this
                     // context
-                    if(removalInfo.removedSeq !== undefined
+                    if(removalInfo !== undefined
                         && removalInfo.removedSeq !== UnassignedSequenceNumber) {
                         return undefined;
                     }
@@ -2305,13 +2293,6 @@ export class MergeTree {
         }
     }
 
-    private addOverlappingClient(removalInfo: IRemovalInfo, clientId: number) {
-        if (!removalInfo.removedClientOverlap) {
-            removalInfo.removedClientOverlap = <number[]>[];
-        }
-        removalInfo.removedClientOverlap.push(clientId);
-    }
-
     /**
      * Annotate a range with properties
      * @param start - The inclusive start position of the range to annotate
@@ -2382,21 +2363,24 @@ export class MergeTree {
         const savedLocalRefs: LocalReferenceCollection[] = [];
         const localSeq = seq === UnassignedSequenceNumber ? ++this.collabWindow.localSeq : undefined;
         const markRemoved = (segment: ISegment, pos: number, _start: number, _end: number) => {
-            const removalInfo: IRemovalInfo = segment;
-            if (removalInfo.removedSeq !== undefined) {
+            const existingRemovalInfo = toRemovalInfo(segment);
+            if (existingRemovalInfo !== undefined) {
                 _overwrite = true;
-                if (removalInfo.removedSeq === UnassignedSequenceNumber) {
-                    // replace because comes later
-                    removalInfo.removedClientId = clientId;
-                    removalInfo.removedSeq = seq;
+                if (existingRemovalInfo.removedSeq === UnassignedSequenceNumber) {
+                    // we removed this locally, but someone else removed it first
+                    // so put them at the head of the list
+                    // the list isn't ordered, but we
+                    // keep first removal at the head.
+                    existingRemovalInfo.removedClientIds.unshift(clientId);
+                    existingRemovalInfo.removedSeq = seq;
                     segment.localRemovedSeq = undefined;
                 } else {
                     // Do not replace earlier sequence number for remove
-                    this.addOverlappingClient(removalInfo, clientId);
+                    existingRemovalInfo.removedClientIds.push(clientId);
                 }
             } else {
-                removalInfo.removedClientId = clientId;
-                removalInfo.removedSeq = seq;
+                segment.removedClientIds = [clientId];
+                segment.removedSeq = seq;
                 segment.localRemovedSeq = localSeq;
 
                 removedSegments.push({ segment });
@@ -2408,9 +2392,7 @@ export class MergeTree {
 
             // Save segment so can assign removed sequence number when acked by server
             if (this.collabWindow.collaborating) {
-                // Use removal information
-                const _removalInfo: IRemovalInfo = segment;
-                if (_removalInfo.removedSeq === UnassignedSequenceNumber && clientId === this.collabWindow.clientId) {
+                if (segment.removedSeq === UnassignedSequenceNumber && clientId === this.collabWindow.clientId) {
                     segmentGroup = this.addToPendingList(segment, segmentGroup, localSeq);
                 } else {
                     if (MergeTree.options.zamboniSegments) {

package/src/partialLengths.ts

@@ -14,6 +14,7 @@ import {
     IRemovalInfo,
     ISegment,
     MergeTree,
+    toRemovalInfo,
 } from "./mergeTree";
 
 interface IOverlapClient {
@@ -81,7 +82,7 @@ export class PartialSequenceLengths {
         collabWindow: CollaborationWindow,
         recur = false) {
         let combinedPartialLengths = new PartialSequenceLengths(collabWindow.minSeq);
-        PartialSequenceLengths.fromLeaves(mergeTree, combinedPartialLengths, block, collabWindow);
+        PartialSequenceLengths.fromLeaves(combinedPartialLengths, block, collabWindow);
         let prevPartial: PartialSequenceLength | undefined;
 
         function cloneOverlapRemoveClients(oldTree: RedBlackTree<number, IOverlapClient> | undefined) {
@@ -209,13 +210,13 @@ export class PartialSequenceLengths {
     }
 
     private static fromLeaves(
-        mergeTree: MergeTree, combinedPartialLengths: PartialSequenceLengths,
+        combinedPartialLengths: PartialSequenceLengths,
         block: IMergeBlock, collabWindow: CollaborationWindow) {
         combinedPartialLengths.minLength = 0;
         combinedPartialLengths.segmentCount = block.childCount;
 
-        function seqLTE(seq: number, minSeq: number) {
-            return (seq !== UnassignedSequenceNumber) && (seq <= minSeq);
+        function seqLTE(seq: number | undefined, minSeq: number) {
+            return seq !== undefined && seq !== UnassignedSequenceNumber && seq <= minSeq;
         }
 
         for (let i = 0; i < block.childCount; i++) {
@@ -223,21 +224,19 @@ export class PartialSequenceLengths {
             if (child.isLeaf()) {
                 // Leaf segment
                 const segment = child;
-                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                if (seqLTE(segment.seq!, collabWindow.minSeq)) {
+                if (seqLTE(segment.seq, collabWindow.minSeq)) {
                     combinedPartialLengths.minLength += segment.cachedLength;
                 } else {
                     if (segment.seq !== UnassignedSequenceNumber) {
                         PartialSequenceLengths.insertSegment(combinedPartialLengths, segment);
                     }
                 }
-                const removalInfo: IRemovalInfo = segment;
-                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                if (seqLTE(removalInfo.removedSeq!, collabWindow.minSeq)) {
+                const removalInfo = toRemovalInfo(segment);
+                if (seqLTE(removalInfo?.removedSeq, collabWindow.minSeq)) {
                     combinedPartialLengths.minLength -= segment.cachedLength;
                 } else {
-                    if ((removalInfo.removedSeq !== undefined) &&
-                        (removalInfo.removedSeq !== UnassignedSequenceNumber)) {
+                    if (removalInfo !== undefined
+                        && removalInfo.removedSeq !== UnassignedSequenceNumber) {
                         PartialSequenceLengths.insertSegment(
                             combinedPartialLengths,
                             segment,
@@ -300,14 +299,15 @@ export class PartialSequenceLengths {
         let removeClientOverlap: number[] | undefined;
 
         if (removalInfo) {
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            seq = removalInfo.removedSeq!;
+            seq = removalInfo.removedSeq;
             segmentLen = -segmentLen;
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            clientId = removalInfo.removedClientId!;
-            if (removalInfo.removedClientOverlap) {
-                removeClientOverlap = removalInfo.removedClientOverlap;
-            }
+            // this code still assume removed client id and
+            // overlap clients are separate. so we need to pull
+            // then apart first.
+            clientId = removalInfo.removedClientIds[0];
+            removeClientOverlap = removalInfo.removedClientIds.length > 1
+                ? removalInfo.removedClientIds.slice(1)
+                : undefined;
         }
 
         const seqPartials = combinedPartialLengths.partialLengths;
@@ -366,6 +366,8 @@ export class PartialSequenceLengths {
             }
         }
         if (seqPartialLen === undefined) {
+            // len will be assigned below, making this assertion true.
+            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
             seqPartialLen = {
                 clientId,
                 seglen: seqSeglen,
@@ -419,14 +421,14 @@ export class PartialSequenceLengths {
                 segCount += branchPartialLengths.segmentCount;
             } else {
                 const segment = child;
-                const removalInfo: IRemovalInfo = segment;
+                const removalInfo = toRemovalInfo(segment);
 
                 if (segment.seq === seq) {
-                    if (removalInfo.removedSeq !== seq) {
+                    if (removalInfo?.removedSeq !== seq) {
                         seqSeglen += segment.cachedLength;
                     }
                 } else {
-                    if (removalInfo.removedSeq === seq) {
+                    if (removalInfo?.removedSeq === seq) {
                         seqSeglen -= segment.cachedLength;
                     }
                 }

package/src/snapshotChunks.ts

@@ -62,7 +62,11 @@ export interface IJSONSegmentWithMergeInfo {
     json: IJSONSegment;
     client?: string;
     seq?: number;
+    /**
+     * @deprecated - use removedClientIds instead. this only exists for back-compat
+     */
     removedClient?: string;
+    removedClientIds?: string[];
     removedSeq?: number;
 }
 

package/src/snapshotLoader.ts

@@ -102,8 +102,16 @@ export class SnapshotLoader {
             if (spec.removedSeq !== undefined) {
                 seg.removedSeq = spec.removedSeq;
             }
+            // this format had a bug where it didn't store all the overlap clients
+            // this is for back compat, so we change the singular id to an array
+            // this will only cause problems if there is an overlapping delete
+            // spanning the snapshot, which should be rare
             if (spec.removedClient !== undefined) {
-                seg.removedClientId = this.client.getOrAddShortClientId(spec.removedClient);
+                seg.removedClientIds = [this.client.getOrAddShortClientId(spec.removedClient)];
+            }
+            if (spec.removedClientIds !== undefined) {
+                seg.removedClientIds = spec.removedClientIds?.map(
+                    (sid)=> this.client.getOrAddShortClientId(sid));
             }
         } else {
             seg = this.client.specToSegment(spec);

package/src/snapshotV1.ts

@@ -218,8 +218,14 @@ export class SnapshotV1 {
                     assert(segment.removedSeq !== UnassignedSequenceNumber && segment.removedSeq > minSeq,
                         0x065 /* "On removal info preservation, segment has invalid removed sequence number!" */);
                     raw.removedSeq = segment.removedSeq;
-                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                    raw.removedClient = this.getLongClientId(segment.removedClientId!);
+
+                    // back compat for when we split overlap and removed client
+                    raw.removedClient =
+                        segment.removedClientIds !== undefined
+                            ? this.getLongClientId(segment.removedClientIds[0])
+                            : undefined;
+
+                    raw.removedClientIds = segment.removedClientIds?.map((id)=>this.getLongClientId(id));
                 }
 
             // Sanity check that we are preserving either the seq < minSeq or a removed segment's info.