@fluid-tools/fetch-tool 0.53.0-46105

package/dist/fluidFetchSnapshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchSnapshot.d.ts","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACH,gBAAgB,EAEnB,MAAM,oCAAoC,CAAC;AAqR5C,wBAAsB,kBAAkB,CACpC,eAAe,CAAC,EAAE,gBAAgB,EAClC,OAAO,CAAC,EAAE,MAAM,iBA4FnB"}

package/dist/fluidFetchSnapshot.js

@@ -0,0 +1,289 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fluidFetchSnapshot = void 0;
+const fs_1 = __importDefault(require("fs"));
+const util_1 = __importDefault(require("util"));
+const common_utils_1 = require("@fluidframework/common-utils");
+const driver_utils_1 = require("@fluidframework/driver-utils");
+const fluidAnalyzeMessages_1 = require("./fluidAnalyzeMessages");
+const fluidFetchArgs_1 = require("./fluidFetchArgs");
+const fluidFetchInit_1 = require("./fluidFetchInit");
+function isFetchedTree(fetchedData) {
+    return "patched" in fetchedData;
+}
+const blobCache = new Map();
+let blobCachePrevious = new Map();
+let blobCacheCurrent = new Map();
+function fetchBlobs(prefix, tree, storage, blobIdMap) {
+    const result = [];
+    for (const item of Object.keys(tree.blobs)) {
+        const treePath = `${prefix}${item}`;
+        const blobId = tree.blobs[item];
+        if (blobId !== null) {
+            let reused = true;
+            let blob = blobCachePrevious.get(blobId);
+            if (!blob) {
+                reused = false;
+                blob = blobCache.get(blobId);
+                if (blob === undefined) {
+                    blob = storage.readBlob(blobId);
+                    blobCache.set(blobId, blob);
+                }
+            }
+            blobCacheCurrent.set(blobId, blob);
+            // Use the blobIdMap to assign a number for each unique blob
+            // and use it as a prefix for files to avoid case-insensitive fs
+            let index = blobIdMap.get(blobId);
+            if (!index) {
+                index = blobIdMap.size;
+                blobIdMap.set(blobId, index);
+            }
+            const filename = `${index}-${blobId}`;
+            result.push({ treePath, blobId, blob, reused, filename });
+            // patch the tree so that we can write it out to reference the file
+            tree.blobs[item] = filename;
+        }
+    }
+    return result;
+}
+function createTreeBlob(tree, prefix, patched) {
+    var _a;
+    const id = (_a = tree.id) !== null && _a !== void 0 ? _a : "original";
+    const blob = common_utils_1.stringToBuffer(JSON.stringify(tree), "utf8");
+    const filename = patched ? "tree" : `tree-${id}`;
+    const treePath = `${prefix}${filename}`;
+    return { treePath, blobId: "original tree $id", filename, blob, patched, reused: false };
+}
+async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/", perCommitBlobIdMap) {
+    common_utils_1.assert(Object.keys(tree.commits).length === 0 || (prefix === "/"), 0x1be /* "Unexpected tree input to fetch" */);
+    const commit = !perCommitBlobIdMap;
+    if (commit && fluidFetchArgs_1.dumpSnapshotTrees) {
+        console.log(tree);
+    }
+    if (prefix === "/") {
+        blobCachePrevious = blobCacheCurrent;
+        blobCacheCurrent = new Map();
+    }
+    // Create the tree info before fetching blobs (which will modify it)
+    let commitBlob;
+    if (commit) {
+        commitBlob = createTreeBlob(tree, prefix, false);
+    }
+    const blobIdMap = perCommitBlobIdMap !== null && perCommitBlobIdMap !== void 0 ? perCommitBlobIdMap : new Map();
+    let result = fetchBlobs(prefix, tree, storage, blobIdMap);
+    for (const dataStore of Object.keys(tree.commits)) {
+        const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);
+        if (dataStoreVersions.length !== 1) {
+            console.error(`ERROR: Unable to get versions for ${dataStore}`);
+            continue;
+        }
+        const dataStoreSnapShotTree = await reportErrors(`getSnapshotTree ${dataStoreVersions[0].id}`, storage.getSnapshotTree(dataStoreVersions[0]));
+        if (dataStoreSnapShotTree === null) {
+            // eslint-disable-next-line max-len
+            console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);
+            continue;
+        }
+        common_utils_1.assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore], 0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);
+        common_utils_1.assert(tree.commits[dataStore] === dataStoreVersions[0].id, 0x1c0 /* "Mismatch between commit id and fetched tree id" */);
+        const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, dataStoreSnapShotTree, `${prefix}[${dataStore}]/`);
+        result = result.concat(dataStoreBlobs);
+    }
+    for (const subtreeId of Object.keys(tree.trees)) {
+        const subtree = tree.trees[subtreeId];
+        common_utils_1.assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* "Unexpected subtree properties" */);
+        const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, subtree, `${prefix}${subtreeId}/`, blobIdMap);
+        result = result.concat(dataStoreBlobs);
+    }
+    if (commitBlob) {
+        result.push(commitBlob);
+        result.push(createTreeBlob(tree, prefix, true));
+    }
+    return result;
+}
+function getDumpFetchedData(fetchedData) {
+    const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));
+    return sorted.filter((item) => !isFetchedTree(item) || !item.patched);
+}
+async function dumpSnapshotTreeVerbose(name, fetchedData) {
+    let size = 0;
+    const sorted = getDumpFetchedData(fetchedData);
+    let nameLength = 10;
+    for (const item of sorted) {
+        nameLength = Math.max(nameLength, item.treePath.length);
+    }
+    console.log("");
+    console.log(`${"Blob Path".padEnd(nameLength)} | Reused |      Bytes`);
+    console.log("-".repeat(nameLength + 26));
+    for (const item of sorted) {
+        const buffer = await item.blob;
+        if (buffer === undefined) {
+            continue;
+        }
+        const blob = common_utils_1.bufferToString(buffer, "utf8");
+        // eslint-disable-next-line max-len
+        console.log(`${item.treePath.padEnd(nameLength)} |    ${item.reused ? "X" : " "}   | ${fluidAnalyzeMessages_1.formatNumber(blob.length).padStart(10)}`);
+        size += blob.length;
+    }
+    console.log("-".repeat(nameLength + 26));
+    console.log(`${"Total snapshot size".padEnd(nameLength)} |        | ${fluidAnalyzeMessages_1.formatNumber(size).padStart(10)}`);
+}
+async function dumpSnapshotTree(name, fetchedData) {
+    let size = 0;
+    let sizeNew = 0;
+    let blobCountNew = 0;
+    const sorted = getDumpFetchedData(fetchedData);
+    for (const item of sorted) {
+        const buffer = await item.blob;
+        if (buffer === undefined) {
+            continue;
+        }
+        const blob = common_utils_1.bufferToString(buffer, "utf8");
+        if (!item.reused) {
+            sizeNew += blob.length;
+            blobCountNew++;
+        }
+        size += blob.length;
+    }
+    return { blobCountNew, blobCount: sorted.length, size, sizeNew };
+}
+async function saveSnapshot(name, fetchedData, saveDir) {
+    const outDir = `${saveDir}/${name}/`;
+    const mkdir = util_1.default.promisify(fs_1.default.mkdir);
+    await mkdir(`${outDir}/decoded`, { recursive: true });
+    await Promise.all(fetchedData.map(async (item) => {
+        const buffer = await item.blob;
+        if (buffer === undefined) {
+            console.error(`ERROR: Unable to get data for blob ${item.blobId}`);
+            return;
+        }
+        if (!isFetchedTree(item)) {
+            // Just write the data as is.
+            fs_1.default.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));
+            // we assume that the buffer is utf8 here, which currently is true for
+            // all of our snapshot blobs.  It doesn't necessary be true in the future
+            let decoded = common_utils_1.bufferToString(buffer, "utf8");
+            try {
+                if (!fluidFetchArgs_1.paramActualFormatting) {
+                    decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);
+                }
+            }
+            catch (e) {
+            }
+            fs_1.default.writeFileSync(`${outDir}/decoded/${item.filename}.json`, decoded);
+        }
+        else {
+            // Write out same data for tree decoded or not, except for formatting
+            const treeString = common_utils_1.bufferToString(buffer, "utf8");
+            fs_1.default.writeFileSync(`${outDir}/${item.filename}.json`, treeString);
+            fs_1.default.writeFileSync(`${outDir}/decoded/${item.filename}.json`, fluidFetchArgs_1.paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));
+        }
+    }));
+}
+async function fetchBlobsFromVersion(storage, version) {
+    const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));
+    if (!tree) {
+        return Promise.reject(new Error("Failed to load snapshot tree"));
+    }
+    return fetchBlobsFromSnapshotTree(storage, tree);
+}
+async function reportErrors(message, res) {
+    try {
+        return await res;
+    }
+    catch (error) {
+        console.error(`Error calling ${message}`);
+        throw error;
+    }
+}
+async function fluidFetchSnapshot(documentService, saveDir) {
+    if (!fluidFetchArgs_1.dumpSnapshotStats && !fluidFetchArgs_1.dumpSnapshotTrees && !fluidFetchArgs_1.dumpSnapshotVersions && saveDir === undefined) {
+        return;
+    }
+    // --local mode - do not connect to storage.
+    // For now, bail out early.
+    // In future, separate download from analyzes parts and allow offline analyzes
+    if (!documentService) {
+        return;
+    }
+    console.log("\n");
+    let storage = await documentService.connectToStorage();
+    if (fluidFetchArgs_1.paramUnpackAggregatedBlobs) {
+        storage = driver_utils_1.BlobAggregationStorage.wrap(storage, new common_utils_1.TelemetryNullLogger(), false /* allowPacking */);
+    }
+    let version;
+    const versions = await reportErrors(`getVersions ${fluidFetchInit_1.latestVersionsId}`, storage.getVersions(fluidFetchInit_1.latestVersionsId, fluidFetchArgs_1.paramNumSnapshotVersions));
+    if (fluidFetchArgs_1.dumpSnapshotVersions) {
+        console.log("Snapshot versions");
+        console.log(versions);
+    }
+    let blobsToDump;
+    if (fluidFetchArgs_1.paramSnapshotVersionIndex !== undefined) {
+        version = versions[fluidFetchArgs_1.paramSnapshotVersionIndex];
+        if (version === undefined) {
+            console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);
+            return;
+        }
+        if (saveDir !== undefined) {
+            blobsToDump = await fetchBlobsFromVersion(storage, version);
+            const name = version.id;
+            console.log(`Saving snapshot ${name}`);
+            await saveSnapshot(name, blobsToDump, saveDir);
+        }
+    }
+    else {
+        version = versions[0];
+        if (saveDir !== undefined && versions.length > 0) {
+            console.log("  Name          |                    Date |       Size |   New Size |  Blobs | New Blobs");
+            console.log("-".repeat(86));
+            // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots
+            for (let i = versions.length - 1; i >= 0; i--) {
+                const v = versions[i];
+                const blobs = await fetchBlobsFromVersion(storage, v);
+                blobsToDump = blobs;
+                const name = `${i}-${v.id}`;
+                const res = await dumpSnapshotTree(name, blobs);
+                let date = "";
+                if (v.date) {
+                    try {
+                        date = new Date(v.date).toLocaleString();
+                    }
+                    catch (e) {
+                        date = v.date.replace("T", " ");
+                        const index = date.lastIndexOf(".");
+                        if (index > 0) {
+                            date = `${date.substr(0, index)} Z`;
+                        }
+                    }
+                }
+                date = date.padStart(23);
+                const size = fluidAnalyzeMessages_1.formatNumber(res.size).padStart(10);
+                const sizeNew = fluidAnalyzeMessages_1.formatNumber(res.sizeNew).padStart(10);
+                const blobCount = fluidAnalyzeMessages_1.formatNumber(res.blobCount).padStart(6);
+                const blobCountNew = fluidAnalyzeMessages_1.formatNumber(res.blobCountNew).padStart(9);
+                console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);
+                await saveSnapshot(name, blobs, saveDir);
+            }
+        }
+    }
+    if (fluidFetchArgs_1.dumpSnapshotStats || fluidFetchArgs_1.dumpSnapshotTrees) {
+        if (version === undefined) {
+            console.log("No snapshot tree");
+        }
+        else {
+            if (blobsToDump === undefined) {
+                blobsToDump = await fetchBlobsFromVersion(storage, version);
+            }
+            console.log(`\n\nSnapshot version ${version.id}`);
+            await dumpSnapshotTreeVerbose(version.id, blobsToDump);
+        }
+    }
+}
+exports.fluidFetchSnapshot = fluidFetchSnapshot;
+//# sourceMappingURL=fluidFetchSnapshot.js.map

package/dist/fluidFetchSnapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchSnapshot.js","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;AAEH,4CAAoB;AACpB,gDAAwB;AACxB,+DAA2G;AAS3G,+DAAsE;AACtE,iEAAsD;AACtD,qDAQ0B;AAC1B,qDAAoD;AA8BpD,SAAS,aAAa,CAAC,WAAyB;IAC5C,OAAO,SAAS,IAAI,WAAW,CAAC;AACpC,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoC,CAAC;AAC9D,IAAI,iBAAiB,GAAG,IAAI,GAAG,EAAoC,CAAC;AACpE,IAAI,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;AAEnE,SAAS,UAAU,CAAC,MAAc,EAC9B,IAAmB,EACnB,OAAgC,EAChC,SAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QACxC,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,IAAI,EAAE;YACjB,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE;gBACP,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,IAAI,IAAI,KAAK,SAAS,EAAE;oBACpB,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAChC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC/B;aACJ;YACD,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEnC,4DAA4D;YAC5D,gEAAgE;YAChE,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE;gBACR,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;aAChC;YACD,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE1D,mEAAmE;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;SAC/B;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,MAAc,EAAE,OAAgB;;IACzE,MAAM,EAAE,SAAG,IAAI,CAAC,EAAE,mCAAI,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,6BAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;IACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7F,CAAC;AAED,KAAK,UAAU,0BAA0B,CACrC,OAAgC,EAChC,IAAmB,EACnB,SAAiB,GAAG,EACpB,kBAAwC;IACxC,qBAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,EAC7D,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,CAAC,kBAAkB,CAAC;IACnC,IAAI,MAAM,IAAI,kCAAiB,EAAE;QAC7B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;KACrB;IAED,IAAI,MAAM,KAAK,GAAG,EAAE;QAChB,iBAAiB,GAAG,gBAAgB,CAAC;QACrC,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;KAClE;IAED,oEAAoE;IACpE,IAAI,UAAoC,CAAC;IACzC,IAAI,MAAM,EAAE;QACR,UAAU,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;KACpD;IAED,MAAM,SAAS,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAI,GAAG,EAAkB,CAAC;IAClE,IAAI,MAAM,GAAmB,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE1E,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QAC/C,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,OAAO,CAAC,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;YAChE,SAAS;SACZ;QACD,MAAM,qBAAqB,GAAG,MAAM,YAAY,CAC5C,mBAAmB,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAC5C,OAAO,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,qBAAqB,KAAK,IAAI,EAAE;YAChC,mCAAmC;YACnC,OAAO,CAAC,KAAK,CAAC,uCAAuC,SAAS,YAAY,MAAM,eAAe,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1H,SAAS;SACZ;QACD,qBAAM,CAAC,qBAAqB,CAAC,EAAE,KAAK,SAAS,IAAI,qBAAqB,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EACjG,KAAK,CAAC,2DAA2D,CAAC,CAAC;QACvE,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EACtD,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAClE,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,qBAAqB,EACrB,GAAG,MAAM,IAAI,SAAS,IAAI,CAAC,CAAC;QAChC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,qBAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC/F,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,OAAO,EACP,GAAG,MAAM,GAAG,SAAS,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,IAAI,UAAU,EAAE;QACZ,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,WAA2B;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,WAA2B;IAC5E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;KAC3D;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;QAC3C,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,mCAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACjI,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,eAAe,mCAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAA2B;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACd,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;YACvB,YAAY,EAAE,CAAC;SAClB;QACD,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,WAA2B,EAAE,OAAe;IAClF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,KAAK,CAAC,GAAG,MAAM,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO;SACV;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;YACtB,6BAA6B;YAC7B,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,OAAO,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YAC5C,IAAI;gBACA,IAAI,CAAC,sCAAqB,EAAE;oBACxB,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;iBAC/D;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,YAAE,CAAC,aAAa,CACZ,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EAAE,OAAO,CAAC,CAAC;SAC3D;aAAM;YACH,qEAAqE;YACrE,MAAM,UAAU,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YACjD,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,OAAO,EAAE,UAAU,CAAC,CAAC;YAChE,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EACtD,sCAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;SAClG;IACL,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAgC,EAAE,OAAiB;IACpF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,IAAI,CAAC,IAAI,EAAE;QACP,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;KACpE;IACD,OAAO,0BAA0B,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,YAAY,CAAI,OAAe,EAAE,GAAe;IAC3D,IAAI;QACA,OAAO,MAAM,GAAG,CAAC;KACpB;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;KACf;AACL,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACpC,eAAkC,EAClC,OAAgB;IAEhB,IAAI,CAAC,kCAAiB,IAAI,CAAC,kCAAiB,IAAI,CAAC,qCAAoB,IAAI,OAAO,KAAK,SAAS,EAAE;QAC5F,OAAO;KACV;IAED,4CAA4C;IAC5C,2BAA2B;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,eAAe,EAAE;QAClB,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;IACvD,IAAI,2CAA0B,EAAE;QAC5B,OAAO,GAAG,qCAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,kCAAmB,EAAE,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;KACvG;IAED,IAAI,OAA6B,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAC/B,eAAe,iCAAgB,EAAE,EACjC,OAAO,CAAC,WAAW,CAAC,iCAAgB,EAAE,yCAAwB,CAAC,CAAC,CAAC;IACrE,IAAI,qCAAoB,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACzB;IAED,IAAI,WAAuC,CAAC;IAC5C,IAAI,0CAAyB,KAAK,SAAS,EAAE;QACzC,OAAO,GAAG,QAAQ,CAAC,0CAAyB,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,MAAM,iDAAiD,CAAC,CAAC;YAChG,OAAO;SACV;QACD,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;SAClD;KACJ;SAAM;QACH,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,0FAA0F,CAAC,CAAC;YACxG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,2FAA2F;YAC3F,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC3C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtD,WAAW,GAAG,KAAK,CAAC;gBACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAEhD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,CAAC,IAAI,EAAE;oBACR,IAAI;wBACA,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;qBAC5C;oBAAC,OAAO,CAAC,EAAE;wBACR,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBACpC,IAAI,KAAK,GAAG,CAAC,EAAE;4BACX,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;yBACvC;qBACJ;iBACJ;gBACD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,mCAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,mCAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,mCAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,mCAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,IAAI,MAAM,OAAO,MAAM,SAAS,MAAM,YAAY,EAAE,CAAC,CAAC;gBAEpG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;aAC5C;SACJ;KACJ;IAED,IAAI,kCAAiB,IAAI,kCAAiB,EAAE;QACxC,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SACnC;aAAM;YACH,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC3B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aAC/D;YACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,uBAAuB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;SAC1D;KACJ;AACL,CAAC;AA9FD,gDA8FC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport util from \"util\";\nimport { assert, bufferToString, stringToBuffer, TelemetryNullLogger } from \"@fluidframework/common-utils\";\nimport {\n    IDocumentService,\n    IDocumentStorageService,\n} from \"@fluidframework/driver-definitions\";\nimport {\n    ISnapshotTree,\n    IVersion,\n} from \"@fluidframework/protocol-definitions\";\nimport { BlobAggregationStorage } from \"@fluidframework/driver-utils\";\nimport { formatNumber } from \"./fluidAnalyzeMessages\";\nimport {\n    dumpSnapshotStats,\n    dumpSnapshotTrees,\n    dumpSnapshotVersions,\n    paramActualFormatting,\n    paramNumSnapshotVersions,\n    paramSnapshotVersionIndex,\n    paramUnpackAggregatedBlobs,\n} from \"./fluidFetchArgs\";\nimport { latestVersionsId } from \"./fluidFetchInit\";\n\ninterface ISnapshotInfo {\n    blobCountNew: number;\n    blobCount: number;\n    size: number;\n    sizeNew: number;\n}\n\ntype IFetchedData = IFetchedBlob | IFetchedTree;\n\ninterface IFetchedBlob {\n    treePath: string;\n    filename: string;\n    blobId: string;\n    blob: Promise<ArrayBufferLike | undefined>;\n    reused: boolean;\n}\n\ninterface IFetchedTree {\n    treePath: string;\n    blobId: string;\n    filename: string;\n    blob: ArrayBufferLike;\n\n    reused: false;\n\n    patched: boolean;\n}\n\nfunction isFetchedTree(fetchedData: IFetchedData): fetchedData is IFetchedTree {\n    return \"patched\" in fetchedData;\n}\n\nconst blobCache = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCachePrevious = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\nfunction fetchBlobs(prefix: string,\n    tree: ISnapshotTree,\n    storage: IDocumentStorageService,\n    blobIdMap: Map<string, number>,\n) {\n    const result: IFetchedBlob[] = [];\n    for (const item of Object.keys(tree.blobs)) {\n        const treePath = `${prefix}${item}`;\n        const blobId = tree.blobs[item];\n        if (blobId !== null) {\n            let reused = true;\n            let blob = blobCachePrevious.get(blobId);\n            if (!blob) {\n                reused = false;\n                blob = blobCache.get(blobId);\n                if (blob === undefined) {\n                    blob = storage.readBlob(blobId);\n                    blobCache.set(blobId, blob);\n                }\n            }\n            blobCacheCurrent.set(blobId, blob);\n\n            // Use the blobIdMap to assign a number for each unique blob\n            // and use it as a prefix for files to avoid case-insensitive fs\n            let index = blobIdMap.get(blobId);\n            if (!index) {\n                index = blobIdMap.size;\n                blobIdMap.set(blobId, index);\n            }\n            const filename = `${index}-${blobId}`;\n            result.push({ treePath, blobId, blob, reused, filename });\n\n            // patch the tree so that we can write it out to reference the file\n            tree.blobs[item] = filename;\n        }\n    }\n    return result;\n}\n\nfunction createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {\n    const id = tree.id ?? \"original\";\n    const blob = stringToBuffer(JSON.stringify(tree),\"utf8\");\n    const filename = patched ? \"tree\" : `tree-${id}`;\n    const treePath = `${prefix}${filename}`;\n    return { treePath, blobId: \"original tree $id\", filename, blob, patched, reused: false };\n}\n\nasync function fetchBlobsFromSnapshotTree(\n    storage: IDocumentStorageService,\n    tree: ISnapshotTree,\n    prefix: string = \"/\",\n    perCommitBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {\n    assert(Object.keys(tree.commits).length === 0 || (prefix === \"/\"),\n        0x1be /* \"Unexpected tree input to fetch\" */);\n    const commit = !perCommitBlobIdMap;\n    if (commit && dumpSnapshotTrees) {\n        console.log(tree);\n    }\n\n    if (prefix === \"/\") {\n        blobCachePrevious = blobCacheCurrent;\n        blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n    }\n\n    // Create the tree info before fetching blobs (which will modify it)\n    let commitBlob: IFetchedTree | undefined;\n    if (commit) {\n        commitBlob = createTreeBlob(tree, prefix, false);\n    }\n\n    const blobIdMap = perCommitBlobIdMap ?? new Map<string, number>();\n    let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);\n\n    for (const dataStore of Object.keys(tree.commits)) {\n        const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);\n        if (dataStoreVersions.length !== 1) {\n            console.error(`ERROR: Unable to get versions for ${dataStore}`);\n            continue;\n        }\n        const dataStoreSnapShotTree = await reportErrors(\n            `getSnapshotTree ${dataStoreVersions[0].id}`,\n            storage.getSnapshotTree(dataStoreVersions[0]));\n        if (dataStoreSnapShotTree === null) {\n            // eslint-disable-next-line max-len\n            console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);\n            continue;\n        }\n        assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore],\n            0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);\n        assert(tree.commits[dataStore] === dataStoreVersions[0].id,\n            0x1c0 /* \"Mismatch between commit id and fetched tree id\" */);\n        const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n            storage,\n            dataStoreSnapShotTree,\n            `${prefix}[${dataStore}]/`);\n        result = result.concat(dataStoreBlobs);\n    }\n\n    for (const subtreeId of Object.keys(tree.trees)) {\n        const subtree = tree.trees[subtreeId];\n        assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* \"Unexpected subtree properties\" */);\n        const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n            storage,\n            subtree,\n            `${prefix}${subtreeId}/`, blobIdMap);\n        result = result.concat(dataStoreBlobs);\n    }\n\n    if (commitBlob) {\n        result.push(commitBlob);\n        result.push(createTreeBlob(tree, prefix, true));\n    }\n    return result;\n}\n\nfunction getDumpFetchedData(fetchedData: IFetchedData[]) {\n    const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));\n    return sorted.filter((item) => !isFetchedTree(item) || !item.patched);\n}\n\nasync function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {\n    let size = 0;\n    const sorted = getDumpFetchedData(fetchedData);\n\n    let nameLength = 10;\n    for (const item of sorted) {\n        nameLength = Math.max(nameLength, item.treePath.length);\n    }\n\n    console.log(\"\");\n    console.log(`${\"Blob Path\".padEnd(nameLength)} | Reused |      Bytes`);\n    console.log(\"-\".repeat(nameLength + 26));\n    for (const item of sorted) {\n        const buffer = await item.blob;\n        if (buffer === undefined) {\n            continue;\n        }\n        const blob = bufferToString(buffer,\"utf8\");\n        // eslint-disable-next-line max-len\n        console.log(`${item.treePath.padEnd(nameLength)} |    ${item.reused ? \"X\" : \" \"}   | ${formatNumber(blob.length).padStart(10)}`);\n        size += blob.length;\n    }\n\n    console.log(\"-\".repeat(nameLength + 26));\n    console.log(`${\"Total snapshot size\".padEnd(nameLength)} |        | ${formatNumber(size).padStart(10)}`);\n}\n\nasync function dumpSnapshotTree(name: string, fetchedData: IFetchedData[]): Promise<ISnapshotInfo> {\n    let size = 0;\n    let sizeNew = 0;\n    let blobCountNew = 0;\n    const sorted = getDumpFetchedData(fetchedData);\n\n    for (const item of sorted) {\n        const buffer = await item.blob;\n        if (buffer === undefined) {\n            continue;\n        }\n        const blob = bufferToString(buffer, \"utf8\");\n        if (!item.reused) {\n            sizeNew += blob.length;\n            blobCountNew++;\n        }\n        size += blob.length;\n    }\n\n    return { blobCountNew, blobCount: sorted.length, size, sizeNew };\n}\n\nasync function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {\n    const outDir = `${saveDir}/${name}/`;\n    const mkdir = util.promisify(fs.mkdir);\n\n    await mkdir(`${outDir}/decoded`, { recursive: true });\n    await Promise.all(fetchedData.map(async (item) => {\n        const buffer = await item.blob;\n        if (buffer === undefined) {\n            console.error(`ERROR: Unable to get data for blob ${item.blobId}`);\n            return;\n        }\n\n        if (!isFetchedTree(item)) {\n            // Just write the data as is.\n            fs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));\n\n            // we assume that the buffer is utf8 here, which currently is true for\n            // all of our snapshot blobs.  It doesn't necessary be true in the future\n            let decoded = bufferToString(buffer,\"utf8\");\n            try {\n                if (!paramActualFormatting) {\n                    decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);\n                }\n            } catch (e) {\n            }\n            fs.writeFileSync(\n                `${outDir}/decoded/${item.filename}.json`, decoded);\n        } else {\n            // Write out same data for tree decoded or not, except for formatting\n            const treeString = bufferToString(buffer,\"utf8\");\n            fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);\n            fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`,\n                paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));\n        }\n    }));\n}\n\nasync function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {\n    const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));\n    if (!tree) {\n        return Promise.reject(new Error(\"Failed to load snapshot tree\"));\n    }\n    return fetchBlobsFromSnapshotTree(storage, tree);\n}\n\nasync function reportErrors<T>(message: string, res: Promise<T>) {\n    try {\n        return await res;\n    } catch (error) {\n        console.error(`Error calling ${message}`);\n        throw error;\n    }\n}\n\nexport async function fluidFetchSnapshot(\n    documentService?: IDocumentService,\n    saveDir?: string,\n    ) {\n    if (!dumpSnapshotStats && !dumpSnapshotTrees && !dumpSnapshotVersions && saveDir === undefined) {\n        return;\n    }\n\n    // --local mode - do not connect to storage.\n    // For now, bail out early.\n    // In future, separate download from analyzes parts and allow offline analyzes\n    if (!documentService) {\n        return;\n    }\n\n    console.log(\"\\n\");\n\n    let storage = await documentService.connectToStorage();\n    if (paramUnpackAggregatedBlobs) {\n        storage = BlobAggregationStorage.wrap(storage, new TelemetryNullLogger(), false /* allowPacking */);\n    }\n\n    let version: IVersion | undefined;\n    const versions = await reportErrors(\n        `getVersions ${latestVersionsId}`,\n        storage.getVersions(latestVersionsId, paramNumSnapshotVersions));\n    if (dumpSnapshotVersions) {\n        console.log(\"Snapshot versions\");\n        console.log(versions);\n    }\n\n    let blobsToDump: IFetchedData[] | undefined;\n    if (paramSnapshotVersionIndex !== undefined) {\n        version = versions[paramSnapshotVersionIndex];\n        if (version === undefined) {\n            console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);\n            return;\n        }\n        if (saveDir !== undefined) {\n            blobsToDump = await fetchBlobsFromVersion(storage, version);\n            const name = version.id;\n            console.log(`Saving snapshot ${name}`);\n            await saveSnapshot(name, blobsToDump, saveDir);\n        }\n    } else {\n        version = versions[0];\n        if (saveDir !== undefined && versions.length > 0) {\n            console.log(\"  Name          |                    Date |       Size |   New Size |  Blobs | New Blobs\");\n            console.log(\"-\".repeat(86));\n\n            // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots\n            for (let i = versions.length - 1; i >= 0; i--) {\n                const v = versions[i];\n                const blobs = await fetchBlobsFromVersion(storage, v);\n                blobsToDump = blobs;\n                const name = `${i}-${v.id}`;\n                const res = await dumpSnapshotTree(name, blobs);\n\n                let date = \"\";\n                if (v.date) {\n                    try {\n                        date = new Date(v.date).toLocaleString();\n                    } catch (e) {\n                        date = v.date.replace(\"T\", \" \");\n                        const index = date.lastIndexOf(\".\");\n                        if (index > 0) {\n                            date = `${date.substr(0, index)} Z`;\n                        }\n                    }\n                }\n                date = date.padStart(23);\n                const size = formatNumber(res.size).padStart(10);\n                const sizeNew = formatNumber(res.sizeNew).padStart(10);\n                const blobCount = formatNumber(res.blobCount).padStart(6);\n                const blobCountNew = formatNumber(res.blobCountNew).padStart(9);\n\n                console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);\n\n                await saveSnapshot(name, blobs, saveDir);\n            }\n        }\n    }\n\n    if (dumpSnapshotStats || dumpSnapshotTrees) {\n        if (version === undefined) {\n            console.log(\"No snapshot tree\");\n        } else {\n            if (blobsToDump === undefined) {\n                blobsToDump = await fetchBlobsFromVersion(storage, version);\n            }\n            console.log(`\\n\\nSnapshot version ${version.id}`);\n            await dumpSnapshotTreeVerbose(version.id, blobsToDump);\n        }\n    }\n}\n"]}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@fluid-tools/fetch-tool",
+  "version": "0.53.0-46105",
+  "description": "Console tool to fetch Fluid data from relay service",
+  "homepage": "https://fluidframework.com",
+  "repository": "https://github.com/microsoft/FluidFramework",
+  "license": "MIT",
+  "author": "Microsoft and contributors",
+  "main": "dist/fluidFetch.js",
+  "bin": {
+    "fluid-fetch": "bin/fluid-fetch"
+  },
+  "scripts": {
+    "build": "concurrently npm:build:compile npm:lint",
+    "build:compile": "npm run tsc",
+    "build:full": "npm run build",
+    "build:full:compile": "npm run build:compile",
+    "clean": "rimraf dist *.tsbuildinfo *.build.log",
+    "eslint": "eslint --format stylish src",
+    "eslint:fix": "eslint --format stylish src --fix",
+    "lint": "npm run eslint",
+    "lint:fix": "npm run eslint:fix",
+    "tsc": "tsc",
+    "tsfmt": "tsfmt --verify",
+    "tsfmt:fix": "tsfmt --replace"
+  },
+  "dependencies": {
+    "@fluid-tools/fluidapp-odsp-urlresolver": "0.53.0-46105",
+    "@fluidframework/azure-client": "0.53.0-46105",
+    "@fluidframework/common-utils": "^0.32.1",
+    "@fluidframework/container-runtime": "0.53.0-46105",
+    "@fluidframework/datastore": "0.53.0-46105",
+    "@fluidframework/driver-definitions": "^0.43.0-0",
+    "@fluidframework/driver-utils": "0.53.0-46105",
+    "@fluidframework/odsp-doclib-utils": "0.53.0-46105",
+    "@fluidframework/odsp-driver": "0.53.0-46105",
+    "@fluidframework/odsp-driver-definitions": "0.53.0-46105",
+    "@fluidframework/odsp-urlresolver": "0.53.0-46105",
+    "@fluidframework/protocol-definitions": "^0.1026.0",
+    "@fluidframework/routerlicious-driver": "0.53.0-46105",
+    "@fluidframework/routerlicious-urlresolver": "0.53.0-46105",
+    "@fluidframework/runtime-definitions": "0.53.0-46105",
+    "@fluidframework/telemetry-utils": "0.53.0-46105",
+    "@fluidframework/test-client-utils": "0.53.0-46105",
+    "@fluidframework/tool-utils": "0.53.0-46105"
+  },
+  "devDependencies": {
+    "@fluidframework/build-common": "^0.23.0",
+    "@fluidframework/eslint-config-fluid": "^0.24.0",
+    "@types/node": "^12.19.0",
+    "@typescript-eslint/eslint-plugin": "~4.14.0",
+    "@typescript-eslint/parser": "~4.14.0",
+    "concurrently": "^6.2.0",
+    "eslint": "~7.18.0",
+    "eslint-plugin-eslint-comments": "~3.2.0",
+    "eslint-plugin-import": "~2.22.1",
+    "eslint-plugin-no-null": "~1.0.2",
+    "eslint-plugin-prefer-arrow": "~1.2.2",
+    "eslint-plugin-react": "~7.22.0",
+    "eslint-plugin-unicorn": "~26.0.1",
+    "rimraf": "^2.6.2",
+    "typescript": "~4.1.3",
+    "typescript-formatter": "7.1.0"
+  }
+}