@fluid-tools/fetch-tool 2.0.0-dev-rc.2.0.0.246488 → 2.0.0-dev-rc.3.0.0.253463

package/lib/fluidFetchSnapshot.js

@@ -0,0 +1,261 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import fs from "fs";
+import util from "util";
+import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
+import { formatNumber } from "./fluidAnalyzeMessages.js";
+import { dumpSnapshotStats, dumpSnapshotTrees, dumpSnapshotVersions, paramActualFormatting, paramNumSnapshotVersions, paramSnapshotVersionIndex, } from "./fluidFetchArgs.js";
+import { latestVersionsId } from "./fluidFetchInit.js";
+function isFetchedTree(fetchedData) {
+    return "patched" in fetchedData;
+}
+const blobCache = new Map();
+let blobCachePrevious = new Map();
+let blobCacheCurrent = new Map();
+function fetchBlobs(prefix, tree, storage, blobIdMap) {
+    const result = [];
+    for (const item of Object.keys(tree.blobs)) {
+        const treePath = `${prefix}${item}`;
+        const blobId = tree.blobs[item];
+        if (blobId !== null) {
+            let reused = true;
+            let blob = blobCachePrevious.get(blobId);
+            if (!blob) {
+                reused = false;
+                blob = blobCache.get(blobId);
+                if (blob === undefined) {
+                    blob = storage.readBlob(blobId);
+                    blobCache.set(blobId, blob);
+                }
+            }
+            blobCacheCurrent.set(blobId, blob);
+            // Use the blobIdMap to assign a number for each unique blob
+            // and use it as a prefix for files to avoid case-insensitive fs
+            let index = blobIdMap.get(blobId);
+            if (index === undefined) {
+                index = blobIdMap.size;
+                blobIdMap.set(blobId, index);
+            }
+            const filename = `${index}-${blobId}`;
+            result.push({ treePath, blobId, blob, reused, filename });
+            // patch the tree so that we can write it out to reference the file
+            tree.blobs[item] = filename;
+        }
+    }
+    return result;
+}
+function createTreeBlob(tree, prefix, patched) {
+    const id = tree.id ?? "original";
+    const blob = stringToBuffer(JSON.stringify(tree), "utf8");
+    const filename = patched ? "tree" : `tree-${id}`;
+    const treePath = `${prefix}${filename}`;
+    return { treePath, blobId: "original tree $id", filename, blob, patched, reused: false };
+}
+async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/", parentBlobIdMap) {
+    const isTopLevel = !parentBlobIdMap;
+    if (isTopLevel && dumpSnapshotTrees) {
+        console.log(tree);
+    }
+    if (prefix === "/") {
+        blobCachePrevious = blobCacheCurrent;
+        blobCacheCurrent = new Map();
+    }
+    // Create the tree info before fetching blobs (which will modify it)
+    let topLevelBlob;
+    if (isTopLevel) {
+        topLevelBlob = createTreeBlob(tree, prefix, false);
+    }
+    const blobIdMap = parentBlobIdMap ?? new Map();
+    let result = fetchBlobs(prefix, tree, storage, blobIdMap);
+    for (const subtreeId of Object.keys(tree.trees)) {
+        const subtree = tree.trees[subtreeId];
+        const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, subtree, `${prefix}${subtreeId}/`, blobIdMap);
+        result = result.concat(dataStoreBlobs);
+    }
+    if (topLevelBlob) {
+        result.push(topLevelBlob);
+        result.push(createTreeBlob(tree, prefix, true));
+    }
+    return result;
+}
+function getDumpFetchedData(fetchedData) {
+    const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));
+    return sorted.filter((item) => !isFetchedTree(item) || !item.patched);
+}
+async function dumpSnapshotTreeVerbose(name, fetchedData) {
+    let size = 0;
+    const sorted = getDumpFetchedData(fetchedData);
+    let nameLength = 10;
+    for (const item of sorted) {
+        nameLength = Math.max(nameLength, item.treePath.length);
+    }
+    console.log("");
+    console.log(`${"Blob Path".padEnd(nameLength)} | Reused |      Bytes`);
+    console.log("-".repeat(nameLength + 26));
+    for (const item of sorted) {
+        const buffer = await item.blob;
+        if (buffer === undefined) {
+            continue;
+        }
+        const blob = bufferToString(buffer, "utf8");
+        console.log(`${item.treePath.padEnd(nameLength)} |    ${item.reused ? "X" : " "}   | ${formatNumber(blob.length).padStart(10)}`);
+        size += blob.length;
+    }
+    console.log("-".repeat(nameLength + 26));
+    console.log(`${"Total snapshot size".padEnd(nameLength)} |        | ${formatNumber(size).padStart(10)}`);
+}
+async function dumpSnapshotTree(name, fetchedData) {
+    let size = 0;
+    let sizeNew = 0;
+    let blobCountNew = 0;
+    const sorted = getDumpFetchedData(fetchedData);
+    for (const item of sorted) {
+        const buffer = await item.blob;
+        if (buffer === undefined) {
+            continue;
+        }
+        const blob = bufferToString(buffer, "utf8");
+        if (!item.reused) {
+            sizeNew += blob.length;
+            blobCountNew++;
+        }
+        size += blob.length;
+    }
+    return { blobCountNew, blobCount: sorted.length, size, sizeNew };
+}
+async function saveSnapshot(name, fetchedData, saveDir) {
+    const outDir = `${saveDir}/${name}/`;
+    const mkdir = util.promisify(fs.mkdir);
+    await mkdir(`${outDir}/decoded`, { recursive: true });
+    await Promise.all(fetchedData.map(async (item) => {
+        const buffer = await item.blob;
+        if (buffer === undefined) {
+            console.error(`ERROR: Unable to get data for blob ${item.blobId}`);
+            return;
+        }
+        if (!isFetchedTree(item)) {
+            // Just write the data as is.
+            fs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));
+            // we assume that the buffer is utf8 here, which currently is true for
+            // all of our snapshot blobs.  It doesn't necessary be true in the future
+            let decoded = bufferToString(buffer, "utf8");
+            try {
+                if (!paramActualFormatting) {
+                    decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);
+                }
+            }
+            catch (e) { }
+            fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`, decoded);
+        }
+        else {
+            // Write out same data for tree decoded or not, except for formatting
+            const treeString = bufferToString(buffer, "utf8");
+            fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);
+            fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`, paramActualFormatting
+                ? treeString
+                : JSON.stringify(JSON.parse(treeString), undefined, 2));
+        }
+    }));
+}
+async function fetchBlobsFromVersion(storage, version) {
+    const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));
+    if (!tree) {
+        throw new Error("Failed to load snapshot tree");
+    }
+    return fetchBlobsFromSnapshotTree(storage, tree);
+}
+async function reportErrors(message, res) {
+    try {
+        return await res;
+    }
+    catch (error) {
+        console.error(`Error calling ${message}`);
+        throw error;
+    }
+}
+export async function fluidFetchSnapshot(documentService, saveDir) {
+    if (!dumpSnapshotStats &&
+        !dumpSnapshotTrees &&
+        !dumpSnapshotVersions &&
+        saveDir === undefined) {
+        return;
+    }
+    // --local mode - do not connect to storage.
+    // For now, bail out early.
+    // In future, separate download from analyzes parts and allow offline analyzes
+    if (!documentService) {
+        return;
+    }
+    console.log("\n");
+    const storage = await documentService.connectToStorage();
+    let version;
+    const versions = await reportErrors(`getVersions ${latestVersionsId}`, storage.getVersions(latestVersionsId, paramNumSnapshotVersions));
+    if (dumpSnapshotVersions) {
+        console.log("Snapshot versions");
+        console.log(versions);
+    }
+    let blobsToDump;
+    if (paramSnapshotVersionIndex !== undefined) {
+        version = versions[paramSnapshotVersionIndex];
+        if (version === undefined) {
+            console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);
+            return;
+        }
+        if (saveDir !== undefined) {
+            blobsToDump = await fetchBlobsFromVersion(storage, version);
+            const name = version.id;
+            console.log(`Saving snapshot ${name}`);
+            await saveSnapshot(name, blobsToDump, saveDir);
+        }
+    }
+    else {
+        version = versions[0];
+        if (saveDir !== undefined && versions.length > 0) {
+            console.log("  Name          |                    Date |       Size |   New Size |  Blobs | New Blobs");
+            console.log("-".repeat(86));
+            // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots
+            for (let i = versions.length - 1; i >= 0; i--) {
+                const v = versions[i];
+                const blobs = await fetchBlobsFromVersion(storage, v);
+                blobsToDump = blobs;
+                const name = `${i}-${v.id}`;
+                const res = await dumpSnapshotTree(name, blobs);
+                let date = "";
+                if (v.date !== undefined) {
+                    try {
+                        date = new Date(v.date).toLocaleString();
+                    }
+                    catch (e) {
+                        date = v.date.replace("T", " ");
+                        const index = date.lastIndexOf(".");
+                        if (index > 0) {
+                            date = `${date.substr(0, index)} Z`;
+                        }
+                    }
+                }
+                date = date.padStart(23);
+                const size = formatNumber(res.size).padStart(10);
+                const sizeNew = formatNumber(res.sizeNew).padStart(10);
+                const blobCount = formatNumber(res.blobCount).padStart(6);
+                const blobCountNew = formatNumber(res.blobCountNew).padStart(9);
+                console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);
+                await saveSnapshot(name, blobs, saveDir);
+            }
+        }
+    }
+    if (dumpSnapshotStats || dumpSnapshotTrees) {
+        if (version === undefined) {
+            console.log("No snapshot tree");
+        }
+        else {
+            if (blobsToDump === undefined) {
+                blobsToDump = await fetchBlobsFromVersion(storage, version);
+            }
+            console.log(`\n\nSnapshot version ${version.id}`);
+            await dumpSnapshotTreeVerbose(version.id, blobsToDump);
+        }
+    }
+}
+//# sourceMappingURL=fluidFetchSnapshot.js.map

package/lib/fluidFetchSnapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchSnapshot.js","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAO9E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EACN,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,EACxB,yBAAyB,GACzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AA8BvD,SAAS,aAAa,CAAC,WAAyB;IAC/C,OAAO,SAAS,IAAI,WAAW,CAAC;AACjC,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoC,CAAC;AAC9D,IAAI,iBAAiB,GAAG,IAAI,GAAG,EAAoC,CAAC;AACpE,IAAI,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;AAEnE,SAAS,UAAU,CAClB,MAAc,EACd,IAAmB,EACnB,OAAgC,EAChC,SAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC3C,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,IAAI,EAAE;YACpB,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE;gBACV,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,IAAI,IAAI,KAAK,SAAS,EAAE;oBACvB,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAChC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC5B;aACD;YACD,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEnC,4DAA4D;YAC5D,gEAAgE;YAChE,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,KAAK,KAAK,SAAS,EAAE;gBACxB,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;aAC7B;YACD,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE1D,mEAAmE;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;SAC5B;KACD;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,MAAc,EAAE,OAAgB;IAC5E,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;IACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC1F,CAAC;AAED,KAAK,UAAU,0BAA0B,CACxC,OAAgC,EAChC,IAAmB,EACnB,SAAiB,GAAG,EACpB,eAAqC;IAErC,MAAM,UAAU,GAAG,CAAC,eAAe,CAAC;IACpC,IAAI,UAAU,IAAI,iBAAiB,EAAE;QACpC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;KAClB;IAED,IAAI,MAAM,KAAK,GAAG,EAAE;QACnB,iBAAiB,GAAG,gBAAgB,CAAC;QACrC,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;KAC/D;IAED,oEAAoE;IACpE,IAAI,YAAsC,CAAC;IAC3C,IAAI,UAAU,EAAE;QACf,YAAY,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;KACnD;IAED,MAAM,SAAS,GAAG,eAAe,IAAI,IAAI,GAAG,EAAkB,CAAC;IAC/D,IAAI,MAAM,GAAmB,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE1E,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACtD,OAAO,EACP,OAAO,EACP,GAAG,MAAM,GAAG,SAAS,GAAG,EACxB,SAAS,CACT,CAAC;QACF,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KACvC;IAED,IAAI,YAAY,EAAE;QACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;KAChD;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,WAA2B;IACtD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,WAA2B;IAC/E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QAC1B,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;KACxD;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACzB,SAAS;SACT;QACD,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CACV,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,YAAY,CACtF,IAAI,CAAC,MAAM,CACX,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAChB,CAAC;QACF,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACpB;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CACV,GAAG,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,eAAe,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAC3F,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAA2B;IACxE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACzB,SAAS;SACT;QACD,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACjB,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;YACvB,YAAY,EAAE,CAAC;SACf;QACD,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACpB;IAED,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAClE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,WAA2B,EAAE,OAAe;IACrF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,KAAK,CAAC,GAAG,MAAM,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,GAAG,CAChB,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACzB,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO;SACP;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;YACzB,6BAA6B;YAC7B,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC7C,IAAI;gBACH,IAAI,CAAC,qBAAqB,EAAE;oBAC3B,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;iBAC5D;aACD;YAAC,OAAO,CAAC,EAAE,GAAE;YACd,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EAAE,OAAO,CAAC,CAAC;SACrE;aAAM;YACN,qEAAqE;YACrE,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAClD,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,OAAO,EAAE,UAAU,CAAC,CAAC;YAChE,EAAE,CAAC,aAAa,CACf,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EACzC,qBAAqB;gBACpB,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CACvD,CAAC;SACF;IACF,CAAC,CAAC,CACF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAgC,EAAE,OAAiB;IACvF,MAAM,IAAI,GAAG,MAAM,YAAY,CAC9B,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAC/B,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAChC,CAAC;IACF,IAAI,CAAC,IAAI,EAAE;QACV,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;KAChD;IACD,OAAO,0BAA0B,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,YAAY,CAAI,OAAe,EAAE,GAAe;IAC9D,IAAI;QACH,OAAO,MAAM,GAAG,CAAC;KACjB;IAAC,OAAO,KAAK,EAAE;QACf,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;KACZ;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,eAAkC,EAAE,OAAgB;IAC5F,IACC,CAAC,iBAAiB;QAClB,CAAC,iBAAiB;QAClB,CAAC,oBAAoB;QACrB,OAAO,KAAK,SAAS,EACpB;QACD,OAAO;KACP;IAED,4CAA4C;IAC5C,2BAA2B;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,eAAe,EAAE;QACrB,OAAO;KACP;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;IAEzD,IAAI,OAA6B,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAClC,eAAe,gBAAgB,EAAE,EACjC,OAAO,CAAC,WAAW,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAC/D,CAAC;IACF,IAAI,oBAAoB,EAAE;QACzB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACtB;IAED,IAAI,WAAuC,CAAC;IAC5C,IAAI,yBAAyB,KAAK,SAAS,EAAE;QAC5C,OAAO,GAAG,QAAQ,CAAC,yBAAyB,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CACV,kBAAkB,QAAQ,CAAC,MAAM,iDAAiD,CAClF,CAAC;YACF,OAAO;SACP;QACD,IAAI,OAAO,KAAK,SAAS,EAAE;YAC1B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;SAC/C;KACD;SAAM;QACN,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YACjD,OAAO,CAAC,GAAG,CACV,0FAA0F,CAC1F,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,2FAA2F;YAC3F,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC9C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtD,WAAW,GAAG,KAAK,CAAC;gBACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAEhD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE;oBACzB,IAAI;wBACH,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;qBACzC;oBAAC,OAAO,CAAC,EAAE;wBACX,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBACpC,IAAI,KAAK,GAAG,CAAC,EAAE;4BACd,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;yBACpC;qBACD;iBACD;gBACD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAEhE,OAAO,CAAC,GAAG,CACV,GAAG,IAAI,CAAC,MAAM,CACb,EAAE,CACF,MAAM,IAAI,MAAM,IAAI,MAAM,OAAO,MAAM,SAAS,MAAM,YAAY,EAAE,CACrE,CAAC;gBAEF,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;aACzC;SACD;KACD;IAED,IAAI,iBAAiB,IAAI,iBAAiB,EAAE;QAC3C,IAAI,OAAO,KAAK,SAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SAChC;aAAM;YACN,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC9B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aAC5D;YACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,uBAAuB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;SACvD;KACD;AACF,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport util from \"util\";\n\nimport { bufferToString, stringToBuffer } from \"@fluid-internal/client-utils\";\nimport {\n\tIDocumentService,\n\tIDocumentStorageService,\n} from \"@fluidframework/driver-definitions/internal\";\nimport { ISnapshotTree, IVersion } from \"@fluidframework/protocol-definitions\";\n\nimport { formatNumber } from \"./fluidAnalyzeMessages.js\";\nimport {\n\tdumpSnapshotStats,\n\tdumpSnapshotTrees,\n\tdumpSnapshotVersions,\n\tparamActualFormatting,\n\tparamNumSnapshotVersions,\n\tparamSnapshotVersionIndex,\n} from \"./fluidFetchArgs.js\";\nimport { latestVersionsId } from \"./fluidFetchInit.js\";\n\ninterface ISnapshotInfo {\n\tblobCountNew: number;\n\tblobCount: number;\n\tsize: number;\n\tsizeNew: number;\n}\n\ntype IFetchedData = IFetchedBlob | IFetchedTree;\n\ninterface IFetchedBlob {\n\ttreePath: string;\n\tfilename: string;\n\tblobId: string;\n\tblob: Promise<ArrayBufferLike | undefined>;\n\treused: boolean;\n}\n\ninterface IFetchedTree {\n\ttreePath: string;\n\tblobId: string;\n\tfilename: string;\n\tblob: ArrayBufferLike;\n\n\treused: false;\n\n\tpatched: boolean;\n}\n\nfunction isFetchedTree(fetchedData: IFetchedData): fetchedData is IFetchedTree {\n\treturn \"patched\" in fetchedData;\n}\n\nconst blobCache = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCachePrevious = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\nfunction fetchBlobs(\n\tprefix: string,\n\ttree: ISnapshotTree,\n\tstorage: IDocumentStorageService,\n\tblobIdMap: Map<string, number>,\n) {\n\tconst result: IFetchedBlob[] = [];\n\tfor (const item of Object.keys(tree.blobs)) {\n\t\tconst treePath = `${prefix}${item}`;\n\t\tconst blobId = tree.blobs[item];\n\t\tif (blobId !== null) {\n\t\t\tlet reused = true;\n\t\t\tlet blob = blobCachePrevious.get(blobId);\n\t\t\tif (!blob) {\n\t\t\t\treused = false;\n\t\t\t\tblob = blobCache.get(blobId);\n\t\t\t\tif (blob === undefined) {\n\t\t\t\t\tblob = storage.readBlob(blobId);\n\t\t\t\t\tblobCache.set(blobId, blob);\n\t\t\t\t}\n\t\t\t}\n\t\t\tblobCacheCurrent.set(blobId, blob);\n\n\t\t\t// Use the blobIdMap to assign a number for each unique blob\n\t\t\t// and use it as a prefix for files to avoid case-insensitive fs\n\t\t\tlet index = blobIdMap.get(blobId);\n\t\t\tif (index === undefined) {\n\t\t\t\tindex = blobIdMap.size;\n\t\t\t\tblobIdMap.set(blobId, index);\n\t\t\t}\n\t\t\tconst filename = `${index}-${blobId}`;\n\t\t\tresult.push({ treePath, blobId, blob, reused, filename });\n\n\t\t\t// patch the tree so that we can write it out to reference the file\n\t\t\ttree.blobs[item] = filename;\n\t\t}\n\t}\n\treturn result;\n}\n\nfunction createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {\n\tconst id = tree.id ?? \"original\";\n\tconst blob = stringToBuffer(JSON.stringify(tree), \"utf8\");\n\tconst filename = patched ? \"tree\" : `tree-${id}`;\n\tconst treePath = `${prefix}${filename}`;\n\treturn { treePath, blobId: \"original tree $id\", filename, blob, patched, reused: false };\n}\n\nasync function fetchBlobsFromSnapshotTree(\n\tstorage: IDocumentStorageService,\n\ttree: ISnapshotTree,\n\tprefix: string = \"/\",\n\tparentBlobIdMap?: Map<string, number>,\n): Promise<IFetchedData[]> {\n\tconst isTopLevel = !parentBlobIdMap;\n\tif (isTopLevel && dumpSnapshotTrees) {\n\t\tconsole.log(tree);\n\t}\n\n\tif (prefix === \"/\") {\n\t\tblobCachePrevious = blobCacheCurrent;\n\t\tblobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\t}\n\n\t// Create the tree info before fetching blobs (which will modify it)\n\tlet topLevelBlob: IFetchedTree | undefined;\n\tif (isTopLevel) {\n\t\ttopLevelBlob = createTreeBlob(tree, prefix, false);\n\t}\n\n\tconst blobIdMap = parentBlobIdMap ?? new Map<string, number>();\n\tlet result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);\n\n\tfor (const subtreeId of Object.keys(tree.trees)) {\n\t\tconst subtree = tree.trees[subtreeId];\n\t\tconst dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n\t\t\tstorage,\n\t\t\tsubtree,\n\t\t\t`${prefix}${subtreeId}/`,\n\t\t\tblobIdMap,\n\t\t);\n\t\tresult = result.concat(dataStoreBlobs);\n\t}\n\n\tif (topLevelBlob) {\n\t\tresult.push(topLevelBlob);\n\t\tresult.push(createTreeBlob(tree, prefix, true));\n\t}\n\treturn result;\n}\n\nfunction getDumpFetchedData(fetchedData: IFetchedData[]) {\n\tconst sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));\n\treturn sorted.filter((item) => !isFetchedTree(item) || !item.patched);\n}\n\nasync function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {\n\tlet size = 0;\n\tconst sorted = getDumpFetchedData(fetchedData);\n\n\tlet nameLength = 10;\n\tfor (const item of sorted) {\n\t\tnameLength = Math.max(nameLength, item.treePath.length);\n\t}\n\n\tconsole.log(\"\");\n\tconsole.log(`${\"Blob Path\".padEnd(nameLength)} | Reused |      Bytes`);\n\tconsole.log(\"-\".repeat(nameLength + 26));\n\tfor (const item of sorted) {\n\t\tconst buffer = await item.blob;\n\t\tif (buffer === undefined) {\n\t\t\tcontinue;\n\t\t}\n\t\tconst blob = bufferToString(buffer, \"utf8\");\n\t\tconsole.log(\n\t\t\t`${item.treePath.padEnd(nameLength)} |    ${item.reused ? \"X\" : \" \"}   | ${formatNumber(\n\t\t\t\tblob.length,\n\t\t\t).padStart(10)}`,\n\t\t);\n\t\tsize += blob.length;\n\t}\n\n\tconsole.log(\"-\".repeat(nameLength + 26));\n\tconsole.log(\n\t\t`${\"Total snapshot size\".padEnd(nameLength)} |        | ${formatNumber(size).padStart(10)}`,\n\t);\n}\n\nasync function dumpSnapshotTree(name: string, fetchedData: IFetchedData[]): Promise<ISnapshotInfo> {\n\tlet size = 0;\n\tlet sizeNew = 0;\n\tlet blobCountNew = 0;\n\tconst sorted = getDumpFetchedData(fetchedData);\n\n\tfor (const item of sorted) {\n\t\tconst buffer = await item.blob;\n\t\tif (buffer === undefined) {\n\t\t\tcontinue;\n\t\t}\n\t\tconst blob = bufferToString(buffer, \"utf8\");\n\t\tif (!item.reused) {\n\t\t\tsizeNew += blob.length;\n\t\t\tblobCountNew++;\n\t\t}\n\t\tsize += blob.length;\n\t}\n\n\treturn { blobCountNew, blobCount: sorted.length, size, sizeNew };\n}\n\nasync function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {\n\tconst outDir = `${saveDir}/${name}/`;\n\tconst mkdir = util.promisify(fs.mkdir);\n\n\tawait mkdir(`${outDir}/decoded`, { recursive: true });\n\tawait Promise.all(\n\t\tfetchedData.map(async (item) => {\n\t\t\tconst buffer = await item.blob;\n\t\t\tif (buffer === undefined) {\n\t\t\t\tconsole.error(`ERROR: Unable to get data for blob ${item.blobId}`);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (!isFetchedTree(item)) {\n\t\t\t\t// Just write the data as is.\n\t\t\t\tfs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));\n\n\t\t\t\t// we assume that the buffer is utf8 here, which currently is true for\n\t\t\t\t// all of our snapshot blobs.  It doesn't necessary be true in the future\n\t\t\t\tlet decoded = bufferToString(buffer, \"utf8\");\n\t\t\t\ttry {\n\t\t\t\t\tif (!paramActualFormatting) {\n\t\t\t\t\t\tdecoded = JSON.stringify(JSON.parse(decoded), undefined, 2);\n\t\t\t\t\t}\n\t\t\t\t} catch (e) {}\n\t\t\t\tfs.writeFileSync(`${outDir}/decoded/${item.filename}.json`, decoded);\n\t\t\t} else {\n\t\t\t\t// Write out same data for tree decoded or not, except for formatting\n\t\t\t\tconst treeString = bufferToString(buffer, \"utf8\");\n\t\t\t\tfs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);\n\t\t\t\tfs.writeFileSync(\n\t\t\t\t\t`${outDir}/decoded/${item.filename}.json`,\n\t\t\t\t\tparamActualFormatting\n\t\t\t\t\t\t? treeString\n\t\t\t\t\t\t: JSON.stringify(JSON.parse(treeString), undefined, 2),\n\t\t\t\t);\n\t\t\t}\n\t\t}),\n\t);\n}\n\nasync function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {\n\tconst tree = await reportErrors(\n\t\t`getSnapshotTree ${version.id}`,\n\t\tstorage.getSnapshotTree(version),\n\t);\n\tif (!tree) {\n\t\tthrow new Error(\"Failed to load snapshot tree\");\n\t}\n\treturn fetchBlobsFromSnapshotTree(storage, tree);\n}\n\nasync function reportErrors<T>(message: string, res: Promise<T>) {\n\ttry {\n\t\treturn await res;\n\t} catch (error) {\n\t\tconsole.error(`Error calling ${message}`);\n\t\tthrow error;\n\t}\n}\n\nexport async function fluidFetchSnapshot(documentService?: IDocumentService, saveDir?: string) {\n\tif (\n\t\t!dumpSnapshotStats &&\n\t\t!dumpSnapshotTrees &&\n\t\t!dumpSnapshotVersions &&\n\t\tsaveDir === undefined\n\t) {\n\t\treturn;\n\t}\n\n\t// --local mode - do not connect to storage.\n\t// For now, bail out early.\n\t// In future, separate download from analyzes parts and allow offline analyzes\n\tif (!documentService) {\n\t\treturn;\n\t}\n\n\tconsole.log(\"\\n\");\n\n\tconst storage = await documentService.connectToStorage();\n\n\tlet version: IVersion | undefined;\n\tconst versions = await reportErrors(\n\t\t`getVersions ${latestVersionsId}`,\n\t\tstorage.getVersions(latestVersionsId, paramNumSnapshotVersions),\n\t);\n\tif (dumpSnapshotVersions) {\n\t\tconsole.log(\"Snapshot versions\");\n\t\tconsole.log(versions);\n\t}\n\n\tlet blobsToDump: IFetchedData[] | undefined;\n\tif (paramSnapshotVersionIndex !== undefined) {\n\t\tversion = versions[paramSnapshotVersionIndex];\n\t\tif (version === undefined) {\n\t\t\tconsole.log(\n\t\t\t\t`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`,\n\t\t\t);\n\t\t\treturn;\n\t\t}\n\t\tif (saveDir !== undefined) {\n\t\t\tblobsToDump = await fetchBlobsFromVersion(storage, version);\n\t\t\tconst name = version.id;\n\t\t\tconsole.log(`Saving snapshot ${name}`);\n\t\t\tawait saveSnapshot(name, blobsToDump, saveDir);\n\t\t}\n\t} else {\n\t\tversion = versions[0];\n\t\tif (saveDir !== undefined && versions.length > 0) {\n\t\t\tconsole.log(\n\t\t\t\t\"  Name          |                    Date |       Size |   New Size |  Blobs | New Blobs\",\n\t\t\t);\n\t\t\tconsole.log(\"-\".repeat(86));\n\n\t\t\t// Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots\n\t\t\tfor (let i = versions.length - 1; i >= 0; i--) {\n\t\t\t\tconst v = versions[i];\n\t\t\t\tconst blobs = await fetchBlobsFromVersion(storage, v);\n\t\t\t\tblobsToDump = blobs;\n\t\t\t\tconst name = `${i}-${v.id}`;\n\t\t\t\tconst res = await dumpSnapshotTree(name, blobs);\n\n\t\t\t\tlet date = \"\";\n\t\t\t\tif (v.date !== undefined) {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tdate = new Date(v.date).toLocaleString();\n\t\t\t\t\t} catch (e) {\n\t\t\t\t\t\tdate = v.date.replace(\"T\", \" \");\n\t\t\t\t\t\tconst index = date.lastIndexOf(\".\");\n\t\t\t\t\t\tif (index > 0) {\n\t\t\t\t\t\t\tdate = `${date.substr(0, index)} Z`;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tdate = date.padStart(23);\n\t\t\t\tconst size = formatNumber(res.size).padStart(10);\n\t\t\t\tconst sizeNew = formatNumber(res.sizeNew).padStart(10);\n\t\t\t\tconst blobCount = formatNumber(res.blobCount).padStart(6);\n\t\t\t\tconst blobCountNew = formatNumber(res.blobCountNew).padStart(9);\n\n\t\t\t\tconsole.log(\n\t\t\t\t\t`${name.padEnd(\n\t\t\t\t\t\t15,\n\t\t\t\t\t)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`,\n\t\t\t\t);\n\n\t\t\t\tawait saveSnapshot(name, blobs, saveDir);\n\t\t\t}\n\t\t}\n\t}\n\n\tif (dumpSnapshotStats || dumpSnapshotTrees) {\n\t\tif (version === undefined) {\n\t\t\tconsole.log(\"No snapshot tree\");\n\t\t} else {\n\t\t\tif (blobsToDump === undefined) {\n\t\t\t\tblobsToDump = await fetchBlobsFromVersion(storage, version);\n\t\t\t}\n\t\t\tconsole.log(`\\n\\nSnapshot version ${version.id}`);\n\t\t\tawait dumpSnapshotTreeVerbose(version.id, blobsToDump);\n\t\t}\n\t}\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluid-tools/fetch-tool",
-  "version": "2.0.0-dev-rc.2.0.0.246488",
+  "version": "2.0.0-dev-rc.3.0.0.253463",
   "description": "Console tool to fetch Fluid data from relay service",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -10,36 +10,36 @@
   },
   "license": "MIT",
   "author": "Microsoft and contributors",
-  "main": "dist/fluidFetch.js",
-  "types": "dist/fluidFetch.d.ts",
+  "type": "module",
   "bin": {
     "fluid-fetch": "bin/fluid-fetch"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/container-runtime": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/core-interfaces": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/core-utils": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/datastore": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/driver-definitions": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/odsp-doclib-utils": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/odsp-driver": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/odsp-driver-definitions": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/odsp-urlresolver": "2.0.0-dev-rc.2.0.0.246488",
+    "@fluid-internal/client-utils": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/container-runtime": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/core-interfaces": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/core-utils": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/datastore": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/driver-definitions": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/odsp-doclib-utils": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/odsp-driver": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/odsp-driver-definitions": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/odsp-urlresolver": "2.0.0-dev-rc.3.0.0.253463",
     "@fluidframework/protocol-definitions": "^3.2.0",
-    "@fluidframework/routerlicious-driver": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/routerlicious-urlresolver": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/runtime-definitions": "2.0.0-dev-rc.2.0.0.246488",
-    "@fluidframework/tool-utils": "2.0.0-dev-rc.2.0.0.246488"
+    "@fluidframework/routerlicious-driver": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/routerlicious-urlresolver": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/runtime-definitions": "2.0.0-dev-rc.3.0.0.253463",
+    "@fluidframework/tool-utils": "2.0.0-dev-rc.3.0.0.253463"
   },
   "devDependencies": {
-    "@arethetypeswrong/cli": "^0.13.3",
+    "@biomejs/biome": "^1.6.2",
     "@fluid-tools/build-cli": "^0.34.0",
     "@fluid-tools/fetch-tool-previous": "npm:@fluid-tools/fetch-tool@2.0.0-internal.8.0.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.34.0",
     "@fluidframework/eslint-config-fluid": "^5.1.0",
     "@types/node": "^18.19.0",
+    "copyfiles": "^2.4.1",
     "eslint": "~8.55.0",
     "prettier": "~3.0.3",
     "rimraf": "^4.4.0",
@@ -51,15 +51,15 @@
   "scripts": {
     "build": "fluid-build . --task build",
     "build:compile": "fluid-build . --task compile",
-    "check:are-the-types-wrong": "attw --pack",
-    "clean": "rimraf --glob dist \"**/*.tsbuildinfo\" \"**/*.build.log\"",
+    "build:esnext": "tsc --project ./tsconfig.json",
+    "check:prettier": "prettier --check . --cache --ignore-path ../../../.prettierignore",
+    "clean": "rimraf --glob dist \"**/*.tsbuildinfo\" \"**/*.build.log\" lib",
     "eslint": "eslint --format stylish src",
     "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
-    "format": "npm run prettier:fix",
-    "lint": "npm run prettier && npm run eslint",
-    "lint:fix": "npm run prettier:fix && npm run eslint:fix",
-    "prettier": "prettier --check . --cache --ignore-path ../../../.prettierignore",
-    "prettier:fix": "prettier --write . --cache --ignore-path ../../../.prettierignore",
-    "tsc": "tsc"
+    "format": "fluid-build --task format .",
+    "format:prettier": "prettier --write . --cache --ignore-path ../../../.prettierignore",
+    "lint": "fluid-build . --task lint",
+    "lint:fix": "fluid-build . --task eslint:fix --task format",
+    "tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && copyfiles -f ../../../common/build/build-common/src/cjs/package.json ./dist"
   }
 }

package/src/fluidAnalyzeMessages.ts

@@ -3,7 +3,13 @@
  * Licensed under the MIT License.
  */
 
-import { assert, unreachableCase } from "@fluidframework/core-utils";
+import {
+	ContainerMessageType,
+	IChunkedOp,
+	unpackRuntimeMessage,
+} from "@fluidframework/container-runtime/internal";
+import { assert, unreachableCase } from "@fluidframework/core-utils/internal";
+import { DataStoreMessageType } from "@fluidframework/datastore/internal";
 import {
 	ISequencedDocumentMessage,
 	ISummaryAck,
@@ -12,13 +18,7 @@ import {
 	MessageType,
 	TreeEntry,
 } from "@fluidframework/protocol-definitions";
-import { IAttachMessage, IEnvelope } from "@fluidframework/runtime-definitions";
-import {
-	IChunkedOp,
-	ContainerMessageType,
-	unpackRuntimeMessage,
-} from "@fluidframework/container-runtime";
-import { DataStoreMessageType } from "@fluidframework/datastore";
+import { IAttachMessage, IEnvelope } from "@fluidframework/runtime-definitions/internal";
 
 const noClientName = "No Client";
 const objectTypePrefix = "https://graph.microsoft.com/types/";
@@ -555,6 +555,9 @@ function processOp(
 			case ContainerMessageType.GC: {
 				break;
 			}
+			case ContainerMessageType.DocumentSchemaChange: {
+				break;
+			}
 			case ContainerMessageType.ChunkedOp: {
 				const chunk = runtimeMessage.contents as IChunkedOp;
 				// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
@@ -580,8 +583,8 @@ function processOp(
 					opCount = chunk.totalChunks; // 1 op for each chunk.
 					const patchedMessage = Object.create(runtimeMessage);
 					patchedMessage.contents = chunks.join("");
-					patchedMessage.type = chunk.originalType;
-					type = chunk.originalType;
+					type = (chunk as any).originalType;
+					patchedMessage.type = type;
 					totalMsgSize = value.totalSize;
 					chunkMap.delete(patchedMessage.clientId);
 				} else {

package/src/fluidFetch.ts

@@ -5,12 +5,14 @@
 
 import fs from "fs";
 import util from "util";
-import { isOdspHostname, IOdspDriveItem } from "@fluidframework/odsp-doclib-utils/internal";
-import { paramSaveDir, paramURL, parseArguments } from "./fluidFetchArgs";
-import { connectionInfo, fluidFetchInit } from "./fluidFetchInit";
-import { fluidFetchMessages } from "./fluidFetchMessages";
-import { getSharepointFiles, getSingleSharePointFile } from "./fluidFetchSharePoint";
-import { fluidFetchSnapshot } from "./fluidFetchSnapshot";
+
+import { IOdspDriveItem, isOdspHostname } from "@fluidframework/odsp-doclib-utils/internal";
+
+import { paramSaveDir, paramURL, parseArguments } from "./fluidFetchArgs.js";
+import { connectionInfo, fluidFetchInit } from "./fluidFetchInit.js";
+import { fluidFetchMessages } from "./fluidFetchMessages.js";
+import { getSharepointFiles, getSingleSharePointFile } from "./fluidFetchSharePoint.js";
+import { fluidFetchSnapshot } from "./fluidFetchSnapshot.js";
 
 async function fluidFetchOneFile(urlStr: string, name?: string) {
 	const documentService = await fluidFetchInit(urlStr);

package/src/fluidFetchInit.ts

@@ -4,19 +4,23 @@
  */
 
 import { IRequest } from "@fluidframework/core-interfaces";
-import { IResolvedUrl } from "@fluidframework/driver-definitions";
+import { IResolvedUrl } from "@fluidframework/driver-definitions/internal";
 import { IClientConfig, IOdspAuthRequestInfo } from "@fluidframework/odsp-doclib-utils/internal";
-import * as odsp from "@fluidframework/odsp-driver";
+import * as odsp from "@fluidframework/odsp-driver/internal";
 import {
 	IOdspResolvedUrl,
 	OdspResourceTokenFetchOptions,
-} from "@fluidframework/odsp-driver-definitions";
-import { FluidAppOdspUrlResolver, OdspUrlResolver } from "@fluidframework/odsp-urlresolver";
-import * as r11s from "@fluidframework/routerlicious-driver";
-import { RouterliciousUrlResolver } from "@fluidframework/routerlicious-urlresolver";
-import { getMicrosoftConfiguration } from "@fluidframework/tool-utils";
-import { localDataOnly, paramJWT } from "./fluidFetchArgs";
-import { resolveWrapper } from "./fluidFetchSharePoint";
+} from "@fluidframework/odsp-driver-definitions/internal";
+import {
+	FluidAppOdspUrlResolver,
+	OdspUrlResolver,
+} from "@fluidframework/odsp-urlresolver/internal";
+import * as r11s from "@fluidframework/routerlicious-driver/internal";
+import { RouterliciousUrlResolver } from "@fluidframework/routerlicious-urlresolver/internal";
+import { getMicrosoftConfiguration } from "@fluidframework/tool-utils/internal";
+
+import { localDataOnly, paramJWT } from "./fluidFetchArgs.js";
+import { resolveWrapper } from "./fluidFetchSharePoint.js";
 
 export let latestVersionsId: string = "";
 export let connectionInfo: any;

package/src/fluidFetchMessages.ts

@@ -4,23 +4,25 @@
  */
 
 import fs from "fs";
-import { assert } from "@fluidframework/core-utils";
-import { IDocumentService } from "@fluidframework/driver-definitions";
+
+import { assert } from "@fluidframework/core-utils/internal";
+import { IDocumentService } from "@fluidframework/driver-definitions/internal";
 import {
 	IClient,
 	ISequencedDocumentMessage,
 	MessageType,
 	ScopeType,
 } from "@fluidframework/protocol-definitions";
-import { printMessageStats } from "./fluidAnalyzeMessages";
+
+import { printMessageStats } from "./fluidAnalyzeMessages.js";
 import {
 	connectToWebSocket,
-	dumpMessages,
 	dumpMessageStats,
+	dumpMessages,
+	messageTypeFilter,
 	overWrite,
 	paramActualFormatting,
-	messageTypeFilter,
-} from "./fluidFetchArgs";
+} from "./fluidFetchArgs.js";
 
 function filenameFromIndex(index: number): string {
 	return index === 0 ? "" : index.toString(); // support old tools...

package/src/fluidFetchSharePoint.ts

@@ -4,24 +4,26 @@
  */
 
 import child_process from "child_process";
+
 import { DriverErrorTypes } from "@fluidframework/driver-definitions";
 import {
+	IClientConfig,
+	IOdspAuthRequestInfo,
+	IOdspDriveItem,
 	getChildrenByDriveItem,
 	getDriveItemByServerRelativePath,
 	getDriveItemFromDriveAndItem,
-	IClientConfig,
-	IOdspDriveItem,
 	getOdspRefreshTokenFn,
-	IOdspAuthRequestInfo,
 } from "@fluidframework/odsp-doclib-utils/internal";
 import {
-	getMicrosoftConfiguration,
+	IOdspTokenManagerCacheKey,
+	OdspTokenConfig,
 	OdspTokenManager,
+	getMicrosoftConfiguration,
 	odspTokensCache,
-	OdspTokenConfig,
-	IOdspTokenManagerCacheKey,
-} from "@fluidframework/tool-utils";
-import { getForceTokenReauth } from "./fluidFetchArgs";
+} from "@fluidframework/tool-utils/internal";
+
+import { getForceTokenReauth } from "./fluidFetchArgs.js";
 
 export async function resolveWrapper<T>(
 	callback: (authRequestInfo: IOdspAuthRequestInfo) => Promise<T>,

package/src/fluidFetchSnapshot.ts

@@ -7,10 +7,13 @@ import fs from "fs";
 import util from "util";
 
 import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
-import { IDocumentService, IDocumentStorageService } from "@fluidframework/driver-definitions";
+import {
+	IDocumentService,
+	IDocumentStorageService,
+} from "@fluidframework/driver-definitions/internal";
 import { ISnapshotTree, IVersion } from "@fluidframework/protocol-definitions";
 
-import { formatNumber } from "./fluidAnalyzeMessages";
+import { formatNumber } from "./fluidAnalyzeMessages.js";
 import {
 	dumpSnapshotStats,
 	dumpSnapshotTrees,
@@ -18,8 +21,8 @@ import {
 	paramActualFormatting,
 	paramNumSnapshotVersions,
 	paramSnapshotVersionIndex,
-} from "./fluidFetchArgs";
-import { latestVersionsId } from "./fluidFetchInit";
+} from "./fluidFetchArgs.js";
+import { latestVersionsId } from "./fluidFetchInit.js";
 
 interface ISnapshotInfo {
 	blobCountNew: number;

package/tsconfig.cjs.json

@@ -0,0 +1,7 @@
+{
+	// This config must be used in a "type": "commonjs" environment. (Use `fluid-tsc commonjs`.)
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"outDir": "./dist",
+	},
+}

package/tsconfig.json

@@ -1,13 +1,10 @@
 {
-	"extends": [
-		"../../../common/build/build-common/tsconfig.base.json",
-		"../../../common/build/build-common/tsconfig.cjs.json",
-	],
+	"extends": "../../../common/build/build-common/tsconfig.node16.json",
 	"include": ["src/**/*.ts"],
 	"exclude": ["dist", "node_modules"],
 	"compilerOptions": {
 		"rootDir": "./src",
-		"outDir": "./dist",
+		"outDir": "./lib",
 		"types": ["node"],
 	},
 }