@fluid-tools/fetch-tool 0.58.3000-61081 → 0.59.2000-61729
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchSnapshot.d.ts","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACH,gBAAgB,EAEnB,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"fluidFetchSnapshot.d.ts","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACH,gBAAgB,EAEnB,MAAM,oCAAoC,CAAC;AAyP5C,wBAAsB,kBAAkB,CACpC,eAAe,CAAC,EAAE,gBAAgB,EAClC,OAAO,CAAC,EAAE,MAAM,iBA4FnB"}
|
|
@@ -61,10 +61,9 @@ function createTreeBlob(tree, prefix, patched) {
|
|
|
61
61
|
const treePath = `${prefix}${filename}`;
|
|
62
62
|
return { treePath, blobId: "original tree $id", filename, blob, patched, reused: false };
|
|
63
63
|
}
|
|
64
|
-
async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/",
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (commit && fluidFetchArgs_1.dumpSnapshotTrees) {
|
|
64
|
+
async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/", parentBlobIdMap) {
|
|
65
|
+
const isTopLevel = !parentBlobIdMap;
|
|
66
|
+
if (isTopLevel && fluidFetchArgs_1.dumpSnapshotTrees) {
|
|
68
67
|
console.log(tree);
|
|
69
68
|
}
|
|
70
69
|
if (prefix === "/") {
|
|
@@ -72,37 +71,19 @@ async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/", perCommit
|
|
|
72
71
|
blobCacheCurrent = new Map();
|
|
73
72
|
}
|
|
74
73
|
// Create the tree info before fetching blobs (which will modify it)
|
|
75
|
-
let
|
|
76
|
-
if (
|
|
77
|
-
|
|
74
|
+
let topLevelBlob;
|
|
75
|
+
if (isTopLevel) {
|
|
76
|
+
topLevelBlob = createTreeBlob(tree, prefix, false);
|
|
78
77
|
}
|
|
79
|
-
const blobIdMap =
|
|
78
|
+
const blobIdMap = parentBlobIdMap !== null && parentBlobIdMap !== void 0 ? parentBlobIdMap : new Map();
|
|
80
79
|
let result = fetchBlobs(prefix, tree, storage, blobIdMap);
|
|
81
|
-
for (const dataStore of Object.keys(tree.commits)) {
|
|
82
|
-
const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);
|
|
83
|
-
if (dataStoreVersions.length !== 1) {
|
|
84
|
-
console.error(`ERROR: Unable to get versions for ${dataStore}`);
|
|
85
|
-
continue;
|
|
86
|
-
}
|
|
87
|
-
const dataStoreSnapShotTree = await reportErrors(`getSnapshotTree ${dataStoreVersions[0].id}`, storage.getSnapshotTree(dataStoreVersions[0]));
|
|
88
|
-
if (dataStoreSnapShotTree === null) {
|
|
89
|
-
// eslint-disable-next-line max-len
|
|
90
|
-
console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
common_utils_1.assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore], 0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);
|
|
94
|
-
common_utils_1.assert(tree.commits[dataStore] === dataStoreVersions[0].id, 0x1c0 /* "Mismatch between commit id and fetched tree id" */);
|
|
95
|
-
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, dataStoreSnapShotTree, `${prefix}[${dataStore}]/`);
|
|
96
|
-
result = result.concat(dataStoreBlobs);
|
|
97
|
-
}
|
|
98
80
|
for (const subtreeId of Object.keys(tree.trees)) {
|
|
99
81
|
const subtree = tree.trees[subtreeId];
|
|
100
|
-
common_utils_1.assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* "Unexpected subtree properties" */);
|
|
101
82
|
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, subtree, `${prefix}${subtreeId}/`, blobIdMap);
|
|
102
83
|
result = result.concat(dataStoreBlobs);
|
|
103
84
|
}
|
|
104
|
-
if (
|
|
105
|
-
result.push(
|
|
85
|
+
if (topLevelBlob) {
|
|
86
|
+
result.push(topLevelBlob);
|
|
106
87
|
result.push(createTreeBlob(tree, prefix, true));
|
|
107
88
|
}
|
|
108
89
|
return result;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchSnapshot.js","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;AAEH,4CAAoB;AACpB,gDAAwB;AACxB,+DAA2G;AAS3G,+DAAsE;AACtE,iEAAsD;AACtD,qDAQ0B;AAC1B,qDAAoD;AA8BpD,SAAS,aAAa,CAAC,WAAyB;IAC5C,OAAO,SAAS,IAAI,WAAW,CAAC;AACpC,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoC,CAAC;AAC9D,IAAI,iBAAiB,GAAG,IAAI,GAAG,EAAoC,CAAC;AACpE,IAAI,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;AAEnE,SAAS,UAAU,CAAC,MAAc,EAC9B,IAAmB,EACnB,OAAgC,EAChC,SAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QACxC,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,IAAI,EAAE;YACjB,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE;gBACP,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,IAAI,IAAI,KAAK,SAAS,EAAE;oBACpB,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAChC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC/B;aACJ;YACD,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEnC,4DAA4D;YAC5D,gEAAgE;YAChE,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE;gBACR,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;aAChC;YACD,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE1D,mEAAmE;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;SAC/B;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,MAAc,EAAE,OAAgB;;IACzE,MAAM,EAAE,SAAG,IAAI,CAAC,EAAE,mCAAI,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,6BAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;IACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7F,CAAC;AAED,KAAK,UAAU,0BAA0B,CACrC,OAAgC,EAChC,IAAmB,EACnB,SAAiB,GAAG,EACpB,kBAAwC;IACxC,qBAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,EAC7D,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,CAAC,kBAAkB,CAAC;IACnC,IAAI,MAAM,IAAI,kCAAiB,EAAE;QAC7B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;KACrB;IAED,IAAI,MAAM,KAAK,GAAG,EAAE;QAChB,iBAAiB,GAAG,gBAAgB,CAAC;QACrC,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;KAClE;IAED,oEAAoE;IACpE,IAAI,UAAoC,CAAC;IACzC,IAAI,MAAM,EAAE;QACR,UAAU,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;KACpD;IAED,MAAM,SAAS,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAI,GAAG,EAAkB,CAAC;IAClE,IAAI,MAAM,GAAmB,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE1E,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QAC/C,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,OAAO,CAAC,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;YAChE,SAAS;SACZ;QACD,MAAM,qBAAqB,GAAG,MAAM,YAAY,CAC5C,mBAAmB,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAC5C,OAAO,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,qBAAqB,KAAK,IAAI,EAAE;YAChC,mCAAmC;YACnC,OAAO,CAAC,KAAK,CAAC,uCAAuC,SAAS,YAAY,MAAM,eAAe,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1H,SAAS;SACZ;QACD,qBAAM,CAAC,qBAAqB,CAAC,EAAE,KAAK,SAAS,IAAI,qBAAqB,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EACjG,KAAK,CAAC,2DAA2D,CAAC,CAAC;QACvE,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EACtD,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAClE,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,qBAAqB,EACrB,GAAG,MAAM,IAAI,SAAS,IAAI,CAAC,CAAC;QAChC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,qBAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC/F,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,OAAO,EACP,GAAG,MAAM,GAAG,SAAS,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,IAAI,UAAU,EAAE;QACZ,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,WAA2B;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,WAA2B;IAC5E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;KAC3D;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;QAC3C,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,mCAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACjI,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,eAAe,mCAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAA2B;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACd,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;YACvB,YAAY,EAAE,CAAC;SAClB;QACD,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,WAA2B,EAAE,OAAe;IAClF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,KAAK,CAAC,GAAG,MAAM,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO;SACV;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;YACtB,6BAA6B;YAC7B,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,OAAO,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YAC5C,IAAI;gBACA,IAAI,CAAC,sCAAqB,EAAE;oBACxB,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;iBAC/D;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,YAAE,CAAC,aAAa,CACZ,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EAAE,OAAO,CAAC,CAAC;SAC3D;aAAM;YACH,qEAAqE;YACrE,MAAM,UAAU,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YACjD,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,OAAO,EAAE,UAAU,CAAC,CAAC;YAChE,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EACtD,sCAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;SAClG;IACL,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAgC,EAAE,OAAiB;IACpF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,IAAI,CAAC,IAAI,EAAE;QACP,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;KACpE;IACD,OAAO,0BAA0B,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,YAAY,CAAI,OAAe,EAAE,GAAe;IAC3D,IAAI;QACA,OAAO,MAAM,GAAG,CAAC;KACpB;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;KACf;AACL,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACpC,eAAkC,EAClC,OAAgB;IAEhB,IAAI,CAAC,kCAAiB,IAAI,CAAC,kCAAiB,IAAI,CAAC,qCAAoB,IAAI,OAAO,KAAK,SAAS,EAAE;QAC5F,OAAO;KACV;IAED,4CAA4C;IAC5C,2BAA2B;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,eAAe,EAAE;QAClB,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;IACvD,IAAI,2CAA0B,EAAE;QAC5B,OAAO,GAAG,qCAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,kCAAmB,EAAE,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;KACvG;IAED,IAAI,OAA6B,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAC/B,eAAe,iCAAgB,EAAE,EACjC,OAAO,CAAC,WAAW,CAAC,iCAAgB,EAAE,yCAAwB,CAAC,CAAC,CAAC;IACrE,IAAI,qCAAoB,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACzB;IAED,IAAI,WAAuC,CAAC;IAC5C,IAAI,0CAAyB,KAAK,SAAS,EAAE;QACzC,OAAO,GAAG,QAAQ,CAAC,0CAAyB,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,MAAM,iDAAiD,CAAC,CAAC;YAChG,OAAO;SACV;QACD,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;SAClD;KACJ;SAAM;QACH,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,0FAA0F,CAAC,CAAC;YACxG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,2FAA2F;YAC3F,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC3C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtD,WAAW,GAAG,KAAK,CAAC;gBACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAEhD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,CAAC,IAAI,EAAE;oBACR,IAAI;wBACA,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;qBAC5C;oBAAC,OAAO,CAAC,EAAE;wBACR,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBACpC,IAAI,KAAK,GAAG,CAAC,EAAE;4BACX,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;yBACvC;qBACJ;iBACJ;gBACD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,mCAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,mCAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,mCAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,mCAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,IAAI,MAAM,OAAO,MAAM,SAAS,MAAM,YAAY,EAAE,CAAC,CAAC;gBAEpG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;aAC5C;SACJ;KACJ;IAED,IAAI,kCAAiB,IAAI,kCAAiB,EAAE;QACxC,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SACnC;aAAM;YACH,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC3B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aAC/D;YACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,uBAAuB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;SAC1D;KACJ;AACL,CAAC;AA9FD,gDA8FC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport util from \"util\";\nimport { assert, bufferToString, stringToBuffer, TelemetryNullLogger } from \"@fluidframework/common-utils\";\nimport {\n IDocumentService,\n IDocumentStorageService,\n} from \"@fluidframework/driver-definitions\";\nimport {\n ISnapshotTree,\n IVersion,\n} from \"@fluidframework/protocol-definitions\";\nimport { BlobAggregationStorage } from \"@fluidframework/driver-utils\";\nimport { formatNumber } from \"./fluidAnalyzeMessages\";\nimport {\n dumpSnapshotStats,\n dumpSnapshotTrees,\n dumpSnapshotVersions,\n paramActualFormatting,\n paramNumSnapshotVersions,\n paramSnapshotVersionIndex,\n paramUnpackAggregatedBlobs,\n} from \"./fluidFetchArgs\";\nimport { latestVersionsId } from \"./fluidFetchInit\";\n\ninterface ISnapshotInfo {\n blobCountNew: number;\n blobCount: number;\n size: number;\n sizeNew: number;\n}\n\ntype IFetchedData = IFetchedBlob | IFetchedTree;\n\ninterface IFetchedBlob {\n treePath: string;\n filename: string;\n blobId: string;\n blob: Promise<ArrayBufferLike | undefined>;\n reused: boolean;\n}\n\ninterface IFetchedTree {\n treePath: string;\n blobId: string;\n filename: string;\n blob: ArrayBufferLike;\n\n reused: false;\n\n patched: boolean;\n}\n\nfunction isFetchedTree(fetchedData: IFetchedData): fetchedData is IFetchedTree {\n return \"patched\" in fetchedData;\n}\n\nconst blobCache = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCachePrevious = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\nfunction fetchBlobs(prefix: string,\n tree: ISnapshotTree,\n storage: IDocumentStorageService,\n blobIdMap: Map<string, number>,\n) {\n const result: IFetchedBlob[] = [];\n for (const item of Object.keys(tree.blobs)) {\n const treePath = `${prefix}${item}`;\n const blobId = tree.blobs[item];\n if (blobId !== null) {\n let reused = true;\n let blob = blobCachePrevious.get(blobId);\n if (!blob) {\n reused = false;\n blob = blobCache.get(blobId);\n if (blob === undefined) {\n blob = storage.readBlob(blobId);\n blobCache.set(blobId, blob);\n }\n }\n blobCacheCurrent.set(blobId, blob);\n\n // Use the blobIdMap to assign a number for each unique blob\n // and use it as a prefix for files to avoid case-insensitive fs\n let index = blobIdMap.get(blobId);\n if (!index) {\n index = blobIdMap.size;\n blobIdMap.set(blobId, index);\n }\n const filename = `${index}-${blobId}`;\n result.push({ treePath, blobId, blob, reused, filename });\n\n // patch the tree so that we can write it out to reference the file\n tree.blobs[item] = filename;\n }\n }\n return result;\n}\n\nfunction createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {\n const id = tree.id ?? \"original\";\n const blob = stringToBuffer(JSON.stringify(tree),\"utf8\");\n const filename = patched ? \"tree\" : `tree-${id}`;\n const treePath = `${prefix}${filename}`;\n return { treePath, blobId: \"original tree $id\", filename, blob, patched, reused: false };\n}\n\nasync function fetchBlobsFromSnapshotTree(\n storage: IDocumentStorageService,\n tree: ISnapshotTree,\n prefix: string = \"/\",\n perCommitBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {\n assert(Object.keys(tree.commits).length === 0 || (prefix === \"/\"),\n 0x1be /* \"Unexpected tree input to fetch\" */);\n const commit = !perCommitBlobIdMap;\n if (commit && dumpSnapshotTrees) {\n console.log(tree);\n }\n\n if (prefix === \"/\") {\n blobCachePrevious = blobCacheCurrent;\n blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n }\n\n // Create the tree info before fetching blobs (which will modify it)\n let commitBlob: IFetchedTree | undefined;\n if (commit) {\n commitBlob = createTreeBlob(tree, prefix, false);\n }\n\n const blobIdMap = perCommitBlobIdMap ?? new Map<string, number>();\n let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);\n\n for (const dataStore of Object.keys(tree.commits)) {\n const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);\n if (dataStoreVersions.length !== 1) {\n console.error(`ERROR: Unable to get versions for ${dataStore}`);\n continue;\n }\n const dataStoreSnapShotTree = await reportErrors(\n `getSnapshotTree ${dataStoreVersions[0].id}`,\n storage.getSnapshotTree(dataStoreVersions[0]));\n if (dataStoreSnapShotTree === null) {\n // eslint-disable-next-line max-len\n console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);\n continue;\n }\n assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore],\n 0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);\n assert(tree.commits[dataStore] === dataStoreVersions[0].id,\n 0x1c0 /* \"Mismatch between commit id and fetched tree id\" */);\n const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n storage,\n dataStoreSnapShotTree,\n `${prefix}[${dataStore}]/`);\n result = result.concat(dataStoreBlobs);\n }\n\n for (const subtreeId of Object.keys(tree.trees)) {\n const subtree = tree.trees[subtreeId];\n assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* \"Unexpected subtree properties\" */);\n const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n storage,\n subtree,\n `${prefix}${subtreeId}/`, blobIdMap);\n result = result.concat(dataStoreBlobs);\n }\n\n if (commitBlob) {\n result.push(commitBlob);\n result.push(createTreeBlob(tree, prefix, true));\n }\n return result;\n}\n\nfunction getDumpFetchedData(fetchedData: IFetchedData[]) {\n const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));\n return sorted.filter((item) => !isFetchedTree(item) || !item.patched);\n}\n\nasync function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {\n let size = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n let nameLength = 10;\n for (const item of sorted) {\n nameLength = Math.max(nameLength, item.treePath.length);\n }\n\n console.log(\"\");\n console.log(`${\"Blob Path\".padEnd(nameLength)} | Reused | Bytes`);\n console.log(\"-\".repeat(nameLength + 26));\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer,\"utf8\");\n // eslint-disable-next-line max-len\n console.log(`${item.treePath.padEnd(nameLength)} | ${item.reused ? \"X\" : \" \"} | ${formatNumber(blob.length).padStart(10)}`);\n size += blob.length;\n }\n\n console.log(\"-\".repeat(nameLength + 26));\n console.log(`${\"Total snapshot size\".padEnd(nameLength)} | | ${formatNumber(size).padStart(10)}`);\n}\n\nasync function dumpSnapshotTree(name: string, fetchedData: IFetchedData[]): Promise<ISnapshotInfo> {\n let size = 0;\n let sizeNew = 0;\n let blobCountNew = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer, \"utf8\");\n if (!item.reused) {\n sizeNew += blob.length;\n blobCountNew++;\n }\n size += blob.length;\n }\n\n return { blobCountNew, blobCount: sorted.length, size, sizeNew };\n}\n\nasync function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {\n const outDir = `${saveDir}/${name}/`;\n const mkdir = util.promisify(fs.mkdir);\n\n await mkdir(`${outDir}/decoded`, { recursive: true });\n await Promise.all(fetchedData.map(async (item) => {\n const buffer = await item.blob;\n if (buffer === undefined) {\n console.error(`ERROR: Unable to get data for blob ${item.blobId}`);\n return;\n }\n\n if (!isFetchedTree(item)) {\n // Just write the data as is.\n fs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));\n\n // we assume that the buffer is utf8 here, which currently is true for\n // all of our snapshot blobs. It doesn't necessary be true in the future\n let decoded = bufferToString(buffer,\"utf8\");\n try {\n if (!paramActualFormatting) {\n decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);\n }\n } catch (e) {\n }\n fs.writeFileSync(\n `${outDir}/decoded/${item.filename}.json`, decoded);\n } else {\n // Write out same data for tree decoded or not, except for formatting\n const treeString = bufferToString(buffer,\"utf8\");\n fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);\n fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`,\n paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));\n }\n }));\n}\n\nasync function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {\n const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));\n if (!tree) {\n return Promise.reject(new Error(\"Failed to load snapshot tree\"));\n }\n return fetchBlobsFromSnapshotTree(storage, tree);\n}\n\nasync function reportErrors<T>(message: string, res: Promise<T>) {\n try {\n return await res;\n } catch (error) {\n console.error(`Error calling ${message}`);\n throw error;\n }\n}\n\nexport async function fluidFetchSnapshot(\n documentService?: IDocumentService,\n saveDir?: string,\n ) {\n if (!dumpSnapshotStats && !dumpSnapshotTrees && !dumpSnapshotVersions && saveDir === undefined) {\n return;\n }\n\n // --local mode - do not connect to storage.\n // For now, bail out early.\n // In future, separate download from analyzes parts and allow offline analyzes\n if (!documentService) {\n return;\n }\n\n console.log(\"\\n\");\n\n let storage = await documentService.connectToStorage();\n if (paramUnpackAggregatedBlobs) {\n storage = BlobAggregationStorage.wrap(storage, new TelemetryNullLogger(), false /* allowPacking */);\n }\n\n let version: IVersion | undefined;\n const versions = await reportErrors(\n `getVersions ${latestVersionsId}`,\n storage.getVersions(latestVersionsId, paramNumSnapshotVersions));\n if (dumpSnapshotVersions) {\n console.log(\"Snapshot versions\");\n console.log(versions);\n }\n\n let blobsToDump: IFetchedData[] | undefined;\n if (paramSnapshotVersionIndex !== undefined) {\n version = versions[paramSnapshotVersionIndex];\n if (version === undefined) {\n console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);\n return;\n }\n if (saveDir !== undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n const name = version.id;\n console.log(`Saving snapshot ${name}`);\n await saveSnapshot(name, blobsToDump, saveDir);\n }\n } else {\n version = versions[0];\n if (saveDir !== undefined && versions.length > 0) {\n console.log(\" Name | Date | Size | New Size | Blobs | New Blobs\");\n console.log(\"-\".repeat(86));\n\n // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots\n for (let i = versions.length - 1; i >= 0; i--) {\n const v = versions[i];\n const blobs = await fetchBlobsFromVersion(storage, v);\n blobsToDump = blobs;\n const name = `${i}-${v.id}`;\n const res = await dumpSnapshotTree(name, blobs);\n\n let date = \"\";\n if (v.date) {\n try {\n date = new Date(v.date).toLocaleString();\n } catch (e) {\n date = v.date.replace(\"T\", \" \");\n const index = date.lastIndexOf(\".\");\n if (index > 0) {\n date = `${date.substr(0, index)} Z`;\n }\n }\n }\n date = date.padStart(23);\n const size = formatNumber(res.size).padStart(10);\n const sizeNew = formatNumber(res.sizeNew).padStart(10);\n const blobCount = formatNumber(res.blobCount).padStart(6);\n const blobCountNew = formatNumber(res.blobCountNew).padStart(9);\n\n console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);\n\n await saveSnapshot(name, blobs, saveDir);\n }\n }\n }\n\n if (dumpSnapshotStats || dumpSnapshotTrees) {\n if (version === undefined) {\n console.log(\"No snapshot tree\");\n } else {\n if (blobsToDump === undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n }\n console.log(`\\n\\nSnapshot version ${version.id}`);\n await dumpSnapshotTreeVerbose(version.id, blobsToDump);\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"fluidFetchSnapshot.js","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;AAEH,4CAAoB;AACpB,gDAAwB;AACxB,+DAAmG;AASnG,+DAAsE;AACtE,iEAAsD;AACtD,qDAQ0B;AAC1B,qDAAoD;AA8BpD,SAAS,aAAa,CAAC,WAAyB;IAC5C,OAAO,SAAS,IAAI,WAAW,CAAC;AACpC,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoC,CAAC;AAC9D,IAAI,iBAAiB,GAAG,IAAI,GAAG,EAAoC,CAAC;AACpE,IAAI,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;AAEnE,SAAS,UAAU,CAAC,MAAc,EAC9B,IAAmB,EACnB,OAAgC,EAChC,SAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QACxC,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,IAAI,EAAE;YACjB,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE;gBACP,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,IAAI,IAAI,KAAK,SAAS,EAAE;oBACpB,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAChC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC/B;aACJ;YACD,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEnC,4DAA4D;YAC5D,gEAAgE;YAChE,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE;gBACR,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;aAChC;YACD,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE1D,mEAAmE;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;SAC/B;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,MAAc,EAAE,OAAgB;;IACzE,MAAM,EAAE,SAAG,IAAI,CAAC,EAAE,mCAAI,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,6BAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;IACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7F,CAAC;AAED,KAAK,UAAU,0BAA0B,CACrC,OAAgC,EAChC,IAAmB,EACnB,SAAiB,GAAG,EACpB,eAAqC;IACrC,MAAM,UAAU,GAAG,CAAC,eAAe,CAAC;IACpC,IAAI,UAAU,IAAI,kCAAiB,EAAE;QACjC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;KACrB;IAED,IAAI,MAAM,KAAK,GAAG,EAAE;QAChB,iBAAiB,GAAG,gBAAgB,CAAC;QACrC,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;KAClE;IAED,oEAAoE;IACpE,IAAI,YAAsC,CAAC;IAC3C,IAAI,UAAU,EAAE;QACZ,YAAY,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;KACtD;IAED,MAAM,SAAS,GAAG,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,IAAI,GAAG,EAAkB,CAAC;IAC/D,IAAI,MAAM,GAAmB,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE1E,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,OAAO,EACP,GAAG,MAAM,GAAG,SAAS,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,IAAI,YAAY,EAAE;QACd,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,WAA2B;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,WAA2B;IAC5E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;KAC3D;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;QAC3C,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,mCAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACjI,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,eAAe,mCAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAA2B;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACd,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;YACvB,YAAY,EAAE,CAAC;SAClB;QACD,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,WAA2B,EAAE,OAAe;IAClF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,KAAK,CAAC,GAAG,MAAM,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO;SACV;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;YACtB,6BAA6B;YAC7B,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,OAAO,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YAC5C,IAAI;gBACA,IAAI,CAAC,sCAAqB,EAAE;oBACxB,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;iBAC/D;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,YAAE,CAAC,aAAa,CACZ,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EAAE,OAAO,CAAC,CAAC;SAC3D;aAAM;YACH,qEAAqE;YACrE,MAAM,UAAU,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YACjD,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,OAAO,EAAE,UAAU,CAAC,CAAC;YAChE,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EACtD,sCAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;SAClG;IACL,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAgC,EAAE,OAAiB;IACpF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,IAAI,CAAC,IAAI,EAAE;QACP,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;KACpE;IACD,OAAO,0BAA0B,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,YAAY,CAAI,OAAe,EAAE,GAAe;IAC3D,IAAI;QACA,OAAO,MAAM,GAAG,CAAC;KACpB;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;KACf;AACL,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACpC,eAAkC,EAClC,OAAgB;IAEhB,IAAI,CAAC,kCAAiB,IAAI,CAAC,kCAAiB,IAAI,CAAC,qCAAoB,IAAI,OAAO,KAAK,SAAS,EAAE;QAC5F,OAAO;KACV;IAED,4CAA4C;IAC5C,2BAA2B;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,eAAe,EAAE;QAClB,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;IACvD,IAAI,2CAA0B,EAAE;QAC5B,OAAO,GAAG,qCAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,kCAAmB,EAAE,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;KACvG;IAED,IAAI,OAA6B,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAC/B,eAAe,iCAAgB,EAAE,EACjC,OAAO,CAAC,WAAW,CAAC,iCAAgB,EAAE,yCAAwB,CAAC,CAAC,CAAC;IACrE,IAAI,qCAAoB,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACzB;IAED,IAAI,WAAuC,CAAC;IAC5C,IAAI,0CAAyB,KAAK,SAAS,EAAE;QACzC,OAAO,GAAG,QAAQ,CAAC,0CAAyB,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,MAAM,iDAAiD,CAAC,CAAC;YAChG,OAAO;SACV;QACD,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;SAClD;KACJ;SAAM;QACH,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,0FAA0F,CAAC,CAAC;YACxG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,2FAA2F;YAC3F,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC3C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtD,WAAW,GAAG,KAAK,CAAC;gBACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAEhD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,CAAC,IAAI,EAAE;oBACR,IAAI;wBACA,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;qBAC5C;oBAAC,OAAO,CAAC,EAAE;wBACR,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBACpC,IAAI,KAAK,GAAG,CAAC,EAAE;4BACX,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;yBACvC;qBACJ;iBACJ;gBACD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,mCAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,mCAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,mCAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,mCAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,IAAI,MAAM,OAAO,MAAM,SAAS,MAAM,YAAY,EAAE,CAAC,CAAC;gBAEpG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;aAC5C;SACJ;KACJ;IAED,IAAI,kCAAiB,IAAI,kCAAiB,EAAE;QACxC,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SACnC;aAAM;YACH,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC3B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aAC/D;YACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,uBAAuB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;SAC1D;KACJ;AACL,CAAC;AA9FD,gDA8FC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport util from \"util\";\nimport { bufferToString, stringToBuffer, TelemetryNullLogger } from \"@fluidframework/common-utils\";\nimport {\n IDocumentService,\n IDocumentStorageService,\n} from \"@fluidframework/driver-definitions\";\nimport {\n ISnapshotTree,\n IVersion,\n} from \"@fluidframework/protocol-definitions\";\nimport { BlobAggregationStorage } from \"@fluidframework/driver-utils\";\nimport { formatNumber } from \"./fluidAnalyzeMessages\";\nimport {\n dumpSnapshotStats,\n dumpSnapshotTrees,\n dumpSnapshotVersions,\n paramActualFormatting,\n paramNumSnapshotVersions,\n paramSnapshotVersionIndex,\n paramUnpackAggregatedBlobs,\n} from \"./fluidFetchArgs\";\nimport { latestVersionsId } from \"./fluidFetchInit\";\n\ninterface ISnapshotInfo {\n blobCountNew: number;\n blobCount: number;\n size: number;\n sizeNew: number;\n}\n\ntype IFetchedData = IFetchedBlob | IFetchedTree;\n\ninterface IFetchedBlob {\n treePath: string;\n filename: string;\n blobId: string;\n blob: Promise<ArrayBufferLike | undefined>;\n reused: boolean;\n}\n\ninterface IFetchedTree {\n treePath: string;\n blobId: string;\n filename: string;\n blob: ArrayBufferLike;\n\n reused: false;\n\n patched: boolean;\n}\n\nfunction isFetchedTree(fetchedData: IFetchedData): fetchedData is IFetchedTree {\n return \"patched\" in fetchedData;\n}\n\nconst blobCache = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCachePrevious = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\nfunction fetchBlobs(prefix: string,\n tree: ISnapshotTree,\n storage: IDocumentStorageService,\n blobIdMap: Map<string, number>,\n) {\n const result: IFetchedBlob[] = [];\n for (const item of Object.keys(tree.blobs)) {\n const treePath = `${prefix}${item}`;\n const blobId = tree.blobs[item];\n if (blobId !== null) {\n let reused = true;\n let blob = blobCachePrevious.get(blobId);\n if (!blob) {\n reused = false;\n blob = blobCache.get(blobId);\n if (blob === undefined) {\n blob = storage.readBlob(blobId);\n blobCache.set(blobId, blob);\n }\n }\n blobCacheCurrent.set(blobId, blob);\n\n // Use the blobIdMap to assign a number for each unique blob\n // and use it as a prefix for files to avoid case-insensitive fs\n let index = blobIdMap.get(blobId);\n if (!index) {\n index = blobIdMap.size;\n blobIdMap.set(blobId, index);\n }\n const filename = `${index}-${blobId}`;\n result.push({ treePath, blobId, blob, reused, filename });\n\n // patch the tree so that we can write it out to reference the file\n tree.blobs[item] = filename;\n }\n }\n return result;\n}\n\nfunction createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {\n const id = tree.id ?? \"original\";\n const blob = stringToBuffer(JSON.stringify(tree),\"utf8\");\n const filename = patched ? \"tree\" : `tree-${id}`;\n const treePath = `${prefix}${filename}`;\n return { treePath, blobId: \"original tree $id\", filename, blob, patched, reused: false };\n}\n\nasync function fetchBlobsFromSnapshotTree(\n storage: IDocumentStorageService,\n tree: ISnapshotTree,\n prefix: string = \"/\",\n parentBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {\n const isTopLevel = !parentBlobIdMap;\n if (isTopLevel && dumpSnapshotTrees) {\n console.log(tree);\n }\n\n if (prefix === \"/\") {\n blobCachePrevious = blobCacheCurrent;\n blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n }\n\n // Create the tree info before fetching blobs (which will modify it)\n let topLevelBlob: IFetchedTree | undefined;\n if (isTopLevel) {\n topLevelBlob = createTreeBlob(tree, prefix, false);\n }\n\n const blobIdMap = parentBlobIdMap ?? new Map<string, number>();\n let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);\n\n for (const subtreeId of Object.keys(tree.trees)) {\n const subtree = tree.trees[subtreeId];\n const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n storage,\n subtree,\n `${prefix}${subtreeId}/`, blobIdMap);\n result = result.concat(dataStoreBlobs);\n }\n\n if (topLevelBlob) {\n result.push(topLevelBlob);\n result.push(createTreeBlob(tree, prefix, true));\n }\n return result;\n}\n\nfunction getDumpFetchedData(fetchedData: IFetchedData[]) {\n const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));\n return sorted.filter((item) => !isFetchedTree(item) || !item.patched);\n}\n\nasync function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {\n let size = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n let nameLength = 10;\n for (const item of sorted) {\n nameLength = Math.max(nameLength, item.treePath.length);\n }\n\n console.log(\"\");\n console.log(`${\"Blob Path\".padEnd(nameLength)} | Reused | Bytes`);\n console.log(\"-\".repeat(nameLength + 26));\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer,\"utf8\");\n // eslint-disable-next-line max-len\n console.log(`${item.treePath.padEnd(nameLength)} | ${item.reused ? \"X\" : \" \"} | ${formatNumber(blob.length).padStart(10)}`);\n size += blob.length;\n }\n\n console.log(\"-\".repeat(nameLength + 26));\n console.log(`${\"Total snapshot size\".padEnd(nameLength)} | | ${formatNumber(size).padStart(10)}`);\n}\n\nasync function dumpSnapshotTree(name: string, fetchedData: IFetchedData[]): Promise<ISnapshotInfo> {\n let size = 0;\n let sizeNew = 0;\n let blobCountNew = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer, \"utf8\");\n if (!item.reused) {\n sizeNew += blob.length;\n blobCountNew++;\n }\n size += blob.length;\n }\n\n return { blobCountNew, blobCount: sorted.length, size, sizeNew };\n}\n\nasync function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {\n const outDir = `${saveDir}/${name}/`;\n const mkdir = util.promisify(fs.mkdir);\n\n await mkdir(`${outDir}/decoded`, { recursive: true });\n await Promise.all(fetchedData.map(async (item) => {\n const buffer = await item.blob;\n if (buffer === undefined) {\n console.error(`ERROR: Unable to get data for blob ${item.blobId}`);\n return;\n }\n\n if (!isFetchedTree(item)) {\n // Just write the data as is.\n fs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));\n\n // we assume that the buffer is utf8 here, which currently is true for\n // all of our snapshot blobs. It doesn't necessary be true in the future\n let decoded = bufferToString(buffer,\"utf8\");\n try {\n if (!paramActualFormatting) {\n decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);\n }\n } catch (e) {\n }\n fs.writeFileSync(\n `${outDir}/decoded/${item.filename}.json`, decoded);\n } else {\n // Write out same data for tree decoded or not, except for formatting\n const treeString = bufferToString(buffer,\"utf8\");\n fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);\n fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`,\n paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));\n }\n }));\n}\n\nasync function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {\n const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));\n if (!tree) {\n return Promise.reject(new Error(\"Failed to load snapshot tree\"));\n }\n return fetchBlobsFromSnapshotTree(storage, tree);\n}\n\nasync function reportErrors<T>(message: string, res: Promise<T>) {\n try {\n return await res;\n } catch (error) {\n console.error(`Error calling ${message}`);\n throw error;\n }\n}\n\nexport async function fluidFetchSnapshot(\n documentService?: IDocumentService,\n saveDir?: string,\n ) {\n if (!dumpSnapshotStats && !dumpSnapshotTrees && !dumpSnapshotVersions && saveDir === undefined) {\n return;\n }\n\n // --local mode - do not connect to storage.\n // For now, bail out early.\n // In future, separate download from analyzes parts and allow offline analyzes\n if (!documentService) {\n return;\n }\n\n console.log(\"\\n\");\n\n let storage = await documentService.connectToStorage();\n if (paramUnpackAggregatedBlobs) {\n storage = BlobAggregationStorage.wrap(storage, new TelemetryNullLogger(), false /* allowPacking */);\n }\n\n let version: IVersion | undefined;\n const versions = await reportErrors(\n `getVersions ${latestVersionsId}`,\n storage.getVersions(latestVersionsId, paramNumSnapshotVersions));\n if (dumpSnapshotVersions) {\n console.log(\"Snapshot versions\");\n console.log(versions);\n }\n\n let blobsToDump: IFetchedData[] | undefined;\n if (paramSnapshotVersionIndex !== undefined) {\n version = versions[paramSnapshotVersionIndex];\n if (version === undefined) {\n console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);\n return;\n }\n if (saveDir !== undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n const name = version.id;\n console.log(`Saving snapshot ${name}`);\n await saveSnapshot(name, blobsToDump, saveDir);\n }\n } else {\n version = versions[0];\n if (saveDir !== undefined && versions.length > 0) {\n console.log(\" Name | Date | Size | New Size | Blobs | New Blobs\");\n console.log(\"-\".repeat(86));\n\n // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots\n for (let i = versions.length - 1; i >= 0; i--) {\n const v = versions[i];\n const blobs = await fetchBlobsFromVersion(storage, v);\n blobsToDump = blobs;\n const name = `${i}-${v.id}`;\n const res = await dumpSnapshotTree(name, blobs);\n\n let date = \"\";\n if (v.date) {\n try {\n date = new Date(v.date).toLocaleString();\n } catch (e) {\n date = v.date.replace(\"T\", \" \");\n const index = date.lastIndexOf(\".\");\n if (index > 0) {\n date = `${date.substr(0, index)} Z`;\n }\n }\n }\n date = date.padStart(23);\n const size = formatNumber(res.size).padStart(10);\n const sizeNew = formatNumber(res.sizeNew).padStart(10);\n const blobCount = formatNumber(res.blobCount).padStart(6);\n const blobCountNew = formatNumber(res.blobCountNew).padStart(9);\n\n console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);\n\n await saveSnapshot(name, blobs, saveDir);\n }\n }\n }\n\n if (dumpSnapshotStats || dumpSnapshotTrees) {\n if (version === undefined) {\n console.log(\"No snapshot tree\");\n } else {\n if (blobsToDump === undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n }\n console.log(`\\n\\nSnapshot version ${version.id}`);\n await dumpSnapshotTreeVerbose(version.id, blobsToDump);\n }\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fluid-tools/fetch-tool",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.59.2000-61729",
|
|
4
4
|
"description": "Console tool to fetch Fluid data from relay service",
|
|
5
5
|
"homepage": "https://fluidframework.com",
|
|
6
6
|
"repository": {
|
|
@@ -29,28 +29,28 @@
|
|
|
29
29
|
"tsfmt:fix": "tsfmt --replace"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@fluid-tools/fluidapp-odsp-urlresolver": "0.
|
|
33
|
-
"@fluidframework/azure-client": "0.
|
|
32
|
+
"@fluid-tools/fluidapp-odsp-urlresolver": "0.59.2000-61729",
|
|
33
|
+
"@fluidframework/azure-client": "0.59.2000-61729",
|
|
34
34
|
"@fluidframework/common-utils": "^0.32.1",
|
|
35
|
-
"@fluidframework/container-runtime": "0.
|
|
36
|
-
"@fluidframework/datastore": "0.
|
|
37
|
-
"@fluidframework/driver-definitions": "^0.
|
|
38
|
-
"@fluidframework/driver-utils": "0.
|
|
39
|
-
"@fluidframework/odsp-doclib-utils": "0.
|
|
40
|
-
"@fluidframework/odsp-driver": "0.
|
|
41
|
-
"@fluidframework/odsp-driver-definitions": "0.
|
|
42
|
-
"@fluidframework/odsp-urlresolver": "0.
|
|
43
|
-
"@fluidframework/protocol-definitions": "^0.
|
|
44
|
-
"@fluidframework/routerlicious-driver": "0.
|
|
45
|
-
"@fluidframework/routerlicious-urlresolver": "0.
|
|
46
|
-
"@fluidframework/runtime-definitions": "0.
|
|
47
|
-
"@fluidframework/telemetry-utils": "0.
|
|
48
|
-
"@fluidframework/test-client-utils": "0.
|
|
49
|
-
"@fluidframework/tool-utils": "0.
|
|
35
|
+
"@fluidframework/container-runtime": "0.59.2000-61729",
|
|
36
|
+
"@fluidframework/datastore": "0.59.2000-61729",
|
|
37
|
+
"@fluidframework/driver-definitions": "^0.46.1000-0",
|
|
38
|
+
"@fluidframework/driver-utils": "0.59.2000-61729",
|
|
39
|
+
"@fluidframework/odsp-doclib-utils": "0.59.2000-61729",
|
|
40
|
+
"@fluidframework/odsp-driver": "0.59.2000-61729",
|
|
41
|
+
"@fluidframework/odsp-driver-definitions": "0.59.2000-61729",
|
|
42
|
+
"@fluidframework/odsp-urlresolver": "0.59.2000-61729",
|
|
43
|
+
"@fluidframework/protocol-definitions": "^0.1028.1000-0",
|
|
44
|
+
"@fluidframework/routerlicious-driver": "0.59.2000-61729",
|
|
45
|
+
"@fluidframework/routerlicious-urlresolver": "0.59.2000-61729",
|
|
46
|
+
"@fluidframework/runtime-definitions": "0.59.2000-61729",
|
|
47
|
+
"@fluidframework/telemetry-utils": "0.59.2000-61729",
|
|
48
|
+
"@fluidframework/test-client-utils": "0.59.2000-61729",
|
|
49
|
+
"@fluidframework/tool-utils": "0.59.2000-61729"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@fluidframework/build-common": "^0.23.0",
|
|
53
|
-
"@fluidframework/eslint-config-fluid": "^0.
|
|
53
|
+
"@fluidframework/eslint-config-fluid": "^0.28.1000-61189",
|
|
54
54
|
"@rushstack/eslint-config": "^2.5.1",
|
|
55
55
|
"@types/node": "^14.18.0",
|
|
56
56
|
"@typescript-eslint/eslint-plugin": "~5.9.0",
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import fs from "fs";
|
|
7
7
|
import util from "util";
|
|
8
|
-
import {
|
|
8
|
+
import { bufferToString, stringToBuffer, TelemetryNullLogger } from "@fluidframework/common-utils";
|
|
9
9
|
import {
|
|
10
10
|
IDocumentService,
|
|
11
11
|
IDocumentStorageService,
|
|
@@ -114,11 +114,9 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
114
114
|
storage: IDocumentStorageService,
|
|
115
115
|
tree: ISnapshotTree,
|
|
116
116
|
prefix: string = "/",
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const commit = !perCommitBlobIdMap;
|
|
121
|
-
if (commit && dumpSnapshotTrees) {
|
|
117
|
+
parentBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {
|
|
118
|
+
const isTopLevel = !parentBlobIdMap;
|
|
119
|
+
if (isTopLevel && dumpSnapshotTrees) {
|
|
122
120
|
console.log(tree);
|
|
123
121
|
}
|
|
124
122
|
|
|
@@ -128,42 +126,16 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
128
126
|
}
|
|
129
127
|
|
|
130
128
|
// Create the tree info before fetching blobs (which will modify it)
|
|
131
|
-
let
|
|
132
|
-
if (
|
|
133
|
-
|
|
129
|
+
let topLevelBlob: IFetchedTree | undefined;
|
|
130
|
+
if (isTopLevel) {
|
|
131
|
+
topLevelBlob = createTreeBlob(tree, prefix, false);
|
|
134
132
|
}
|
|
135
133
|
|
|
136
|
-
const blobIdMap =
|
|
134
|
+
const blobIdMap = parentBlobIdMap ?? new Map<string, number>();
|
|
137
135
|
let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);
|
|
138
136
|
|
|
139
|
-
for (const dataStore of Object.keys(tree.commits)) {
|
|
140
|
-
const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);
|
|
141
|
-
if (dataStoreVersions.length !== 1) {
|
|
142
|
-
console.error(`ERROR: Unable to get versions for ${dataStore}`);
|
|
143
|
-
continue;
|
|
144
|
-
}
|
|
145
|
-
const dataStoreSnapShotTree = await reportErrors(
|
|
146
|
-
`getSnapshotTree ${dataStoreVersions[0].id}`,
|
|
147
|
-
storage.getSnapshotTree(dataStoreVersions[0]));
|
|
148
|
-
if (dataStoreSnapShotTree === null) {
|
|
149
|
-
// eslint-disable-next-line max-len
|
|
150
|
-
console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);
|
|
151
|
-
continue;
|
|
152
|
-
}
|
|
153
|
-
assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore],
|
|
154
|
-
0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);
|
|
155
|
-
assert(tree.commits[dataStore] === dataStoreVersions[0].id,
|
|
156
|
-
0x1c0 /* "Mismatch between commit id and fetched tree id" */);
|
|
157
|
-
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(
|
|
158
|
-
storage,
|
|
159
|
-
dataStoreSnapShotTree,
|
|
160
|
-
`${prefix}[${dataStore}]/`);
|
|
161
|
-
result = result.concat(dataStoreBlobs);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
137
|
for (const subtreeId of Object.keys(tree.trees)) {
|
|
165
138
|
const subtree = tree.trees[subtreeId];
|
|
166
|
-
assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* "Unexpected subtree properties" */);
|
|
167
139
|
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(
|
|
168
140
|
storage,
|
|
169
141
|
subtree,
|
|
@@ -171,8 +143,8 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
171
143
|
result = result.concat(dataStoreBlobs);
|
|
172
144
|
}
|
|
173
145
|
|
|
174
|
-
if (
|
|
175
|
-
result.push(
|
|
146
|
+
if (topLevelBlob) {
|
|
147
|
+
result.push(topLevelBlob);
|
|
176
148
|
result.push(createTreeBlob(tree, prefix, true));
|
|
177
149
|
}
|
|
178
150
|
return result;
|