@fluidframework/driver-utils 2.0.0-internal.3.1.0 → 2.0.0-internal.3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/dist/blobAggregationStorage.d.ts +13 -0
  2. package/dist/blobAggregationStorage.d.ts.map +1 -1
  3. package/dist/blobAggregationStorage.js +11 -7
  4. package/dist/blobAggregationStorage.js.map +1 -1
  5. package/dist/packageVersion.d.ts +1 -1
  6. package/dist/packageVersion.js +1 -1
  7. package/dist/packageVersion.js.map +1 -1
  8. package/lib/blobAggregationStorage.d.ts +13 -0
  9. package/lib/blobAggregationStorage.d.ts.map +1 -1
  10. package/lib/blobAggregationStorage.js +11 -7
  11. package/lib/blobAggregationStorage.js.map +1 -1
  12. package/lib/packageVersion.d.ts +1 -1
  13. package/lib/packageVersion.js +1 -1
  14. package/lib/packageVersion.js.map +1 -1
  15. package/package.json +107 -129
  16. package/src/blobAggregationStorage.ts +11 -7
  17. package/src/packageVersion.ts +1 -1
package/dist/blobAggregationStorage.d.ts CHANGED
@@ -5,6 +5,12 @@
5
5
  import { FetchSource, IDocumentStorageService, IDocumentStorageServicePolicies, ISummaryContext } from "@fluidframework/driver-definitions";
6
6
  import { ICreateBlobResponse, ISnapshotTree, ISummaryHandle, ISummaryTree, IVersion } from "@fluidframework/protocol-definitions";
7
7
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
8
+ /**
9
+ * Base class that deals with unpacking snapshots (in place) containing aggregated blobs
10
+ * It relies on abstract methods for reads and storing unpacked blobs.
11
+ *
12
+ * @deprecated This class was introduced experimentally and is no longer used.
13
+ */
8
14
  export declare abstract class SnapshotExtractor {
9
15
  protected readonly aggregatedBlobName = "__big";
10
16
  protected readonly virtualIdPrefix = "__";
@@ -14,6 +20,13 @@ export declare abstract class SnapshotExtractor {
14
20
  abstract setBlob(id: string, tree: ISnapshotTree, content: string): any;
15
21
  unpackSnapshotCore(snapshot: ISnapshotTree, level?: number): Promise<void>;
16
22
  }
23
+ /**
24
+ * Snapshot packer and extractor.
25
+ * When summary is written it will find and aggregate small blobs into bigger blobs
26
+ * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.
27
+ *
28
+ * @deprecated This class was introduced experimentally and is no longer used.
29
+ */
17
30
  export declare class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {
18
31
  private readonly storage;
19
32
  private readonly logger;
package/dist/blobAggregationStorage.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"blobAggregationStorage.d.ts","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,WAAW,EACX,uBAAuB,EACvB,+BAA+B,EAC/B,eAAe,EACf,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACN,mBAAmB,EACnB,aAAa,EACb,cAAc,EACd,YAAY,EACZ,QAAQ,EAER,MAAM,sCAAsC,CAAC;AAS9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AA0CtE,8BAAsB,iBAAiB;IACtC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,WAAW;IAChD,SAAS,CAAC,QAAQ,CAAC,eAAe,QAAQ;IAG1C,SAAS,CAAC,gBAAgB,SAAK;IAC/B,SAAS,CAAC,gBAAgB;IAI1B,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAC3E,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAEpD,kBAAkB,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAsClF;AAoCD,qBAAa,sBAAuB,SAAQ,iBAAkB,YAAW,uBAAuB;IAiE9F,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IA1DjC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,QAAQ;IAE9C,SAAS,CAAC,iBAAiB,UAAS;IAEpC,SAAS,CAAC,YAAY,+BAAsC;IAE5D,MAAM,CAAC,IAAI,CACV,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,CAAC,EAAE,OAAO,EACtB,YAAY,SAAI;WAsBJ,cAAc,CAAC,QAAQ,EAAE,aAAa;IAKnD,IAAW,QAAQ,IAAI,+BAA+B,GAAG,SAAS,CAKjE;IAEY,cAAc,CAAC,QAAQ,EAAE,aAAa;IASnD,SAAS,aACS,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,EAAE,OAAO,EACrB,YAAY,EAAE,MAAM,EACpB,cAAc,CAAC,oBAAQ;IAKlC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAIlD,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAO/E,IAAW,aAAa,WAEvB;IACY,WAAW,CACvB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,WAAW;IAKb,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;IAM9D,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAI/D,eAAe,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAQlE,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAiB9C,wBAAwB,CACpC,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,eAAe,GACtB,OAAO,CAAC,MAAM,CAAC;YAiBJ,kBAAkB;IAkHhC,SAAS,CAAC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;CAG9C"}
1
+ {"version":3,"file":"blobAggregationStorage.d.ts","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,WAAW,EACX,uBAAuB,EACvB,+BAA+B,EAC/B,eAAe,EACf,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACN,mBAAmB,EACnB,aAAa,EACb,cAAc,EACd,YAAY,EACZ,QAAQ,EAER,MAAM,sCAAsC,CAAC;AAS9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAsCtE;;;;;GAKG;AACH,8BAAsB,iBAAiB;IACtC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,WAAW;IAChD,SAAS,CAAC,QAAQ,CAAC,eAAe,QAAQ;IAG1C,SAAS,CAAC,gBAAgB,SAAK;IAC/B,SAAS,CAAC,gBAAgB;IAI1B,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAC3E,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAEpD,kBAAkB,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAsClF;AA+BD;;;;;;GAMG;AACH,qBAAa,sBAAuB,SAAQ,iBAAkB,YAAW,uBAAuB;IAiE9F,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IA1DjC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,QAAQ;IAE9C,SAAS,CAAC,iBAAiB,UAAS;IAEpC,SAAS,CAAC,YAAY,+BAAsC;IAE5D,MAAM,CAAC,IAAI,CACV,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,CAAC,EAAE,OAAO,EACtB,YAAY,SAAI;WAsBJ,cAAc,CAAC,QAAQ,EAAE,aAAa;IAKnD,IAAW,QAAQ,IAAI,+BAA+B,GAAG,SAAS,CAKjE;IAEY,cAAc,CAAC,QAAQ,EAAE,aAAa;IASnD,SAAS,aACS,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,EAAE,OAAO,EACrB,YAAY,EAAE,MAAM,EACpB,cAAc,CAAC,oBAAQ;IAKlC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAIlD,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAO/E,IAAW,aAAa,WAEvB;IACY,WAAW,CACvB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,WAAW;IAKb,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;IAM9D,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAI/D,eAAe,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAQlE,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAiB9C,wBAAwB,CACpC,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,eAAe,GACtB,OAAO,CAAC,MAAM,CAAC;YAiBJ,kBAAkB;IAkHhC,SAAS,CAAC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;CAG9C"}
package/dist/blobAggregationStorage.js CHANGED
@@ -40,9 +40,11 @@ class BlobAggregator {
40
40
  return JSON.parse(data);
41
41
  }
42
42
  }
43
- /*
43
+ /**
44
44
  * Base class that deals with unpacking snapshots (in place) containing aggregated blobs
45
45
  * It relies on abstract methods for reads and storing unpacked blobs.
46
+ *
47
+ * @deprecated This class was introduced experimentally and is no longer used.
46
48
  */
47
49
  class SnapshotExtractor {
48
50
  constructor() {
@@ -114,10 +116,12 @@ class SnapshotExtractorInPlace extends SnapshotExtractor {
114
116
  tree.blobs[id] = (0, common_utils_1.fromUtf8ToBase64)(content);
115
117
  }
116
118
  }
117
- /*
119
+ /**
118
120
  * Snapshot packer and extractor.
119
121
  * When summary is written it will find and aggregate small blobs into bigger blobs
120
122
  * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.
123
+ *
124
+ * @deprecated This class was introduced experimentally and is no longer used.
121
125
  */
122
126
  class BlobAggregationStorage extends SnapshotExtractor {
123
127
  constructor(storage, logger, allowPacking, packingLevel, blobCutOffSize) {
@@ -214,12 +218,12 @@ class BlobAggregationStorage extends SnapshotExtractor {
214
218
  // - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)
215
219
  // summary produced by data stores.
216
220
  // These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated
217
- // blob Or contain aggregated blob that stays relevant for that sub-tree)
221
+ // blob or contain aggregated blob that stays relevant for that sub-tree)
218
222
  // Note:
219
- // From perf perspective, it makes sense to place aggregated blobs one level up in the tree not to create extra
223
+ // From perf perspective, it makes sense to place aggregated blobs one level up in the tree to not create extra
220
224
  // tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at
221
225
  // incremental summary time - we would need to understand handles and parse them. In current design we can skip
222
- // that step because if data store is reused, the hole sub-tree is reused included aggregated blob embedded into it
226
+ // that step because if data store is reused, the whole sub-tree is reused including aggregated blob embedded into it
223
227
  // and that means we can do nothing and be correct!
224
228
  async compressSmallBlobs(summary, path = "", level = 0, aggregatorArg) {
225
229
  if (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {
@@ -247,10 +251,10 @@ class BlobAggregationStorage extends SnapshotExtractor {
247
251
  case protocol_definitions_1.SummaryType.Tree:
248
252
  // If client created empty tree, keep it as is
249
253
  // Also do not package search blobs - they are part of storage contract
250
- if (obj.tree !== {} && key !== "__search") {
254
+ if (Object.keys(obj).length !== 0 && key !== "__search") {
251
255
  const tree = await this.compressSmallBlobs(obj, newPath, level + 1, aggregator);
252
256
  newSummary.tree[key] = tree;
253
- if (tree.tree === {}) {
257
+ if (Object.keys(obj).length === 0) {
254
258
  // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
255
259
  delete newSummary.tree[key];
256
260
  }
package/dist/blobAggregationStorage.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"blobAggregationStorage.js","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAQH,+EAO8C;AAC9C,+DAOsC;AAEtC,qEAA4E;AAE5E;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAqB,EAAE,QAA4B;IAC3E,IAAI,IAAI,YAAY,UAAU,EAAE;QAC/B,8DAA8D;QAC9D,OAAO,IAAA,iCAAkB,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;KAC1C;IACD,OAAO,IAAA,6BAAc,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,cAAc;IAApB;QACkB,YAAO,GAAuB,EAAE,CAAC;IAiBnD,CAAC;IAfO,OAAO,CAAC,GAAW,EAAE,OAAe;QAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACnC,CAAC;IAEM,wBAAwB;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,OAAO,SAAS,CAAC;SACjB;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAsB;QACjC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAC/C,CAAC;CACD;AAED;;;GAGG;AACH,MAAsB,iBAAiB;IAAvC;QACoB,uBAAkB,GAAG,OAAO,CAAC;QAC7B,oBAAe,GAAG,IAAI,CAAC;QAE1C,gDAAgD;QACtC,qBAAgB,GAAG,CAAC,CAAC;IA8ChC,CAAC;IA7CU,gBAAgB;QACzB,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5D,CAAC;IAKM,KAAK,CAAC,kBAAkB,CAAC,QAAuB,EAAE,KAAK,GAAG,CAAC;QACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;SAC9C;QAED,4EAA4E;QAC5E,2CAA2C;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;gBAC7C,SAAS;aACT;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,MAAM,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAClD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACtD,MAAM,EAAE,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACnC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;oBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,OAAO,GAAG,QAAQ,CAAC;oBACvB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;wBAC/D,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE;4BACzC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;yBAClD;wBACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;qBACjC;oBACD,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACjD,IAAA,qBAAM,EACL,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EACrC,KAAK,CAAC,2BAA2B,CACjC,CAAC;oBACF,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;iBAC7B;gBACD,gEAAgE;gBAChE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;aAC/C;SACD;IACF,CAAC;CACD;AAnDD,8CAmDC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,wBAAyB,SAAQ,iBAAiB;IAChD,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAA,qBAAM,EAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,IAAA,6BAAc,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,IAAA,qBAAM,EACL,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,SAAS,EAC5B,KAAK,CAAC,kDAAkD,CACxD,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,IAAA,+BAAgB,EAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;CACD;AAED;;;;GAIG;AACH,MAAa,sBAAuB,SAAQ,iBAAiB;IAgE5D,YACkB,OAAgC,EAChC,MAAwB,EACxB,YAAqB,EACrB,YAAoB,EACpB,cAAuB;QAExC,KAAK,EAAE,CAAC;QANS,YAAO,GAAP,OAAO,CAAyB;QAChC,WAAM,GAAN,MAAM,CAAkB;QACxB,iBAAY,GAAZ,YAAY,CAAS;QACrB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,mBAAc,GAAd,cAAc,CAAS;QAxD/B,sBAAiB,GAAG,KAAK,CAAC;QAE1B,iBAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;IAyD5D,CAAC;IAvDD,MAAM,CAAC,IAAI,CACV,OAAgC,EAChC,MAAwB,EACxB,YAAsB,EACtB,YAAY,GAAG,CAAC;;QAEhB,IAAI,OAAO,YAAY,sBAAsB,EAAE;YAC9C,OAAO,OAAO,CAAC;SACf;QACD,MAAM,EAAE,GAAG,IAAA,2CAAyB,EAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,kBAAkB,GACvB,MAAA,MAAA,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,mCAAI,YAAY,mCAAI,KAAK,CAAC;QAEtE,kFAAkF;QAClF,4FAA4F;QAC5F,wDAAwD;QACxD,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,QAAQ,0CAAE,WAAW,CAAC;QAClD,OAAO,IAAI,sBAAsB,CAChC,OAAO,EACP,MAAM,EACN,kBAAkB,EAClB,YAAY,EACZ,WAAW,CACX,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,MAAM,SAAS,GAAG,IAAI,wBAAwB,EAAE,CAAC;QACjD,MAAM,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED,IAAW,QAAQ;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACvC,IAAI,QAAQ,EAAE;YACb,uCAAY,QAAQ,KAAE,WAAW,EAAE,SAAS,IAAG;SAC/C;IACF,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,8FAA8F;QAC9F,qCAAqC;QACrC,6DAA6D;QAE7D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAYM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,IAAA,6BAAc,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,IAAW,aAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACnC,CAAC;IACM,KAAK,CAAC,WAAW,CACvB,SAAwB,EACxB,KAAa,EACb,YAAqB,EACrB,WAAyB;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAC9E,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,MAAsB;QAClD,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,wFAAwF;IACxF,yBAAyB;IAClB,KAAK,CAAC,UAAU,CAAC,IAAqB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAkB;QAC9C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,IAAI,EAAE;YACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,EAAU;QAC/B,IAAI,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;SACjC;QACD,iEAAiE;QACjE,6FAA6F;QAC7F,oGAAoG;QACpG,uCAAuC;QAEvC,mGAAmG;QACnG,eAAe;QACf,IAAA,qBAAM,EAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvC,IAAA,qBAAM,EAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,wBAAwB,CACpC,OAAqB,EACrB,OAAwB;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,qCAAqC;IACrC,8DAA8D;IAC9D,8GAA8G;IAC9G,mCAAmC;IACnC,gHAAgH;IAChH,yEAAyE;IACzE,QAAQ;IACR,+GAA+G;IAC/G,8GAA8G;IAC9G,+GAA+G;IAC/G,mHAAmH;IACnH,mDAAmD;IAC3C,KAAK,CAAC,kBAAkB,CAC/B,OAAqB,EACrB,IAAI,GAAG,EAAE,EACT,KAAK,GAAG,CAAC,EACT,aAA8B;QAE9B,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;YACjE,OAAO,OAAO,CAAC;SACf;QAED,IAAI,cAAc,GAAY,KAAK,CAAC;QAEpC,IAAI,UAAU,GAAG,aAAa,CAAC;QAC/B,+EAA+E;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE;YACrD,IAAA,qBAAM,EAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAC1E,IAAA,qBAAM,EACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;YACF,UAAU,GAAG,IAAI,cAAc,EAAE,CAAC;YAClC,cAAc,GAAG,IAAI,CAAC;SACtB;aAAM;YACN,IAAA,qBAAM,EACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;SACF;QAED,MAAM,UAAU,qBAAsB,OAAO,CAAE,CAAC;QAChD,UAAU,CAAC,IAAI,qBAAQ,UAAU,CAAC,IAAI,CAAE,CAAC;QACzC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,oEAAoE;YACpE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;YACxD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,kCAAW,CAAC,IAAI;oBACpB,8CAA8C;oBAC9C,uEAAuE;oBACvE,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE,IAAI,GAAG,KAAK,UAAU,EAAE;wBAC1C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CACzC,GAAG,EACH,OAAO,EACP,KAAK,GAAG,CAAC,EACT,UAAU,CACV,CAAC;wBACF,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;wBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE;4BACrB,gEAAgE;4BAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;yBAC5B;qBACD;oBACD,MAAM;gBACP,KAAK,kCAAW,CAAC,IAAI;oBACpB,IACC,UAAU;wBACV,OAAO,GAAG,CAAC,OAAO,IAAI,QAAQ;wBAC9B,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EACvC;wBACD,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;wBACzC,gEAAgE;wBAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAC5B;oBACD,MAAM;gBACP,KAAK,kCAAW,CAAC,MAAM,CAAC,CAAC;oBACxB,oBAAoB;oBACpB,yBAAyB;oBACzB,2EAA2E;oBAC3E,wFAAwF;oBACxF,IAAI,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC;oBAC5B,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;wBAC/B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;qBAClC;oBACD,2EAA2E;oBAC3E,IAAA,qBAAM,EAAC,KAAK,KAAK,CAAC,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBAC7D,IAAA,qBAAM,EACL,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EACzB,KAAK,CAAC,sDAAsD,CAC5D,CAAC;oBACF,MAAM;iBACN;gBACD,KAAK,kCAAW,CAAC,UAAU;oBAC1B,IAAA,qBAAM,EACL,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,EAC5B,KAAK,CAAC,oCAAoC,CAC1C,CAAC;oBACF,MAAM;gBACP;oBACC,IAAA,8BAAe,EAAC,GAAG,EAAE,iBAAkB,GAAW,CAAC,IAAI,EAAE,CAAC,CAAC;aAC5D;SACD;QAED,IAAA,qBAAM,EACL,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,SAAS,EACtD,KAAK,CAAC,gCAAgC,CACtC,CAAC;QACF,IAAI,cAAc,EAAE;YACnB,yFAAyF;YACzF,4FAA4F;YAC5F,qGAAqG;YACrG,iEAAiE;YACjE,+CAA+C;YAC/C,IAAA,qBAAM,EAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,UAAU,CAAC,wBAAwB,EAAE,CAAC;YACtD,IAAI,OAAO,KAAK,SAAS,EAAE;gBAC1B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG;oBAC1C,IAAI,EAAE,kCAAW,CAAC,IAAI;oBACtB,OAAO;iBACP,CAAC;aACF;SACD;QACD,OAAO,UAAU,CAAC;IACnB,CAAC;IAES,eAAe,CAAC,EAAU;QACnC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;;AA5QF,wDA6QC;AA5QA,6FAA6F;AAC7F,8BAA8B;AAC9B,uGAAuG;AACvG,yGAAyG;AACzG,4FAA4F;AAC5F,qGAAqG;AACrG,oGAAoG;AACpG,mCAAmC;AACnC,yGAAyG;AACzG,mFAAmF;AACnE,6CAAsB,GAAG,IAAI,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport {\n\tFetchSource,\n\tIDocumentStorageService,\n\tIDocumentStorageServicePolicies,\n\tISummaryContext,\n} from \"@fluidframework/driver-definitions\";\nimport {\n\tICreateBlobResponse,\n\tISnapshotTree,\n\tISummaryHandle,\n\tISummaryTree,\n\tIVersion,\n\tSummaryType,\n} from \"@fluidframework/protocol-definitions\";\nimport {\n\tassert,\n\tbufferToString,\n\tstringToBuffer,\n\tunreachableCase,\n\tfromUtf8ToBase64,\n\tUint8ArrayToString,\n} from \"@fluidframework/common-utils\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { loggerToMonitoringContext } from \"@fluidframework/telemetry-utils\";\n\n/*\n * Work around for bufferToString having a bug - it can't consume IsoBuffer!\n * To be removed once bufferToString is fixed!\n */\nfunction bufferToString2(blob: ArrayBufferLike, encoding: \"utf-8\" | \"base64\"): string {\n\tif (blob instanceof Uint8Array) {\n\t\t// IsoBuffer does not have ctor, so it's not in proto chain :(\n\t\treturn Uint8ArrayToString(blob, encoding);\n\t}\n\treturn bufferToString(blob, encoding);\n}\n\n/**\n * Class responsible for aggregating smaller blobs into one and unpacking it later on.\n */\nclass BlobAggregator {\n\tprivate readonly content: [string, string][] = [];\n\n\tpublic addBlob(key: string, content: string) {\n\t\tthis.content.push([key, content]);\n\t}\n\n\tpublic getAggregatedBlobContent() {\n\t\tif (this.content.length === 0) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn JSON.stringify(this.content);\n\t}\n\n\tstatic load(input: ArrayBufferLike) {\n\t\tconst data = bufferToString2(input, \"utf-8\");\n\t\treturn JSON.parse(data) as [string, string][];\n\t}\n}\n\n/*\n * Base class that deals with unpacking snapshots (in place) containing aggregated blobs\n * It relies on abstract methods for reads and storing unpacked blobs.\n */\nexport abstract class SnapshotExtractor {\n\tprotected readonly aggregatedBlobName = \"__big\";\n\tprotected readonly virtualIdPrefix = \"__\";\n\n\t// counter for generation of virtual storage IDs\n\tprotected virtualIdCounter = 0;\n\tprotected getNextVirtualId() {\n\t\treturn `${this.virtualIdPrefix}${++this.virtualIdCounter}`;\n\t}\n\n\tabstract getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike>;\n\tabstract setBlob(id: string, tree: ISnapshotTree, content: string);\n\n\tpublic async unpackSnapshotCore(snapshot: ISnapshotTree, level = 0): Promise<void> {\n\t\tfor (const key of Object.keys(snapshot.trees)) {\n\t\t\tconst obj = snapshot.trees[key];\n\t\t\tawait this.unpackSnapshotCore(obj, level + 1);\n\t\t}\n\n\t\t// For future proof, we will support multiple aggregated blobs with any name\n\t\t// that starts with this.aggregatedBlobName\n\t\tfor (const key of Object.keys(snapshot.blobs)) {\n\t\t\tif (!key.startsWith(this.aggregatedBlobName)) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst blobId = snapshot.blobs[key];\n\t\t\tif (blobId !== undefined) {\n\t\t\t\tconst blob = await this.getBlob(blobId, snapshot);\n\t\t\t\tfor (const [path, value] of BlobAggregator.load(blob)) {\n\t\t\t\t\tconst id = this.getNextVirtualId();\n\t\t\t\t\tthis.setBlob(id, snapshot, value);\n\t\t\t\t\tconst pathSplit = path.split(\"/\");\n\t\t\t\t\tlet subTree = snapshot;\n\t\t\t\t\tfor (const subPath of pathSplit.slice(0, pathSplit.length - 1)) {\n\t\t\t\t\t\tif (subTree.trees[subPath] === undefined) {\n\t\t\t\t\t\t\tsubTree.trees[subPath] = { blobs: {}, trees: {} };\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsubTree = subTree.trees[subPath];\n\t\t\t\t\t}\n\t\t\t\t\tconst blobName = pathSplit[pathSplit.length - 1];\n\t\t\t\t\tassert(\n\t\t\t\t\t\tsubTree.blobs[blobName] === undefined,\n\t\t\t\t\t\t0x0f6 /* \"real blob ID exists\" */,\n\t\t\t\t\t);\n\t\t\t\t\tsubTree.blobs[blobName] = id;\n\t\t\t\t}\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\tdelete snapshot.blobs[this.aggregatedBlobName];\n\t\t\t}\n\t\t}\n\t}\n}\n\n/*\n * Snapshot extractor class that works in place, i.e. patches snapshot that has\n * blob content in ISnapshotTree.blobs itself, not in storage.\n * As result, it implements reading and writing of blobs to/from snapshot itself.\n * It follows existing pattern that mixes concerns - ISnapshotTree.blobs is used for two\n * purposes:\n * 1. map path name to blob ID\n * 2. map blob ID to blob content\n * #2 is what storage (IDocumentStorageService) is for, but in places where we do not have it\n * (like loading serialized earlier draft content), blob content is put directly into snapshot.\n * Ideally this should be fixed by using BlobCacheStorageService or something similar and\n * fixing existing flows to allow switching of storage.\n */\nclass SnapshotExtractorInPlace extends SnapshotExtractor {\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\tconst blob = tree.blobs[id];\n\t\tassert(blob !== undefined, 0x0f7 /* \"aggregate blob missing\" */);\n\t\treturn stringToBuffer(blob, \"base64\");\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tassert(\n\t\t\ttree.blobs[id] === undefined,\n\t\t\t0x0f8 /* \"blob from aggregate blob exists on its own\" */,\n\t\t);\n\t\ttree.blobs[id] = fromUtf8ToBase64(content);\n\t}\n}\n\n/*\n * Snapshot packer and extractor.\n * When summary is written it will find and aggregate small blobs into bigger blobs\n * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.\n */\nexport class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {\n\t// Tells data store if it can use incremental summary (i.e. reuse DDSes from previous summary\n\t// when only one DDS changed).\n\t// The answer has to be know long before we enable actual packing. The reason for the is the following:\n\t// A the moment when we enable packing, we should assume that all clients out there wil already have bits\n\t// that can unpack properly (i.e. enough time passed since we deployed bits that can unpack)\n\t// But we can still have clients where some of them already pack, and some do not. If one summary was\n\t// using packing, then it relies on non-incremental summaries going forward, even if next client who\n\t// produced summary is not packing!\n\t// This can have slight improvement by enabling it per file (based on \"did summary we loaded from contain\n\t// aggregated blobs\"), but that's harder to make reliable, so going for simplicity.\n\tstatic readonly fullDataStoreSummaries = true;\n\n\tprotected loadedFromSummary = false;\n\n\tprotected virtualBlobs = new Map<string, ArrayBufferLike>();\n\n\tstatic wrap(\n\t\tstorage: IDocumentStorageService,\n\t\tlogger: ITelemetryLogger,\n\t\tallowPacking?: boolean,\n\t\tpackingLevel = 2,\n\t) {\n\t\tif (storage instanceof BlobAggregationStorage) {\n\t\t\treturn storage;\n\t\t}\n\t\tconst mc = loggerToMonitoringContext(logger);\n\t\tconst realAllowPackaging =\n\t\t\tmc.config.getBoolean(\"FluidAggregateBlobs\") ?? allowPacking ?? false;\n\n\t\t// Always create BlobAggregationStorage even if storage is not asking for packing.\n\t\t// This is mostly to avoid cases where future changes in policy would result in inability to\n\t\t// load old files that were created with aggregation on.\n\t\tconst minBlobSize = storage.policies?.minBlobSize;\n\t\treturn new BlobAggregationStorage(\n\t\t\tstorage,\n\t\t\tlogger,\n\t\t\trealAllowPackaging,\n\t\t\tpackingLevel,\n\t\t\tminBlobSize,\n\t\t);\n\t}\n\n\tstatic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\tconst converter = new SnapshotExtractorInPlace();\n\t\tawait converter.unpackSnapshotCore(snapshot);\n\t}\n\n\tpublic get policies(): IDocumentStorageServicePolicies | undefined {\n\t\tconst policies = this.storage.policies;\n\t\tif (policies) {\n\t\t\treturn { ...policies, minBlobSize: undefined };\n\t\t}\n\t}\n\n\tpublic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\t// SummarizerNodeWithGC.refreshLatestSummary can call it when this.loadedFromSummary === false\n\t\t// (I assumed after file was created)\n\t\t// assert(!this.loadedFromSummary, \"unpack without summary\");\n\n\t\tthis.loadedFromSummary = true;\n\t\tawait this.unpackSnapshotCore(snapshot);\n\t}\n\n\tprotected constructor(\n\t\tprivate readonly storage: IDocumentStorageService,\n\t\tprivate readonly logger: ITelemetryLogger,\n\t\tprivate readonly allowPacking: boolean,\n\t\tprivate readonly packingLevel: number,\n\t\tprivate readonly blobCutOffSize?: number,\n\t) {\n\t\tsuper();\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tthis.virtualBlobs.set(id, stringToBuffer(content, \"utf-8\"));\n\t}\n\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\treturn this.readBlob(id).catch((error) => {\n\t\t\tthis.logger.sendErrorEvent({ eventName: \"BlobDedupNoAggregateBlob\" }, error);\n\t\t\tthrow error;\n\t\t});\n\t}\n\n\tpublic get repositoryUrl() {\n\t\treturn this.storage.repositoryUrl;\n\t}\n\tpublic async getVersions(\n\t\tversionId: string | null,\n\t\tcount: number,\n\t\tscenarioName?: string,\n\t\tfetchSource?: FetchSource,\n\t) {\n\t\treturn this.storage.getVersions(versionId, count, scenarioName, fetchSource);\n\t}\n\n\tpublic async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {\n\t\tthrow new Error(\"NYI\");\n\t}\n\n\t// for now we are not optimizing these blobs, with assumption that this API is used only\n\t// for big blobs (images)\n\tpublic async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {\n\t\treturn this.storage.createBlob(file);\n\t}\n\n\tpublic async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {\n\t\tconst tree = await this.storage.getSnapshotTree(version);\n\t\tif (tree) {\n\t\t\tawait this.unpackSnapshot(tree);\n\t\t}\n\t\treturn tree;\n\t}\n\n\tpublic async readBlob(id: string): Promise<ArrayBufferLike> {\n\t\tif (this.isRealStorageId(id)) {\n\t\t\treturn this.storage.readBlob(id);\n\t\t}\n\t\t// We support only reading blobs from the summary we loaded from.\n\t\t// This may need to be extended to any general summary in the future as runtime usage pattern\n\t\t// of storage changes (for example, data stores start to load from recent summary, not from original\n\t\t// summary whole container loaded from)\n\n\t\t// are there other ways we can get here? createFile is one flow, but we should not be reading blobs\n\t\t// in such flow\n\t\tassert(this.loadedFromSummary, 0x0f9 /* \"never read summary\" */);\n\t\tconst blob = this.virtualBlobs.get(id);\n\t\tassert(blob !== undefined, 0x0fa /* \"virtual blob not found\" */);\n\t\treturn blob;\n\t}\n\n\tpublic async uploadSummaryWithContext(\n\t\tsummary: ISummaryTree,\n\t\tcontext: ISummaryContext,\n\t): Promise<string> {\n\t\tconst summaryNew = this.allowPacking ? await this.compressSmallBlobs(summary) : summary;\n\t\treturn this.storage.uploadSummaryWithContext(summaryNew, context);\n\t}\n\n\t// For simplification, we assume that\n\t// - blob aggregation is done at data store level only for now\n\t// - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)\n\t// summary produced by data stores.\n\t// These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated\n\t// blob Or contain aggregated blob that stays relevant for that sub-tree)\n\t// Note:\n\t// From perf perspective, it makes sense to place aggregated blobs one level up in the tree not to create extra\n\t// tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at\n\t// incremental summary time - we would need to understand handles and parse them. In current design we can skip\n\t// that step because if data store is reused, the hole sub-tree is reused included aggregated blob embedded into it\n\t// and that means we can do nothing and be correct!\n\tprivate async compressSmallBlobs(\n\t\tsummary: ISummaryTree,\n\t\tpath = \"\",\n\t\tlevel = 0,\n\t\taggregatorArg?: BlobAggregator,\n\t): Promise<ISummaryTree> {\n\t\tif (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {\n\t\t\treturn summary;\n\t\t}\n\n\t\tlet shouldCompress: boolean = false;\n\n\t\tlet aggregator = aggregatorArg;\n\t\t// checking if this is a dataStore tree, since we only pack at data store level\n\t\tif (Object.keys(summary.tree).includes(\".component\")) {\n\t\t\tassert(aggregator === undefined, 0x0fb /* \"logic err with aggregator\" */);\n\t\t\tassert(\n\t\t\t\tlevel === this.packingLevel,\n\t\t\t\t0x23b /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t\taggregator = new BlobAggregator();\n\t\t\tshouldCompress = true;\n\t\t} else {\n\t\t\tassert(\n\t\t\t\tlevel !== this.packingLevel,\n\t\t\t\t0x23c /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t}\n\n\t\tconst newSummary: ISummaryTree = { ...summary };\n\t\tnewSummary.tree = { ...newSummary.tree };\n\t\tfor (const key of Object.keys(summary.tree)) {\n\t\t\tconst obj = summary.tree[key];\n\t\t\t// Get path relative to root of data store (where we do aggregation)\n\t\t\tconst newPath = shouldCompress ? key : `${path}/${key}`;\n\t\t\tswitch (obj.type) {\n\t\t\t\tcase SummaryType.Tree:\n\t\t\t\t\t// If client created empty tree, keep it as is\n\t\t\t\t\t// Also do not package search blobs - they are part of storage contract\n\t\t\t\t\tif (obj.tree !== {} && key !== \"__search\") {\n\t\t\t\t\t\tconst tree = await this.compressSmallBlobs(\n\t\t\t\t\t\t\tobj,\n\t\t\t\t\t\t\tnewPath,\n\t\t\t\t\t\t\tlevel + 1,\n\t\t\t\t\t\t\taggregator,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tnewSummary.tree[key] = tree;\n\t\t\t\t\t\tif (tree.tree === {}) {\n\t\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Blob:\n\t\t\t\t\tif (\n\t\t\t\t\t\taggregator &&\n\t\t\t\t\t\ttypeof obj.content == \"string\" &&\n\t\t\t\t\t\tobj.content.length < this.blobCutOffSize\n\t\t\t\t\t) {\n\t\t\t\t\t\taggregator.addBlob(newPath, obj.content);\n\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Handle: {\n\t\t\t\t\t// Would be nice to:\n\t\t\t\t\t// Trees: expand the tree\n\t\t\t\t\t// Blobs: parse handle and ensure it points to real blob, not virtual blob.\n\t\t\t\t\t// We can avoid it for now given data store is the granularity of incremental summaries.\n\t\t\t\t\tlet handlePath = obj.handle;\n\t\t\t\t\tif (handlePath.startsWith(\"/\")) {\n\t\t\t\t\t\thandlePath = handlePath.substr(1);\n\t\t\t\t\t}\n\t\t\t\t\t// Ensure only whole data stores can be reused, no reusing at deeper level!\n\t\t\t\t\tassert(level === 0, 0x0fc /* \"tree reuse at lower level\" */);\n\t\t\t\t\tassert(\n\t\t\t\t\t\t!handlePath.includes(\"/\"),\n\t\t\t\t\t\t0x0fd /* \"data stores are writing incremental summaries!\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase SummaryType.Attachment:\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.isRealStorageId(obj.id),\n\t\t\t\t\t\t0x0fe /* \"attachment is aggregate blob\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\tdefault:\n\t\t\t\t\tunreachableCase(obj, `Unknown type: ${(obj as any).type}`);\n\t\t\t}\n\t\t}\n\n\t\tassert(\n\t\t\tnewSummary.tree[this.aggregatedBlobName] === undefined,\n\t\t\t0x0ff /* \"duplicate aggregate blob\" */,\n\t\t);\n\t\tif (shouldCompress) {\n\t\t\t// Note: It would be great to add code here to unpack aggregate blob back to normal blobs\n\t\t\t// If only one blob made it into aggregate. Currently that does not happen as we always have\n\t\t\t// at least one .component blob and at least one DDS that has .attributes blob, so it's not an issue.\n\t\t\t// But it's possible that in future that would be great addition!\n\t\t\t// Good news - it's backward compatible change.\n\t\t\tassert(aggregator !== undefined, 0x100 /* \"logic error\" */);\n\t\t\tconst content = aggregator.getAggregatedBlobContent();\n\t\t\tif (content !== undefined) {\n\t\t\t\tnewSummary.tree[this.aggregatedBlobName] = {\n\t\t\t\t\ttype: SummaryType.Blob,\n\t\t\t\t\tcontent,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t\treturn newSummary;\n\t}\n\n\tprotected isRealStorageId(id: string): boolean {\n\t\treturn !id.startsWith(this.virtualIdPrefix);\n\t}\n}\n"]}
1
+ {"version":3,"file":"blobAggregationStorage.js","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAQH,+EAO8C;AAC9C,+DAOsC;AAEtC,qEAA4E;AAE5E;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAqB,EAAE,QAA4B;IAC3E,IAAI,IAAI,YAAY,UAAU,EAAE;QAC/B,8DAA8D;QAC9D,OAAO,IAAA,iCAAkB,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;KAC1C;IACD,OAAO,IAAA,6BAAc,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,cAAc;IAApB;QACkB,YAAO,GAAuB,EAAE,CAAC;IAiBnD,CAAC;IAfO,OAAO,CAAC,GAAW,EAAE,OAAe;QAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACnC,CAAC;IAEM,wBAAwB;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,OAAO,SAAS,CAAC;SACjB;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAsB;QACjC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAC/C,CAAC;CACD;AAED;;;;;GAKG;AACH,MAAsB,iBAAiB;IAAvC;QACoB,uBAAkB,GAAG,OAAO,CAAC;QAC7B,oBAAe,GAAG,IAAI,CAAC;QAE1C,gDAAgD;QACtC,qBAAgB,GAAG,CAAC,CAAC;IA8ChC,CAAC;IA7CU,gBAAgB;QACzB,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5D,CAAC;IAKM,KAAK,CAAC,kBAAkB,CAAC,QAAuB,EAAE,KAAK,GAAG,CAAC;QACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;SAC9C;QAED,4EAA4E;QAC5E,2CAA2C;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;gBAC7C,SAAS;aACT;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,MAAM,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAClD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACtD,MAAM,EAAE,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACnC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;oBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,OAAO,GAAG,QAAQ,CAAC;oBACvB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;wBAC/D,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE;4BACzC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;yBAClD;wBACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;qBACjC;oBACD,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACjD,IAAA,qBAAM,EACL,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EACrC,KAAK,CAAC,2BAA2B,CACjC,CAAC;oBACF,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;iBAC7B;gBACD,gEAAgE;gBAChE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;aAC/C;SACD;IACF,CAAC;CACD;AAnDD,8CAmDC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,wBAAyB,SAAQ,iBAAiB;IAChD,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAA,qBAAM,EAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,IAAA,6BAAc,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,IAAA,qBAAM,EACL,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,SAAS,EAC5B,KAAK,CAAC,kDAAkD,CACxD,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,IAAA,+BAAgB,EAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;CACD;AAED;;;;;;GAMG;AACH,MAAa,sBAAuB,SAAQ,iBAAiB;IAgE5D,YACkB,OAAgC,EAChC,MAAwB,EACxB,YAAqB,EACrB,YAAoB,EACpB,cAAuB;QAExC,KAAK,EAAE,CAAC;QANS,YAAO,GAAP,OAAO,CAAyB;QAChC,WAAM,GAAN,MAAM,CAAkB;QACxB,iBAAY,GAAZ,YAAY,CAAS;QACrB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,mBAAc,GAAd,cAAc,CAAS;QAxD/B,sBAAiB,GAAG,KAAK,CAAC;QAE1B,iBAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;IAyD5D,CAAC;IAvDD,MAAM,CAAC,IAAI,CACV,OAAgC,EAChC,MAAwB,EACxB,YAAsB,EACtB,YAAY,GAAG,CAAC;;QAEhB,IAAI,OAAO,YAAY,sBAAsB,EAAE;YAC9C,OAAO,OAAO,CAAC;SACf;QACD,MAAM,EAAE,GAAG,IAAA,2CAAyB,EAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,kBAAkB,GACvB,MAAA,MAAA,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,mCAAI,YAAY,mCAAI,KAAK,CAAC;QAEtE,kFAAkF;QAClF,4FAA4F;QAC5F,wDAAwD;QACxD,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,QAAQ,0CAAE,WAAW,CAAC;QAClD,OAAO,IAAI,sBAAsB,CAChC,OAAO,EACP,MAAM,EACN,kBAAkB,EAClB,YAAY,EACZ,WAAW,CACX,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,MAAM,SAAS,GAAG,IAAI,wBAAwB,EAAE,CAAC;QACjD,MAAM,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED,IAAW,QAAQ;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACvC,IAAI,QAAQ,EAAE;YACb,uCAAY,QAAQ,KAAE,WAAW,EAAE,SAAS,IAAG;SAC/C;IACF,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,8FAA8F;QAC9F,qCAAqC;QACrC,6DAA6D;QAE7D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAYM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,IAAA,6BAAc,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,IAAW,aAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACnC,CAAC;IACM,KAAK,CAAC,WAAW,CACvB,SAAwB,EACxB,KAAa,EACb,YAAqB,EACrB,WAAyB;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAC9E,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,MAAsB;QAClD,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,wFAAwF;IACxF,yBAAyB;IAClB,KAAK,CAAC,UAAU,CAAC,IAAqB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAkB;QAC9C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,IAAI,EAAE;YACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,EAAU;QAC/B,IAAI,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;SACjC;QACD,iEAAiE;QACjE,6FAA6F;QAC7F,oGAAoG;QACpG,uCAAuC;QAEvC,mGAAmG;QACnG,eAAe;QACf,IAAA,qBAAM,EAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvC,IAAA,qBAAM,EAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,wBAAwB,CACpC,OAAqB,EACrB,OAAwB;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,qCAAqC;IACrC,8DAA8D;IAC9D,8GAA8G;IAC9G,mCAAmC;IACnC,gHAAgH;IAChH,yEAAyE;IACzE,QAAQ;IACR,+GAA+G;IAC/G,8GAA8G;IAC9G,+GAA+G;IAC/G,qHAAqH;IACrH,mDAAmD;IAC3C,KAAK,CAAC,kBAAkB,CAC/B,OAAqB,EACrB,IAAI,GAAG,EAAE,EACT,KAAK,GAAG,CAAC,EACT,aAA8B;QAE9B,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;YACjE,OAAO,OAAO,CAAC;SACf;QAED,IAAI,cAAc,GAAY,KAAK,CAAC;QAEpC,IAAI,UAAU,GAAG,aAAa,CAAC;QAC/B,+EAA+E;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE;YACrD,IAAA,qBAAM,EAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAC1E,IAAA,qBAAM,EACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;YACF,UAAU,GAAG,IAAI,cAAc,EAAE,CAAC;YAClC,cAAc,GAAG,IAAI,CAAC;SACtB;aAAM;YACN,IAAA,qBAAM,EACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;SACF;QAED,MAAM,UAAU,qBAAsB,OAAO,CAAE,CAAC;QAChD,UAAU,CAAC,IAAI,qBAAQ,UAAU,CAAC,IAAI,CAAE,CAAC;QACzC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,oEAAoE;YACpE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;YACxD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,kCAAW,CAAC,IAAI;oBACpB,8CAA8C;oBAC9C,uEAAuE;oBACvE,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,KAAK,UAAU,EAAE;wBACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CACzC,GAAG,EACH,OAAO,EACP,KAAK,GAAG,CAAC,EACT,UAAU,CACV,CAAC;wBACF,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;wBAC5B,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;4BAClC,gEAAgE;4BAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;yBAC5B;qBACD;oBACD,MAAM;gBACP,KAAK,kCAAW,CAAC,IAAI;oBACpB,IACC,UAAU;wBACV,OAAO,GAAG,CAAC,OAAO,IAAI,QAAQ;wBAC9B,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EACvC;wBACD,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;wBACzC,gEAAgE;wBAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAC5B;oBACD,MAAM;gBACP,KAAK,kCAAW,CAAC,MAAM,CAAC,CAAC;oBACxB,oBAAoB;oBACpB,yBAAyB;oBACzB,2EAA2E;oBAC3E,wFAAwF;oBACxF,IAAI,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC;oBAC5B,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;wBAC/B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;qBAClC;oBACD,2EAA2E;oBAC3E,IAAA,qBAAM,EAAC,KAAK,KAAK,CAAC,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBAC7D,IAAA,qBAAM,EACL,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EACzB,KAAK,CAAC,sDAAsD,CAC5D,CAAC;oBACF,MAAM;iBACN;gBACD,KAAK,kCAAW,CAAC,UAAU;oBAC1B,IAAA,qBAAM,EACL,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,EAC5B,KAAK,CAAC,oCAAoC,CAC1C,CAAC;oBACF,MAAM;gBACP;oBACC,IAAA,8BAAe,EAAC,GAAG,EAAE,iBAAkB,GAAW,CAAC,IAAI,EAAE,CAAC,CAAC;aAC5D;SACD;QAED,IAAA,qBAAM,EACL,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,SAAS,EACtD,KAAK,CAAC,gCAAgC,CACtC,CAAC;QACF,IAAI,cAAc,EAAE;YACnB,yFAAyF;YACzF,4FAA4F;YAC5F,qGAAqG;YACrG,iEAAiE;YACjE,+CAA+C;YAC/C,IAAA,qBAAM,EAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,UAAU,CAAC,wBAAwB,EAAE,CAAC;YACtD,IAAI,OAAO,KAAK,SAAS,EAAE;gBAC1B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG;oBAC1C,IAAI,EAAE,kCAAW,CAAC,IAAI;oBACtB,OAAO;iBACP,CAAC;aACF;SACD;QACD,OAAO,UAAU,CAAC;IACnB,CAAC;IAES,eAAe,CAAC,EAAU;QACnC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;;AA5QF,wDA6QC;AA5QA,6FAA6F;AAC7F,8BAA8B;AAC9B,uGAAuG;AACvG,yGAAyG;AACzG,4FAA4F;AAC5F,qGAAqG;AACrG,oGAAoG;AACpG,mCAAmC;AACnC,yGAAyG;AACzG,mFAAmF;AACnE,6CAAsB,GAAG,IAAI,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport {\n\tFetchSource,\n\tIDocumentStorageService,\n\tIDocumentStorageServicePolicies,\n\tISummaryContext,\n} from \"@fluidframework/driver-definitions\";\nimport {\n\tICreateBlobResponse,\n\tISnapshotTree,\n\tISummaryHandle,\n\tISummaryTree,\n\tIVersion,\n\tSummaryType,\n} from \"@fluidframework/protocol-definitions\";\nimport {\n\tassert,\n\tbufferToString,\n\tstringToBuffer,\n\tunreachableCase,\n\tfromUtf8ToBase64,\n\tUint8ArrayToString,\n} from \"@fluidframework/common-utils\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { loggerToMonitoringContext } from \"@fluidframework/telemetry-utils\";\n\n/*\n * Work around for bufferToString having a bug - it can't consume IsoBuffer!\n * To be removed once bufferToString is fixed!\n */\nfunction bufferToString2(blob: ArrayBufferLike, encoding: \"utf-8\" | \"base64\"): string {\n\tif (blob instanceof Uint8Array) {\n\t\t// IsoBuffer does not have ctor, so it's not in proto chain :(\n\t\treturn Uint8ArrayToString(blob, encoding);\n\t}\n\treturn bufferToString(blob, encoding);\n}\n\n/**\n * Class responsible for aggregating smaller blobs into one and unpacking it later on.\n */\nclass BlobAggregator {\n\tprivate readonly content: [string, string][] = [];\n\n\tpublic addBlob(key: string, content: string) {\n\t\tthis.content.push([key, content]);\n\t}\n\n\tpublic getAggregatedBlobContent() {\n\t\tif (this.content.length === 0) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn JSON.stringify(this.content);\n\t}\n\n\tstatic load(input: ArrayBufferLike) {\n\t\tconst data = bufferToString2(input, \"utf-8\");\n\t\treturn JSON.parse(data) as [string, string][];\n\t}\n}\n\n/**\n * Base class that deals with unpacking snapshots (in place) containing aggregated blobs\n * It relies on abstract methods for reads and storing unpacked blobs.\n *\n * @deprecated This class was introduced experimentally and is no longer used.\n */\nexport abstract class SnapshotExtractor {\n\tprotected readonly aggregatedBlobName = \"__big\";\n\tprotected readonly virtualIdPrefix = \"__\";\n\n\t// counter for generation of virtual storage IDs\n\tprotected virtualIdCounter = 0;\n\tprotected getNextVirtualId() {\n\t\treturn `${this.virtualIdPrefix}${++this.virtualIdCounter}`;\n\t}\n\n\tabstract getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike>;\n\tabstract setBlob(id: string, tree: ISnapshotTree, content: string);\n\n\tpublic async unpackSnapshotCore(snapshot: ISnapshotTree, level = 0): Promise<void> {\n\t\tfor (const key of Object.keys(snapshot.trees)) {\n\t\t\tconst obj = snapshot.trees[key];\n\t\t\tawait this.unpackSnapshotCore(obj, level + 1);\n\t\t}\n\n\t\t// For future proof, we will support multiple aggregated blobs with any name\n\t\t// that starts with this.aggregatedBlobName\n\t\tfor (const key of Object.keys(snapshot.blobs)) {\n\t\t\tif (!key.startsWith(this.aggregatedBlobName)) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst blobId = snapshot.blobs[key];\n\t\t\tif (blobId !== undefined) {\n\t\t\t\tconst blob = await this.getBlob(blobId, snapshot);\n\t\t\t\tfor (const [path, value] of BlobAggregator.load(blob)) {\n\t\t\t\t\tconst id = this.getNextVirtualId();\n\t\t\t\t\tthis.setBlob(id, snapshot, value);\n\t\t\t\t\tconst pathSplit = path.split(\"/\");\n\t\t\t\t\tlet subTree = snapshot;\n\t\t\t\t\tfor (const subPath of pathSplit.slice(0, pathSplit.length - 1)) {\n\t\t\t\t\t\tif (subTree.trees[subPath] === undefined) {\n\t\t\t\t\t\t\tsubTree.trees[subPath] = { blobs: {}, trees: {} };\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsubTree = subTree.trees[subPath];\n\t\t\t\t\t}\n\t\t\t\t\tconst blobName = pathSplit[pathSplit.length - 1];\n\t\t\t\t\tassert(\n\t\t\t\t\t\tsubTree.blobs[blobName] === undefined,\n\t\t\t\t\t\t0x0f6 /* \"real blob ID exists\" */,\n\t\t\t\t\t);\n\t\t\t\t\tsubTree.blobs[blobName] = id;\n\t\t\t\t}\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\tdelete snapshot.blobs[this.aggregatedBlobName];\n\t\t\t}\n\t\t}\n\t}\n}\n\n/*\n * Snapshot extractor class that works in place, i.e. patches snapshot that has\n * blob content in ISnapshotTree.blobs itself, not in storage.\n * As result, it implements reading and writing of blobs to/from snapshot itself.\n * It follows existing pattern that mixes concerns - ISnapshotTree.blobs is used for two\n * purposes:\n * 1. map path name to blob ID\n * 2. map blob ID to blob content\n * #2 is what storage (IDocumentStorageService) is for, but in places where we do not have it\n * (like loading serialized earlier draft content), blob content is put directly into snapshot.\n * Ideally this should be fixed by using BlobCacheStorageService or something similar and\n * fixing existing flows to allow switching of storage.\n */\nclass SnapshotExtractorInPlace extends SnapshotExtractor {\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\tconst blob = tree.blobs[id];\n\t\tassert(blob !== undefined, 0x0f7 /* \"aggregate blob missing\" */);\n\t\treturn stringToBuffer(blob, \"base64\");\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tassert(\n\t\t\ttree.blobs[id] === undefined,\n\t\t\t0x0f8 /* \"blob from aggregate blob exists on its own\" */,\n\t\t);\n\t\ttree.blobs[id] = fromUtf8ToBase64(content);\n\t}\n}\n\n/**\n * Snapshot packer and extractor.\n * When summary is written it will find and aggregate small blobs into bigger blobs\n * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.\n *\n * @deprecated This class was introduced experimentally and is no longer used.\n */\nexport class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {\n\t// Tells data store if it can use incremental summary (i.e. reuse DDSes from previous summary\n\t// when only one DDS changed).\n\t// The answer has to be know long before we enable actual packing. The reason for the is the following:\n\t// A the moment when we enable packing, we should assume that all clients out there wil already have bits\n\t// that can unpack properly (i.e. enough time passed since we deployed bits that can unpack)\n\t// But we can still have clients where some of them already pack, and some do not. If one summary was\n\t// using packing, then it relies on non-incremental summaries going forward, even if next client who\n\t// produced summary is not packing!\n\t// This can have slight improvement by enabling it per file (based on \"did summary we loaded from contain\n\t// aggregated blobs\"), but that's harder to make reliable, so going for simplicity.\n\tstatic readonly fullDataStoreSummaries = true;\n\n\tprotected loadedFromSummary = false;\n\n\tprotected virtualBlobs = new Map<string, ArrayBufferLike>();\n\n\tstatic wrap(\n\t\tstorage: IDocumentStorageService,\n\t\tlogger: ITelemetryLogger,\n\t\tallowPacking?: boolean,\n\t\tpackingLevel = 2,\n\t) {\n\t\tif (storage instanceof BlobAggregationStorage) {\n\t\t\treturn storage;\n\t\t}\n\t\tconst mc = loggerToMonitoringContext(logger);\n\t\tconst realAllowPackaging =\n\t\t\tmc.config.getBoolean(\"FluidAggregateBlobs\") ?? allowPacking ?? false;\n\n\t\t// Always create BlobAggregationStorage even if storage is not asking for packing.\n\t\t// This is mostly to avoid cases where future changes in policy would result in inability to\n\t\t// load old files that were created with aggregation on.\n\t\tconst minBlobSize = storage.policies?.minBlobSize;\n\t\treturn new BlobAggregationStorage(\n\t\t\tstorage,\n\t\t\tlogger,\n\t\t\trealAllowPackaging,\n\t\t\tpackingLevel,\n\t\t\tminBlobSize,\n\t\t);\n\t}\n\n\tstatic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\tconst converter = new SnapshotExtractorInPlace();\n\t\tawait converter.unpackSnapshotCore(snapshot);\n\t}\n\n\tpublic get policies(): IDocumentStorageServicePolicies | undefined {\n\t\tconst policies = this.storage.policies;\n\t\tif (policies) {\n\t\t\treturn { ...policies, minBlobSize: undefined };\n\t\t}\n\t}\n\n\tpublic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\t// SummarizerNodeWithGC.refreshLatestSummary can call it when this.loadedFromSummary === false\n\t\t// (I assumed after file was created)\n\t\t// assert(!this.loadedFromSummary, \"unpack without summary\");\n\n\t\tthis.loadedFromSummary = true;\n\t\tawait this.unpackSnapshotCore(snapshot);\n\t}\n\n\tprotected constructor(\n\t\tprivate readonly storage: IDocumentStorageService,\n\t\tprivate readonly logger: ITelemetryLogger,\n\t\tprivate readonly allowPacking: boolean,\n\t\tprivate readonly packingLevel: number,\n\t\tprivate readonly blobCutOffSize?: number,\n\t) {\n\t\tsuper();\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tthis.virtualBlobs.set(id, stringToBuffer(content, \"utf-8\"));\n\t}\n\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\treturn this.readBlob(id).catch((error) => {\n\t\t\tthis.logger.sendErrorEvent({ eventName: \"BlobDedupNoAggregateBlob\" }, error);\n\t\t\tthrow error;\n\t\t});\n\t}\n\n\tpublic get repositoryUrl() {\n\t\treturn this.storage.repositoryUrl;\n\t}\n\tpublic async getVersions(\n\t\tversionId: string | null,\n\t\tcount: number,\n\t\tscenarioName?: string,\n\t\tfetchSource?: FetchSource,\n\t) {\n\t\treturn this.storage.getVersions(versionId, count, scenarioName, fetchSource);\n\t}\n\n\tpublic async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {\n\t\tthrow new Error(\"NYI\");\n\t}\n\n\t// for now we are not optimizing these blobs, with assumption that this API is used only\n\t// for big blobs (images)\n\tpublic async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {\n\t\treturn this.storage.createBlob(file);\n\t}\n\n\tpublic async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {\n\t\tconst tree = await this.storage.getSnapshotTree(version);\n\t\tif (tree) {\n\t\t\tawait this.unpackSnapshot(tree);\n\t\t}\n\t\treturn tree;\n\t}\n\n\tpublic async readBlob(id: string): Promise<ArrayBufferLike> {\n\t\tif (this.isRealStorageId(id)) {\n\t\t\treturn this.storage.readBlob(id);\n\t\t}\n\t\t// We support only reading blobs from the summary we loaded from.\n\t\t// This may need to be extended to any general summary in the future as runtime usage pattern\n\t\t// of storage changes (for example, data stores start to load from recent summary, not from original\n\t\t// summary whole container loaded from)\n\n\t\t// are there other ways we can get here? createFile is one flow, but we should not be reading blobs\n\t\t// in such flow\n\t\tassert(this.loadedFromSummary, 0x0f9 /* \"never read summary\" */);\n\t\tconst blob = this.virtualBlobs.get(id);\n\t\tassert(blob !== undefined, 0x0fa /* \"virtual blob not found\" */);\n\t\treturn blob;\n\t}\n\n\tpublic async uploadSummaryWithContext(\n\t\tsummary: ISummaryTree,\n\t\tcontext: ISummaryContext,\n\t): Promise<string> {\n\t\tconst summaryNew = this.allowPacking ? await this.compressSmallBlobs(summary) : summary;\n\t\treturn this.storage.uploadSummaryWithContext(summaryNew, context);\n\t}\n\n\t// For simplification, we assume that\n\t// - blob aggregation is done at data store level only for now\n\t// - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)\n\t// summary produced by data stores.\n\t// These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated\n\t// blob or contain aggregated blob that stays relevant for that sub-tree)\n\t// Note:\n\t// From perf perspective, it makes sense to place aggregated blobs one level up in the tree to not create extra\n\t// tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at\n\t// incremental summary time - we would need to understand handles and parse them. In current design we can skip\n\t// that step because if data store is reused, the whole sub-tree is reused including aggregated blob embedded into it\n\t// and that means we can do nothing and be correct!\n\tprivate async compressSmallBlobs(\n\t\tsummary: ISummaryTree,\n\t\tpath = \"\",\n\t\tlevel = 0,\n\t\taggregatorArg?: BlobAggregator,\n\t): Promise<ISummaryTree> {\n\t\tif (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {\n\t\t\treturn summary;\n\t\t}\n\n\t\tlet shouldCompress: boolean = false;\n\n\t\tlet aggregator = aggregatorArg;\n\t\t// checking if this is a dataStore tree, since we only pack at data store level\n\t\tif (Object.keys(summary.tree).includes(\".component\")) {\n\t\t\tassert(aggregator === undefined, 0x0fb /* \"logic err with aggregator\" */);\n\t\t\tassert(\n\t\t\t\tlevel === this.packingLevel,\n\t\t\t\t0x23b /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t\taggregator = new BlobAggregator();\n\t\t\tshouldCompress = true;\n\t\t} else {\n\t\t\tassert(\n\t\t\t\tlevel !== this.packingLevel,\n\t\t\t\t0x23c /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t}\n\n\t\tconst newSummary: ISummaryTree = { ...summary };\n\t\tnewSummary.tree = { ...newSummary.tree };\n\t\tfor (const key of Object.keys(summary.tree)) {\n\t\t\tconst obj = summary.tree[key];\n\t\t\t// Get path relative to root of data store (where we do aggregation)\n\t\t\tconst newPath = shouldCompress ? key : `${path}/${key}`;\n\t\t\tswitch (obj.type) {\n\t\t\t\tcase SummaryType.Tree:\n\t\t\t\t\t// If client created empty tree, keep it as is\n\t\t\t\t\t// Also do not package search blobs - they are part of storage contract\n\t\t\t\t\tif (Object.keys(obj).length !== 0 && key !== \"__search\") {\n\t\t\t\t\t\tconst tree = await this.compressSmallBlobs(\n\t\t\t\t\t\t\tobj,\n\t\t\t\t\t\t\tnewPath,\n\t\t\t\t\t\t\tlevel + 1,\n\t\t\t\t\t\t\taggregator,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tnewSummary.tree[key] = tree;\n\t\t\t\t\t\tif (Object.keys(obj).length === 0) {\n\t\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Blob:\n\t\t\t\t\tif (\n\t\t\t\t\t\taggregator &&\n\t\t\t\t\t\ttypeof obj.content == \"string\" &&\n\t\t\t\t\t\tobj.content.length < this.blobCutOffSize\n\t\t\t\t\t) {\n\t\t\t\t\t\taggregator.addBlob(newPath, obj.content);\n\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Handle: {\n\t\t\t\t\t// Would be nice to:\n\t\t\t\t\t// Trees: expand the tree\n\t\t\t\t\t// Blobs: parse handle and ensure it points to real blob, not virtual blob.\n\t\t\t\t\t// We can avoid it for now given data store is the granularity of incremental summaries.\n\t\t\t\t\tlet handlePath = obj.handle;\n\t\t\t\t\tif (handlePath.startsWith(\"/\")) {\n\t\t\t\t\t\thandlePath = handlePath.substr(1);\n\t\t\t\t\t}\n\t\t\t\t\t// Ensure only whole data stores can be reused, no reusing at deeper level!\n\t\t\t\t\tassert(level === 0, 0x0fc /* \"tree reuse at lower level\" */);\n\t\t\t\t\tassert(\n\t\t\t\t\t\t!handlePath.includes(\"/\"),\n\t\t\t\t\t\t0x0fd /* \"data stores are writing incremental summaries!\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase SummaryType.Attachment:\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.isRealStorageId(obj.id),\n\t\t\t\t\t\t0x0fe /* \"attachment is aggregate blob\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\tdefault:\n\t\t\t\t\tunreachableCase(obj, `Unknown type: ${(obj as any).type}`);\n\t\t\t}\n\t\t}\n\n\t\tassert(\n\t\t\tnewSummary.tree[this.aggregatedBlobName] === undefined,\n\t\t\t0x0ff /* \"duplicate aggregate blob\" */,\n\t\t);\n\t\tif (shouldCompress) {\n\t\t\t// Note: It would be great to add code here to unpack aggregate blob back to normal blobs\n\t\t\t// If only one blob made it into aggregate. Currently that does not happen as we always have\n\t\t\t// at least one .component blob and at least one DDS that has .attributes blob, so it's not an issue.\n\t\t\t// But it's possible that in future that would be great addition!\n\t\t\t// Good news - it's backward compatible change.\n\t\t\tassert(aggregator !== undefined, 0x100 /* \"logic error\" */);\n\t\t\tconst content = aggregator.getAggregatedBlobContent();\n\t\t\tif (content !== undefined) {\n\t\t\t\tnewSummary.tree[this.aggregatedBlobName] = {\n\t\t\t\t\ttype: SummaryType.Blob,\n\t\t\t\t\tcontent,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t\treturn newSummary;\n\t}\n\n\tprotected isRealStorageId(id: string): boolean {\n\t\treturn !id.startsWith(this.virtualIdPrefix);\n\t}\n}\n"]}
package/dist/packageVersion.d.ts CHANGED
@@ -5,5 +5,5 @@
5
5
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
6
6
  */
7
7
  export declare const pkgName = "@fluidframework/driver-utils";
8
- export declare const pkgVersion = "2.0.0-internal.3.1.0";
8
+ export declare const pkgVersion = "2.0.0-internal.3.2.1";
9
9
  //# sourceMappingURL=packageVersion.d.ts.map
package/dist/packageVersion.js CHANGED
@@ -8,5 +8,5 @@
8
8
  Object.defineProperty(exports, "__esModule", { value: true });
9
9
  exports.pkgVersion = exports.pkgName = void 0;
10
10
  exports.pkgName = "@fluidframework/driver-utils";
11
- exports.pkgVersion = "2.0.0-internal.3.1.0";
11
+ exports.pkgVersion = "2.0.0-internal.3.2.1";
12
12
  //# sourceMappingURL=packageVersion.js.map
package/dist/packageVersion.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,OAAO,GAAG,8BAA8B,CAAC;AACzC,QAAA,UAAU,GAAG,sBAAsB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/driver-utils\";\nexport const pkgVersion = \"2.0.0-internal.3.1.0\";\n"]}
1
+ {"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,OAAO,GAAG,8BAA8B,CAAC;AACzC,QAAA,UAAU,GAAG,sBAAsB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/driver-utils\";\nexport const pkgVersion = \"2.0.0-internal.3.2.1\";\n"]}
package/lib/blobAggregationStorage.d.ts CHANGED
@@ -5,6 +5,12 @@
5
5
  import { FetchSource, IDocumentStorageService, IDocumentStorageServicePolicies, ISummaryContext } from "@fluidframework/driver-definitions";
6
6
  import { ICreateBlobResponse, ISnapshotTree, ISummaryHandle, ISummaryTree, IVersion } from "@fluidframework/protocol-definitions";
7
7
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
8
+ /**
9
+ * Base class that deals with unpacking snapshots (in place) containing aggregated blobs
10
+ * It relies on abstract methods for reads and storing unpacked blobs.
11
+ *
12
+ * @deprecated This class was introduced experimentally and is no longer used.
13
+ */
8
14
  export declare abstract class SnapshotExtractor {
9
15
  protected readonly aggregatedBlobName = "__big";
10
16
  protected readonly virtualIdPrefix = "__";
@@ -14,6 +20,13 @@ export declare abstract class SnapshotExtractor {
14
20
  abstract setBlob(id: string, tree: ISnapshotTree, content: string): any;
15
21
  unpackSnapshotCore(snapshot: ISnapshotTree, level?: number): Promise<void>;
16
22
  }
23
+ /**
24
+ * Snapshot packer and extractor.
25
+ * When summary is written it will find and aggregate small blobs into bigger blobs
26
+ * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.
27
+ *
28
+ * @deprecated This class was introduced experimentally and is no longer used.
29
+ */
17
30
  export declare class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {
18
31
  private readonly storage;
19
32
  private readonly logger;
package/lib/blobAggregationStorage.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"blobAggregationStorage.d.ts","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,WAAW,EACX,uBAAuB,EACvB,+BAA+B,EAC/B,eAAe,EACf,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACN,mBAAmB,EACnB,aAAa,EACb,cAAc,EACd,YAAY,EACZ,QAAQ,EAER,MAAM,sCAAsC,CAAC;AAS9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AA0CtE,8BAAsB,iBAAiB;IACtC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,WAAW;IAChD,SAAS,CAAC,QAAQ,CAAC,eAAe,QAAQ;IAG1C,SAAS,CAAC,gBAAgB,SAAK;IAC/B,SAAS,CAAC,gBAAgB;IAI1B,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAC3E,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAEpD,kBAAkB,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAsClF;AAoCD,qBAAa,sBAAuB,SAAQ,iBAAkB,YAAW,uBAAuB;IAiE9F,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IA1DjC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,QAAQ;IAE9C,SAAS,CAAC,iBAAiB,UAAS;IAEpC,SAAS,CAAC,YAAY,+BAAsC;IAE5D,MAAM,CAAC,IAAI,CACV,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,CAAC,EAAE,OAAO,EACtB,YAAY,SAAI;WAsBJ,cAAc,CAAC,QAAQ,EAAE,aAAa;IAKnD,IAAW,QAAQ,IAAI,+BAA+B,GAAG,SAAS,CAKjE;IAEY,cAAc,CAAC,QAAQ,EAAE,aAAa;IASnD,SAAS,aACS,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,EAAE,OAAO,EACrB,YAAY,EAAE,MAAM,EACpB,cAAc,CAAC,oBAAQ;IAKlC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAIlD,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAO/E,IAAW,aAAa,WAEvB;IACY,WAAW,CACvB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,WAAW;IAKb,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;IAM9D,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAI/D,eAAe,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAQlE,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAiB9C,wBAAwB,CACpC,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,eAAe,GACtB,OAAO,CAAC,MAAM,CAAC;YAiBJ,kBAAkB;IAkHhC,SAAS,CAAC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;CAG9C"}
1
+ {"version":3,"file":"blobAggregationStorage.d.ts","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,WAAW,EACX,uBAAuB,EACvB,+BAA+B,EAC/B,eAAe,EACf,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACN,mBAAmB,EACnB,aAAa,EACb,cAAc,EACd,YAAY,EACZ,QAAQ,EAER,MAAM,sCAAsC,CAAC;AAS9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAsCtE;;;;;GAKG;AACH,8BAAsB,iBAAiB;IACtC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,WAAW;IAChD,SAAS,CAAC,QAAQ,CAAC,eAAe,QAAQ;IAG1C,SAAS,CAAC,gBAAgB,SAAK;IAC/B,SAAS,CAAC,gBAAgB;IAI1B,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAC3E,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAEpD,kBAAkB,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAsClF;AA+BD;;;;;;GAMG;AACH,qBAAa,sBAAuB,SAAQ,iBAAkB,YAAW,uBAAuB;IAiE9F,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IA1DjC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,QAAQ;IAE9C,SAAS,CAAC,iBAAiB,UAAS;IAEpC,SAAS,CAAC,YAAY,+BAAsC;IAE5D,MAAM,CAAC,IAAI,CACV,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,CAAC,EAAE,OAAO,EACtB,YAAY,SAAI;WAsBJ,cAAc,CAAC,QAAQ,EAAE,aAAa;IAKnD,IAAW,QAAQ,IAAI,+BAA+B,GAAG,SAAS,CAKjE;IAEY,cAAc,CAAC,QAAQ,EAAE,aAAa;IASnD,SAAS,aACS,OAAO,EAAE,uBAAuB,EAChC,MAAM,EAAE,gBAAgB,EACxB,YAAY,EAAE,OAAO,EACrB,YAAY,EAAE,MAAM,EACpB,cAAc,CAAC,oBAAQ;IAKlC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM;IAIlD,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAO/E,IAAW,aAAa,WAEvB;IACY,WAAW,CACvB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,WAAW;IAKb,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;IAM9D,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAI/D,eAAe,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAQlE,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAiB9C,wBAAwB,CACpC,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,eAAe,GACtB,OAAO,CAAC,MAAM,CAAC;YAiBJ,kBAAkB;IAkHhC,SAAS,CAAC,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;CAG9C"}
package/lib/blobAggregationStorage.js CHANGED
@@ -37,9 +37,11 @@ class BlobAggregator {
37
37
  return JSON.parse(data);
38
38
  }
39
39
  }
40
- /*
40
+ /**
41
41
  * Base class that deals with unpacking snapshots (in place) containing aggregated blobs
42
42
  * It relies on abstract methods for reads and storing unpacked blobs.
43
+ *
44
+ * @deprecated This class was introduced experimentally and is no longer used.
43
45
  */
44
46
  export class SnapshotExtractor {
45
47
  constructor() {
@@ -110,10 +112,12 @@ class SnapshotExtractorInPlace extends SnapshotExtractor {
110
112
  tree.blobs[id] = fromUtf8ToBase64(content);
111
113
  }
112
114
  }
113
- /*
115
+ /**
114
116
  * Snapshot packer and extractor.
115
117
  * When summary is written it will find and aggregate small blobs into bigger blobs
116
118
  * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.
119
+ *
120
+ * @deprecated This class was introduced experimentally and is no longer used.
117
121
  */
118
122
  export class BlobAggregationStorage extends SnapshotExtractor {
119
123
  constructor(storage, logger, allowPacking, packingLevel, blobCutOffSize) {
@@ -210,12 +214,12 @@ export class BlobAggregationStorage extends SnapshotExtractor {
210
214
  // - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)
211
215
  // summary produced by data stores.
212
216
  // These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated
213
- // blob Or contain aggregated blob that stays relevant for that sub-tree)
217
+ // blob or contain aggregated blob that stays relevant for that sub-tree)
214
218
  // Note:
215
- // From perf perspective, it makes sense to place aggregated blobs one level up in the tree not to create extra
219
+ // From perf perspective, it makes sense to place aggregated blobs one level up in the tree to not create extra
216
220
  // tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at
217
221
  // incremental summary time - we would need to understand handles and parse them. In current design we can skip
218
- // that step because if data store is reused, the hole sub-tree is reused included aggregated blob embedded into it
222
+ // that step because if data store is reused, the whole sub-tree is reused including aggregated blob embedded into it
219
223
  // and that means we can do nothing and be correct!
220
224
  async compressSmallBlobs(summary, path = "", level = 0, aggregatorArg) {
221
225
  if (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {
@@ -243,10 +247,10 @@ export class BlobAggregationStorage extends SnapshotExtractor {
243
247
  case SummaryType.Tree:
244
248
  // If client created empty tree, keep it as is
245
249
  // Also do not package search blobs - they are part of storage contract
246
- if (obj.tree !== {} && key !== "__search") {
250
+ if (Object.keys(obj).length !== 0 && key !== "__search") {
247
251
  const tree = await this.compressSmallBlobs(obj, newPath, level + 1, aggregator);
248
252
  newSummary.tree[key] = tree;
249
- if (tree.tree === {}) {
253
+ if (Object.keys(obj).length === 0) {
250
254
  // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
251
255
  delete newSummary.tree[key];
252
256
  }
package/lib/blobAggregationStorage.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"blobAggregationStorage.js","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,EAMN,WAAW,GACX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACN,MAAM,EACN,cAAc,EACd,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAE5E;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAqB,EAAE,QAA4B;IAC3E,IAAI,IAAI,YAAY,UAAU,EAAE;QAC/B,8DAA8D;QAC9D,OAAO,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;KAC1C;IACD,OAAO,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,cAAc;IAApB;QACkB,YAAO,GAAuB,EAAE,CAAC;IAiBnD,CAAC;IAfO,OAAO,CAAC,GAAW,EAAE,OAAe;QAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACnC,CAAC;IAEM,wBAAwB;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,OAAO,SAAS,CAAC;SACjB;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAsB;QACjC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAC/C,CAAC;CACD;AAED;;;GAGG;AACH,MAAM,OAAgB,iBAAiB;IAAvC;QACoB,uBAAkB,GAAG,OAAO,CAAC;QAC7B,oBAAe,GAAG,IAAI,CAAC;QAE1C,gDAAgD;QACtC,qBAAgB,GAAG,CAAC,CAAC;IA8ChC,CAAC;IA7CU,gBAAgB;QACzB,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5D,CAAC;IAKM,KAAK,CAAC,kBAAkB,CAAC,QAAuB,EAAE,KAAK,GAAG,CAAC;QACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;SAC9C;QAED,4EAA4E;QAC5E,2CAA2C;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;gBAC7C,SAAS;aACT;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,MAAM,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAClD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACtD,MAAM,EAAE,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACnC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;oBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,OAAO,GAAG,QAAQ,CAAC;oBACvB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;wBAC/D,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE;4BACzC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;yBAClD;wBACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;qBACjC;oBACD,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACjD,MAAM,CACL,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EACrC,KAAK,CAAC,2BAA2B,CACjC,CAAC;oBACF,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;iBAC7B;gBACD,gEAAgE;gBAChE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;aAC/C;SACD;IACF,CAAC;CACD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,wBAAyB,SAAQ,iBAAiB;IAChD,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,MAAM,CACL,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,SAAS,EAC5B,KAAK,CAAC,kDAAkD,CACxD,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;CACD;AAED;;;;GAIG;AACH,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IAgE5D,YACkB,OAAgC,EAChC,MAAwB,EACxB,YAAqB,EACrB,YAAoB,EACpB,cAAuB;QAExC,KAAK,EAAE,CAAC;QANS,YAAO,GAAP,OAAO,CAAyB;QAChC,WAAM,GAAN,MAAM,CAAkB;QACxB,iBAAY,GAAZ,YAAY,CAAS;QACrB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,mBAAc,GAAd,cAAc,CAAS;QAxD/B,sBAAiB,GAAG,KAAK,CAAC;QAE1B,iBAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;IAyD5D,CAAC;IAvDD,MAAM,CAAC,IAAI,CACV,OAAgC,EAChC,MAAwB,EACxB,YAAsB,EACtB,YAAY,GAAG,CAAC;;QAEhB,IAAI,OAAO,YAAY,sBAAsB,EAAE;YAC9C,OAAO,OAAO,CAAC;SACf;QACD,MAAM,EAAE,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,kBAAkB,GACvB,MAAA,MAAA,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,mCAAI,YAAY,mCAAI,KAAK,CAAC;QAEtE,kFAAkF;QAClF,4FAA4F;QAC5F,wDAAwD;QACxD,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,QAAQ,0CAAE,WAAW,CAAC;QAClD,OAAO,IAAI,sBAAsB,CAChC,OAAO,EACP,MAAM,EACN,kBAAkB,EAClB,YAAY,EACZ,WAAW,CACX,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,MAAM,SAAS,GAAG,IAAI,wBAAwB,EAAE,CAAC;QACjD,MAAM,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED,IAAW,QAAQ;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACvC,IAAI,QAAQ,EAAE;YACb,uCAAY,QAAQ,KAAE,WAAW,EAAE,SAAS,IAAG;SAC/C;IACF,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,8FAA8F;QAC9F,qCAAqC;QACrC,6DAA6D;QAE7D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAYM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,IAAW,aAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACnC,CAAC;IACM,KAAK,CAAC,WAAW,CACvB,SAAwB,EACxB,KAAa,EACb,YAAqB,EACrB,WAAyB;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAC9E,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,MAAsB;QAClD,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,wFAAwF;IACxF,yBAAyB;IAClB,KAAK,CAAC,UAAU,CAAC,IAAqB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAkB;QAC9C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,IAAI,EAAE;YACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,EAAU;QAC/B,IAAI,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;SACjC;QACD,iEAAiE;QACjE,6FAA6F;QAC7F,oGAAoG;QACpG,uCAAuC;QAEvC,mGAAmG;QACnG,eAAe;QACf,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,wBAAwB,CACpC,OAAqB,EACrB,OAAwB;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,qCAAqC;IACrC,8DAA8D;IAC9D,8GAA8G;IAC9G,mCAAmC;IACnC,gHAAgH;IAChH,yEAAyE;IACzE,QAAQ;IACR,+GAA+G;IAC/G,8GAA8G;IAC9G,+GAA+G;IAC/G,mHAAmH;IACnH,mDAAmD;IAC3C,KAAK,CAAC,kBAAkB,CAC/B,OAAqB,EACrB,IAAI,GAAG,EAAE,EACT,KAAK,GAAG,CAAC,EACT,aAA8B;QAE9B,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;YACjE,OAAO,OAAO,CAAC;SACf;QAED,IAAI,cAAc,GAAY,KAAK,CAAC;QAEpC,IAAI,UAAU,GAAG,aAAa,CAAC;QAC/B,+EAA+E;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE;YACrD,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAC1E,MAAM,CACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;YACF,UAAU,GAAG,IAAI,cAAc,EAAE,CAAC;YAClC,cAAc,GAAG,IAAI,CAAC;SACtB;aAAM;YACN,MAAM,CACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;SACF;QAED,MAAM,UAAU,qBAAsB,OAAO,CAAE,CAAC;QAChD,UAAU,CAAC,IAAI,qBAAQ,UAAU,CAAC,IAAI,CAAE,CAAC;QACzC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,oEAAoE;YACpE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;YACxD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,WAAW,CAAC,IAAI;oBACpB,8CAA8C;oBAC9C,uEAAuE;oBACvE,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE,IAAI,GAAG,KAAK,UAAU,EAAE;wBAC1C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CACzC,GAAG,EACH,OAAO,EACP,KAAK,GAAG,CAAC,EACT,UAAU,CACV,CAAC;wBACF,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;wBAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE;4BACrB,gEAAgE;4BAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;yBAC5B;qBACD;oBACD,MAAM;gBACP,KAAK,WAAW,CAAC,IAAI;oBACpB,IACC,UAAU;wBACV,OAAO,GAAG,CAAC,OAAO,IAAI,QAAQ;wBAC9B,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EACvC;wBACD,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;wBACzC,gEAAgE;wBAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAC5B;oBACD,MAAM;gBACP,KAAK,WAAW,CAAC,MAAM,CAAC,CAAC;oBACxB,oBAAoB;oBACpB,yBAAyB;oBACzB,2EAA2E;oBAC3E,wFAAwF;oBACxF,IAAI,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC;oBAC5B,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;wBAC/B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;qBAClC;oBACD,2EAA2E;oBAC3E,MAAM,CAAC,KAAK,KAAK,CAAC,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBAC7D,MAAM,CACL,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EACzB,KAAK,CAAC,sDAAsD,CAC5D,CAAC;oBACF,MAAM;iBACN;gBACD,KAAK,WAAW,CAAC,UAAU;oBAC1B,MAAM,CACL,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,EAC5B,KAAK,CAAC,oCAAoC,CAC1C,CAAC;oBACF,MAAM;gBACP;oBACC,eAAe,CAAC,GAAG,EAAE,iBAAkB,GAAW,CAAC,IAAI,EAAE,CAAC,CAAC;aAC5D;SACD;QAED,MAAM,CACL,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,SAAS,EACtD,KAAK,CAAC,gCAAgC,CACtC,CAAC;QACF,IAAI,cAAc,EAAE;YACnB,yFAAyF;YACzF,4FAA4F;YAC5F,qGAAqG;YACrG,iEAAiE;YACjE,+CAA+C;YAC/C,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,UAAU,CAAC,wBAAwB,EAAE,CAAC;YACtD,IAAI,OAAO,KAAK,SAAS,EAAE;gBAC1B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG;oBAC1C,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,OAAO;iBACP,CAAC;aACF;SACD;QACD,OAAO,UAAU,CAAC;IACnB,CAAC;IAES,eAAe,CAAC,EAAU;QACnC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;;AA3QD,6FAA6F;AAC7F,8BAA8B;AAC9B,uGAAuG;AACvG,yGAAyG;AACzG,4FAA4F;AAC5F,qGAAqG;AACrG,oGAAoG;AACpG,mCAAmC;AACnC,yGAAyG;AACzG,mFAAmF;AACnE,6CAAsB,GAAG,IAAI,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport {\n\tFetchSource,\n\tIDocumentStorageService,\n\tIDocumentStorageServicePolicies,\n\tISummaryContext,\n} from \"@fluidframework/driver-definitions\";\nimport {\n\tICreateBlobResponse,\n\tISnapshotTree,\n\tISummaryHandle,\n\tISummaryTree,\n\tIVersion,\n\tSummaryType,\n} from \"@fluidframework/protocol-definitions\";\nimport {\n\tassert,\n\tbufferToString,\n\tstringToBuffer,\n\tunreachableCase,\n\tfromUtf8ToBase64,\n\tUint8ArrayToString,\n} from \"@fluidframework/common-utils\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { loggerToMonitoringContext } from \"@fluidframework/telemetry-utils\";\n\n/*\n * Work around for bufferToString having a bug - it can't consume IsoBuffer!\n * To be removed once bufferToString is fixed!\n */\nfunction bufferToString2(blob: ArrayBufferLike, encoding: \"utf-8\" | \"base64\"): string {\n\tif (blob instanceof Uint8Array) {\n\t\t// IsoBuffer does not have ctor, so it's not in proto chain :(\n\t\treturn Uint8ArrayToString(blob, encoding);\n\t}\n\treturn bufferToString(blob, encoding);\n}\n\n/**\n * Class responsible for aggregating smaller blobs into one and unpacking it later on.\n */\nclass BlobAggregator {\n\tprivate readonly content: [string, string][] = [];\n\n\tpublic addBlob(key: string, content: string) {\n\t\tthis.content.push([key, content]);\n\t}\n\n\tpublic getAggregatedBlobContent() {\n\t\tif (this.content.length === 0) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn JSON.stringify(this.content);\n\t}\n\n\tstatic load(input: ArrayBufferLike) {\n\t\tconst data = bufferToString2(input, \"utf-8\");\n\t\treturn JSON.parse(data) as [string, string][];\n\t}\n}\n\n/*\n * Base class that deals with unpacking snapshots (in place) containing aggregated blobs\n * It relies on abstract methods for reads and storing unpacked blobs.\n */\nexport abstract class SnapshotExtractor {\n\tprotected readonly aggregatedBlobName = \"__big\";\n\tprotected readonly virtualIdPrefix = \"__\";\n\n\t// counter for generation of virtual storage IDs\n\tprotected virtualIdCounter = 0;\n\tprotected getNextVirtualId() {\n\t\treturn `${this.virtualIdPrefix}${++this.virtualIdCounter}`;\n\t}\n\n\tabstract getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike>;\n\tabstract setBlob(id: string, tree: ISnapshotTree, content: string);\n\n\tpublic async unpackSnapshotCore(snapshot: ISnapshotTree, level = 0): Promise<void> {\n\t\tfor (const key of Object.keys(snapshot.trees)) {\n\t\t\tconst obj = snapshot.trees[key];\n\t\t\tawait this.unpackSnapshotCore(obj, level + 1);\n\t\t}\n\n\t\t// For future proof, we will support multiple aggregated blobs with any name\n\t\t// that starts with this.aggregatedBlobName\n\t\tfor (const key of Object.keys(snapshot.blobs)) {\n\t\t\tif (!key.startsWith(this.aggregatedBlobName)) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst blobId = snapshot.blobs[key];\n\t\t\tif (blobId !== undefined) {\n\t\t\t\tconst blob = await this.getBlob(blobId, snapshot);\n\t\t\t\tfor (const [path, value] of BlobAggregator.load(blob)) {\n\t\t\t\t\tconst id = this.getNextVirtualId();\n\t\t\t\t\tthis.setBlob(id, snapshot, value);\n\t\t\t\t\tconst pathSplit = path.split(\"/\");\n\t\t\t\t\tlet subTree = snapshot;\n\t\t\t\t\tfor (const subPath of pathSplit.slice(0, pathSplit.length - 1)) {\n\t\t\t\t\t\tif (subTree.trees[subPath] === undefined) {\n\t\t\t\t\t\t\tsubTree.trees[subPath] = { blobs: {}, trees: {} };\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsubTree = subTree.trees[subPath];\n\t\t\t\t\t}\n\t\t\t\t\tconst blobName = pathSplit[pathSplit.length - 1];\n\t\t\t\t\tassert(\n\t\t\t\t\t\tsubTree.blobs[blobName] === undefined,\n\t\t\t\t\t\t0x0f6 /* \"real blob ID exists\" */,\n\t\t\t\t\t);\n\t\t\t\t\tsubTree.blobs[blobName] = id;\n\t\t\t\t}\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\tdelete snapshot.blobs[this.aggregatedBlobName];\n\t\t\t}\n\t\t}\n\t}\n}\n\n/*\n * Snapshot extractor class that works in place, i.e. patches snapshot that has\n * blob content in ISnapshotTree.blobs itself, not in storage.\n * As result, it implements reading and writing of blobs to/from snapshot itself.\n * It follows existing pattern that mixes concerns - ISnapshotTree.blobs is used for two\n * purposes:\n * 1. map path name to blob ID\n * 2. map blob ID to blob content\n * #2 is what storage (IDocumentStorageService) is for, but in places where we do not have it\n * (like loading serialized earlier draft content), blob content is put directly into snapshot.\n * Ideally this should be fixed by using BlobCacheStorageService or something similar and\n * fixing existing flows to allow switching of storage.\n */\nclass SnapshotExtractorInPlace extends SnapshotExtractor {\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\tconst blob = tree.blobs[id];\n\t\tassert(blob !== undefined, 0x0f7 /* \"aggregate blob missing\" */);\n\t\treturn stringToBuffer(blob, \"base64\");\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tassert(\n\t\t\ttree.blobs[id] === undefined,\n\t\t\t0x0f8 /* \"blob from aggregate blob exists on its own\" */,\n\t\t);\n\t\ttree.blobs[id] = fromUtf8ToBase64(content);\n\t}\n}\n\n/*\n * Snapshot packer and extractor.\n * When summary is written it will find and aggregate small blobs into bigger blobs\n * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.\n */\nexport class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {\n\t// Tells data store if it can use incremental summary (i.e. reuse DDSes from previous summary\n\t// when only one DDS changed).\n\t// The answer has to be know long before we enable actual packing. The reason for the is the following:\n\t// A the moment when we enable packing, we should assume that all clients out there wil already have bits\n\t// that can unpack properly (i.e. enough time passed since we deployed bits that can unpack)\n\t// But we can still have clients where some of them already pack, and some do not. If one summary was\n\t// using packing, then it relies on non-incremental summaries going forward, even if next client who\n\t// produced summary is not packing!\n\t// This can have slight improvement by enabling it per file (based on \"did summary we loaded from contain\n\t// aggregated blobs\"), but that's harder to make reliable, so going for simplicity.\n\tstatic readonly fullDataStoreSummaries = true;\n\n\tprotected loadedFromSummary = false;\n\n\tprotected virtualBlobs = new Map<string, ArrayBufferLike>();\n\n\tstatic wrap(\n\t\tstorage: IDocumentStorageService,\n\t\tlogger: ITelemetryLogger,\n\t\tallowPacking?: boolean,\n\t\tpackingLevel = 2,\n\t) {\n\t\tif (storage instanceof BlobAggregationStorage) {\n\t\t\treturn storage;\n\t\t}\n\t\tconst mc = loggerToMonitoringContext(logger);\n\t\tconst realAllowPackaging =\n\t\t\tmc.config.getBoolean(\"FluidAggregateBlobs\") ?? allowPacking ?? false;\n\n\t\t// Always create BlobAggregationStorage even if storage is not asking for packing.\n\t\t// This is mostly to avoid cases where future changes in policy would result in inability to\n\t\t// load old files that were created with aggregation on.\n\t\tconst minBlobSize = storage.policies?.minBlobSize;\n\t\treturn new BlobAggregationStorage(\n\t\t\tstorage,\n\t\t\tlogger,\n\t\t\trealAllowPackaging,\n\t\t\tpackingLevel,\n\t\t\tminBlobSize,\n\t\t);\n\t}\n\n\tstatic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\tconst converter = new SnapshotExtractorInPlace();\n\t\tawait converter.unpackSnapshotCore(snapshot);\n\t}\n\n\tpublic get policies(): IDocumentStorageServicePolicies | undefined {\n\t\tconst policies = this.storage.policies;\n\t\tif (policies) {\n\t\t\treturn { ...policies, minBlobSize: undefined };\n\t\t}\n\t}\n\n\tpublic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\t// SummarizerNodeWithGC.refreshLatestSummary can call it when this.loadedFromSummary === false\n\t\t// (I assumed after file was created)\n\t\t// assert(!this.loadedFromSummary, \"unpack without summary\");\n\n\t\tthis.loadedFromSummary = true;\n\t\tawait this.unpackSnapshotCore(snapshot);\n\t}\n\n\tprotected constructor(\n\t\tprivate readonly storage: IDocumentStorageService,\n\t\tprivate readonly logger: ITelemetryLogger,\n\t\tprivate readonly allowPacking: boolean,\n\t\tprivate readonly packingLevel: number,\n\t\tprivate readonly blobCutOffSize?: number,\n\t) {\n\t\tsuper();\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tthis.virtualBlobs.set(id, stringToBuffer(content, \"utf-8\"));\n\t}\n\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\treturn this.readBlob(id).catch((error) => {\n\t\t\tthis.logger.sendErrorEvent({ eventName: \"BlobDedupNoAggregateBlob\" }, error);\n\t\t\tthrow error;\n\t\t});\n\t}\n\n\tpublic get repositoryUrl() {\n\t\treturn this.storage.repositoryUrl;\n\t}\n\tpublic async getVersions(\n\t\tversionId: string | null,\n\t\tcount: number,\n\t\tscenarioName?: string,\n\t\tfetchSource?: FetchSource,\n\t) {\n\t\treturn this.storage.getVersions(versionId, count, scenarioName, fetchSource);\n\t}\n\n\tpublic async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {\n\t\tthrow new Error(\"NYI\");\n\t}\n\n\t// for now we are not optimizing these blobs, with assumption that this API is used only\n\t// for big blobs (images)\n\tpublic async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {\n\t\treturn this.storage.createBlob(file);\n\t}\n\n\tpublic async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {\n\t\tconst tree = await this.storage.getSnapshotTree(version);\n\t\tif (tree) {\n\t\t\tawait this.unpackSnapshot(tree);\n\t\t}\n\t\treturn tree;\n\t}\n\n\tpublic async readBlob(id: string): Promise<ArrayBufferLike> {\n\t\tif (this.isRealStorageId(id)) {\n\t\t\treturn this.storage.readBlob(id);\n\t\t}\n\t\t// We support only reading blobs from the summary we loaded from.\n\t\t// This may need to be extended to any general summary in the future as runtime usage pattern\n\t\t// of storage changes (for example, data stores start to load from recent summary, not from original\n\t\t// summary whole container loaded from)\n\n\t\t// are there other ways we can get here? createFile is one flow, but we should not be reading blobs\n\t\t// in such flow\n\t\tassert(this.loadedFromSummary, 0x0f9 /* \"never read summary\" */);\n\t\tconst blob = this.virtualBlobs.get(id);\n\t\tassert(blob !== undefined, 0x0fa /* \"virtual blob not found\" */);\n\t\treturn blob;\n\t}\n\n\tpublic async uploadSummaryWithContext(\n\t\tsummary: ISummaryTree,\n\t\tcontext: ISummaryContext,\n\t): Promise<string> {\n\t\tconst summaryNew = this.allowPacking ? await this.compressSmallBlobs(summary) : summary;\n\t\treturn this.storage.uploadSummaryWithContext(summaryNew, context);\n\t}\n\n\t// For simplification, we assume that\n\t// - blob aggregation is done at data store level only for now\n\t// - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)\n\t// summary produced by data stores.\n\t// These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated\n\t// blob Or contain aggregated blob that stays relevant for that sub-tree)\n\t// Note:\n\t// From perf perspective, it makes sense to place aggregated blobs one level up in the tree not to create extra\n\t// tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at\n\t// incremental summary time - we would need to understand handles and parse them. In current design we can skip\n\t// that step because if data store is reused, the hole sub-tree is reused included aggregated blob embedded into it\n\t// and that means we can do nothing and be correct!\n\tprivate async compressSmallBlobs(\n\t\tsummary: ISummaryTree,\n\t\tpath = \"\",\n\t\tlevel = 0,\n\t\taggregatorArg?: BlobAggregator,\n\t): Promise<ISummaryTree> {\n\t\tif (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {\n\t\t\treturn summary;\n\t\t}\n\n\t\tlet shouldCompress: boolean = false;\n\n\t\tlet aggregator = aggregatorArg;\n\t\t// checking if this is a dataStore tree, since we only pack at data store level\n\t\tif (Object.keys(summary.tree).includes(\".component\")) {\n\t\t\tassert(aggregator === undefined, 0x0fb /* \"logic err with aggregator\" */);\n\t\t\tassert(\n\t\t\t\tlevel === this.packingLevel,\n\t\t\t\t0x23b /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t\taggregator = new BlobAggregator();\n\t\t\tshouldCompress = true;\n\t\t} else {\n\t\t\tassert(\n\t\t\t\tlevel !== this.packingLevel,\n\t\t\t\t0x23c /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t}\n\n\t\tconst newSummary: ISummaryTree = { ...summary };\n\t\tnewSummary.tree = { ...newSummary.tree };\n\t\tfor (const key of Object.keys(summary.tree)) {\n\t\t\tconst obj = summary.tree[key];\n\t\t\t// Get path relative to root of data store (where we do aggregation)\n\t\t\tconst newPath = shouldCompress ? key : `${path}/${key}`;\n\t\t\tswitch (obj.type) {\n\t\t\t\tcase SummaryType.Tree:\n\t\t\t\t\t// If client created empty tree, keep it as is\n\t\t\t\t\t// Also do not package search blobs - they are part of storage contract\n\t\t\t\t\tif (obj.tree !== {} && key !== \"__search\") {\n\t\t\t\t\t\tconst tree = await this.compressSmallBlobs(\n\t\t\t\t\t\t\tobj,\n\t\t\t\t\t\t\tnewPath,\n\t\t\t\t\t\t\tlevel + 1,\n\t\t\t\t\t\t\taggregator,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tnewSummary.tree[key] = tree;\n\t\t\t\t\t\tif (tree.tree === {}) {\n\t\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Blob:\n\t\t\t\t\tif (\n\t\t\t\t\t\taggregator &&\n\t\t\t\t\t\ttypeof obj.content == \"string\" &&\n\t\t\t\t\t\tobj.content.length < this.blobCutOffSize\n\t\t\t\t\t) {\n\t\t\t\t\t\taggregator.addBlob(newPath, obj.content);\n\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Handle: {\n\t\t\t\t\t// Would be nice to:\n\t\t\t\t\t// Trees: expand the tree\n\t\t\t\t\t// Blobs: parse handle and ensure it points to real blob, not virtual blob.\n\t\t\t\t\t// We can avoid it for now given data store is the granularity of incremental summaries.\n\t\t\t\t\tlet handlePath = obj.handle;\n\t\t\t\t\tif (handlePath.startsWith(\"/\")) {\n\t\t\t\t\t\thandlePath = handlePath.substr(1);\n\t\t\t\t\t}\n\t\t\t\t\t// Ensure only whole data stores can be reused, no reusing at deeper level!\n\t\t\t\t\tassert(level === 0, 0x0fc /* \"tree reuse at lower level\" */);\n\t\t\t\t\tassert(\n\t\t\t\t\t\t!handlePath.includes(\"/\"),\n\t\t\t\t\t\t0x0fd /* \"data stores are writing incremental summaries!\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase SummaryType.Attachment:\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.isRealStorageId(obj.id),\n\t\t\t\t\t\t0x0fe /* \"attachment is aggregate blob\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\tdefault:\n\t\t\t\t\tunreachableCase(obj, `Unknown type: ${(obj as any).type}`);\n\t\t\t}\n\t\t}\n\n\t\tassert(\n\t\t\tnewSummary.tree[this.aggregatedBlobName] === undefined,\n\t\t\t0x0ff /* \"duplicate aggregate blob\" */,\n\t\t);\n\t\tif (shouldCompress) {\n\t\t\t// Note: It would be great to add code here to unpack aggregate blob back to normal blobs\n\t\t\t// If only one blob made it into aggregate. Currently that does not happen as we always have\n\t\t\t// at least one .component blob and at least one DDS that has .attributes blob, so it's not an issue.\n\t\t\t// But it's possible that in future that would be great addition!\n\t\t\t// Good news - it's backward compatible change.\n\t\t\tassert(aggregator !== undefined, 0x100 /* \"logic error\" */);\n\t\t\tconst content = aggregator.getAggregatedBlobContent();\n\t\t\tif (content !== undefined) {\n\t\t\t\tnewSummary.tree[this.aggregatedBlobName] = {\n\t\t\t\t\ttype: SummaryType.Blob,\n\t\t\t\t\tcontent,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t\treturn newSummary;\n\t}\n\n\tprotected isRealStorageId(id: string): boolean {\n\t\treturn !id.startsWith(this.virtualIdPrefix);\n\t}\n}\n"]}
1
+ {"version":3,"file":"blobAggregationStorage.js","sourceRoot":"","sources":["../src/blobAggregationStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,EAMN,WAAW,GACX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACN,MAAM,EACN,cAAc,EACd,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAE5E;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAqB,EAAE,QAA4B;IAC3E,IAAI,IAAI,YAAY,UAAU,EAAE;QAC/B,8DAA8D;QAC9D,OAAO,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;KAC1C;IACD,OAAO,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,cAAc;IAApB;QACkB,YAAO,GAAuB,EAAE,CAAC;IAiBnD,CAAC;IAfO,OAAO,CAAC,GAAW,EAAE,OAAe;QAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACnC,CAAC;IAEM,wBAAwB;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,OAAO,SAAS,CAAC;SACjB;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAsB;QACjC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAC/C,CAAC;CACD;AAED;;;;;GAKG;AACH,MAAM,OAAgB,iBAAiB;IAAvC;QACoB,uBAAkB,GAAG,OAAO,CAAC;QAC7B,oBAAe,GAAG,IAAI,CAAC;QAE1C,gDAAgD;QACtC,qBAAgB,GAAG,CAAC,CAAC;IA8ChC,CAAC;IA7CU,gBAAgB;QACzB,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5D,CAAC;IAKM,KAAK,CAAC,kBAAkB,CAAC,QAAuB,EAAE,KAAK,GAAG,CAAC;QACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;SAC9C;QAED,4EAA4E;QAC5E,2CAA2C;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YAC9C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;gBAC7C,SAAS;aACT;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,MAAM,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAClD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACtD,MAAM,EAAE,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACnC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;oBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,OAAO,GAAG,QAAQ,CAAC;oBACvB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;wBAC/D,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE;4BACzC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;yBAClD;wBACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;qBACjC;oBACD,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACjD,MAAM,CACL,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,EACrC,KAAK,CAAC,2BAA2B,CACjC,CAAC;oBACF,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;iBAC7B;gBACD,gEAAgE;gBAChE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;aAC/C;SACD;IACF,CAAC;CACD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,wBAAyB,SAAQ,iBAAiB;IAChD,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,MAAM,CACL,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,SAAS,EAC5B,KAAK,CAAC,kDAAkD,CACxD,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;CACD;AAED;;;;;;GAMG;AACH,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IAgE5D,YACkB,OAAgC,EAChC,MAAwB,EACxB,YAAqB,EACrB,YAAoB,EACpB,cAAuB;QAExC,KAAK,EAAE,CAAC;QANS,YAAO,GAAP,OAAO,CAAyB;QAChC,WAAM,GAAN,MAAM,CAAkB;QACxB,iBAAY,GAAZ,YAAY,CAAS;QACrB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,mBAAc,GAAd,cAAc,CAAS;QAxD/B,sBAAiB,GAAG,KAAK,CAAC;QAE1B,iBAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;IAyD5D,CAAC;IAvDD,MAAM,CAAC,IAAI,CACV,OAAgC,EAChC,MAAwB,EACxB,YAAsB,EACtB,YAAY,GAAG,CAAC;;QAEhB,IAAI,OAAO,YAAY,sBAAsB,EAAE;YAC9C,OAAO,OAAO,CAAC;SACf;QACD,MAAM,EAAE,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,kBAAkB,GACvB,MAAA,MAAA,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,mCAAI,YAAY,mCAAI,KAAK,CAAC;QAEtE,kFAAkF;QAClF,4FAA4F;QAC5F,wDAAwD;QACxD,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,QAAQ,0CAAE,WAAW,CAAC;QAClD,OAAO,IAAI,sBAAsB,CAChC,OAAO,EACP,MAAM,EACN,kBAAkB,EAClB,YAAY,EACZ,WAAW,CACX,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,MAAM,SAAS,GAAG,IAAI,wBAAwB,EAAE,CAAC;QACjD,MAAM,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED,IAAW,QAAQ;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACvC,IAAI,QAAQ,EAAE;YACb,uCAAY,QAAQ,KAAE,WAAW,EAAE,SAAS,IAAG;SAC/C;IACF,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,QAAuB;QAClD,8FAA8F;QAC9F,qCAAqC;QACrC,6DAA6D;QAE7D,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAYM,OAAO,CAAC,EAAU,EAAE,IAAmB,EAAE,OAAe;QAC9D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAmB;QACnD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,IAAW,aAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACnC,CAAC;IACM,KAAK,CAAC,WAAW,CACvB,SAAwB,EACxB,KAAa,EACb,YAAqB,EACrB,WAAyB;QAEzB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAC9E,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,MAAsB;QAClD,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,wFAAwF;IACxF,yBAAyB;IAClB,KAAK,CAAC,UAAU,CAAC,IAAqB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAkB;QAC9C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,IAAI,EAAE;YACT,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;SAChC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,EAAU;QAC/B,IAAI,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;SACjC;QACD,iEAAiE;QACjE,6FAA6F;QAC7F,oGAAoG;QACpG,uCAAuC;QAEvC,mGAAmG;QACnG,eAAe;QACf,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,wBAAwB,CACpC,OAAqB,EACrB,OAAwB;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,OAAO,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED,qCAAqC;IACrC,8DAA8D;IAC9D,8GAA8G;IAC9G,mCAAmC;IACnC,gHAAgH;IAChH,yEAAyE;IACzE,QAAQ;IACR,+GAA+G;IAC/G,8GAA8G;IAC9G,+GAA+G;IAC/G,qHAAqH;IACrH,mDAAmD;IAC3C,KAAK,CAAC,kBAAkB,CAC/B,OAAqB,EACrB,IAAI,GAAG,EAAE,EACT,KAAK,GAAG,CAAC,EACT,aAA8B;QAE9B,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE;YACjE,OAAO,OAAO,CAAC;SACf;QAED,IAAI,cAAc,GAAY,KAAK,CAAC;QAEpC,IAAI,UAAU,GAAG,aAAa,CAAC;QAC/B,+EAA+E;QAC/E,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE;YACrD,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAC1E,MAAM,CACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;YACF,UAAU,GAAG,IAAI,cAAc,EAAE,CAAC;YAClC,cAAc,GAAG,IAAI,CAAC;SACtB;aAAM;YACN,MAAM,CACL,KAAK,KAAK,IAAI,CAAC,YAAY,EAC3B,KAAK,CAAC,6CAA6C,CACnD,CAAC;SACF;QAED,MAAM,UAAU,qBAAsB,OAAO,CAAE,CAAC;QAChD,UAAU,CAAC,IAAI,qBAAQ,UAAU,CAAC,IAAI,CAAE,CAAC;QACzC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,oEAAoE;YACpE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;YACxD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,WAAW,CAAC,IAAI;oBACpB,8CAA8C;oBAC9C,uEAAuE;oBACvE,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,KAAK,UAAU,EAAE;wBACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CACzC,GAAG,EACH,OAAO,EACP,KAAK,GAAG,CAAC,EACT,UAAU,CACV,CAAC;wBACF,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;wBAC5B,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;4BAClC,gEAAgE;4BAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;yBAC5B;qBACD;oBACD,MAAM;gBACP,KAAK,WAAW,CAAC,IAAI;oBACpB,IACC,UAAU;wBACV,OAAO,GAAG,CAAC,OAAO,IAAI,QAAQ;wBAC9B,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EACvC;wBACD,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;wBACzC,gEAAgE;wBAChE,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAC5B;oBACD,MAAM;gBACP,KAAK,WAAW,CAAC,MAAM,CAAC,CAAC;oBACxB,oBAAoB;oBACpB,yBAAyB;oBACzB,2EAA2E;oBAC3E,wFAAwF;oBACxF,IAAI,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC;oBAC5B,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;wBAC/B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;qBAClC;oBACD,2EAA2E;oBAC3E,MAAM,CAAC,KAAK,KAAK,CAAC,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBAC7D,MAAM,CACL,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EACzB,KAAK,CAAC,sDAAsD,CAC5D,CAAC;oBACF,MAAM;iBACN;gBACD,KAAK,WAAW,CAAC,UAAU;oBAC1B,MAAM,CACL,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,EAC5B,KAAK,CAAC,oCAAoC,CAC1C,CAAC;oBACF,MAAM;gBACP;oBACC,eAAe,CAAC,GAAG,EAAE,iBAAkB,GAAW,CAAC,IAAI,EAAE,CAAC,CAAC;aAC5D;SACD;QAED,MAAM,CACL,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,SAAS,EACtD,KAAK,CAAC,gCAAgC,CACtC,CAAC;QACF,IAAI,cAAc,EAAE;YACnB,yFAAyF;YACzF,4FAA4F;YAC5F,qGAAqG;YACrG,iEAAiE;YACjE,+CAA+C;YAC/C,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,UAAU,CAAC,wBAAwB,EAAE,CAAC;YACtD,IAAI,OAAO,KAAK,SAAS,EAAE;gBAC1B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG;oBAC1C,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,OAAO;iBACP,CAAC;aACF;SACD;QACD,OAAO,UAAU,CAAC;IACnB,CAAC;IAES,eAAe,CAAC,EAAU;QACnC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;;AA3QD,6FAA6F;AAC7F,8BAA8B;AAC9B,uGAAuG;AACvG,yGAAyG;AACzG,4FAA4F;AAC5F,qGAAqG;AACrG,oGAAoG;AACpG,mCAAmC;AACnC,yGAAyG;AACzG,mFAAmF;AACnE,6CAAsB,GAAG,IAAI,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport {\n\tFetchSource,\n\tIDocumentStorageService,\n\tIDocumentStorageServicePolicies,\n\tISummaryContext,\n} from \"@fluidframework/driver-definitions\";\nimport {\n\tICreateBlobResponse,\n\tISnapshotTree,\n\tISummaryHandle,\n\tISummaryTree,\n\tIVersion,\n\tSummaryType,\n} from \"@fluidframework/protocol-definitions\";\nimport {\n\tassert,\n\tbufferToString,\n\tstringToBuffer,\n\tunreachableCase,\n\tfromUtf8ToBase64,\n\tUint8ArrayToString,\n} from \"@fluidframework/common-utils\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { loggerToMonitoringContext } from \"@fluidframework/telemetry-utils\";\n\n/*\n * Work around for bufferToString having a bug - it can't consume IsoBuffer!\n * To be removed once bufferToString is fixed!\n */\nfunction bufferToString2(blob: ArrayBufferLike, encoding: \"utf-8\" | \"base64\"): string {\n\tif (blob instanceof Uint8Array) {\n\t\t// IsoBuffer does not have ctor, so it's not in proto chain :(\n\t\treturn Uint8ArrayToString(blob, encoding);\n\t}\n\treturn bufferToString(blob, encoding);\n}\n\n/**\n * Class responsible for aggregating smaller blobs into one and unpacking it later on.\n */\nclass BlobAggregator {\n\tprivate readonly content: [string, string][] = [];\n\n\tpublic addBlob(key: string, content: string) {\n\t\tthis.content.push([key, content]);\n\t}\n\n\tpublic getAggregatedBlobContent() {\n\t\tif (this.content.length === 0) {\n\t\t\treturn undefined;\n\t\t}\n\t\treturn JSON.stringify(this.content);\n\t}\n\n\tstatic load(input: ArrayBufferLike) {\n\t\tconst data = bufferToString2(input, \"utf-8\");\n\t\treturn JSON.parse(data) as [string, string][];\n\t}\n}\n\n/**\n * Base class that deals with unpacking snapshots (in place) containing aggregated blobs\n * It relies on abstract methods for reads and storing unpacked blobs.\n *\n * @deprecated This class was introduced experimentally and is no longer used.\n */\nexport abstract class SnapshotExtractor {\n\tprotected readonly aggregatedBlobName = \"__big\";\n\tprotected readonly virtualIdPrefix = \"__\";\n\n\t// counter for generation of virtual storage IDs\n\tprotected virtualIdCounter = 0;\n\tprotected getNextVirtualId() {\n\t\treturn `${this.virtualIdPrefix}${++this.virtualIdCounter}`;\n\t}\n\n\tabstract getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike>;\n\tabstract setBlob(id: string, tree: ISnapshotTree, content: string);\n\n\tpublic async unpackSnapshotCore(snapshot: ISnapshotTree, level = 0): Promise<void> {\n\t\tfor (const key of Object.keys(snapshot.trees)) {\n\t\t\tconst obj = snapshot.trees[key];\n\t\t\tawait this.unpackSnapshotCore(obj, level + 1);\n\t\t}\n\n\t\t// For future proof, we will support multiple aggregated blobs with any name\n\t\t// that starts with this.aggregatedBlobName\n\t\tfor (const key of Object.keys(snapshot.blobs)) {\n\t\t\tif (!key.startsWith(this.aggregatedBlobName)) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst blobId = snapshot.blobs[key];\n\t\t\tif (blobId !== undefined) {\n\t\t\t\tconst blob = await this.getBlob(blobId, snapshot);\n\t\t\t\tfor (const [path, value] of BlobAggregator.load(blob)) {\n\t\t\t\t\tconst id = this.getNextVirtualId();\n\t\t\t\t\tthis.setBlob(id, snapshot, value);\n\t\t\t\t\tconst pathSplit = path.split(\"/\");\n\t\t\t\t\tlet subTree = snapshot;\n\t\t\t\t\tfor (const subPath of pathSplit.slice(0, pathSplit.length - 1)) {\n\t\t\t\t\t\tif (subTree.trees[subPath] === undefined) {\n\t\t\t\t\t\t\tsubTree.trees[subPath] = { blobs: {}, trees: {} };\n\t\t\t\t\t\t}\n\t\t\t\t\t\tsubTree = subTree.trees[subPath];\n\t\t\t\t\t}\n\t\t\t\t\tconst blobName = pathSplit[pathSplit.length - 1];\n\t\t\t\t\tassert(\n\t\t\t\t\t\tsubTree.blobs[blobName] === undefined,\n\t\t\t\t\t\t0x0f6 /* \"real blob ID exists\" */,\n\t\t\t\t\t);\n\t\t\t\t\tsubTree.blobs[blobName] = id;\n\t\t\t\t}\n\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\tdelete snapshot.blobs[this.aggregatedBlobName];\n\t\t\t}\n\t\t}\n\t}\n}\n\n/*\n * Snapshot extractor class that works in place, i.e. patches snapshot that has\n * blob content in ISnapshotTree.blobs itself, not in storage.\n * As result, it implements reading and writing of blobs to/from snapshot itself.\n * It follows existing pattern that mixes concerns - ISnapshotTree.blobs is used for two\n * purposes:\n * 1. map path name to blob ID\n * 2. map blob ID to blob content\n * #2 is what storage (IDocumentStorageService) is for, but in places where we do not have it\n * (like loading serialized earlier draft content), blob content is put directly into snapshot.\n * Ideally this should be fixed by using BlobCacheStorageService or something similar and\n * fixing existing flows to allow switching of storage.\n */\nclass SnapshotExtractorInPlace extends SnapshotExtractor {\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\tconst blob = tree.blobs[id];\n\t\tassert(blob !== undefined, 0x0f7 /* \"aggregate blob missing\" */);\n\t\treturn stringToBuffer(blob, \"base64\");\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tassert(\n\t\t\ttree.blobs[id] === undefined,\n\t\t\t0x0f8 /* \"blob from aggregate blob exists on its own\" */,\n\t\t);\n\t\ttree.blobs[id] = fromUtf8ToBase64(content);\n\t}\n}\n\n/**\n * Snapshot packer and extractor.\n * When summary is written it will find and aggregate small blobs into bigger blobs\n * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.\n *\n * @deprecated This class was introduced experimentally and is no longer used.\n */\nexport class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {\n\t// Tells data store if it can use incremental summary (i.e. reuse DDSes from previous summary\n\t// when only one DDS changed).\n\t// The answer has to be know long before we enable actual packing. The reason for the is the following:\n\t// A the moment when we enable packing, we should assume that all clients out there wil already have bits\n\t// that can unpack properly (i.e. enough time passed since we deployed bits that can unpack)\n\t// But we can still have clients where some of them already pack, and some do not. If one summary was\n\t// using packing, then it relies on non-incremental summaries going forward, even if next client who\n\t// produced summary is not packing!\n\t// This can have slight improvement by enabling it per file (based on \"did summary we loaded from contain\n\t// aggregated blobs\"), but that's harder to make reliable, so going for simplicity.\n\tstatic readonly fullDataStoreSummaries = true;\n\n\tprotected loadedFromSummary = false;\n\n\tprotected virtualBlobs = new Map<string, ArrayBufferLike>();\n\n\tstatic wrap(\n\t\tstorage: IDocumentStorageService,\n\t\tlogger: ITelemetryLogger,\n\t\tallowPacking?: boolean,\n\t\tpackingLevel = 2,\n\t) {\n\t\tif (storage instanceof BlobAggregationStorage) {\n\t\t\treturn storage;\n\t\t}\n\t\tconst mc = loggerToMonitoringContext(logger);\n\t\tconst realAllowPackaging =\n\t\t\tmc.config.getBoolean(\"FluidAggregateBlobs\") ?? allowPacking ?? false;\n\n\t\t// Always create BlobAggregationStorage even if storage is not asking for packing.\n\t\t// This is mostly to avoid cases where future changes in policy would result in inability to\n\t\t// load old files that were created with aggregation on.\n\t\tconst minBlobSize = storage.policies?.minBlobSize;\n\t\treturn new BlobAggregationStorage(\n\t\t\tstorage,\n\t\t\tlogger,\n\t\t\trealAllowPackaging,\n\t\t\tpackingLevel,\n\t\t\tminBlobSize,\n\t\t);\n\t}\n\n\tstatic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\tconst converter = new SnapshotExtractorInPlace();\n\t\tawait converter.unpackSnapshotCore(snapshot);\n\t}\n\n\tpublic get policies(): IDocumentStorageServicePolicies | undefined {\n\t\tconst policies = this.storage.policies;\n\t\tif (policies) {\n\t\t\treturn { ...policies, minBlobSize: undefined };\n\t\t}\n\t}\n\n\tpublic async unpackSnapshot(snapshot: ISnapshotTree) {\n\t\t// SummarizerNodeWithGC.refreshLatestSummary can call it when this.loadedFromSummary === false\n\t\t// (I assumed after file was created)\n\t\t// assert(!this.loadedFromSummary, \"unpack without summary\");\n\n\t\tthis.loadedFromSummary = true;\n\t\tawait this.unpackSnapshotCore(snapshot);\n\t}\n\n\tprotected constructor(\n\t\tprivate readonly storage: IDocumentStorageService,\n\t\tprivate readonly logger: ITelemetryLogger,\n\t\tprivate readonly allowPacking: boolean,\n\t\tprivate readonly packingLevel: number,\n\t\tprivate readonly blobCutOffSize?: number,\n\t) {\n\t\tsuper();\n\t}\n\n\tpublic setBlob(id: string, tree: ISnapshotTree, content: string) {\n\t\tthis.virtualBlobs.set(id, stringToBuffer(content, \"utf-8\"));\n\t}\n\n\tpublic async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {\n\t\treturn this.readBlob(id).catch((error) => {\n\t\t\tthis.logger.sendErrorEvent({ eventName: \"BlobDedupNoAggregateBlob\" }, error);\n\t\t\tthrow error;\n\t\t});\n\t}\n\n\tpublic get repositoryUrl() {\n\t\treturn this.storage.repositoryUrl;\n\t}\n\tpublic async getVersions(\n\t\tversionId: string | null,\n\t\tcount: number,\n\t\tscenarioName?: string,\n\t\tfetchSource?: FetchSource,\n\t) {\n\t\treturn this.storage.getVersions(versionId, count, scenarioName, fetchSource);\n\t}\n\n\tpublic async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {\n\t\tthrow new Error(\"NYI\");\n\t}\n\n\t// for now we are not optimizing these blobs, with assumption that this API is used only\n\t// for big blobs (images)\n\tpublic async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {\n\t\treturn this.storage.createBlob(file);\n\t}\n\n\tpublic async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {\n\t\tconst tree = await this.storage.getSnapshotTree(version);\n\t\tif (tree) {\n\t\t\tawait this.unpackSnapshot(tree);\n\t\t}\n\t\treturn tree;\n\t}\n\n\tpublic async readBlob(id: string): Promise<ArrayBufferLike> {\n\t\tif (this.isRealStorageId(id)) {\n\t\t\treturn this.storage.readBlob(id);\n\t\t}\n\t\t// We support only reading blobs from the summary we loaded from.\n\t\t// This may need to be extended to any general summary in the future as runtime usage pattern\n\t\t// of storage changes (for example, data stores start to load from recent summary, not from original\n\t\t// summary whole container loaded from)\n\n\t\t// are there other ways we can get here? createFile is one flow, but we should not be reading blobs\n\t\t// in such flow\n\t\tassert(this.loadedFromSummary, 0x0f9 /* \"never read summary\" */);\n\t\tconst blob = this.virtualBlobs.get(id);\n\t\tassert(blob !== undefined, 0x0fa /* \"virtual blob not found\" */);\n\t\treturn blob;\n\t}\n\n\tpublic async uploadSummaryWithContext(\n\t\tsummary: ISummaryTree,\n\t\tcontext: ISummaryContext,\n\t): Promise<string> {\n\t\tconst summaryNew = this.allowPacking ? await this.compressSmallBlobs(summary) : summary;\n\t\treturn this.storage.uploadSummaryWithContext(summaryNew, context);\n\t}\n\n\t// For simplification, we assume that\n\t// - blob aggregation is done at data store level only for now\n\t// - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)\n\t// summary produced by data stores.\n\t// These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated\n\t// blob or contain aggregated blob that stays relevant for that sub-tree)\n\t// Note:\n\t// From perf perspective, it makes sense to place aggregated blobs one level up in the tree to not create extra\n\t// tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at\n\t// incremental summary time - we would need to understand handles and parse them. In current design we can skip\n\t// that step because if data store is reused, the whole sub-tree is reused including aggregated blob embedded into it\n\t// and that means we can do nothing and be correct!\n\tprivate async compressSmallBlobs(\n\t\tsummary: ISummaryTree,\n\t\tpath = \"\",\n\t\tlevel = 0,\n\t\taggregatorArg?: BlobAggregator,\n\t): Promise<ISummaryTree> {\n\t\tif (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {\n\t\t\treturn summary;\n\t\t}\n\n\t\tlet shouldCompress: boolean = false;\n\n\t\tlet aggregator = aggregatorArg;\n\t\t// checking if this is a dataStore tree, since we only pack at data store level\n\t\tif (Object.keys(summary.tree).includes(\".component\")) {\n\t\t\tassert(aggregator === undefined, 0x0fb /* \"logic err with aggregator\" */);\n\t\t\tassert(\n\t\t\t\tlevel === this.packingLevel,\n\t\t\t\t0x23b /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t\taggregator = new BlobAggregator();\n\t\t\tshouldCompress = true;\n\t\t} else {\n\t\t\tassert(\n\t\t\t\tlevel !== this.packingLevel,\n\t\t\t\t0x23c /* \"we are not packing at the right level\" */,\n\t\t\t);\n\t\t}\n\n\t\tconst newSummary: ISummaryTree = { ...summary };\n\t\tnewSummary.tree = { ...newSummary.tree };\n\t\tfor (const key of Object.keys(summary.tree)) {\n\t\t\tconst obj = summary.tree[key];\n\t\t\t// Get path relative to root of data store (where we do aggregation)\n\t\t\tconst newPath = shouldCompress ? key : `${path}/${key}`;\n\t\t\tswitch (obj.type) {\n\t\t\t\tcase SummaryType.Tree:\n\t\t\t\t\t// If client created empty tree, keep it as is\n\t\t\t\t\t// Also do not package search blobs - they are part of storage contract\n\t\t\t\t\tif (Object.keys(obj).length !== 0 && key !== \"__search\") {\n\t\t\t\t\t\tconst tree = await this.compressSmallBlobs(\n\t\t\t\t\t\t\tobj,\n\t\t\t\t\t\t\tnewPath,\n\t\t\t\t\t\t\tlevel + 1,\n\t\t\t\t\t\t\taggregator,\n\t\t\t\t\t\t);\n\t\t\t\t\t\tnewSummary.tree[key] = tree;\n\t\t\t\t\t\tif (Object.keys(obj).length === 0) {\n\t\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Blob:\n\t\t\t\t\tif (\n\t\t\t\t\t\taggregator &&\n\t\t\t\t\t\ttypeof obj.content == \"string\" &&\n\t\t\t\t\t\tobj.content.length < this.blobCutOffSize\n\t\t\t\t\t) {\n\t\t\t\t\t\taggregator.addBlob(newPath, obj.content);\n\t\t\t\t\t\t// eslint-disable-next-line @typescript-eslint/no-dynamic-delete\n\t\t\t\t\t\tdelete newSummary.tree[key];\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t\tcase SummaryType.Handle: {\n\t\t\t\t\t// Would be nice to:\n\t\t\t\t\t// Trees: expand the tree\n\t\t\t\t\t// Blobs: parse handle and ensure it points to real blob, not virtual blob.\n\t\t\t\t\t// We can avoid it for now given data store is the granularity of incremental summaries.\n\t\t\t\t\tlet handlePath = obj.handle;\n\t\t\t\t\tif (handlePath.startsWith(\"/\")) {\n\t\t\t\t\t\thandlePath = handlePath.substr(1);\n\t\t\t\t\t}\n\t\t\t\t\t// Ensure only whole data stores can be reused, no reusing at deeper level!\n\t\t\t\t\tassert(level === 0, 0x0fc /* \"tree reuse at lower level\" */);\n\t\t\t\t\tassert(\n\t\t\t\t\t\t!handlePath.includes(\"/\"),\n\t\t\t\t\t\t0x0fd /* \"data stores are writing incremental summaries!\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tcase SummaryType.Attachment:\n\t\t\t\t\tassert(\n\t\t\t\t\t\tthis.isRealStorageId(obj.id),\n\t\t\t\t\t\t0x0fe /* \"attachment is aggregate blob\" */,\n\t\t\t\t\t);\n\t\t\t\t\tbreak;\n\t\t\t\tdefault:\n\t\t\t\t\tunreachableCase(obj, `Unknown type: ${(obj as any).type}`);\n\t\t\t}\n\t\t}\n\n\t\tassert(\n\t\t\tnewSummary.tree[this.aggregatedBlobName] === undefined,\n\t\t\t0x0ff /* \"duplicate aggregate blob\" */,\n\t\t);\n\t\tif (shouldCompress) {\n\t\t\t// Note: It would be great to add code here to unpack aggregate blob back to normal blobs\n\t\t\t// If only one blob made it into aggregate. Currently that does not happen as we always have\n\t\t\t// at least one .component blob and at least one DDS that has .attributes blob, so it's not an issue.\n\t\t\t// But it's possible that in future that would be great addition!\n\t\t\t// Good news - it's backward compatible change.\n\t\t\tassert(aggregator !== undefined, 0x100 /* \"logic error\" */);\n\t\t\tconst content = aggregator.getAggregatedBlobContent();\n\t\t\tif (content !== undefined) {\n\t\t\t\tnewSummary.tree[this.aggregatedBlobName] = {\n\t\t\t\t\ttype: SummaryType.Blob,\n\t\t\t\t\tcontent,\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t\treturn newSummary;\n\t}\n\n\tprotected isRealStorageId(id: string): boolean {\n\t\treturn !id.startsWith(this.virtualIdPrefix);\n\t}\n}\n"]}
package/lib/packageVersion.d.ts CHANGED
@@ -5,5 +5,5 @@
5
5
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
6
6
  */
7
7
  export declare const pkgName = "@fluidframework/driver-utils";
8
- export declare const pkgVersion = "2.0.0-internal.3.1.0";
8
+ export declare const pkgVersion = "2.0.0-internal.3.2.1";
9
9
  //# sourceMappingURL=packageVersion.d.ts.map
package/lib/packageVersion.js CHANGED
@@ -5,5 +5,5 @@
5
5
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
6
6
  */
7
7
  export const pkgName = "@fluidframework/driver-utils";
8
- export const pkgVersion = "2.0.0-internal.3.1.0";
8
+ export const pkgVersion = "2.0.0-internal.3.2.1";
9
9
  //# sourceMappingURL=packageVersion.js.map
package/lib/packageVersion.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,8BAA8B,CAAC;AACtD,MAAM,CAAC,MAAM,UAAU,GAAG,sBAAsB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/driver-utils\";\nexport const pkgVersion = \"2.0.0-internal.3.1.0\";\n"]}
1
+ {"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,8BAA8B,CAAC;AACtD,MAAM,CAAC,MAAM,UAAU,GAAG,sBAAsB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/driver-utils\";\nexport const pkgVersion = \"2.0.0-internal.3.2.1\";\n"]}
package/package.json CHANGED
@@ -1,130 +1,108 @@
1
1
  {
2
- "name": "@fluidframework/driver-utils",
3
- "version": "2.0.0-internal.3.1.0",
4
- "description": "Collection of utility functions for Fluid drivers",
5
- "homepage": "https://fluidframework.com",
6
- "repository": {
7
- "type": "git",
8
- "url": "https://github.com/microsoft/FluidFramework.git",
9
- "directory": "packages/loader/driver-utils"
10
- },
11
- "license": "MIT",
12
- "author": "Microsoft and contributors",
13
- "sideEffects": false,
14
- "main": "dist/index.js",
15
- "module": "lib/index.js",
16
- "types": "dist/index.d.ts",
17
- "scripts": {
18
- "build": "npm run build:genver && concurrently npm:build:compile npm:lint && npm run build:docs",
19
- "build:commonjs": "npm run tsc && npm run typetests:gen && npm run build:test",
20
- "build:compile": "concurrently npm:build:commonjs npm:build:esnext",
21
- "build:docs": "api-extractor run --local --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
22
- "build:esnext": "tsc --project ./tsconfig.esnext.json",
23
- "build:full": "npm run build",
24
- "build:full:compile": "npm run build:compile",
25
- "build:genver": "gen-version",
26
- "build:test": "tsc --project ./src/test/tsconfig.json",
27
- "ci:build:docs": "api-extractor run --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/* ../../../_api-extractor-temp/",
28
- "clean": "rimraf dist lib *.tsbuildinfo *.build.log",
29
- "eslint": "eslint --format stylish src",
30
- "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
31
- "format": "npm run prettier:fix",
32
- "lint": "npm run prettier && npm run eslint",
33
- "lint:fix": "npm run prettier:fix && npm run eslint:fix",
34
- "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
35
- "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
36
- "test": "npm run test:mocha",
37
- "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
38
- "test:mocha": "mocha --ignore 'dist/test/types/*' --recursive dist/test -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
39
- "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
40
- "tsc": "tsc",
41
- "typetests:gen": "flub generate typetests --generate --dir .",
42
- "typetests:prepare": "flub generate typetests --prepare --dir . --pin"
43
- },
44
- "nyc": {
45
- "all": true,
46
- "cache-dir": "nyc/.cache",
47
- "exclude": [
48
- "src/test/**/*.ts",
49
- "dist/test/**/*.js"
50
- ],
51
- "exclude-after-remap": false,
52
- "include": [
53
- "src/**/*.ts",
54
- "dist/**/*.js"
55
- ],
56
- "report-dir": "nyc/report",
57
- "reporter": [
58
- "cobertura",
59
- "html",
60
- "text"
61
- ],
62
- "temp-directory": "nyc/.nyc_output"
63
- },
64
- "dependencies": {
65
- "@fluidframework/common-definitions": "^0.20.1",
66
- "@fluidframework/common-utils": "^1.0.0",
67
- "@fluidframework/core-interfaces": ">=2.0.0-internal.3.1.0 <2.0.0-internal.4.0.0",
68
- "@fluidframework/driver-definitions": ">=2.0.0-internal.3.1.0 <2.0.0-internal.4.0.0",
69
- "@fluidframework/gitresources": "^0.1038.2000",
70
- "@fluidframework/protocol-base": "^0.1038.2000",
71
- "@fluidframework/protocol-definitions": "^1.1.0",
72
- "@fluidframework/telemetry-utils": ">=2.0.0-internal.3.1.0 <2.0.0-internal.4.0.0",
73
- "axios": "^0.26.0",
74
- "url": "^0.11.0",
75
- "uuid": "^8.3.1"
76
- },
77
- "devDependencies": {
78
- "@fluid-tools/build-cli": "^0.9.0",
79
- "@fluidframework/build-common": "^1.1.0",
80
- "@fluidframework/build-tools": "^0.9.0",
81
- "@fluidframework/driver-utils-previous": "npm:@fluidframework/driver-utils@2.0.0-internal.3.0.0",
82
- "@fluidframework/eslint-config-fluid": "^2.0.0",
83
- "@fluidframework/mocha-test-setup": ">=2.0.0-internal.3.1.0 <2.0.0-internal.4.0.0",
84
- "@fluidframework/runtime-utils": ">=2.0.0-internal.3.1.0 <2.0.0-internal.4.0.0",
85
- "@microsoft/api-extractor": "^7.22.2",
86
- "@rushstack/eslint-config": "^2.5.1",
87
- "@types/mocha": "^9.1.1",
88
- "@types/node": "^14.18.36",
89
- "@types/sinon": "^7.0.13",
90
- "concurrently": "^6.2.0",
91
- "copyfiles": "^2.4.1",
92
- "cross-env": "^7.0.2",
93
- "eslint": "~8.6.0",
94
- "mocha": "^10.0.0",
95
- "nyc": "^15.0.0",
96
- "prettier": "~2.6.2",
97
- "rimraf": "^2.6.2",
98
- "sinon": "^7.4.2",
99
- "typescript": "~4.5.5"
100
- },
101
- "typeValidation": {
102
- "version": "2.0.0-internal.3.1.0",
103
- "previousVersionStyle": "~previousMinor",
104
- "baselineRange": ">=2.0.0-internal.3.0.0 <2.0.0-internal.3.1.0",
105
- "baselineVersion": "2.0.0-internal.3.0.0",
106
- "broken": {
107
- "ClassDeclaration_AuthorizationError": {
108
- "backCompat": false
109
- },
110
- "ClassDeclaration_DeltaStreamConnectionForbiddenError": {
111
- "backCompat": false
112
- },
113
- "ClassDeclaration_FluidInvalidSchemaError": {
114
- "backCompat": false
115
- },
116
- "ClassDeclaration_GenericNetworkError": {
117
- "backCompat": false
118
- },
119
- "ClassDeclaration_LocationRedirectionError": {
120
- "backCompat": false
121
- },
122
- "ClassDeclaration_ThrottlingError": {
123
- "backCompat": false
124
- },
125
- "ClassDeclaration_UsageError": {
126
- "backCompat": false
127
- }
128
- }
129
- }
130
- }
2
+ "name": "@fluidframework/driver-utils",
3
+ "version": "2.0.0-internal.3.2.1",
4
+ "description": "Collection of utility functions for Fluid drivers",
5
+ "homepage": "https://fluidframework.com",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "https://github.com/microsoft/FluidFramework.git",
9
+ "directory": "packages/loader/driver-utils"
10
+ },
11
+ "license": "MIT",
12
+ "author": "Microsoft and contributors",
13
+ "sideEffects": false,
14
+ "main": "dist/index.js",
15
+ "module": "lib/index.js",
16
+ "types": "dist/index.d.ts",
17
+ "nyc": {
18
+ "all": true,
19
+ "cache-dir": "nyc/.cache",
20
+ "exclude": [
21
+ "src/test/**/*.ts",
22
+ "dist/test/**/*.js"
23
+ ],
24
+ "exclude-after-remap": false,
25
+ "include": [
26
+ "src/**/*.ts",
27
+ "dist/**/*.js"
28
+ ],
29
+ "report-dir": "nyc/report",
30
+ "reporter": [
31
+ "cobertura",
32
+ "html",
33
+ "text"
34
+ ],
35
+ "temp-directory": "nyc/.nyc_output"
36
+ },
37
+ "dependencies": {
38
+ "@fluidframework/common-definitions": "^0.20.1",
39
+ "@fluidframework/common-utils": "^1.1.1",
40
+ "@fluidframework/core-interfaces": ">=2.0.0-internal.3.2.1 <2.0.0-internal.4.0.0",
41
+ "@fluidframework/driver-definitions": ">=2.0.0-internal.3.2.1 <2.0.0-internal.4.0.0",
42
+ "@fluidframework/gitresources": "^0.1038.2000",
43
+ "@fluidframework/protocol-base": "^0.1038.2000",
44
+ "@fluidframework/protocol-definitions": "^1.1.0",
45
+ "@fluidframework/telemetry-utils": ">=2.0.0-internal.3.2.1 <2.0.0-internal.4.0.0",
46
+ "axios": "^0.26.0",
47
+ "url": "^0.11.0",
48
+ "uuid": "^8.3.1"
49
+ },
50
+ "devDependencies": {
51
+ "@fluid-tools/build-cli": "^0.10.0",
52
+ "@fluidframework/build-common": "^1.1.0",
53
+ "@fluidframework/build-tools": "^0.10.0",
54
+ "@fluidframework/driver-utils-previous": "npm:@fluidframework/driver-utils@2.0.0-internal.3.1.0",
55
+ "@fluidframework/eslint-config-fluid": "^2.0.0",
56
+ "@fluidframework/mocha-test-setup": ">=2.0.0-internal.3.2.1 <2.0.0-internal.4.0.0",
57
+ "@fluidframework/runtime-utils": ">=2.0.0-internal.3.2.1 <2.0.0-internal.4.0.0",
58
+ "@microsoft/api-extractor": "^7.22.2",
59
+ "@rushstack/eslint-config": "^2.5.1",
60
+ "@types/mocha": "^9.1.1",
61
+ "@types/node": "^14.18.36",
62
+ "@types/sinon": "^7.0.13",
63
+ "concurrently": "^6.2.0",
64
+ "copyfiles": "^2.4.1",
65
+ "cross-env": "^7.0.2",
66
+ "eslint": "~8.6.0",
67
+ "mocha": "^10.0.0",
68
+ "nyc": "^15.0.0",
69
+ "prettier": "~2.6.2",
70
+ "rimraf": "^2.6.2",
71
+ "sinon": "^7.4.2",
72
+ "typescript": "~4.5.5"
73
+ },
74
+ "typeValidation": {
75
+ "version": "2.0.0-internal.3.2.0",
76
+ "previousVersionStyle": "~previousMinor",
77
+ "baselineRange": ">=2.0.0-internal.3.1.0 <2.0.0-internal.3.2.0",
78
+ "baselineVersion": "2.0.0-internal.3.1.0",
79
+ "broken": {}
80
+ },
81
+ "scripts": {
82
+ "build": "npm run build:genver && concurrently npm:build:compile npm:lint && npm run build:docs",
83
+ "build:commonjs": "npm run tsc && npm run typetests:gen && npm run build:test",
84
+ "build:compile": "concurrently npm:build:commonjs npm:build:esnext",
85
+ "build:docs": "api-extractor run --local --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/doc-models/* ../../../_api-extractor-temp/",
86
+ "build:esnext": "tsc --project ./tsconfig.esnext.json",
87
+ "build:full": "npm run build",
88
+ "build:full:compile": "npm run build:compile",
89
+ "build:genver": "gen-version",
90
+ "build:test": "tsc --project ./src/test/tsconfig.json",
91
+ "ci:build:docs": "api-extractor run --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/* ../../../_api-extractor-temp/",
92
+ "clean": "rimraf dist lib *.tsbuildinfo *.build.log",
93
+ "eslint": "eslint --format stylish src",
94
+ "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
95
+ "format": "npm run prettier:fix",
96
+ "lint": "npm run prettier && npm run eslint",
97
+ "lint:fix": "npm run prettier:fix && npm run eslint:fix",
98
+ "prettier": "prettier --check . --ignore-path ../../../.prettierignore",
99
+ "prettier:fix": "prettier --write . --ignore-path ../../../.prettierignore",
100
+ "test": "npm run test:mocha",
101
+ "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
102
+ "test:mocha": "mocha --ignore 'dist/test/types/*' --recursive dist/test -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
103
+ "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
104
+ "tsc": "tsc",
105
+ "typetests:gen": "flub generate typetests --generate --dir .",
106
+ "typetests:prepare": "flub generate typetests --prepare --dir . --pin"
107
+ }
108
+ }
package/src/blobAggregationStorage.ts CHANGED
@@ -63,9 +63,11 @@ class BlobAggregator {
63
63
  }
64
64
  }
65
65
 
66
- /*
66
+ /**
67
67
  * Base class that deals with unpacking snapshots (in place) containing aggregated blobs
68
68
  * It relies on abstract methods for reads and storing unpacked blobs.
69
+ *
70
+ * @deprecated This class was introduced experimentally and is no longer used.
69
71
  */
70
72
  export abstract class SnapshotExtractor {
71
73
  protected readonly aggregatedBlobName = "__big";
@@ -149,10 +151,12 @@ class SnapshotExtractorInPlace extends SnapshotExtractor {
149
151
  }
150
152
  }
151
153
 
152
- /*
154
+ /**
153
155
  * Snapshot packer and extractor.
154
156
  * When summary is written it will find and aggregate small blobs into bigger blobs
155
157
  * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.
158
+ *
159
+ * @deprecated This class was introduced experimentally and is no longer used.
156
160
  */
157
161
  export class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {
158
162
  // Tells data store if it can use incremental summary (i.e. reuse DDSes from previous summary
@@ -299,12 +303,12 @@ export class BlobAggregationStorage extends SnapshotExtractor implements IDocume
299
303
  // - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)
300
304
  // summary produced by data stores.
301
305
  // These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated
302
- // blob Or contain aggregated blob that stays relevant for that sub-tree)
306
+ // blob or contain aggregated blob that stays relevant for that sub-tree)
303
307
  // Note:
304
- // From perf perspective, it makes sense to place aggregated blobs one level up in the tree not to create extra
308
+ // From perf perspective, it makes sense to place aggregated blobs one level up in the tree to not create extra
305
309
  // tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at
306
310
  // incremental summary time - we would need to understand handles and parse them. In current design we can skip
307
- // that step because if data store is reused, the hole sub-tree is reused included aggregated blob embedded into it
311
+ // that step because if data store is reused, the whole sub-tree is reused including aggregated blob embedded into it
308
312
  // and that means we can do nothing and be correct!
309
313
  private async compressSmallBlobs(
310
314
  summary: ISummaryTree,
@@ -345,7 +349,7 @@ export class BlobAggregationStorage extends SnapshotExtractor implements IDocume
345
349
  case SummaryType.Tree:
346
350
  // If client created empty tree, keep it as is
347
351
  // Also do not package search blobs - they are part of storage contract
348
- if (obj.tree !== {} && key !== "__search") {
352
+ if (Object.keys(obj).length !== 0 && key !== "__search") {
349
353
  const tree = await this.compressSmallBlobs(
350
354
  obj,
351
355
  newPath,
@@ -353,7 +357,7 @@ export class BlobAggregationStorage extends SnapshotExtractor implements IDocume
353
357
  aggregator,
354
358
  );
355
359
  newSummary.tree[key] = tree;
356
- if (tree.tree === {}) {
360
+ if (Object.keys(obj).length === 0) {
357
361
  // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
358
362
  delete newSummary.tree[key];
359
363
  }
package/src/packageVersion.ts CHANGED
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/driver-utils";
9
- export const pkgVersion = "2.0.0-internal.3.1.0";
9
+ export const pkgVersion = "2.0.0-internal.3.2.1";