@fluidframework/driver-utils 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/CHANGELOG.md +5 -0
  2. package/README.md +68 -1
  3. package/dist/fluidResolvedUrl.d.ts +8 -0
  4. package/dist/fluidResolvedUrl.d.ts.map +1 -1
  5. package/dist/fluidResolvedUrl.js +8 -0
  6. package/dist/fluidResolvedUrl.js.map +1 -1
  7. package/dist/index.d.ts +4 -11
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +2 -21
  10. package/dist/index.js.map +1 -1
  11. package/dist/messageRecognition.d.ts +0 -20
  12. package/dist/messageRecognition.d.ts.map +1 -1
  13. package/dist/messageRecognition.js +1 -37
  14. package/dist/messageRecognition.js.map +1 -1
  15. package/dist/network.d.ts +0 -15
  16. package/dist/network.d.ts.map +1 -1
  17. package/dist/network.js.map +1 -1
  18. package/dist/networkUtils.d.ts +0 -10
  19. package/dist/networkUtils.d.ts.map +1 -1
  20. package/dist/networkUtils.js +1 -32
  21. package/dist/networkUtils.js.map +1 -1
  22. package/dist/packageVersion.d.ts +1 -1
  23. package/dist/packageVersion.js +1 -1
  24. package/dist/packageVersion.js.map +1 -1
  25. package/dist/parallelRequests.d.ts.map +1 -1
  26. package/dist/parallelRequests.js +45 -14
  27. package/dist/parallelRequests.js.map +1 -1
  28. package/dist/summaryForCreateNew.d.ts +21 -1
  29. package/dist/summaryForCreateNew.d.ts.map +1 -1
  30. package/dist/summaryForCreateNew.js +27 -5
  31. package/dist/summaryForCreateNew.js.map +1 -1
  32. package/dist/treeConversions.d.ts.map +1 -1
  33. package/dist/treeConversions.js +8 -9
  34. package/dist/treeConversions.js.map +1 -1
  35. package/lib/fluidResolvedUrl.d.ts +8 -0
  36. package/lib/fluidResolvedUrl.d.ts.map +1 -1
  37. package/lib/fluidResolvedUrl.js +8 -0
  38. package/lib/fluidResolvedUrl.js.map +1 -1
  39. package/lib/index.d.ts +4 -11
  40. package/lib/index.d.ts.map +1 -1
  41. package/lib/index.js +3 -10
  42. package/lib/index.js.map +1 -1
  43. package/lib/messageRecognition.d.ts +0 -20
  44. package/lib/messageRecognition.d.ts.map +1 -1
  45. package/lib/messageRecognition.js +0 -35
  46. package/lib/messageRecognition.js.map +1 -1
  47. package/lib/network.d.ts +0 -15
  48. package/lib/network.d.ts.map +1 -1
  49. package/lib/network.js.map +1 -1
  50. package/lib/networkUtils.d.ts +0 -10
  51. package/lib/networkUtils.d.ts.map +1 -1
  52. package/lib/networkUtils.js +0 -30
  53. package/lib/networkUtils.js.map +1 -1
  54. package/lib/packageVersion.d.ts +1 -1
  55. package/lib/packageVersion.js +1 -1
  56. package/lib/packageVersion.js.map +1 -1
  57. package/lib/parallelRequests.d.ts.map +1 -1
  58. package/lib/parallelRequests.js +46 -15
  59. package/lib/parallelRequests.js.map +1 -1
  60. package/lib/summaryForCreateNew.d.ts +21 -1
  61. package/lib/summaryForCreateNew.d.ts.map +1 -1
  62. package/lib/summaryForCreateNew.js +25 -4
  63. package/lib/summaryForCreateNew.js.map +1 -1
  64. package/lib/treeConversions.d.ts.map +1 -1
  65. package/lib/treeConversions.js +8 -9
  66. package/lib/treeConversions.js.map +1 -1
  67. package/package.json +52 -75
  68. package/src/fluidResolvedUrl.ts +8 -0
  69. package/src/index.ts +4 -19
  70. package/src/messageRecognition.ts +0 -36
  71. package/src/network.ts +0 -16
  72. package/src/networkUtils.ts +0 -32
  73. package/src/packageVersion.ts +1 -1
  74. package/src/parallelRequests.ts +54 -19
  75. package/src/summaryForCreateNew.ts +49 -5
  76. package/src/treeConversions.ts +8 -9
  77. package/dist/blobAggregationStorage.d.ts +0 -43
  78. package/dist/blobAggregationStorage.d.ts.map +0 -1
  79. package/dist/blobAggregationStorage.js +0 -323
  80. package/dist/blobAggregationStorage.js.map +0 -1
  81. package/dist/blobCacheStorageService.d.ts +0 -16
  82. package/dist/blobCacheStorageService.d.ts.map +0 -1
  83. package/dist/blobCacheStorageService.js +0 -29
  84. package/dist/blobCacheStorageService.js.map +0 -1
  85. package/dist/emptyDocumentDeltaStorageService.d.ts +0 -13
  86. package/dist/emptyDocumentDeltaStorageService.d.ts.map +0 -1
  87. package/dist/emptyDocumentDeltaStorageService.js +0 -20
  88. package/dist/emptyDocumentDeltaStorageService.js.map +0 -1
  89. package/dist/mapWithExpiration.d.ts +0 -34
  90. package/dist/mapWithExpiration.d.ts.map +0 -1
  91. package/dist/mapWithExpiration.js +0 -105
  92. package/dist/mapWithExpiration.js.map +0 -1
  93. package/dist/multiDocumentServiceFactory.d.ts +0 -16
  94. package/dist/multiDocumentServiceFactory.d.ts.map +0 -1
  95. package/dist/multiDocumentServiceFactory.js +0 -63
  96. package/dist/multiDocumentServiceFactory.js.map +0 -1
  97. package/dist/multiUrlResolver.d.ts +0 -20
  98. package/dist/multiUrlResolver.d.ts.map +0 -1
  99. package/dist/multiUrlResolver.js +0 -45
  100. package/dist/multiUrlResolver.js.map +0 -1
  101. package/dist/treeUtils.d.ts +0 -51
  102. package/dist/treeUtils.d.ts.map +0 -1
  103. package/dist/treeUtils.js +0 -85
  104. package/dist/treeUtils.js.map +0 -1
  105. package/lib/blobAggregationStorage.d.ts +0 -43
  106. package/lib/blobAggregationStorage.d.ts.map +0 -1
  107. package/lib/blobAggregationStorage.js +0 -318
  108. package/lib/blobAggregationStorage.js.map +0 -1
  109. package/lib/blobCacheStorageService.d.ts +0 -16
  110. package/lib/blobCacheStorageService.d.ts.map +0 -1
  111. package/lib/blobCacheStorageService.js +0 -25
  112. package/lib/blobCacheStorageService.js.map +0 -1
  113. package/lib/emptyDocumentDeltaStorageService.d.ts +0 -13
  114. package/lib/emptyDocumentDeltaStorageService.d.ts.map +0 -1
  115. package/lib/emptyDocumentDeltaStorageService.js +0 -16
  116. package/lib/emptyDocumentDeltaStorageService.js.map +0 -1
  117. package/lib/mapWithExpiration.d.ts +0 -34
  118. package/lib/mapWithExpiration.d.ts.map +0 -1
  119. package/lib/mapWithExpiration.js +0 -101
  120. package/lib/mapWithExpiration.js.map +0 -1
  121. package/lib/multiDocumentServiceFactory.d.ts +0 -16
  122. package/lib/multiDocumentServiceFactory.d.ts.map +0 -1
  123. package/lib/multiDocumentServiceFactory.js +0 -59
  124. package/lib/multiDocumentServiceFactory.js.map +0 -1
  125. package/lib/multiUrlResolver.d.ts +0 -20
  126. package/lib/multiUrlResolver.d.ts.map +0 -1
  127. package/lib/multiUrlResolver.js +0 -40
  128. package/lib/multiUrlResolver.js.map +0 -1
  129. package/lib/treeUtils.d.ts +0 -51
  130. package/lib/treeUtils.d.ts.map +0 -1
  131. package/lib/treeUtils.js +0 -80
  132. package/lib/treeUtils.js.map +0 -1
  133. package/src/blobAggregationStorage.ts +0 -420
  134. package/src/blobCacheStorageService.ts +0 -35
  135. package/src/emptyDocumentDeltaStorageService.ts +0 -25
  136. package/src/mapWithExpiration.ts +0 -124
  137. package/src/multiDocumentServiceFactory.ts +0 -91
  138. package/src/multiUrlResolver.ts +0 -48
  139. package/src/treeUtils.ts +0 -107
@@ -1,420 +0,0 @@
1
- /*!
2
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
- * Licensed under the MIT License.
4
- */
5
-
6
- import {
7
- IDocumentStorageService,
8
- IDocumentStorageServicePolicies,
9
- ISummaryContext,
10
- } from "@fluidframework/driver-definitions";
11
- import {
12
- ICreateBlobResponse,
13
- ISnapshotTree,
14
- ISummaryHandle,
15
- ISummaryTree,
16
- IVersion,
17
- SummaryType,
18
- } from "@fluidframework/protocol-definitions";
19
- import {
20
- assert,
21
- bufferToString,
22
- stringToBuffer,
23
- unreachableCase,
24
- fromUtf8ToBase64,
25
- Uint8ArrayToString,
26
- } from "@fluidframework/common-utils";
27
- import { ITelemetryLogger } from "@fluidframework/common-definitions";
28
- import { loggerToMonitoringContext } from "@fluidframework/telemetry-utils";
29
-
30
- /*
31
- * Work around for bufferToString having a bug - it can't consume IsoBuffer!
32
- * To be removed once bufferToString is fixed!
33
- */
34
- function bufferToString2(blob: ArrayBufferLike, encoding: "utf-8" | "base64"): string {
35
- if (blob instanceof Uint8Array) {
36
- // IsoBuffer does not have ctor, so it's not in proto chain :(
37
- return Uint8ArrayToString(blob, encoding);
38
- }
39
- return bufferToString(blob, encoding);
40
- }
41
-
42
- /**
43
- * Class responsible for aggregating smaller blobs into one and unpacking it later on.
44
- */
45
- class BlobAggregator {
46
- private readonly content: [string, string][] = [];
47
-
48
- public addBlob(key: string, content: string) {
49
- this.content.push([key, content]);
50
- }
51
-
52
- public getAggregatedBlobContent() {
53
- if (this.content.length === 0) {
54
- return undefined;
55
- }
56
- return JSON.stringify(this.content);
57
- }
58
-
59
- static load(input: ArrayBufferLike) {
60
- const data = bufferToString2(input, "utf-8");
61
- return JSON.parse(data) as [string, string][];
62
- }
63
- }
64
-
65
- /*
66
- * Base class that deals with unpacking snapshots (in place) containing aggregated blobs
67
- * It relies on abstract methods for reads and storing unpacked blobs.
68
- */
69
- export abstract class SnapshotExtractor {
70
- protected readonly aggregatedBlobName = "__big";
71
- protected readonly virtualIdPrefix = "__";
72
-
73
- // counter for generation of virtual storage IDs
74
- protected virtualIdCounter = 0;
75
- protected getNextVirtualId() {
76
- return `${this.virtualIdPrefix}${++this.virtualIdCounter}`;
77
- }
78
-
79
- abstract getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike>;
80
- abstract setBlob(id: string, tree: ISnapshotTree, content: string);
81
-
82
- public async unpackSnapshotCore(snapshot: ISnapshotTree, level = 0): Promise<void> {
83
- for (const key of Object.keys(snapshot.trees)) {
84
- const obj = snapshot.trees[key];
85
- await this.unpackSnapshotCore(obj, level + 1);
86
- }
87
-
88
- // For future proof, we will support multiple aggregated blobs with any name
89
- // that starts with this.aggregatedBlobName
90
- for (const key of Object.keys(snapshot.blobs)) {
91
- if (!key.startsWith(this.aggregatedBlobName)) {
92
- continue;
93
- }
94
- const blobId = snapshot.blobs[key];
95
- if (blobId !== undefined) {
96
- const blob = await this.getBlob(blobId, snapshot);
97
- for (const [path, value] of BlobAggregator.load(blob)) {
98
- const id = this.getNextVirtualId();
99
- this.setBlob(id, snapshot, value);
100
- const pathSplit = path.split("/");
101
- let subTree = snapshot;
102
- for (const subPath of pathSplit.slice(0, pathSplit.length - 1)) {
103
- if (subTree.trees[subPath] === undefined) {
104
- subTree.trees[subPath] = { blobs: {}, trees: {} };
105
- }
106
- subTree = subTree.trees[subPath];
107
- }
108
- const blobName = pathSplit[pathSplit.length - 1];
109
- assert(
110
- subTree.blobs[blobName] === undefined,
111
- 0x0f6 /* "real blob ID exists" */,
112
- );
113
- subTree.blobs[blobName] = id;
114
- }
115
- // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
116
- delete snapshot.blobs[this.aggregatedBlobName];
117
- }
118
- }
119
- }
120
- }
121
-
122
- /*
123
- * Snapshot extractor class that works in place, i.e. patches snapshot that has
124
- * blob content in ISnapshotTree.blobs itself, not in storage.
125
- * As result, it implements reading and writing of blobs to/from snapshot itself.
126
- * It follows existing pattern that mixes concerns - ISnapshotTree.blobs is used for two
127
- * purposes:
128
- * 1. map path name to blob ID
129
- * 2. map blob ID to blob content
130
- * #2 is what storage (IDocumentStorageService) is for, but in places where we do not have it
131
- * (like loading serialized earlier draft content), blob content is put directly into snapshot.
132
- * Ideally this should be fixed by using BlobCacheStorageService or something similar and
133
- * fixing existing flows to allow switching of storage.
134
- */
135
- class SnapshotExtractorInPlace extends SnapshotExtractor {
136
- public async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {
137
- const blob = tree.blobs[id];
138
- assert(blob !== undefined, 0x0f7 /* "aggregate blob missing" */);
139
- return stringToBuffer(blob, "base64");
140
- }
141
-
142
- public setBlob(id: string, tree: ISnapshotTree, content: string) {
143
- assert(
144
- tree.blobs[id] === undefined,
145
- 0x0f8 /* "blob from aggregate blob exists on its own" */,
146
- );
147
- tree.blobs[id] = fromUtf8ToBase64(content);
148
- }
149
- }
150
-
151
- /*
152
- * Snapshot packer and extractor.
153
- * When summary is written it will find and aggregate small blobs into bigger blobs
154
- * When snapshot is read, it will unpack aggregated blobs and provide them transparently to caller.
155
- */
156
- export class BlobAggregationStorage extends SnapshotExtractor implements IDocumentStorageService {
157
- // Tells data store if it can use incremental summary (i.e. reuse DDSes from previous summary
158
- // when only one DDS changed).
159
- // The answer has to be know long before we enable actual packing. The reason for the is the following:
160
- // A the moment when we enable packing, we should assume that all clients out there wil already have bits
161
- // that can unpack properly (i.e. enough time passed since we deployed bits that can unpack)
162
- // But we can still have clients where some of them already pack, and some do not. If one summary was
163
- // using packing, then it relies on non-incremental summaries going forward, even if next client who
164
- // produced summary is not packing!
165
- // This can have slight improvement by enabling it per file (based on "did summary we loaded from contain
166
- // aggregated blobs"), but that's harder to make reliable, so going for simplicity.
167
- static readonly fullDataStoreSummaries = true;
168
-
169
- protected loadedFromSummary = false;
170
-
171
- protected virtualBlobs = new Map<string, ArrayBufferLike>();
172
-
173
- static wrap(
174
- storage: IDocumentStorageService,
175
- logger: ITelemetryLogger,
176
- allowPacking?: boolean,
177
- packingLevel = 2,
178
- ) {
179
- if (storage instanceof BlobAggregationStorage) {
180
- return storage;
181
- }
182
- const mc = loggerToMonitoringContext(logger);
183
- const realAllowPackaging =
184
- mc.config.getBoolean("FluidAggregateBlobs") ?? allowPacking ?? false;
185
-
186
- // Always create BlobAggregationStorage even if storage is not asking for packing.
187
- // This is mostly to avoid cases where future changes in policy would result in inability to
188
- // load old files that were created with aggregation on.
189
- const minBlobSize = storage.policies?.minBlobSize;
190
- return new BlobAggregationStorage(
191
- storage,
192
- logger,
193
- realAllowPackaging,
194
- packingLevel,
195
- minBlobSize,
196
- );
197
- }
198
-
199
- static async unpackSnapshot(snapshot: ISnapshotTree) {
200
- const converter = new SnapshotExtractorInPlace();
201
- await converter.unpackSnapshotCore(snapshot);
202
- }
203
-
204
- public get policies(): IDocumentStorageServicePolicies | undefined {
205
- const policies = this.storage.policies;
206
- if (policies) {
207
- return { ...policies, minBlobSize: undefined };
208
- }
209
- }
210
-
211
- public async unpackSnapshot(snapshot: ISnapshotTree) {
212
- // SummarizerNodeWithGC.refreshLatestSummary can call it when this.loadedFromSummary === false
213
- // (I assumed after file was created)
214
- // assert(!this.loadedFromSummary, "unpack without summary");
215
-
216
- this.loadedFromSummary = true;
217
- await this.unpackSnapshotCore(snapshot);
218
- }
219
-
220
- protected constructor(
221
- private readonly storage: IDocumentStorageService,
222
- private readonly logger: ITelemetryLogger,
223
- private readonly allowPacking: boolean,
224
- private readonly packingLevel: number,
225
- private readonly blobCutOffSize?: number,
226
- ) {
227
- super();
228
- }
229
-
230
- public setBlob(id: string, tree: ISnapshotTree, content: string) {
231
- this.virtualBlobs.set(id, stringToBuffer(content, "utf-8"));
232
- }
233
-
234
- public async getBlob(id: string, tree: ISnapshotTree): Promise<ArrayBufferLike> {
235
- return this.readBlob(id).catch((error) => {
236
- this.logger.sendErrorEvent({ eventName: "BlobDedupNoAggregateBlob" }, error);
237
- throw error;
238
- });
239
- }
240
-
241
- public get repositoryUrl() {
242
- return this.storage.repositoryUrl;
243
- }
244
- public async getVersions(versionId: string | null, count: number) {
245
- return this.storage.getVersions(versionId, count);
246
- }
247
-
248
- public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
249
- throw new Error("NYI");
250
- }
251
-
252
- // for now we are not optimizing these blobs, with assumption that this API is used only
253
- // for big blobs (images)
254
- public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
255
- return this.storage.createBlob(file);
256
- }
257
-
258
- public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {
259
- const tree = await this.storage.getSnapshotTree(version);
260
- if (tree) {
261
- await this.unpackSnapshot(tree);
262
- }
263
- return tree;
264
- }
265
-
266
- public async readBlob(id: string): Promise<ArrayBufferLike> {
267
- if (this.isRealStorageId(id)) {
268
- return this.storage.readBlob(id);
269
- }
270
- // We support only reading blobs from the summary we loaded from.
271
- // This may need to be extended to any general summary in the future as runtime usage pattern
272
- // of storage changes (for example, data stores start to load from recent summary, not from original
273
- // summary whole container loaded from)
274
-
275
- // are there other ways we can get here? createFile is one flow, but we should not be reading blobs
276
- // in such flow
277
- assert(this.loadedFromSummary, 0x0f9 /* "never read summary" */);
278
- const blob = this.virtualBlobs.get(id);
279
- assert(blob !== undefined, 0x0fa /* "virtual blob not found" */);
280
- return blob;
281
- }
282
-
283
- public async uploadSummaryWithContext(
284
- summary: ISummaryTree,
285
- context: ISummaryContext,
286
- ): Promise<string> {
287
- const summaryNew = this.allowPacking ? await this.compressSmallBlobs(summary) : summary;
288
- return this.storage.uploadSummaryWithContext(summaryNew, context);
289
- }
290
-
291
- // For simplification, we assume that
292
- // - blob aggregation is done at data store level only for now
293
- // - data store either reuses previous summary, or generates full summary, i.e. there is no partial (some DDS)
294
- // summary produced by data stores.
295
- // These simplifications allow us not to touch handles, as they are self-contained (either do not use aggregated
296
- // blob Or contain aggregated blob that stays relevant for that sub-tree)
297
- // Note:
298
- // From perf perspective, it makes sense to place aggregated blobs one level up in the tree not to create extra
299
- // tree nodes (i.e. have shallow tree with less edges). But that creates problems with reusability of trees at
300
- // incremental summary time - we would need to understand handles and parse them. In current design we can skip
301
- // that step because if data store is reused, the hole sub-tree is reused included aggregated blob embedded into it
302
- // and that means we can do nothing and be correct!
303
- private async compressSmallBlobs(
304
- summary: ISummaryTree,
305
- path = "",
306
- level = 0,
307
- aggregatorArg?: BlobAggregator,
308
- ): Promise<ISummaryTree> {
309
- if (this.blobCutOffSize === undefined || this.blobCutOffSize < 0) {
310
- return summary;
311
- }
312
-
313
- let shouldCompress: boolean = false;
314
-
315
- let aggregator = aggregatorArg;
316
- // checking if this is a dataStore tree, since we only pack at data store level
317
- if (Object.keys(summary.tree).includes(".component")) {
318
- assert(aggregator === undefined, 0x0fb /* "logic err with aggregator" */);
319
- assert(
320
- level === this.packingLevel,
321
- 0x23b /* "we are not packing at the right level" */,
322
- );
323
- aggregator = new BlobAggregator();
324
- shouldCompress = true;
325
- } else {
326
- assert(
327
- level !== this.packingLevel,
328
- 0x23c /* "we are not packing at the right level" */,
329
- );
330
- }
331
-
332
- const newSummary: ISummaryTree = { ...summary };
333
- newSummary.tree = { ...newSummary.tree };
334
- for (const key of Object.keys(summary.tree)) {
335
- const obj = summary.tree[key];
336
- // Get path relative to root of data store (where we do aggregation)
337
- const newPath = shouldCompress ? key : `${path}/${key}`;
338
- switch (obj.type) {
339
- case SummaryType.Tree:
340
- // If client created empty tree, keep it as is
341
- // Also do not package search blobs - they are part of storage contract
342
- if (obj.tree !== {} && key !== "__search") {
343
- const tree = await this.compressSmallBlobs(
344
- obj,
345
- newPath,
346
- level + 1,
347
- aggregator,
348
- );
349
- newSummary.tree[key] = tree;
350
- if (tree.tree === {}) {
351
- // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
352
- delete newSummary.tree[key];
353
- }
354
- }
355
- break;
356
- case SummaryType.Blob:
357
- if (
358
- aggregator &&
359
- typeof obj.content == "string" &&
360
- obj.content.length < this.blobCutOffSize
361
- ) {
362
- aggregator.addBlob(newPath, obj.content);
363
- // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
364
- delete newSummary.tree[key];
365
- }
366
- break;
367
- case SummaryType.Handle: {
368
- // Would be nice to:
369
- // Trees: expand the tree
370
- // Blobs: parse handle and ensure it points to real blob, not virtual blob.
371
- // We can avoid it for now given data store is the granularity of incremental summaries.
372
- let handlePath = obj.handle;
373
- if (handlePath.startsWith("/")) {
374
- handlePath = handlePath.substr(1);
375
- }
376
- // Ensure only whole data stores can be reused, no reusing at deeper level!
377
- assert(level === 0, 0x0fc /* "tree reuse at lower level" */);
378
- assert(
379
- !handlePath.includes("/"),
380
- 0x0fd /* "data stores are writing incremental summaries!" */,
381
- );
382
- break;
383
- }
384
- case SummaryType.Attachment:
385
- assert(
386
- this.isRealStorageId(obj.id),
387
- 0x0fe /* "attachment is aggregate blob" */,
388
- );
389
- break;
390
- default:
391
- unreachableCase(obj, `Unknown type: ${(obj as any).type}`);
392
- }
393
- }
394
-
395
- assert(
396
- newSummary.tree[this.aggregatedBlobName] === undefined,
397
- 0x0ff /* "duplicate aggregate blob" */,
398
- );
399
- if (shouldCompress) {
400
- // Note: It would be great to add code here to unpack aggregate blob back to normal blobs
401
- // If only one blob made it into aggregate. Currently that does not happen as we always have
402
- // at least one .component blob and at least one DDS that has .attributes blob, so it's not an issue.
403
- // But it's possible that in future that would be great addition!
404
- // Good news - it's backward compatible change.
405
- assert(aggregator !== undefined, 0x100 /* "logic error" */);
406
- const content = aggregator.getAggregatedBlobContent();
407
- if (content !== undefined) {
408
- newSummary.tree[this.aggregatedBlobName] = {
409
- type: SummaryType.Blob,
410
- content,
411
- };
412
- }
413
- }
414
- return newSummary;
415
- }
416
-
417
- protected isRealStorageId(id: string): boolean {
418
- return !id.startsWith(this.virtualIdPrefix);
419
- }
420
- }
@@ -1,35 +0,0 @@
1
- /*!
2
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
- * Licensed under the MIT License.
4
- */
5
-
6
- import {
7
- IDocumentStorageService,
8
- IDocumentStorageServicePolicies,
9
- } from "@fluidframework/driver-definitions";
10
- import { DocumentStorageServiceProxy } from "./documentStorageServiceProxy";
11
-
12
- /**
13
- * IDocumentStorageService adapter with pre-cached blobs.
14
- */
15
- export class BlobCacheStorageService extends DocumentStorageServiceProxy {
16
- constructor(
17
- internalStorageService: IDocumentStorageService,
18
- private readonly blobs: Map<string, ArrayBufferLike>,
19
- ) {
20
- super(internalStorageService);
21
- }
22
-
23
- public get policies(): IDocumentStorageServicePolicies | undefined {
24
- return this.internalStorageService.policies;
25
- }
26
-
27
- public async readBlob(id: string): Promise<ArrayBufferLike> {
28
- const blob = this.blobs.get(id);
29
- if (blob !== undefined) {
30
- return blob;
31
- }
32
-
33
- return this.internalStorageService.readBlob(id);
34
- }
35
- }
@@ -1,25 +0,0 @@
1
- /*!
2
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
- * Licensed under the MIT License.
4
- */
5
-
6
- import { IDocumentDeltaStorageService, IStream } from "@fluidframework/driver-definitions";
7
- import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
8
- import { Queue } from "./parallelRequests";
9
-
10
- /**
11
- * Implementation of IDocumentDeltaStorageService that will always return an empty message queue when fetching messages
12
- */
13
- export class EmptyDocumentDeltaStorageService implements IDocumentDeltaStorageService {
14
- public fetchMessages(
15
- from: number,
16
- _to: number | undefined,
17
- _abortSignal?: AbortSignal,
18
- _cachedOnly?: boolean,
19
- _fetchReason?: string,
20
- ): IStream<ISequencedDocumentMessage[]> {
21
- const queue = new Queue<ISequencedDocumentMessage[]>();
22
- queue.pushDone();
23
- return queue;
24
- }
25
- }
@@ -1,124 +0,0 @@
1
- /*!
2
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
- * Licensed under the MIT License.
4
- */
5
-
6
- import { assert } from "@fluidframework/common-utils";
7
-
8
- /**
9
- * An extension of Map that expires (deletes) entries after a period of inactivity.
10
- * The policy is based on the last time a key was written to.
11
- */
12
- export class MapWithExpiration<TKey = any, TValue = any> extends Map<TKey, TValue> {
13
- /** Timestamps (as epoch ms numbers) of when each key was last refreshed */
14
- private readonly lastRefreshedTimes = new Map<TKey, number>();
15
-
16
- constructor(private readonly expiryMs: number) {
17
- super();
18
- }
19
-
20
- private refresh(key: TKey): void {
21
- this.lastRefreshedTimes.set(key, new Date().valueOf());
22
- }
23
-
24
- /**
25
- * Returns true if the key is present and expired, false if it's not expired, and undefined if it's not found
26
- * If cleanUp is passed as true, then delete any expired entry before returning.
27
- */
28
- private checkExpiry(key: TKey, cleanUp: boolean = false): boolean | undefined {
29
- const refreshTime = this.lastRefreshedTimes.get(key);
30
- assert(
31
- (refreshTime !== undefined) === super.has(key),
32
- 0x50c /* freshness map out of sync */,
33
- );
34
-
35
- if (refreshTime === undefined) {
36
- return undefined;
37
- }
38
- const expired = new Date().valueOf() - refreshTime >= this.expiryMs;
39
- if (expired && cleanUp) {
40
- this.delete(key);
41
- }
42
- return expired;
43
- }
44
-
45
- private clearExpiredEntries() {
46
- // forEach clears out any expired entries
47
- this.forEach(() => {});
48
- }
49
-
50
- get size(): number {
51
- this.clearExpiredEntries();
52
- return super.size;
53
- }
54
-
55
- has(key: TKey): boolean {
56
- this.checkExpiry(key, true /* cleanUp */);
57
- return super.has(key);
58
- }
59
-
60
- get(key: TKey): TValue | undefined {
61
- this.checkExpiry(key, true /* cleanUp */);
62
- return super.get(key);
63
- }
64
-
65
- set(key: TKey, value: TValue): this {
66
- // Sliding window expiration policy (on write)
67
- this.refresh(key);
68
- return super.set(key, value);
69
- }
70
-
71
- delete(key: TKey): boolean {
72
- this.lastRefreshedTimes.delete(key);
73
- return super.delete(key);
74
- }
75
-
76
- clear(): void {
77
- this.lastRefreshedTimes.clear();
78
- super.clear();
79
- }
80
-
81
- forEach(
82
- callbackfn: (value: TValue, key: TKey, map: Map<TKey, TValue>) => void,
83
- thisArg?: any,
84
- ): void {
85
- const expiredKeys: TKey[] = [];
86
- super.forEach(
87
- (v, k, m) => {
88
- if (this.checkExpiry(k) === true) {
89
- expiredKeys.push(k);
90
- } else {
91
- callbackfn.bind(thisArg)(v, k, m);
92
- }
93
- },
94
- // Note we don't pass thisArg here, since we bind it directly to callbackFn and don't need it otherwise
95
- );
96
-
97
- // Clean up keys we know are expired now that we're done iterating
98
- expiredKeys.forEach((key: TKey) => {
99
- this.delete(key);
100
- });
101
- }
102
-
103
- entries(): IterableIterator<[TKey, TValue]> {
104
- this.clearExpiredEntries();
105
- return super.entries();
106
- }
107
- keys(): IterableIterator<TKey> {
108
- this.clearExpiredEntries();
109
- return super.keys();
110
- }
111
- values(): IterableIterator<TValue> {
112
- this.clearExpiredEntries();
113
- return super.values();
114
- }
115
- [Symbol.iterator](): IterableIterator<[TKey, TValue]> {
116
- this.clearExpiredEntries();
117
- return super[Symbol.iterator]();
118
- }
119
-
120
- valueOf() {
121
- this.clearExpiredEntries();
122
- return super.valueOf();
123
- }
124
- }
@@ -1,91 +0,0 @@
1
- /*!
2
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
- * Licensed under the MIT License.
4
- */
5
-
6
- import { parse } from "url";
7
- import {
8
- IDocumentServiceFactory,
9
- IResolvedUrl,
10
- IDocumentService,
11
- } from "@fluidframework/driver-definitions";
12
- import { ISummaryTree } from "@fluidframework/protocol-definitions";
13
- import { ITelemetryBaseLogger } from "@fluidframework/common-definitions";
14
- import { ensureFluidResolvedUrl } from "./fluidResolvedUrl";
15
-
16
- export class MultiDocumentServiceFactory implements IDocumentServiceFactory {
17
- public static create(
18
- documentServiceFactory: IDocumentServiceFactory | IDocumentServiceFactory[],
19
- ) {
20
- if (Array.isArray(documentServiceFactory)) {
21
- const factories: IDocumentServiceFactory[] = [];
22
- documentServiceFactory.forEach((factory) => {
23
- const maybeMulti = factory as MultiDocumentServiceFactory;
24
- if (maybeMulti.protocolToDocumentFactoryMap !== undefined) {
25
- factories.push(...maybeMulti.protocolToDocumentFactoryMap.values());
26
- } else {
27
- factories.push(factory);
28
- }
29
- });
30
- if (factories.length === 1) {
31
- return factories[0];
32
- }
33
- return new MultiDocumentServiceFactory(factories);
34
- }
35
- return documentServiceFactory;
36
- }
37
-
38
- private readonly protocolToDocumentFactoryMap: Map<string, IDocumentServiceFactory>;
39
-
40
- constructor(documentServiceFactories: IDocumentServiceFactory[]) {
41
- this.protocolToDocumentFactoryMap = new Map();
42
- documentServiceFactories.forEach((factory: IDocumentServiceFactory) => {
43
- this.protocolToDocumentFactoryMap.set(factory.protocolName, factory);
44
- });
45
- }
46
- public readonly protocolName = "none:";
47
- async createDocumentService(
48
- resolvedUrl: IResolvedUrl,
49
- logger?: ITelemetryBaseLogger,
50
- clientIsSummarizer?: boolean,
51
- ): Promise<IDocumentService> {
52
- ensureFluidResolvedUrl(resolvedUrl);
53
- const urlObj = parse(resolvedUrl.url);
54
- if (urlObj.protocol === undefined || urlObj.protocol === null) {
55
- throw new Error("No protocol provided");
56
- }
57
- const factory: IDocumentServiceFactory | undefined = this.protocolToDocumentFactoryMap.get(
58
- urlObj.protocol,
59
- );
60
- if (factory === undefined) {
61
- throw new Error("Unknown Fluid protocol");
62
- }
63
-
64
- return factory.createDocumentService(resolvedUrl, logger, clientIsSummarizer);
65
- }
66
-
67
- public async createContainer(
68
- createNewSummary: ISummaryTree,
69
- createNewResolvedUrl: IResolvedUrl,
70
- logger?: ITelemetryBaseLogger,
71
- clientIsSummarizer?: boolean,
72
- ): Promise<IDocumentService> {
73
- ensureFluidResolvedUrl(createNewResolvedUrl);
74
- const urlObj = parse(createNewResolvedUrl.url);
75
- if (urlObj.protocol === undefined || urlObj.protocol === null) {
76
- throw new Error("No protocol provided");
77
- }
78
- const factory: IDocumentServiceFactory | undefined = this.protocolToDocumentFactoryMap.get(
79
- urlObj.protocol,
80
- );
81
- if (factory === undefined) {
82
- throw new Error("Unknown Fluid protocol");
83
- }
84
- return factory.createContainer(
85
- createNewSummary,
86
- createNewResolvedUrl,
87
- logger,
88
- clientIsSummarizer,
89
- );
90
- }
91
- }