@fluidframework/container-loader 2.100.0 → 2.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/api-report/container-loader.legacy.alpha.api.md +13 -1
- package/dist/captureReferencedContents.d.ts +154 -0
- package/dist/captureReferencedContents.d.ts.map +1 -0
- package/dist/captureReferencedContents.js +349 -0
- package/dist/captureReferencedContents.js.map +1 -0
- package/dist/connectionManager.d.ts.map +1 -1
- package/dist/connectionManager.js +25 -7
- package/dist/connectionManager.js.map +1 -1
- package/dist/connectionStateHandler.d.ts.map +1 -1
- package/dist/connectionStateHandler.js +3 -1
- package/dist/connectionStateHandler.js.map +1 -1
- package/dist/container.d.ts.map +1 -1
- package/dist/container.js +6 -1
- package/dist/container.js.map +1 -1
- package/dist/containerStorageAdapter.d.ts +19 -1
- package/dist/containerStorageAdapter.d.ts.map +1 -1
- package/dist/containerStorageAdapter.js.map +1 -1
- package/dist/createAndLoadContainerUtils.d.ts +95 -0
- package/dist/createAndLoadContainerUtils.d.ts.map +1 -1
- package/dist/createAndLoadContainerUtils.js +137 -11
- package/dist/createAndLoadContainerUtils.js.map +1 -1
- package/dist/frozenServices.d.ts +113 -30
- package/dist/frozenServices.d.ts.map +1 -1
- package/dist/frozenServices.js +236 -58
- package/dist/frozenServices.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/legacyAlpha.d.ts +2 -0
- package/dist/loaderLayerCompatState.d.ts +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingLocalStateStore.d.ts.map +1 -1
- package/dist/pendingLocalStateStore.js +9 -3
- package/dist/pendingLocalStateStore.js.map +1 -1
- package/dist/serializedStateManager.d.ts +16 -1
- package/dist/serializedStateManager.d.ts.map +1 -1
- package/dist/serializedStateManager.js +11 -1
- package/dist/serializedStateManager.js.map +1 -1
- package/lib/captureReferencedContents.d.ts +154 -0
- package/lib/captureReferencedContents.d.ts.map +1 -0
- package/lib/captureReferencedContents.js +338 -0
- package/lib/captureReferencedContents.js.map +1 -0
- package/lib/connectionManager.d.ts.map +1 -1
- package/lib/connectionManager.js +26 -8
- package/lib/connectionManager.js.map +1 -1
- package/lib/connectionStateHandler.d.ts.map +1 -1
- package/lib/connectionStateHandler.js +3 -1
- package/lib/connectionStateHandler.js.map +1 -1
- package/lib/container.d.ts.map +1 -1
- package/lib/container.js +6 -1
- package/lib/container.js.map +1 -1
- package/lib/containerStorageAdapter.d.ts +19 -1
- package/lib/containerStorageAdapter.d.ts.map +1 -1
- package/lib/containerStorageAdapter.js.map +1 -1
- package/lib/createAndLoadContainerUtils.d.ts +95 -0
- package/lib/createAndLoadContainerUtils.d.ts.map +1 -1
- package/lib/createAndLoadContainerUtils.js +128 -3
- package/lib/createAndLoadContainerUtils.js.map +1 -1
- package/lib/frozenServices.d.ts +113 -30
- package/lib/frozenServices.d.ts.map +1 -1
- package/lib/frozenServices.js +233 -57
- package/lib/frozenServices.js.map +1 -1
- package/lib/index.d.ts +2 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -1
- package/lib/index.js.map +1 -1
- package/lib/legacyAlpha.d.ts +2 -0
- package/lib/loaderLayerCompatState.d.ts +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingLocalStateStore.d.ts.map +1 -1
- package/lib/pendingLocalStateStore.js +9 -3
- package/lib/pendingLocalStateStore.js.map +1 -1
- package/lib/serializedStateManager.d.ts +16 -1
- package/lib/serializedStateManager.d.ts.map +1 -1
- package/lib/serializedStateManager.js +11 -1
- package/lib/serializedStateManager.js.map +1 -1
- package/package.json +11 -11
- package/src/captureReferencedContents.ts +446 -0
- package/src/connectionManager.ts +30 -8
- package/src/connectionStateHandler.ts +14 -9
- package/src/container.ts +6 -0
- package/src/containerStorageAdapter.ts +20 -1
- package/src/createAndLoadContainerUtils.ts +229 -2
- package/src/frozenServices.ts +285 -64
- package/src/index.ts +7 -0
- package/src/packageVersion.ts +1 -1
- package/src/pendingLocalStateStore.ts +8 -1
- package/src/serializedStateManager.ts +28 -1
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { bufferToString } from "@fluid-internal/client-utils";
|
|
7
|
+
import type {
|
|
8
|
+
IDocumentStorageService,
|
|
9
|
+
ISequencedDocumentMessage,
|
|
10
|
+
ISnapshot,
|
|
11
|
+
ISnapshotTree,
|
|
12
|
+
} from "@fluidframework/driver-definitions/internal";
|
|
13
|
+
import { readAndParse } from "@fluidframework/driver-utils/internal";
|
|
14
|
+
|
|
15
|
+
import type {
|
|
16
|
+
IBase64BlobContents,
|
|
17
|
+
ISerializableBlobContents,
|
|
18
|
+
} from "./containerStorageAdapter.js";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Wire-format constants this module needs to walk and filter snapshots.
|
|
22
|
+
* Authoritative definitions live in `container-runtime` and
|
|
23
|
+
* `runtime-definitions`; the values are duplicated here to avoid a
|
|
24
|
+
* loader → runtime layering dependency. A contract test in
|
|
25
|
+
* `packages/test/local-server-tests` asserts these match the authoritative
|
|
26
|
+
* values; do not change them in isolation.
|
|
27
|
+
*
|
|
28
|
+
* Authoritative sources:
|
|
29
|
+
* - `blobsTreeName`, `redirectTableBlobName`: `packages/runtime/container-runtime/src/blobManager/blobManagerSnapSum.ts`
|
|
30
|
+
* - `blobManagerBasePath`: `packages/runtime/container-runtime/src/blobManager/blobManager.ts`
|
|
31
|
+
* - `gcTreeKey`, `gcBlobPrefix`, `gcTombstoneBlobKey`, `gcDeletedBlobKey`: `packages/runtime/runtime-definitions/src/garbageCollectionDefinitions.ts`
|
|
32
|
+
*
|
|
33
|
+
* @internal
|
|
34
|
+
*/
|
|
35
|
+
export const wireFormatConstants = {
|
|
36
|
+
blobsTreeName: ".blobs",
|
|
37
|
+
redirectTableBlobName: ".redirectTable",
|
|
38
|
+
blobManagerBasePath: "_blobs",
|
|
39
|
+
gcTreeKey: "gc",
|
|
40
|
+
gcBlobPrefix: "__gc",
|
|
41
|
+
gcTombstoneBlobKey: "__tombstones",
|
|
42
|
+
gcDeletedBlobKey: "__deletedNodes",
|
|
43
|
+
} as const;
|
|
44
|
+
|
|
45
|
+
const {
|
|
46
|
+
blobsTreeName,
|
|
47
|
+
redirectTableBlobName,
|
|
48
|
+
blobManagerBasePath,
|
|
49
|
+
gcTreeKey,
|
|
50
|
+
gcBlobPrefix,
|
|
51
|
+
gcTombstoneBlobKey,
|
|
52
|
+
gcDeletedBlobKey,
|
|
53
|
+
} = wireFormatConstants;
|
|
54
|
+
|
|
55
|
+
interface IGcNodeData {
|
|
56
|
+
outboundRoutes: string[];
|
|
57
|
+
unreferencedTimestampMs?: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
interface IGcState {
|
|
61
|
+
gcNodes: { [id: string]: IGcNodeData };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The parsed subset of the `gc` subtree that drives reachability decisions.
|
|
66
|
+
*/
|
|
67
|
+
export interface IGcSnapshotData {
|
|
68
|
+
gcState: IGcState | undefined;
|
|
69
|
+
tombstones: string[] | undefined;
|
|
70
|
+
deletedNodes: string[] | undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Reader that returns a blob's contents for a given storage id. */
|
|
74
|
+
type BlobReader = (id: string) => Promise<ArrayBufferLike>;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Upper bound on concurrent `readBlob` calls. Driver/service back-pressure is
|
|
78
|
+
* real for large documents, and unbounded `Promise.all` can trigger throttling
|
|
79
|
+
* or spike memory. The value is a pragmatic middle ground — high enough to
|
|
80
|
+
* keep a typical driver's request pipeline full, low enough to avoid storms.
|
|
81
|
+
*/
|
|
82
|
+
const maxReadConcurrency = 32;
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Runs `fn` over `items` with at most `limit` promises in flight. Preserves
|
|
86
|
+
* input order on output (not that any caller depends on it today).
|
|
87
|
+
*
|
|
88
|
+
* Exported for unit tests; not part of the package public API.
|
|
89
|
+
*
|
|
90
|
+
* @internal
|
|
91
|
+
*/
|
|
92
|
+
export async function mapWithConcurrency<T, R>(
|
|
93
|
+
items: readonly T[],
|
|
94
|
+
limit: number,
|
|
95
|
+
fn: (item: T) => Promise<R>,
|
|
96
|
+
): Promise<R[]> {
|
|
97
|
+
const results: R[] = Array.from({ length: items.length });
|
|
98
|
+
let cursor = 0;
|
|
99
|
+
const workerCount = Math.min(limit, items.length);
|
|
100
|
+
const workers = Array.from({ length: workerCount }, async () => {
|
|
101
|
+
while (cursor < items.length) {
|
|
102
|
+
const index = cursor++;
|
|
103
|
+
const item = items[index];
|
|
104
|
+
if (item !== undefined) {
|
|
105
|
+
results[index] = await fn(item);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
await Promise.all(workers);
|
|
110
|
+
return results;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Parses the `gc` subtree of a base snapshot. Returns `undefined` if the
|
|
115
|
+
* snapshot has no GC tree (GC disabled or pre-GC document).
|
|
116
|
+
*/
|
|
117
|
+
export async function parseGcSnapshotData(
|
|
118
|
+
baseSnapshot: ISnapshotTree,
|
|
119
|
+
storage: Pick<IDocumentStorageService, "readBlob">,
|
|
120
|
+
): Promise<IGcSnapshotData | undefined> {
|
|
121
|
+
const gcSnapshotTree: ISnapshotTree | undefined = baseSnapshot.trees[gcTreeKey];
|
|
122
|
+
if (gcSnapshotTree === undefined) {
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
let gcState: IGcState | undefined;
|
|
126
|
+
let tombstones: string[] | undefined;
|
|
127
|
+
let deletedNodes: string[] | undefined;
|
|
128
|
+
for (const [key, blobId] of Object.entries(gcSnapshotTree.blobs)) {
|
|
129
|
+
if (key === gcDeletedBlobKey) {
|
|
130
|
+
deletedNodes = await readAndParse<string[]>(storage, blobId);
|
|
131
|
+
} else if (key === gcTombstoneBlobKey) {
|
|
132
|
+
tombstones = await readAndParse<string[]>(storage, blobId);
|
|
133
|
+
} else if (key.startsWith(gcBlobPrefix)) {
|
|
134
|
+
const partial = await readAndParse<IGcState>(storage, blobId);
|
|
135
|
+
if (gcState === undefined) {
|
|
136
|
+
gcState = { gcNodes: { ...partial.gcNodes } };
|
|
137
|
+
} else {
|
|
138
|
+
for (const [nodeId, nodeData] of Object.entries(partial.gcNodes)) {
|
|
139
|
+
gcState.gcNodes[nodeId] ??= nodeData;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return { gcState, tombstones, deletedNodes };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Walks a snapshot and inlines the contents of every blob reachable without
|
|
149
|
+
* crossing an `unreferenced` subtree boundary. Subtrees flagged
|
|
150
|
+
* `unreferenced: true` are skipped entirely — the summarizer sets that flag
|
|
151
|
+
* from GC state, so honouring it filters out dead subtrees without a
|
|
152
|
+
* separate GC-path traversal.
|
|
153
|
+
*
|
|
154
|
+
* The root-level `.blobs` subtree is special-cased: only its `.redirectTable`
|
|
155
|
+
* blob is read, because attachment blob contents are captured separately via
|
|
156
|
+
* {@link captureReferencedAttachmentBlobs}.
|
|
157
|
+
*/
|
|
158
|
+
export async function readReferencedSnapshotBlobs(
|
|
159
|
+
snapshot: ISnapshot | ISnapshotTree,
|
|
160
|
+
storage: Pick<IDocumentStorageService, "readBlob">,
|
|
161
|
+
): Promise<ISerializableBlobContents> {
|
|
162
|
+
const { tree, read } = toTreeAndReader(snapshot, storage);
|
|
163
|
+
const ids = new Set<string>();
|
|
164
|
+
collectReferencedBlobIds(tree, true, ids);
|
|
165
|
+
const blobs: ISerializableBlobContents = {};
|
|
166
|
+
await mapWithConcurrency([...ids], maxReadConcurrency, async (id) => {
|
|
167
|
+
const data = await read(id);
|
|
168
|
+
blobs[id] = bufferToString(data, "utf8");
|
|
169
|
+
});
|
|
170
|
+
return blobs;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Synchronously walks the snapshot tree and gathers the set of blob ids that
|
|
175
|
+
* should be inlined. Subtrees flagged `unreferenced: true` are skipped
|
|
176
|
+
* entirely. The root-level `.blobs` subtree is special-cased: only its
|
|
177
|
+
* `.redirectTable` id is collected, because attachment blob contents are
|
|
178
|
+
* captured separately via {@link captureReferencedAttachmentBlobs}.
|
|
179
|
+
*/
|
|
180
|
+
function collectReferencedBlobIds(
|
|
181
|
+
tree: ISnapshotTree,
|
|
182
|
+
isRoot: boolean,
|
|
183
|
+
ids: Set<string>,
|
|
184
|
+
): void {
|
|
185
|
+
if (tree.unreferenced === true) {
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
for (const blobId of Object.values(tree.blobs)) {
|
|
189
|
+
ids.add(blobId);
|
|
190
|
+
}
|
|
191
|
+
for (const [key, subTree] of Object.entries(tree.trees)) {
|
|
192
|
+
if (isRoot && key === blobsTreeName) {
|
|
193
|
+
const tableBlobId = subTree.blobs[redirectTableBlobName];
|
|
194
|
+
if (tableBlobId !== undefined) {
|
|
195
|
+
ids.add(tableBlobId);
|
|
196
|
+
}
|
|
197
|
+
} else {
|
|
198
|
+
collectReferencedBlobIds(subTree, false, ids);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function toTreeAndReader(
|
|
204
|
+
snapshot: ISnapshot | ISnapshotTree,
|
|
205
|
+
storage: Pick<IDocumentStorageService, "readBlob">,
|
|
206
|
+
): { tree: ISnapshotTree; read: BlobReader } {
|
|
207
|
+
if ("snapshotTree" in snapshot) {
|
|
208
|
+
const blobContents = snapshot.blobContents;
|
|
209
|
+
return {
|
|
210
|
+
tree: snapshot.snapshotTree,
|
|
211
|
+
read: async (id) => blobContents.get(id) ?? storage.readBlob(id),
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
return { tree: snapshot, read: async (id) => storage.readBlob(id) };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Fetches attachment blob contents from a snapshot, filtered by GC
|
|
219
|
+
* reachability. Blobs GC has explicitly marked unreferenced, tombstoned, or
|
|
220
|
+
* deleted are skipped. Blobs absent from the GC graph are kept — GC state
|
|
221
|
+
* lags behind recent attachments and dropping them would lose live data.
|
|
222
|
+
* If `gcData` is `undefined`, every attachment blob is returned.
|
|
223
|
+
*
|
|
224
|
+
* The returned map is keyed by attachment blob storage id. Values are the
|
|
225
|
+
* raw bytes encoded as **base64** strings — attachment blobs may carry
|
|
226
|
+
* arbitrary binary payloads (images, encrypted data, etc.) and a
|
|
227
|
+
* UTF-8 round-trip would silently corrupt non-UTF-8 byte sequences with
|
|
228
|
+
* replacement characters. The runtime's own pending-blob serializer uses
|
|
229
|
+
* base64 for the same reason. This diverges from the structural-blob path
|
|
230
|
+
* in {@link readReferencedSnapshotBlobs}, which encodes UTF-8 because those
|
|
231
|
+
* blobs are JSON or other text the runtime authored. Callers must keep the
|
|
232
|
+
* two encodings on separate fields of the pending state so the load side
|
|
233
|
+
* can decode each correctly.
|
|
234
|
+
*/
|
|
235
|
+
export async function captureReferencedAttachmentBlobs(
|
|
236
|
+
baseSnapshot: ISnapshotTree,
|
|
237
|
+
storage: Pick<IDocumentStorageService, "readBlob">,
|
|
238
|
+
gcData: IGcSnapshotData | undefined,
|
|
239
|
+
): Promise<IBase64BlobContents> {
|
|
240
|
+
const blobsTree: ISnapshotTree | undefined = baseSnapshot.trees[blobsTreeName];
|
|
241
|
+
if (blobsTree === undefined) {
|
|
242
|
+
return {};
|
|
243
|
+
}
|
|
244
|
+
const localIdToStorageId = await readRedirectTable(blobsTree, storage);
|
|
245
|
+
if (localIdToStorageId.size === 0) {
|
|
246
|
+
return {};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const unreferencedLocalIds =
|
|
250
|
+
gcData === undefined ? undefined : collectUnreferencedBlobLocalIds(gcData);
|
|
251
|
+
|
|
252
|
+
const storageIdsToFetch = new Set<string>();
|
|
253
|
+
for (const [localId, storageId] of localIdToStorageId) {
|
|
254
|
+
if (unreferencedLocalIds?.has(localId) !== true) {
|
|
255
|
+
storageIdsToFetch.add(storageId);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const contents: IBase64BlobContents = {};
|
|
260
|
+
await mapWithConcurrency([...storageIdsToFetch], maxReadConcurrency, async (storageId) => {
|
|
261
|
+
const buffer = await storage.readBlob(storageId);
|
|
262
|
+
contents[storageId] = bufferToString(buffer, "base64");
|
|
263
|
+
});
|
|
264
|
+
return contents;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Reconstructs the BlobManager's redirect table from a `.blobs` subtree.
|
|
269
|
+
* Mirrors `toRedirectTable` in blobManagerSnapSum.ts.
|
|
270
|
+
*/
|
|
271
|
+
async function readRedirectTable(
|
|
272
|
+
blobsTree: ISnapshotTree,
|
|
273
|
+
storage: Pick<IDocumentStorageService, "readBlob">,
|
|
274
|
+
): Promise<Map<string, string>> {
|
|
275
|
+
const redirectTable = new Map<string, string>();
|
|
276
|
+
const tableBlobId: string | undefined = blobsTree.blobs[redirectTableBlobName];
|
|
277
|
+
if (tableBlobId !== undefined) {
|
|
278
|
+
const entries = await readAndParse<[string, string][]>(storage, tableBlobId);
|
|
279
|
+
for (const [localId, storageId] of entries) {
|
|
280
|
+
redirectTable.set(localId, storageId);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
for (const [key, storageId] of Object.entries(blobsTree.blobs)) {
|
|
284
|
+
if (key !== redirectTableBlobName) {
|
|
285
|
+
// Identity mapping: storage ids referenced directly in handles (legacy).
|
|
286
|
+
redirectTable.set(storageId, storageId);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return redirectTable;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Collects the set of blob localIds that GC has explicitly marked as
|
|
294
|
+
* unreferenced (via `unreferencedTimestampMs` on a gc node), tombstoned, or
|
|
295
|
+
* deleted. Tombstones and deletedNodes are applied regardless of whether
|
|
296
|
+
* `gcState` is present — they are authoritative on their own and must not
|
|
297
|
+
* be silently dropped when gc state is absent but tombstone/deleted lists
|
|
298
|
+
* exist.
|
|
299
|
+
*/
|
|
300
|
+
function collectUnreferencedBlobLocalIds(gcData: IGcSnapshotData): Set<string> {
|
|
301
|
+
const blobPathPrefix = `/${blobManagerBasePath}/`;
|
|
302
|
+
const unreferenced = new Set<string>();
|
|
303
|
+
if (gcData.gcState !== undefined) {
|
|
304
|
+
for (const [nodePath, nodeData] of Object.entries(gcData.gcState.gcNodes)) {
|
|
305
|
+
if (
|
|
306
|
+
nodePath.startsWith(blobPathPrefix) &&
|
|
307
|
+
nodeData.unreferencedTimestampMs !== undefined
|
|
308
|
+
) {
|
|
309
|
+
unreferenced.add(nodePath.slice(blobPathPrefix.length));
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
for (const nodePath of [...(gcData.tombstones ?? []), ...(gcData.deletedNodes ?? [])]) {
|
|
314
|
+
if (nodePath.startsWith(blobPathPrefix)) {
|
|
315
|
+
unreferenced.add(nodePath.slice(blobPathPrefix.length));
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
return unreferenced;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* A blob reference extracted from a `BlobAttach` op. `localId` is the
|
|
323
|
+
* `BlobManager` GC identity for the blob; `storageId` is the id used for
|
|
324
|
+
* `IDocumentStorageService.readBlob`.
|
|
325
|
+
*
|
|
326
|
+
* @internal
|
|
327
|
+
*/
|
|
328
|
+
export interface IBlobAttachReference {
|
|
329
|
+
readonly localId: string;
|
|
330
|
+
readonly storageId: string;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
interface IBlobAttachLikeMetadata {
|
|
334
|
+
readonly localId: string;
|
|
335
|
+
readonly blobId: string;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function isBlobAttachLikeMetadata(metadata: unknown): metadata is IBlobAttachLikeMetadata {
|
|
339
|
+
if (typeof metadata !== "object" || metadata === null) {
|
|
340
|
+
return false;
|
|
341
|
+
}
|
|
342
|
+
const candidate = metadata as { localId?: unknown; blobId?: unknown };
|
|
343
|
+
return typeof candidate.localId === "string" && typeof candidate.blobId === "string";
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Extracts every `BlobAttach` reference an op carries. Returns an empty array
|
|
348
|
+
* for non-blobAttach ops.
|
|
349
|
+
*
|
|
350
|
+
* This is the single place in the loader that interprets the BlobAttach
|
|
351
|
+
* wire format. Capture and load-side reasoning about ops should call into
|
|
352
|
+
* this function rather than reading `op.metadata` directly, so a future
|
|
353
|
+
* protocol change touches one site.
|
|
354
|
+
*
|
|
355
|
+
* BlobAttach ops carry `(localId, storageId)` directly on
|
|
356
|
+
* `ISequencedDocumentMessage.metadata` and are not grouped — the container
|
|
357
|
+
* runtime routes them through a separate `outbox.submitBlobAttach` lane,
|
|
358
|
+
* and `OpGroupingManager.groupBatch` asserts (0x5dd) that no op carrying
|
|
359
|
+
* non-batch metadata enters a grouped batch. If either guarantee changes,
|
|
360
|
+
* extend this function rather than each call site.
|
|
361
|
+
*
|
|
362
|
+
* @internal
|
|
363
|
+
*/
|
|
364
|
+
export function extractBlobAttachReferences(
|
|
365
|
+
op: Pick<ISequencedDocumentMessage, "metadata">,
|
|
366
|
+
): IBlobAttachReference[] {
|
|
367
|
+
if (!isBlobAttachLikeMetadata(op.metadata)) {
|
|
368
|
+
return [];
|
|
369
|
+
}
|
|
370
|
+
return [{ localId: op.metadata.localId, storageId: op.metadata.blobId }];
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Set of attachment-blob localIds that GC has marked unreferenced,
|
|
375
|
+
* tombstoned, or deleted in the base snapshot. `undefined` if `gcData`
|
|
376
|
+
* is `undefined` (GC disabled / pre-GC document).
|
|
377
|
+
*
|
|
378
|
+
* @internal
|
|
379
|
+
*/
|
|
380
|
+
export function unreferencedAttachmentBlobLocalIds(
|
|
381
|
+
gcData: IGcSnapshotData | undefined,
|
|
382
|
+
): Set<string> | undefined {
|
|
383
|
+
return gcData === undefined ? undefined : collectUnreferencedBlobLocalIds(gcData);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Inline attachment blob contents for the given `(localId, storageId)`
|
|
388
|
+
* references. Skips entries already present in `existing` (de-dupe with
|
|
389
|
+
* the snapshot path) and entries whose `localId` is in
|
|
390
|
+
* `unreferencedLocalIds`. Returns only the freshly-read entries; the
|
|
391
|
+
* caller merges them into the existing map.
|
|
392
|
+
*
|
|
393
|
+
* @internal
|
|
394
|
+
*/
|
|
395
|
+
export async function inlineAttachmentBlobsByReference(
|
|
396
|
+
references: readonly IBlobAttachReference[],
|
|
397
|
+
storage: Pick<IDocumentStorageService, "readBlob">,
|
|
398
|
+
unreferencedLocalIds: ReadonlySet<string> | undefined,
|
|
399
|
+
existing: Readonly<IBase64BlobContents>,
|
|
400
|
+
): Promise<IBase64BlobContents> {
|
|
401
|
+
const storageIdsToFetch = new Set<string>();
|
|
402
|
+
for (const { localId, storageId } of references) {
|
|
403
|
+
if (unreferencedLocalIds?.has(localId) === true) {
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
if (existing[storageId] !== undefined) {
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
storageIdsToFetch.add(storageId);
|
|
410
|
+
}
|
|
411
|
+
const added: IBase64BlobContents = {};
|
|
412
|
+
if (storageIdsToFetch.size === 0) {
|
|
413
|
+
return added;
|
|
414
|
+
}
|
|
415
|
+
await mapWithConcurrency([...storageIdsToFetch], maxReadConcurrency, async (storageId) => {
|
|
416
|
+
const buffer = await storage.readBlob(storageId);
|
|
417
|
+
added[storageId] = bufferToString(buffer, "base64");
|
|
418
|
+
});
|
|
419
|
+
return added;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Returns true if any referenced subtree of `baseSnapshot` declares a
|
|
424
|
+
* `groupId` — the snapshot-tree wire field that carries the runtime's
|
|
425
|
+
* loading-group identifier. Subtrees flagged `unreferenced` are skipped —
|
|
426
|
+
* a dead subtree's `groupId` would not be loaded by the runtime either.
|
|
427
|
+
*
|
|
428
|
+
* `captureFullContainerState` does not yet support loading groups: prefetching
|
|
429
|
+
* per-group snapshots adds a code path that has no end-to-end coverage and no
|
|
430
|
+
* known production consumer. Callers use this to fail fast with a `UsageError`
|
|
431
|
+
* rather than silently producing a pending state that omits group data.
|
|
432
|
+
*/
|
|
433
|
+
export function snapshotHasLoadingGroups(baseSnapshot: ISnapshotTree): boolean {
|
|
434
|
+
if (baseSnapshot.unreferenced === true) {
|
|
435
|
+
return false;
|
|
436
|
+
}
|
|
437
|
+
if (baseSnapshot.groupId !== undefined) {
|
|
438
|
+
return true;
|
|
439
|
+
}
|
|
440
|
+
for (const child of Object.values(baseSnapshot.trees)) {
|
|
441
|
+
if (snapshotHasLoadingGroups(child)) {
|
|
442
|
+
return true;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
return false;
|
|
446
|
+
}
|
package/src/connectionManager.ts
CHANGED
|
@@ -63,7 +63,11 @@ import {
|
|
|
63
63
|
ReconnectMode,
|
|
64
64
|
} from "./contracts.js";
|
|
65
65
|
import { DeltaQueue } from "./deltaQueue.js";
|
|
66
|
-
import {
|
|
66
|
+
import {
|
|
67
|
+
FrozenDeltaStream,
|
|
68
|
+
isFrozenDeltaStreamConnection,
|
|
69
|
+
isWritableFrozenDeltaStreamConnection,
|
|
70
|
+
} from "./frozenServices.js";
|
|
67
71
|
import { SignalType } from "./protocol.js";
|
|
68
72
|
import { isDeltaStreamConnectionForbiddenError } from "./utils.js";
|
|
69
73
|
|
|
@@ -582,9 +586,9 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
582
586
|
LogLevel.verbose,
|
|
583
587
|
);
|
|
584
588
|
if (isDeltaStreamConnectionForbiddenError(origError)) {
|
|
585
|
-
connection = new FrozenDeltaStream(
|
|
586
|
-
|
|
587
|
-
error: origError,
|
|
589
|
+
connection = new FrozenDeltaStream({
|
|
590
|
+
storageOnlyReason: origError.storageOnlyReason,
|
|
591
|
+
readonlyConnectionReason: { text: origError.message, error: origError },
|
|
588
592
|
});
|
|
589
593
|
requestedMode = "read";
|
|
590
594
|
break;
|
|
@@ -592,11 +596,10 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
592
596
|
isFluidError(origError) &&
|
|
593
597
|
origError.errorType === DriverErrorTypes.outOfStorageError
|
|
594
598
|
) {
|
|
595
|
-
// If we get out of storage error from calling joinsession, then use the
|
|
599
|
+
// If we get out of storage error from calling joinsession, then use the FrozenDeltaStream object so
|
|
596
600
|
// that user can at least load the container.
|
|
597
|
-
connection = new FrozenDeltaStream(
|
|
598
|
-
text: origError.message,
|
|
599
|
-
error: origError,
|
|
601
|
+
connection = new FrozenDeltaStream({
|
|
602
|
+
readonlyConnectionReason: { text: origError.message, error: origError },
|
|
600
603
|
});
|
|
601
604
|
requestedMode = "read";
|
|
602
605
|
break;
|
|
@@ -1089,6 +1092,25 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
1089
1092
|
|
|
1090
1093
|
public sendMessages(messages: IDocumentMessage[]): void {
|
|
1091
1094
|
assert(this.connected, 0x2b4 /* "not connected on sending ops!" */);
|
|
1095
|
+
// WritableFrozenDeltaStream short-circuit: writable-frozen containers
|
|
1096
|
+
// (`loadFrozenContainerFromPendingState({ readOnly: false })`) attach a
|
|
1097
|
+
// WritableFrozenDeltaStream as the live connection. Its `mode` is "read" (advertising
|
|
1098
|
+
// "write" would imply quorum membership we cannot honor), so a runtime submit
|
|
1099
|
+
// would otherwise fall into the read-mode reconnect branch below. That branch
|
|
1100
|
+
// schedules `reconnect("write")`, which under `ReconnectMode.Never`
|
|
1101
|
+
// (`allowReconnect: false`) calls `closeHandler` and closes the container — the
|
|
1102
|
+
// opposite of what writable-frozen wants. Drop the messages here: the runtime's
|
|
1103
|
+
// outbox keeps them in `pendingStateManager` so `getPendingLocalState()` can
|
|
1104
|
+
// capture them, which is the entire point of the writable-frozen flow.
|
|
1105
|
+
//
|
|
1106
|
+
// Match only the writable variant (a sibling class, not a subclass) so the read-only
|
|
1107
|
+
// `FrozenDeltaStream` retains its `submit` 403-nack tripwire — a stray submit on a
|
|
1108
|
+
// storage-only frozen connection signals an upstream invariant break and should
|
|
1109
|
+
// remain observable. The read-only variant shouldn't reach here in normal flow anyway
|
|
1110
|
+
// (its `storageOnly` policy keeps the runtime from submitting).
|
|
1111
|
+
if (isWritableFrozenDeltaStreamConnection(this.connection)) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1092
1114
|
// If connection is "read" or implicit "read" (got leave op for "write" connection),
|
|
1093
1115
|
// then op can't make it through - we will get a nack if op is sent.
|
|
1094
1116
|
// We can short-circuit this process.
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import type { IDeltaManager } from "@fluidframework/container-definitions/internal";
|
|
7
7
|
import type { ITelemetryBaseProperties } from "@fluidframework/core-interfaces";
|
|
8
|
+
import { LogLevel } from "@fluidframework/core-interfaces";
|
|
8
9
|
import { assert, Timer } from "@fluidframework/core-utils/internal";
|
|
9
10
|
import type { IClient, ISequencedClient } from "@fluidframework/driver-definitions";
|
|
10
11
|
import type { IAnyDriverError } from "@fluidframework/driver-definitions/internal";
|
|
@@ -687,15 +688,19 @@ export class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
687
688
|
this.prevClientLeftTimer.restart();
|
|
688
689
|
} else {
|
|
689
690
|
// Adding this event temporarily so that we can get help debugging if something goes wrong.
|
|
690
|
-
this.handler.logger.sendTelemetryEvent(
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
691
|
+
this.handler.logger.sendTelemetryEvent(
|
|
692
|
+
{
|
|
693
|
+
eventName: "noWaitOnDisconnected",
|
|
694
|
+
details: JSON.stringify({
|
|
695
|
+
clientId: this._clientId,
|
|
696
|
+
inQuorum: currentClientInQuorum,
|
|
697
|
+
waitingForLeaveOp: this.waitingForLeaveOp,
|
|
698
|
+
hadOutstandingOps: this.handler.shouldClientJoinWrite(),
|
|
699
|
+
}),
|
|
700
|
+
},
|
|
701
|
+
undefined, // error
|
|
702
|
+
LogLevel.info,
|
|
703
|
+
);
|
|
699
704
|
}
|
|
700
705
|
}
|
|
701
706
|
|
package/src/container.ts
CHANGED
|
@@ -1070,6 +1070,7 @@ export class Container
|
|
|
1070
1070
|
|
|
1071
1071
|
this.connectionStateHandler.dispose();
|
|
1072
1072
|
this.serializedStateManager.dispose();
|
|
1073
|
+
this._runtime?.close?.();
|
|
1073
1074
|
} catch (newError) {
|
|
1074
1075
|
this.mc.logger.sendErrorEvent({ eventName: "ContainerCloseException" }, newError);
|
|
1075
1076
|
}
|
|
@@ -1104,6 +1105,8 @@ export class Container
|
|
|
1104
1105
|
eventName: "ContainerDispose",
|
|
1105
1106
|
// Only log error if container isn't closed
|
|
1106
1107
|
category: !this.closed && error !== undefined ? "error" : "generic",
|
|
1108
|
+
isDirty: this.isDirty,
|
|
1109
|
+
lastSequenceNumber: this._deltaManager.lastSequenceNumber,
|
|
1107
1110
|
},
|
|
1108
1111
|
error,
|
|
1109
1112
|
);
|
|
@@ -2394,6 +2397,9 @@ export class Container
|
|
|
2394
2397
|
this.subLogger,
|
|
2395
2398
|
{ eventName: "CodeLoad" },
|
|
2396
2399
|
async () => this.codeLoader.load(codeDetails),
|
|
2400
|
+
undefined, // markers
|
|
2401
|
+
undefined, // sampleThreshold
|
|
2402
|
+
LogLevel.info,
|
|
2397
2403
|
);
|
|
2398
2404
|
|
|
2399
2405
|
this._loadedModule = {
|
|
@@ -36,13 +36,32 @@ import type {
|
|
|
36
36
|
import { convertSnapshotInfoToSnapshot } from "./utils.js";
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
|
-
* Stringified blobs from a summary/snapshot tree.
|
|
39
|
+
* Stringified blobs from a summary/snapshot tree, keyed by blob id.
|
|
40
|
+
* Values are **UTF-8-encoded** — this is the right encoding for JSON or
|
|
41
|
+
* other text the runtime authors and consumes through this map. For
|
|
42
|
+
* arbitrary binary payloads (e.g. attachment blob contents), use
|
|
43
|
+
* {@link IBase64BlobContents} instead; a UTF-8 round-trip silently
|
|
44
|
+
* corrupts non-UTF-8 byte sequences with replacement characters.
|
|
40
45
|
* @internal
|
|
41
46
|
*/
|
|
42
47
|
export interface ISerializableBlobContents {
|
|
43
48
|
[id: string]: string;
|
|
44
49
|
}
|
|
45
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Stringified blobs inlined in a summary/snapshot tree, keyed by blob id.
|
|
53
|
+
* Values are **base64-encoded** raw bytes. Used for attachment-blob
|
|
54
|
+
* payloads, which may carry arbitrary binary data (images, encrypted
|
|
55
|
+
* blobs, etc.). Mirrors the encoding used by the runtime's own
|
|
56
|
+
* pending-blob serializer in `BlobManager`. Structurally identical to
|
|
57
|
+
* {@link ISerializableBlobContents}; the two types exist to keep the
|
|
58
|
+
* encoding contract visible at every call site.
|
|
59
|
+
* @internal
|
|
60
|
+
*/
|
|
61
|
+
export interface IBase64BlobContents {
|
|
62
|
+
[id: string]: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
46
65
|
/**
|
|
47
66
|
* This class wraps the actual storage and make sure no wrong apis are called according to
|
|
48
67
|
* container attach state.
|