@fluidframework/container-loader 2.117.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -2
- package/README.md +53 -50
- package/{api-extractor.json → api-extractor/api-extractor-model.json} +1 -1
- package/api-report/container-loader.legacy.alpha.api.md +4 -3
- package/dist/attachment.d.ts.map +1 -1
- package/dist/audience.js +3 -2
- package/dist/audience.js.map +1 -1
- package/dist/captureReferencedContents.js +9 -9
- package/dist/captureReferencedContents.js.map +1 -1
- package/dist/catchUpMonitor.js +11 -8
- package/dist/catchUpMonitor.js.map +1 -1
- package/dist/connectionManager.d.ts.map +1 -1
- package/dist/connectionManager.js +86 -51
- package/dist/connectionManager.js.map +1 -1
- package/dist/connectionStateHandler.js +40 -14
- package/dist/connectionStateHandler.js.map +1 -1
- package/dist/container.d.ts +1 -1
- package/dist/container.d.ts.map +1 -1
- package/dist/container.js +193 -143
- package/dist/container.js.map +1 -1
- package/dist/containerContext.js +48 -11
- package/dist/containerContext.js.map +1 -1
- package/dist/containerStorageAdapter.js +25 -17
- package/dist/containerStorageAdapter.js.map +1 -1
- package/dist/contracts.d.ts.map +1 -1
- package/dist/createAndLoadContainerUtils.d.ts +34 -13
- package/dist/createAndLoadContainerUtils.d.ts.map +1 -1
- package/dist/createAndLoadContainerUtils.js +97 -37
- package/dist/createAndLoadContainerUtils.js.map +1 -1
- package/dist/debugLogger.d.ts +7 -22
- package/dist/debugLogger.d.ts.map +1 -1
- package/dist/debugLogger.js +34 -34
- package/dist/debugLogger.js.map +1 -1
- package/dist/deltaManager.d.ts +16 -4
- package/dist/deltaManager.d.ts.map +1 -1
- package/dist/deltaManager.js +75 -49
- package/dist/deltaManager.js.map +1 -1
- package/dist/deltaManagerFactory.d.ts +23 -0
- package/dist/deltaManagerFactory.d.ts.map +1 -0
- package/dist/deltaManagerFactory.js +17 -0
- package/dist/deltaManagerFactory.js.map +1 -0
- package/dist/deltaQueue.js +14 -7
- package/dist/deltaQueue.js.map +1 -1
- package/dist/disposal.js +1 -2
- package/dist/disposal.js.map +1 -1
- package/dist/error.js +5 -4
- package/dist/error.js.map +1 -1
- package/dist/frozenServices.d.ts +2 -2
- package/dist/frozenServices.d.ts.map +1 -1
- package/dist/frozenServices.js +52 -34
- package/dist/frozenServices.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/loadContainerToSequenceNumber.d.ts +6 -5
- package/dist/loadContainerToSequenceNumber.d.ts.map +1 -1
- package/dist/loadContainerToSequenceNumber.js +5 -6
- package/dist/loadContainerToSequenceNumber.js.map +1 -1
- package/dist/loadPaused.js +1 -2
- package/dist/loadPaused.js.map +1 -1
- package/dist/loader.js +5 -1
- package/dist/loader.js.map +1 -1
- package/dist/loaderLayerCompatState.d.ts +25 -1
- package/dist/loaderLayerCompatState.d.ts.map +1 -1
- package/dist/loaderLayerCompatState.js +79 -11
- package/dist/loaderLayerCompatState.js.map +1 -1
- package/dist/location-redirection-utilities/resolveWithLocationRedirection.js +2 -3
- package/dist/location-redirection-utilities/resolveWithLocationRedirection.js.map +1 -1
- package/dist/memoryBlobStorage.js +2 -3
- package/dist/memoryBlobStorage.js.map +1 -1
- package/dist/noopHeuristic.js +3 -1
- package/dist/noopHeuristic.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingLocalStateStore.d.ts +4 -3
- package/dist/pendingLocalStateStore.d.ts.map +1 -1
- package/dist/pendingLocalStateStore.js +26 -37
- package/dist/pendingLocalStateStore.js.map +1 -1
- package/dist/pointInTimeServices.js +4 -2
- package/dist/pointInTimeServices.js.map +1 -1
- package/dist/protocol/protocol.js +3 -0
- package/dist/protocol/protocol.js.map +1 -1
- package/dist/protocol/quorum.js +26 -5
- package/dist/protocol/quorum.js.map +1 -1
- package/dist/protocol.js +5 -3
- package/dist/protocol.js.map +1 -1
- package/dist/protocolTreeDocumentStorageService.js +10 -0
- package/dist/protocolTreeDocumentStorageService.js.map +1 -1
- package/dist/quorum.js +1 -2
- package/dist/quorum.js.map +1 -1
- package/dist/retriableDocumentStorageService.d.ts.map +1 -1
- package/dist/retriableDocumentStorageService.js +5 -1
- package/dist/retriableDocumentStorageService.js.map +1 -1
- package/dist/serializedStateManager.d.ts +6 -0
- package/dist/serializedStateManager.d.ts.map +1 -1
- package/dist/serializedStateManager.js +19 -24
- package/dist/serializedStateManager.js.map +1 -1
- package/dist/snapshotRefresher.js +31 -36
- package/dist/snapshotRefresher.js.map +1 -1
- package/dist/utils.d.ts +1 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +13 -13
- package/dist/utils.js.map +1 -1
- package/lib/attachment.d.ts.map +1 -1
- package/lib/audience.js +3 -2
- package/lib/audience.js.map +1 -1
- package/lib/catchUpMonitor.js +11 -8
- package/lib/catchUpMonitor.js.map +1 -1
- package/lib/connectionManager.d.ts.map +1 -1
- package/lib/connectionManager.js +86 -51
- package/lib/connectionManager.js.map +1 -1
- package/lib/connectionStateHandler.js +37 -11
- package/lib/connectionStateHandler.js.map +1 -1
- package/lib/container.d.ts +1 -1
- package/lib/container.d.ts.map +1 -1
- package/lib/container.js +191 -141
- package/lib/container.js.map +1 -1
- package/lib/containerContext.js +48 -11
- package/lib/containerContext.js.map +1 -1
- package/lib/containerStorageAdapter.js +22 -14
- package/lib/containerStorageAdapter.js.map +1 -1
- package/lib/contracts.d.ts.map +1 -1
- package/lib/createAndLoadContainerUtils.d.ts +34 -13
- package/lib/createAndLoadContainerUtils.d.ts.map +1 -1
- package/lib/createAndLoadContainerUtils.js +93 -32
- package/lib/createAndLoadContainerUtils.js.map +1 -1
- package/lib/debugLogger.d.ts +7 -22
- package/lib/debugLogger.d.ts.map +1 -1
- package/lib/debugLogger.js +34 -33
- package/lib/debugLogger.js.map +1 -1
- package/lib/deltaManager.d.ts +16 -4
- package/lib/deltaManager.d.ts.map +1 -1
- package/lib/deltaManager.js +75 -49
- package/lib/deltaManager.js.map +1 -1
- package/lib/deltaManagerFactory.d.ts +23 -0
- package/lib/deltaManagerFactory.d.ts.map +1 -0
- package/lib/deltaManagerFactory.js +14 -0
- package/lib/deltaManagerFactory.js.map +1 -0
- package/lib/deltaQueue.js +14 -7
- package/lib/deltaQueue.js.map +1 -1
- package/lib/error.js +5 -4
- package/lib/error.js.map +1 -1
- package/lib/frozenServices.d.ts +2 -2
- package/lib/frozenServices.d.ts.map +1 -1
- package/lib/frozenServices.js +48 -30
- package/lib/frozenServices.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +1 -1
- package/lib/legacyAlpha.d.ts +1 -1
- package/lib/loadContainerToSequenceNumber.d.ts +6 -5
- package/lib/loadContainerToSequenceNumber.d.ts.map +1 -1
- package/lib/loadContainerToSequenceNumber.js +4 -4
- package/lib/loadContainerToSequenceNumber.js.map +1 -1
- package/lib/loader.js +6 -2
- package/lib/loader.js.map +1 -1
- package/lib/loaderLayerCompatState.d.ts +25 -1
- package/lib/loaderLayerCompatState.d.ts.map +1 -1
- package/lib/loaderLayerCompatState.js +76 -9
- package/lib/loaderLayerCompatState.js.map +1 -1
- package/lib/noopHeuristic.js +3 -1
- package/lib/noopHeuristic.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingLocalStateStore.d.ts +4 -3
- package/lib/pendingLocalStateStore.d.ts.map +1 -1
- package/lib/pendingLocalStateStore.js +26 -37
- package/lib/pendingLocalStateStore.js.map +1 -1
- package/lib/pointInTimeServices.js +2 -0
- package/lib/pointInTimeServices.js.map +1 -1
- package/lib/protocol/protocol.js +3 -0
- package/lib/protocol/protocol.js.map +1 -1
- package/lib/protocol/quorum.js +26 -5
- package/lib/protocol/quorum.js.map +1 -1
- package/lib/protocol.js +2 -0
- package/lib/protocol.js.map +1 -1
- package/lib/protocolTreeDocumentStorageService.js +10 -0
- package/lib/protocolTreeDocumentStorageService.js.map +1 -1
- package/lib/public.d.ts +1 -1
- package/lib/retriableDocumentStorageService.d.ts.map +1 -1
- package/lib/retriableDocumentStorageService.js +5 -1
- package/lib/retriableDocumentStorageService.js.map +1 -1
- package/lib/serializedStateManager.d.ts +6 -0
- package/lib/serializedStateManager.d.ts.map +1 -1
- package/lib/serializedStateManager.js +15 -20
- package/lib/serializedStateManager.js.map +1 -1
- package/lib/snapshotRefresher.js +31 -36
- package/lib/snapshotRefresher.js.map +1 -1
- package/lib/utils.d.ts +1 -1
- package/lib/utils.d.ts.map +1 -1
- package/lib/utils.js +3 -3
- package/lib/utils.js.map +1 -1
- package/package.json +44 -106
- package/src/container.ts +35 -22
- package/src/createAndLoadContainerUtils.ts +131 -46
- package/src/debugLogger.ts +39 -40
- package/src/deltaManager.ts +18 -5
- package/src/deltaManagerFactory.ts +53 -0
- package/src/frozenServices.ts +6 -7
- package/src/index.ts +1 -0
- package/src/loadContainerToSequenceNumber.ts +7 -6
- package/src/loader.ts +2 -2
- package/src/loaderLayerCompatState.ts +85 -20
- package/src/packageVersion.ts +1 -1
- package/src/pendingLocalStateStore.ts +7 -3
- package/src/pointInTime/DEV.md +47 -4
- package/src/serializedStateManager.ts +6 -0
- package/src/utils.ts +3 -3
- package/tsconfig.json +1 -1
- package/internal.d.ts +0 -11
- package/legacy/alpha.d.ts +0 -11
- package/legacy.d.ts +0 -11
- /package/api-extractor/{api-extractor.current.json → api-extractor-report.current.json} +0 -0
- /package/api-extractor/{api-extractor.legacy.json → api-extractor-report.legacy.json} +0 -0
|
@@ -18,7 +18,7 @@ import type {
|
|
|
18
18
|
} from "@fluidframework/core-interfaces";
|
|
19
19
|
import type { AllOrNone } from "@fluidframework/core-interfaces/internal";
|
|
20
20
|
import { validateAllOrNone } from "@fluidframework/core-utils/internal";
|
|
21
|
-
import type { IClientDetails } from "@fluidframework/driver-definitions";
|
|
21
|
+
import type { IClient, IClientDetails } from "@fluidframework/driver-definitions";
|
|
22
22
|
import type {
|
|
23
23
|
IDocumentServiceFactory,
|
|
24
24
|
IResolvedUrl,
|
|
@@ -32,6 +32,7 @@ import { getSnapshotTree } from "@fluidframework/driver-utils/internal";
|
|
|
32
32
|
import {
|
|
33
33
|
GenericError,
|
|
34
34
|
UsageError,
|
|
35
|
+
createChildLogger,
|
|
35
36
|
normalizeError,
|
|
36
37
|
createChildMonitoringContext,
|
|
37
38
|
mixinMonitoringContext,
|
|
@@ -50,8 +51,11 @@ import {
|
|
|
50
51
|
snapshotHasLoadingGroups,
|
|
51
52
|
unreferencedAttachmentBlobLocalIds,
|
|
52
53
|
type IBlobAttachReference,
|
|
54
|
+
type IGcSnapshotData,
|
|
53
55
|
} from "./captureReferencedContents.js";
|
|
54
|
-
import {
|
|
56
|
+
import { CatchUpMonitor } from "./catchUpMonitor.js";
|
|
57
|
+
import { mixinDebugLogger } from "./debugLogger.js";
|
|
58
|
+
import { createDeltaManager } from "./deltaManagerFactory.js";
|
|
55
59
|
import { createFrozenDocumentServiceFactory } from "./frozenServices.js";
|
|
56
60
|
import { Loader } from "./loader.js";
|
|
57
61
|
import { pkgVersion } from "./packageVersion.js";
|
|
@@ -321,19 +325,19 @@ export async function loadExistingContainer(
|
|
|
321
325
|
* resolver's external URL format is unknown without a real `IUrlResolver`.
|
|
322
326
|
*
|
|
323
327
|
* **Offline form precondition.** With no driver wiring there is no live
|
|
324
|
-
* storage to read
|
|
325
|
-
* during load must already be inlined into
|
|
326
|
-
* state must therefore be produced by
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
328
|
+
* storage to read blobs from, so every structural and attachment blob the
|
|
329
|
+
* runtime dereferences during load must already be inlined into
|
|
330
|
+
* `pendingLocalState`. The pending state must therefore be produced by
|
|
331
|
+
* {@link captureFullContainerState} with its default `"inline"` blob capture
|
|
332
|
+
* mode rather than its `"reference"` mode or
|
|
333
|
+
* `IContainer.getPendingLocalState` / `getRequiredPendingLocalState`. Missing
|
|
334
|
+
* blob payloads fail the load with `UsageError`.
|
|
331
335
|
*
|
|
332
336
|
* **URL shape requirement.** In the offline form the captured
|
|
333
337
|
* `pendingLocalState.url` is the only URL available; it is parsed in place of
|
|
334
338
|
* a real `IUrlResolver.resolve()` call, so it must satisfy
|
|
335
339
|
* {@link tryParseCompatibleResolvedUrl}'s contract — a resolved URL of shape
|
|
336
|
-
* `protocol
|
|
340
|
+
* `protocol://{host}/.../..?{queryString}`. This is the format that
|
|
337
341
|
* Fluid-shipped drivers emit; drivers that emit a non-standard resolved-URL
|
|
338
342
|
* shape will surface as a `UsageError` at load time. The online form has no
|
|
339
343
|
* such constraint because the supplied resolver controls URL parsing.
|
|
@@ -418,6 +422,11 @@ export async function loadFrozenContainerFromPendingState(
|
|
|
418
422
|
// returns a resolved URL whose `url` equals `pendingLocalState.url`, so the
|
|
419
423
|
// identity guard in `Loader.resolveCore` is trivially satisfied.
|
|
420
424
|
const pending = getAttachedContainerStateFromSerializedContainer(pendingLocalState);
|
|
425
|
+
if (pending.blobContentsMode === "reference") {
|
|
426
|
+
throw new UsageError(
|
|
427
|
+
`${fnName}: pending state does not contain inlined snapshot blob contents. Reference-only state captured by ${captureFullContainerState.name} requires online driver wiring.`,
|
|
428
|
+
);
|
|
429
|
+
}
|
|
421
430
|
const parsed = tryParseCompatibleResolvedUrl(pending.url);
|
|
422
431
|
if (parsed === undefined) {
|
|
423
432
|
throw new UsageError(
|
|
@@ -505,6 +514,18 @@ export interface ICaptureFullContainerStateProps {
|
|
|
505
514
|
* Optional logger for driver-side telemetry.
|
|
506
515
|
*/
|
|
507
516
|
readonly logger?: ITelemetryBaseLogger | undefined;
|
|
517
|
+
/**
|
|
518
|
+
* Controls whether blob payloads are included in the captured state.
|
|
519
|
+
*
|
|
520
|
+
* `"inline"` (the default) includes blob contents, producing self-contained
|
|
521
|
+
* state that supports fully offline loading with {@link loadFrozenContainerFromPendingState}.
|
|
522
|
+
*
|
|
523
|
+
* `"reference"` preserves blob IDs in the snapshot tree but omits their
|
|
524
|
+
* payloads. This reduces the captured state size, but loading requires live
|
|
525
|
+
* storage through {@link loadExistingContainer} or the online form of
|
|
526
|
+
* {@link loadFrozenContainerFromPendingState}.
|
|
527
|
+
*/
|
|
528
|
+
readonly blobCaptureMode?: "inline" | "reference" | undefined;
|
|
508
529
|
}
|
|
509
530
|
|
|
510
531
|
/**
|
|
@@ -514,11 +535,19 @@ export interface ICaptureFullContainerStateProps {
|
|
|
514
535
|
* format produced by a live container's pending-state serialization, and can
|
|
515
536
|
* be handed to {@link loadExistingContainer} as `pendingLocalState`.
|
|
516
537
|
*
|
|
517
|
-
*
|
|
518
|
-
* the latest snapshot, inlined contents of every blob reachable through
|
|
538
|
+
* By default, the output is a self-contained view of the container's referenced
|
|
539
|
+
* graph: the latest snapshot, inlined contents of every blob reachable through
|
|
519
540
|
* referenced subtrees, inlined contents of every referenced attachment blob
|
|
520
541
|
* keyed by storage id, and all ops with sequence numbers after the base
|
|
521
|
-
* snapshot's sequence number
|
|
542
|
+
* snapshot's sequence number through the point observed when the delta stream
|
|
543
|
+
* connects. The same DeltaManager catch-up path used by container loading
|
|
544
|
+
* combines delta-storage and delta-stream ops, so recently sequenced ops do
|
|
545
|
+
* not need to have reached durable delta storage before capture.
|
|
546
|
+
*
|
|
547
|
+
* When `blobCaptureMode` is `"reference"`, the snapshot tree retains the blob
|
|
548
|
+
* IDs needed to load the container, but structural and attachment payloads are
|
|
549
|
+
* omitted. Reference-only state requires live storage during load and cannot be
|
|
550
|
+
* used with the fully offline form of {@link loadFrozenContainerFromPendingState}.
|
|
522
551
|
*
|
|
523
552
|
* Reachability respects GC. Snapshot subtrees flagged `unreferenced: true`
|
|
524
553
|
* are skipped (their contents are not inlined). Attachment blobs that GC has
|
|
@@ -539,7 +568,8 @@ export interface ICaptureFullContainerStateProps {
|
|
|
539
568
|
* is a known consumer and end-to-end coverage, the capture refuses rather
|
|
540
569
|
* than silently producing pending state that omits group data.
|
|
541
570
|
*
|
|
542
|
-
*
|
|
571
|
+
* @privateRemarks
|
|
572
|
+
* Note: if a new snapshot lands between the snapshot fetch and delta catch-up,
|
|
543
573
|
* the returned state may not reflect the very latest snapshot, but remains
|
|
544
574
|
* internally consistent: ops are anchored to the snapshot that was captured.
|
|
545
575
|
*
|
|
@@ -556,20 +586,20 @@ export async function captureFullContainerState({
|
|
|
556
586
|
documentServiceFactory,
|
|
557
587
|
request,
|
|
558
588
|
logger,
|
|
589
|
+
blobCaptureMode = "inline",
|
|
559
590
|
}: ICaptureFullContainerStateProps): Promise<string> {
|
|
560
591
|
const resolvedUrl = await urlResolver.resolve(request);
|
|
561
592
|
if (resolvedUrl === undefined) {
|
|
562
593
|
throw new UsageError("Failed to resolve request to a Fluid URL");
|
|
563
594
|
}
|
|
564
595
|
|
|
565
|
-
//
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
if (tryParseCompatibleResolvedUrl(resolvedUrl.url) === undefined) {
|
|
596
|
+
// Inline state supports the offline form of `loadFrozenContainerFromPendingState`,
|
|
597
|
+
// which requires `tryParseCompatibleResolvedUrl` to succeed on `pendingLocalState.url`.
|
|
598
|
+
// Reference-only state always uses the caller's live resolver, so it has no such constraint.
|
|
599
|
+
if (
|
|
600
|
+
blobCaptureMode === "inline" &&
|
|
601
|
+
tryParseCompatibleResolvedUrl(resolvedUrl.url) === undefined
|
|
602
|
+
) {
|
|
573
603
|
throw new UsageError(
|
|
574
604
|
`${captureFullContainerState.name}: resolved URL is not in the shape required by tryParseCompatibleResolvedUrl (protocol://<string>/<tenantId>/<docId>?<querystring>); captured state would not rehydrate offline (${resolvedUrl.url})`,
|
|
575
605
|
);
|
|
@@ -579,6 +609,43 @@ export async function captureFullContainerState({
|
|
|
579
609
|
resolvedUrl,
|
|
580
610
|
logger,
|
|
581
611
|
);
|
|
612
|
+
const captureLogger = createChildLogger({ logger, namespace: "captureFullContainerState" });
|
|
613
|
+
const client: IClient = {
|
|
614
|
+
details: { capabilities: { interactive: true } },
|
|
615
|
+
mode: "read",
|
|
616
|
+
permission: [],
|
|
617
|
+
scopes: [],
|
|
618
|
+
user: { id: "" },
|
|
619
|
+
};
|
|
620
|
+
const deltaManager = createDeltaManager({
|
|
621
|
+
serviceProvider: () => documentService,
|
|
622
|
+
logger: captureLogger,
|
|
623
|
+
active: () => false,
|
|
624
|
+
containerDirty: () => false,
|
|
625
|
+
client,
|
|
626
|
+
reconnectAllowed: true,
|
|
627
|
+
});
|
|
628
|
+
const connectedP = new Promise<void>((resolve) => {
|
|
629
|
+
deltaManager.once("connect", () => resolve());
|
|
630
|
+
});
|
|
631
|
+
const closedP = new Promise<never>((_resolve, reject) => {
|
|
632
|
+
deltaManager.once("closed", (error) => {
|
|
633
|
+
reject(
|
|
634
|
+
error === undefined
|
|
635
|
+
? new GenericError("DeltaManager closed while capturing container state")
|
|
636
|
+
: normalizeError(error),
|
|
637
|
+
);
|
|
638
|
+
});
|
|
639
|
+
});
|
|
640
|
+
deltaManager.connect({
|
|
641
|
+
reason: { text: "captureFullContainerState" },
|
|
642
|
+
mode: "read",
|
|
643
|
+
// Ops will be fetched from storage using the later "attachOpHandler" call. Attempting to fetch here is a no-op anyway as
|
|
644
|
+
// no op handler is yet attached.
|
|
645
|
+
// Fetching the snapshot first such that an op handler could be attached using its sequence number information may simplify
|
|
646
|
+
// the code, but doing it this way allows parallelization of snapshot fetch and establishing a connection.
|
|
647
|
+
fetchOpsFromStorage: false,
|
|
648
|
+
});
|
|
582
649
|
try {
|
|
583
650
|
const storage = await documentService.connectToStorage();
|
|
584
651
|
|
|
@@ -610,42 +677,46 @@ export async function captureFullContainerState({
|
|
|
610
677
|
);
|
|
611
678
|
}
|
|
612
679
|
const attributes = await getDocumentAttributes(storage, baseSnapshot);
|
|
613
|
-
|
|
680
|
+
let gcData: IGcSnapshotData | undefined;
|
|
681
|
+
let snapshotBlobs = {};
|
|
682
|
+
let attachmentBlobContents = {};
|
|
614
683
|
// Structural snapshot blobs (JSON/text the runtime authored) are
|
|
615
684
|
// UTF-8-encoded; attachment blobs may carry arbitrary binary bytes
|
|
616
685
|
// and are base64-encoded. Keep them on separate fields of the
|
|
617
686
|
// pending state so the load side can apply the matching decoder
|
|
618
687
|
// without ambiguity. See IPendingContainerState.attachmentBlobContents.
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
undefined,
|
|
628
|
-
undefined,
|
|
629
|
-
false,
|
|
630
|
-
"captureFullContainerState",
|
|
631
|
-
);
|
|
688
|
+
if (blobCaptureMode === "inline") {
|
|
689
|
+
gcData = await parseGcSnapshotData(baseSnapshot, storage);
|
|
690
|
+
[snapshotBlobs, attachmentBlobContents] = await Promise.all([
|
|
691
|
+
readReferencedSnapshotBlobs(snapshot, storage), // utf8 encoded
|
|
692
|
+
captureReferencedAttachmentBlobs(baseSnapshot, storage, gcData), // base64 encoded
|
|
693
|
+
]);
|
|
694
|
+
}
|
|
695
|
+
|
|
632
696
|
const savedOps: ISequencedDocumentMessage[] = [];
|
|
697
|
+
const attachP = deltaManager.attachOpHandler(
|
|
698
|
+
attributes.minimumSequenceNumber,
|
|
699
|
+
attributes.sequenceNumber,
|
|
700
|
+
{
|
|
701
|
+
process: (message) => savedOps.push(message),
|
|
702
|
+
processSignal: () => {},
|
|
703
|
+
},
|
|
704
|
+
"all",
|
|
705
|
+
);
|
|
706
|
+
await Promise.race([Promise.all([connectedP, attachP]), closedP]);
|
|
707
|
+
await Promise.race([waitForCatchUp(deltaManager), closedP]);
|
|
708
|
+
deltaManager.dispose();
|
|
709
|
+
|
|
633
710
|
const postSnapshotBlobReferences: IBlobAttachReference[] = [];
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
for (const op of opsResult.value) {
|
|
637
|
-
savedOps.push(op);
|
|
711
|
+
if (blobCaptureMode === "inline") {
|
|
712
|
+
for (const op of savedOps) {
|
|
638
713
|
// Blobs uploaded after the base snapshot are not in its
|
|
639
714
|
// `.blobs` redirect table, so `captureReferencedAttachmentBlobs`
|
|
640
715
|
// did not see them. The wire-format BlobAttach op carries
|
|
641
716
|
// `(localId, storageId)` in its metadata; collect those here so
|
|
642
717
|
// we can backfill the bytes before sealing the artifact.
|
|
643
|
-
|
|
644
|
-
if (refs.length > 0) {
|
|
645
|
-
postSnapshotBlobReferences.push(...refs);
|
|
646
|
-
}
|
|
718
|
+
postSnapshotBlobReferences.push(...extractBlobAttachReferences(op));
|
|
647
719
|
}
|
|
648
|
-
opsResult = await opsStream.read();
|
|
649
720
|
}
|
|
650
721
|
|
|
651
722
|
if (postSnapshotBlobReferences.length > 0) {
|
|
@@ -662,6 +733,7 @@ export async function captureFullContainerState({
|
|
|
662
733
|
attached: true,
|
|
663
734
|
baseSnapshot,
|
|
664
735
|
snapshotBlobs,
|
|
736
|
+
blobContentsMode: blobCaptureMode === "reference" ? "reference" : undefined,
|
|
665
737
|
attachmentBlobContents:
|
|
666
738
|
Object.keys(attachmentBlobContents).length === 0 ? undefined : attachmentBlobContents,
|
|
667
739
|
loadedGroupIdSnapshots: undefined,
|
|
@@ -671,10 +743,23 @@ export async function captureFullContainerState({
|
|
|
671
743
|
};
|
|
672
744
|
return JSON.stringify(pendingState);
|
|
673
745
|
} finally {
|
|
746
|
+
deltaManager.dispose();
|
|
674
747
|
documentService.dispose();
|
|
675
748
|
}
|
|
676
749
|
}
|
|
677
750
|
|
|
751
|
+
async function waitForCatchUp(
|
|
752
|
+
deltaManager: ReturnType<typeof createDeltaManager>,
|
|
753
|
+
): Promise<void> {
|
|
754
|
+
await new Promise<void>((resolve) => {
|
|
755
|
+
const monitor = new CatchUpMonitor(deltaManager, () => {
|
|
756
|
+
monitor.dispose();
|
|
757
|
+
resolve();
|
|
758
|
+
});
|
|
759
|
+
monitor.start();
|
|
760
|
+
});
|
|
761
|
+
}
|
|
762
|
+
|
|
678
763
|
/**
|
|
679
764
|
* Loads a summarizer container with the required headers, triggers an on-demand summary, and then closes it.
|
|
680
765
|
* Returns success/failure and an optional error for host-side handling.
|
|
@@ -708,7 +793,7 @@ async function loadSummarizerContainerAndMakeSummaryInternal(
|
|
|
708
793
|
};
|
|
709
794
|
|
|
710
795
|
const subMc = mixinMonitoringContext(
|
|
711
|
-
|
|
796
|
+
mixinDebugLogger("fluid:telemetry", logger, {
|
|
712
797
|
all: telemetryProps,
|
|
713
798
|
}),
|
|
714
799
|
sessionStorageConfigProvider.value,
|
package/src/debugLogger.ts
CHANGED
|
@@ -25,46 +25,8 @@ const { debug: registerDebug } = debugPkg;
|
|
|
25
25
|
/**
|
|
26
26
|
* Implementation of debug logger
|
|
27
27
|
*/
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
* Mix in debug logger with another logger.
|
|
31
|
-
* Returned logger will output events to both newly created debug logger, as well as base logger
|
|
32
|
-
* @param namespace - Telemetry event name prefix to add to all events
|
|
33
|
-
* @param properties - Base properties to add to all events
|
|
34
|
-
* @param propertyGetters - Getters to add additional properties to all events
|
|
35
|
-
* @param baseLogger - Base logger to output events (in addition to debug logger being created). Can be undefined.
|
|
36
|
-
*/
|
|
37
|
-
public static mixinDebugLogger(
|
|
38
|
-
namespace: string,
|
|
39
|
-
baseLogger?: ITelemetryBaseLogger,
|
|
40
|
-
properties?: ITelemetryLoggerPropertyBags,
|
|
41
|
-
): TelemetryLoggerExt {
|
|
42
|
-
// Setup base logger upfront, such that host can disable it (if needed)
|
|
43
|
-
const debug = registerDebug(namespace);
|
|
44
|
-
|
|
45
|
-
// Create one for errors that is always enabled
|
|
46
|
-
// It can be silenced by replacing console.error if the debug namespace is not enabled.
|
|
47
|
-
const debugErr = registerDebug(namespace);
|
|
48
|
-
debugErr.log = function (...args: unknown[]): void {
|
|
49
|
-
if (debug.enabled === true) {
|
|
50
|
-
// if the namespace is enabled, just use the default logger
|
|
51
|
-
registerDebug.log(...args);
|
|
52
|
-
} else {
|
|
53
|
-
// other wise, use the console logger (which could be replaced and silenced)
|
|
54
|
-
console.error(...args);
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
debugErr.enabled = true;
|
|
58
|
-
|
|
59
|
-
return createMultiSinkLogger({
|
|
60
|
-
namespace,
|
|
61
|
-
loggers: [baseLogger, new DebugLogger(debug, debugErr)],
|
|
62
|
-
properties,
|
|
63
|
-
tryInheritProperties: true,
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
private constructor(
|
|
28
|
+
class DebugLogger implements ITelemetryBaseLogger {
|
|
29
|
+
public constructor(
|
|
68
30
|
private readonly debug: IDebugger,
|
|
69
31
|
private readonly debugErr: IDebugger,
|
|
70
32
|
) {}
|
|
@@ -119,3 +81,40 @@ export class DebugLogger implements ITelemetryBaseLogger {
|
|
|
119
81
|
logger(`${name} ${payload} ${tick} ${stack}`);
|
|
120
82
|
}
|
|
121
83
|
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Mix in debug logger with another logger.
|
|
87
|
+
* Returned logger will output events to both newly created debug logger, as well as base logger
|
|
88
|
+
* @param namespace - Telemetry event name prefix to add to all events
|
|
89
|
+
* @param baseLogger - Base logger to output events (in addition to debug logger being created). Can be undefined.
|
|
90
|
+
* @param properties - Base properties to add to all events
|
|
91
|
+
*/
|
|
92
|
+
export function mixinDebugLogger(
|
|
93
|
+
namespace: string,
|
|
94
|
+
baseLogger?: ITelemetryBaseLogger,
|
|
95
|
+
properties?: ITelemetryLoggerPropertyBags,
|
|
96
|
+
): TelemetryLoggerExt {
|
|
97
|
+
// Setup base logger upfront, such that host can disable it (if needed)
|
|
98
|
+
const debug = registerDebug(namespace);
|
|
99
|
+
|
|
100
|
+
// Create one for errors that is always enabled
|
|
101
|
+
// It can be silenced by replacing console.error if the debug namespace is not enabled.
|
|
102
|
+
const debugErr = registerDebug(namespace);
|
|
103
|
+
debugErr.log = function (...args: unknown[]): void {
|
|
104
|
+
if (debug.enabled === true) {
|
|
105
|
+
// if the namespace is enabled, just use the default logger
|
|
106
|
+
registerDebug.log(...args);
|
|
107
|
+
} else {
|
|
108
|
+
// other wise, use the console logger (which could be replaced and silenced)
|
|
109
|
+
console.error(...args);
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
debugErr.enabled = true;
|
|
113
|
+
|
|
114
|
+
return createMultiSinkLogger({
|
|
115
|
+
namespace,
|
|
116
|
+
loggers: [baseLogger, new DebugLogger(debug, debugErr)],
|
|
117
|
+
properties,
|
|
118
|
+
tryInheritProperties: true,
|
|
119
|
+
});
|
|
120
|
+
}
|
package/src/deltaManager.ts
CHANGED
|
@@ -16,6 +16,7 @@ import type {
|
|
|
16
16
|
ITelemetryBaseEvent,
|
|
17
17
|
ITelemetryBaseProperties,
|
|
18
18
|
} from "@fluidframework/core-interfaces";
|
|
19
|
+
import { LogLevel } from "@fluidframework/core-interfaces";
|
|
19
20
|
import { JsonParse } from "@fluidframework/core-interfaces/internal";
|
|
20
21
|
import type { IThrottlingWarning, JsonString } from "@fluidframework/core-interfaces/internal";
|
|
21
22
|
import { assert } from "@fluidframework/core-utils/internal";
|
|
@@ -143,7 +144,7 @@ function logIfFalse(
|
|
|
143
144
|
typeof event === "string"
|
|
144
145
|
? { eventName: event, category: "error" }
|
|
145
146
|
: { category: "error", ...event };
|
|
146
|
-
logger.send(newEvent);
|
|
147
|
+
logger.send(newEvent, LogLevel.essential);
|
|
147
148
|
return false;
|
|
148
149
|
}
|
|
149
150
|
|
|
@@ -566,10 +567,22 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
|
|
|
566
567
|
}
|
|
567
568
|
|
|
568
569
|
/**
|
|
569
|
-
*
|
|
570
|
-
*
|
|
571
|
-
*
|
|
572
|
-
*
|
|
570
|
+
* Initializes sequence number tracking, attaches the handler for inbound ops and signals, and
|
|
571
|
+
* resumes the inbound queues.
|
|
572
|
+
*
|
|
573
|
+
* @param minSequenceNumber - The document's current minimum sequence number. This is used for client-side
|
|
574
|
+
* protocol validation.
|
|
575
|
+
* @param snapshotSequenceNumber - The sequence number of the snapshot from which the document
|
|
576
|
+
* was loaded.
|
|
577
|
+
* @param handler - The op/signal handler to attach.
|
|
578
|
+
* @param prefetchType - Controls how missing ops are fetched before the returned promise resolves:
|
|
579
|
+
* `"none"` does not initiate a fetch; `"all"` waits for all available missing ops from storage;
|
|
580
|
+
* and `"cached"` waits only for cached ops before continuing the remaining storage fetch in the
|
|
581
|
+
* background.
|
|
582
|
+
* @param lastProcessedSequenceNumber - The latest sequence number already reflected in the
|
|
583
|
+
* loaded state. It defaults to `snapshotSequenceNumber`. Offline loads may provide a later
|
|
584
|
+
* sequence number so the DeltaManager skips downloading and processing ops already included in
|
|
585
|
+
* that state.
|
|
573
586
|
*/
|
|
574
587
|
public async attachOpHandler(
|
|
575
588
|
minSequenceNumber: number,
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { IClient } from "@fluidframework/driver-definitions";
|
|
7
|
+
import type { IDocumentService } from "@fluidframework/driver-definitions/internal";
|
|
8
|
+
import {
|
|
9
|
+
createChildLogger,
|
|
10
|
+
type TelemetryLoggerExt,
|
|
11
|
+
} from "@fluidframework/telemetry-utils/internal";
|
|
12
|
+
|
|
13
|
+
import { ConnectionManager } from "./connectionManager.js";
|
|
14
|
+
import { DeltaManager } from "./deltaManager.js";
|
|
15
|
+
|
|
16
|
+
export interface ICreateDeltaManagerProps {
|
|
17
|
+
readonly serviceProvider: () => IDocumentService | undefined;
|
|
18
|
+
readonly logger: TelemetryLoggerExt;
|
|
19
|
+
readonly active: () => boolean;
|
|
20
|
+
readonly containerDirty: () => boolean;
|
|
21
|
+
readonly client: IClient;
|
|
22
|
+
readonly reconnectAllowed: boolean;
|
|
23
|
+
readonly maxInitialConnectionAttempts?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Creates the DeltaManager and ConnectionManager pair used by container loading.
|
|
28
|
+
*/
|
|
29
|
+
export function createDeltaManager({
|
|
30
|
+
serviceProvider,
|
|
31
|
+
logger,
|
|
32
|
+
active,
|
|
33
|
+
containerDirty,
|
|
34
|
+
client,
|
|
35
|
+
reconnectAllowed,
|
|
36
|
+
maxInitialConnectionAttempts,
|
|
37
|
+
}: ICreateDeltaManagerProps): DeltaManager<ConnectionManager> {
|
|
38
|
+
return new DeltaManager<ConnectionManager>(
|
|
39
|
+
serviceProvider,
|
|
40
|
+
createChildLogger({ logger, namespace: "DeltaManager" }),
|
|
41
|
+
active,
|
|
42
|
+
(props) =>
|
|
43
|
+
new ConnectionManager(
|
|
44
|
+
serviceProvider,
|
|
45
|
+
containerDirty,
|
|
46
|
+
client,
|
|
47
|
+
reconnectAllowed,
|
|
48
|
+
createChildLogger({ logger, namespace: "ConnectionManager" }),
|
|
49
|
+
props,
|
|
50
|
+
maxInitialConnectionAttempts,
|
|
51
|
+
),
|
|
52
|
+
);
|
|
53
|
+
}
|
package/src/frozenServices.ts
CHANGED
|
@@ -171,16 +171,15 @@ const frozenDocumentStorageServiceHandler = (): never => {
|
|
|
171
171
|
|
|
172
172
|
/**
|
|
173
173
|
* Distinct from {@link frozenDocumentStorageServiceHandler} because callers
|
|
174
|
-
* that hit this path are
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
* message names the missing precondition and the API that produces it.
|
|
174
|
+
* that hit this path are exercising a fully-offline frozen load whose pending
|
|
175
|
+
* state omits a required structural or attachment blob. A generic "operation
|
|
176
|
+
* not supported" is technically true but unhelpful; this message names the
|
|
177
|
+
* missing precondition and the API mode that satisfies it.
|
|
179
178
|
*/
|
|
180
179
|
const frozenReadBlobOfflineHandler = async (): Promise<never> => {
|
|
181
180
|
throw new UsageError(
|
|
182
|
-
"Attempted to read
|
|
183
|
-
"Fully-offline frozen loads must use pending state produced by `captureFullContainerState
|
|
181
|
+
"Attempted to read a blob from a frozen-loaded container without a live storage service. " +
|
|
182
|
+
"Fully-offline frozen loads must use pending state produced by `captureFullContainerState` with the default inline blob capture mode.",
|
|
184
183
|
);
|
|
185
184
|
};
|
|
186
185
|
|
package/src/index.ts
CHANGED
|
@@ -24,8 +24,9 @@ import {
|
|
|
24
24
|
* This is distinct from normal container loading. The supplied
|
|
25
25
|
* {@link IContainerDriverServices.documentServiceFactory} must be able to materialize the document
|
|
26
26
|
* at {@link ILoadContainerToSequenceNumberProps.loadToSequenceNumber} - i.e. it must implement the
|
|
27
|
-
* point-in-time capability the loader detects. For ODSP,
|
|
28
|
-
* `
|
|
27
|
+
* point-in-time capability the loader detects. For ODSP, call
|
|
28
|
+
* `createOdspDocumentServiceFactory` with the implementation imported from
|
|
29
|
+
* `@fluidframework/odsp-driver/legacy/point-in-time`.
|
|
29
30
|
*
|
|
30
31
|
* @legacy @beta
|
|
31
32
|
*/
|
|
@@ -58,9 +59,9 @@ export interface ILoadContainerToSequenceNumberProps
|
|
|
58
59
|
* The supplied {@link IContainerDriverServices.documentServiceFactory} must support point-in-time
|
|
59
60
|
* loading: it must be able to serve a snapshot at or before
|
|
60
61
|
* {@link ILoadContainerToSequenceNumberProps.loadToSequenceNumber} and replay the document forward
|
|
61
|
-
* through that sequence number. For ODSP,
|
|
62
|
-
*
|
|
63
|
-
*
|
|
62
|
+
* through that sequence number. For ODSP, inject `createPointInTimeDocumentService` from the
|
|
63
|
+
* dedicated point-in-time entrypoint through `createOdspDocumentServiceFactory` options. The loader
|
|
64
|
+
* materializes the point-in-time view itself, so no wrapping or decoration is required.
|
|
64
65
|
*
|
|
65
66
|
* @param props - The load options, point-in-time-capable driver services, target sequence number, and
|
|
66
67
|
* optional cancellation signal.
|
|
@@ -81,7 +82,7 @@ export async function loadContainerToSequenceNumber(
|
|
|
81
82
|
const capableFactory = asPointInTimeCapableFactory(documentServiceFactory);
|
|
82
83
|
if (capableFactory === undefined) {
|
|
83
84
|
throw new UsageError(
|
|
84
|
-
"The provided documentServiceFactory does not support point-in-time loading. For ODSP,
|
|
85
|
+
"The provided documentServiceFactory does not support point-in-time loading. For ODSP, call createOdspDocumentServiceFactory with a point-in-time implementation.",
|
|
85
86
|
);
|
|
86
87
|
}
|
|
87
88
|
|
package/src/loader.ts
CHANGED
|
@@ -38,7 +38,7 @@ import type { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils/legacy
|
|
|
38
38
|
import { v4 as uuid } from "uuid";
|
|
39
39
|
|
|
40
40
|
import { Container } from "./container.js";
|
|
41
|
-
import {
|
|
41
|
+
import { mixinDebugLogger } from "./debugLogger.js";
|
|
42
42
|
import { pkgVersion } from "./packageVersion.js";
|
|
43
43
|
import type { ProtocolHandlerBuilder } from "./protocol.js";
|
|
44
44
|
import type { IPendingContainerState } from "./serializedStateManager.js";
|
|
@@ -255,7 +255,7 @@ export class Loader implements IHostLoader {
|
|
|
255
255
|
};
|
|
256
256
|
|
|
257
257
|
const subMc = mixinMonitoringContext(
|
|
258
|
-
|
|
258
|
+
mixinDebugLogger("fluid:telemetry", logger, {
|
|
259
259
|
all: telemetryProps,
|
|
260
260
|
}),
|
|
261
261
|
sessionStorageConfigProvider.value,
|