@fluidframework/container-runtime 2.51.0 → 2.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/api-report/container-runtime.legacy.alpha.api.md +1 -2
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +15 -7
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +72 -186
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/containerCompatibility.d.ts +34 -0
- package/dist/containerCompatibility.d.ts.map +1 -0
- package/dist/containerCompatibility.js +125 -0
- package/dist/containerCompatibility.js.map +1 -0
- package/dist/containerRuntime.d.ts +27 -15
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +175 -136
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +6 -6
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metadata.d.ts +3 -2
- package/dist/metadata.d.ts.map +1 -1
- package/dist/metadata.js +7 -1
- package/dist/metadata.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +40 -5
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -1
- package/dist/storageServiceWithAttachBlobs.js +56 -5
- package/dist/storageServiceWithAttachBlobs.js.map +1 -1
- package/dist/summary/documentSchema.d.ts +1 -1
- package/dist/summary/documentSchema.d.ts.map +1 -1
- package/dist/summary/documentSchema.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +3 -3
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts +15 -7
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +39 -153
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/containerCompatibility.d.ts +34 -0
- package/lib/containerCompatibility.d.ts.map +1 -0
- package/lib/containerCompatibility.js +120 -0
- package/lib/containerCompatibility.js.map +1 -0
- package/lib/containerRuntime.d.ts +27 -15
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +103 -64
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +6 -6
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +1 -1
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/index.d.ts +5 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/metadata.d.ts +3 -2
- package/lib/metadata.d.ts.map +1 -1
- package/lib/metadata.js +5 -0
- package/lib/metadata.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +40 -5
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -1
- package/lib/storageServiceWithAttachBlobs.js +56 -5
- package/lib/storageServiceWithAttachBlobs.js.map +1 -1
- package/lib/summary/documentSchema.d.ts +1 -1
- package/lib/summary/documentSchema.d.ts.map +1 -1
- package/lib/summary/documentSchema.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +3 -3
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js.map +1 -1
- package/package.json +20 -20
- package/src/blobManager/blobManager.ts +53 -195
- package/src/containerCompatibility.ts +176 -0
- package/src/containerRuntime.ts +157 -122
- package/src/dataStoreContext.ts +13 -5
- package/src/index.ts +6 -1
- package/src/metadata.ts +10 -2
- package/src/packageVersion.ts +1 -1
- package/src/storageServiceWithAttachBlobs.ts +92 -10
- package/src/summary/documentSchema.ts +1 -1
- package/src/summary/summaryFormat.ts +2 -2
- package/dist/compatUtils.d.ts +0 -106
- package/dist/compatUtils.d.ts.map +0 -1
- package/dist/compatUtils.js +0 -251
- package/dist/compatUtils.js.map +0 -1
- package/lib/compatUtils.d.ts +0 -106
- package/lib/compatUtils.d.ts.map +0 -1
- package/lib/compatUtils.js +0 -242
- package/lib/compatUtils.js.map +0 -1
- package/src/compatUtils.ts +0 -365
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
|
+
import { type IContainerStorageService } from "@fluidframework/container-definitions/internal";
|
|
5
6
|
import { IContainerRuntime, IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions/internal";
|
|
6
7
|
import type { IEventProvider, IFluidHandleContext, IFluidHandleInternalPayloadPending, ILocalFluidHandle, ILocalFluidHandleEvents, Listenable, PayloadState } from "@fluidframework/core-interfaces/internal";
|
|
7
|
-
import { IDocumentStorageService, ICreateBlobResponse } from "@fluidframework/driver-definitions/internal";
|
|
8
8
|
import { IGarbageCollectionData, ISummaryTreeWithStats, ITelemetryContext, type ISequencedMessageEnvelope } from "@fluidframework/runtime-definitions/internal";
|
|
9
9
|
import { FluidHandleBase } from "@fluidframework/runtime-utils/internal";
|
|
10
10
|
import { type IBlobManagerLoadInfo } from "./blobManagerSnapSum.js";
|
|
@@ -77,20 +77,17 @@ export declare class BlobManager {
|
|
|
77
77
|
*/
|
|
78
78
|
private readonly opsInFlight;
|
|
79
79
|
private readonly sendBlobAttachOp;
|
|
80
|
-
private stopAttaching;
|
|
81
80
|
private readonly routeContext;
|
|
82
81
|
private readonly storage;
|
|
83
82
|
private readonly blobRequested;
|
|
84
83
|
private readonly isBlobDeleted;
|
|
85
84
|
private readonly runtime;
|
|
86
85
|
private readonly localBlobIdGenerator;
|
|
87
|
-
private readonly pendingStashedBlobs;
|
|
88
|
-
readonly stashedBlobsUploadP: Promise<(void | ICreateBlobResponse)[]>;
|
|
89
86
|
private readonly createBlobPayloadPending;
|
|
90
87
|
constructor(props: {
|
|
91
88
|
readonly routeContext: IFluidHandleContext;
|
|
92
89
|
blobManagerLoadInfo: IBlobManagerLoadInfo;
|
|
93
|
-
readonly storage:
|
|
90
|
+
readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
|
|
94
91
|
/**
|
|
95
92
|
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
|
|
96
93
|
* deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
|
|
@@ -101,7 +98,7 @@ export declare class BlobManager {
|
|
|
101
98
|
* knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
|
|
102
99
|
* before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
|
|
103
100
|
*/
|
|
104
|
-
sendBlobAttachOp: (localId: string, storageId
|
|
101
|
+
sendBlobAttachOp: (localId: string, storageId: string) => void;
|
|
105
102
|
readonly blobRequested: (blobPath: string) => void;
|
|
106
103
|
readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
107
104
|
readonly runtime: IBlobManagerRuntime;
|
|
@@ -112,7 +109,6 @@ export declare class BlobManager {
|
|
|
112
109
|
get allBlobsAttached(): boolean;
|
|
113
110
|
get hasPendingBlobs(): boolean;
|
|
114
111
|
private createAbortError;
|
|
115
|
-
hasPendingStashedUploads(): boolean;
|
|
116
112
|
hasBlob(blobId: string): boolean;
|
|
117
113
|
/**
|
|
118
114
|
* Retrieve the blob with the given local blob id.
|
|
@@ -179,6 +175,18 @@ export declare class BlobManager {
|
|
|
179
175
|
*/
|
|
180
176
|
private verifyBlobNotDeleted;
|
|
181
177
|
setRedirectTable(table: Map<string, string>): void;
|
|
178
|
+
/**
|
|
179
|
+
* To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
|
|
180
|
+
* pending upload and/or their BlobAttach op, which can be given to a new BlobManager to
|
|
181
|
+
* resume work.
|
|
182
|
+
*
|
|
183
|
+
* @privateRemarks
|
|
184
|
+
* For now, we don't track any pending blobs since the getPendingBlobs flow doesn't enable
|
|
185
|
+
* restoring to a state where an accessible handle has been stored by the customer (and we'll
|
|
186
|
+
* just drop any BlobAttach ops on the ground during reSubmit). However, once we add support
|
|
187
|
+
* for payload-pending handles, this will return the blobs associated with those handles.
|
|
188
|
+
*/
|
|
189
|
+
getPendingBlobs(): IPendingBlobs | undefined;
|
|
182
190
|
/**
|
|
183
191
|
* Part of container serialization when imminent closure is enabled (Currently when calling closeAndGetPendingLocalState).
|
|
184
192
|
* This asynchronous function resolves all pending createBlob calls and waits for each blob
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../../src/blobManager/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../../src/blobManager/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAEN,KAAK,wBAAwB,EAC7B,MAAM,gDAAgD,CAAC;AACxD,OAAO,EACN,iBAAiB,EACjB,uBAAuB,EACvB,MAAM,wDAAwD,CAAC;AAChE,OAAO,KAAK,EAEX,cAAc,EACd,mBAAmB,EACnB,kCAAkC,EAClC,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,YAAY,EACZ,MAAM,0CAA0C,CAAC;AAIlD,OAAO,EACN,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,KAAK,yBAAyB,EAC9B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACN,eAAe,EAIf,MAAM,wCAAwC,CAAC;AAahD,OAAO,EAIN,KAAK,oBAAoB,EACzB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,qBAAa,UACZ,SAAQ,eAAe,CAAC,eAAe,CACvC,YACC,iBAAiB,CAAC,eAAe,CAAC,EAClC,kCAAkC,CAAC,eAAe,CAAC;aAgCnC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC;aAC1B,cAAc,EAAE,OAAO;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAlChC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,OAAO,CAAC,OAAO,CAEF;IACb,IAAW,MAAM,IAAI,UAAU,CAAC,uBAAuB,CAAC,CAEvD;IAED,OAAO,CAAC,MAAM,CAA2B;IACzC,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED;;;OAGG;IACH,OAAO,CAAC,kBAAkB,CAAU;IACpC,IAAW,iBAAiB,IAAI,OAAO,CAEtC;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGpB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC,EAC1B,cAAc,EAAE,OAAO,EACtB,aAAa,CAAC,SAAQ,IAAI,aAAA;IAM5C,SAAgB,YAAY,QAAO,IAAI,CAGrC;IAEF,SAAgB,YAAY,UAAW,OAAO,KAAG,IAAI,CAGnD;IAEK,WAAW,IAAI,IAAI;CAM1B;AAID,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACrC,iBAAiB,EACjB,aAAa,GAAG,WAAW,GAAG,YAAY,GAAG,eAAe,GAAG,UAAU,CACzE,GACA,cAAc,CAAC,uBAAuB,CAAC,CAAC;AAkBzC,MAAM,WAAW,aAAa;IAC7B,CAAC,OAAO,EAAE,MAAM,GAAG;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KAChB,CAAC;CACF;AAED,MAAM,WAAW,kBAAkB;IAClC,cAAc,EAAE,MAAM,IAAI,CAAC;CAC3B;AAQD,eAAO,MAAM,mBAAmB,UAAoB,CAAC;AAErD,qBAAa,WAAW;IACvB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IACpE,IAAW,MAAM,IAAI,UAAU,CAAC,kBAAkB,CAAC,CAElD;IACD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA+C;IAE9E;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IAEnE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA+C;IAEhF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4D;IAGpF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA6B;IAG3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAe;IAEpD,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAU;gBAEhC,KAAK,EAAE;QACzB,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAAC;QAE3C,mBAAmB,EAAE,oBAAoB,CAAC;QAC1C,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,YAAY,GAAG,UAAU,CAAC,CAAC;QAC5E;;;;;;;;;WASG;QACH,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;QAG/D,QAAQ,CAAC,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QAGnD,QAAQ,CAAC,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;QACtD,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;QACtC,YAAY,EAAE,aAAa,GAAG,SAAS,CAAC;QACxC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;QAC3D,QAAQ,CAAC,wBAAwB,EAAE,OAAO,CAAC;KAC3C;IAgED,IAAW,gBAAgB,IAAI,OAAO,CAOrC;IAED,IAAW,eAAe,IAAI,OAAO,CAKpC;IAED,OAAO,CAAC,gBAAgB;IAOjB,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAIvC;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IA8DvF,OAAO,CAAC,aAAa;YAwBP,kBAAkB;IAUnB,UAAU,CACtB,IAAI,EAAE,eAAe,EACrB,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;YAmBjD,gBAAgB;IAkC9B,OAAO,CAAC,4BAA4B;YA0CtB,UAAU;IAgDxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,eAAe;IAwDvB;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI;IAc7D,wBAAwB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAiDlF,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAI7E;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAejE;;;;;OAKG;IACI,qBAAqB,CAAC,oBAAoB,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,MAAM,EAAE;IAKxF;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,4BAA4B;IA8CpC;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAqBrB,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAiBzD;;;;;;;;;;OAUG;IACI,eAAe,IAAI,aAAa,GAAG,SAAS;IAInD;;;;;;;;;OASG;IACU,wBAAwB,CACpC,uBAAuB,CAAC,EAAE,WAAW,GACnC,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;CAGrC;AAmBD;;GAEG;AACH,eAAO,MAAM,UAAU,SAAU,MAAM,gCACL,CAAC"}
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
6
|
-
import { AttachState } from "@fluidframework/container-definitions";
|
|
5
|
+
import { createEmitter } from "@fluid-internal/client-utils";
|
|
6
|
+
import { AttachState, } from "@fluidframework/container-definitions/internal";
|
|
7
7
|
import { assert, Deferred } from "@fluidframework/core-utils/internal";
|
|
8
8
|
import { canRetryOnError, runWithRetry } from "@fluidframework/driver-utils/internal";
|
|
9
9
|
import { FluidHandleBase, createResponseError, generateHandleContextPath, responseToException, } from "@fluidframework/runtime-utils/internal";
|
|
10
|
-
import { LoggingError, PerformanceEvent, createChildMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils/internal";
|
|
10
|
+
import { LoggingError, PerformanceEvent, UsageError, createChildMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils/internal";
|
|
11
11
|
import { v4 as uuid } from "uuid";
|
|
12
|
+
import { isBlobMetadata } from "../metadata.js";
|
|
12
13
|
import { getStorageIds, summarizeBlobManagerState, toRedirectTable, } from "./blobManagerSnapSum.js";
|
|
13
14
|
/**
|
|
14
15
|
* This class represents blob (long string)
|
|
@@ -56,12 +57,6 @@ export class BlobHandle extends FluidHandleBase {
|
|
|
56
57
|
}
|
|
57
58
|
}
|
|
58
59
|
}
|
|
59
|
-
const stashedPendingBlobOverrides = {
|
|
60
|
-
stashedUpload: true,
|
|
61
|
-
storageId: undefined,
|
|
62
|
-
minTTLInSeconds: undefined,
|
|
63
|
-
uploadTime: undefined,
|
|
64
|
-
};
|
|
65
60
|
export const blobManagerBasePath = "_blobs";
|
|
66
61
|
export class BlobManager {
|
|
67
62
|
get events() {
|
|
@@ -80,9 +75,7 @@ export class BlobManager {
|
|
|
80
75
|
* because we know that the server will not delete the blob corresponding to that storage ID.
|
|
81
76
|
*/
|
|
82
77
|
this.opsInFlight = new Map();
|
|
83
|
-
|
|
84
|
-
this.pendingStashedBlobs = new Map();
|
|
85
|
-
const { routeContext, blobManagerLoadInfo, storage, sendBlobAttachOp, blobRequested, isBlobDeleted, runtime, stashedBlobs, localBlobIdGenerator, createBlobPayloadPending, } = props;
|
|
78
|
+
const { routeContext, blobManagerLoadInfo, storage, sendBlobAttachOp, blobRequested, isBlobDeleted, runtime, localBlobIdGenerator, createBlobPayloadPending, } = props;
|
|
86
79
|
this.routeContext = routeContext;
|
|
87
80
|
this.storage = storage;
|
|
88
81
|
this.blobRequested = blobRequested;
|
|
@@ -95,39 +88,6 @@ export class BlobManager {
|
|
|
95
88
|
namespace: "BlobManager",
|
|
96
89
|
});
|
|
97
90
|
this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger, this.runtime.attachState);
|
|
98
|
-
// Begin uploading stashed blobs from previous container instance
|
|
99
|
-
for (const [localId, entry] of Object.entries(stashedBlobs ?? {})) {
|
|
100
|
-
const { acked, storageId, minTTLInSeconds, uploadTime } = entry;
|
|
101
|
-
const blob = stringToBuffer(entry.blob, "base64");
|
|
102
|
-
const pendingEntry = {
|
|
103
|
-
blob,
|
|
104
|
-
opsent: true,
|
|
105
|
-
handleP: new Deferred(),
|
|
106
|
-
storageId,
|
|
107
|
-
uploadP: undefined,
|
|
108
|
-
uploadTime,
|
|
109
|
-
minTTLInSeconds,
|
|
110
|
-
attached: true,
|
|
111
|
-
acked,
|
|
112
|
-
};
|
|
113
|
-
this.pendingBlobs.set(localId, pendingEntry);
|
|
114
|
-
if (storageId !== undefined && minTTLInSeconds && uploadTime) {
|
|
115
|
-
const timeLapseSinceLocalUpload = (Date.now() - uploadTime) / 1000;
|
|
116
|
-
// stashed entries with more than half-life in storage will not be reuploaded
|
|
117
|
-
if (minTTLInSeconds - timeLapseSinceLocalUpload > minTTLInSeconds / 2) {
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
this.pendingStashedBlobs.set(localId, this.uploadBlob(localId, blob));
|
|
122
|
-
this.pendingBlobs.set(localId, {
|
|
123
|
-
...pendingEntry,
|
|
124
|
-
...stashedPendingBlobOverrides,
|
|
125
|
-
uploadP: this.pendingStashedBlobs.get(localId),
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
this.stashedBlobsUploadP = PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "BlobUploadProcessStashedChanges", count: this.pendingStashedBlobs.size }, async () => Promise.all(this.pendingStashedBlobs.values()), { start: true, end: true }).finally(() => {
|
|
129
|
-
this.pendingStashedBlobs.clear();
|
|
130
|
-
});
|
|
131
91
|
this.sendBlobAttachOp = (localId, blobId) => {
|
|
132
92
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
133
93
|
assert(pendingEntry !== undefined, 0x725 /* Must have pending blob entry for upcoming op */);
|
|
@@ -175,9 +135,6 @@ export class BlobManager {
|
|
|
175
135
|
uploadTime: pending?.uploadTime,
|
|
176
136
|
});
|
|
177
137
|
}
|
|
178
|
-
hasPendingStashedUploads() {
|
|
179
|
-
return [...this.pendingBlobs.values()].some((e) => e.stashedUpload === true);
|
|
180
|
-
}
|
|
181
138
|
hasBlob(blobId) {
|
|
182
139
|
return this.redirectTable.get(blobId) !== undefined;
|
|
183
140
|
}
|
|
@@ -203,7 +160,7 @@ export class BlobManager {
|
|
|
203
160
|
if (this.runtime.attachState === AttachState.Detached) {
|
|
204
161
|
assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);
|
|
205
162
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
206
|
-
// The '
|
|
163
|
+
// The 'IContainerStorageService.readBlob()' call below will retrieve these via localId.
|
|
207
164
|
storageId = blobId;
|
|
208
165
|
}
|
|
209
166
|
else {
|
|
@@ -256,7 +213,7 @@ export class BlobManager {
|
|
|
256
213
|
}
|
|
257
214
|
async createBlobDetached(blob) {
|
|
258
215
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
259
|
-
// The '
|
|
216
|
+
// The 'IContainerStorageService.createBlob()' call below will respond with a localId.
|
|
260
217
|
const response = await this.storage.createBlob(blob);
|
|
261
218
|
this.setRedirection(response.id, undefined);
|
|
262
219
|
return this.getBlobHandle(response.id);
|
|
@@ -385,17 +342,8 @@ export class BlobManager {
|
|
|
385
342
|
}
|
|
386
343
|
onUploadResolve(localId, response) {
|
|
387
344
|
const entry = this.pendingBlobs.get(localId);
|
|
388
|
-
if (entry === undefined && this.pendingStashedBlobs.has(localId)) {
|
|
389
|
-
// The blob was already processed and deleted. This can happen if the blob was reuploaded by
|
|
390
|
-
// the stashing process and the original upload was processed before the stashed upload.
|
|
391
|
-
this.mc.logger.sendTelemetryEvent({
|
|
392
|
-
eventName: "StashedBlobAlreadyProcessed",
|
|
393
|
-
localId,
|
|
394
|
-
});
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
345
|
assert(entry !== undefined, 0x6c8 /* pending blob entry not found for uploaded blob */);
|
|
398
|
-
if (
|
|
346
|
+
if (entry.abortSignal?.aborted === true && !entry.opsent) {
|
|
399
347
|
this.mc.logger.sendTelemetryEvent({
|
|
400
348
|
eventName: "BlobAborted",
|
|
401
349
|
localId,
|
|
@@ -404,7 +352,6 @@ export class BlobManager {
|
|
|
404
352
|
return;
|
|
405
353
|
}
|
|
406
354
|
assert(entry.storageId === undefined, 0x386 /* Must have pending blob entry for uploaded blob */);
|
|
407
|
-
entry.stashedUpload = undefined;
|
|
408
355
|
entry.storageId = response.id;
|
|
409
356
|
entry.uploadTime = Date.now();
|
|
410
357
|
entry.minTTLInSeconds = response.minTTLInSeconds;
|
|
@@ -448,38 +395,29 @@ export class BlobManager {
|
|
|
448
395
|
* @param metadata - op metadata containing storage and/or local IDs
|
|
449
396
|
*/
|
|
450
397
|
reSubmit(metadata) {
|
|
451
|
-
assert(
|
|
398
|
+
assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
|
|
452
399
|
const { localId, blobId } = metadata;
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
400
|
+
// Any blob that we're actively trying to advance to attached state must have a
|
|
401
|
+
// pendingBlobs entry. Decline to resubmit for anything else.
|
|
402
|
+
// For example, we might be asked to resubmit stashed ops for blobs that never had
|
|
403
|
+
// their handle attached - these won't have a pendingBlobs entry and we shouldn't
|
|
404
|
+
// try to attach them since they won't be accessible to the customer and would just
|
|
405
|
+
// be considered garbage immediately.
|
|
406
|
+
if (this.pendingBlobs.has(localId)) {
|
|
407
|
+
this.sendBlobAttachOp(localId, blobId);
|
|
459
408
|
}
|
|
460
|
-
return this.sendBlobAttachOp(localId, blobId);
|
|
461
409
|
}
|
|
462
410
|
processBlobAttachMessage(message, local) {
|
|
463
|
-
|
|
464
|
-
const blobId = message.metadata
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
assert(blobId !== undefined, 0x12a /* "Missing blob id on metadata" */);
|
|
473
|
-
// Set up a mapping from local ID to storage ID. This is crucial since without this the blob cannot be
|
|
474
|
-
// requested from the server.
|
|
475
|
-
// Note: The check for undefined is needed for back-compat when localId was not part of the BlobAttach op that
|
|
476
|
-
// was sent when online.
|
|
477
|
-
if (localId !== undefined) {
|
|
478
|
-
this.setRedirection(localId, blobId);
|
|
411
|
+
assert(isBlobMetadata(message.metadata), 0xc02 /* Expected blob metadata for a BlobAttach op */);
|
|
412
|
+
const { localId, blobId } = message.metadata;
|
|
413
|
+
const pendingEntry = this.pendingBlobs.get(localId);
|
|
414
|
+
if (pendingEntry?.abortSignal?.aborted) {
|
|
415
|
+
this.deletePendingBlob(localId);
|
|
416
|
+
return;
|
|
479
417
|
}
|
|
418
|
+
this.setRedirection(localId, blobId);
|
|
480
419
|
// set identity (id -> id) entry
|
|
481
420
|
this.setRedirection(blobId, blobId);
|
|
482
|
-
assert(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
|
|
483
421
|
if (local) {
|
|
484
422
|
const waitingBlobs = this.opsInFlight.get(blobId);
|
|
485
423
|
if (waitingBlobs !== undefined) {
|
|
@@ -626,6 +564,20 @@ export class BlobManager {
|
|
|
626
564
|
this.setRedirection(storageId, storageId);
|
|
627
565
|
}
|
|
628
566
|
}
|
|
567
|
+
/**
|
|
568
|
+
* To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
|
|
569
|
+
* pending upload and/or their BlobAttach op, which can be given to a new BlobManager to
|
|
570
|
+
* resume work.
|
|
571
|
+
*
|
|
572
|
+
* @privateRemarks
|
|
573
|
+
* For now, we don't track any pending blobs since the getPendingBlobs flow doesn't enable
|
|
574
|
+
* restoring to a state where an accessible handle has been stored by the customer (and we'll
|
|
575
|
+
* just drop any BlobAttach ops on the ground during reSubmit). However, once we add support
|
|
576
|
+
* for payload-pending handles, this will return the blobs associated with those handles.
|
|
577
|
+
*/
|
|
578
|
+
getPendingBlobs() {
|
|
579
|
+
return undefined;
|
|
580
|
+
}
|
|
629
581
|
/**
|
|
630
582
|
* Part of container serialization when imminent closure is enabled (Currently when calling closeAndGetPendingLocalState).
|
|
631
583
|
* This asynchronous function resolves all pending createBlob calls and waits for each blob
|
|
@@ -637,73 +589,7 @@ export class BlobManager {
|
|
|
637
589
|
* or undefined if no blobs were processed.
|
|
638
590
|
*/
|
|
639
591
|
async attachAndGetPendingBlobs(stopBlobAttachingSignal) {
|
|
640
|
-
|
|
641
|
-
if (this.pendingBlobs.size === 0) {
|
|
642
|
-
return;
|
|
643
|
-
}
|
|
644
|
-
const blobs = {};
|
|
645
|
-
const localBlobs = new Set();
|
|
646
|
-
// This while is used to stash blobs created while attaching and getting blobs
|
|
647
|
-
while (localBlobs.size < this.pendingBlobs.size) {
|
|
648
|
-
const attachHandlesP = [];
|
|
649
|
-
for (const [localId, entry] of this.pendingBlobs) {
|
|
650
|
-
if (!localBlobs.has(entry)) {
|
|
651
|
-
localBlobs.add(entry);
|
|
652
|
-
// In order to follow natural blob creation flow we need to:
|
|
653
|
-
// 1 send the blob attach op
|
|
654
|
-
// 2 resolve the blob handle
|
|
655
|
-
// 3 wait for op referencing the blob
|
|
656
|
-
if (!entry.opsent) {
|
|
657
|
-
this.sendBlobAttachOp(localId, entry.storageId);
|
|
658
|
-
}
|
|
659
|
-
// Resolving the blob handle to let hosts continue with their operations (it will resolve
|
|
660
|
-
// original createBlob call) and let them attach the blob. This is a lie we told since the upload
|
|
661
|
-
// hasn't finished yet, but it's fine since we will retry on rehydration.
|
|
662
|
-
entry.handleP.resolve(this.getBlobHandle(localId));
|
|
663
|
-
// Array of promises that will resolve when handles get attached.
|
|
664
|
-
attachHandlesP.push(new Promise((resolve, reject) => {
|
|
665
|
-
stopBlobAttachingSignal?.addEventListener("abort", () => {
|
|
666
|
-
this.stopAttaching = true;
|
|
667
|
-
reject(new Error("Operation aborted"));
|
|
668
|
-
}, { once: true });
|
|
669
|
-
const onHandleAttached = (attachedEntry) => {
|
|
670
|
-
if (attachedEntry === entry) {
|
|
671
|
-
this.internalEvents.off("handleAttached", onHandleAttached);
|
|
672
|
-
resolve();
|
|
673
|
-
}
|
|
674
|
-
};
|
|
675
|
-
if (entry.attached) {
|
|
676
|
-
resolve();
|
|
677
|
-
}
|
|
678
|
-
else {
|
|
679
|
-
this.internalEvents.on("handleAttached", onHandleAttached);
|
|
680
|
-
}
|
|
681
|
-
}));
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
// Wait for all blobs to be attached. This is important, otherwise serialized container
|
|
685
|
-
// could send the blobAttach op without any op that references the blob, making it useless.
|
|
686
|
-
await Promise.allSettled(attachHandlesP);
|
|
687
|
-
}
|
|
688
|
-
for (const [localId, entry] of this.pendingBlobs) {
|
|
689
|
-
if (stopBlobAttachingSignal?.aborted && !entry.attached) {
|
|
690
|
-
this.mc.logger.sendTelemetryEvent({
|
|
691
|
-
eventName: "UnableToStashBlob",
|
|
692
|
-
id: localId,
|
|
693
|
-
});
|
|
694
|
-
continue;
|
|
695
|
-
}
|
|
696
|
-
assert(entry.attached === true, 0x790 /* stashed blob should be attached */);
|
|
697
|
-
blobs[localId] = {
|
|
698
|
-
blob: bufferToString(entry.blob, "base64"),
|
|
699
|
-
storageId: entry.storageId,
|
|
700
|
-
acked: entry.acked,
|
|
701
|
-
minTTLInSeconds: entry.minTTLInSeconds,
|
|
702
|
-
uploadTime: entry.uploadTime,
|
|
703
|
-
};
|
|
704
|
-
}
|
|
705
|
-
return Object.keys(blobs).length > 0 ? blobs : undefined;
|
|
706
|
-
});
|
|
592
|
+
throw new UsageError("attachAndGetPendingBlobs is no longer supported");
|
|
707
593
|
}
|
|
708
594
|
}
|
|
709
595
|
/**
|