@fluidframework/container-loader 2.0.0-internal.1.1.2 → 2.0.0-internal.1.2.0

package/src/containerContext.ts

@@ -4,7 +4,7 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert, LazyPromise } from "@fluidframework/common-utils";
+import { LazyPromise } from "@fluidframework/common-utils";
 import {
     IAudience,
     IContainerContext,
@@ -22,6 +22,7 @@ import {
     ICodeDetailsLoader,
     IFluidModuleWithDetails,
     ISnapshotTreeWithBlobContents,
+    IBatchMessage,
 } from "@fluidframework/container-definitions";
 import {
     IRequest,
@@ -42,6 +43,7 @@ import {
     ISummaryTree,
     IVersion,
     MessageType,
+    ISummaryContent,
 } from "@fluidframework/protocol-definitions";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { Container } from "./container";
@@ -59,6 +61,8 @@ export class ContainerContext implements IContainerContext {
         quorum: IQuorum,
         loader: ILoader,
         submitFn: (type: MessageType, contents: any, batch: boolean, appData: any) => number,
+        submitSummaryFn: (summaryOp: ISummaryContent) => number,
+        submitBatchFn: (batch: IBatchMessage[]) => number,
         submitSignalFn: (contents: any) => void,
         closeFn: (error?: ICriticalContainerError) => void,
         version: string,
@@ -76,6 +80,8 @@ export class ContainerContext implements IContainerContext {
             quorum,
             loader,
             submitFn,
+            submitSummaryFn,
+            submitBatchFn,
             submitSignalFn,
             closeFn,
             version,
@@ -107,8 +113,13 @@ export class ContainerContext implements IContainerContext {
         return this.container.clientDetails;
     }
 
+    private _connected: boolean;
+    /**
+     * When true, ops are free to flow
+     * When false, ops should be kept as pending or rejected
+     */
     public get connected(): boolean {
-        return this.container.connected;
+        return this._connected;
     }
 
     public get canSummarize(): boolean {
@@ -166,6 +177,9 @@ export class ContainerContext implements IContainerContext {
         quorum: IQuorum,
         public readonly loader: ILoader,
         public readonly submitFn: (type: MessageType, contents: any, batch: boolean, appData: any) => number,
+        public readonly submitSummaryFn: (summaryOp: ISummaryContent) => number,
+        /** @returns clientSequenceNumber of last message in a batch */
+        public readonly submitBatchFn: (batch: IBatchMessage[]) => number,
         public readonly submitSignalFn: (contents: any) => void,
         public readonly closeFn: (error?: ICriticalContainerError) => void,
         public readonly version: string,
@@ -174,6 +188,7 @@ export class ContainerContext implements IContainerContext {
         public readonly pendingLocalState?: unknown,
 
     ) {
+        this._connected = this.container.connected;
         this._quorum = quorum;
         this.taggedLogger = container.subLogger;
         this._fluidModuleP = new LazyPromise<IFluidModuleWithDetails>(
@@ -183,9 +198,10 @@ export class ContainerContext implements IContainerContext {
     }
 
     /**
-     * @deprecated - Temporary migratory API, to be removed when customers no longer need it.  When removed,
-     * ContainerContext should only take an IQuorumClients rather than an IQuorum.  See IContainerContext for more
-     * details.
+     * @deprecated Temporary migratory API, to be removed when customers no longer need it.
+     * When removed, `ContainerContext` should only take an {@link @fluidframework/container-definitions#IQuorumClients}
+     * rather than an {@link @fluidframework/protocol-definitions#IQuorum}.
+     * See {@link @fluidframework/container-definitions#IContainerContext} for more details.
      */
     public getSpecifiedCodeDetails(): IFluidCodeDetails | undefined {
         return (this._quorum.get("code") ?? this._quorum.get("code2")) as IFluidCodeDetails | undefined;
@@ -223,9 +239,7 @@ export class ContainerContext implements IContainerContext {
 
     public setConnectionState(connected: boolean, clientId?: string) {
         const runtime = this.runtime;
-
-        assert(connected === this.connected, 0x0de /* "Mismatch in connection state while setting" */);
-
+        this._connected = connected;
         runtime.setConnectionState(connected, clientId);
     }
 

package/src/containerStorageAdapter.ts

@@ -3,14 +3,17 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
+import { assert } from "@fluidframework/common-utils";
 import { ISnapshotTreeWithBlobContents } from "@fluidframework/container-definitions";
 import {
     FetchSource,
+    IDocumentService,
     IDocumentStorageService,
     IDocumentStorageServicePolicies,
     ISummaryContext,
 } from "@fluidframework/driver-definitions";
+import { UsageError } from "@fluidframework/driver-utils";
 import {
     ICreateBlobResponse,
     ISnapshotTree,
@@ -19,14 +22,52 @@ import {
     IVersion,
 } from "@fluidframework/protocol-definitions";
 import { IDetachedBlobStorage } from "./loader";
+import { ProtocolTreeStorageService } from "./protocolTreeDocumentStorageService";
+import { RetriableDocumentStorageService } from "./retriableDocumentStorageService";
 
 /**
  * This class wraps the actual storage and make sure no wrong apis are called according to
  * container attach state.
  */
-export class ContainerStorageAdapter implements IDocumentStorageService {
+export class ContainerStorageAdapter implements IDocumentStorageService, IDisposable {
     private readonly blobContents: { [id: string]: ArrayBufferLike; } = {};
-    constructor(private readonly storageGetter: () => IDocumentStorageService) {}
+    private _storageService: IDocumentStorageService & Partial<IDisposable>;
+
+    constructor(
+        detachedBlobStorage: IDetachedBlobStorage | undefined,
+        private readonly logger: ITelemetryLogger,
+        private readonly captureProtocolSummary?: () => ISummaryTree,
+    ) {
+        this._storageService = new BlobOnlyStorage(detachedBlobStorage, logger);
+    }
+
+    disposed: boolean = false;
+    dispose(error?: Error): void {
+        this._storageService?.dispose?.(error);
+        this.disposed = true;
+    }
+
+    public async connectToService(service: IDocumentService): Promise<void> {
+        if (!(this._storageService instanceof BlobOnlyStorage)) {
+            return;
+        }
+
+        const storageService = await service.connectToStorage();
+        const retriableStorage = this._storageService =
+            new RetriableDocumentStorageService(
+                storageService,
+                this.logger);
+
+        if (this.captureProtocolSummary !== undefined) {
+            this.logger.sendTelemetryEvent({ eventName: "summarizeProtocolTreeEnabled" });
+            this._storageService =
+                new ProtocolTreeStorageService(retriableStorage, this.captureProtocolSummary);
+        }
+
+        // ensure we did not lose that policy in the process of wrapping
+        assert(storageService.policies?.minBlobSize === this._storageService.policies?.minBlobSize,
+            0x0e0 /* "lost minBlobSize policy" */);
+    }
 
     public loadSnapshotForRehydratingContainer(snapshotTree: ISnapshotTreeWithBlobContents) {
         this.getBlobContents(snapshotTree);
@@ -45,17 +86,17 @@ export class ContainerStorageAdapter implements IDocumentStorageService {
         // back-compat 0.40 containerRuntime requests policies even in detached container if storage is present
         // and storage is always present in >=0.41.
         try {
-            return this.storageGetter().policies;
+            return this._storageService.policies;
         } catch (e) {}
         return undefined;
     }
 
     public get repositoryUrl(): string {
-        return this.storageGetter().repositoryUrl;
+        return this._storageService.repositoryUrl;
     }
 
     public async getSnapshotTree(version?: IVersion, scenarioName?: string): Promise<ISnapshotTree | null> {
-        return this.storageGetter().getSnapshotTree(version, scenarioName);
+        return this._storageService.getSnapshotTree(version, scenarioName);
     }
 
     public async readBlob(id: string): Promise<ArrayBufferLike> {
@@ -63,7 +104,7 @@ export class ContainerStorageAdapter implements IDocumentStorageService {
         if (blob !== undefined) {
             return blob;
         }
-        return this.storageGetter().readBlob(id);
+        return this._storageService.readBlob(id);
     }
 
     public async getVersions(
@@ -72,19 +113,19 @@ export class ContainerStorageAdapter implements IDocumentStorageService {
         scenarioName?: string,
         fetchSource?: FetchSource,
     ): Promise<IVersion[]> {
-        return this.storageGetter().getVersions(versionId, count, scenarioName, fetchSource);
+        return this._storageService.getVersions(versionId, count, scenarioName, fetchSource);
     }
 
     public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
-        return this.storageGetter().uploadSummaryWithContext(summary, context);
+        return this._storageService.uploadSummaryWithContext(summary, context);
     }
 
     public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
-        return this.storageGetter().downloadSummary(handle);
+        return this._storageService.downloadSummary(handle);
     }
 
     public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
-        return this.storageGetter().createBlob(file);
+        return this._storageService.createBlob(file);
     }
 }
 
@@ -92,18 +133,25 @@ export class ContainerStorageAdapter implements IDocumentStorageService {
  * Storage which only supports createBlob() and readBlob(). This is used with IDetachedBlobStorage to support
  * blobs in detached containers.
  */
-export class BlobOnlyStorage implements IDocumentStorageService {
+class BlobOnlyStorage implements IDocumentStorageService {
     constructor(
-        private readonly blobStorage: IDetachedBlobStorage,
+        private readonly detachedStorage: IDetachedBlobStorage | undefined,
         private readonly logger: ITelemetryLogger,
     ) { }
 
     public async createBlob(content: ArrayBufferLike): Promise<ICreateBlobResponse> {
-        return this.blobStorage.createBlob(content);
+        return this.verifyStorage().createBlob(content);
     }
 
     public async readBlob(blobId: string): Promise<ArrayBufferLike> {
-        return this.blobStorage.readBlob(blobId);
+        return this.verifyStorage().readBlob(blobId);
+    }
+
+    private verifyStorage(): IDetachedBlobStorage {
+        if (this.detachedStorage === undefined) {
+            throw new UsageError("Real storage calls not allowed in Unattached container");
+        }
+        return this.detachedStorage;
     }
 
     public get policies(): IDocumentStorageServicePolicies | undefined {
@@ -123,6 +171,7 @@ export class BlobOnlyStorage implements IDocumentStorageService {
     /* eslint-enable @typescript-eslint/unbound-method */
 
     private notCalled(): never {
+        this.verifyStorage();
         try {
             // some browsers may not populate stack unless exception is thrown
             throw new Error("BlobOnlyStorage not implemented method used");

package/src/deltaManager.ts

@@ -199,12 +199,12 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     public get readOnlyInfo() { return this.connectionManager.readOnlyInfo; }
     public get clientDetails() { return this.connectionManager.clientDetails; }
 
-    public submit(type: MessageType, contents: any, batch = false, metadata?: any) {
+    public submit(type: MessageType, contents?: string, batch = false, metadata?: any) {
         if (this.currentlyProcessingOps && this.preventConcurrentOpSend) {
             this.close(new UsageError("Making changes to data model is disallowed while processing ops."));
         }
         const messagePartial: Omit<IDocumentMessage, "clientSequenceNumber"> = {
-            contents: JSON.stringify(contents),
+            contents,
             metadata,
             referenceSequenceNumber: this.lastProcessedSequenceNumber,
             type,
@@ -218,7 +218,9 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
             return -1;
         }
 
-        this.opsSize += message.contents.length;
+        if (contents !== undefined) {
+            this.opsSize += contents.length;
+        }
 
         this.messageBuffer.push(message);
 
@@ -233,15 +235,26 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     public submitSignal(content: any) { return this.connectionManager.submitSignal(content); }
 
     public flush() {
-        if (this.messageBuffer.length === 0) {
+        const batch = this.messageBuffer;
+        if (batch.length === 0) {
             return;
         }
 
+        this.messageBuffer = [];
+
         // The prepareFlush event allows listeners to append metadata to the batch prior to submission.
-        this.emit("prepareSend", this.messageBuffer);
+        this.emit("prepareSend", batch);
 
-        this.connectionManager.sendMessages(this.messageBuffer);
-        this.messageBuffer = [];
+        if (batch.length === 1) {
+            assert(batch[0].metadata?.batch === undefined, 0x3c9 /* no batch markup on single message */);
+        } else {
+            assert(batch[0].metadata?.batch === true, 0x3ca /* no start batch markup */);
+            assert(batch[batch.length - 1].metadata?.batch === false, 0x3cb /* no end batch markup */);
+        }
+
+        this.connectionManager.sendMessages(batch);
+
+        assert(this.messageBuffer.length === 0, 0x3cc /* reentrancy */);
     }
 
     public get connectionProps(): ITelemetryProperties {

package/src/loader.ts

@@ -133,7 +133,7 @@ export interface ILoaderOptions extends ILoaderOptions1 {
 
 /**
  * @deprecated IFluidModuleWithDetails interface is moved to
- * {@link @fluidframework/container-definition#IFluidModuleWithDetails}
+ * {@link @fluidframework/container-definitions#IFluidModuleWithDetails}
  * to have all the code loading modules in one package. #8193
  * Encapsulates a module entry point with corresponding code details.
  */

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-loader";
-export const pkgVersion = "2.0.0-internal.1.1.2";
+export const pkgVersion = "2.0.0-internal.1.2.0";

package/src/protocol.ts

@@ -18,6 +18,7 @@ import {
     ISignalMessage,
     MessageType,
 } from "@fluidframework/protocol-definitions";
+import { canBeCoalescedByService } from "@fluidframework/driver-utils";
 
 /**
  * Function to be used for creating a protocol handler.
@@ -68,7 +69,7 @@ export class ProtocolHandler extends ProtocolOpHandler implements IProtocolHandl
                 throw new Error("Remote message's clientId is missing from the quorum");
             }
 
-            if (client?.shouldHaveLeft === true && message.type !== MessageType.NoOp) {
+            if (client?.shouldHaveLeft === true && !canBeCoalescedByService(message)) {
                 // pre-0.58 error message: messageClientIdShouldHaveLeft
                 throw new Error("Remote message's clientId already should have left");
             }