@fluidframework/container-loader 2.0.0-internal.1.1.0 → 2.0.0-internal.1.2.0.93071

package/src/collabWindowTracker.ts

@@ -5,7 +5,7 @@
 
 import { assert, Timer } from "@fluidframework/common-utils";
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
-import { isRuntimeMessage } from "@fluidframework/driver-utils";
+import { isRuntimeMessage, MessageType2 } from "@fluidframework/driver-utils";
 
 const defaultNoopTimeFrequency = 2000;
 const defaultNoopCountFrequency = 50;
@@ -34,7 +34,7 @@ export class CollabWindowTracker {
     private readonly timer: Timer | undefined;
 
     constructor(
-        private readonly submit: (type: MessageType, contents: any) => void,
+        private readonly submit: (type: MessageType) => void,
         NoopTimeFrequency: number = defaultNoopTimeFrequency,
         private readonly NoopCountFrequency: number = defaultNoopCountFrequency,
     ) {
@@ -93,7 +93,8 @@ export class CollabWindowTracker {
 
     private submitNoop(immediate: boolean) {
         // Anything other than null is immediate noop
-        this.submit(MessageType.NoOp, immediate ? "" : null);
+        // ADO:1385: Remove cast and use MessageType once definition changes propagate
+        this.submit(immediate ? (MessageType2.Accept as unknown as MessageType) : MessageType.NoOp);
         assert(this.opsCountSinceNoop === 0,
             0x243 /* "stopSequenceNumberUpdate should be called as result of sending any op!" */);
     }

package/src/connectionManager.ts

@@ -31,7 +31,7 @@ import {
     waitForConnectedState,
     DeltaStreamConnectionForbiddenError,
     logNetworkFailure,
-    // isRuntimeMessage,
+    isRuntimeMessage,
 } from "@fluidframework/driver-utils";
 import {
     ConnectionMode,
@@ -168,7 +168,7 @@ export class ConnectionManager implements IConnectionManager {
     private clientSequenceNumber = 0;
     private clientSequenceNumberObserved = 0;
     /** Counts the number of noops sent by the client which may not be acked. */
-    private trailingNoopCount = 0;
+    private localOpsToIgnore = 0;
 
     /** track clientId used last time when we sent any ops */
     private lastSubmittedClientId: string | undefined;
@@ -247,7 +247,7 @@ export class ConnectionManager implements IConnectionManager {
 
     public shouldJoinWrite(): boolean {
         // We don't have to wait for ack for topmost NoOps. So subtract those.
-        return this.clientSequenceNumberObserved < (this.clientSequenceNumber - this.trailingNoopCount);
+        return this.clientSequenceNumberObserved < (this.clientSequenceNumber - this.localOpsToIgnore);
     }
 
     /**
@@ -819,10 +819,10 @@ export class ConnectionManager implements IConnectionManager {
             this.clientSequenceNumberObserved = 0;
         }
 
-        if (message.type === MessageType.NoOp) {
-            this.trailingNoopCount++;
+        if (!isRuntimeMessage(message)) {
+            this.localOpsToIgnore++;
         } else {
-            this.trailingNoopCount = 0;
+            this.localOpsToIgnore = 0;
         }
 
         return {

package/src/container.ts

@@ -29,10 +29,9 @@ import {
     IContainerLoadMode,
     IFluidCodeDetails,
     isFluidCodeDetails,
+    IBatchMessage,
 } from "@fluidframework/container-definitions";
 import {
-    DataCorruptionError,
-    extractSafePropertiesFromMessage,
     GenericError,
     UsageError,
 } from "@fluidframework/container-utils";
@@ -50,8 +49,6 @@ import {
     combineAppAndProtocolSummary,
     runWithRetry,
     isFluidResolvedUrl,
-    isRuntimeMessage,
-    isUnpackedRuntimeMessage,
 } from "@fluidframework/driver-utils";
 import { IQuorumSnapshot } from "@fluidframework/protocol-base";
 import {
@@ -61,7 +58,6 @@ import {
     ICommittedProposal,
     IDocumentAttributes,
     IDocumentMessage,
-    IProcessMessageResult,
     IProtocolState,
     IQuorumClients,
     IQuorumProposals,
@@ -1416,7 +1412,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         const protocol = protocolHandlerBuilder(
             attributes,
             quorumSnapshot,
-            (key, value) => this.submitMessage(MessageType.Propose, { key, value }),
+            (key, value) => this.submitMessage(MessageType.Propose, JSON.stringify({ key, value })),
             this._initialClients ?? [],
         );
 
@@ -1585,6 +1581,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         });
 
         deltaManager.on("readonly", (readonly) => {
+            this.setContextConnectedState(this.connectionState === ConnectionState.Connected, readonly);
             this.emit("readonly", readonly);
         });
 
@@ -1682,9 +1679,7 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
 
         // Both protocol and context should not be undefined if we got so far.
 
-        if (this._context?.disposed === false) {
-            this.context.setConnectionState(state, this.clientId);
-        }
+        this.setContextConnectedState(state, this._deltaManager.connectionManager.readOnlyInfo.readonly ?? false);
         this.protocolHandler.setConnectionState(state, this.clientId);
         raiseConnectedEvent(this.mc.logger, this, state, this.clientId);
 
@@ -1694,35 +1689,53 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         }
     }
 
+    // back-compat: ADO #1385: Remove in the future, summary op should come through submitSummaryMessage()
     private submitContainerMessage(type: MessageType, contents: any, batch?: boolean, metadata?: any): number {
-        const outboundMessageType: string = type;
-        switch (outboundMessageType) {
+        switch (type) {
             case MessageType.Operation:
-            case MessageType.RemoteHelp:
-                break;
-            case MessageType.Summarize: {
-                // github #6451: this is only needed for staging so the server
-                // know when the protocol tree is included
-                // this can be removed once all clients send
-                // protocol tree by default
-                const summary = contents as ISummaryContent;
-                if (summary.details === undefined) {
-                    summary.details = {};
-                }
-                summary.details.includesProtocolTree =
-                    this.options.summarizeProtocolTree === true;
-                break;
-            }
+                return this.submitMessage(
+                    type,
+                    JSON.stringify(contents),
+                    batch,
+                    metadata);
+            case MessageType.Summarize:
+                return this.submitSummaryMessage(contents as unknown as ISummaryContent);
             default:
                 this.close(new GenericError("invalidContainerSubmitOpType",
                     undefined /* error */,
                     { messageType: type }));
                 return -1;
         }
-        return this.submitMessage(type, contents, batch, metadata);
     }
 
-    private submitMessage(type: MessageType, contents: any, batch?: boolean, metadata?: any): number {
+    /** @returns clientSequenceNumber of last message in a batch */
+    private submitBatch(batch: IBatchMessage[]): number {
+        let clientSequenceNumber = -1;
+        for (const message of batch) {
+            clientSequenceNumber = this.submitMessage(
+                MessageType.Operation,
+                message.contents,
+                true, // batch
+                message.metadata);
+        }
+        this._deltaManager.flush();
+        return clientSequenceNumber;
+    }
+
+    private submitSummaryMessage(summary: ISummaryContent) {
+        // github #6451: this is only needed for staging so the server
+        // know when the protocol tree is included
+        // this can be removed once all clients send
+        // protocol tree by default
+        if (summary.details === undefined) {
+            summary.details = {};
+        }
+        summary.details.includesProtocolTree =
+            this.options.summarizeProtocolTree === true;
+        return this.submitMessage(MessageType.Summarize, JSON.stringify(summary), false /* batch */);
+    }
+
+    private submitMessage(type: MessageType, contents?: string, batch?: boolean, metadata?: any): number {
         if (this.connectionState !== ConnectionState.Connected) {
             this.mc.logger.sendErrorEvent({ eventName: "SubmitMessageWithNoConnection", type });
             return -1;
@@ -1733,28 +1746,14 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         return this._deltaManager.submit(type, contents, batch, metadata);
     }
 
-    private processRemoteMessage(message: ISequencedDocumentMessage): IProcessMessageResult {
+    private processRemoteMessage(message: ISequencedDocumentMessage) {
         const local = this.clientId === message.clientId;
 
         // Allow the protocol handler to process the message
-        let result: IProcessMessageResult = { immediateNoOp: false };
-        try {
-            result = this.protocolHandler.processMessage(message, local);
-        } catch (error) {
-            this.close(wrapError(error, (errorMessage) =>
-                new DataCorruptionError(errorMessage, extractSafePropertiesFromMessage(message))));
-        }
+        const result = this.protocolHandler.processMessage(message, local);
 
-        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
-        if (isUnpackedRuntimeMessage(message) && !isRuntimeMessage(message)) {
-            this.mc.logger.sendTelemetryEvent(
-                { eventName: "UnpackedRuntimeMessage", type: message.type });
-        }
-        // Forward non system messages to the loaded runtime for processing
-        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
-        if (isRuntimeMessage(message) || isUnpackedRuntimeMessage(message)) {
-            this.context.process(message, local, undefined);
-        }
+        // Forward messages to the loaded runtime for processing
+        this.context.process(message, local, undefined);
 
         // Inactive (not in quorum or not writers) clients don't take part in the minimum sequence number calculation.
         if (this.activeConnection()) {
@@ -1767,10 +1766,10 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
                     this.serviceConfiguration !== undefined,
                     0x2e4 /* "there should be service config for active connection" */);
                 this.collabWindowTracker = new CollabWindowTracker(
-                    (type, contents) => {
+                    (type) => {
                         assert(this.activeConnection(),
                             0x241 /* "disconnect should result in stopSequenceNumberUpdate() call" */);
-                        this.submitMessage(type, contents);
+                        this.submitMessage(type);
                     },
                     this.serviceConfiguration.noopTimeFrequency,
                     this.serviceConfiguration.noopCountFrequency,
@@ -1780,8 +1779,6 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
         }
 
         this.emit("op", message);
-
-        return result;
     }
 
     private submitSignal(message: any) {
@@ -1857,6 +1854,8 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
             new QuorumProxy(this.protocolHandler.quorum),
             loader,
             (type, contents, batch, metadata) => this.submitContainerMessage(type, contents, batch, metadata),
+            (summaryOp: ISummaryContent) => this.submitSummaryMessage(summaryOp),
+            (batch: IBatchMessage[]) => this.submitBatch(batch),
             (message) => this.submitSignal(message),
             (error?: ICriticalContainerError) => this.close(error),
             Container.version,
@@ -1879,4 +1878,22 @@ export class Container extends EventEmitterWithErrorHandling<IContainerEvents> i
     private logContainerError(warning: ContainerWarning) {
         this.mc.logger.sendErrorEvent({ eventName: "ContainerWarning" }, warning);
     }
+
+    /**
+     * Set the connected state of the ContainerContext
+     * This controls the "connected" state of the ContainerRuntime as well
+     * @param state - Is the container currently connected?
+     * @param readonly - Is the container in readonly mode?
+     */
+    private setContextConnectedState(state: boolean, readonly: boolean): void {
+        if (this._context?.disposed === false) {
+            /**
+             * We want to lie to the ContainerRuntime when we are in readonly mode to prevent issues with pending
+             * ops getting through to the DeltaManager.
+             * The ContainerRuntime's "connected" state simply means it is ok to send ops
+             * See https://dev.azure.com/fluidframework/internal/_workitems/edit/1246
+             */
+            this.context.setConnectionState(state && !readonly, this.clientId);
+        }
+    }
 }

package/src/containerContext.ts

@@ -4,7 +4,7 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert, LazyPromise } from "@fluidframework/common-utils";
+import { LazyPromise } from "@fluidframework/common-utils";
 import {
     IAudience,
     IContainerContext,
@@ -22,6 +22,7 @@ import {
     ICodeDetailsLoader,
     IFluidModuleWithDetails,
     ISnapshotTreeWithBlobContents,
+    IBatchMessage,
 } from "@fluidframework/container-definitions";
 import {
     IRequest,
@@ -42,6 +43,7 @@ import {
     ISummaryTree,
     IVersion,
     MessageType,
+    ISummaryContent,
 } from "@fluidframework/protocol-definitions";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { Container } from "./container";
@@ -59,6 +61,8 @@ export class ContainerContext implements IContainerContext {
         quorum: IQuorum,
         loader: ILoader,
         submitFn: (type: MessageType, contents: any, batch: boolean, appData: any) => number,
+        submitSummaryFn: (summaryOp: ISummaryContent) => number,
+        submitBatchFn: (batch: IBatchMessage[]) => number,
         submitSignalFn: (contents: any) => void,
         closeFn: (error?: ICriticalContainerError) => void,
         version: string,
@@ -76,6 +80,8 @@ export class ContainerContext implements IContainerContext {
             quorum,
             loader,
             submitFn,
+            submitSummaryFn,
+            submitBatchFn,
             submitSignalFn,
             closeFn,
             version,
@@ -107,8 +113,13 @@ export class ContainerContext implements IContainerContext {
         return this.container.clientDetails;
     }
 
+    private _connected: boolean;
+    /**
+     * When true, ops are free to flow
+     * When false, ops should be kept as pending or rejected
+     */
     public get connected(): boolean {
-        return this.container.connected;
+        return this._connected;
     }
 
     public get canSummarize(): boolean {
@@ -166,6 +177,9 @@ export class ContainerContext implements IContainerContext {
         quorum: IQuorum,
         public readonly loader: ILoader,
         public readonly submitFn: (type: MessageType, contents: any, batch: boolean, appData: any) => number,
+        public readonly submitSummaryFn: (summaryOp: ISummaryContent) => number,
+        /** @returns clientSequenceNumber of last message in a batch */
+        public readonly submitBatchFn: (batch: IBatchMessage[]) => number,
         public readonly submitSignalFn: (contents: any) => void,
         public readonly closeFn: (error?: ICriticalContainerError) => void,
         public readonly version: string,
@@ -174,6 +188,7 @@ export class ContainerContext implements IContainerContext {
         public readonly pendingLocalState?: unknown,
 
     ) {
+        this._connected = this.container.connected;
         this._quorum = quorum;
         this.taggedLogger = container.subLogger;
         this._fluidModuleP = new LazyPromise<IFluidModuleWithDetails>(
@@ -183,9 +198,10 @@ export class ContainerContext implements IContainerContext {
     }
 
     /**
-     * @deprecated - Temporary migratory API, to be removed when customers no longer need it.  When removed,
-     * ContainerContext should only take an IQuorumClients rather than an IQuorum.  See IContainerContext for more
-     * details.
+     * @deprecated Temporary migratory API, to be removed when customers no longer need it.
+     * When removed, `ContainerContext` should only take an {@link @fluidframework/container-definitions#IQuorumClients}
+     * rather than an {@link @fluidframework/protocol-definitions#IQuorum}.
+     * See {@link @fluidframework/container-definitions#IContainerContext} for more details.
      */
     public getSpecifiedCodeDetails(): IFluidCodeDetails | undefined {
         return (this._quorum.get("code") ?? this._quorum.get("code2")) as IFluidCodeDetails | undefined;
@@ -223,9 +239,7 @@ export class ContainerContext implements IContainerContext {
 
     public setConnectionState(connected: boolean, clientId?: string) {
         const runtime = this.runtime;
-
-        assert(connected === this.connected, 0x0de /* "Mismatch in connection state while setting" */);
-
+        this._connected = connected;
         runtime.setConnectionState(connected, clientId);
     }
 

package/src/deltaManager.ts

@@ -199,12 +199,12 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     public get readOnlyInfo() { return this.connectionManager.readOnlyInfo; }
     public get clientDetails() { return this.connectionManager.clientDetails; }
 
-    public submit(type: MessageType, contents: any, batch = false, metadata?: any) {
+    public submit(type: MessageType, contents?: string, batch = false, metadata?: any) {
         if (this.currentlyProcessingOps && this.preventConcurrentOpSend) {
             this.close(new UsageError("Making changes to data model is disallowed while processing ops."));
         }
         const messagePartial: Omit<IDocumentMessage, "clientSequenceNumber"> = {
-            contents: JSON.stringify(contents),
+            contents,
             metadata,
             referenceSequenceNumber: this.lastProcessedSequenceNumber,
             type,
@@ -218,7 +218,9 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
             return -1;
         }
 
-        this.opsSize += message.contents.length;
+        if (contents !== undefined) {
+            this.opsSize += contents.length;
+        }
 
         this.messageBuffer.push(message);
 
@@ -233,15 +235,26 @@ export class DeltaManager<TConnectionManager extends IConnectionManager>
     public submitSignal(content: any) { return this.connectionManager.submitSignal(content); }
 
     public flush() {
-        if (this.messageBuffer.length === 0) {
+        const batch = this.messageBuffer;
+        if (batch.length === 0) {
             return;
         }
 
+        this.messageBuffer = [];
+
         // The prepareFlush event allows listeners to append metadata to the batch prior to submission.
-        this.emit("prepareSend", this.messageBuffer);
+        this.emit("prepareSend", batch);
 
-        this.connectionManager.sendMessages(this.messageBuffer);
-        this.messageBuffer = [];
+        if (batch.length === 1) {
+            assert(batch[0].metadata?.batch === undefined, "no batch markup on single message");
+        } else {
+            assert(batch[0].metadata?.batch === true, "no start batch markup");
+            assert(batch[batch.length - 1].metadata?.batch === false, "no end batch markup");
+        }
+
+        this.connectionManager.sendMessages(batch);
+
+        assert(this.messageBuffer.length === 0, "reentrancy");
     }
 
     public get connectionProps(): ITelemetryProperties {

package/src/loader.ts

@@ -133,7 +133,7 @@ export interface ILoaderOptions extends ILoaderOptions1 {
 
 /**
  * @deprecated IFluidModuleWithDetails interface is moved to
- * {@link @fluidframework/container-definition#IFluidModuleWithDetails}
+ * {@link @fluidframework/container-definitions#IFluidModuleWithDetails}
  * to have all the code loading modules in one package. #8193
  * Encapsulates a module entry point with corresponding code details.
  */

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-loader";
-export const pkgVersion = "2.0.0-internal.1.1.0";
+export const pkgVersion = "2.0.0-internal.1.2.0.93071";

package/src/protocol.ts

@@ -18,6 +18,7 @@ import {
     ISignalMessage,
     MessageType,
 } from "@fluidframework/protocol-definitions";
+import { canBeCoalescedByService } from "@fluidframework/driver-utils";
 
 /**
  * Function to be used for creating a protocol handler.
@@ -68,7 +69,7 @@ export class ProtocolHandler extends ProtocolOpHandler implements IProtocolHandl
                 throw new Error("Remote message's clientId is missing from the quorum");
             }
 
-            if (client?.shouldHaveLeft === true && message.type !== MessageType.NoOp) {
+            if (client?.shouldHaveLeft === true && !canBeCoalescedByService(message)) {
                 // pre-0.58 error message: messageClientIdShouldHaveLeft
                 throw new Error("Remote message's clientId already should have left");
             }