@fluidframework/container-runtime 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.225277

package/src/summaryCollection.ts

@@ -1,406 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-import { IDisposable, IEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
-import { Deferred, assert, TypedEventEmitter } from "@fluidframework/common-utils";
-import { IDeltaManager } from "@fluidframework/container-definitions";
-import {
-    IDocumentMessage,
-    ISequencedDocumentMessage,
-    ISummaryAck,
-    ISummaryContent,
-    ISummaryNack,
-    MessageType,
-} from "@fluidframework/protocol-definitions";
-
-/**
- * Interface for summary op messages with typed contents.
- */
-export interface ISummaryOpMessage extends ISequencedDocumentMessage {
-    type: MessageType.Summarize;
-    contents: ISummaryContent;
-}
-
-/**
- * Interface for summary ack messages with typed contents.
- */
-export interface ISummaryAckMessage extends ISequencedDocumentMessage {
-    type: MessageType.SummaryAck;
-    contents: ISummaryAck;
-}
-
-/**
- * Interface for summary nack messages with typed contents.
- */
-export interface ISummaryNackMessage extends ISequencedDocumentMessage {
-    type: MessageType.SummaryNack;
-    contents: ISummaryNack;
-}
-
-/**
- * A single summary which can be tracked as it goes through its life cycle.
- * The life cycle is: Local to Broadcast to Acked/Nacked.
- */
-export interface ISummary {
-    readonly clientId: string;
-    readonly clientSequenceNumber: number;
-    waitBroadcast(): Promise<ISummaryOpMessage>;
-    waitAckNack(): Promise<ISummaryAckMessage | ISummaryNackMessage>;
-}
-
-/**
- * A single summary which has already been acked by the server.
- */
-export interface IAckedSummary {
-    readonly summaryOp: ISummaryOpMessage;
-    readonly summaryAck: ISummaryAckMessage;
-}
-
-enum SummaryState {
-    Local = 0,
-    Broadcast = 1,
-    Acked = 2,
-    Nacked = -1,
-}
-
-class Summary implements ISummary {
-    public static createLocal(clientId: string, clientSequenceNumber: number) {
-        return new Summary(clientId, clientSequenceNumber);
-    }
-    public static createFromOp(op: ISummaryOpMessage) {
-        const summary = new Summary(op.clientId, op.clientSequenceNumber);
-        summary.broadcast(op);
-        return summary;
-    }
-
-    private state = SummaryState.Local;
-
-    private _summaryOp?: ISummaryOpMessage;
-    private _summaryAckNack?: ISummaryAckMessage | ISummaryNackMessage;
-
-    private readonly defSummaryOp = new Deferred<void>();
-    private readonly defSummaryAck = new Deferred<void>();
-
-    public get summaryOp() { return this._summaryOp; }
-    public get summaryAckNack() { return this._summaryAckNack; }
-
-    private constructor(
-        public readonly clientId: string,
-        public readonly clientSequenceNumber: number) { }
-
-    public hasBeenAcked(): this is IAckedSummary {
-        return this.state === SummaryState.Acked;
-    }
-
-    public broadcast(op: ISummaryOpMessage) {
-        assert(this.state === SummaryState.Local, 0x175 /* "Can only broadcast if summarizer starts in local state" */);
-        this._summaryOp = op;
-        this.defSummaryOp.resolve();
-        this.state = SummaryState.Broadcast;
-        return true;
-    }
-
-    public ackNack(op: ISummaryAckMessage | ISummaryNackMessage) {
-        assert(this.state === SummaryState.Broadcast,
-            0x176 /* "Can only ack/nack if summarizer is in broadcasting state" */);
-        this._summaryAckNack = op;
-        this.defSummaryAck.resolve();
-        this.state = op.type === MessageType.SummaryAck ? SummaryState.Acked : SummaryState.Nacked;
-        return true;
-    }
-
-    public async waitBroadcast(): Promise<ISummaryOpMessage> {
-        await this.defSummaryOp.promise;
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        return this._summaryOp!;
-    }
-
-    public async waitAckNack(): Promise<ISummaryAckMessage | ISummaryNackMessage> {
-        await this.defSummaryAck.promise;
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        return this._summaryAckNack!;
-    }
-}
-
-/**
- * Watches summaries created by a specific client.
- */
-export interface IClientSummaryWatcher extends IDisposable {
-    watchSummary(clientSequenceNumber: number): ISummary;
-    waitFlushed(): Promise<IAckedSummary | undefined>;
-}
-
-/**
- * This class watches summaries created by a specific client.
- * It should be created and managed from a SummaryCollection.
- */
-class ClientSummaryWatcher implements IClientSummaryWatcher {
-    // key: clientSeqNum
-    private readonly localSummaries = new Map<number, Summary>();
-    private _disposed = false;
-
-    public get disposed() { return this._disposed; }
-
-    public constructor(
-        public readonly clientId: string,
-        private readonly summaryCollection: SummaryCollection,
-    ) { }
-
-    /**
-     * Watches for a specific sent summary op.
-     * @param clientSequenceNumber - client sequence number of sent summary op
-     */
-    public watchSummary(clientSequenceNumber: number): ISummary {
-        let summary = this.localSummaries.get(clientSequenceNumber);
-        if (!summary) {
-            summary = Summary.createLocal(this.clientId, clientSequenceNumber);
-            this.localSummaries.set(summary.clientSequenceNumber, summary);
-        }
-        return summary;
-    }
-
-    /**
-     * Waits until all of the pending summaries in the underlying SummaryCollection
-     * are acked/nacked.
-     */
-    // eslint-disable-next-line @typescript-eslint/promise-function-async
-    public waitFlushed() {
-        return this.summaryCollection.waitFlushed();
-    }
-
-    /**
-     * Gets a watched summary or returns undefined if not watched.
-     * @param clientSequenceNumber - client sequence number of sent summary op
-     */
-    public tryGetSummary(clientSequenceNumber: number) {
-        return this.localSummaries.get(clientSequenceNumber);
-    }
-
-    /**
-     * Starts watching a summary made by this client.
-     * @param summary - summary to start watching
-     */
-    public setSummary(summary: Summary) {
-        this.localSummaries.set(summary.clientSequenceNumber, summary);
-    }
-
-    public dispose() {
-        this.summaryCollection.removeWatcher(this.clientId);
-        this._disposed = true;
-    }
-}
-
-export type OpActionEventName = MessageType.Summarize | MessageType.SummaryAck | MessageType.SummaryNack | "default";
-export type OpActionEventListener = (op: ISequencedDocumentMessage) => void;
-export interface ISummaryCollectionOpEvents extends IEvent {
-    (event: OpActionEventName, listener: OpActionEventListener);
-}
-
-/**
- * Data structure that looks at the op stream to track summaries as they
- * are broadcast, acked and nacked.
- * It provides functionality for watching specific summaries.
- */
-export class SummaryCollection extends TypedEventEmitter<ISummaryCollectionOpEvents> {
-    // key: clientId
-    private readonly summaryWatchers = new Map<string, ClientSummaryWatcher>();
-    // key: summarySeqNum
-    private readonly pendingSummaries = new Map<number, Summary>();
-    private refreshWaitNextAck = new Deferred<void>();
-
-    private lastSummaryTimestamp: number | undefined;
-    private maxAckWaitTime: number | undefined;
-    private pendingAckTimerTimeoutCallback: (() => void) | undefined;
-    private lastAck: IAckedSummary | undefined;
-
-    public get latestAck(): IAckedSummary | undefined { return this.lastAck; }
-
-    public emit(event: OpActionEventName, ...args: Parameters<OpActionEventListener>): boolean {
-        return super.emit(event, ...args);
-    }
-
-    public get opsSinceLastAck() {
-        return this.deltaManager.lastSequenceNumber -
-            (this.lastAck?.summaryAck.sequenceNumber ?? this.deltaManager.initialSequenceNumber);
-    }
-
-    public addOpListener(listener: () => void) {
-        this.deltaManager.on("op", listener);
-    }
-
-    public removeOpListener(listener: () => void) {
-        this.deltaManager.off("op", listener);
-    }
-
-    public constructor(
-        private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
-        private readonly logger: ITelemetryLogger,
-    ) {
-        super();
-        this.deltaManager.on(
-            "op",
-            (op) => this.handleOp(op));
-    }
-
-    /**
-     * Creates and returns a summary watcher for a specific client.
-     * This will allow for local sent summaries to be better tracked.
-     * @param clientId - client id for watcher
-     */
-    public createWatcher(clientId: string): IClientSummaryWatcher {
-        const watcher = new ClientSummaryWatcher(clientId, this);
-        this.summaryWatchers.set(clientId, watcher);
-        return watcher;
-    }
-
-    public removeWatcher(clientId: string) {
-        this.summaryWatchers.delete(clientId);
-    }
-
-    public setPendingAckTimerTimeoutCallback(maxAckWaitTime: number, timeoutCallback: () => void) {
-        this.maxAckWaitTime = maxAckWaitTime;
-        this.pendingAckTimerTimeoutCallback = timeoutCallback;
-    }
-
-    public unsetPendingAckTimerTimeoutCallback() {
-        this.maxAckWaitTime = undefined;
-        this.pendingAckTimerTimeoutCallback = undefined;
-    }
-
-    /**
-     * Returns a promise that resolves once all pending summary ops
-     * have been acked or nacked.
-     */
-    public async waitFlushed(): Promise<IAckedSummary | undefined> {
-        while (this.pendingSummaries.size > 0) {
-            // eslint-disable-next-line @typescript-eslint/promise-function-async
-            const promises = Array.from(this.pendingSummaries, ([, summary]) => summary.waitAckNack());
-            await Promise.all(promises);
-        }
-        return this.lastAck;
-    }
-
-    /**
-     * Returns a promise that resolves once a summary is acked that has a reference
-     * sequence number greater than or equal to the passed in sequence number.
-     * @param referenceSequenceNumber - reference sequence number to wait for
-     * @returns The latest acked summary
-     */
-    public async waitSummaryAck(referenceSequenceNumber: number): Promise<IAckedSummary> {
-        while (!this.lastAck || this.lastAck.summaryOp.referenceSequenceNumber < referenceSequenceNumber) {
-            await this.refreshWaitNextAck.promise;
-        }
-        return this.lastAck;
-    }
-
-    /**
-     * Handler for ops; only handles ops relating to summaries.
-     * @param op - op message to handle
-     */
-    private handleOp(op: ISequencedDocumentMessage) {
-        switch (op.type) {
-            case MessageType.Summarize: {
-                this.handleSummaryOp(op as ISummaryOpMessage);
-                return;
-            }
-            case MessageType.SummaryAck: {
-                this.handleSummaryAck(op as ISummaryAckMessage);
-                return;
-            }
-            case MessageType.SummaryNack: {
-                this.handleSummaryNack(op as ISummaryNackMessage);
-                return;
-            }
-            default: {
-                // If the difference between timestamp of current op and last summary op is greater than
-                // the maxAckWaitTime, then we need to inform summarizer to not wait and summarize
-                // immediately as we have already waited for maxAckWaitTime.
-                const lastOpTimestamp = op.timestamp;
-                if (this.lastSummaryTimestamp !== undefined &&
-                    this.maxAckWaitTime !== undefined &&
-                    lastOpTimestamp - this.lastSummaryTimestamp >= this.maxAckWaitTime
-                ) {
-                    this.pendingAckTimerTimeoutCallback?.();
-                }
-                this.emit("default", op);
-
-                return;
-            }
-        }
-    }
-
-    private handleSummaryOp(op: ISummaryOpMessage) {
-        let summary: Summary | undefined;
-
-        // Check if summary already being watched, broadcast if so
-        const watcher = this.summaryWatchers.get(op.clientId);
-        if (watcher) {
-            summary = watcher.tryGetSummary(op.clientSequenceNumber);
-            if (summary) {
-                summary.broadcast(op);
-            }
-        }
-
-        // If not watched, create from op
-        if (!summary) {
-            summary = Summary.createFromOp(op);
-            if (watcher) {
-                watcher.setSummary(summary);
-            }
-        }
-        this.pendingSummaries.set(op.sequenceNumber, summary);
-        this.lastSummaryTimestamp = op.timestamp;
-        this.emit(MessageType.Summarize, op);
-    }
-
-    private handleSummaryAck(op: ISummaryAckMessage) {
-        const seq = op.contents.summaryProposal.summarySequenceNumber;
-        const summary = this.pendingSummaries.get(seq);
-        if (!summary || summary.summaryOp === undefined) {
-            // Summary ack without an op should be rare. We could fetch the
-            // reference sequence number from the snapshot, but instead we
-            // will not emit this ack. It should be the case that the summary
-            // op that this ack is for is earlier than this file was loaded
-            // from. i.e. initialSequenceNumber > summarySequenceNumber.
-            // We really don't care about it for now, since it is older than
-            // the one we loaded from.
-            if (seq >= this.deltaManager.initialSequenceNumber) {
-                // Potential causes for it to be later than our initialSequenceNumber
-                // are that the summaryOp was nacked then acked, double-acked, or
-                // the summarySequenceNumber is incorrect.
-                this.logger.sendErrorEvent({
-                    eventName: "SummaryAckWithoutOp",
-                    sequenceNumber: op.sequenceNumber, // summary ack seq #
-                    summarySequenceNumber: seq, // missing summary seq #
-                    initialSequenceNumber: this.deltaManager.initialSequenceNumber,
-                });
-            }
-            return;
-        }
-        summary.ackNack(op);
-        this.pendingSummaries.delete(seq);
-
-        // Track latest ack
-        if (!this.lastAck || seq > this.lastAck.summaryAck.contents.summaryProposal.summarySequenceNumber) {
-            this.lastAck = {
-                summaryOp: summary.summaryOp,
-                summaryAck: op,
-            };
-            this.refreshWaitNextAck.resolve();
-            this.refreshWaitNextAck = new Deferred<void>();
-            this.emit(MessageType.SummaryAck, op);
-        }
-    }
-
-    private handleSummaryNack(op: ISummaryNackMessage) {
-        const seq = op.contents.summaryProposal.summarySequenceNumber;
-        const summary = this.pendingSummaries.get(seq);
-        if (summary) {
-            summary.ackNack(op);
-            this.pendingSummaries.delete(seq);
-            this.emit(MessageType.SummaryNack, op);
-        }
-    }
-}

package/src/summaryFormat.ts

@@ -1,239 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-import { assert } from "@fluidframework/common-utils";
-import { IDocumentStorageService } from "@fluidframework/driver-definitions";
-import { readAndParse } from "@fluidframework/driver-utils";
-import { ISequencedDocumentMessage, ISnapshotTree, SummaryType } from "@fluidframework/protocol-definitions";
-import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
-import { gcTreeKey } from "./garbageCollection";
-
-type OmitAttributesVersions<T> = Omit<T, "snapshotFormatVersion" | "summaryFormatVersion">;
-interface IFluidDataStoreAttributes0 {
-    readonly snapshotFormatVersion?: undefined;
-    readonly summaryFormatVersion?: undefined;
-    pkg: string;
-    /**
-     * This tells whether a data store is root. Root data stores are never collected.
-     * Non-root data stores may be collected if they are not used. If this is not present, default it to
-     * true. This will ensure that older data stores are incorrectly collected.
-     */
-    readonly isRootDataStore?: boolean;
-}
-interface IFluidDataStoreAttributes1 extends OmitAttributesVersions<IFluidDataStoreAttributes0> {
-    readonly snapshotFormatVersion: "0.1";
-    readonly summaryFormatVersion?: undefined;
-}
-interface IFluidDataStoreAttributes2 extends OmitAttributesVersions<IFluidDataStoreAttributes1> {
-    /** Switch from snapshotFormatVersion to summaryFormatVersion */
-    readonly snapshotFormatVersion?: undefined;
-    readonly summaryFormatVersion: 2;
-    /**
-     * True if channels are not isolated in .channels subtrees, otherwise isolated.
-     * This is required in both datastore attributes as well as the root container,
-     * because reused summary handles may cause different format versions in each
-     * datastore subtree within the summary.
-     */
-    readonly disableIsolatedChannels?: true;
-}
-/**
- * Added IFluidDataStoreAttributes similar to IChannelAttributes which will tell the attributes of a
- * store like the package, snapshotFormatVersion to take different decisions based on a particular
- * snapshotFormatVersion.
- */
-export type ReadFluidDataStoreAttributes =
-    | IFluidDataStoreAttributes0
-    | IFluidDataStoreAttributes1
-    | IFluidDataStoreAttributes2;
-export type WriteFluidDataStoreAttributes = IFluidDataStoreAttributes1 | IFluidDataStoreAttributes2;
-
-export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttributes): number {
-    if (attributes.summaryFormatVersion) {
-        /**
-         * Version 2+: Introduces .channels trees for isolation of
-         * channel trees from data store objects.
-         */
-        return attributes.summaryFormatVersion;
-    } else if (attributes.snapshotFormatVersion === "0.1") {
-        /**
-         * Version 1: from this version the pkg within the data store
-         * attributes blob is a JSON array rather than a string.
-         */
-        return 1;
-    }
-    /**
-     * Version 0: format version is missing from summary.
-     * This indicates it is an older version.
-     */
-    return 0;
-}
-
-export function hasIsolatedChannels(attributes: ReadFluidDataStoreAttributes): boolean {
-    return !!attributes.summaryFormatVersion && !attributes.disableIsolatedChannels;
-}
-
-export interface IContainerRuntimeMetadata extends ICreateContainerMetadata, IGCMetadata {
-    readonly summaryFormatVersion: 1;
-    /** The last message processed at the time of summary. Only primitive property types are added to the summary. */
-    readonly message: ISummaryMetadataMessage | undefined;
-    /** True if channels are not isolated in .channels subtrees, otherwise isolated. */
-    readonly disableIsolatedChannels?: true;
-    /** The summary number for a container's summary. Incremented on summaries throughout its lifetime. */
-    readonly summaryNumber?: number;
-    /**
-     * @deprecated - User summaryNumber instead.
-     * Counter of the last summary happened, increments every time we summarize
-     */
-    readonly summaryCount?: number;
-}
-
-export interface ICreateContainerMetadata {
-    /** Runtime version of the container when it was first created */
-    createContainerRuntimeVersion?: string;
-    /** Timestamp of the container when it was first created */
-    createContainerTimestamp?: number;
-}
-
-export type GCVersion = number;
-export interface IGCMetadata {
-    /**
-     * The version of the GC code that was run to generate the GC data that is written in the summary.
-     * Also, used to determine whether GC is enabled for this container or not:
-     * - A value of 0 or undefined means GC is disabled.
-     * - A value greater than 0 means GC is enabled.
-     */
-    readonly gcFeature?: GCVersion;
-    /** If this is present, the session for this container will expire after this time and the container will close */
-    readonly sessionExpiryTimeoutMs?: number;
-    /**
-     * Tells whether the GC sweep phase is enabled for this container.
-     * - True means sweep phase is enabled.
-     * - False means sweep phase is disabled. If GC is disabled as per gcFeature, sweep is also disabled.
-     */
-    readonly sweepEnabled?: boolean;
-}
-
-/** The properties of an ISequencedDocumentMessage to be stored in the metadata blob in summary. */
-export type ISummaryMetadataMessage = Pick<ISequencedDocumentMessage,
-    | "clientId"
-    | "clientSequenceNumber"
-    | "minimumSequenceNumber"
-    | "referenceSequenceNumber"
-    | "sequenceNumber"
-    | "timestamp"
-    | "type">;
-
-/**
- * Extracts the properties from an ISequencedDocumentMessage as defined by ISummaryMetadataMessage. This message is
- * added to the metadata blob in summary.
- */
-export const extractSummaryMetadataMessage = (
-    message?: ISequencedDocumentMessage,
-): ISummaryMetadataMessage | undefined => message === undefined ? undefined : {
-    clientId: message.clientId,
-    clientSequenceNumber: message.clientSequenceNumber,
-    minimumSequenceNumber: message.minimumSequenceNumber,
-    referenceSequenceNumber: message.referenceSequenceNumber,
-    sequenceNumber: message.sequenceNumber,
-    timestamp: message.timestamp,
-    type: message.type,
-};
-
-export function getMetadataFormatVersion(metadata?: IContainerRuntimeMetadata): number {
-    /**
-     * Version 2+: Introduces runtime sequence number for data verification.
-     *
-     * Version 1+: Introduces .metadata blob and .channels trees for isolation of
-     * data store trees from container-level objects.
-     * Also introduces enableGC option stored in the summary.
-     *
-     * Version 0: metadata blob missing; format version is missing from summary.
-     * This indicates it is an older version.
-     */
-    return metadata?.summaryFormatVersion ?? 0;
-}
-
-export const aliasBlobName = ".aliases";
-export const metadataBlobName = ".metadata";
-export const chunksBlobName = ".chunks";
-export const electedSummarizerBlobName = ".electedSummarizer";
-export const blobsTreeName = ".blobs";
-
-export function rootHasIsolatedChannels(metadata?: IContainerRuntimeMetadata): boolean {
-    return !!metadata && !metadata.disableIsolatedChannels;
-}
-
-export function getGCVersion(metadata?: IGCMetadata): GCVersion {
-    if (!metadata) {
-        // Force to 0/disallowed in prior versions
-        return 0;
-    }
-    return metadata.gcFeature ?? 0;
-}
-
-export const protocolTreeName = ".protocol";
-
-/**
- * List of tree IDs at the container level which are reserved.
- * This is for older versions of summaries that do not yet have an
- * isolated data stores namespace. Without the namespace, this must
- * be used to prevent name collisions with data store IDs.
- */
-export const nonDataStorePaths = [protocolTreeName, ".logTail", ".serviceProtocol", blobsTreeName, gcTreeKey];
-
-export const dataStoreAttributesBlobName = ".component";
-
-/**
- * Modifies summary tree and stats to put tree under .channels tree.
- *
- * @param summarizeResult - Summary tree and stats to modify
- *
- * @example
- * Converts from:
- * ```typescript
- * {
- *     type: SummaryType.Tree,
- *     tree: { a: {...}, b: {...}, c: {...} },
- * }
- * ```
- *
- * to:
- *
- * ```typescript
- * {
- *     type: SummaryType.Tree,
- *     tree: {
- *         ".channels": {
- *             type: SummaryType.Tree,
- *             tree: { a: {...}, b: {...}, c: {...} }
- *         },
- *     },
- * }
- * ```
- * And adds +1 to treeNodeCount in stats.
- */
-export function wrapSummaryInChannelsTree(summarizeResult: ISummaryTreeWithStats): void {
-    summarizeResult.summary = {
-        type: SummaryType.Tree,
-        tree: { [channelsTreeName]: summarizeResult.summary },
-    };
-    summarizeResult.stats.treeNodeCount++;
-}
-
-export async function getFluidDataStoreAttributes(
-    storage: IDocumentStorageService,
-    snapshot: ISnapshotTree,
-): Promise<ReadFluidDataStoreAttributes> {
-    const attributes = await readAndParse<ReadFluidDataStoreAttributes>(
-        storage, snapshot.blobs[dataStoreAttributesBlobName]);
-    // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot.
-    // For snapshotFormatVersion = "0.1" (1) or above, pkg is jsonified, otherwise it is just a string.
-    // However the feature of loading a detached container from snapshot, is added when the
-    // snapshotFormatVersion is at least "0.1" (1), so we don't expect it to be anything else.
-    const formatVersion = getAttributesFormatVersion(attributes);
-    assert(formatVersion > 0,
-        0x1d5 /* Invalid snapshot format version */);
-    return attributes;
-}