@fluid-experimental/attributor 2.0.0-internal.3.3.0

package/src/encoders.ts

@@ -0,0 +1,112 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { assert } from "@fluidframework/common-utils";
+import { Jsonable } from "@fluidframework/datastore-definitions";
+import { IUser } from "@fluidframework/protocol-definitions";
+import { AttributionInfo } from "@fluidframework/runtime-definitions";
+import { IAttributor } from "./attributor";
+import { InternedStringId, MutableStringInterner } from "./stringInterner";
+
+export interface Encoder<TDecoded, TEncoded> {
+	encode(decoded: TDecoded): TEncoded;
+
+	decode(encoded: TEncoded): TDecoded;
+}
+
+// Note: the encoded format doesn't matter as long as it's serializable;
+// these types could be weakened.
+export type TimestampEncoder = Encoder<number[], number[]>;
+
+export const deltaEncoder: TimestampEncoder = {
+	encode: (timestamps: number[]) => {
+		const deltaTimestamps: number[] = new Array(timestamps.length);
+		let prev = 0;
+		for (let i = 0; i < timestamps.length; i++) {
+			deltaTimestamps[i] = timestamps[i] - prev;
+			prev = timestamps[i];
+		}
+		return deltaTimestamps;
+	},
+	decode: (encoded: Jsonable) => {
+		assert(
+			Array.isArray(encoded),
+			0x4b0 /* Encoded timestamps should be an array of nummbers */,
+		);
+		const timestamps: number[] = new Array(encoded.length);
+		let cumulativeSum = 0;
+		for (let i = 0; i < encoded.length; i++) {
+			cumulativeSum += encoded[i];
+			timestamps[i] = cumulativeSum;
+		}
+		return timestamps;
+	},
+};
+
+export type IAttributorSerializer = Encoder<IAttributor, SerializedAttributor>;
+
+export interface SerializedAttributor {
+	interner: readonly string[] /* result of calling getSerializable() on a StringInterner */;
+	seqs: number[];
+	timestamps: number[];
+	attributionRefs: InternedStringId[];
+}
+
+export class AttributorSerializer implements IAttributorSerializer {
+	constructor(
+		private readonly makeAttributor: (
+			entries: Iterable<[number, AttributionInfo]>,
+		) => IAttributor,
+		private readonly timestampEncoder: TimestampEncoder,
+	) {}
+
+	public encode(attributor: IAttributor): SerializedAttributor {
+		const interner = new MutableStringInterner();
+		const seqs: number[] = [];
+		const timestamps: number[] = [];
+		const attributionRefs: InternedStringId[] = [];
+		for (const [seq, { user, timestamp }] of attributor.entries()) {
+			seqs.push(seq);
+			timestamps.push(timestamp);
+			const ref = interner.getOrCreateInternedId(JSON.stringify(user));
+			attributionRefs.push(ref);
+		}
+
+		const serialized: SerializedAttributor = {
+			interner: interner.getSerializable(),
+			seqs,
+			timestamps: this.timestampEncoder.encode(timestamps),
+			attributionRefs,
+		};
+
+		return serialized;
+	}
+
+	public decode(encoded: SerializedAttributor): IAttributor {
+		const interner = new MutableStringInterner(encoded.interner);
+		const { seqs, timestamps: encodedTimestamps, attributionRefs } = encoded;
+		const timestamps = this.timestampEncoder.decode(encodedTimestamps);
+		assert(
+			seqs.length === timestamps.length && timestamps.length === attributionRefs.length,
+			0x4b1 /* serialized attribution columns should have the same length */,
+		);
+		const entries = new Array(seqs.length);
+		for (let i = 0; i < seqs.length; i++) {
+			const key = seqs[i];
+			const timestamp = timestamps[i];
+			const ref = attributionRefs[i];
+			const user: IUser = JSON.parse(interner.getString(ref));
+			entries[i] = [key, { user, timestamp }];
+		}
+		return this.makeAttributor(entries);
+	}
+}
+
+/**
+ * @returns an encoder which composes `a` and `b`.
+ */
+export const chain = <T1, T2, T3>(a: Encoder<T1, T2>, b: Encoder<T2, T3>): Encoder<T1, T3> => ({
+	encode: (content) => b.encode(a.encode(content)),
+	decode: (content) => a.decode(b.decode(content)),
+});

package/src/index.ts

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+export { Attributor, OpStreamAttributor, IAttributor } from "./attributor";
+export {
+	createRuntimeAttributor,
+	enableOnNewFileKey,
+	IProvideRuntimeAttributor,
+	IRuntimeAttributor,
+	mixinAttributor,
+} from "./mixinAttributor";

package/src/lz4Encoder.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { compress, decompress } from "lz4js";
+import { bufferToString, stringToBuffer } from "@fluidframework/common-utils";
+import { Jsonable } from "@fluidframework/datastore-definitions";
+import { Encoder } from "./encoders";
+
+/**
+ * @alpha
+ */
+export function makeLZ4Encoder<T>(): Encoder<Jsonable<T>, string> {
+	return {
+		encode: (decoded: Jsonable<T>) => {
+			const uncompressed = new TextEncoder().encode(JSON.stringify(decoded));
+			const compressed = compress(uncompressed);
+			return bufferToString(compressed, "base64");
+		},
+		decode: (serializedSummary: string): Jsonable<T> => {
+			const compressed = new Uint8Array(stringToBuffer(serializedSummary, "base64"));
+			const uncompressed = decompress(compressed);
+			const decoded: Jsonable<T> = JSON.parse(new TextDecoder().decode(uncompressed));
+			return decoded;
+		},
+	};
+}

package/src/mixinAttributor.ts

@@ -0,0 +1,283 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import {
+	IDocumentMessage,
+	ISequencedDocumentMessage,
+	ISnapshotTree,
+} from "@fluidframework/protocol-definitions";
+import { IAudience, IContainerContext, IDeltaManager } from "@fluidframework/container-definitions";
+import { ContainerRuntime } from "@fluidframework/container-runtime";
+import type { IContainerRuntimeOptions } from "@fluidframework/container-runtime";
+import {
+	AttributionInfo,
+	AttributionKey,
+	ISummaryTreeWithStats,
+	ITelemetryContext,
+	NamedFluidDataStoreRegistryEntries,
+} from "@fluidframework/runtime-definitions";
+import { addSummarizeResultToSummary, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
+import { IContainerRuntime } from "@fluidframework/container-runtime-definitions";
+import { IRequest, IResponse, FluidObject } from "@fluidframework/core-interfaces";
+import { assert, bufferToString, unreachableCase } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
+import { loggerToMonitoringContext } from "@fluidframework/telemetry-utils";
+import { Attributor, IAttributor, OpStreamAttributor } from "./attributor";
+import { AttributorSerializer, chain, deltaEncoder, Encoder } from "./encoders";
+import { makeLZ4Encoder } from "./lz4Encoder";
+
+// Summary tree keys
+const attributorTreeName = ".attributor";
+const opBlobName = "op";
+
+/**
+ * @alpha
+ * Feature Gate Key -
+ * Whether or not a container runtime instantiated using `mixinAttributor`'s load should generate an attributor on
+ * new files. See package README for more notes on integration.
+ */
+export const enableOnNewFileKey = "Fluid.Attribution.EnableOnNewFile";
+
+/**
+ * @alpha
+ */
+export const IRuntimeAttributor: keyof IProvideRuntimeAttributor = "IRuntimeAttributor";
+
+/**
+ * @alpha
+ */
+export interface IProvideRuntimeAttributor {
+	readonly IRuntimeAttributor: IRuntimeAttributor;
+}
+
+/**
+ * Provides access to attribution information stored on the container runtime.
+ *
+ * Attributors are only populated after the container runtime they are injected into has initialized.
+ * @sealed
+ * @alpha
+ */
+export interface IRuntimeAttributor extends IProvideRuntimeAttributor {
+	/**
+	 * @throws - If no AttributionInfo exists for this key.
+	 */
+	get(key: AttributionKey): AttributionInfo;
+
+	/**
+	 * @returns - Whether any AttributionInfo exists for the provided key.
+	 */
+	has(key: AttributionKey): boolean;
+
+	/**
+	 * @returns - Whether the runtime is currently tracking attribution information for the loaded container.
+	 * See {@link mixinAttributor} for more details on when this happens.
+	 */
+	readonly isEnabled: boolean;
+}
+
+/**
+ * @returns an IRuntimeAttributor for usage with `mixinAttributor`. The attributor will only be populated with data
+ * once it's passed via scope to a container runtime load flow. See {@link mixinAttributor}.
+ * @alpha
+ */
+export function createRuntimeAttributor(): IRuntimeAttributor {
+	return new RuntimeAttributor();
+}
+
+/**
+ * Mixes in logic to load and store runtime-based attribution functionality.
+ *
+ * The `scope` passed to `load` should implement `IProvideRuntimeAttributor`.
+ *
+ * Existing documents without stored attributors will not start storing attribution information: if an
+ * IRuntimeAttributor is passed via scope to load a document that never previously had attribution information,
+ * that attributor's `has` method will always return `false`.
+ * @param Base - base class, inherits from FluidAttributorRuntime
+ * @alpha
+ */
+export const mixinAttributor = (Base: typeof ContainerRuntime = ContainerRuntime) =>
+	class ContainerRuntimeWithAttributor extends Base {
+		public static async load(
+			context: IContainerContext,
+			registryEntries: NamedFluidDataStoreRegistryEntries,
+			requestHandler?:
+				| ((request: IRequest, runtime: IContainerRuntime) => Promise<IResponse>)
+				| undefined,
+			runtimeOptions: IContainerRuntimeOptions | undefined = {},
+			containerScope: FluidObject | undefined = context.scope,
+			existing?: boolean | undefined,
+			ctor: typeof ContainerRuntime = ContainerRuntimeWithAttributor as unknown as typeof ContainerRuntime,
+		): Promise<ContainerRuntime> {
+			const runtimeAttributor = (
+				containerScope as FluidObject<IProvideRuntimeAttributor> | undefined
+			)?.IRuntimeAttributor;
+			if (!runtimeAttributor) {
+				throw new UsageError(
+					"ContainerRuntimeWithAttributor must be passed a scope implementing IProvideRuntimeAttributor",
+				);
+			}
+
+			const pendingRuntimeState = context.pendingLocalState as {
+				baseSnapshot?: ISnapshotTree;
+			};
+			const baseSnapshot: ISnapshotTree | undefined =
+				pendingRuntimeState?.baseSnapshot ?? context.baseSnapshot;
+
+			const { audience, deltaManager } = context;
+			assert(
+				audience !== undefined,
+				0x508 /* Audience must exist when instantiating attribution-providing runtime */,
+			);
+
+			const mc = loggerToMonitoringContext(context.taggedLogger);
+			const shouldTrackAttribution = mc.config.getBoolean(enableOnNewFileKey) ?? false;
+			if (shouldTrackAttribution) {
+				(context.options.attribution ??= {}).track = true;
+			}
+
+			const runtime = (await Base.load(
+				context,
+				registryEntries,
+				requestHandler,
+				runtimeOptions,
+				containerScope,
+				existing,
+				ctor,
+			)) as ContainerRuntimeWithAttributor;
+			runtime.runtimeAttributor = runtimeAttributor as RuntimeAttributor;
+
+			// Note: this fetches attribution blobs relatively eagerly in the load flow; we may want to optimize
+			// this to avoid blocking on such information until application actually requests some op-based attribution
+			// info or we need to summarize. All that really needs to happen immediately is to start recording
+			// op seq# -> attributionInfo for new ops.
+			await runtime.runtimeAttributor.initialize(
+				deltaManager,
+				audience,
+				baseSnapshot,
+				async (id) => runtime.storage.readBlob(id),
+				shouldTrackAttribution,
+			);
+			return runtime;
+		}
+
+		private runtimeAttributor: RuntimeAttributor | undefined;
+
+		protected addContainerStateToSummary(
+			summaryTree: ISummaryTreeWithStats,
+			fullTree: boolean,
+			trackState: boolean,
+			telemetryContext?: ITelemetryContext,
+		) {
+			super.addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext);
+			const attributorSummary = this.runtimeAttributor?.summarize();
+			if (attributorSummary) {
+				addSummarizeResultToSummary(summaryTree, attributorTreeName, attributorSummary);
+			}
+		}
+	} as unknown as typeof ContainerRuntime;
+
+class RuntimeAttributor implements IRuntimeAttributor {
+	public get IRuntimeAttributor(): IRuntimeAttributor {
+		return this;
+	}
+
+	public get(key: AttributionKey): AttributionInfo {
+		assert(
+			this.opAttributor !== undefined,
+			0x509 /* RuntimeAttributor must be initialized before getAttributionInfo can be called */,
+		);
+
+		if (key.type === "detached") {
+			throw new Error("Attribution of detached keys is not yet supported.");
+		}
+
+		if (key.type === "local") {
+			// Note: we can *almost* orchestrate this correctly with internal-only changes by looking up the current
+			// client id in the audience. However, for read->write client transition, the container might have not yet
+			// received a client id. This is left as a TODO as it might be more easily solved once the detached case
+			// is settled (e.g. if it's reasonable for the host to know the current user information at container
+			// creation time, we could just use that here as well).
+			throw new Error("Attribution of local keys is not yet supported.");
+		}
+
+		return this.opAttributor.getAttributionInfo(key.seq);
+	}
+
+	public has(key: AttributionKey): boolean {
+		if (key.type === "detached") {
+			return false;
+		}
+
+		if (key.type === "local") {
+			return false;
+		}
+
+		return this.opAttributor?.tryGetAttributionInfo(key.seq) !== undefined;
+	}
+
+	private encoder: Encoder<IAttributor, string> = {
+		encode: unreachableCase,
+		decode: unreachableCase,
+	};
+
+	private opAttributor: IAttributor | undefined;
+	public isEnabled = false;
+
+	public async initialize(
+		deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
+		audience: IAudience,
+		baseSnapshot: ISnapshotTree | undefined,
+		readBlob: (id: string) => Promise<ArrayBufferLike>,
+		shouldAddAttributorOnNewFile: boolean,
+	): Promise<void> {
+		const attributorTree = baseSnapshot?.trees[attributorTreeName];
+		// Existing documents that don't already have a snapshot containing runtime attribution info shouldn't
+		// inject any for now--this causes some back-compat integration problems that aren't fully worked out.
+		const shouldExcludeAttributor =
+			(baseSnapshot !== undefined && attributorTree === undefined) ||
+			(baseSnapshot === undefined && !shouldAddAttributorOnNewFile);
+		if (shouldExcludeAttributor) {
+			// This gives a consistent error for calls to `get` on keys that don't exist.
+			this.opAttributor = new Attributor();
+			return;
+		}
+
+		this.isEnabled = true;
+		this.encoder = chain(
+			new AttributorSerializer(
+				(entries) => new OpStreamAttributor(deltaManager, audience, entries),
+				deltaEncoder,
+			),
+			makeLZ4Encoder(),
+		);
+
+		if (attributorTree !== undefined) {
+			const id = attributorTree.blobs[opBlobName];
+			assert(
+				id !== undefined,
+				0x50a /* Attributor tree should have op attributor summary blob. */,
+			);
+			const blobContents = await readBlob(id);
+			const attributorSnapshot = bufferToString(blobContents, "utf8");
+			this.opAttributor = this.encoder.decode(attributorSnapshot);
+		} else {
+			this.opAttributor = new OpStreamAttributor(deltaManager, audience);
+		}
+	}
+
+	public summarize(): ISummaryTreeWithStats | undefined {
+		if (!this.isEnabled) {
+			// Loaded existing document without attributor data: avoid injecting any data.
+			return undefined;
+		}
+
+		assert(
+			this.opAttributor !== undefined,
+			0x50b /* RuntimeAttributor should be initialized before summarization */,
+		);
+		const builder = new SummaryTreeBuilder();
+		builder.addBlob(opBlobName, this.encoder.encode(this.opAttributor));
+		return builder.getSummaryTree();
+	}
+}

package/src/stringInterner.ts

@@ -0,0 +1,86 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { UsageError } from "@fluidframework/container-utils";
+
+/**
+ * The ID of the string that has been interned, which can be used by a {@link StringInterner} to retrieve the
+ * original string.
+ * @public
+ */
+export type InternedStringId = number & {
+	readonly InternedStringId: "e221abc9-9d17-4493-8db0-70c871a1c27c";
+};
+
+/**
+ * Interns strings as integers.
+ */
+export interface StringInterner {
+	getInternedId(input: string): InternedStringId | undefined;
+	getString(internedId: number): string;
+	getSerializable(): readonly string[];
+}
+
+/**
+ * Interns strings as integers.
+ * Given a string, this class will produce a unique integer associated with that string that can then be used to
+ * retrieve the string.
+ */
+export class MutableStringInterner implements StringInterner {
+	private readonly stringToInternedIdMap = new Map<string, InternedStringId>();
+	private readonly internedStrings: string[] = [];
+
+	/**
+	 * @param inputStrings - A list of strings to intern in the order given. Can be used to rehydrate from a previous
+	 * `StringInterner`'s {@link StringInterner.getSerializable} return value.
+	 */
+	constructor(inputStrings: readonly string[] = []) {
+		for (const value of inputStrings) {
+			this.getOrCreateInternedId(value);
+		}
+	}
+
+	/**
+	 * @param input - The string to get the associated intern ID for
+	 * @returns an intern ID that is uniquely associated with the input string
+	 */
+	public getOrCreateInternedId(input: string): InternedStringId {
+		return this.getInternedId(input) ?? this.createNewId(input);
+	}
+
+	public getInternedId(input: string): InternedStringId | undefined {
+		return this.stringToInternedIdMap.get(input);
+	}
+
+	/**
+	 *
+	 * @param internId - The intern ID to get the associated string for. Can only retrieve strings that have been
+	 * used as inputs to calls of `getInternId`.
+	 * @returns a string that is uniquely associated with the given intern ID
+	 */
+	public getString(internId: number): string {
+		const result = this.internedStrings[internId];
+		if (!result) {
+			throw new UsageError(`No string associated with ${internId}.`);
+		}
+		return result;
+	}
+
+	/**
+	 * @returns The list of strings interned where the indices map to the associated {@link InternedStringId} of
+	 * each string.
+	 */
+	public getSerializable(): readonly string[] {
+		return this.internedStrings;
+	}
+
+	/** Create a new interned id. Assumes without validation that the input doesn't already have an interned id. */
+	private createNewId(input: string): InternedStringId {
+		const internedId = this.stringToInternedIdMap.size as InternedStringId;
+		this.stringToInternedIdMap.set(input, internedId);
+		this.internedStrings.push(input);
+		return internedId;
+	}
+}

package/tsconfig.esnext.json

@@ -0,0 +1,7 @@
+{
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"outDir": "./lib",
+		"module": "esnext",
+	},
+}

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+	"extends": "@fluidframework/build-common/ts-common-config.json",
+	"exclude": ["src/test/**/*"],
+	"compilerOptions": {
+		"rootDir": "./src",
+		"outDir": "./dist",
+		"composite": true,
+	},
+	"include": ["src/**/*"],
+}