@fluidframework/fluid-runner 2.0.0-dev-rc.1.0.0.224419

package/src/fluidRunner.ts

@@ -0,0 +1,135 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as yargs from "yargs";
+import { exportFile } from "./exportFile";
+import { IFluidFileConverter } from "./codeLoaderBundle";
+import { parseBundleAndExportFile } from "./parseBundleAndExportFile";
+// eslint-disable-next-line import/no-internal-modules
+import { validateAndParseTelemetryOptions } from "./logger/loggerUtils";
+import { validateCommandLineArgs } from "./utils";
+
+/**
+ * @param fluidFileConverter - needs to be provided if "codeLoaderBundle" is not and vice versa
+ * @internal
+ */
+export function fluidRunner(fluidFileConverter?: IFluidFileConverter) {
+	// eslint-disable-next-line @typescript-eslint/no-unused-expressions
+	yargs
+		.strict()
+		.version(false)
+		.command(
+			"exportFile",
+			"Generate an output for a local ODSP snapshot",
+			// eslint-disable-next-line @typescript-eslint/no-shadow
+			(yargs) =>
+				yargs
+					.option("codeLoader", {
+						describe:
+							'Path to code loader bundle. Required if this application is being called without modification.\nSee "README.md" for more details.',
+						type: "string",
+						demandOption: false,
+					})
+					.option("inputFile", {
+						describe: "Path to local ODSP snapshot",
+						type: "string",
+						demandOption: true,
+					})
+					.option("outputFile", {
+						describe:
+							"Path of output file (cannot already exist).\nExecution result will be written here",
+						type: "string",
+						demandOption: true,
+					})
+					.option("telemetryFile", {
+						describe:
+							"Path of telemetry file for config and session data (cannot already exist)",
+						type: "string",
+						demandOption: true,
+					})
+					.option("options", {
+						describe: "Additional options passed to container on execution",
+						type: "string",
+						demandOption: false,
+					})
+					.option("telemetryFormat", {
+						describe:
+							'Output format for telemetry. Current options are: ["JSON", "CSV"]',
+						type: "string",
+						demandOption: false,
+						default: "JSON",
+					})
+					.option("telemetryProp", {
+						describe:
+							'Property to add to every telemetry entry. Formatted like "--telemetryProp prop1 value1 --telemetryProp prop2 \\"value 2\\"".',
+						type: "array",
+						demandOption: false,
+					})
+					.option("eventsPerFlush", {
+						describe:
+							"Number of telemetry events per flush to telemetryFile (only applicable for JSON format)",
+						type: "number",
+						demandOption: false,
+					})
+					.option("timeout", {
+						describe: "Allowed timeout in ms before process is automatically cancelled",
+						type: "number",
+						demandOption: false,
+					})
+					.option("disableNetworkFetch", {
+						describe: "Should network fetch calls be explicitly disabled?",
+						type: "boolean",
+						demandOption: false,
+						default: false,
+					}),
+			// eslint-disable-next-line @typescript-eslint/no-misused-promises
+			async (argv) => {
+				const argsError = validateCommandLineArgs(argv.codeLoader, fluidFileConverter);
+				if (argsError) {
+					console.error(argsError);
+					process.exit(1);
+				}
+				const telemetryOptionsResult = validateAndParseTelemetryOptions(
+					argv.telemetryFormat,
+					argv.telemetryProp,
+					argv.eventsPerFlush,
+				);
+				if (!telemetryOptionsResult.success) {
+					console.error(telemetryOptionsResult.error);
+					process.exit(1);
+				}
+
+				const result = await (argv.codeLoader
+					? parseBundleAndExportFile(
+							argv.codeLoader,
+							argv.inputFile,
+							argv.outputFile,
+							argv.telemetryFile,
+							argv.options,
+							telemetryOptionsResult.telemetryOptions,
+							argv.timeout,
+							argv.disableNetworkFetch,
+					  )
+					: exportFile(
+							fluidFileConverter!,
+							argv.inputFile,
+							argv.outputFile,
+							argv.telemetryFile,
+							argv.options,
+							telemetryOptionsResult.telemetryOptions,
+							argv.timeout,
+							argv.disableNetworkFetch,
+					  ));
+
+				if (!result.success) {
+					console.error(`${result.eventName}: ${result.errorMessage}`);
+					process.exit(1);
+				}
+				process.exit(0);
+			},
+		)
+		.help()
+		.demandCommand().argv;
+}

package/src/index.ts

@@ -0,0 +1,18 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/* eslint-disable import/no-internal-modules */
+export { ICodeLoaderBundle, IFluidFileConverter } from "./codeLoaderBundle";
+export { createContainerAndExecute, exportFile, IExportFileResponse } from "./exportFile";
+export { fluidRunner } from "./fluidRunner";
+export { OutputFormat, ITelemetryOptions } from "./logger/fileLogger";
+export {
+	createLogger,
+	getTelemetryFileValidationError,
+	validateAndParseTelemetryOptions,
+} from "./logger/loggerUtils";
+export { parseBundleAndExportFile } from "./parseBundleAndExportFile";
+export { getSnapshotFileContent } from "./utils";
+/* eslint-enable import/no-internal-modules */

package/src/logger/baseFileLogger.ts

@@ -0,0 +1,58 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { ITelemetryBaseEvent } from "@fluidframework/core-interfaces";
+import { IFileLogger } from "./fileLogger";
+
+/**
+ * @internal
+ */
+export abstract class BaseFileLogger implements IFileLogger {
+	public supportsTags?: true | undefined;
+
+	/** Hold events in memory until flushed */
+	protected events: any[] = [];
+	protected hasWrittenToFile = false;
+
+	/**
+	 * @param filePath - file path to write logs to
+	 * @param eventsPerFlush - number of events per flush
+	 * @param defaultProps - default properties to add to every telemetry event
+	 */
+	public constructor(
+		protected readonly filePath: string,
+		protected readonly eventsPerFlush: number = 50,
+		protected readonly defaultProps?: Record<string, string | number>,
+	) {}
+
+	public send(event: ITelemetryBaseEvent): void {
+		// eslint-disable-next-line no-param-reassign
+		event = { ...event, ...this.defaultProps };
+		this.events.push(event);
+
+		if (this.events.length >= this.eventsPerFlush || event.category === "error") {
+			// eslint-disable-next-line @typescript-eslint/no-floating-promises
+			this.flush();
+		}
+	}
+
+	protected async flush(): Promise<void> {
+		if (this.events.length > 0) {
+			const contentToWrite = this.events.map((it) => JSON.stringify(it)).join(",");
+			if (this.hasWrittenToFile) {
+				fs.appendFileSync(this.filePath, `,${contentToWrite}`);
+			} else {
+				fs.appendFileSync(this.filePath, contentToWrite);
+			}
+			this.events = [];
+			this.hasWrittenToFile = true;
+		}
+	}
+
+	public async close(): Promise<void> {
+		await this.flush();
+	}
+}

package/src/logger/csvFileLogger.ts

@@ -0,0 +1,40 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { parse } from "json2csv";
+import { ITelemetryBaseEvent } from "@fluidframework/core-interfaces";
+import { BaseFileLogger } from "./baseFileLogger";
+
+/**
+ * FileLogger that writes events into a defined CSV file
+ * @internal
+ */
+export class CSVFileLogger extends BaseFileLogger {
+	/** Store the column names to write as the CSV header */
+	private readonly columns = new Set();
+
+	protected async flush(): Promise<void> {
+		// No flushing is performed since we need all log entries to determine set of CSV columns
+	}
+
+	public send(event: ITelemetryBaseEvent): void {
+		// eslint-disable-next-line guard-for-in, no-restricted-syntax
+		for (const prop in event) {
+			this.columns.add(prop);
+		}
+		super.send(event);
+	}
+
+	public async close(): Promise<void> {
+		await super.close();
+		// eslint-disable-next-line guard-for-in, no-restricted-syntax
+		for (const field in this.defaultProps) {
+			this.columns.add(field);
+		}
+
+		fs.writeFileSync(this.filePath, parse(this.events, Array.from(this.columns)));
+	}
+}

package/src/logger/fileLogger.ts

@@ -0,0 +1,51 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+
+/**
+ * Contract for logger that writes telemetry to a file
+ * @internal
+ */
+export interface IFileLogger extends ITelemetryBaseLogger {
+	/**
+	 * This method acts as a "dispose" and should be explicitly called at the end of execution
+	 */
+	close(): Promise<void>;
+}
+
+/**
+ * Desired output format for the telemetry
+ * @alpha
+ */
+export enum OutputFormat {
+	JSON,
+	CSV,
+}
+
+/* eslint-disable tsdoc/syntax */
+/**
+ * Options to provide upon creation of IFileLogger
+ * @internal
+ */
+export interface ITelemetryOptions {
+	/** Desired output format used to create a specific IFileLogger implementation */
+	outputFormat?: OutputFormat;
+
+	/**
+	 * Properties that should be added to every telemetry event
+	 *
+	 * @example
+	 *
+	 * ```JSON
+	 * { "prop1": "value1", "prop2": 10.0 }
+	 * ```
+	 */
+	defaultProps?: Record<string, string | number>;
+
+	/** Number of telemetry events per flush to telemetry file */
+	eventsPerFlush?: number;
+}
+/* eslint-enable tsdoc/syntax */

package/src/logger/jsonFileLogger.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { BaseFileLogger } from "./baseFileLogger";
+
+/**
+ * FileLogger that writes events into a defined CSV file
+ * @internal
+ */
+export class JSONFileLogger extends BaseFileLogger {
+	constructor(
+		filePath: string,
+		eventsPerFlush: number = 50,
+		defaultProps?: Record<string, string | number>,
+	) {
+		super(filePath, eventsPerFlush, defaultProps);
+		fs.appendFileSync(this.filePath, "[");
+	}
+
+	public async close(): Promise<void> {
+		await super.close();
+		fs.appendFileSync(this.filePath, "]");
+	}
+}

package/src/logger/loggerUtils.ts

@@ -0,0 +1,108 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { ITelemetryLoggerExt, createChildLogger } from "@fluidframework/telemetry-utils";
+import { CSVFileLogger } from "./csvFileLogger";
+import { IFileLogger, ITelemetryOptions, OutputFormat } from "./fileLogger";
+import { JSONFileLogger } from "./jsonFileLogger";
+
+/**
+ * Create an {@link @fluidframework/telemetry-utils#ITelemetryLoggerExt} wrapped around provided {@link IFileLogger}.
+ *
+ * @remarks
+ *
+ * It is expected that all events be sent through the returned "logger" value.
+ *
+ * The "fileLogger" value should have its "close()" method called at the end of execution.
+ *
+ * Note: if an output format is not supplied, default is JSON.
+ *
+ * @returns Both the `IFileLogger` implementation and `ITelemetryLoggerExt` wrapper to be called.
+ * @internal
+ */
+export function createLogger(
+	filePath: string,
+	options?: ITelemetryOptions,
+): { logger: ITelemetryLoggerExt; fileLogger: IFileLogger } {
+	const fileLogger =
+		options?.outputFormat === OutputFormat.CSV
+			? new CSVFileLogger(filePath, options?.eventsPerFlush, options?.defaultProps)
+			: new JSONFileLogger(filePath, options?.eventsPerFlush, options?.defaultProps);
+
+	const logger = createChildLogger({
+		logger: fileLogger,
+		namespace: "LocalSnapshotRunnerApp",
+		properties: {
+			all: { Event_Time: () => Date.now() },
+		},
+	});
+
+	return { logger, fileLogger };
+}
+
+/**
+ * Validate the telemetryFile command line argument
+ * @param telemetryFile - path where telemetry will be written
+ * @internal
+ */
+export function getTelemetryFileValidationError(telemetryFile: string): string | undefined {
+	if (!telemetryFile) {
+		return "Telemetry file argument is missing.";
+	} else if (fs.existsSync(telemetryFile)) {
+		return `Telemetry file already exists [${telemetryFile}].`;
+	}
+
+	return undefined;
+}
+
+/**
+ * Validate the provided output format and default properties
+ * @param format - desired output format of the telemetry
+ * @param props - default properties to be added to every telemetry entry
+ * @internal
+ */
+export function validateAndParseTelemetryOptions(
+	format?: string,
+	props?: (string | number)[],
+	eventsPerFlush?: number,
+): { success: false; error: string } | { success: true; telemetryOptions: ITelemetryOptions } {
+	let outputFormat: OutputFormat | undefined;
+	const defaultProps: Record<string, string | number> = {};
+
+	if (format) {
+		outputFormat = OutputFormat[format];
+		if (outputFormat === undefined) {
+			return { success: false, error: `Invalid telemetry format [${format}]` };
+		}
+	}
+
+	if (props && props.length > 0) {
+		if (props.length % 2 !== 0) {
+			return {
+				success: false,
+				error: `Invalid number of telemetry properties to add [${props.length}]`,
+			};
+		}
+		for (let i = 0; i < props.length; i += 2) {
+			if (typeof props[i] === "number") {
+				return {
+					success: false,
+					error: `Property name cannot be number at index [${i}] -> [${props[i]}]`,
+				};
+			}
+			defaultProps[props[i]] = props[i + 1];
+		}
+	}
+
+	if (eventsPerFlush !== undefined && isNaN(eventsPerFlush)) {
+		return {
+			success: false,
+			error: "Invalid eventsPerFlush",
+		};
+	}
+
+	return { success: true, telemetryOptions: { outputFormat, defaultProps, eventsPerFlush } };
+}

package/src/parseBundleAndExportFile.ts

@@ -0,0 +1,92 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { PerformanceEvent } from "@fluidframework/telemetry-utils";
+import { isCodeLoaderBundle, isFluidFileConverter } from "./codeLoaderBundle";
+import { createContainerAndExecute, IExportFileResponse } from "./exportFile";
+/* eslint-disable import/no-internal-modules */
+import { ITelemetryOptions } from "./logger/fileLogger";
+import { createLogger, getTelemetryFileValidationError } from "./logger/loggerUtils";
+/* eslint-enable import/no-internal-modules */
+import { getSnapshotFileContent, getArgsValidationError } from "./utils";
+
+const clientArgsValidationError = "Client_ArgsValidationError";
+
+/**
+ * Parse a provided JS bundle, execute code on Container based on ODSP snapshot, and write result to file
+ * @param codeLoader - path to provided JS bundle that implements ICodeLoaderBundle (see codeLoaderBundle.ts)
+ * @internal
+ */
+export async function parseBundleAndExportFile(
+	codeLoader: string,
+	inputFile: string,
+	outputFile: string,
+	telemetryFile: string,
+	options?: string,
+	telemetryOptions?: ITelemetryOptions,
+	timeout?: number,
+	disableNetworkFetch?: boolean,
+): Promise<IExportFileResponse> {
+	const telemetryArgError = getTelemetryFileValidationError(telemetryFile);
+	if (telemetryArgError) {
+		const eventName = clientArgsValidationError;
+		return { success: false, eventName, errorMessage: telemetryArgError };
+	}
+	const { fileLogger, logger } = createLogger(telemetryFile, telemetryOptions);
+
+	try {
+		return await PerformanceEvent.timedExecAsync(
+			logger,
+			{ eventName: "ParseBundleAndExportFile" },
+			async () => {
+				// eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+				const codeLoaderBundle = require(codeLoader);
+				if (!isCodeLoaderBundle(codeLoaderBundle)) {
+					const eventName = clientArgsValidationError;
+					const errorMessage = "Code loader bundle is not of type ICodeLoaderBundle";
+					logger.sendErrorEvent({ eventName, message: errorMessage });
+					return { success: false, eventName, errorMessage };
+				}
+
+				const fluidExport = await codeLoaderBundle.fluidExport;
+				if (!isFluidFileConverter(fluidExport)) {
+					const eventName = clientArgsValidationError;
+					const errorMessage =
+						"Fluid export from CodeLoaderBundle is not of type IFluidFileConverter";
+					logger.sendErrorEvent({ eventName, message: errorMessage });
+					return { success: false, eventName, errorMessage };
+				}
+
+				const argsValidationError = getArgsValidationError(inputFile, outputFile, timeout);
+				if (argsValidationError) {
+					const eventName = clientArgsValidationError;
+					logger.sendErrorEvent({ eventName, message: argsValidationError });
+					return { success: false, eventName, errorMessage: argsValidationError };
+				}
+
+				fs.writeFileSync(
+					outputFile,
+					await createContainerAndExecute(
+						getSnapshotFileContent(inputFile),
+						fluidExport,
+						logger,
+						options,
+						timeout,
+						disableNetworkFetch,
+					),
+				);
+
+				return { success: true };
+			},
+		);
+	} catch (error) {
+		const eventName = "Client_UnexpectedError";
+		logger.sendErrorEvent({ eventName }, error);
+		return { success: false, eventName, errorMessage: "Unexpected error", error };
+	} finally {
+		await fileLogger.close();
+	}
+}

package/src/utils.ts

@@ -0,0 +1,100 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { IFluidFileConverter } from "./codeLoaderBundle";
+
+/**
+ * Is the given snapshot in JSON format
+ * @param content - snapshot file content
+ * @internal
+ */
+export function isJsonSnapshot(content: Buffer): boolean {
+	return content.toString(undefined, 0, 1) === "{";
+}
+
+/**
+ * Get the ODSP snapshot file content
+ * Works on both JSON and binary snapshot formats
+ * @param filePath - path to the ODSP snapshot file
+ * @internal
+ */
+export function getSnapshotFileContent(filePath: string): string | Buffer {
+	// TODO: read file stream
+	const content = fs.readFileSync(filePath);
+	return isJsonSnapshot(content) ? content.toString() : content;
+}
+
+/**
+ * Validate provided command line arguments
+ * @internal
+ */
+export function validateCommandLineArgs(
+	codeLoader?: string,
+	fluidFileConverter?: IFluidFileConverter,
+): string | undefined {
+	if (codeLoader && fluidFileConverter !== undefined) {
+		return '"codeLoader" and "fluidFileConverter" cannot both be provided. See README for details.';
+	}
+	if (!codeLoader && fluidFileConverter === undefined) {
+		return '"codeLoader" must be provided if there is no explicit "fluidFileConverter". See README for details.';
+	}
+	return undefined;
+}
+
+/**
+ * @internal
+ */
+export function getArgsValidationError(
+	inputFile: string,
+	outputFile: string,
+	timeout?: number,
+): string | undefined {
+	// Validate input file
+	if (!inputFile) {
+		return "Input file name argument is missing.";
+	} else if (!fs.existsSync(inputFile)) {
+		return "Input file does not exist.";
+	}
+
+	// Validate output file
+	if (!outputFile) {
+		return "Output file argument is missing.";
+	} else if (fs.existsSync(outputFile)) {
+		return `Output file already exists [${outputFile}].`;
+	}
+
+	if (timeout !== undefined && (isNaN(timeout) || timeout < 0)) {
+		return "Invalid timeout";
+	}
+
+	return undefined;
+}
+
+/**
+ * @internal
+ */
+export async function timeoutPromise<T = void>(
+	executor: (
+		resolve: (value: T | PromiseLike<T>) => void,
+		reject: (reason?: any) => void,
+	) => void,
+	timeout: number,
+): Promise<T> {
+	return new Promise<T>((resolve, reject) => {
+		const timer = setTimeout(() => reject(new Error(`Timed out (${timeout}ms)`)), timeout);
+
+		executor(
+			(value) => {
+				clearTimeout(timer);
+				resolve(value);
+			},
+			(reason) => {
+				clearTimeout(timer);
+				reject(reason);
+			},
+		);
+	});
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"extends": [
+		"../../../common/build/build-common/tsconfig.base.json",
+		"../../../common/build/build-common/tsconfig.cjs.json",
+	],
+	"include": ["src/**/*"],
+	"exclude": ["src/test/**/*"],
+	"compilerOptions": {
+		"rootDir": "./src",
+		"outDir": "./dist",
+		"types": ["node"],
+	},
+}