@fluidframework/fluid-runner 2.0.0-dev.1.3.0.96595

package/package.json

@@ -0,0 +1,87 @@
+{
+  "name": "@fluidframework/fluid-runner",
+  "version": "2.0.0-dev.1.3.0.96595",
+  "description": "Utility for running various functionality inside a Fluid Framework environment",
+  "homepage": "https://fluidframework.com",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/FluidFramework.git",
+    "directory": "packages/tools/fluid-runner"
+  },
+  "license": "MIT",
+  "author": "Microsoft and contributors",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "fluid-runner": "bin/fluid-runner"
+  },
+  "scripts": {
+    "build": "concurrently npm:build:compile npm:lint",
+    "build:commonjs": "npm run tsc && npm run build:test",
+    "build:compile": "concurrently npm:build:commonjs npm:build:esnext",
+    "build:esnext": "tsc --project ./tsconfig.esnext.json",
+    "build:full": "npm run build",
+    "build:full:compile": "npm run build:compile",
+    "build:test": "tsc --project ./src/test/tsconfig.json",
+    "clean": "rimraf dist lib *.tsbuildinfo *.build.log",
+    "eslint": "eslint --format stylish src",
+    "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
+    "lint": "npm run eslint",
+    "lint:fix": "npm run eslint:fix",
+    "test": "npm run test:mocha",
+    "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
+    "test:mocha": "mocha --ignore 'dist/test/types/*' --recursive dist/test -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
+    "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
+    "tsc": "tsc",
+    "tsfmt": "tsfmt --verify",
+    "tsfmt:fix": "tsfmt --replace"
+  },
+  "nyc": {
+    "all": true,
+    "cache-dir": "nyc/.cache",
+    "exclude": [
+      "src/test/**/*.ts",
+      "dist/test/**/*.js"
+    ],
+    "exclude-after-remap": false,
+    "include": [
+      "src/**/*.ts",
+      "dist/**/*.js"
+    ],
+    "report-dir": "nyc/report",
+    "reporter": [
+      "cobertura",
+      "html",
+      "text"
+    ],
+    "temp-directory": "nyc/.nyc_output"
+  },
+  "dependencies": {
+    "@fluidframework/aqueduct": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/common-definitions": "^0.20.1",
+    "@fluidframework/container-definitions": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/container-loader": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/core-interfaces": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/driver-definitions": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/odsp-driver": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/odsp-driver-definitions": "2.0.0-dev.1.3.0.96595",
+    "@fluidframework/telemetry-utils": "2.0.0-dev.1.3.0.96595",
+    "json2csv": "^5.0.7",
+    "yargs": "13.2.2"
+  },
+  "devDependencies": {
+    "@fluidframework/build-common": "^1.0.0",
+    "@fluidframework/eslint-config-fluid": "^1.0.0",
+    "@fluidframework/mocha-test-setup": "2.0.0-dev.1.3.0.96595",
+    "@rushstack/eslint-config": "^2.5.1",
+    "@types/mocha": "^9.1.1",
+    "@types/node": "^14.18.0",
+    "concurrently": "^6.2.0",
+    "cross-env": "^7.0.2",
+    "eslint": "~8.6.0",
+    "mocha": "^10.0.0",
+    "nyc": "^15.0.0",
+    "rimraf": "^2.6.2",
+    "typescript": "~4.5.5"
+  }
+}

package/src/codeLoaderBundle.ts

@@ -0,0 +1,57 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryBaseLogger } from "@fluidframework/common-definitions";
+import { ICodeDetailsLoader, IContainer } from "@fluidframework/container-definitions";
+import { FluidObject } from "@fluidframework/core-interfaces";
+
+/**
+ * Contract that defines the necessary exports for the bundle provided at runtime
+ * For an example, see "src/test/sampleCodeLoaders/sampleCodeLoader.ts"
+ */
+export interface ICodeLoaderBundle {
+    /**
+     * Fluid export of all the required objects and functions
+     */
+    fluidExport: Promise<IFluidFileConverter>;
+}
+
+/**
+ * Instance that holds all the details for Fluid file conversion
+ */
+export interface IFluidFileConverter {
+    /**
+     * Get code loader details to provide at Loader creation
+     * @param logger - created logger object to pass to code loader
+     */
+    getCodeLoader(logger: ITelemetryBaseLogger): Promise<ICodeDetailsLoader>;
+
+    /**
+     * Scope object to provide at Loader creation
+     */
+    scope?: FluidObject;
+
+    /**
+     * Executes code on container and returns the result
+     * @param container - container created by this application
+     * @param options - additional options
+     */
+    execute(container: IContainer, options?: string): Promise<string>;
+}
+
+/**
+ * Type cast to ensure necessary methods are present in the provided bundle
+ * @param bundle - bundle provided to this application
+ */
+export function isCodeLoaderBundle(bundle: any): bundle is ICodeLoaderBundle {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return bundle?.fluidExport && typeof bundle.fluidExport === "object";
+}
+
+export function isFluidFileConverter(obj: any): obj is IFluidFileConverter {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return obj?.getCodeLoader && typeof obj.getCodeLoader === "function"
+        && obj.execute && typeof obj.execute === "function";
+}

package/src/exportFile.ts

@@ -0,0 +1,104 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { LoaderHeader } from "@fluidframework/container-definitions";
+import { Loader } from "@fluidframework/container-loader";
+import { createLocalOdspDocumentServiceFactory } from "@fluidframework/odsp-driver";
+import { PerformanceEvent } from "@fluidframework/telemetry-utils";
+import { getArgsValidationError } from "./getArgsValidationError";
+import { IFluidFileConverter } from "./codeLoaderBundle";
+import { FakeUrlResolver } from "./fakeUrlResolver";
+import { getSnapshotFileContent } from "./utils";
+/* eslint-disable import/no-internal-modules */
+import { ITelemetryOptions } from "./logger/fileLogger";
+import { createLogger, getTelemetryFileValidationError } from "./logger/loggerUtils";
+/* eslint-enable import/no-internal-modules */
+
+export type IExportFileResponse = IExportFileResponseSuccess | IExportFileResponseFailure;
+
+interface IExportFileResponseSuccess {
+    success: true;
+}
+
+interface IExportFileResponseFailure {
+    success: false;
+    eventName: string;
+    errorMessage: string;
+    error?: any;
+}
+
+const clientArgsValidationError = "Client_ArgsValidationError";
+
+/**
+ * Execute code on Container based on ODSP snapshot and write result to file
+ */
+export async function exportFile(
+    fluidFileConverter: IFluidFileConverter,
+    inputFile: string,
+    outputFile: string,
+    telemetryFile: string,
+    options?: string,
+    telemetryOptions?: ITelemetryOptions,
+): Promise<IExportFileResponse> {
+    const telemetryArgError = getTelemetryFileValidationError(telemetryFile);
+    if (telemetryArgError) {
+        const eventName = clientArgsValidationError;
+        return { success: false, eventName, errorMessage: telemetryArgError };
+    }
+    const { fileLogger, logger } = createLogger(telemetryFile, telemetryOptions);
+
+    try {
+        return await PerformanceEvent.timedExecAsync(logger, { eventName: "ExportFile" }, async () => {
+            const argsValidationError = getArgsValidationError(inputFile, outputFile);
+            if (argsValidationError) {
+                const eventName = clientArgsValidationError;
+                logger.sendErrorEvent({ eventName, message: argsValidationError });
+                return { success: false, eventName, errorMessage: argsValidationError };
+            }
+
+            fs.writeFileSync(outputFile, await createContainerAndExecute(
+                getSnapshotFileContent(inputFile),
+                fluidFileConverter,
+                logger,
+                options,
+            ));
+
+            return { success: true };
+        });
+    } catch (error) {
+        const eventName = "Client_UnexpectedError";
+        logger.sendErrorEvent({ eventName }, error);
+        return { success: false, eventName, errorMessage: "Unexpected error", error };
+    } finally {
+        await fileLogger.close();
+    }
+}
+
+/**
+ * Create the container based on an ODSP snapshot and execute code on it
+ * @returns result of execution
+ */
+export async function createContainerAndExecute(
+    localOdspSnapshot: string | Uint8Array,
+    fluidFileConverter: IFluidFileConverter,
+    logger: ITelemetryLogger,
+    options?: string,
+): Promise<string> {
+    const loader = new Loader({
+        urlResolver: new FakeUrlResolver(),
+        documentServiceFactory: createLocalOdspDocumentServiceFactory(localOdspSnapshot),
+        codeLoader: await fluidFileConverter.getCodeLoader(logger),
+        scope: fluidFileConverter.scope,
+        logger,
+    });
+
+    const container = await loader.resolve({ url: "/fakeUrl/", headers: {
+        [LoaderHeader.loadMode]: { opsBeforeReturn: "cached" } } });
+
+    return PerformanceEvent.timedExecAsync(logger, { eventName: "ExportFile" }, async () =>
+        fluidFileConverter.execute(container, options));
+}

package/src/fakeUrlResolver.ts

@@ -0,0 +1,50 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { IRequest } from "@fluidframework/core-interfaces";
+import { IContainerPackageInfo, IResolvedUrl, IUrlResolver } from "@fluidframework/driver-definitions";
+import { IOdspResolvedUrl } from "@fluidframework/odsp-driver-definitions";
+
+const fakeId = "FakeUrlResolver";
+const fakeUrl = "/FakeUrlResolver/";
+
+/**
+ * Fake URL resolver that returns hard coded values on every request
+ * @internal
+ */
+export class FakeUrlResolver implements IUrlResolver {
+    public async resolve(_request: IRequest): Promise<IResolvedUrl | undefined> {
+        const fakeOdspResolvedUrl: IOdspResolvedUrl = {
+            type: "fluid",
+            odspResolvedUrl: true,
+            id: fakeId,
+            siteUrl: fakeUrl,
+            driveId: fakeId,
+            itemId: fakeId,
+            url: fakeUrl,
+            hashedDocumentId: fakeId,
+            endpoints: {
+                snapshotStorageUrl: fakeUrl,
+                attachmentPOSTStorageUrl: fakeUrl,
+                attachmentGETStorageUrl: fakeUrl,
+                deltaStorageUrl: fakeUrl,
+            },
+            tokens: {},
+            fileName: fakeId,
+            summarizer: false,
+            fileVersion: fakeId,
+        };
+
+        return fakeOdspResolvedUrl;
+    }
+
+    public async getAbsoluteUrl(
+        _resolvedUrl: IResolvedUrl,
+        _relativeUrl: string,
+        _packageInfoSource?: IContainerPackageInfo,
+    ): Promise<string> {
+        return "";
+    }
+}

package/src/fluidRunner.ts

@@ -0,0 +1,106 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/* eslint-disable max-len */
+import * as yargs from "yargs";
+import { exportFile } from "./exportFile";
+import { IFluidFileConverter } from "./codeLoaderBundle";
+import { parseBundleAndExportFile } from "./parseBundleAndExportFile";
+// eslint-disable-next-line import/no-internal-modules
+import { validateAndParseTelemetryOptions } from "./logger/loggerUtils";
+import { validateCommandLineArgs } from "./utils";
+
+/**
+ * @param fluidFileConverter - needs to be provided if "codeLoaderBundle" is not and vice versa
+ */
+export function fluidRunner(fluidFileConverter?: IFluidFileConverter) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+    yargs
+        .strict()
+        .version(false)
+        .command(
+            "exportFile",
+            "Generate an output for a local ODSP snapshot",
+            // eslint-disable-next-line @typescript-eslint/no-shadow
+            (yargs) =>
+                yargs
+                    .option("codeLoader", {
+                        describe: "Path to code loader bundle. Required if this application is being called without modification.\nSee \"README.md\" for more details.",
+                        type: "string",
+                        demandOption: false,
+                    })
+                    .option("inputFile", {
+                        describe: "Path to local ODSP snapshot",
+                        type: "string",
+                        demandOption: true,
+                    })
+                    .option("outputFile", {
+                        describe: "Path of output file (cannot already exist).\nExecution result will be written here",
+                        type: "string",
+                        demandOption: true,
+                    })
+                    .option("telemetryFile", {
+                        describe: "Path of telemetry file for config and session data (cannot already exist)",
+                        type: "string",
+                        demandOption: true,
+                    })
+                    .option("options", {
+                        describe: "Additional options passed to container on execution",
+                        type: "string",
+                        demandOption: false,
+                    })
+                    .option("telemetryFormat", {
+                        describe: "Output format for telemetry. Current options are: [\"JSON\", \"CSV\"]",
+                        type: "string",
+                        demandOption: false,
+                        default: "JSON",
+                    })
+                    .option("telemetryProp", {
+                        describe: "Property to add to every telemetry entry. Formatted like \"--telemetryProp prop1 value1 --telemetryProp prop2 \\\"value 2\\\"\".",
+                        type: "array",
+                        demandOption: false,
+                    }),
+            // eslint-disable-next-line @typescript-eslint/no-misused-promises
+            async (argv) => {
+                const argsError = validateCommandLineArgs(argv.codeLoader, fluidFileConverter);
+                if (argsError) {
+                    console.error(argsError);
+                    process.exit(1);
+                }
+                const telemetryOptionsResult = validateAndParseTelemetryOptions(argv.telemetryFormat, argv.telemetryProp);
+                if (!telemetryOptionsResult.success) {
+                    console.error(telemetryOptionsResult.error);
+                    process.exit(1);
+                }
+
+                const result = await (argv.codeLoader
+                    ? parseBundleAndExportFile(
+                        argv.codeLoader,
+                        argv.inputFile,
+                        argv.outputFile,
+                        argv.telemetryFile,
+                        argv.options,
+                        telemetryOptionsResult.telemetryOptions,
+                    ) : exportFile(
+                        fluidFileConverter!,
+                        argv.inputFile,
+                        argv.outputFile,
+                        argv.telemetryFile,
+                        argv.options,
+                        telemetryOptionsResult.telemetryOptions,
+                    ));
+
+                if (!result.success) {
+                    console.error(`${result.eventName}: ${result.errorMessage}`);
+                    process.exit(1);
+                }
+            },
+        )
+        .help()
+        .demandCommand().argv;
+}
+
+fluidRunner();
+/* eslint-enable max-len */

package/src/getArgsValidationError.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+
+export function getArgsValidationError(
+    inputFile: string,
+    outputFile: string,
+): string | undefined {
+    // Validate input file
+    if (!inputFile) {
+        return "Input file name argument is missing.";
+    } else if (!fs.existsSync(inputFile)) {
+        return "Input file does not exist.";
+    }
+
+    // Validate output file
+    if (!outputFile) {
+        return "Output file argument is missing.";
+    } else if (fs.existsSync(outputFile)) {
+        return `Output file already exists [${outputFile}].`;
+    }
+
+    return undefined;
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/* eslint-disable import/no-internal-modules */
+export { ICodeLoaderBundle, IFluidFileConverter } from "./codeLoaderBundle";
+export * from "./exportFile";
+export { fluidRunner } from "./fluidRunner";
+export { OutputFormat } from "./logger/fileLogger";
+export { createLogger, getTelemetryFileValidationError } from "./logger/loggerUtils";
+export * from "./parseBundleAndExportFile";
+export { getSnapshotFileContent } from "./utils";
+/* eslint-enable import/no-internal-modules */

package/src/logger/baseFileLogger.ts

@@ -0,0 +1,58 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { ITelemetryBaseEvent } from "@fluidframework/common-definitions";
+import { IFileLogger } from "./fileLogger";
+
+/**
+ * @internal
+ */
+export abstract class BaseFileLogger implements IFileLogger {
+    public supportsTags?: true | undefined;
+
+    /** Hold events in memory until flushed */
+    protected events: any[] = [];
+    protected hasWrittenToFile = false;
+
+    /**
+     * @param filePath - file path to write logs to
+     * @param eventsPerFlush - number of events per flush
+     * @param defaultProps - default properties to add to every telemetry event
+     */
+     public constructor(
+        protected readonly filePath: string,
+        protected readonly eventsPerFlush: number = 50,
+        protected readonly defaultProps?: Record<string, string | number>,
+    ) { }
+
+    public send(event: ITelemetryBaseEvent): void {
+        // eslint-disable-next-line no-param-reassign
+        event = { ...event, ...this.defaultProps };
+        this.events.push(event);
+
+        if (this.events.length >= this.eventsPerFlush || event.category === "error") {
+            // eslint-disable-next-line @typescript-eslint/no-floating-promises
+            this.flush();
+        }
+    }
+
+    protected async flush(): Promise<void> {
+        if (this.events.length > 0) {
+            const contentToWrite = this.events.map((it) => JSON.stringify(it)).join(",");
+            if (this.hasWrittenToFile) {
+                fs.appendFileSync(this.filePath, `,${contentToWrite}`);
+            } else {
+                fs.appendFileSync(this.filePath, contentToWrite);
+            }
+            this.events = [];
+            this.hasWrittenToFile = true;
+        }
+    }
+
+    public async close(): Promise<void> {
+        await this.flush();
+    }
+}

package/src/logger/csvFileLogger.ts

@@ -0,0 +1,40 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { parse } from "json2csv";
+import { ITelemetryBaseEvent } from "@fluidframework/common-definitions";
+import { BaseFileLogger } from "./baseFileLogger";
+
+/**
+ * FileLogger that writes events into a defined CSV file
+ * @internal
+ */
+export class CSVFileLogger extends BaseFileLogger {
+    /** Store the column names to write as the CSV header */
+    private readonly columns = new Set();
+
+    public async flush(): Promise<void> {
+        // Do nothing
+    }
+
+    public send(event: ITelemetryBaseEvent): void {
+        // eslint-disable-next-line guard-for-in, no-restricted-syntax
+        for (const prop in event) {
+            this.columns.add(prop);
+        }
+        super.send(event);
+    }
+
+    public async close(): Promise<void> {
+        await super.close();
+        // eslint-disable-next-line guard-for-in, no-restricted-syntax
+        for (const field in this.defaultProps) {
+            this.columns.add(field);
+        }
+
+        fs.writeFileSync(this.filePath, parse(this.events, Array.from(this.columns)));
+    }
+}

package/src/logger/fileLogger.ts

@@ -0,0 +1,45 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryBaseLogger } from "@fluidframework/common-definitions";
+
+/**
+ * Contract for logger that writes telemetry to a file
+ * @internal
+ */
+export interface IFileLogger extends ITelemetryBaseLogger {
+    /**
+     * This method acts as a "dispose" and should be explicitly called at the end of execution
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * Desired output format for the telemetry
+ */
+export enum OutputFormat {
+    JSON,
+    CSV,
+}
+
+/* eslint-disable tsdoc/syntax */
+/**
+ * Options to provide upon creation of IFileLogger
+ * @internal
+ */
+export interface ITelemetryOptions {
+    /** Desired output format used to create a specific IFileLogger implementation */
+    outputFormat?: OutputFormat;
+
+    /**
+     * Properties that should be added to every telemetry event
+     * Example: { "prop1": "value1", "prop2": 10.0 }
+     */
+    defaultProps?: Record<string, string | number>;
+
+    /** Number of telemetry events per flush to telemetry file */
+    eventsPerFlush?: number;
+}
+/* eslint-enable tsdoc/syntax */

package/src/logger/jsonFileLogger.ts

@@ -0,0 +1,27 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import * as fs from "fs";
+import { BaseFileLogger } from "./baseFileLogger";
+
+/**
+ * FileLogger that writes events into a defined CSV file
+ * @internal
+ */
+export class JSONFileLogger extends BaseFileLogger {
+    constructor(
+        filePath: string,
+        eventsPerFlush: number = 50,
+        defaultProps?: Record<string, string | number>,
+    ) {
+        super(filePath, eventsPerFlush, defaultProps);
+        fs.appendFileSync(this.filePath, "[");
+    }
+
+    public async close(): Promise<void> {
+        await super.close();
+        fs.appendFileSync(this.filePath, "]");
+    }
+}