@eagleoutice/flowr 1.5.2 → 2.0.0

package/benchmark/stopwatch.d.ts

@@ -0,0 +1,35 @@
+export interface IStoppableStopwatch {
+    /** Stop the given stopwatch. */
+    stop(): void;
+}
+/**
+ * Allows to measure keys of type `T` with a `Stopwatch`.
+ *
+ * Measure with {@link start}, {@link measure} or {@link measureAsync}, retrieve the final measurements with {@link get}.
+ */
+export declare class Measurements<T> {
+    private measurements;
+    /**
+   * Start a timer for the given key, and guards that this is the first time this key is started.
+   * Call {@link IStoppableStopwatch#stop} on the returned stopwatch to stop the timer.
+   */
+    start(key: T): IStoppableStopwatch;
+    /**
+   * Automatically call {@link Measurements#start | start} and the corresponding stop to measure the execution time of the given function.
+   * @see {@link measureAsync}
+   */
+    measure<Out>(key: T, fn: () => Out): Out;
+    /**
+   * Similar to {@link measure}, but await the promise as part of the measurement
+   *
+     * @param key - The key to write the resulting measurement to
+     * @param fn  - The function to measure
+     *
+   * @see measure
+   */
+    measureAsync<Out>(key: T, fn: () => Promise<Out>): Promise<Out>;
+    /**
+   * Retrieve all measure-results, requires that all stop-watches that have been started have also been stopped.
+   */
+    get(): Map<T, bigint>;
+}

package/benchmark/stopwatch.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Measurements = void 0;
+const assert_1 = require("../util/assert");
+/** unguarded start-stop wrapper */
+class Stopwatch {
+    timeStart;
+    timeEnd;
+    stopped = false;
+    start() {
+        this.timeStart = process.hrtime.bigint();
+    }
+    stop() {
+        this.timeEnd = process.hrtime.bigint();
+        // check after to not affect measurements
+        (0, assert_1.guard)(!this.stopped, 'cannot stop a stopwatch twice');
+        this.stopped = true;
+    }
+    get() {
+        (0, assert_1.guard)(this.timeStart !== undefined && this.timeEnd !== undefined, 'cannot get elapsed time as the stopwatch has not been started/stopped');
+        return this.timeEnd - this.timeStart;
+    }
+}
+/**
+ * Allows to measure keys of type `T` with a `Stopwatch`.
+ *
+ * Measure with {@link start}, {@link measure} or {@link measureAsync}, retrieve the final measurements with {@link get}.
+ */
+class Measurements {
+    measurements = new Map();
+    /**
+   * Start a timer for the given key, and guards that this is the first time this key is started.
+   * Call {@link IStoppableStopwatch#stop} on the returned stopwatch to stop the timer.
+   */
+    start(key) {
+        // we guard *before* starting so there is no additional time penalty
+        (0, assert_1.guard)(!this.measurements.has(key), `already started stop watch for ${JSON.stringify(key)}`);
+        const stopwatch = new Stopwatch();
+        this.measurements.set(key, stopwatch);
+        stopwatch.start();
+        return stopwatch;
+    }
+    /**
+   * Automatically call {@link Measurements#start | start} and the corresponding stop to measure the execution time of the given function.
+   * @see {@link measureAsync}
+   */
+    measure(key, fn) {
+        const stopwatch = this.start(key);
+        const result = fn();
+        stopwatch.stop();
+        return result;
+    }
+    /**
+   * Similar to {@link measure}, but await the promise as part of the measurement
+   *
+     * @param key - The key to write the resulting measurement to
+     * @param fn  - The function to measure
+     *
+   * @see measure
+   */
+    async measureAsync(key, fn) {
+        const stopwatch = this.start(key);
+        const result = await fn();
+        stopwatch.stop();
+        return result;
+    }
+    /**
+   * Retrieve all measure-results, requires that all stop-watches that have been started have also been stopped.
+   */
+    get() {
+        const result = new Map();
+        for (const [key, stopwatch] of this.measurements) {
+            result.set(key, stopwatch.get());
+        }
+        return result;
+    }
+}
+exports.Measurements = Measurements;
+//# sourceMappingURL=stopwatch.js.map

package/benchmark/summarizer/data.d.ts

@@ -0,0 +1,54 @@
+import type { SummarizedMeasurement } from '../../util/summarizer';
+import type { CommonSlicerMeasurements, PerSliceMeasurements, SlicerStats, SlicerStatsDataflow, SlicerStatsInput } from '../stats/stats';
+export interface SliceSizeCollection {
+    lines: number[];
+    characters: number[];
+    nonWhitespaceCharacters: number[];
+    /** like library statements during reconstruction */
+    autoSelected: number[];
+    dataflowNodes: number[];
+    tokens: number[];
+    normalizedTokens: number[];
+}
+/**
+ * @see SlicerStats
+ * @see summarizeSlicerStats
+ */
+export type SummarizedSlicerStats = {
+    perSliceMeasurements: SummarizedPerSliceStats;
+} & Omit<SlicerStats, 'perSliceMeasurements'>;
+export interface Reduction<T = number> {
+    numberOfLines: T;
+    numberOfLinesNoAutoSelection: T;
+    numberOfCharacters: T;
+    numberOfNonWhitespaceCharacters: T;
+    numberOfRTokens: T;
+    numberOfNormalizedTokens: T;
+    numberOfDataflowNodes: T;
+}
+export interface SummarizedPerSliceStats {
+    /** number of total slicing calls */
+    numberOfSlices: number;
+    /** statistics on the used slicing criteria (number of ids within criteria etc.) */
+    sliceCriteriaSizes: SummarizedMeasurement;
+    measurements: Map<PerSliceMeasurements, SummarizedMeasurement>;
+    reduction: Reduction<SummarizedMeasurement>;
+    failedToRepParse: number;
+    timesHitThreshold: number;
+    sliceSize: {
+        [K in keyof SliceSizeCollection]: SummarizedMeasurement;
+    };
+}
+export interface UltimateSlicerStats {
+    totalRequests: number;
+    totalSlices: number;
+    commonMeasurements: Map<CommonSlicerMeasurements, SummarizedMeasurement>;
+    perSliceMeasurements: Map<PerSliceMeasurements, SummarizedMeasurement>;
+    /** sum */
+    failedToRepParse: number;
+    /** sum */
+    timesHitThreshold: number;
+    reduction: Reduction<SummarizedMeasurement>;
+    input: SlicerStatsInput<SummarizedMeasurement>;
+    dataflow: SlicerStatsDataflow<SummarizedMeasurement>;
+}

package/benchmark/summarizer/first-phase/input.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="node" />
+export declare function processRunMeasurement(line: Buffer, fileNum: number, lineNum: number, summarizedText: string, outputPath: string): Promise<void>;
+export declare function processSummarizedFileMeasurement(file: string, summariesFile: string, outputPath: string): void;

package/benchmark/summarizer/first-phase/input.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processSummarizedFileMeasurement = exports.processRunMeasurement = void 0;
+const fs_1 = __importDefault(require("fs"));
+const process_1 = require("../second-phase/process");
+const process_2 = require("./process");
+const assert_1 = require("../../../util/assert");
+const ansi_1 = require("../../../util/ansi");
+const json_1 = require("../../../util/json");
+const files_1 = require("../../../util/files");
+const print_1 = require("../../stats/print");
+async function processRunMeasurement(line, fileNum, lineNum, summarizedText, outputPath) {
+    let got = JSON.parse(line.toString());
+    console.log(`[file ${fileNum}, line ${lineNum}] Summarize for ${got.filename}`);
+    // now we have to recover the maps and bigints :C
+    got = {
+        filename: got.filename,
+        'file-id': got['file-id'],
+        'run-num': got['run-num'],
+        stats: {
+            input: got.stats.input,
+            request: got.stats.request,
+            dataflow: got.stats.dataflow,
+            commonMeasurements: new Map(got.stats.commonMeasurements
+                .map(([k, v]) => {
+                (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
+                return [k, BigInt(v.slice(0, -1))];
+            })),
+            perSliceMeasurements: new Map(got.stats.perSliceMeasurements
+                .map(([k, v]) => mapPerSliceStats(k, v)))
+        }
+    };
+    const totalSlices = got.stats.perSliceMeasurements.size;
+    console.log(`Summarizing ${totalSlices} slices...`);
+    let atSliceNumber = 0;
+    const summarized = await (0, process_2.summarizeSlicerStats)(got.stats, (criterion, stats) => {
+        console.log(`${ansi_1.escape}1F${ansi_1.escape}1G${ansi_1.escape}2K    [${++atSliceNumber}/${totalSlices}] Summarizing ${JSON.stringify(criterion)} (reconstructed has ${stats.reconstructedCode.code.length} characters)`);
+    });
+    console.log(`    - Append raw summary to ${outputPath}`);
+    fs_1.default.appendFileSync(outputPath, `${JSON.stringify({
+        filename: got.filename,
+        'file-id': got['file-id'],
+        'run-num': got['run-num'],
+        summarize: summarized
+    }, json_1.jsonReplacer)}\n`);
+    console.log(`    - Append textual summary to ${summarizedText}`);
+    fs_1.default.appendFileSync(summarizedText, `${(0, print_1.stats2string)(summarized)}\n`);
+}
+exports.processRunMeasurement = processRunMeasurement;
+function processSummarizedFileMeasurement(file, summariesFile, outputPath) {
+    console.log(`Summarize all runs for ${file}`);
+    const summaries = [];
+    (0, files_1.readLineByLineSync)(summariesFile, l => (0, process_1.processNextSummary)(l, summaries));
+    fs_1.default.appendFileSync(outputPath, `${JSON.stringify({
+        filename: file,
+        summarize: (0, process_1.summarizeAllSummarizedStats)(summaries)
+    }, json_1.jsonReplacer)}\n`);
+}
+exports.processSummarizedFileMeasurement = processSummarizedFileMeasurement;
+function mapPerSliceStats(k, v) {
+    return [k, {
+            reconstructedCode: v.reconstructedCode,
+            slicingCriteria: v.slicingCriteria,
+            timesHitThreshold: v.timesHitThreshold,
+            measurements: new Map(v.measurements
+                .map(([k, v]) => {
+                (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
+                return [k, BigInt(v.slice(0, -1))];
+            })),
+            numberOfDataflowNodesSliced: v.numberOfDataflowNodesSliced
+        }];
+}
+//# sourceMappingURL=input.js.map

package/benchmark/summarizer/first-phase/process.d.ts

@@ -0,0 +1,11 @@
+import type { Reduction, SummarizedSlicerStats } from '../data';
+import type { SummarizedMeasurement } from '../../../util/summarizer';
+import type { PerSliceStats, SlicerStats } from '../../stats/stats';
+import type { SlicingCriteria } from '../../../slicing/criterion/parse';
+/**
+ * Summarizes the given stats by calculating the min, max, median, mean, and the standard deviation for each measurement.
+ * @see Slicer
+ */
+export declare function summarizeSlicerStats(stats: SlicerStats, report?: (criteria: SlicingCriteria, stats: PerSliceStats) => void): Promise<Readonly<SummarizedSlicerStats>>;
+export declare function summarizeSummarizedMeasurement(data: SummarizedMeasurement[]): SummarizedMeasurement;
+export declare function summarizeReductions(reductions: Reduction<SummarizedMeasurement>[]): Reduction<SummarizedMeasurement>;

package/benchmark/summarizer/first-phase/process.js

@@ -0,0 +1,205 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarizeReductions = exports.summarizeSummarizedMeasurement = exports.summarizeSlicerStats = void 0;
+const tmp = __importStar(require("tmp"));
+const fs_1 = __importDefault(require("fs"));
+const defaultmap_1 = require("../../../util/defaultmap");
+const log_1 = require("../../../util/log");
+const strings_1 = require("../../../util/strings");
+const summarizer_1 = require("../../../util/summarizer");
+const assert_1 = require("../../../util/assert");
+const shell_1 = require("../../../r-bridge/shell");
+const retriever_1 = require("../../../r-bridge/retriever");
+const visitor_1 = require("../../../r-bridge/lang-4.x/ast/model/processing/visitor");
+const tempfile = (() => {
+    let _tempfile = undefined;
+    return () => {
+        if (_tempfile === undefined) {
+            _tempfile = tmp.fileSync({ postfix: '.R', keep: false });
+            process.on('beforeExit', () => _tempfile?.removeCallback());
+        }
+        return _tempfile;
+    };
+})();
+function safeDivPercentage(a, b) {
+    if (isNaN(a) || isNaN(b)) {
+        return undefined;
+    }
+    else if (b === 0) {
+        return a === 0 ? 0 : undefined;
+    }
+    else {
+        const result = 1 - (a / b);
+        if (isNaN(result)) {
+            log_1.log.error(`NaN for ${a} and ${b}\n`);
+            return undefined;
+        }
+        else {
+            return result;
+        }
+    }
+}
+function calculateReductionForSlice(input, dataflow, perSlice) {
+    return {
+        numberOfLines: safeDivPercentage(perSlice.lines, input.numberOfLines),
+        numberOfLinesNoAutoSelection: safeDivPercentage(perSlice.lines - perSlice.autoSelected, input.numberOfLines),
+        numberOfCharacters: safeDivPercentage(perSlice.characters, input.numberOfCharacters),
+        numberOfNonWhitespaceCharacters: safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
+        numberOfRTokens: safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
+        numberOfNormalizedTokens: safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
+        numberOfDataflowNodes: safeDivPercentage(perSlice.dataflowNodes, dataflow.numberOfNodes)
+    };
+}
+/**
+ * Summarizes the given stats by calculating the min, max, median, mean, and the standard deviation for each measurement.
+ * @see Slicer
+ */
+async function summarizeSlicerStats(stats, report = () => {
+}) {
+    const perSliceStats = stats.perSliceMeasurements;
+    const collect = new defaultmap_1.DefaultMap(() => []);
+    const sizeOfSliceCriteria = [];
+    const reParseShellSession = new shell_1.RShell();
+    const reductions = [];
+    let failedOutputs = 0;
+    const sliceSize = {
+        lines: [],
+        autoSelected: [],
+        characters: [],
+        nonWhitespaceCharacters: [],
+        tokens: [],
+        normalizedTokens: [],
+        dataflowNodes: []
+    };
+    let timesHitThreshold = 0;
+    for (const [criteria, perSliceStat] of perSliceStats) {
+        report(criteria, perSliceStat);
+        for (const measure of perSliceStat.measurements) {
+            collect.get(measure[0]).push(Number(measure[1]));
+        }
+        sizeOfSliceCriteria.push(perSliceStat.slicingCriteria.length);
+        timesHitThreshold += perSliceStat.timesHitThreshold > 0 ? 1 : 0;
+        const { code: output, autoSelected } = perSliceStat.reconstructedCode;
+        sliceSize.autoSelected.push(autoSelected);
+        const lines = output.split('\n').length;
+        sliceSize.lines.push(lines);
+        sliceSize.characters.push(output.length);
+        const nonWhitespace = (0, strings_1.withoutWhitespace)(output).length;
+        sliceSize.nonWhitespaceCharacters.push(nonWhitespace);
+        // reparse the output to get the number of tokens
+        try {
+            // there seem to be encoding issues, therefore, we dump to a temp file
+            fs_1.default.writeFileSync(tempfile().name, output);
+            const reParsed = await (0, retriever_1.retrieveNormalizedAstFromRCode)({ request: 'file', content: tempfile().name }, reParseShellSession);
+            let numberOfNormalizedTokens = 0;
+            (0, visitor_1.visitAst)(reParsed.ast, _ => {
+                numberOfNormalizedTokens++;
+                return false;
+            });
+            sliceSize.normalizedTokens.push(numberOfNormalizedTokens);
+            const numberOfRTokens = await (0, retriever_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession);
+            sliceSize.tokens.push(numberOfRTokens);
+            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, {
+                lines: lines,
+                characters: output.length,
+                nonWhitespaceCharacters: nonWhitespace,
+                autoSelected: autoSelected,
+                tokens: numberOfRTokens,
+                normalizedTokens: numberOfNormalizedTokens,
+                dataflowNodes: perSliceStat.numberOfDataflowNodesSliced
+            }));
+        }
+        catch (e) {
+            console.error(`    ! Failed to re-parse the output of the slicer for ${JSON.stringify(criteria)}`); //, e
+            console.error(`      Code: ${output}\n`);
+            failedOutputs++;
+        }
+        sliceSize.dataflowNodes.push(perSliceStat.numberOfDataflowNodesSliced);
+    }
+    // summarize all measurements:
+    const summarized = new Map();
+    for (const [criterion, measurements] of collect.entries()) {
+        summarized.set(criterion, (0, summarizer_1.summarizeMeasurement)(measurements));
+    }
+    reParseShellSession.close();
+    return {
+        ...stats,
+        perSliceMeasurements: {
+            numberOfSlices: perSliceStats.size,
+            sliceCriteriaSizes: (0, summarizer_1.summarizeMeasurement)(sizeOfSliceCriteria),
+            measurements: summarized,
+            failedToRepParse: failedOutputs,
+            timesHitThreshold,
+            reduction: {
+                numberOfLines: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
+                numberOfLinesNoAutoSelection: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
+                numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
+                numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
+                numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
+                numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
+                numberOfDataflowNodes: (0, summarizer_1.summarizeMeasurement)(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
+            },
+            sliceSize: {
+                lines: (0, summarizer_1.summarizeMeasurement)(sliceSize.lines),
+                characters: (0, summarizer_1.summarizeMeasurement)(sliceSize.characters),
+                nonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(sliceSize.nonWhitespaceCharacters),
+                autoSelected: (0, summarizer_1.summarizeMeasurement)(sliceSize.autoSelected),
+                tokens: (0, summarizer_1.summarizeMeasurement)(sliceSize.tokens),
+                normalizedTokens: (0, summarizer_1.summarizeMeasurement)(sliceSize.normalizedTokens),
+                dataflowNodes: (0, summarizer_1.summarizeMeasurement)(sliceSize.dataflowNodes)
+            }
+        }
+    };
+}
+exports.summarizeSlicerStats = summarizeSlicerStats;
+function summarizeSummarizedMeasurement(data) {
+    const min = data.map(d => d.min).filter(assert_1.isNotUndefined).reduce((a, b) => Math.min(a, b), Infinity);
+    const max = data.map(d => d.max).filter(assert_1.isNotUndefined).reduce((a, b) => Math.max(a, b), -Infinity);
+    // get most average
+    const median = data.map(d => d.median).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    const mean = data.map(d => d.mean).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    // Method 1 of https://www.statology.org/averaging-standard-deviations/
+    const std = Math.sqrt(data.map(d => d.std ** 2).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length);
+    const total = data.map(d => d.total).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0);
+    return { min, max, median, mean, std, total };
+}
+exports.summarizeSummarizedMeasurement = summarizeSummarizedMeasurement;
+function summarizeReductions(reductions) {
+    return {
+        numberOfDataflowNodes: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfDataflowNodes)),
+        numberOfLines: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfLines)),
+        numberOfCharacters: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfCharacters)),
+        numberOfNonWhitespaceCharacters: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfNonWhitespaceCharacters)),
+        numberOfLinesNoAutoSelection: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfLinesNoAutoSelection)),
+        numberOfNormalizedTokens: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfNormalizedTokens)),
+        numberOfRTokens: summarizeSummarizedMeasurement(reductions.map(r => r.numberOfRTokens))
+    };
+}
+exports.summarizeReductions = summarizeReductions;
+//# sourceMappingURL=process.js.map

package/benchmark/summarizer/second-phase/graph.d.ts

@@ -0,0 +1,2 @@
+import type { UltimateSlicerStats } from '../data';
+export declare function writeGraphOutput(ultimate: UltimateSlicerStats, outputGraphPath: string): void;

package/benchmark/summarizer/second-phase/graph.js

@@ -0,0 +1,54 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeGraphOutput = void 0;
+const fs_1 = __importDefault(require("fs"));
+const json_1 = require("../../../util/json");
+function writeGraphOutput(ultimate, outputGraphPath) {
+    console.log(`Producing benchmark graph data (${outputGraphPath})...`);
+    const data = [];
+    for (const { name, measurements } of [{ name: 'per-file', measurements: ultimate.commonMeasurements }, { name: 'per-slice', measurements: ultimate.perSliceMeasurements }]) {
+        for (const [point, measurement] of measurements) {
+            if (point === 'close R session' || point === 'initialize R session') {
+                continue;
+            }
+            const pointName = point === 'total' ? `total ${name}` : point;
+            data.push({
+                name: pointName[0].toUpperCase() + pointName.slice(1),
+                unit: 'ms',
+                value: Number(measurement.mean / 1e6),
+                range: Number(measurement.std / 1e6),
+                extra: `median: ${(measurement.median / 1e6).toFixed(2)}ms`
+            });
+        }
+    }
+    data.push({
+        name: 'failed to reconstruct/re-parse',
+        unit: '#',
+        value: ultimate.failedToRepParse,
+        extra: `out of ${ultimate.totalSlices} slices`
+    });
+    data.push({
+        name: 'times hit threshold',
+        unit: '#',
+        value: ultimate.timesHitThreshold
+    });
+    data.push({
+        name: 'reduction (characters)',
+        unit: '#',
+        value: ultimate.reduction.numberOfCharacters.mean,
+        extra: `std: ${ultimate.reduction.numberOfCharacters.std}`
+    });
+    data.push({
+        name: 'reduction (normalized tokens)',
+        unit: '#',
+        value: ultimate.reduction.numberOfNormalizedTokens.mean,
+        extra: `std: ${ultimate.reduction.numberOfNormalizedTokens.std}`
+    });
+    // write the output file
+    fs_1.default.writeFileSync(outputGraphPath, JSON.stringify(data, json_1.jsonReplacer));
+}
+exports.writeGraphOutput = writeGraphOutput;
+//# sourceMappingURL=graph.js.map

package/benchmark/summarizer/second-phase/process.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="node" />
+import type { SummarizedSlicerStats, UltimateSlicerStats } from '../data';
+export declare function summarizeAllSummarizedStats(stats: SummarizedSlicerStats[]): UltimateSlicerStats;
+export declare function summarizeAllUltimateStats(stats: UltimateSlicerStats[]): UltimateSlicerStats;
+export declare function processNextSummary(line: Buffer, allSummarized: SummarizedSlicerStats[]): void;
+export declare function processNextUltimateSummary(line: Buffer, allSummarized: UltimateSlicerStats[]): void;

package/benchmark/summarizer/second-phase/process.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processNextUltimateSummary = exports.processNextSummary = exports.summarizeAllUltimateStats = exports.summarizeAllSummarizedStats = void 0;
+const process_1 = require("../first-phase/process");
+const defaultmap_1 = require("../../../util/defaultmap");
+const summarizer_1 = require("../../../util/summarizer");
+const assert_1 = require("../../../util/assert");
+const stats_1 = require("../../stats/stats");
+function summarizeAllSummarizedStats(stats) {
+    const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
+    const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
+    const reductions = [];
+    const inputs = [];
+    const dataflows = [];
+    let failedToRepParse = 0;
+    let timesHitThreshold = 0;
+    let totalSlices = 0;
+    for (const stat of stats) {
+        for (const [k, v] of stat.commonMeasurements) {
+            commonMeasurements.get(k).push(Number(v));
+        }
+        for (const [k, v] of stat.perSliceMeasurements.measurements) {
+            perSliceMeasurements.get(k).push(v);
+        }
+        reductions.push(stat.perSliceMeasurements.reduction);
+        inputs.push(stat.input);
+        dataflows.push(stat.dataflow);
+        failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
+        totalSlices += stat.perSliceMeasurements.numberOfSlices;
+        timesHitThreshold += stat.perSliceMeasurements.timesHitThreshold;
+    }
+    return {
+        totalRequests: stats.length,
+        totalSlices: totalSlices,
+        commonMeasurements: new Map([...commonMeasurements.entries()].map(([k, v]) => [k, (0, summarizer_1.summarizeMeasurement)(v)])),
+        perSliceMeasurements: new Map([...perSliceMeasurements.entries()].map(([k, v]) => [k, (0, process_1.summarizeSummarizedMeasurement)(v)])),
+        failedToRepParse,
+        timesHitThreshold,
+        reduction: (0, process_1.summarizeReductions)(reductions),
+        input: {
+            numberOfLines: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfLines)),
+            numberOfCharacters: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfCharacters)),
+            numberOfNonWhitespaceCharacters: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonWhitespaceCharacters)),
+            numberOfRTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfRTokens)),
+            numberOfNormalizedTokens: (0, summarizer_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokens))
+        },
+        dataflow: {
+            numberOfNodes: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfNodes)),
+            numberOfFunctionDefinitions: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfFunctionDefinitions)),
+            numberOfCalls: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfCalls)),
+            numberOfEdges: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.numberOfEdges))
+        }
+    };
+}
+exports.summarizeAllSummarizedStats = summarizeAllSummarizedStats;
+function summarizeAllUltimateStats(stats) {
+    return {
+        // these should be deterministic, so we don't technically need to use max, but we do just in case something unexpected happens :)
+        totalRequests: Math.max(...stats.map(s => s.totalRequests)),
+        totalSlices: Math.max(...stats.map(s => s.totalSlices)),
+        failedToRepParse: Math.max(...stats.map(s => s.failedToRepParse)),
+        timesHitThreshold: Math.max(...stats.map(s => s.timesHitThreshold)),
+        // average out / summarize other measurements
+        commonMeasurements: new Map(stats_1.CommonSlicerMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.commonMeasurements.get(m)))])),
+        perSliceMeasurements: new Map(stats_1.PerSliceMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.perSliceMeasurements.get(m)))])),
+        reduction: (0, process_1.summarizeReductions)(stats.map(s => s.reduction)),
+        input: {
+            numberOfLines: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfLines)),
+            numberOfCharacters: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfCharacters)),
+            numberOfNonWhitespaceCharacters: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNonWhitespaceCharacters)),
+            numberOfRTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfRTokens)),
+            numberOfNormalizedTokens: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.input.numberOfNormalizedTokens))
+        },
+        dataflow: {
+            numberOfNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfNodes)),
+            numberOfFunctionDefinitions: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfFunctionDefinitions)),
+            numberOfCalls: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfCalls)),
+            numberOfEdges: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.numberOfEdges))
+        }
+    };
+}
+exports.summarizeAllUltimateStats = summarizeAllUltimateStats;
+function processNextSummary(line, allSummarized) {
+    let got = JSON.parse(line.toString());
+    got = {
+        summarize: {
+            input: got.summarize.input,
+            request: got.summarize.request,
+            dataflow: got.summarize.dataflow,
+            commonMeasurements: new Map(got.summarize.commonMeasurements
+                .map(([k, v]) => {
+                (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
+                return [k, BigInt(v.slice(0, -1))];
+            })),
+            perSliceMeasurements: {
+                numberOfSlices: got.summarize.perSliceMeasurements.numberOfSlices,
+                sliceCriteriaSizes: got.summarize.perSliceMeasurements.sliceCriteriaSizes,
+                measurements: new Map(got.summarize.perSliceMeasurements.measurements),
+                reduction: got.summarize.perSliceMeasurements.reduction,
+                timesHitThreshold: got.summarize.perSliceMeasurements.timesHitThreshold,
+                failedToRepParse: got.summarize.perSliceMeasurements.failedToRepParse,
+                sliceSize: got.summarize.perSliceMeasurements.sliceSize
+            }
+        }
+    };
+    allSummarized.push(got.summarize);
+}
+exports.processNextSummary = processNextSummary;
+function processNextUltimateSummary(line, allSummarized) {
+    let got = JSON.parse(line.toString());
+    got = {
+        summarize: {
+            totalRequests: got.summarize.totalRequests,
+            totalSlices: got.summarize.totalSlices,
+            commonMeasurements: new Map(got.summarize.commonMeasurements),
+            perSliceMeasurements: new Map(got.summarize.perSliceMeasurements),
+            failedToRepParse: got.summarize.failedToRepParse,
+            timesHitThreshold: got.summarize.timesHitThreshold,
+            reduction: got.summarize.reduction,
+            input: got.summarize.input,
+            dataflow: got.summarize.dataflow,
+        }
+    };
+    allSummarized.push(got.summarize);
+}
+exports.processNextUltimateSummary = processNextUltimateSummary;
+//# sourceMappingURL=process.js.map