@eagleoutice/flowr 1.3.11

package/util/summarizer/benchmark/first-phase/process.js

@@ -0,0 +1,208 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarizeSummarizedMeasurement = exports.summarizeMeasurement = exports.summarizeSlicerStats = void 0;
+const tmp = __importStar(require("tmp"));
+const assert_1 = require("../../../assert");
+const log_1 = require("../../../log");
+const defaultmap_1 = require("../../../defaultmap");
+const r_bridge_1 = require("../../../../r-bridge");
+const strings_1 = require("../../../strings");
+const fs_1 = __importDefault(require("fs"));
+const arrays_1 = require("../../../arrays");
+const tempfile = (() => {
+    let _tempfile = undefined;
+    return () => {
+        if (_tempfile === undefined) {
+            _tempfile = tmp.fileSync({ postfix: '.R', keep: false });
+            process.on('beforeExit', () => _tempfile?.removeCallback());
+        }
+        return _tempfile;
+    };
+})();
+function safeDivPercentage(a, b) {
+    if (isNaN(a) || isNaN(b)) {
+        return undefined;
+    }
+    else if (b === 0) {
+        return a === 0 ? 0 : undefined;
+    }
+    else {
+        const result = 1 - (a / b);
+        if (isNaN(result)) {
+            log_1.log.error(`NaN for ${a} and ${b}\n`);
+            return undefined;
+        }
+        else {
+            return result;
+        }
+    }
+}
+function calculateReductionForSlice(input, dataflow, perSlice) {
+    return {
+        numberOfLines: safeDivPercentage(perSlice.lines, input.numberOfLines),
+        numberOfLinesNoAutoSelection: safeDivPercentage(perSlice.lines - perSlice.autoSelected, input.numberOfLines),
+        numberOfCharacters: safeDivPercentage(perSlice.characters, input.numberOfCharacters),
+        numberOfNonWhitespaceCharacters: safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
+        numberOfRTokens: safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
+        numberOfNormalizedTokens: safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
+        numberOfDataflowNodes: safeDivPercentage(perSlice.dataflowNodes, dataflow.numberOfNodes)
+    };
+}
+/**
+ * Summarizes the given stats by calculating the min, max, median, mean, and the standard deviation for each measurement.
+ * @see Slicer
+ */
+async function summarizeSlicerStats(stats, report = () => {
+}) {
+    const perSliceStats = stats.perSliceMeasurements;
+    const collect = new defaultmap_1.DefaultMap(() => []);
+    const sizeOfSliceCriteria = [];
+    const reParseShellSession = new r_bridge_1.RShell();
+    reParseShellSession.tryToInjectHomeLibPath();
+    const reductions = [];
+    let failedOutputs = 0;
+    const sliceSize = {
+        lines: [],
+        autoSelected: [],
+        characters: [],
+        nonWhitespaceCharacters: [],
+        tokens: [],
+        normalizedTokens: [],
+        dataflowNodes: []
+    };
+    let first = true;
+    let timesHitThreshold = 0;
+    for (const [criteria, perSliceStat] of perSliceStats) {
+        report(criteria, perSliceStat);
+        for (const measure of perSliceStat.measurements) {
+            collect.get(measure[0]).push(Number(measure[1]));
+        }
+        sizeOfSliceCriteria.push(perSliceStat.slicingCriteria.length);
+        timesHitThreshold += perSliceStat.timesHitThreshold > 0 ? 1 : 0;
+        const { code: output, autoSelected } = perSliceStat.reconstructedCode;
+        sliceSize.autoSelected.push(autoSelected);
+        const lines = output.split('\n').length;
+        sliceSize.lines.push(lines);
+        sliceSize.characters.push(output.length);
+        const nonWhitespace = (0, strings_1.withoutWhitespace)(output).length;
+        sliceSize.nonWhitespaceCharacters.push(nonWhitespace);
+        // reparse the output to get the number of tokens
+        try {
+            // there seem to be encoding issues, therefore, we dump to a temp file
+            fs_1.default.writeFileSync(tempfile().name, output);
+            const reParsed = await (0, r_bridge_1.retrieveNormalizedAstFromRCode)({ request: 'file', content: tempfile().name, ensurePackageInstalled: first }, reParseShellSession);
+            first = false;
+            let numberOfNormalizedTokens = 0;
+            (0, r_bridge_1.visitAst)(reParsed.ast, _ => {
+                numberOfNormalizedTokens++;
+                return false;
+            });
+            sliceSize.normalizedTokens.push(numberOfNormalizedTokens);
+            const numberOfRTokens = await (0, r_bridge_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession);
+            sliceSize.tokens.push(numberOfRTokens);
+            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, {
+                lines: lines,
+                characters: output.length,
+                nonWhitespaceCharacters: nonWhitespace,
+                autoSelected: autoSelected,
+                tokens: numberOfRTokens,
+                normalizedTokens: numberOfNormalizedTokens,
+                dataflowNodes: perSliceStat.numberOfDataflowNodesSliced
+            }));
+        }
+        catch (e) {
+            console.error(`    ! Failed to re-parse the output of the slicer for ${JSON.stringify(criteria)}`); //, e
+            console.error(`      Code: ${output}\n`);
+            failedOutputs++;
+        }
+        sliceSize.dataflowNodes.push(perSliceStat.numberOfDataflowNodesSliced);
+    }
+    // summarize all measurements:
+    const summarized = new Map();
+    for (const [criterion, measurements] of collect.entries()) {
+        summarized.set(criterion, summarizeMeasurement(measurements));
+    }
+    reParseShellSession.close();
+    return {
+        ...stats,
+        perSliceMeasurements: {
+            numberOfSlices: perSliceStats.size,
+            sliceCriteriaSizes: summarizeMeasurement(sizeOfSliceCriteria),
+            measurements: summarized,
+            failedToRepParse: failedOutputs,
+            timesHitThreshold,
+            reduction: {
+                numberOfLines: summarizeMeasurement(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
+                numberOfLinesNoAutoSelection: summarizeMeasurement(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
+                numberOfCharacters: summarizeMeasurement(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
+                numberOfNonWhitespaceCharacters: summarizeMeasurement(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
+                numberOfRTokens: summarizeMeasurement(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
+                numberOfNormalizedTokens: summarizeMeasurement(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
+                numberOfDataflowNodes: summarizeMeasurement(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
+            },
+            sliceSize: {
+                lines: summarizeMeasurement(sliceSize.lines),
+                characters: summarizeMeasurement(sliceSize.characters),
+                nonWhitespaceCharacters: summarizeMeasurement(sliceSize.nonWhitespaceCharacters),
+                autoSelected: summarizeMeasurement(sliceSize.autoSelected),
+                tokens: summarizeMeasurement(sliceSize.tokens),
+                normalizedTokens: summarizeMeasurement(sliceSize.normalizedTokens),
+                dataflowNodes: summarizeMeasurement(sliceSize.dataflowNodes)
+            }
+        }
+    };
+}
+exports.summarizeSlicerStats = summarizeSlicerStats;
+function summarizeMeasurement(data, totalNumberOfDataPoints) {
+    // just to avoid in-place modification
+    const sorted = [...data].sort((a, b) => a - b);
+    const min = sorted[0];
+    const max = sorted[sorted.length - 1];
+    const median = sorted[Math.floor(sorted.length / 2)];
+    const total = (0, arrays_1.sum)(sorted);
+    const length = totalNumberOfDataPoints ?? sorted.length;
+    const mean = total / length;
+    // sqrt(sum(x-mean)^2 / n)
+    const std = Math.sqrt(sorted.map(x => (x - mean) ** 2).reduce((a, b) => a + b, 0) / length);
+    return { min, max, median, mean, std, total };
+}
+exports.summarizeMeasurement = summarizeMeasurement;
+function summarizeSummarizedMeasurement(data) {
+    const min = data.map(d => d.min).filter(assert_1.isNotUndefined).reduce((a, b) => Math.min(a, b), Infinity);
+    const max = data.map(d => d.max).filter(assert_1.isNotUndefined).reduce((a, b) => Math.max(a, b), -Infinity);
+    // get most average
+    const median = data.map(d => d.median).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    const mean = data.map(d => d.mean).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    // Method 1 of https://www.statology.org/averaging-standard-deviations/
+    const std = Math.sqrt(data.map(d => d.std ** 2).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length);
+    const total = data.map(d => d.total).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0);
+    return { min, max, median, mean, std, total };
+}
+exports.summarizeSummarizedMeasurement = summarizeSummarizedMeasurement;
+//# sourceMappingURL=process.js.map

package/util/summarizer/benchmark/second-phase/graph.d.ts

@@ -0,0 +1,2 @@
+import { UltimateSlicerStats } from '../data';
+export declare function writeGraphOutput(ultimate: UltimateSlicerStats, outputGraphPath: string): void;

package/util/summarizer/benchmark/second-phase/graph.js

@@ -0,0 +1,54 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeGraphOutput = void 0;
+const fs_1 = __importDefault(require("fs"));
+const json_1 = require("../../../json");
+function writeGraphOutput(ultimate, outputGraphPath) {
+    console.log(`Producing benchmark graph data (${outputGraphPath})...`);
+    const data = [];
+    for (const { name, measurements } of [{ name: 'per-file', measurements: ultimate.commonMeasurements }, { name: 'per-slice', measurements: ultimate.perSliceMeasurements }]) {
+        for (const [point, measurement] of measurements) {
+            if (point === 'close R session' || point === 'initialize R session' || point === 'inject home path' || point === 'ensure installation of xmlparsedata' || point === 'retrieve token map') {
+                continue;
+            }
+            const pointName = point === 'total' ? `total ${name}` : point;
+            data.push({
+                name: pointName[0].toUpperCase() + pointName.slice(1),
+                unit: 'ms',
+                value: Number(measurement.mean / 1e6),
+                range: Number(measurement.std / 1e6),
+                extra: `median: ${(measurement.median / 1e6).toFixed(2)}ms`
+            });
+        }
+    }
+    data.push({
+        name: 'failed to reconstruct/re-parse',
+        unit: '#',
+        value: ultimate.failedToRepParse,
+        extra: `out of ${ultimate.totalSlices} slices`
+    });
+    data.push({
+        name: 'times hit threshold',
+        unit: '#',
+        value: ultimate.timesHitThreshold
+    });
+    data.push({
+        name: 'reduction (characters)',
+        unit: '#',
+        value: ultimate.reduction.numberOfCharacters.mean,
+        extra: `std: ${ultimate.reduction.numberOfCharacters.std}`
+    });
+    data.push({
+        name: 'reduction (normalized tokens)',
+        unit: '#',
+        value: ultimate.reduction.numberOfNormalizedTokens.mean,
+        extra: `std: ${ultimate.reduction.numberOfNormalizedTokens.std}`
+    });
+    // write the output file
+    fs_1.default.writeFileSync(outputGraphPath, JSON.stringify(data, json_1.jsonReplacer));
+}
+exports.writeGraphOutput = writeGraphOutput;
+//# sourceMappingURL=graph.js.map

package/util/summarizer/benchmark/second-phase/process.d.ts

@@ -0,0 +1,4 @@
+/// <reference types="node" />
+import { SummarizedSlicerStats, UltimateSlicerStats } from '../data';
+export declare function summarizeAllSummarizedStats(stats: SummarizedSlicerStats[]): UltimateSlicerStats;
+export declare function processNextSummary(line: Buffer, allSummarized: SummarizedSlicerStats[]): void;

package/util/summarizer/benchmark/second-phase/process.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processNextSummary = exports.summarizeAllSummarizedStats = void 0;
+const defaultmap_1 = require("../../../defaultmap");
+const benchmark_1 = require("../../../../benchmark");
+const assert_1 = require("../../../assert");
+function summarizeAllSummarizedStats(stats) {
+    const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
+    const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
+    const reductions = [];
+    const inputs = [];
+    const dataflows = [];
+    let failedToRepParse = 0;
+    let timesHitThreshold = 0;
+    let totalSlices = 0;
+    for (const stat of stats) {
+        for (const [k, v] of stat.commonMeasurements) {
+            commonMeasurements.get(k).push(Number(v));
+        }
+        for (const [k, v] of stat.perSliceMeasurements.measurements) {
+            perSliceMeasurements.get(k).push(v);
+        }
+        reductions.push(stat.perSliceMeasurements.reduction);
+        inputs.push(stat.input);
+        dataflows.push(stat.dataflow);
+        failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
+        totalSlices += stat.perSliceMeasurements.numberOfSlices;
+        timesHitThreshold += stat.perSliceMeasurements.timesHitThreshold;
+    }
+    return {
+        totalRequests: stats.length,
+        totalSlices: totalSlices,
+        commonMeasurements: new Map([...commonMeasurements.entries()].map(([k, v]) => [k, (0, benchmark_1.summarizeMeasurement)(v)])),
+        perSliceMeasurements: new Map([...perSliceMeasurements.entries()].map(([k, v]) => [k, (0, benchmark_1.summarizeSummarizedMeasurement)(v)])),
+        failedToRepParse,
+        timesHitThreshold,
+        reduction: {
+            numberOfDataflowNodes: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfDataflowNodes)),
+            numberOfLines: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfLines)),
+            numberOfCharacters: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfCharacters)),
+            numberOfNonWhitespaceCharacters: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfNonWhitespaceCharacters)),
+            numberOfLinesNoAutoSelection: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfLinesNoAutoSelection)),
+            numberOfNormalizedTokens: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfNormalizedTokens)),
+            numberOfRTokens: (0, benchmark_1.summarizeSummarizedMeasurement)(reductions.map(r => r.numberOfRTokens))
+        },
+        input: {
+            numberOfLines: (0, benchmark_1.summarizeMeasurement)(inputs.map(i => i.numberOfLines)),
+            numberOfCharacters: (0, benchmark_1.summarizeMeasurement)(inputs.map(i => i.numberOfCharacters)),
+            numberOfNonWhitespaceCharacters: (0, benchmark_1.summarizeMeasurement)(inputs.map(i => i.numberOfNonWhitespaceCharacters)),
+            numberOfRTokens: (0, benchmark_1.summarizeMeasurement)(inputs.map(i => i.numberOfRTokens)),
+            numberOfNormalizedTokens: (0, benchmark_1.summarizeMeasurement)(inputs.map(i => i.numberOfNormalizedTokens))
+        },
+        dataflow: {
+            numberOfNodes: (0, benchmark_1.summarizeMeasurement)(dataflows.map(d => d.numberOfNodes)),
+            numberOfFunctionDefinitions: (0, benchmark_1.summarizeMeasurement)(dataflows.map(d => d.numberOfFunctionDefinitions)),
+            numberOfCalls: (0, benchmark_1.summarizeMeasurement)(dataflows.map(d => d.numberOfCalls)),
+            numberOfEdges: (0, benchmark_1.summarizeMeasurement)(dataflows.map(d => d.numberOfEdges))
+        }
+    };
+}
+exports.summarizeAllSummarizedStats = summarizeAllSummarizedStats;
+function processNextSummary(line, allSummarized) {
+    let got = JSON.parse(line.toString());
+    got = {
+        filename: got.filename,
+        summarize: {
+            input: got.summarize.input,
+            request: got.summarize.request,
+            dataflow: got.summarize.dataflow,
+            commonMeasurements: new Map(got.summarize.commonMeasurements
+                .map(([k, v]) => {
+                (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
+                return [k, BigInt(v.slice(0, -1))];
+            })),
+            perSliceMeasurements: {
+                numberOfSlices: got.summarize.perSliceMeasurements.numberOfSlices,
+                sliceCriteriaSizes: got.summarize.perSliceMeasurements.sliceCriteriaSizes,
+                measurements: new Map(got.summarize.perSliceMeasurements.measurements),
+                reduction: got.summarize.perSliceMeasurements.reduction,
+                timesHitThreshold: got.summarize.perSliceMeasurements.timesHitThreshold,
+                failedToRepParse: got.summarize.perSliceMeasurements.failedToRepParse,
+                sliceSize: got.summarize.perSliceMeasurements.sliceSize
+            }
+        }
+    };
+    allSummarized.push(got.summarize);
+}
+exports.processNextSummary = processNextSummary;
+//# sourceMappingURL=process.js.map

package/util/summarizer/benchmark/summarizer.d.ts

@@ -0,0 +1,34 @@
+/**
+ * This module is tasked with processing the results of the benchmarking (see {@link SlicerStats}).
+ * @module
+ */
+import { CommonSummarizerConfiguration, Summarizer } from '../summarizer';
+import { UltimateSlicerStats } from './data';
+export interface BenchmarkSummarizerConfiguration extends CommonSummarizerConfiguration {
+    /**
+     * If given, produce graph data output (e.g. for the benchmark visualization) to the given path
+     */
+    graphOutputPath?: string;
+    /**
+     * The input path to read from
+     */
+    inputPath: string;
+    /**
+     * Path for the intermediate results of the preparation phase
+     */
+    intermediateOutputPath: string;
+    /**
+     * Path for the final results of the summarization phase
+     */
+    outputLogPath?: string;
+    /**
+     * Path for the final results of the summarization phase
+     */
+    outputPath: string;
+}
+export declare class BenchmarkSummarizer extends Summarizer<UltimateSlicerStats, BenchmarkSummarizerConfiguration> {
+    constructor(config: BenchmarkSummarizerConfiguration);
+    preparationPhase(): Promise<void>;
+    summarizePhase(): Promise<UltimateSlicerStats>;
+    private removeIfExists;
+}

package/util/summarizer/benchmark/summarizer.js

@@ -0,0 +1,53 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BenchmarkSummarizer = void 0;
+/**
+ * This module is tasked with processing the results of the benchmarking (see {@link SlicerStats}).
+ * @module
+ */
+const summarizer_1 = require("../summarizer");
+const fs_1 = __importDefault(require("fs"));
+const input_1 = require("./first-phase/input");
+const json_1 = require("../../json");
+const benchmark_1 = require("../../../benchmark");
+const process_1 = require("./second-phase/process");
+const graph_1 = require("./second-phase/graph");
+const files_1 = require("../../files");
+class BenchmarkSummarizer extends summarizer_1.Summarizer {
+    constructor(config) {
+        super(config);
+    }
+    async preparationPhase() {
+        this.removeIfExists(this.config.intermediateOutputPath);
+        this.removeIfExists(this.config.outputLogPath);
+        await (0, files_1.readLineByLine)(this.config.inputPath, (line, lineNumber) => (0, input_1.processNestMeasurement)(line, lineNumber, `${this.config.intermediateOutputPath}.log`, this.config.intermediateOutputPath));
+        this.log('Done summarizing');
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await -- just to obey the structure
+    async summarizePhase() {
+        this.log(`Summarizing all summaries from ${this.config.inputPath}...`);
+        this.removeIfExists(this.config.outputPath);
+        const allSummarized = [];
+        (0, files_1.readLineByLineSync)(this.config.intermediateOutputPath, line => (0, process_1.processNextSummary)(line, allSummarized));
+        // summarizedRaw
+        const ultimate = (0, process_1.summarizeAllSummarizedStats)(allSummarized);
+        this.log(`Writing ultimate summary to ${this.config.outputPath}`);
+        fs_1.default.writeFileSync(this.config.outputPath, JSON.stringify(ultimate, json_1.jsonReplacer));
+        console.log((0, benchmark_1.ultimateStats2String)(ultimate));
+        if (this.config.graphOutputPath) {
+            (0, graph_1.writeGraphOutput)(ultimate, this.config.graphOutputPath);
+        }
+        return ultimate;
+    }
+    removeIfExists(path) {
+        if (path && fs_1.default.existsSync(path)) {
+            this.log(`Removing existing ${path}`);
+            fs_1.default.unlinkSync(path);
+        }
+    }
+}
+exports.BenchmarkSummarizer = BenchmarkSummarizer;
+//# sourceMappingURL=summarizer.js.map

package/util/summarizer/statistics/first-phase/process.d.ts

@@ -0,0 +1,6 @@
+export declare class FileMigrator {
+    private readonly writeHandles;
+    private finished;
+    migrate(sourceFolderContent: Map<string, string>, targetFolder: string, originalFile: string | undefined): Promise<void>;
+    finish(): void;
+}

package/util/summarizer/statistics/first-phase/process.js

@@ -0,0 +1,81 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileMigrator = void 0;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const assert_1 = require("../../../assert");
+class FileMigrator {
+    writeHandles = new Map();
+    finished = false;
+    async migrate(sourceFolderContent, targetFolder, originalFile) {
+        (0, assert_1.guard)(!this.finished, () => 'migrator is already marked as finished!');
+        if (!node_fs_1.default.existsSync(targetFolder)) {
+            node_fs_1.default.mkdirSync(targetFolder, { recursive: true });
+        }
+        const promises = [];
+        for (const [filepath, content] of sourceFolderContent.entries()) {
+            const target = node_path_1.default.join(targetFolder, filepath);
+            let targetStream = this.writeHandles.get(target);
+            if (targetStream === undefined) {
+                if (!node_fs_1.default.existsSync(node_path_1.default.dirname(target))) {
+                    node_fs_1.default.mkdirSync(node_path_1.default.dirname(target), { recursive: true });
+                }
+                targetStream = node_fs_1.default.createWriteStream(target, { flags: 'a' });
+                this.writeHandles.set(target, targetStream);
+            }
+            // before we write said content we have to group {value: string, context: string} by context (while we can safely assume that there is only one context per file,
+            // i want to be sure
+            let data;
+            // regex matches failed due to encoding errors
+            if (filepath.endsWith('meta/stats.txt') || filepath.endsWith('meta/features.txt')) {
+                data = `{"file":"${originalFile ?? ''}","content":${content.trimEnd()}}\n`;
+            }
+            else {
+                const grouped = groupByContext(content);
+                data = grouped === undefined ? content : grouped.map(s => JSON.stringify(s)).join('\n') + '\n';
+            }
+            promises.push(new Promise((resolve, reject) => targetStream.write(data, 'utf-8', err => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve();
+                }
+            })));
+        }
+        await Promise.all(promises);
+    }
+    finish() {
+        for (const handle of this.writeHandles.values()) {
+            handle.close();
+        }
+        this.finished = true;
+    }
+}
+exports.FileMigrator = FileMigrator;
+function groupByContext(input) {
+    if (input === undefined) {
+        return [];
+    }
+    const parsed = input.split('\n').filter(s => s && s !== '').map(s => JSON.parse(s));
+    const grouped = new Map();
+    for (const content of parsed) {
+        if (!Array.isArray(content)) {
+            // in this case it is a meta file or other which does not have to be grouped
+            return undefined;
+        }
+        const [value, context] = content;
+        const get = grouped.get(context);
+        if (get === undefined) {
+            grouped.set(context, [value]);
+        }
+        else {
+            get.push(value);
+        }
+    }
+    return [...grouped.entries()].map(([context, values]) => [values, context]);
+}
+//# sourceMappingURL=process.js.map

package/util/summarizer/statistics/post-process/clusterer.d.ts

@@ -0,0 +1,26 @@
+import { DefaultMap } from '../../../defaultmap';
+import { NodeId } from '../../../../r-bridge';
+import { MergeableRecord } from '../../../objects';
+export type ContextsWithCount = DefaultMap<NodeId, number>;
+export type ClusterContextIdMap = DefaultMap<string | undefined, NodeId>;
+export type ClusterValueInfoMap = DefaultMap<string, ContextsWithCount>;
+/** Produced by {@link clusterStatisticsOutput} */
+export interface ClusterReport extends MergeableRecord {
+    /** The input file which has been clustered */
+    readonly filepath: string;
+    /** Maps each context encountered (i.e., every file which contains something associated with the feature) to a unique id, used in the {@link ClusterReport#valueInfoMap | valueInfoMap}. */
+    contextIdMap: ClusterContextIdMap;
+    /**
+   * Counts which contexts contained which values of a feature.
+   * For example, that `<-` occurred in files with ids `[12, 42, 19, 19]` (i.e., the context with the id 19 contained it twice).
+   */
+    valueInfoMap: ClusterValueInfoMap;
+}
+/**
+ * Takes a statistics file like `statistics-out/top-2023-01-01-00-00-00/Assignments/assignmentOperator.txt` and clusters the values by context
+ *
+ * @param filepath     - Filepath of the statistics file
+ * @param contextIdMap - The id map to use, can use an existing one to reuse ids for same contexts spreading over multiple input files.
+ *  `undefined` is used for unknown contexts. This map allows us to reference contexts with a way shorter identifier (vs. the full file path).
+ */
+export declare function clusterStatisticsOutput(filepath: string, contextIdMap?: ClusterContextIdMap): ClusterReport;

package/util/summarizer/statistics/post-process/clusterer.js

@@ -0,0 +1,43 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clusterStatisticsOutput = void 0;
+/**
+ * Reading the given file line by line and expecting constructs of {@link StatisticsOutputFormat},
+ * this module is responsible for identifying interesting groups of same data.
+ *
+ * @module
+ */
+const n_readlines_1 = __importDefault(require("n-readlines"));
+const defaultmap_1 = require("../../../defaultmap");
+const r_bridge_1 = require("../../../../r-bridge");
+/**
+ * Takes a statistics file like `statistics-out/top-2023-01-01-00-00-00/Assignments/assignmentOperator.txt` and clusters the values by context
+ *
+ * @param filepath     - Filepath of the statistics file
+ * @param contextIdMap - The id map to use, can use an existing one to reuse ids for same contexts spreading over multiple input files.
+ *  `undefined` is used for unknown contexts. This map allows us to reference contexts with a way shorter identifier (vs. the full file path).
+ */
+function clusterStatisticsOutput(filepath, contextIdMap = new defaultmap_1.DefaultMap((0, r_bridge_1.deterministicCountingIdGenerator)())) {
+    const lineReader = new n_readlines_1.default(filepath);
+    // for each value we store the context ids it was seen in (may list the same context multiple times if more often) - this serves as a counter as well
+    const valueInfoMap = new defaultmap_1.DefaultMap(() => new defaultmap_1.DefaultMap(() => 0));
+    let line;
+    // eslint-disable-next-line no-cond-assign
+    while (line = lineReader.next()) {
+        const json = JSON.parse(line.toString());
+        const contextId = contextIdMap.get(json[1]);
+        const value = valueInfoMap.get(json[0]);
+        // step the counter accordingly
+        value.set(contextId, value.get(contextId) + 1);
+    }
+    return {
+        filepath,
+        contextIdMap,
+        valueInfoMap
+    };
+}
+exports.clusterStatisticsOutput = clusterStatisticsOutput;
+//# sourceMappingURL=clusterer.js.map

package/util/summarizer/statistics/post-process/file-based-count.d.ts

@@ -0,0 +1,17 @@
+import { ClusterReport } from './clusterer';
+export interface FileBasedTable {
+    header: string[];
+    rows: string[][];
+}
+/**
+ * The purpose of this function is to reformat {@link ClusterReport} in way that lists file-based contributions.
+ * E.g., "the file with id 12 contained the assignment with `<-` 3 times".
+ * Feature Values are listed in the header.
+ *
+ * @param report - the report to reformat
+ */
+export declare function fileBasedCount(report: ClusterReport): FileBasedTable;
+/**
+ * The threshold will cap of values larger to the threshold.
+ */
+export declare function writeFileBasedCountToFile(table: FileBasedTable, filepath: string): void;