@eagleoutice/flowr 1.4.1 → 1.5.0

package/util/summarizer/statistics/post-process/clusterer.js

@@ -1,43 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.clusterStatisticsOutput = void 0;
-/**
- * Reading the given file line by line and expecting constructs of {@link StatisticsOutputFormat},
- * this module is responsible for identifying interesting groups of same data.
- *
- * @module
- */
-const n_readlines_1 = __importDefault(require("n-readlines"));
-const defaultmap_1 = require("../../../defaultmap");
-const r_bridge_1 = require("../../../../r-bridge");
-/**
- * Takes a statistics file like `statistics-out/top-2023-01-01-00-00-00/Assignments/assignmentOperator.txt` and clusters the values by context
- *
- * @param filepath     - Filepath of the statistics file
- * @param contextIdMap - The id map to use, can use an existing one to reuse ids for same contexts spreading over multiple input files.
- *  `undefined` is used for unknown contexts. This map allows us to reference contexts with a way shorter identifier (vs. the full file path).
- */
-function clusterStatisticsOutput(filepath, contextIdMap = new defaultmap_1.DefaultMap((0, r_bridge_1.deterministicCountingIdGenerator)())) {
-    const lineReader = new n_readlines_1.default(filepath);
-    // for each value we store the context ids it was seen in (may list the same context multiple times if more often) - this serves as a counter as well
-    const valueInfoMap = new defaultmap_1.DefaultMap(() => new defaultmap_1.DefaultMap(() => 0));
-    let line;
-    // eslint-disable-next-line no-cond-assign
-    while (line = lineReader.next()) {
-        const json = JSON.parse(line.toString());
-        const contextId = contextIdMap.get(json[1]);
-        const value = valueInfoMap.get(json[0]);
-        // step the counter accordingly
-        value.set(contextId, value.get(contextId) + 1);
-    }
-    return {
-        filepath,
-        contextIdMap,
-        valueInfoMap
-    };
-}
-exports.clusterStatisticsOutput = clusterStatisticsOutput;
-//# sourceMappingURL=clusterer.js.map

package/util/summarizer/statistics/post-process/file-based-count.d.ts

@@ -1,17 +0,0 @@
-import type { ClusterReport } from './clusterer';
-export interface FileBasedTable {
-    header: string[];
-    rows: string[][];
-}
-/**
- * The purpose of this function is to reformat {@link ClusterReport} in way that lists file-based contributions.
- * E.g., "the file with id 12 contained the assignment with `<-` 3 times".
- * Feature Values are listed in the header.
- *
- * @param report - the report to reformat
- */
-export declare function fileBasedCount(report: ClusterReport): FileBasedTable;
-/**
- * The threshold will cap of values larger to the threshold.
- */
-export declare function writeFileBasedCountToFile(table: FileBasedTable, filepath: string): void;

package/util/summarizer/statistics/post-process/file-based-count.js

@@ -1,49 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.writeFileBasedCountToFile = exports.fileBasedCount = void 0;
-const fs_1 = __importDefault(require("fs"));
-/**
- * The purpose of this function is to reformat {@link ClusterReport} in way that lists file-based contributions.
- * E.g., "the file with id 12 contained the assignment with `<-` 3 times".
- * Feature Values are listed in the header.
- *
- * @param report - the report to reformat
- */
-function fileBasedCount(report) {
-    const values = report.valueInfoMap;
-    const contexts = [...report.valueInfoMap.values()];
-    const header = [...values.keys()].map(k => `"${k}"`);
-    const rows = [];
-    for (const id of report.contextIdMap.values()) {
-        rows.push(contexts.map(c => `${c.get(id)}`));
-    }
-    return {
-        header: header,
-        rows: rows
-    };
-}
-exports.fileBasedCount = fileBasedCount;
-/**
- * The threshold will cap of values larger to the threshold.
- */
-function writeFileBasedCountToFile(table, filepath) {
-    const handle = fs_1.default.openSync(filepath, 'w');
-    const header = table.header.join('\t');
-    fs_1.default.writeSync(handle, `${header}\n`);
-    let max = 0;
-    function processEntry(r) {
-        const val = Number(r);
-        max = Math.max(val, max);
-        return r;
-    }
-    for (const row of table.rows) {
-        fs_1.default.writeSync(handle, row.map(processEntry).join('\t') + '\n');
-    }
-    fs_1.default.writeSync(handle, `%%% max: ${max}\n`);
-    fs_1.default.closeSync(handle);
-}
-exports.writeFileBasedCountToFile = writeFileBasedCountToFile;
-//# sourceMappingURL=file-based-count.js.map

package/util/summarizer/statistics/post-process/histogram.d.ts

@@ -1,59 +0,0 @@
-import type { ClusterReport } from './clusterer';
-import type { Table } from '../../../files';
-/**
- * A conventional histogram (e.g., created by {@link histogramFromNumbers}).
- * Can be converted to a {@link Table} by {@link histograms2table}.
- * As described in {@link histogramFromNumbers}, there always will be a special bin for minimum.
- */
-export interface Histogram {
-    /** A name intended for humans to know what the histogram is about. */
-    readonly name: string;
-    /** Values located in each bin */
-    bins: number[];
-    /** The configured size of each bin (stored explicitly to avoid semantic confusion with floating point arithmetic/problems with different rounding schemes) */
-    binSize: number;
-    /** Minimum value encountered (inclusive minimum of the underlying value range) */
-    min: number;
-    /** Maximum value encountered (inclusive maximum of the underlying value range) */
-    max: number;
-    /** Average of the included numbers */
-    mean: number;
-    /** Standard deviation of the included numbers */
-    std: number;
-    /** Median of the included numbers */
-    median: number;
-}
-/**
- * Produces column-wise histogram-information based on a {@link ClusterReport}.
- *
- * Let's suppose you want histograms for the Assignments feature.
- * By default, for each clustered value, a histogram is produced (can be configured by `filter`).
- *
- * @param report  - The report to collect histogram information from
- * @param binSize - Size of each bin (see {@link histogramFromNumbers} for details on why we do not specify the bin-count)
- * @param relateValuesToNumberOfLines - If true, each value (like `<-` appeared in file 'x' exactly `N` times) will be divided by the number of lines in the file 'x'.
- * @param filter  - If given, only produce histograms for the given values
- */
-export declare function histogramsFromClusters(report: ClusterReport, binSize: number, relateValuesToNumberOfLines: boolean, ...filter: string[]): Histogram[];
-/**
- * Produces a histogram from a list of numbers.
- * Because we need to create several histograms of different datasets and want to compare them, we do not accept the
- * number of bins desired and calculate the bin-size from the data (via `Math.ceil((max - min + 1) / bins)`).
- * Instead, we require the bin-size to be given.
- * There *always* will be an extra bin for the minimum value.
- */
-export declare function histogramFromNumbers(name: string, binSize: number, values: number[]): Histogram;
-/**
- * Takes an array of histograms created by {@link histogramFromNumbers} and produces a CSV table from it.
- * They must have the same bin-size for this function to work.
- *
- * The table has the following columns:
- * - `bin`  - The corresponding bin number
- * - `from` - The exclusive lower bound of the bin
- * - `to`   - The inclusive upper bound of the bin
- * - a column with the name of each histogram, containing its count of values in the corresponding bin
- *
- * @param histograms     - The histogram to convert (assumed to have the same ranges and bins)
- * @param countAsDensity - If true, the count is divided by the total number of values (individually for each histogram, similar to pgfplots `hist/density` option)
- */
-export declare function histograms2table(histograms: Histogram[], countAsDensity?: boolean): Table;

package/util/summarizer/statistics/post-process/histogram.js

@@ -1,128 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.histograms2table = exports.histogramFromNumbers = exports.histogramsFromClusters = void 0;
-const defaultmap_1 = require("../../../defaultmap");
-const assert_1 = require("../../../assert");
-const bimap_1 = require("../../../bimap");
-const fs_1 = __importDefault(require("fs"));
-const benchmark_1 = require("../../../../benchmark");
-/**
- * Produces column-wise histogram-information based on a {@link ClusterReport}.
- *
- * Let's suppose you want histograms for the Assignments feature.
- * By default, for each clustered value, a histogram is produced (can be configured by `filter`).
- *
- * @param report  - The report to collect histogram information from
- * @param binSize - Size of each bin (see {@link histogramFromNumbers} for details on why we do not specify the bin-count)
- * @param relateValuesToNumberOfLines - If true, each value (like `<-` appeared in file 'x' exactly `N` times) will be divided by the number of lines in the file 'x'.
- * @param filter  - If given, only produce histograms for the given values
- */
-function histogramsFromClusters(report, binSize, relateValuesToNumberOfLines, ...filter) {
-    const contexts = [...report.valueInfoMap.entries()];
-    const filenameFromId = new bimap_1.BiMap(report.contextIdMap.entries());
-    // first, we collect the number of appearances for each value
-    const valueCounts = new defaultmap_1.DefaultMap(() => []);
-    for (const id of report.contextIdMap.values()) {
-        // calculate the number of lines within the file given by the id
-        const filename = filenameFromId.getKey(id);
-        (0, assert_1.guard)(filename !== undefined, `filename for id ${id} is undefined`);
-        const numberOfLines = relateValuesToNumberOfLines ? fs_1.default.readFileSync(filename, 'utf-8').split('\n').length : 1;
-        for (const [value, counts] of contexts) {
-            valueCounts.get(value).push(counts.get(id) / numberOfLines);
-        }
-    }
-    return [...valueCounts.entries()].map(([name, counts]) => filter.length === 0 || filter.includes(name) ? histogramFromNumbers(name, binSize, counts) : undefined).filter(assert_1.isNotUndefined);
-}
-exports.histogramsFromClusters = histogramsFromClusters;
-/**
- * Produces a histogram from a list of numbers.
- * Because we need to create several histograms of different datasets and want to compare them, we do not accept the
- * number of bins desired and calculate the bin-size from the data (via `Math.ceil((max - min + 1) / bins)`).
- * Instead, we require the bin-size to be given.
- * There *always* will be an extra bin for the minimum value.
- */
-function histogramFromNumbers(name, binSize, values) {
-    (0, assert_1.guard)(binSize > 0, `binSize must be greater than 0, but was ${binSize}`);
-    (0, assert_1.guard)(values.length > 0, 'values must not be empty');
-    const summarized = (0, benchmark_1.summarizeMeasurement)(values);
-    const numberOfBins = Math.ceil((summarized.max - summarized.min + 1) / binSize) + 1;
-    const histogram = new Array(numberOfBins).fill(0);
-    for (const v of values) {
-        const bin = v === summarized.min ? 0 : Math.floor((v - summarized.min) / binSize) + 1;
-        histogram[bin]++;
-    }
-    return {
-        name: name,
-        bins: histogram,
-        binSize,
-        ...summarized
-    };
-}
-exports.histogramFromNumbers = histogramFromNumbers;
-/**
- * Takes an array of histograms created by {@link histogramFromNumbers} and produces a CSV table from it.
- * They must have the same bin-size for this function to work.
- *
- * The table has the following columns:
- * - `bin`  - The corresponding bin number
- * - `from` - The exclusive lower bound of the bin
- * - `to`   - The inclusive upper bound of the bin
- * - a column with the name of each histogram, containing its count of values in the corresponding bin
- *
- * @param histograms     - The histogram to convert (assumed to have the same ranges and bins)
- * @param countAsDensity - If true, the count is divided by the total number of values (individually for each histogram, similar to pgfplots `hist/density` option)
- */
-function histograms2table(histograms, countAsDensity = false) {
-    (0, assert_1.guard)(histograms.length > 0, 'there must be at least one histogram to convert to a table');
-    const mostBins = guardForLargestBinSize(histograms);
-    const header = ['bin', 'from', 'to', ...histograms.map(h => JSON.stringify(h.name))];
-    const sums = histograms.map(h => h.bins.reduce((a, b) => a + b, 0));
-    const rows = [];
-    for (let binIndex = 0; binIndex < mostBins; binIndex++) {
-        const row = new Array(histograms.length + 3);
-        row[0] = String(binIndex);
-        if (binIndex === 0) {
-            row[1] = histograms[0].min.toFixed(3);
-            row[2] = histograms[0].min.toFixed(3);
-        }
-        else {
-            row[1] = String((binIndex - 1) * histograms[0].binSize + histograms[0].min);
-            row[2] = String((binIndex) * histograms[0].binSize + histograms[0].min);
-        }
-        // fill remaining columns
-        writeRoResultsForHistograms(histograms, binIndex, row, countAsDensity, sums);
-        rows.push(row);
-    }
-    return {
-        header: header,
-        rows: rows
-    };
-}
-exports.histograms2table = histograms2table;
-function guardForLargestBinSize(histograms) {
-    const first = histograms[0];
-    let mostBins = first.bins.length;
-    for (let i = 1; i < histograms.length; i++) {
-        (0, assert_1.guard)(histograms[i].binSize === first.binSize, `histograms must have the same bin-size, but ${histograms[i].name} has ${histograms[i].binSize} instead of ${first.binSize}`);
-        if (histograms[i].bins.length > mostBins) {
-            mostBins = histograms[i].bins.length;
-        }
-    }
-    return mostBins;
-}
-function writeRoResultsForHistograms(histograms, binIndex, row, countAsDensity, sums) {
-    for (let j = 0; j < histograms.length; j++) {
-        const bins = histograms[j].bins;
-        // does not have to be performant...
-        if (binIndex >= bins.length) {
-            row[j + 3] = '0'; /* in a histogram, 0 is the best default value for bins that are not present -- no value appeared in the corresponding bin */
-        }
-        else {
-            row[j + 3] = String(countAsDensity ? bins[binIndex] / sums[j] : bins[binIndex]);
-        }
-    }
-}
-//# sourceMappingURL=histogram.js.map

package/util/summarizer/statistics/post-process/index.d.ts

@@ -1,4 +0,0 @@
-export * from './clusterer';
-export * from './post-process-output';
-export * from './histogram';
-export * from './file-based-count';

package/util/summarizer/statistics/post-process/index.js

@@ -1,21 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./clusterer"), exports);
-__exportStar(require("./post-process-output"), exports);
-__exportStar(require("./histogram"), exports);
-__exportStar(require("./file-based-count"), exports);
-//# sourceMappingURL=index.js.map

package/util/summarizer/statistics/post-process/post-process-output.d.ts

@@ -1,16 +0,0 @@
-import type { FeatureSelection } from '../../../../statistics';
-import type { ClusterReport } from './clusterer';
-/**
- * Post process the collections in a given folder, reducing them in a memory preserving way.
- *
- * @param filepath - Path to the root file of the data collection like `statistics-out/top-2023-01-01-00-00-00/`
- * @param features - Collection of features to post process, expects corresponding folders to exist
- *
- * @returns non-aggregated reports for each sub-key of each feature
- */
-export declare function postProcessFeatureFolder(filepath: string, features: FeatureSelection): ClusterReport[];
-/**
- * Prints the report to the console, but limits the output to the `limit` entries with the highest counts.
- * The names of these entries (like `->`) are returned, so they can be used to filter the following histograms.
- */
-export declare function printClusterReport(report: ClusterReport, limit?: number): string[];

package/util/summarizer/statistics/post-process/post-process-output.js

@@ -1,104 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.printClusterReport = exports.postProcessFeatureFolder = void 0;
-const statistics_1 = require("../../../../statistics");
-const path_1 = __importDefault(require("path"));
-const log_1 = require("../../../log");
-const fs_1 = __importDefault(require("fs"));
-const clusterer_1 = require("./clusterer");
-const statistics_2 = require("../../../../statistics");
-const r_bridge_1 = require("../../../../r-bridge");
-const defaultmap_1 = require("../../../defaultmap");
-/**
- * Post process the collections in a given folder, reducing them in a memory preserving way.
- *
- * @param filepath - Path to the root file of the data collection like `statistics-out/top-2023-01-01-00-00-00/`
- * @param features - Collection of features to post process, expects corresponding folders to exist
- *
- * @returns non-aggregated reports for each sub-key of each feature
- */
-function postProcessFeatureFolder(filepath, features) {
-    if (!fs_1.default.existsSync(filepath)) {
-        log_1.log.warn(`Folder for ${filepath} does not exist, skipping post processing`);
-        return [];
-    }
-    const results = [];
-    for (const feature of features) {
-        const result = processFeatureFolder(filepath, feature);
-        if (result.length > 0) {
-            results.push(...result);
-        }
-    }
-    return results;
-}
-exports.postProcessFeatureFolder = postProcessFeatureFolder;
-/**
- * Process a single feature folder like `Assignments/`
- *
- * @param filepath - Same as the input to {@link postProcessFeatureFolder}
- * @param feature  - The (single) feature to process
- */
-function processFeatureFolder(filepath, feature) {
-    const featureInfo = statistics_1.ALL_FEATURES[feature];
-    const targetPath = path_1.default.join(filepath, featureInfo.name);
-    if (!fs_1.default.existsSync(targetPath)) {
-        log_1.log.warn(`Folder for ${feature} does not exist at ${targetPath} skipping post processing of this feature`);
-        return [];
-    }
-    log_1.log.info(`Processing ${feature} at ${targetPath}`);
-    const contextIdMap = new defaultmap_1.DefaultMap((0, r_bridge_1.deterministicCountingIdGenerator)());
-    const featureSubKeys = Object.keys(featureInfo.initialValue);
-    const reports = [];
-    for (const subKey of featureSubKeys) {
-        const value = processFeatureSubKey(targetPath, subKey, contextIdMap);
-        if (value !== undefined) {
-            reports.push(value);
-        }
-    }
-    return reports;
-}
-function processFeatureSubKey(featurePath, subKey, contextIdMap) {
-    const targetPath = path_1.default.join(featurePath, `${subKey}${statistics_2.defaultStatisticsFileSuffix}`);
-    if (!fs_1.default.existsSync(targetPath)) {
-        log_1.log.warn(`Folder for ${subKey} does not exist at ${targetPath} skipping post processing of this key`);
-        return undefined;
-    }
-    return (0, clusterer_1.clusterStatisticsOutput)(targetPath, contextIdMap);
-}
-/**
- * Prints the report to the console, but limits the output to the `limit` entries with the highest counts.
- * The names of these entries (like `->`) are returned, so they can be used to filter the following histograms.
- */
-function printClusterReport(report, limit = 1000) {
-    console.log('\n\n\n');
-    console.log(report.filepath);
-    const shortStats = [...report.valueInfoMap.entries()].map(([name, values]) => {
-        return {
-            name,
-            count: [...values.values()].reduce((a, b) => a + b, 0),
-            unique: values.size()
-        };
-    }).sort((a, b) => b.count - a.count).slice(0, limit);
-    const { longestName, longestCount, longestUnique } = shortStats.reduce((acc, { name, count, unique }) => {
-        return {
-            longestName: Math.max(acc.longestName, name.length),
-            longestCount: Math.max(acc.longestCount, count.toLocaleString().length),
-            longestUnique: Math.max(acc.longestUnique, unique.toLocaleString().length),
-        };
-    }, { longestName: 0, longestCount: 0, longestUnique: 0 });
-    for (const { name, count, unique } of shortStats) {
-        const strId = `${name}`.padEnd(longestName, ' ');
-        const strCount = count.toLocaleString().padStart(longestCount, ' ');
-        const strUnique = unique.toLocaleString().padStart(longestUnique, ' ');
-        const uniqueSuffix = `\t (${strUnique} ${statistics_2.formatter.format('unique', { color: 7 /* Colors.White */, effect: statistics_2.ColorEffect.Foreground })})`;
-        console.log(`\t${statistics_2.formatter.format(strId, { style: 1 /* FontStyles.Bold */ })}\t ${strCount} ` +
-            `${statistics_2.formatter.format('total', { color: 7 /* Colors.White */, effect: statistics_2.ColorEffect.Foreground })}`
-            + (count !== unique ? uniqueSuffix : ''));
-    }
-    return shortStats.map(({ name }) => name);
-}
-exports.printClusterReport = printClusterReport;
-//# sourceMappingURL=post-process-output.js.map

package/util/summarizer/statistics/second-phase/process.d.ts

@@ -1,11 +0,0 @@
-import type { CommonSummarizerConfiguration } from '../../summarizer';
-import type { StatisticsSummarizerConfiguration } from '../summarizer';
-/**
- * Post process the collections in a given folder, retrieving the final summaries.
- *
- * @param logger       - The logger to use for outputs
- * @param filepath     - Path to the root file of the data collection (contains all the archives)
- * @param config       - Configuration of the summarizer
- * @param outputPath   - The final outputPath to write the result to (may differ from the configured root folder)
- */
-export declare function postProcessFeatureFolder(logger: CommonSummarizerConfiguration['logger'], filepath: string, config: StatisticsSummarizerConfiguration, outputPath: string): void;

package/util/summarizer/statistics/second-phase/process.js

@@ -1,117 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.postProcessFeatureFolder = void 0;
-const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
-const statistics_1 = require("../../../../statistics");
-const files_1 = require("../../../files");
-const assert_1 = require("../../../assert");
-const time_1 = require("../../../time");
-const data_1 = require("../../benchmark/data");
-const process_1 = require("../../benchmark/first-phase/process");
-const arrays_1 = require("../../../arrays");
-function postProcessFeatures(config, filepath, outputPath, logger, metaFeatureInformation) {
-    for (const featureName of config.featuresToUse) {
-        const featureInfo = statistics_1.ALL_FEATURES[featureName];
-        const targetPath = path_1.default.join(filepath, featureInfo.name);
-        const targetFeature = path_1.default.join(outputPath, featureInfo.name);
-        if (!featureInfo.postProcess) {
-            logger(`    Skipping post processing of ${featureName} as no post processing behavior is defined`);
-            continue;
-        }
-        logger(`    Post processing of ${featureName}...`);
-        if (!fs_1.default.existsSync(targetFeature)) {
-            fs_1.default.mkdirSync(targetFeature, { recursive: true });
-        }
-        if (global.gc) {
-            logger(`    [${(0, time_1.date2string)(new Date())}] Running garbage collection (--expose-gc)`);
-            global.gc();
-        }
-        featureInfo.postProcess(targetPath, metaFeatureInformation, targetFeature, config);
-    }
-}
-function postProcessMeta(config, filepath, outputPath, logger, metaFeatureInformation) {
-    const fileStatisticsSummary = {
-        successfulParsed: [],
-        processingTimeMs: [],
-        failedRequests: [],
-        // min lengths of 1 etc. could come from different line endings
-        lines: [],
-        characters: [],
-        numberOfNormalizedNodes: []
-    };
-    if (!fs_1.default.existsSync(path_1.default.join(outputPath, 'meta'))) {
-        fs_1.default.mkdirSync(path_1.default.join(outputPath, 'meta'), { recursive: true });
-    }
-    const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'meta', 'stats.csv'));
-    out.write(`file,successfulParsed,${(0, data_1.summarizedMeasurement2CsvHeader)('processing')},failedRequests,${(0, data_1.summarizedMeasurement2CsvHeader)('line-length')},${(0, data_1.summarizedMeasurement2CsvHeader)('lines')},${(0, data_1.summarizedMeasurement2CsvHeader)('characters')},numberOfNormalizedNodes\n`);
-    for (const [file, info] of metaFeatureInformation) {
-        // we could retrieve these by summing later as well :thinking: however, this makes it more explicit
-        const characters = (0, arrays_1.sum)(info.stats.lines[0]);
-        out.write(`${JSON.stringify(file)},${info.stats.successfulParsed},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(info.stats.processingTimeMs))},`
-            + `${info.stats.failedRequests.length},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(info.stats.lines[0]))},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)([info.stats.lines[0].length]))},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)([characters]))},${info.stats.numberOfNormalizedNodes[0]}\n`);
-        fileStatisticsSummary.successfulParsed.push(info.stats.successfulParsed);
-        fileStatisticsSummary.processingTimeMs.push(...info.stats.processingTimeMs);
-        fileStatisticsSummary.failedRequests.push(info.stats.failedRequests.length);
-        fileStatisticsSummary.lines.push(info.stats.lines[0]);
-        fileStatisticsSummary.characters.push(characters);
-        fileStatisticsSummary.numberOfNormalizedNodes.push(info.stats.numberOfNormalizedNodes[0]);
-    }
-    out.write(`all,${(0, arrays_1.sum)(fileStatisticsSummary.successfulParsed)},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(fileStatisticsSummary.processingTimeMs))},`
-        + `${(0, arrays_1.sum)(fileStatisticsSummary.failedRequests)},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(fileStatisticsSummary.lines.flat()))},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(fileStatisticsSummary.lines.map(l => l.length)))},${(0, data_1.summarizedMeasurement2Csv)((0, process_1.summarizeMeasurement)(fileStatisticsSummary.characters))},${(0, arrays_1.sum)(fileStatisticsSummary.numberOfNormalizedNodes)}\n`);
-    out.close();
-}
-/**
- * Post process the collections in a given folder, retrieving the final summaries.
- *
- * @param logger       - The logger to use for outputs
- * @param filepath     - Path to the root file of the data collection (contains all the archives)
- * @param config       - Configuration of the summarizer
- * @param outputPath   - The final outputPath to write the result to (may differ from the configured root folder)
- */
-function postProcessFeatureFolder(logger, filepath, config, outputPath) {
-    if (!fs_1.default.existsSync(filepath)) {
-        logger(`    Folder for ${filepath} does not exist, skipping post processing`);
-        return;
-    }
-    if (!fs_1.default.existsSync(outputPath)) {
-        fs_1.default.mkdirSync(outputPath, { recursive: true });
-    }
-    const metaFeatureInformation = extractMetaInformationFrom(logger, path_1.default.join(filepath, 'meta', 'features.txt'), path_1.default.join(filepath, 'meta', 'stats.txt'));
-    postProcessFeatures(config, filepath, outputPath, logger, metaFeatureInformation);
-    postProcessMeta(config, filepath, outputPath, logger, metaFeatureInformation);
-}
-exports.postProcessFeatureFolder = postProcessFeatureFolder;
-function extractMetaInformationFrom(logger, metaFeaturesPath, metaStatsPath) {
-    const storage = new Map();
-    logger(`    [${(0, time_1.date2string)(new Date())}] Collect feature statistics`);
-    (0, files_1.readLineByLineSync)(metaFeaturesPath, (line, lineNumber) => {
-        if (line.length === 0) {
-            return;
-        }
-        if (lineNumber % 2_500 === 0) {
-            logger(`    [${(0, time_1.date2string)(new Date())}] ${lineNumber} meta feature lines processed`);
-        }
-        const meta = JSON.parse(line.toString());
-        storage.set(meta.file, meta.content);
-    });
-    logger(`    [${(0, time_1.date2string)(new Date())}] Collect meta statistics`);
-    (0, files_1.readLineByLineSync)(metaStatsPath, (line, lineNumber) => {
-        if (line.length === 0) {
-            return;
-        }
-        if (lineNumber % 2_500 === 0) {
-            logger(`    [${(0, time_1.date2string)(new Date())}] ${lineNumber} meta statistics lines processed`);
-        }
-        const meta = JSON.parse(line.toString());
-        const existing = storage.get(meta.file);
-        (0, assert_1.guard)(existing !== undefined, () => `Expected to find meta information for ${meta.file} in line ${lineNumber + 1} of ${metaFeaturesPath}`);
-        existing.stats = meta.content;
-    });
-    logger(`    [${(0, time_1.date2string)(new Date())}] Done collecting meta information`);
-    return storage;
-}
-//# sourceMappingURL=process.js.map

package/util/summarizer/statistics/summarizer.d.ts

@@ -1,35 +0,0 @@
-import type { CommonSummarizerConfiguration } from '../summarizer';
-import { Summarizer } from '../summarizer';
-import type { FeatureSelection } from '../../../statistics';
-export interface StatisticsSummarizerConfiguration extends CommonSummarizerConfiguration {
-    /**
-     * The input path to read all zips from
-     */
-    inputPath: string;
-    /**
-     * Features to extract the summaries for
-     */
-    featuresToUse: FeatureSelection;
-    /**
-     * Path for the intermediate results of the preparation phase
-     */
-    intermediateOutputPath: string;
-    /**
-     * Path for the final results of the summarization phase
-     */
-    outputPath: string;
-    /**
-     * How many folders to skip to find the project root
-     */
-    projectSkip: number;
-}
-export declare const statisticsFileNameRegex: RegExp;
-export declare class StatisticsSummarizer extends Summarizer<unknown, StatisticsSummarizerConfiguration> {
-    constructor(config: StatisticsSummarizerConfiguration);
-    private removeIfExists;
-    /**
-     * The preparation phase essentially merges all files into one by just attaching lines together!
-     */
-    preparationPhase(useTypeClassification: boolean): Promise<void>;
-    summarizePhase(): Promise<unknown>;
-}