@eagleoutice/flowr 1.5.2 → 2.0.1

package/statistics/output/statistics-file.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendStatisticsFile = exports.initDummyFileProvider = exports.initFileProvider = exports.statisticsFileProvider = exports.extractNodeContent = void 0;
+const file_provider_1 = require("./file-provider");
+const log_1 = require("../../util/log");
+/**
+ * Requires source information to be attached on parsing!
+ * <p>
+ * Returns the content of the node (i.e., the text content excluding the children)
+ */
+function extractNodeContent(node) {
+    let result = node.textContent;
+    if (node.hasChildNodes()) {
+        const firstChild = node.childNodes.item(0);
+        if (firstChild.nodeType === 3 /* text node */) {
+            result = firstChild.textContent;
+        }
+    }
+    return result?.trim()?.replaceAll('\n', '\\n') ?? '<unknown>';
+}
+exports.extractNodeContent = extractNodeContent;
+initDummyFileProvider();
+/**
+ * Make the statistics write to a given output directory.
+ */
+function initFileProvider(outputDirectory) {
+    log_1.log.debug(`Initializing file provider for output directory ${outputDirectory}`);
+    exports.statisticsFileProvider = new file_provider_1.StatisticFileProvider(outputDirectory);
+}
+exports.initFileProvider = initFileProvider;
+/**
+ * Either ignore the statistics or write them to a given map (e.g., for testing).
+ *
+ * @param map - The map to write to, will not persist calls if no map is given
+ */
+function initDummyFileProvider(map) {
+    exports.statisticsFileProvider = new file_provider_1.DummyAppendProvider(map);
+}
+exports.initDummyFileProvider = initDummyFileProvider;
+/**
+ * Append the content of all nodes to the storage file for the given feature
+ * @param name    - The name of the feature {@link Feature#name}
+ * @param fn      - The name of the feature-aspect to record
+ * @param nodes   - The nodes to append, you may pass already transformed string contents
+ * @param context - The context of the information retrieval (e.g. the name of the file that contained the R source code)
+ * @param unique  - Should duplicate entries be removed on addition
+ */
+function appendStatisticsFile(name, fn, nodes, context, unique = false) {
+    if (nodes.length === 0) {
+        return;
+    }
+    let values;
+    if (typeof nodes[0] === 'string') {
+        values = nodes;
+    }
+    else if ('nodeType' in nodes[0]) {
+        values = nodes.map(extractNodeContent);
+    }
+    else {
+        values = nodes;
+    }
+    if (unique) {
+        values = [...new Set(values)];
+    }
+    values = values.map(value => JSON.stringify(context === undefined ? [value] : [value, context]));
+    exports.statisticsFileProvider.append(name, fn, values.join('\n'));
+}
+exports.appendStatisticsFile = appendStatisticsFile;
+//# sourceMappingURL=statistics-file.js.map

package/statistics/statistics.d.ts

@@ -0,0 +1,28 @@
+import type { MetaStatistics } from './meta-statistics';
+import type { RParseRequestFromFile, RParseRequestFromText } from '../r-bridge/retriever';
+import type { PipelineOutput } from '../core/steps/pipeline/pipeline';
+import { DEFAULT_DATAFLOW_PIPELINE } from '../core/steps/pipeline/default-pipelines';
+import type { RShell } from '../r-bridge/shell';
+import type { FeatureSelection, FeatureStatistics } from './features/feature';
+/**
+ * By default, {@link extractUsageStatistics} requires a generator, but sometimes you already know all the files
+ * that you want to process. This function simply reps your requests as a generator.
+ */
+export declare function staticRequests(...requests: (RParseRequestFromText | RParseRequestFromFile)[]): AsyncGenerator<RParseRequestFromText | RParseRequestFromFile>;
+type DataflowResult = PipelineOutput<typeof DEFAULT_DATAFLOW_PIPELINE>;
+/**
+ * Extract all wanted statistic information from a set of requests using the presented R session.
+ *
+ * @param shell     - The R session to use
+ * @param onRequest - A callback that is called at the beginning of each request, this may be used to debug the requests.
+ * @param features  - The features to extract (see {@link allFeatureNames}).
+ * @param requests  - The requests to extract the features from. May generate them on demand (e.g., by traversing a folder).
+ * 										If your request is statically known, you can use {@link staticRequests} to create this generator.
+ * @param rootPath  - The root path to the project, this is used to relativize the file paths in the statistics.
+ */
+export declare function extractUsageStatistics<T extends RParseRequestFromText | RParseRequestFromFile>(shell: RShell, onRequest: (request: T) => void, features: FeatureSelection, requests: AsyncGenerator<T>, rootPath?: string): Promise<{
+    features: FeatureStatistics;
+    meta: MetaStatistics;
+    outputs: Map<T, DataflowResult>;
+}>;
+export {};

package/statistics/statistics.js

@@ -0,0 +1,108 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractUsageStatistics = exports.staticRequests = void 0;
+const xmldom_1 = require("@xmldom/xmldom");
+const fs_1 = __importDefault(require("fs"));
+const meta_statistics_1 = require("./meta-statistics");
+const log_1 = require("../util/log");
+const json_1 = require("../util/json");
+const pipeline_executor_1 = require("../core/pipeline-executor");
+const default_pipelines_1 = require("../core/steps/pipeline/default-pipelines");
+const feature_1 = require("./features/feature");
+const convert_values_1 = require("../r-bridge/lang-4.x/convert-values");
+/**
+ * By default, {@link extractUsageStatistics} requires a generator, but sometimes you already know all the files
+ * that you want to process. This function simply reps your requests as a generator.
+ */
+function staticRequests(...requests) {
+    // eslint-disable-next-line @typescript-eslint/require-await
+    return async function* () {
+        for (const request of requests) {
+            yield request;
+        }
+    }();
+}
+exports.staticRequests = staticRequests;
+/**
+ * Extract all wanted statistic information from a set of requests using the presented R session.
+ *
+ * @param shell     - The R session to use
+ * @param onRequest - A callback that is called at the beginning of each request, this may be used to debug the requests.
+ * @param features  - The features to extract (see {@link allFeatureNames}).
+ * @param requests  - The requests to extract the features from. May generate them on demand (e.g., by traversing a folder).
+ * 										If your request is statically known, you can use {@link staticRequests} to create this generator.
+ * @param rootPath  - The root path to the project, this is used to relativize the file paths in the statistics.
+ */
+async function extractUsageStatistics(shell, onRequest, features, requests, rootPath) {
+    let result = initializeFeatureStatistics();
+    const meta = (0, meta_statistics_1.initialMetaStatistics)();
+    const outputs = new Map();
+    for await (const request of requests) {
+        onRequest(request);
+        const start = performance.now();
+        const suffix = request.request === 'file' ? request.content.replace(new RegExp('^' + (rootPath ?? '')), '') : undefined;
+        try {
+            let output;
+            ({ stats: result, output } = await extractSingle(result, shell, request, features, suffix));
+            outputs.set(request, output);
+            processMetaOnSuccessful(meta, request);
+            meta.numberOfNormalizedNodes.push(output.normalize.idMap.size);
+        }
+        catch (e) {
+            log_1.log.error('for request: ', request, e);
+            processMetaOnUnsuccessful(meta, request);
+        }
+        meta.processingTimeMs.push(performance.now() - start);
+    }
+    return { features: result, meta, outputs };
+}
+exports.extractUsageStatistics = extractUsageStatistics;
+function initializeFeatureStatistics() {
+    const result = {};
+    for (const key of feature_1.allFeatureNames) {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        result[key] = JSON.parse(JSON.stringify(feature_1.ALL_FEATURES[key].initialValue, json_1.jsonReplacer), json_1.jsonRetriever);
+    }
+    return result;
+}
+function processMetaOnUnsuccessful(meta, request) {
+    meta.failedRequests.push(request);
+}
+function processMetaOnSuccessful(meta, request) {
+    meta.successfulParsed++;
+    if (request.request === 'text') {
+        meta.lines.push(request.content.split('\n').map(l => l.length));
+    }
+    else {
+        meta.lines.push(fs_1.default.readFileSync(request.content, 'utf-8').split('\n').map(l => l.length));
+    }
+}
+const parser = new xmldom_1.DOMParser();
+async function extractSingle(result, shell, request, features, suffixFilePath) {
+    const slicerOutput = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_DATAFLOW_PIPELINE, {
+        request, shell
+    }).allRemainingSteps();
+    // retrieve parsed xml through (legacy) xmlparsedata
+    const suffix = request.request === 'file' ? ', encoding="utf-8"' : '';
+    shell.sendCommands(`try(flowr_parsed<-parse(${request.request}=${JSON.stringify(request.content)},keep.source=TRUE${suffix}),silent=FALSE)`, 'try(flowr_output<-xmlparsedata::xml_parse_data(flowr_parsed,includeText=TRUE,pretty=FALSE),silent=FALSE)');
+    const parsed = (await shell.sendCommandWithOutput(`cat(flowr_output,${(0, convert_values_1.ts2r)(shell.options.eol)})`)).join(shell.options.eol);
+    const doc = parser.parseFromString(parsed, 'text/xml');
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    for (const [key, feature] of Object.entries(feature_1.ALL_FEATURES)) {
+        if (features !== 'all' && !features.has(key)) {
+            continue;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        result[key] = feature.process(result[key], {
+            parsedRAst: doc,
+            dataflow: slicerOutput.dataflow,
+            normalizedRAst: slicerOutput.normalize,
+            filepath: suffixFilePath
+        });
+    }
+    return { stats: result, output: slicerOutput };
+}
+//# sourceMappingURL=statistics.js.map

package/statistics/summarizer/auto-detect.d.ts

@@ -0,0 +1,2 @@
+import { SummarizerType } from '../../util/summarizer';
+export declare function detectSummarizationType(inputPath: string): Promise<SummarizerType>;

package/statistics/summarizer/auto-detect.js

@@ -0,0 +1,32 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectSummarizationType = void 0;
+const fs_1 = __importDefault(require("fs"));
+const summarizer_1 = require("./summarizer");
+const log_1 = require("../../util/log");
+async function detectSummarizationType(inputPath) {
+    if (fs_1.default.statSync(inputPath).isFile()) {
+        log_1.log.info(`Detected benchmark summarization with single file ${inputPath}`);
+        return "benchmark" /* SummarizerType.Benchmark */;
+    }
+    // current heuristic: search for a tar.gz with two minus signs :D
+    const dir = await fs_1.default.promises.opendir(inputPath);
+    const thresholdInit = 60;
+    let threshold = thresholdInit;
+    for await (const dirent of dir) {
+        if (summarizer_1.statisticsFileNameRegex.test(dirent.name)) {
+            log_1.log.info(`Detected statistics summarization by file ${dirent.name} matching ${summarizer_1.statisticsFileNameRegex.source}`);
+            return "statistics" /* SummarizerType.Statistics */;
+        }
+        else if (threshold-- < 0) {
+            break;
+        }
+    }
+    log_1.log.info(`Detected benchmark summarization with no file (first ${thresholdInit}) matching ${summarizer_1.statisticsFileNameRegex.source}`);
+    return "benchmark" /* SummarizerType.Benchmark */;
+}
+exports.detectSummarizationType = detectSummarizationType;
+//# sourceMappingURL=auto-detect.js.map

package/statistics/summarizer/first-phase/process.d.ts

@@ -0,0 +1,6 @@
+export declare class FileMigrator {
+    private readonly writeHandles;
+    private finished;
+    migrate(sourceFolderContent: Map<string, string>, targetFolder: string, originalFile: string | undefined): Promise<void>;
+    finish(): void;
+}

package/statistics/summarizer/first-phase/process.js

@@ -0,0 +1,81 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileMigrator = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const assert_1 = require("../../../util/assert");
+class FileMigrator {
+    writeHandles = new Map();
+    finished = false;
+    async migrate(sourceFolderContent, targetFolder, originalFile) {
+        (0, assert_1.guard)(!this.finished, () => 'migrator is already marked as finished!');
+        if (!fs_1.default.existsSync(targetFolder)) {
+            fs_1.default.mkdirSync(targetFolder, { recursive: true });
+        }
+        const promises = [];
+        for (const [filepath, content] of sourceFolderContent.entries()) {
+            const target = path_1.default.join(targetFolder, filepath);
+            let targetStream = this.writeHandles.get(target);
+            if (targetStream === undefined) {
+                if (!fs_1.default.existsSync(path_1.default.dirname(target))) {
+                    fs_1.default.mkdirSync(path_1.default.dirname(target), { recursive: true });
+                }
+                targetStream = fs_1.default.createWriteStream(target, { flags: 'a' });
+                this.writeHandles.set(target, targetStream);
+            }
+            // before we write said content we have to group {value: string, context: string} by context (while we can safely assume that there is only one context per file,
+            // i want to be sure
+            let data;
+            // regex matches failed due to encoding errors
+            if (filepath.endsWith('meta/stats.txt') || filepath.endsWith('meta/features.txt')) {
+                data = `{"file":"${originalFile ?? ''}","content":${content.trimEnd()}}\n`;
+            }
+            else {
+                const grouped = groupByContext(content);
+                data = grouped === undefined ? content : grouped.map(s => JSON.stringify(s)).join('\n') + '\n';
+            }
+            promises.push(new Promise((resolve, reject) => targetStream.write(data, 'utf-8', err => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve();
+                }
+            })));
+        }
+        await Promise.all(promises);
+    }
+    finish() {
+        for (const handle of this.writeHandles.values()) {
+            handle.close();
+        }
+        this.finished = true;
+    }
+}
+exports.FileMigrator = FileMigrator;
+function groupByContext(input) {
+    if (input === undefined) {
+        return [];
+    }
+    const parsed = input.split('\n').filter(s => s && s !== '').map(s => JSON.parse(s));
+    const grouped = new Map();
+    for (const content of parsed) {
+        if (!Array.isArray(content)) {
+            // in this case it is a meta file or other which does not have to be grouped
+            return undefined;
+        }
+        const [value, context] = content;
+        const get = grouped.get(context);
+        if (get === undefined) {
+            grouped.set(context, [value]);
+        }
+        else {
+            get.push(value);
+        }
+    }
+    return [...grouped.entries()].map(([context, values]) => [values, context]);
+}
+//# sourceMappingURL=process.js.map

package/statistics/summarizer/post-process/clusterer.d.ts

@@ -0,0 +1,26 @@
+import { DefaultMap } from '../../../util/defaultmap';
+import type { MergeableRecord } from '../../../util/objects';
+import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';
+export type ContextsWithCount = DefaultMap<NodeId, number>;
+export type ClusterContextIdMap = DefaultMap<string | undefined, NodeId>;
+export type ClusterValueInfoMap = DefaultMap<string, ContextsWithCount>;
+/** Produced by {@link clusterStatisticsOutput} */
+export interface ClusterReport extends MergeableRecord {
+    /** The input file which has been clustered */
+    readonly filepath: string;
+    /** Maps each context encountered (i.e., every file which contains something associated with the feature) to a unique id, used in the {@link ClusterReport#valueInfoMap | valueInfoMap}. */
+    contextIdMap: ClusterContextIdMap;
+    /**
+   * Counts which contexts contained which values of a feature.
+   * For example, that `<-` occurred in files with ids `[12, 42, 19, 19]` (i.e., the context with the id 19 contained it twice).
+   */
+    valueInfoMap: ClusterValueInfoMap;
+}
+/**
+ * Takes a statistics file like `statistics-out/top-2023-01-01-00-00-00/Assignments/assignmentOperator.txt` and clusters the values by context
+ *
+ * @param filepath     - Filepath of the statistics file
+ * @param contextIdMap - The id map to use, can use an existing one to reuse ids for same contexts spreading over multiple input files.
+ *  `undefined` is used for unknown contexts. This map allows us to reference contexts with a way shorter identifier (vs. the full file path).
+ */
+export declare function clusterStatisticsOutput(filepath: string, contextIdMap?: ClusterContextIdMap): ClusterReport;

package/statistics/summarizer/post-process/clusterer.js

@@ -0,0 +1,43 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clusterStatisticsOutput = void 0;
+/**
+ * Reading the given file line by line and expecting constructs of {@link StatisticsOutputFormat},
+ * this module is responsible for identifying interesting groups of same data.
+ *
+ * @module
+ */
+const n_readlines_1 = __importDefault(require("n-readlines"));
+const defaultmap_1 = require("../../../util/defaultmap");
+const decorate_1 = require("../../../r-bridge/lang-4.x/ast/model/processing/decorate");
+/**
+ * Takes a statistics file like `statistics-out/top-2023-01-01-00-00-00/Assignments/assignmentOperator.txt` and clusters the values by context
+ *
+ * @param filepath     - Filepath of the statistics file
+ * @param contextIdMap - The id map to use, can use an existing one to reuse ids for same contexts spreading over multiple input files.
+ *  `undefined` is used for unknown contexts. This map allows us to reference contexts with a way shorter identifier (vs. the full file path).
+ */
+function clusterStatisticsOutput(filepath, contextIdMap = new defaultmap_1.DefaultMap((0, decorate_1.deterministicCountingIdGenerator)())) {
+    const lineReader = new n_readlines_1.default(filepath);
+    // for each value we store the context ids it was seen in (may list the same context multiple times if more often) - this serves as a counter as well
+    const valueInfoMap = new defaultmap_1.DefaultMap(() => new defaultmap_1.DefaultMap(() => 0));
+    let line;
+    // eslint-disable-next-line no-cond-assign
+    while (line = lineReader.next()) {
+        const json = JSON.parse(line.toString());
+        const contextId = contextIdMap.get(json[1]);
+        const value = valueInfoMap.get(json[0]);
+        // step the counter accordingly
+        value.set(contextId, value.get(contextId) + 1);
+    }
+    return {
+        filepath,
+        contextIdMap,
+        valueInfoMap
+    };
+}
+exports.clusterStatisticsOutput = clusterStatisticsOutput;
+//# sourceMappingURL=clusterer.js.map

package/statistics/summarizer/post-process/file-based-count.d.ts

@@ -0,0 +1,17 @@
+import type { ClusterReport } from './clusterer';
+export interface FileBasedTable {
+    header: string[];
+    rows: string[][];
+}
+/**
+ * The purpose of this function is to reformat {@link ClusterReport} in way that lists file-based contributions.
+ * E.g., "the file with id 12 contained the assignment with `<-` 3 times".
+ * Feature Values are listed in the header.
+ *
+ * @param report - the report to reformat
+ */
+export declare function fileBasedCount(report: ClusterReport): FileBasedTable;
+/**
+ * The threshold will cap of values larger to the threshold.
+ */
+export declare function writeFileBasedCountToFile(table: FileBasedTable, filepath: string): void;

package/statistics/summarizer/post-process/file-based-count.js

@@ -0,0 +1,49 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeFileBasedCountToFile = exports.fileBasedCount = void 0;
+const fs_1 = __importDefault(require("fs"));
+/**
+ * The purpose of this function is to reformat {@link ClusterReport} in way that lists file-based contributions.
+ * E.g., "the file with id 12 contained the assignment with `<-` 3 times".
+ * Feature Values are listed in the header.
+ *
+ * @param report - the report to reformat
+ */
+function fileBasedCount(report) {
+    const values = report.valueInfoMap;
+    const contexts = [...report.valueInfoMap.values()];
+    const header = [...values.keys()].map(k => `"${k}"`);
+    const rows = [];
+    for (const id of report.contextIdMap.values()) {
+        rows.push(contexts.map(c => `${c.get(id)}`));
+    }
+    return {
+        header: header,
+        rows: rows
+    };
+}
+exports.fileBasedCount = fileBasedCount;
+/**
+ * The threshold will cap of values larger to the threshold.
+ */
+function writeFileBasedCountToFile(table, filepath) {
+    const handle = fs_1.default.openSync(filepath, 'w');
+    const header = table.header.join('\t');
+    fs_1.default.writeSync(handle, `${header}\n`);
+    let max = 0;
+    function processEntry(r) {
+        const val = Number(r);
+        max = Math.max(val, max);
+        return r;
+    }
+    for (const row of table.rows) {
+        fs_1.default.writeSync(handle, row.map(processEntry).join('\t') + '\n');
+    }
+    fs_1.default.writeSync(handle, `%%% max: ${max}\n`);
+    fs_1.default.closeSync(handle);
+}
+exports.writeFileBasedCountToFile = writeFileBasedCountToFile;
+//# sourceMappingURL=file-based-count.js.map

package/statistics/summarizer/post-process/histogram.d.ts

@@ -0,0 +1,59 @@
+import type { ClusterReport } from './clusterer';
+import type { Table } from '../../../util/files';
+/**
+ * A conventional histogram (e.g., created by {@link histogramFromNumbers}).
+ * Can be converted to a {@link Table} by {@link histograms2table}.
+ * As described in {@link histogramFromNumbers}, there always will be a special bin for minimum.
+ */
+export interface Histogram {
+    /** A name intended for humans to know what the histogram is about. */
+    readonly name: string;
+    /** Values located in each bin */
+    bins: number[];
+    /** The configured size of each bin (stored explicitly to avoid semantic confusion with floating point arithmetic/problems with different rounding schemes) */
+    binSize: number;
+    /** Minimum value encountered (inclusive minimum of the underlying value range) */
+    min: number;
+    /** Maximum value encountered (inclusive maximum of the underlying value range) */
+    max: number;
+    /** Average of the included numbers */
+    mean: number;
+    /** Standard deviation of the included numbers */
+    std: number;
+    /** Median of the included numbers */
+    median: number;
+}
+/**
+ * Produces column-wise histogram-information based on a {@link ClusterReport}.
+ *
+ * Let's suppose you want histograms for the Assignments feature.
+ * By default, for each clustered value, a histogram is produced (can be configured by `filter`).
+ *
+ * @param report  - The report to collect histogram information from
+ * @param binSize - Size of each bin (see {@link histogramFromNumbers} for details on why we do not specify the bin-count)
+ * @param relateValuesToNumberOfLines - If true, each value (like `<-` appeared in file 'x' exactly `N` times) will be divided by the number of lines in the file 'x'.
+ * @param filter  - If given, only produce histograms for the given values
+ */
+export declare function histogramsFromClusters(report: ClusterReport, binSize: number, relateValuesToNumberOfLines: boolean, ...filter: string[]): Histogram[];
+/**
+ * Produces a histogram from a list of numbers.
+ * Because we need to create several histograms of different datasets and want to compare them, we do not accept the
+ * number of bins desired and calculate the bin-size from the data (via `Math.ceil((max - min + 1) / bins)`).
+ * Instead, we require the bin-size to be given.
+ * There *always* will be an extra bin for the minimum value.
+ */
+export declare function histogramFromNumbers(name: string, binSize: number, values: number[]): Histogram;
+/**
+ * Takes an array of histograms created by {@link histogramFromNumbers} and produces a CSV table from it.
+ * They must have the same bin-size for this function to work.
+ *
+ * The table has the following columns:
+ * - `bin`  - The corresponding bin number
+ * - `from` - The exclusive lower bound of the bin
+ * - `to`   - The inclusive upper bound of the bin
+ * - a column with the name of each histogram, containing its count of values in the corresponding bin
+ *
+ * @param histograms     - The histogram to convert (assumed to have the same ranges and bins)
+ * @param countAsDensity - If true, the count is divided by the total number of values (individually for each histogram, similar to pgfplots `hist/density` option)
+ */
+export declare function histograms2table(histograms: Histogram[], countAsDensity?: boolean): Table;

package/statistics/summarizer/post-process/histogram.js

@@ -0,0 +1,128 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.histograms2table = exports.histogramFromNumbers = exports.histogramsFromClusters = void 0;
+const fs_1 = __importDefault(require("fs"));
+const bimap_1 = require("../../../util/bimap");
+const defaultmap_1 = require("../../../util/defaultmap");
+const assert_1 = require("../../../util/assert");
+const summarizer_1 = require("../../../util/summarizer");
+/**
+ * Produces column-wise histogram-information based on a {@link ClusterReport}.
+ *
+ * Let's suppose you want histograms for the Assignments feature.
+ * By default, for each clustered value, a histogram is produced (can be configured by `filter`).
+ *
+ * @param report  - The report to collect histogram information from
+ * @param binSize - Size of each bin (see {@link histogramFromNumbers} for details on why we do not specify the bin-count)
+ * @param relateValuesToNumberOfLines - If true, each value (like `<-` appeared in file 'x' exactly `N` times) will be divided by the number of lines in the file 'x'.
+ * @param filter  - If given, only produce histograms for the given values
+ */
+function histogramsFromClusters(report, binSize, relateValuesToNumberOfLines, ...filter) {
+    const contexts = [...report.valueInfoMap.entries()];
+    const filenameFromId = new bimap_1.BiMap(report.contextIdMap.entries());
+    // first, we collect the number of appearances for each value
+    const valueCounts = new defaultmap_1.DefaultMap(() => []);
+    for (const id of report.contextIdMap.values()) {
+        // calculate the number of lines within the file given by the id
+        const filename = filenameFromId.getKey(id);
+        (0, assert_1.guard)(filename !== undefined, `filename for id ${id} is undefined`);
+        const numberOfLines = relateValuesToNumberOfLines ? fs_1.default.readFileSync(filename, 'utf-8').split('\n').length : 1;
+        for (const [value, counts] of contexts) {
+            valueCounts.get(value).push(counts.get(id) / numberOfLines);
+        }
+    }
+    return [...valueCounts.entries()].map(([name, counts]) => filter.length === 0 || filter.includes(name) ? histogramFromNumbers(name, binSize, counts) : undefined).filter(assert_1.isNotUndefined);
+}
+exports.histogramsFromClusters = histogramsFromClusters;
+/**
+ * Produces a histogram from a list of numbers.
+ * Because we need to create several histograms of different datasets and want to compare them, we do not accept the
+ * number of bins desired and calculate the bin-size from the data (via `Math.ceil((max - min + 1) / bins)`).
+ * Instead, we require the bin-size to be given.
+ * There *always* will be an extra bin for the minimum value.
+ */
+function histogramFromNumbers(name, binSize, values) {
+    (0, assert_1.guard)(binSize > 0, `binSize must be greater than 0, but was ${binSize}`);
+    (0, assert_1.guard)(values.length > 0, 'values must not be empty');
+    const summarized = (0, summarizer_1.summarizeMeasurement)(values);
+    const numberOfBins = Math.ceil((summarized.max - summarized.min + 1) / binSize) + 1;
+    const histogram = new Array(numberOfBins).fill(0);
+    for (const v of values) {
+        const bin = v === summarized.min ? 0 : Math.floor((v - summarized.min) / binSize) + 1;
+        histogram[bin]++;
+    }
+    return {
+        name: name,
+        bins: histogram,
+        binSize,
+        ...summarized
+    };
+}
+exports.histogramFromNumbers = histogramFromNumbers;
+/**
+ * Takes an array of histograms created by {@link histogramFromNumbers} and produces a CSV table from it.
+ * They must have the same bin-size for this function to work.
+ *
+ * The table has the following columns:
+ * - `bin`  - The corresponding bin number
+ * - `from` - The exclusive lower bound of the bin
+ * - `to`   - The inclusive upper bound of the bin
+ * - a column with the name of each histogram, containing its count of values in the corresponding bin
+ *
+ * @param histograms     - The histogram to convert (assumed to have the same ranges and bins)
+ * @param countAsDensity - If true, the count is divided by the total number of values (individually for each histogram, similar to pgfplots `hist/density` option)
+ */
+function histograms2table(histograms, countAsDensity = false) {
+    (0, assert_1.guard)(histograms.length > 0, 'there must be at least one histogram to convert to a table');
+    const mostBins = guardForLargestBinSize(histograms);
+    const header = ['bin', 'from', 'to', ...histograms.map(h => JSON.stringify(h.name))];
+    const sums = histograms.map(h => h.bins.reduce((a, b) => a + b, 0));
+    const rows = [];
+    for (let binIndex = 0; binIndex < mostBins; binIndex++) {
+        const row = new Array(histograms.length + 3);
+        row[0] = String(binIndex);
+        if (binIndex === 0) {
+            row[1] = histograms[0].min.toFixed(3);
+            row[2] = histograms[0].min.toFixed(3);
+        }
+        else {
+            row[1] = String((binIndex - 1) * histograms[0].binSize + histograms[0].min);
+            row[2] = String((binIndex) * histograms[0].binSize + histograms[0].min);
+        }
+        // fill remaining columns
+        writeRoResultsForHistograms(histograms, binIndex, row, countAsDensity, sums);
+        rows.push(row);
+    }
+    return {
+        header: header,
+        rows: rows
+    };
+}
+exports.histograms2table = histograms2table;
+function guardForLargestBinSize(histograms) {
+    const first = histograms[0];
+    let mostBins = first.bins.length;
+    for (let i = 1; i < histograms.length; i++) {
+        (0, assert_1.guard)(histograms[i].binSize === first.binSize, `histograms must have the same bin-size, but ${histograms[i].name} has ${histograms[i].binSize} instead of ${first.binSize}`);
+        if (histograms[i].bins.length > mostBins) {
+            mostBins = histograms[i].bins.length;
+        }
+    }
+    return mostBins;
+}
+function writeRoResultsForHistograms(histograms, binIndex, row, countAsDensity, sums) {
+    for (let j = 0; j < histograms.length; j++) {
+        const bins = histograms[j].bins;
+        // does not have to be performant...
+        if (binIndex >= bins.length) {
+            row[j + 3] = '0'; /* in a histogram, 0 is the best default value for bins that are not present -- no value appeared in the corresponding bin */
+        }
+        else {
+            row[j + 3] = String(countAsDensity ? bins[binIndex] / sums[j] : bins[binIndex]);
+        }
+    }
+}
+//# sourceMappingURL=histogram.js.map