@eagleoutice/flowr 1.3.11

package/util/set.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setMinus = exports.setEquals = void 0;
+/**
+ * Given both sets, this checks if they contain the same elements.
+ */
+function setEquals(a, b) {
+    if (a.size !== b.size) {
+        return false;
+    }
+    for (const item of a) {
+        if (!b.has(item)) {
+            return false;
+        }
+    }
+    return true;
+}
+exports.setEquals = setEquals;
+/**
+ * Returns `A – B`
+ */
+function setMinus(a, b) {
+    const result = new Set();
+    for (const item of a) {
+        if (!b.has(item)) {
+            result.add(item);
+        }
+    }
+    return result;
+}
+exports.setMinus = setMinus;
+//# sourceMappingURL=set.js.map

package/util/strings.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Check if the given string starts and ends with the given letter
+ */
+export declare function startAndEndsWith(str: string, letter: string): boolean;
+/**
+ * Removes all whitespace in the given string
+ */
+export declare function withoutWhitespace(output: string): string;
+/**
+ * Find the longest common prefix in an array of strings
+ */
+export declare function longestCommonPrefix(strings: string[]): string;

package/util/strings.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.longestCommonPrefix = exports.withoutWhitespace = exports.startAndEndsWith = void 0;
+/**
+ * Check if the given string starts and ends with the given letter
+ */
+function startAndEndsWith(str, letter) {
+    return str.startsWith(letter) && str.endsWith(letter);
+}
+exports.startAndEndsWith = startAndEndsWith;
+/**
+ * Removes all whitespace in the given string
+ */
+function withoutWhitespace(output) {
+    return output.replace(/\s/g, '');
+}
+exports.withoutWhitespace = withoutWhitespace;
+/**
+ * Find the longest common prefix in an array of strings
+ */
+function longestCommonPrefix(strings) {
+    if (strings.length === 0) {
+        return '';
+    }
+    let prefix = strings[0];
+    for (const str of strings) {
+        if (prefix.length === 0) {
+            break;
+        }
+        let i = 0;
+        while (i < prefix.length && prefix[i] === str[i]) {
+            i++;
+        }
+        if (i !== prefix.length) {
+            prefix = prefix.slice(0, i);
+        }
+    }
+    return prefix;
+}
+exports.longestCommonPrefix = longestCommonPrefix;
+//# sourceMappingURL=strings.js.map

package/util/summarizer/auto-detect.d.ts

@@ -0,0 +1,2 @@
+import { SummarizerType } from './summarizer';
+export declare function detectSummarizationType(inputPath: string): Promise<SummarizerType>;

package/util/summarizer/auto-detect.js

@@ -0,0 +1,32 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectSummarizationType = void 0;
+const fs_1 = __importDefault(require("fs"));
+const log_1 = require("../log");
+const summarizer_1 = require("./statistics/summarizer");
+async function detectSummarizationType(inputPath) {
+    if (fs_1.default.statSync(inputPath).isFile()) {
+        log_1.log.info(`Detected benchmark summarization with single file ${inputPath}`);
+        return "benchmark" /* SummarizerType.Benchmark */;
+    }
+    // current heuristic: search for a tar.gz with two minus signs :D
+    const dir = await fs_1.default.promises.opendir(inputPath);
+    const thresholdInit = 60;
+    let threshold = thresholdInit;
+    for await (const dirent of dir) {
+        if (summarizer_1.statisticsFileNameRegex.test(dirent.name)) {
+            log_1.log.info(`Detected statistics summarization by file ${dirent.name} matching ${summarizer_1.statisticsFileNameRegex.source}`);
+            return "statistics" /* SummarizerType.Statistics */;
+        }
+        else if (threshold-- < 0) {
+            break;
+        }
+    }
+    log_1.log.info(`Detected benchmark summarization with no file (first ${thresholdInit}) matching ${summarizer_1.statisticsFileNameRegex.source}`);
+    return "benchmark" /* SummarizerType.Benchmark */;
+}
+exports.detectSummarizationType = detectSummarizationType;
+//# sourceMappingURL=auto-detect.js.map

package/util/summarizer/benchmark/benchmark-summarizer.js

@@ -0,0 +1,208 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarizeSummarizedMeasurement = exports.summarizeMeasurement = exports.summarizeSlicerStats = void 0;
+const tmp = __importStar(require("tmp"));
+const assert_1 = require("../../assert");
+const log_1 = require("../../log");
+const defaultmap_1 = require("../../defaultmap");
+const r_bridge_1 = require("../../../r-bridge");
+const strings_1 = require("../../strings");
+const fs_1 = __importDefault(require("fs"));
+const arrays_1 = require("../../arrays");
+const tempfile = (() => {
+    let _tempfile = undefined;
+    return () => {
+        if (_tempfile === undefined) {
+            _tempfile = tmp.fileSync({ postfix: '.R', keep: false });
+            process.on('beforeExit', () => _tempfile?.removeCallback());
+        }
+        return _tempfile;
+    };
+})();
+function safeDivPercentage(a, b) {
+    if (isNaN(a) || isNaN(b)) {
+        return undefined;
+    }
+    else if (b === 0) {
+        return a === 0 ? 0 : undefined;
+    }
+    else {
+        const result = 1 - (a / b);
+        if (isNaN(result)) {
+            log_1.log.error(`NaN for ${a} and ${b}\n`);
+            return undefined;
+        }
+        else {
+            return result;
+        }
+    }
+}
+function calculateReductionForSlice(input, dataflow, perSlice) {
+    return {
+        numberOfLines: safeDivPercentage(perSlice.lines, input.numberOfLines),
+        numberOfLinesNoAutoSelection: safeDivPercentage(perSlice.lines - perSlice.autoSelected, input.numberOfLines),
+        numberOfCharacters: safeDivPercentage(perSlice.characters, input.numberOfCharacters),
+        numberOfNonWhitespaceCharacters: safeDivPercentage(perSlice.nonWhitespaceCharacters, input.numberOfNonWhitespaceCharacters),
+        numberOfRTokens: safeDivPercentage(perSlice.tokens, input.numberOfRTokens),
+        numberOfNormalizedTokens: safeDivPercentage(perSlice.normalizedTokens, input.numberOfNormalizedTokens),
+        numberOfDataflowNodes: safeDivPercentage(perSlice.dataflowNodes, dataflow.numberOfNodes)
+    };
+}
+/**
+ * Summarizes the given stats by calculating the min, max, median, mean, and the standard deviation for each measurement.
+ * @see Slicer
+ */
+async function summarizeSlicerStats(stats, report = () => {
+}) {
+    const perSliceStats = stats.perSliceMeasurements;
+    const collect = new defaultmap_1.DefaultMap(() => []);
+    const sizeOfSliceCriteria = [];
+    const reParseShellSession = new r_bridge_1.RShell();
+    reParseShellSession.tryToInjectHomeLibPath();
+    const reductions = [];
+    let failedOutputs = 0;
+    const sliceSize = {
+        lines: [],
+        autoSelected: [],
+        characters: [],
+        nonWhitespaceCharacters: [],
+        tokens: [],
+        normalizedTokens: [],
+        dataflowNodes: []
+    };
+    let first = true;
+    let timesHitThreshold = 0;
+    for (const [criteria, perSliceStat] of perSliceStats) {
+        report(criteria, perSliceStat);
+        for (const measure of perSliceStat.measurements) {
+            collect.get(measure[0]).push(Number(measure[1]));
+        }
+        sizeOfSliceCriteria.push(perSliceStat.slicingCriteria.length);
+        timesHitThreshold += perSliceStat.timesHitThreshold > 0 ? 1 : 0;
+        const { code: output, autoSelected } = perSliceStat.reconstructedCode;
+        sliceSize.autoSelected.push(autoSelected);
+        const lines = output.split('\n').length;
+        sliceSize.lines.push(lines);
+        sliceSize.characters.push(output.length);
+        const nonWhitespace = (0, strings_1.withoutWhitespace)(output).length;
+        sliceSize.nonWhitespaceCharacters.push(nonWhitespace);
+        // reparse the output to get the number of tokens
+        try {
+            // there seem to be encoding issues, therefore, we dump to a temp file
+            fs_1.default.writeFileSync(tempfile().name, output);
+            const reParsed = await (0, r_bridge_1.retrieveNormalizedAstFromRCode)({ request: 'file', content: tempfile().name, ensurePackageInstalled: first }, reParseShellSession);
+            first = false;
+            let numberOfNormalizedTokens = 0;
+            (0, r_bridge_1.visitAst)(reParsed.ast, _ => {
+                numberOfNormalizedTokens++;
+                return false;
+            });
+            sliceSize.normalizedTokens.push(numberOfNormalizedTokens);
+            const numberOfRTokens = await (0, r_bridge_1.retrieveNumberOfRTokensOfLastParse)(reParseShellSession);
+            sliceSize.tokens.push(numberOfRTokens);
+            reductions.push(calculateReductionForSlice(stats.input, stats.dataflow, {
+                lines: lines,
+                characters: output.length,
+                nonWhitespaceCharacters: nonWhitespace,
+                autoSelected: autoSelected,
+                tokens: numberOfRTokens,
+                normalizedTokens: numberOfNormalizedTokens,
+                dataflowNodes: perSliceStat.numberOfDataflowNodesSliced
+            }));
+        }
+        catch (e) {
+            console.error(`    ! Failed to re-parse the output of the slicer for ${JSON.stringify(criteria)}`); //, e
+            console.error(`      Code: ${output}\n`);
+            failedOutputs++;
+        }
+        sliceSize.dataflowNodes.push(perSliceStat.numberOfDataflowNodesSliced);
+    }
+    // summarize all measurements:
+    const summarized = new Map();
+    for (const [criterion, measurements] of collect.entries()) {
+        summarized.set(criterion, summarizeMeasurement(measurements));
+    }
+    reParseShellSession.close();
+    return {
+        ...stats,
+        perSliceMeasurements: {
+            numberOfSlices: perSliceStats.size,
+            sliceCriteriaSizes: summarizeMeasurement(sizeOfSliceCriteria),
+            measurements: summarized,
+            failedToRepParse: failedOutputs,
+            timesHitThreshold,
+            reduction: {
+                numberOfLines: summarizeMeasurement(reductions.map(r => r.numberOfLines).filter(assert_1.isNotUndefined)),
+                numberOfLinesNoAutoSelection: summarizeMeasurement(reductions.map(r => r.numberOfLinesNoAutoSelection).filter(assert_1.isNotUndefined)),
+                numberOfCharacters: summarizeMeasurement(reductions.map(r => r.numberOfCharacters).filter(assert_1.isNotUndefined)),
+                numberOfNonWhitespaceCharacters: summarizeMeasurement(reductions.map(r => r.numberOfNonWhitespaceCharacters).filter(assert_1.isNotUndefined)),
+                numberOfRTokens: summarizeMeasurement(reductions.map(r => r.numberOfRTokens).filter(assert_1.isNotUndefined)),
+                numberOfNormalizedTokens: summarizeMeasurement(reductions.map(r => r.numberOfNormalizedTokens).filter(assert_1.isNotUndefined)),
+                numberOfDataflowNodes: summarizeMeasurement(reductions.map(r => r.numberOfDataflowNodes).filter(assert_1.isNotUndefined))
+            },
+            sliceSize: {
+                lines: summarizeMeasurement(sliceSize.lines),
+                characters: summarizeMeasurement(sliceSize.characters),
+                nonWhitespaceCharacters: summarizeMeasurement(sliceSize.nonWhitespaceCharacters),
+                autoSelected: summarizeMeasurement(sliceSize.autoSelected),
+                tokens: summarizeMeasurement(sliceSize.tokens),
+                normalizedTokens: summarizeMeasurement(sliceSize.normalizedTokens),
+                dataflowNodes: summarizeMeasurement(sliceSize.dataflowNodes)
+            }
+        }
+    };
+}
+exports.summarizeSlicerStats = summarizeSlicerStats;
+function summarizeMeasurement(data, totalNumberOfDataPoints) {
+    // just to avoid in-place modification
+    const sorted = [...data].sort((a, b) => a - b);
+    const min = sorted[0];
+    const max = sorted[sorted.length - 1];
+    const median = sorted[Math.floor(sorted.length / 2)];
+    const total = (0, arrays_1.sum)(sorted);
+    const length = totalNumberOfDataPoints ?? sorted.length;
+    const mean = total / length;
+    // sqrt(sum(x-mean)^2 / n)
+    const std = Math.sqrt(sorted.map(x => (x - mean) ** 2).reduce((a, b) => a + b, 0) / length);
+    return { min, max, median, mean, std, total };
+}
+exports.summarizeMeasurement = summarizeMeasurement;
+function summarizeSummarizedMeasurement(data) {
+    const min = data.map(d => d.min).filter(assert_1.isNotUndefined).reduce((a, b) => Math.min(a, b), Infinity);
+    const max = data.map(d => d.max).filter(assert_1.isNotUndefined).reduce((a, b) => Math.max(a, b), -Infinity);
+    // get most average
+    const median = data.map(d => d.median).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    const mean = data.map(d => d.mean).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length;
+    // Method 1 of https://www.statology.org/averaging-standard-deviations/
+    const std = Math.sqrt(data.map(d => d.std ** 2).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0) / data.length);
+    const total = data.map(d => d.total).filter(assert_1.isNotUndefined).reduce((a, b) => a + b, 0);
+    return { min, max, median, mean, std, total };
+}
+exports.summarizeSummarizedMeasurement = summarizeSummarizedMeasurement;
+//# sourceMappingURL=benchmark-summarizer.js.map

package/util/summarizer/benchmark/data.d.ts

@@ -0,0 +1,66 @@
+import { CommonSlicerMeasurements, PerSliceMeasurements, SlicerStats, SlicerStatsDataflow, SlicerStatsInput } from '../../../benchmark';
+export interface SummarizedMeasurement<T = number> {
+    min: T;
+    max: T;
+    median: T;
+    /** total may be useless for some measurements, especially if they are weighted before (it is just the sum...)*/
+    total: T;
+    /** average */
+    mean: number;
+    /** standard deviation */
+    std: number;
+}
+export declare function summarizedMeasurement2Csv(a: SummarizedMeasurement): string;
+export declare function summarizedMeasurement2CsvHeader(prefix?: string): string;
+export interface SliceSizeCollection {
+    lines: number[];
+    characters: number[];
+    nonWhitespaceCharacters: number[];
+    /** like library statements during reconstruction */
+    autoSelected: number[];
+    dataflowNodes: number[];
+    tokens: number[];
+    normalizedTokens: number[];
+}
+/**
+ * @see SlicerStats
+ * @see summarizeSlicerStats
+ */
+export type SummarizedSlicerStats = {
+    perSliceMeasurements: SummarizedPerSliceStats;
+} & Omit<SlicerStats, 'perSliceMeasurements'>;
+export interface Reduction<T = number> {
+    numberOfLines: T;
+    numberOfLinesNoAutoSelection: T;
+    numberOfCharacters: T;
+    numberOfNonWhitespaceCharacters: T;
+    numberOfRTokens: T;
+    numberOfNormalizedTokens: T;
+    numberOfDataflowNodes: T;
+}
+export interface SummarizedPerSliceStats {
+    /** number of total slicing calls */
+    numberOfSlices: number;
+    /** statistics on the used slicing criteria (number of ids within criteria etc.) */
+    sliceCriteriaSizes: SummarizedMeasurement;
+    measurements: Map<PerSliceMeasurements, SummarizedMeasurement>;
+    reduction: Reduction<SummarizedMeasurement>;
+    failedToRepParse: number;
+    timesHitThreshold: number;
+    sliceSize: {
+        [K in keyof SliceSizeCollection]: SummarizedMeasurement;
+    };
+}
+export interface UltimateSlicerStats {
+    totalRequests: number;
+    totalSlices: number;
+    commonMeasurements: Map<CommonSlicerMeasurements, SummarizedMeasurement>;
+    perSliceMeasurements: Map<PerSliceMeasurements, SummarizedMeasurement>;
+    /** sum */
+    failedToRepParse: number;
+    /** sum */
+    timesHitThreshold: number;
+    reduction: Reduction<SummarizedMeasurement>;
+    input: SlicerStatsInput<SummarizedMeasurement>;
+    dataflow: SlicerStatsDataflow<SummarizedMeasurement>;
+}

package/util/summarizer/benchmark/data.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.summarizedMeasurement2CsvHeader = exports.summarizedMeasurement2Csv = void 0;
+function summarizedMeasurement2Csv(a) {
+    return `${a.min},${a.max},${a.median},${a.mean},${a.std},${a.total}`;
+}
+exports.summarizedMeasurement2Csv = summarizedMeasurement2Csv;
+const summarizedKeys = ['min', 'max', 'median', 'mean', 'std', 'total'];
+function summarizedMeasurement2CsvHeader(prefix) {
+    return summarizedKeys.map(k => prefix ? `${prefix}-${k}` : k).join(',');
+}
+exports.summarizedMeasurement2CsvHeader = summarizedMeasurement2CsvHeader;
+//# sourceMappingURL=data.js.map

package/util/summarizer/benchmark/first-phase/input.d.ts

@@ -0,0 +1,2 @@
+/// <reference types="node" />
+export declare function processNestMeasurement(line: Buffer, counter: number, summarizedText: string, outputPath: string): Promise<void>;

package/util/summarizer/benchmark/first-phase/input.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processNestMeasurement = void 0;
+const benchmark_1 = require("../../../../benchmark");
+const assert_1 = require("../../../assert");
+const statistics_1 = require("../../../../statistics");
+const fs_1 = __importDefault(require("fs"));
+const json_1 = require("../../../json");
+async function processNestMeasurement(line, counter, summarizedText, outputPath) {
+    let got = JSON.parse(line.toString());
+    console.log(`[${counter}] Summarize for ${got.filename}`);
+    // now we have to recover the maps and bigints :C
+    got = {
+        filename: got.filename,
+        stats: {
+            input: got.stats.input,
+            request: got.stats.request,
+            dataflow: got.stats.dataflow,
+            commonMeasurements: new Map(got.stats.commonMeasurements
+                .map(([k, v]) => {
+                (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
+                return [k, BigInt(v.slice(0, -1))];
+            })),
+            perSliceMeasurements: new Map(got.stats.perSliceMeasurements
+                .map(([k, v]) => mapPerSliceStats(k, v)))
+        }
+    };
+    const totalSlices = got.stats.perSliceMeasurements.size;
+    console.log(`Summarizing ${totalSlices} slices...`);
+    let atSliceNumber = 0;
+    const summarized = await (0, benchmark_1.summarizeSlicerStats)(got.stats, (criterion, stats) => {
+        console.log(`${statistics_1.escape}1F${statistics_1.escape}1G${statistics_1.escape}2K    [${++atSliceNumber}/${totalSlices}] Summarizing ${JSON.stringify(criterion)} (reconstructed has ${stats.reconstructedCode.code.length} characters)`);
+    });
+    console.log(`    - Append raw summary to ${outputPath}`);
+    fs_1.default.appendFileSync(outputPath, `${JSON.stringify({
+        filename: got.filename,
+        summarize: summarized
+    }, json_1.jsonReplacer)}\n`);
+    console.log(`    - Append textual summary to ${summarizedText}`);
+    fs_1.default.appendFileSync(summarizedText, `${(0, benchmark_1.stats2string)(summarized)}\n`);
+}
+exports.processNestMeasurement = processNestMeasurement;
+function mapPerSliceStats(k, v) {
+    return [k, {
+            reconstructedCode: v.reconstructedCode,
+            slicingCriteria: v.slicingCriteria,
+            timesHitThreshold: v.timesHitThreshold,
+            measurements: new Map(v.measurements
+                .map(([k, v]) => {
+                (0, assert_1.guard)(v.endsWith('n'), 'Expected a bigint');
+                return [k, BigInt(v.slice(0, -1))];
+            })),
+            numberOfDataflowNodesSliced: v.numberOfDataflowNodesSliced
+        }];
+}
+//# sourceMappingURL=input.js.map

package/util/summarizer/benchmark/first-phase/process.d.ts

@@ -0,0 +1,10 @@
+import { SummarizedMeasurement, SummarizedSlicerStats } from '../data';
+import { PerSliceStats, SlicerStats } from '../../../../benchmark';
+import { SlicingCriteria } from '../../../../slicing';
+/**
+ * Summarizes the given stats by calculating the min, max, median, mean, and the standard deviation for each measurement.
+ * @see Slicer
+ */
+export declare function summarizeSlicerStats(stats: SlicerStats, report?: (criteria: SlicingCriteria, stats: PerSliceStats) => void): Promise<Readonly<SummarizedSlicerStats>>;
+export declare function summarizeMeasurement(data: number[], totalNumberOfDataPoints?: number): SummarizedMeasurement;
+export declare function summarizeSummarizedMeasurement(data: SummarizedMeasurement[]): SummarizedMeasurement;