@eagleoutice/flowr 1.3.11

package/statistics/output/file-provider.js

@@ -0,0 +1,97 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StatisticFileProvider = exports.defaultStatisticsFileSuffix = exports.DummyAppendProvider = void 0;
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const assert_1 = require("../../util/assert");
+const log_1 = require("../../util/log");
+class DummyAppendProvider {
+    map;
+    /**
+     * If you pass a map the dummy will log all append calls to the map, using the feature name and the appendage type as keys
+     *
+     * @param map - The map to log to
+     */
+    constructor(map = undefined) {
+        this.map = map;
+    }
+    append(name, fn, content) {
+        if (log_1.log.settings.minLevel >= 1 /* LogLevel.Trace */) {
+            log_1.log.trace(`DummyAppendProvider: ${name} ${String(fn)} ${content}`);
+        }
+        if (this.map) {
+            const fnMap = this.map.get(name);
+            const contentArr = content.split('\n');
+            if (fnMap) {
+                const contentList = fnMap.get(fn);
+                if (contentList) {
+                    contentList.push(...contentArr);
+                }
+                else {
+                    // just in case the map already had some entries
+                    fnMap.set(fn, contentArr);
+                }
+            }
+            else {
+                this.map.set(name, new Map([[fn, contentArr]]));
+            }
+        }
+    }
+}
+exports.DummyAppendProvider = DummyAppendProvider;
+exports.defaultStatisticsFileSuffix = '.txt';
+/**
+ * Provides cached open connections for all files to connect.
+ * allowing to append to the same file often.
+ * <p>
+ * While we could simply reopen these files, it is safer/more performant to keep the connection open.
+ */
+class StatisticFileProvider {
+    statisticsDirectory;
+    connections = new Map();
+    constructor(statisticsDirectory) {
+        (0, assert_1.guard)(statisticsDirectory !== undefined, 'Please supply an output directory!');
+        this.statisticsDirectory = statisticsDirectory;
+        // just to make sure, that they are closed
+        process.on('beforeExit', () => {
+            this.connections.forEach(fd => {
+                fs_1.default.closeSync(fd);
+            });
+        });
+    }
+    /**
+   * @param name - the name of the feature {@link Feature#name}
+   * @param fn - the name of the feature-aspect to record
+   */
+    statisticsFile(name, fn) {
+        return path_1.default.join(this.statisticsDirectory, name, `${fn}${exports.defaultStatisticsFileSuffix}`);
+    }
+    /**
+   * Append the given content to the information for a feature of the given name and function.
+   */
+    append(name, fn, content) {
+        const descriptor = this.getHandle(name, String(fn));
+        fs_1.default.appendFileSync(descriptor, content + '\n', 'utf8');
+    }
+    getHandle(name, fn) {
+        const key = `${name}-${fn}`;
+        const fileHandle = this.connections.get(key);
+        if (fileHandle) {
+            return fileHandle;
+        }
+        // open the connection and ensure the location
+        const filepath = this.statisticsFile(name, String(fn));
+        const dirpath = path_1.default.dirname(filepath);
+        if (!fs_1.default.existsSync(dirpath)) {
+            fs_1.default.mkdirSync(dirpath, { recursive: true });
+        }
+        const fileDescriptor = fs_1.default.openSync(filepath, 'a');
+        this.connections.set(key, fileDescriptor);
+        return fileDescriptor;
+    }
+}
+exports.StatisticFileProvider = StatisticFileProvider;
+//# sourceMappingURL=file-provider.js.map

package/statistics/output/index.d.ts

@@ -0,0 +1,4 @@
+export * from './print-stats';
+export * from './file-provider';
+export * from './statistics-file';
+export * from './ansi';

package/statistics/output/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./print-stats"), exports);
+__exportStar(require("./file-provider"), exports);
+__exportStar(require("./statistics-file"), exports);
+__exportStar(require("./ansi"), exports);
+//# sourceMappingURL=index.js.map

package/statistics/output/print-stats.d.ts

@@ -0,0 +1,17 @@
+import { FeatureKey, FeatureStatistics } from '../features';
+import { MetaStatistics } from '../meta-statistics';
+interface MinMaxAvgMedian {
+    sum: number;
+    min: number;
+    max: number;
+    avg: number;
+    median: number;
+}
+export declare function minMaxAvgAndMedian(data: number[]): MinMaxAvgMedian;
+export declare function statsString(data: MinMaxAvgMedian, suffix?: string): string;
+export declare function printFeatureStatistics(statistics: {
+    features: FeatureStatistics;
+    meta: MetaStatistics;
+}, features?: 'all' | Set<FeatureKey>): void;
+export declare function printFeatureStatisticsEntry(info: Record<string, unknown>): void;
+export {};

package/statistics/output/print-stats.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.printFeatureStatisticsEntry = exports.printFeatureStatistics = exports.statsString = exports.minMaxAvgAndMedian = void 0;
+const features_1 = require("../features");
+const ansi_1 = require("./ansi");
+const json_1 = require("../../util/json");
+function minMaxAvgAndMedian(data) {
+    data = data.sort((a, b) => a - b);
+    const sum = data.reduce((a, b) => a + b, 0);
+    return {
+        sum,
+        min: data[0],
+        max: data[data.length - 1],
+        avg: sum / data.length,
+        median: data[Math.floor(data.length / 2)]
+    };
+}
+exports.minMaxAvgAndMedian = minMaxAvgAndMedian;
+function formatStatNumber(num) {
+    return num === undefined ? '<?>' : Number(num.toFixed(3)).toLocaleString();
+}
+function statsString(data, suffix = '') {
+    return `[${formatStatNumber(data.min)}${suffix} .. ${formatStatNumber(data.max)}${suffix}] (avg: ${formatStatNumber(data.avg)}${suffix}, median: ${formatStatNumber(data.median)}${suffix})`;
+}
+exports.statsString = statsString;
+function printFeatureStatistics(statistics, features = 'all') {
+    for (const feature of Object.keys(statistics.features)) {
+        if (features !== 'all' && !features.has(feature)) {
+            continue;
+        }
+        const meta = features_1.ALL_FEATURES[feature];
+        console.log(`\n\n-----${meta.name}-------------`);
+        console.log(ansi_1.formatter.format(meta.description, { color: 7 /* Colors.White */, effect: ansi_1.ColorEffect.Foreground }));
+        printFeatureStatisticsEntry(statistics.features[feature]);
+        console.log('\n\n');
+    }
+    const linesPerFile = minMaxAvgAndMedian(statistics.meta.lines.map(l => l.length));
+    const lineLengths = minMaxAvgAndMedian(statistics.meta.lines.flat());
+    const processingTimesPerFile = minMaxAvgAndMedian(statistics.meta.processingTimeMs);
+    console.log(`processed ${statistics.meta.successfulParsed} files (skipped ${statistics.meta.failedRequests.length} due to errors):
+\ttotal processing time: ${processingTimesPerFile.sum} ms
+\t\tprocessing time range: ${statsString(processingTimesPerFile, ' ms')}
+\ttotal number of lines: ${lineLengths.sum}
+\t\tline range: ${statsString(linesPerFile)}
+\t\tline length range: ${statsString(lineLengths, ' chars')}
+  `);
+}
+exports.printFeatureStatistics = printFeatureStatistics;
+const pad = 3;
+function printFeatureStatisticsEntry(info) {
+    let longestKey = 0;
+    let longestValue = 0;
+    const out = new Map();
+    for (const [key, value] of Object.entries(info)) {
+        if (key.length > longestKey) {
+            longestKey = key.length;
+        }
+        const valueString = JSON.stringify(value, json_1.jsonReplacer);
+        out.set(key, valueString);
+        if (valueString.length > longestValue) {
+            longestValue = valueString.length;
+        }
+    }
+    for (const [key, value] of out.entries()) {
+        console.log(`${key.padEnd(longestKey + pad)} ${value.padStart(longestValue)}`);
+    }
+}
+exports.printFeatureStatisticsEntry = printFeatureStatisticsEntry;
+//# sourceMappingURL=print-stats.js.map

package/statistics/output/statistics-file.d.ts

@@ -0,0 +1,37 @@
+import { DummyAppendMemoryMap, StatisticAppendProvider } from './file-provider';
+/**
+ * Requires source information to be attached on parsing!
+ * <p>
+ * Returns the content of the node (i.e., the text content excluding the children)
+ */
+export declare function extractNodeContent(node: Node): string;
+/** by default, we do not write to anything */
+export declare let statisticsFileProvider: StatisticAppendProvider;
+/**
+ * Make the statistics write to a given output directory.
+ */
+export declare function initFileProvider(outputDirectory: string): void;
+/**
+ * Either ignore the statistics or write them to a given map (e.g., for testing).
+ *
+ * @param map - The map to write to, will not persist calls if no map is given
+ */
+export declare function initDummyFileProvider(map?: DummyAppendMemoryMap): void;
+/**
+ * Format used to dump each entry of a feature during collection.
+ */
+export type StatisticsOutputFormat<T = string> = [
+    /** the collected value (like the assignment operator lexeme, ...) */
+    value: T,
+    /** the context of the information retrieval (e.g. the name of the file that contained the R source code) */
+    context: string | undefined
+];
+/**
+ * Append the content of all nodes to the storage file for the given feature
+ * @param name    - The name of the feature {@link Feature#name}
+ * @param fn      - The name of the feature-aspect to record
+ * @param nodes   - The nodes to append, you may pass already transformed string contents
+ * @param context - The context of the information retrieval (e.g. the name of the file that contained the R source code)
+ * @param unique  - Should duplicate entries be removed on addition
+ */
+export declare function appendStatisticsFile<T>(name: string, fn: keyof T, nodes: string[] | Node[] | object[], context: string | undefined, unique?: boolean): void;

package/statistics/output/statistics-file.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendStatisticsFile = exports.initDummyFileProvider = exports.initFileProvider = exports.statisticsFileProvider = exports.extractNodeContent = void 0;
+const file_provider_1 = require("./file-provider");
+const log_1 = require("../../util/log");
+/**
+ * Requires source information to be attached on parsing!
+ * <p>
+ * Returns the content of the node (i.e., the text content excluding the children)
+ */
+function extractNodeContent(node) {
+    let result = node.textContent;
+    if (node.hasChildNodes()) {
+        const firstChild = node.childNodes.item(0);
+        if (firstChild.nodeType === 3 /* text node */) {
+            result = firstChild.textContent;
+        }
+    }
+    return result?.trim()?.replaceAll('\n', '\\n') ?? '<unknown>';
+}
+exports.extractNodeContent = extractNodeContent;
+initDummyFileProvider();
+/**
+ * Make the statistics write to a given output directory.
+ */
+function initFileProvider(outputDirectory) {
+    log_1.log.debug(`Initializing file provider for output directory ${outputDirectory}`);
+    exports.statisticsFileProvider = new file_provider_1.StatisticFileProvider(outputDirectory);
+}
+exports.initFileProvider = initFileProvider;
+/**
+ * Either ignore the statistics or write them to a given map (e.g., for testing).
+ *
+ * @param map - The map to write to, will not persist calls if no map is given
+ */
+function initDummyFileProvider(map) {
+    exports.statisticsFileProvider = new file_provider_1.DummyAppendProvider(map);
+}
+exports.initDummyFileProvider = initDummyFileProvider;
+/**
+ * Append the content of all nodes to the storage file for the given feature
+ * @param name    - The name of the feature {@link Feature#name}
+ * @param fn      - The name of the feature-aspect to record
+ * @param nodes   - The nodes to append, you may pass already transformed string contents
+ * @param context - The context of the information retrieval (e.g. the name of the file that contained the R source code)
+ * @param unique  - Should duplicate entries be removed on addition
+ */
+function appendStatisticsFile(name, fn, nodes, context, unique = false) {
+    if (nodes.length === 0) {
+        return;
+    }
+    let values;
+    if (typeof nodes[0] === 'string') {
+        values = nodes;
+    }
+    else if ('nodeType' in nodes[0]) {
+        values = nodes.map(extractNodeContent);
+    }
+    else {
+        values = nodes;
+    }
+    if (unique) {
+        values = [...new Set(values)];
+    }
+    values = values.map(value => JSON.stringify(context === undefined ? [value] : [value, context]));
+    exports.statisticsFileProvider.append(name, fn, values.join('\n'));
+}
+exports.appendStatisticsFile = appendStatisticsFile;
+//# sourceMappingURL=statistics-file.js.map

package/statistics/statistics.d.ts

@@ -0,0 +1,24 @@
+import { RParseRequestFromFile, RParseRequestFromText, RShell } from '../r-bridge';
+import { FeatureSelection, FeatureStatistics } from './features';
+import { MetaStatistics } from './meta-statistics';
+import { StepResults } from '../core';
+/**
+ * By default, {@link extractUsageStatistics} requires a generator, but sometimes you already know all the files
+ * that you want to process. This function simply reps your requests as a generator.
+ */
+export declare function staticRequests(...requests: (RParseRequestFromText | RParseRequestFromFile)[]): AsyncGenerator<RParseRequestFromText | RParseRequestFromFile>;
+/**
+ * Extract all wanted statistic information from a set of requests using the presented R session.
+ *
+ * @param shell     - The R session to use
+ * @param onRequest - A callback that is called at the beginning of each request, this may be used to debug the requests.
+ * @param features  - The features to extract (see {@link allFeatureNames}).
+ * @param requests  - The requests to extract the features from. May generate them on demand (e.g., by traversing a folder).
+ * 										If your request is statically known, you can use {@link staticRequests} to create this generator.
+ * @param rootPath  - The root path to the project, this is used to relativize the file paths in the statistics.
+ */
+export declare function extractUsageStatistics<T extends RParseRequestFromText | RParseRequestFromFile>(shell: RShell, onRequest: (request: T) => void, features: FeatureSelection, requests: AsyncGenerator<T>, rootPath?: string): Promise<{
+    features: FeatureStatistics;
+    meta: MetaStatistics;
+    outputs: Map<T, StepResults<'dataflow'>>;
+}>;

package/statistics/statistics.js

@@ -0,0 +1,109 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractUsageStatistics = exports.staticRequests = void 0;
+const features_1 = require("./features");
+const xmldom_1 = require("@xmldom/xmldom");
+const fs_1 = __importDefault(require("fs"));
+const log_1 = require("../util/log");
+const meta_statistics_1 = require("./meta-statistics");
+const core_1 = require("../core");
+const json_1 = require("../util/json");
+/**
+ * By default, {@link extractUsageStatistics} requires a generator, but sometimes you already know all the files
+ * that you want to process. This function simply reps your requests as a generator.
+ */
+function staticRequests(...requests) {
+    // eslint-disable-next-line @typescript-eslint/require-await
+    return async function* () {
+        for (const request of requests) {
+            yield request;
+        }
+    }();
+}
+exports.staticRequests = staticRequests;
+/**
+ * Extract all wanted statistic information from a set of requests using the presented R session.
+ *
+ * @param shell     - The R session to use
+ * @param onRequest - A callback that is called at the beginning of each request, this may be used to debug the requests.
+ * @param features  - The features to extract (see {@link allFeatureNames}).
+ * @param requests  - The requests to extract the features from. May generate them on demand (e.g., by traversing a folder).
+ * 										If your request is statically known, you can use {@link staticRequests} to create this generator.
+ * @param rootPath  - The root path to the project, this is used to relativize the file paths in the statistics.
+ */
+async function extractUsageStatistics(shell, onRequest, features, requests, rootPath) {
+    let result = initializeFeatureStatistics();
+    const meta = (0, meta_statistics_1.initialMetaStatistics)();
+    let first = true;
+    const outputs = new Map();
+    for await (const request of requests) {
+        onRequest(request);
+        const start = performance.now();
+        const suffix = request.request === 'file' ? request.content.replace(new RegExp('^' + (rootPath ?? '')), '') : undefined;
+        try {
+            let output;
+            ({ stats: result, output } = await extractSingle(result, shell, {
+                ...request,
+                ensurePackageInstalled: first
+            }, features, suffix));
+            outputs.set(request, output);
+            processMetaOnSuccessful(meta, request);
+            meta.numberOfNormalizedNodes.push(output.normalize.idMap.size);
+            first = false;
+        }
+        catch (e) {
+            log_1.log.error('for request: ', request, e);
+            processMetaOnUnsuccessful(meta, request);
+        }
+        meta.processingTimeMs.push(performance.now() - start);
+    }
+    return { features: result, meta, outputs };
+}
+exports.extractUsageStatistics = extractUsageStatistics;
+function initializeFeatureStatistics() {
+    const result = {};
+    for (const key of features_1.allFeatureNames) {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        result[key] = JSON.parse(JSON.stringify(features_1.ALL_FEATURES[key].initialValue, json_1.jsonReplacer), json_1.jsonRetriever);
+    }
+    return result;
+}
+function processMetaOnUnsuccessful(meta, request) {
+    meta.failedRequests.push(request);
+}
+function processMetaOnSuccessful(meta, request) {
+    meta.successfulParsed++;
+    if (request.request === 'text') {
+        meta.lines.push(request.content.split('\n').map(l => l.length));
+    }
+    else {
+        meta.lines.push(fs_1.default.readFileSync(request.content, 'utf-8').split('\n').map(l => l.length));
+    }
+}
+const parser = new xmldom_1.DOMParser();
+async function extractSingle(result, shell, request, features, suffixFilePath) {
+    const slicerOutput = await new core_1.SteppingSlicer({
+        stepOfInterest: 'dataflow',
+        request, shell
+    }).allRemainingSteps();
+    // await retrieveXmlFromRCode(from, shell)
+    const doc = parser.parseFromString(slicerOutput.parse, 'text/xml');
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    for (const [key, feature] of Object.entries(features_1.ALL_FEATURES)) {
+        if (features !== 'all' && !features.has(key)) {
+            continue;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        result[key] = feature.process(result[key], {
+            parsedRAst: doc,
+            dataflow: slicerOutput.dataflow,
+            normalizedRAst: slicerOutput.normalize,
+            filepath: suffixFilePath
+        });
+    }
+    return { stats: result, output: slicerOutput };
+}
+//# sourceMappingURL=statistics.js.map

package/util/args.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Just to avoid another library for splitting arguments, we use this module to provide what we need.
+ *
+ * @module
+ */
+/**
+ * This splits an input string on the given split string (e.g., ` `) but checks if the string is quoted or escaped.
+ *
+ * Given an input string like `a "b c" d` with a space character as split this splits the arguments similar to common shell interpreters (i.e., `a`, `b c`, and `d`).
+ *
+ * @param inputString - The string to split
+ * @param split       - The **single** character to split on (can not be the backslash or quote)
+ */
+export declare function splitAtEscapeSensitive(inputString: string, split?: string): string[];

package/util/args.js

@@ -0,0 +1,67 @@
+"use strict";
+/**
+ * Just to avoid another library for splitting arguments, we use this module to provide what we need.
+ *
+ * @module
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitAtEscapeSensitive = void 0;
+/**
+ * This splits an input string on the given split string (e.g., ` `) but checks if the string is quoted or escaped.
+ *
+ * Given an input string like `a "b c" d` with a space character as split this splits the arguments similar to common shell interpreters (i.e., `a`, `b c`, and `d`).
+ *
+ * @param inputString - The string to split
+ * @param split       - The **single** character to split on (can not be the backslash or quote)
+ */
+function splitAtEscapeSensitive(inputString, split = ' ') {
+    const args = [];
+    let current = '';
+    let inQuotes = false;
+    let escaped = false;
+    for (const c of inputString) {
+        if (escaped) {
+            escaped = false;
+            switch (c) {
+                case 'n':
+                    current += '\n';
+                    break;
+                case 't':
+                    current += '\t';
+                    break;
+                case 'r':
+                    current += '\r';
+                    break;
+                case 'v':
+                    current += '\v';
+                    break;
+                case 'f':
+                    current += '\f';
+                    break;
+                case 'b':
+                    current += '\b';
+                    break;
+                default: current += c;
+            }
+        }
+        else if (c === split && !inQuotes && current !== '') {
+            args.push(current);
+            current = '';
+        }
+        else if (c === '"' || c === "'") {
+            inQuotes = !inQuotes;
+        }
+        else if (c === '\\') {
+            escaped = true;
+        }
+        else {
+            current += c;
+        }
+    }
+    if (current !== '') {
+        args.push(current);
+    }
+    return args;
+}
+exports.splitAtEscapeSensitive = splitAtEscapeSensitive;
+//# sourceMappingURL=args.js.map

package/util/arrays.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Splits the array every time the given predicate fires.
+ * The element the split appears on will not be included!
+ *
+ * @example with this we can split on all empty strings:
+ * ```
+ * splitArrayOn(['a', '', 'b', '', '', 'c'], elem => elem === '')
+ * // => [['a'], ['b'], [], ['c']]
+ * ```
+ * @example if the last element satisfies the predicate, this will add an empty array element
+ * ```
+ * splitArrayOn([1,2,3], elem => true)
+ * // => [[], [], [], []]
+ * ```
+ */
+export declare function splitArrayOn<T>(arr: T[], predicate: (elem: T) => boolean): T[][];
+/**
+ * Generate all permutations of the given array using Heap's algorithm (with its non-recursive variant).
+ *
+ * @param arr - The array to permute
+ * @see getUniqueCombinationsOfSize
+ */
+export declare function allPermutations<T>(arr: T[]): Generator<T[], void, void>;
+export declare function partition<T>(arr: T[], predicate: (elem: T) => boolean): [T[], T[]];
+/**
+ * Generate all unique combinations of the array with the given size.
+ * In other words, given `[a,b,c]`, as well as `minSize=2` and `maxSize=2`, this will generate `[a,b]`, `[a,c]` and `[b,c]`,
+ * but not, e.g., `[a,a]` or `[b,a]`.
+ *
+ * If `minSize!=maxSize`, the result is guaranteed to be sorted by size.
+ *
+ * @param array   - The array to generate combinations from
+ * @param minSize - The inclusive minimum size of the combinations, must be at least `0` and at most `maxSize`
+ * @param maxSize - The inclusive maximum size of the combinations, must be at least `minSize` and at most `array.length`
+ */
+export declare function getUniqueCombinationsOfSize<T>(array: T[], minSize?: number, maxSize?: number): Generator<T[], void, void>;
+/**
+ * Returns the sum of all elements in the given array
+ */
+export declare function sum(arr: number[]): number;
+/**
+ * Converts an array into a bag data-structure (in the form of a map mapping the entries/keys to their counts)
+ */
+export declare function array2bag<T>(arr: T[]): Map<T, number>;