@eagleoutice/flowr 1.5.2 → 2.0.1

package/statistics/features/supported/used-functions/post-process.js

@@ -0,0 +1,178 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.postProcess = void 0;
+const common_syntax_probability_1 = require("../../common-syntax-probability");
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const used_functions_1 = require("./used-functions");
+const summarizer_1 = require("../../../../util/summarizer");
+const files_1 = require("../../../../util/files");
+const time_1 = require("../../../../util/time");
+const numbers_1 = require("../../../../util/numbers");
+function retrieveFunctionCallInformation(featureRoot, info, config, outputPath) {
+    // each number[][] contains a 'number[]' per file
+    /**
+     * maps fn-name (including namespace) to number of arguments and their location (the number of elements in the array give the number of total call)
+     * we use tuples to reduce the memory!
+     * A function that is defined within the file is _always_ decorated with the filename (as second array element)!
+     */
+    const functionsPerFile = new Map();
+    const importantFunctions = new Map(['parse',
+        'eval',
+        'deparse',
+        'quote',
+        'body',
+        'formals',
+        'body',
+        'environment',
+        'new.env',
+        'assign',
+        'get',
+        'setGeneric',
+        'R6Class'].map(name => [name, fs_1.default.createWriteStream(path_1.default.join(outputPath, `${name}.csv`))]));
+    for (const [, value] of importantFunctions) {
+        value.write('filepath,location,namespace,inspected by,classification,notes\n');
+    }
+    // we collect only `all-calls`
+    (0, files_1.readLineByLineSync)(path_1.default.join(featureRoot, `${used_functions_1.AllCallsFileBase}.txt`), (line, lineNumber) => processNextLine(functionsPerFile, lineNumber, info, JSON.parse(String(line)), config, importantFunctions));
+    for (const [, value] of importantFunctions) {
+        value.close();
+    }
+    importantFunctions.clear();
+    console.log(`    [${(0, time_1.date2string)(new Date())}] Used functions process completed, start to write out function info`);
+    const fnOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-calls.csv'));
+    const prefixes = ['total', 'args', 'line-frac'];
+    const others = prefixes.flatMap(summarizer_1.summarizedMeasurement2CsvHeader).join(',');
+    fnOutStream.write(`function,unique-projects,unique-files,${others}\n`);
+    for (const [key, [uniqueProjects, uniqueFiles, total, args, lineFrac]] of functionsPerFile.entries()) {
+        const totalSum = (0, summarizer_1.summarizeMeasurement)(total.flat(), info.size);
+        const argsSum = (0, summarizer_1.summarizeMeasurement)(args.flat(), info.size);
+        const lineFracSum = (0, summarizer_1.summarizeMeasurement)(lineFrac.flat());
+        // we write in csv style :), we escape the key in case it contains commas (with filenames)etc.
+        fnOutStream.write(`${JSON.stringify(key ?? 'unknown')},${uniqueProjects.size},${uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(totalSum)},${(0, summarizer_1.summarizedMeasurement2Csv)(argsSum)},${(0, summarizer_1.summarizedMeasurement2Csv)(lineFracSum)}\n`);
+    }
+    fnOutStream.close();
+}
+function writeFunctionCallsMetaInformationToCsv(outputPath, data) {
+    const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-calls-meta.csv'));
+    out.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    out.write(`average-call,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.averageCall.flat()))}\n`);
+    out.write(`nested-calls,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.nestedCalls.flat()))}\n`);
+    out.write(`deepest-nesting,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.deepestNesting.flat()))}\n`);
+    out.write(`empty-args,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.emptyArgs.flat()))}\n`);
+    out.write(`unnamed-calls,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.unnamedCalls.flat()))}\n`);
+    out.close();
+}
+function retrieveFunctionCallMetaInformation(info, outputPath) {
+    const data = {
+        averageCall: [],
+        nestedCalls: [],
+        deepestNesting: [],
+        emptyArgs: [],
+        unnamedCalls: [],
+        args: []
+    };
+    for (const meta of info.values()) {
+        const us = meta.usedFunctions;
+        data.averageCall.push([us.allFunctionCalls]);
+        data.nestedCalls.push([us.nestedFunctionCalls]);
+        data.deepestNesting.push([us.deepestNesting]);
+        data.emptyArgs.push([(0, numbers_1.bigint2number)(us.args[0])]);
+        data.unnamedCalls.push([us.unnamedCalls]);
+        for (const [i, val] of Object.entries(us.args)) {
+            if (Number(i) !== 0) {
+                let get = data.args[Number(i)];
+                if (!get) {
+                    get = (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []);
+                    data.args[Number(i)] = get;
+                }
+                (0, common_syntax_probability_1.appendCommonSyntaxTypeCounter)(get, val);
+            }
+        }
+    }
+    console.log(`    [${(0, time_1.date2string)(new Date())}] Used functions metadata reading completed, summarizing and writing to file`);
+    writeFunctionCallsMetaInformationToCsv(outputPath, data);
+    for (const [index, arg] of data.args.entries()) {
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        if (!arg) {
+            // we treat the first/0-argument entry separate for legacy reasons
+            continue;
+        }
+        const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, `function-calls-arg-${index}.csv`));
+        out.write(`kind,name,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+        for (const [name, vals] of Object.entries(arg)) {
+            if (Array.isArray(vals)) {
+                out.write(`${JSON.stringify(name)},"",${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(vals.flat()))}\n`);
+            }
+            else {
+                for (const [keyName, keyValue] of Object.entries(vals)) {
+                    out.write(`${JSON.stringify(name)},${JSON.stringify(keyName)},${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(keyValue.flat()))}\n`);
+                }
+            }
+        }
+        out.close();
+    }
+}
+/**
+ * Note: the summary does not contain a 0 for each function that is _not_ called by a file. Hence, the minimum can not be 0 (division for mean etc. will still be performed on total file count)
+ */
+function postProcess(featureRoot, info, outputPath, config) {
+    retrieveFunctionCallInformation(featureRoot, info, config, outputPath);
+    console.log(`    [${(0, time_1.date2string)(new Date())}] Used functions reading completed, summarizing info...`);
+    retrieveFunctionCallMetaInformation(info, outputPath);
+}
+exports.postProcess = postProcess;
+function processNextLine(data, lineNumber, info, line, config, importants) {
+    if (lineNumber % 2_500 === 0) {
+        console.log(`    [${(0, time_1.date2string)(new Date())}] Used functions processed ${lineNumber} lines`);
+    }
+    const [hits, context] = line;
+    // group hits by fullname
+    const groupedByFunctionName = new Map();
+    for (const [name, loc, args, ns, known] of hits) {
+        const importantWrite = name && importants.get(name);
+        if (importantWrite) {
+            importantWrite.write(`${JSON.stringify(context)},${loc?.[0] ?? '?'}:${loc?.[1] ?? '?'},${ns ?? '""'},,,\n`);
+        }
+        const fullname = ns && ns !== '' ? `${ns}::${name ?? ''}` : name;
+        const key = (fullname ?? '') + (known === 1 ? '-' + (context ?? '') : '');
+        const stats = info.get(context ?? '')?.stats.lines[0].length;
+        let get = groupedByFunctionName.get(key);
+        if (!get) {
+            get = [new Set(), new Set(), [], [], []];
+            groupedByFunctionName.set(key, get);
+        }
+        // we retrieve the first component fo the path
+        const projectName = context?.split(path_1.default.sep)[config.projectSkip];
+        get[0].add(projectName ?? '');
+        get[1].add(context ?? '');
+        get[2].push(1);
+        get[3].push(args);
+        if (loc && stats) {
+            // we reduce by 1 to get flat 0% if it is the first line
+            get[4].push(stats === 1 ? 1 : (loc[0] - 1) / (stats - 1));
+        }
+    }
+    for (const [key, info] of groupedByFunctionName.entries()) {
+        let get = data.get(key);
+        if (!get) {
+            get = [new Set(), new Set(), [], [], []];
+            // an amazing empty structure :D
+            data.set(key, get);
+        }
+        // for total, we only need the number of elements as it will always be one :D
+        for (const context of info[0]) {
+            get[0].add(context);
+        }
+        for (const context of info[1]) {
+            get[1].add(context);
+        }
+        get[2].push([info[2].length]);
+        get[3].push(info[3]);
+        get[4].push(info[4]);
+    }
+}
+//# sourceMappingURL=post-process.js.map

package/statistics/features/supported/used-functions/used-functions.d.ts

@@ -0,0 +1,24 @@
+import type { Feature } from '../../feature';
+import type { Writable } from 'ts-essentials';
+import type { CommonSyntaxTypeCounts } from '../../common-syntax-probability';
+declare const initialFunctionUsageInfo: {
+    allFunctionCalls: number;
+    args: Record<number, bigint | CommonSyntaxTypeCounts<bigint>>;
+    /** `a(b(), c(3, d()))` has 3 (`b`, `c`, `d`) */
+    nestedFunctionCalls: number;
+    deepestNesting: number;
+    unnamedCalls: number;
+};
+export type FunctionUsageInfo = Writable<typeof initialFunctionUsageInfo>;
+export declare const AllCallsFileBase = "all-calls";
+export declare const usedFunctions: Feature<FunctionUsageInfo>;
+export type FunctionCallInformation = [
+    /** the name of the called function, or undefined if this was an unnamed function call */
+    name: string | undefined,
+    location: [line: number, character: number] | undefined,
+    numberOfArguments: number,
+    /** whether this was called from a namespace, like `a::b()` */
+    namespace: string | undefined,
+    knownDefinitionInFile: 0 | 1
+];
+export {};

package/statistics/features/supported/used-functions/used-functions.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.usedFunctions = exports.AllCallsFileBase = void 0;
+const common_syntax_probability_1 = require("../../common-syntax-probability");
+const post_process_1 = require("./post-process");
+const range_1 = require("../../../../util/range");
+const unpack_argument_1 = require("../../../../dataflow/internal/process/functions/call/argument/unpack-argument");
+const visitor_1 = require("../../../../r-bridge/lang-4.x/ast/model/processing/visitor");
+const statistics_file_1 = require("../../../output/statistics-file");
+const edge_1 = require("../../../../dataflow/graph/edge");
+const initialFunctionUsageInfo = {
+    allFunctionCalls: 0,
+    args: {
+        // only if called without arguments
+        0: 0n,
+        1: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)()
+    },
+    /** `a(b(), c(3, d()))` has 3 (`b`, `c`, `d`) */
+    nestedFunctionCalls: 0,
+    deepestNesting: 0,
+    unnamedCalls: 0
+};
+exports.AllCallsFileBase = 'all-calls';
+exports.usedFunctions = {
+    name: 'Used Functions',
+    description: 'All functions called, split into various sub-categories',
+    process(existing, input) {
+        visitCalls(existing, input);
+        return existing;
+    },
+    initialValue: initialFunctionUsageInfo,
+    postProcess: post_process_1.postProcess
+};
+function classifyArguments(args, existing) {
+    if (args.length === 0) {
+        existing[0]++;
+        return;
+    }
+    let i = 1;
+    for (const arg of args) {
+        if (arg === undefined) {
+            existing[0]++;
+            continue;
+        }
+        existing[i] = (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(existing[i] ?? (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(), arg);
+        i++;
+    }
+}
+function visitCalls(info, input) {
+    const calls = [];
+    const allCalls = [];
+    (0, visitor_1.visitAst)(input.normalizedRAst.ast, node => {
+        if (node.type !== "RFunctionCall" /* RType.FunctionCall */) {
+            return;
+        }
+        if (calls.length > 0) {
+            info.nestedFunctionCalls++;
+            (0, statistics_file_1.appendStatisticsFile)(exports.usedFunctions.name, 'nested-calls', [node.lexeme], input.filepath);
+            info.deepestNesting = Math.max(info.deepestNesting, calls.length);
+        }
+        const dataflowNode = input.dataflow.graph.get(node.info.id);
+        let hasCallsEdge = false;
+        if (dataflowNode) {
+            hasCallsEdge = [...dataflowNode[1].values()].some(e => (0, edge_1.edgeIncludesType)(e.types, 4 /* EdgeType.Calls */));
+        }
+        if (node.flavor === 'unnamed') {
+            info.unnamedCalls++;
+            (0, statistics_file_1.appendStatisticsFile)(exports.usedFunctions.name, 'unnamed-calls', [node.lexeme], input.filepath);
+            allCalls.push([
+                undefined,
+                (0, range_1.getRangeStart)(node.location),
+                node.arguments.length,
+                '',
+                hasCallsEdge ? 1 : 0
+            ]);
+        }
+        else {
+            allCalls.push([
+                node.functionName.lexeme,
+                (0, range_1.getRangeStart)(node.location),
+                node.arguments.length,
+                node.functionName.namespace ?? '',
+                hasCallsEdge ? 1 : 0
+            ]);
+        }
+        classifyArguments(node.arguments.map(unpack_argument_1.unpackArgument), info.args);
+        calls.push(node);
+    }, node => {
+        // drop again :D
+        if (node.type === "RFunctionCall" /* RType.FunctionCall */) {
+            calls.pop();
+        }
+    });
+    info.allFunctionCalls += allCalls.length;
+    (0, statistics_file_1.appendStatisticsFile)(exports.usedFunctions.name, exports.AllCallsFileBase, allCalls, input.filepath);
+}
+//# sourceMappingURL=used-functions.js.map

package/statistics/features/supported/used-packages/post-process.d.ts

@@ -0,0 +1,3 @@
+import type { FeatureStatisticsWithMeta } from '../../feature';
+import type { StatisticsSummarizerConfiguration } from '../../../summarizer/summarizer';
+export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;

package/statistics/features/supported/used-packages/post-process.js

@@ -0,0 +1,120 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.postProcess = void 0;
+const post_processing_1 = require("../../post-processing");
+const used_packages_1 = require("./used-packages");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const summarizer_1 = require("../../../../util/summarizer");
+const files_1 = require("../../../../util/files");
+const strings_1 = require("../../../../util/strings");
+const arrays_1 = require("../../../../util/arrays");
+function postProcess(featureRoot, info, outputPath, config) {
+    const collected = {};
+    for (const [filepath, data] of info.entries()) {
+        const value = data.usedPackages;
+        for (const [key, val] of Object.entries(value)) {
+            let get = collected[key];
+            if (!get) {
+                get = (0, post_processing_1.emptySummarizedWithProject)();
+                collected[key] = get;
+            }
+            get.count.push(val);
+            if (val > 0) {
+                (0, post_processing_1.recordFilePath)(get, filepath, config);
+            }
+        }
+    }
+    const variablesOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'loading-functions.csv'));
+    variablesOutStream.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of Object.entries(collected)) {
+        const data = val;
+        const sum = (0, summarizer_1.summarizeMeasurement)(data.count);
+        variablesOutStream.write(`${JSON.stringify(key)},${data.uniqueProjects.size},${data.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(sum)}\n`);
+    }
+    variablesOutStream.close();
+    // now we want to collect the names of the loaded libraries,
+    // we collect and store them separately (per kind) but also, we want store the summarized results in the end!
+    const loadedLibraries = new Map();
+    for (const key of Object.keys(used_packages_1.initialUsedPackageInfos)) {
+        const data = retrieveDataForLoad(key, featureRoot, outputPath, config);
+        for (const [name, val] of data.entries()) {
+            let get = loadedLibraries.get(name);
+            if (!get) {
+                get = (0, post_processing_1.emptySummarizedWithProject)();
+                loadedLibraries.set(name, get);
+            }
+            get.count.push(...val.count);
+            for (const uniqueFile of val.uniqueFiles) {
+                get.uniqueFiles.add(uniqueFile);
+            }
+            for (const uniqueProject of val.uniqueProjects) {
+                get.uniqueProjects.add(uniqueProject);
+            }
+        }
+    }
+    const output = path_1.default.join(outputPath, 'all-operators.csv');
+    const out = fs_1.default.createWriteStream(output);
+    out.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of loadedLibraries.entries()) {
+        const { count, uniqueProjects, uniqueFiles } = val;
+        const sum = (0, summarizer_1.summarizeMeasurement)(count);
+        out.write(`${JSON.stringify(key)},${uniqueProjects.size},${uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(sum)}\n`);
+    }
+    out.close();
+}
+exports.postProcess = postProcess;
+// directly writes the results to the output path
+function retrieveDataForLoad(operator, readFromPath, outputPath, config) {
+    const input = path_1.default.join(readFromPath, `${operator}.txt`);
+    if (!fs_1.default.existsSync(input)) {
+        // if there is nothing with this, just return :)
+        return new Map();
+    }
+    const collected = new Map();
+    (0, files_1.readLineByLineSync)(input, (line, lineNumber) => {
+        if (line.length === 0) {
+            return;
+        }
+        if (lineNumber % 2_500 === 0) {
+            console.log(`    Processing line ${lineNumber} from ${input}`);
+        }
+        const [packages, context] = JSON.parse(line.toString());
+        // first, we have to collect what this file gives us
+        // we normalize surrounding quotation marks
+        const bag = (0, arrays_1.array2bag)(packages.map(p => {
+            if ((0, strings_1.startAndEndsWith)(p, '"') || (0, strings_1.startAndEndsWith)(p, "'") || (0, strings_1.startAndEndsWith)(p, '`')) {
+                return p.slice(1, -1);
+            }
+            else {
+                return p;
+            }
+        }));
+        // now we merge it into the global map (oh gosh this is so horrible
+        for (const [name, count] of bag.entries()) {
+            let get = collected.get(name);
+            if (!get) {
+                get = (0, post_processing_1.emptySummarizedWithProject)();
+                collected.set(name, get);
+            }
+            get.count.push(count);
+            if (count > 0) {
+                (0, post_processing_1.recordFilePath)(get, context, config);
+            }
+        }
+    });
+    const output = path_1.default.join(outputPath, `${operator}.csv`);
+    const out = fs_1.default.createWriteStream(output);
+    out.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of collected.entries()) {
+        const { count, uniqueProjects, uniqueFiles } = val;
+        const sum = (0, summarizer_1.summarizeMeasurement)(count);
+        out.write(`${JSON.stringify(key)},${uniqueProjects.size},${uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(sum)}\n`);
+    }
+    out.close();
+    return collected;
+}
+//# sourceMappingURL=post-process.js.map

package/statistics/features/supported/used-packages/used-packages.d.ts

@@ -0,0 +1,16 @@
+import type { Feature } from '../../feature';
+import type { Writable } from 'ts-essentials';
+export declare const initialUsedPackageInfos: {
+    library: number;
+    require: number;
+    loadNamespace: number;
+    requireNamespace: number;
+    attachNamespace: number;
+    withinApply: number;
+    '::': number;
+    ':::': number;
+    /** just contains all occurrences where it is impossible to statically determine which package is loaded */
+    '<loadedByVariable>': number;
+};
+export type UsedPackageInfo = Writable<typeof initialUsedPackageInfos>;
+export declare const usedPackages: Feature<UsedPackageInfo>;

package/statistics/features/supported/used-packages/used-packages.js

@@ -0,0 +1,130 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.usedPackages = exports.initialUsedPackageInfos = void 0;
+const xpath = __importStar(require("xpath-ts2"));
+const post_process_1 = require("./post-process");
+const statistics_file_1 = require("../../../output/statistics-file");
+exports.initialUsedPackageInfos = {
+    library: 0,
+    require: 0,
+    loadNamespace: 0,
+    requireNamespace: 0,
+    attachNamespace: 0,
+    withinApply: 0,
+    '::': 0,
+    ':::': 0,
+    /** just contains all occurrences where it is impossible to statically determine which package is loaded */
+    '<loadedByVariable>': 0
+};
+// based on the extraction routine of lintr search for function calls which are not character-loads (we can not trace those...)
+const withinApply = xpath.parse(`
+  //SYMBOL_FUNCTION_CALL[contains(.,"apply")]/../..
+    //SYMBOL[text()='require' or text()='library' or text()='loadNamespace' or text()='requireNamespace' or text()='attachNamespace']
+    /../..
+`);
+// horrible ways I found exploratively like loading within `sapply`
+const libraryOrRequire = xpath.parse(`
+  //SYMBOL_FUNCTION_CALL[text() = $variable]
+    /parent::expr
+    /parent::expr[
+      expr[2][STR_CONST]
+      or (
+        expr[2][SYMBOL]
+        and not(
+          SYMBOL_SUB[text() = 'character.only']
+          /following-sibling::expr[1]
+          /NUM_CONST[text() = 'TRUE' or text() = 'T']
+        )
+      )
+    ]/OP-LEFT-PAREN[1]/following-sibling::expr[1][SYMBOL | STR_CONST]
+`);
+// there is no except in xpath 1.0?
+const packageLoadedWithVariableLoadRequire = xpath.parse(`
+    //SYMBOL_FUNCTION_CALL[text() = 'library' or text() = 'require']
+    /parent::expr
+    /parent::expr[
+        expr[2][SYMBOL]
+        and (
+          SYMBOL_SUB[text() = 'character.only']
+          /following-sibling::expr[1]
+          /NUM_CONST[text() = 'TRUE' or text() = 'T']
+        )
+    ]/OP-LEFT-PAREN[1]/following-sibling::expr[1][SYMBOL | STR_CONST]
+`);
+const packageLoadedWithVariableNamespaces = xpath.parse(`
+  //SYMBOL_FUNCTION_CALL[text() = 'loadNamespace' or text() = 'requireNamespace' or text() = 'attachNamespace']/../following-sibling::expr[1][SYMBOL]
+`);
+const queryForFunctionCall = xpath.parse(`
+  //SYMBOL_FUNCTION_CALL[text() = $variable]/../following-sibling::expr[1][STR_CONST]
+`);
+// otherwise, the parser seems to fail
+const queryForNsAccess = xpath.parse(`
+  //NS_GET[text() = $variable]/../SYMBOL_PACKAGE[1]
+  |
+  //NS_GET_INT[text() = $variable]/../SYMBOL_PACKAGE[1]
+`);
+const queries = [
+    {
+        types: ['library', 'require'],
+        query: libraryOrRequire
+    },
+    {
+        types: ['loadNamespace', 'requireNamespace', 'attachNamespace'],
+        query: queryForFunctionCall
+    },
+    {
+        types: ['::', ':::'],
+        query: queryForNsAccess
+    }
+];
+exports.usedPackages = {
+    name: 'Used Packages',
+    description: 'All the packages used in the code',
+    process(existing, input) {
+        // we will unify in the end, so we can count, group etc. but we do not re-count multiple packages in the same file
+        for (const q of queries) {
+            for (const fn of q.types) {
+                const nodes = q.query.select({ node: input.parsedRAst, variables: { variable: fn } });
+                existing[fn] += nodes.length;
+                (0, statistics_file_1.appendStatisticsFile)(this.name, fn, nodes, input.filepath, true);
+            }
+        }
+        const nodesForVariableLoad = [
+            ...packageLoadedWithVariableLoadRequire.select({ node: input.parsedRAst }),
+            ...packageLoadedWithVariableNamespaces.select({ node: input.parsedRAst })
+        ];
+        existing['<loadedByVariable>'] += nodesForVariableLoad.length;
+        // should not be unique as variables may be repeated, and we have no idea
+        (0, statistics_file_1.appendStatisticsFile)(this.name, '<loadedByVariable>', nodesForVariableLoad, input.filepath);
+        const withinApplyNodes = withinApply.select({ node: input.parsedRAst });
+        existing.withinApply += withinApplyNodes.length;
+        (0, statistics_file_1.appendStatisticsFile)(this.name, 'withinApply', withinApplyNodes, input.filepath);
+        return existing;
+    },
+    initialValue: exports.initialUsedPackageInfos,
+    postProcess: post_process_1.postProcess
+};
+//# sourceMappingURL=used-packages.js.map

package/statistics/features/supported/values/post-process.d.ts

@@ -0,0 +1,3 @@
+import type { FeatureStatisticsWithMeta } from '../../feature';
+import type { StatisticsSummarizerConfiguration } from '../../../summarizer/summarizer';
+export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;

package/statistics/features/supported/values/post-process.js

@@ -0,0 +1,71 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.postProcess = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const post_processing_1 = require("../../post-processing");
+const summarizer_1 = require("../../../../util/summarizer");
+const time_1 = require("../../../../util/time");
+const files_1 = require("../../../../util/files");
+const arrays_1 = require("../../../../util/arrays");
+function postProcess(featureRoot, info, outputPath, config) {
+    const collected = {};
+    for (const [filepath, data] of info.entries()) {
+        const value = data.values;
+        for (const [key, val] of Object.entries(value)) {
+            let get = collected[key];
+            if (!get) {
+                get = (0, post_processing_1.emptySummarizedWithProject)();
+                collected[key] = get;
+            }
+            get.count.push(val);
+            if (val > 0) {
+                (0, post_processing_1.recordFilePath)(get, filepath, config);
+            }
+        }
+    }
+    const valuesOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'values.csv'));
+    valuesOutStream.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of Object.entries(collected)) {
+        const data = val;
+        const sum = (0, summarizer_1.summarizeMeasurement)(data.count);
+        valuesOutStream.write(`${JSON.stringify(key)},${data.uniqueProjects.size},${data.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(sum)}\n`);
+    }
+    valuesOutStream.close();
+    // now we read all numeric values to get the top used magic numbers (per file)
+    const valueMap = new Map();
+    (0, files_1.readLineByLineSync)(path_1.default.join(featureRoot, 'numeric.txt'), (line, lineNumber) => {
+        if (line.length === 0) {
+            return;
+        }
+        if (lineNumber % 2_500 === 0) {
+            console.log(`    [${(0, time_1.date2string)(new Date())}] Collecting numeric values ${lineNumber}`);
+        }
+        const [values, context] = JSON.parse(String(line));
+        const bag = (0, arrays_1.array2bag)(values);
+        for (const [key, val] of bag.entries()) {
+            let get = valueMap.get(key);
+            if (!get) {
+                get = { count: [], uniqueFiles: new Set(), uniqueProjects: new Set() };
+                valueMap.set(key, get);
+            }
+            get.count.push(val);
+            if (val > 0) {
+                get.uniqueFiles.add(context ?? '');
+                get.uniqueProjects.add(context?.split(path_1.default.sep)[config.projectSkip] ?? '');
+            }
+        }
+    });
+    const magicNumbersOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'magic-numbers.csv'));
+    magicNumbersOutStream.write(`num,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of valueMap.entries()) {
+        const sum = (0, summarizer_1.summarizeMeasurement)(val.count);
+        magicNumbersOutStream.write(`${JSON.stringify(key)},${val.uniqueProjects.size},${val.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(sum)}\n`);
+    }
+    magicNumbersOutStream.close();
+}
+exports.postProcess = postProcess;
+//# sourceMappingURL=post-process.js.map

package/statistics/features/supported/values/values.d.ts

@@ -0,0 +1,14 @@
+import type { Feature } from '../../feature';
+import type { Writable } from 'ts-essentials';
+declare const initialValueInfo: {
+    allNumerics: number;
+    imaginaryNumbers: number;
+    integers: number;
+    floatHex: number;
+    logical: number;
+    specialConstants: number;
+    strings: number;
+};
+export type ValueInfo = Writable<typeof initialValueInfo>;
+export declare const values: Feature<ValueInfo>;
+export {};