@eagleoutice/flowr 1.5.2 → 2.0.1

package/statistics/features/supported/defined-functions/defined-functions.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.definedFunctions = exports.AllDefinitionsFileBase = void 0;
+const post_process_1 = require("./post-process");
+const range_1 = require("../../../../util/range");
+const assert_1 = require("../../../../util/assert");
+const edge_1 = require("../../../../dataflow/graph/edge");
+const visitor_1 = require("../../../../r-bridge/lang-4.x/ast/model/processing/visitor");
+const statistics_file_1 = require("../../../output/statistics-file");
+const initialFunctionDefinitionInfo = {
+    /** all, anonymous, assigned, non-assigned, ... */
+    total: 0,
+    /** how many are really using OP-Lambda? */
+    lambdasOnly: 0,
+    /** using `<<-`, `<-`, `=`, `->` `->>` */
+    assignedFunctions: 0,
+    nestedFunctions: 0,
+    /** functions that in some easily detectable way call themselves */
+    recursive: 0,
+    deepestNesting: 0
+};
+exports.AllDefinitionsFileBase = 'all-definitions';
+function retrieveAllCallsites(input, node, recursiveCalls) {
+    const dfStart = input.dataflow.graph.outgoingEdges(node.info.id);
+    const callsites = [];
+    for (const [target, edge] of dfStart ?? []) {
+        if (!(0, edge_1.edgeIncludesType)(edge.types, 4 /* EdgeType.Calls */)) {
+            continue;
+        }
+        const loc = input.normalizedRAst.idMap.get(target)?.location;
+        if (loc) {
+            callsites.push((0, range_1.getRangeStart)(loc));
+        }
+    }
+    for (const call of recursiveCalls) {
+        const loc = call.location;
+        if (loc) {
+            callsites.push((0, range_1.getRangeStart)(loc));
+        }
+    }
+    return callsites;
+}
+function visitDefinitions(info, input) {
+    const definitionStack = [];
+    const allDefinitions = [];
+    (0, visitor_1.visitAst)(input.normalizedRAst.ast, node => {
+        if (node.type !== "RFunctionDefinition" /* RType.FunctionDefinition */) {
+            return;
+        }
+        const graph = input.dataflow.graph;
+        const dfNode = graph.get(node.info.id, true);
+        if (dfNode === undefined) {
+            (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, 'no-dataflow-node-found', [node], input.filepath);
+            return;
+        }
+        const [fnDefinition] = dfNode;
+        (0, assert_1.guard)(fnDefinition.tag === "function-definition" /* VertexType.FunctionDefinition */, () => `Dataflow node is not a function definition (${JSON.stringify(fnDefinition)}))})`);
+        const returnTypes = fnDefinition.exitPoints.map(ep => graph.get(ep, true)).filter(assert_1.isNotUndefined)
+            .map(([vertex]) => {
+            const l = graph.idMap?.get(vertex.id)?.location;
+            return {
+                location: l ? (0, range_1.getRangeStart)(l) : [-1, -1]
+            };
+        });
+        if (definitionStack.length > 0) {
+            info.nestedFunctions++;
+            info.deepestNesting = Math.max(info.deepestNesting, definitionStack.length);
+            (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, 'nested-definitions', [node.info.fullLexeme ?? node.lexeme], input.filepath);
+        }
+        // parameter names:
+        const parameterNames = node.parameters.map(p => p.info.fullLexeme ?? p.lexeme);
+        (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, 'usedParameterNames', parameterNames, input.filepath);
+        const isLambda = node.lexeme.startsWith('\\');
+        if (isLambda) {
+            info.lambdasOnly++;
+            (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, 'allLambdas', [node.info.fullLexeme ?? node.lexeme], input.filepath);
+        }
+        definitionStack.push(node);
+        // we find definitions with silly defined-by edges
+        const assigned = new Set();
+        const edges = input.dataflow.graph.ingoingEdges(node.info.id);
+        if (edges !== undefined) {
+            for (const [targetId, edge] of edges) {
+                if ((0, edge_1.edgeIncludesType)(edge.types, 2 /* EdgeType.DefinedBy */)) {
+                    const target = input.normalizedRAst.idMap.get(targetId);
+                    (0, assert_1.guard)(target !== undefined, 'Dataflow edge points to unknown node');
+                    const name = target.info.fullLexeme ?? target.lexeme;
+                    if (name) {
+                        assigned.add(name);
+                    }
+                    info.assignedFunctions++;
+                    (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, 'assignedFunctions', [name ?? '<unknown>'], input.filepath);
+                }
+                if ((0, edge_1.edgeIncludesType)(edge.types, 4 /* EdgeType.Calls */)) {
+                    const target = input.normalizedRAst.idMap.get(targetId);
+                    (0, assert_1.guard)(target !== undefined, 'Dataflow edge points to unknown node');
+                }
+            }
+        }
+        // track all calls with the same name that do not already have a bound calls edge, superfluous if recursive tracking is explicit
+        const recursiveCalls = [];
+        (0, visitor_1.visitAst)(node.body, n => {
+            if (n.type === "RFunctionCall" /* RType.FunctionCall */ && n.flavor === 'named' && assigned.has(n.functionName.lexeme)) {
+                recursiveCalls.push(n);
+            }
+        });
+        // one recursive definition, but we record all
+        info.recursive += recursiveCalls.length > 0 ? 1 : 0;
+        (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, 'recursive', recursiveCalls.map(n => n.info.fullLexeme ?? n.lexeme ?? 'unknown'), input.filepath);
+        const lexeme = node.info.fullLexeme;
+        const lexemeSplit = lexeme?.split('\n');
+        allDefinitions.push({
+            location: (0, range_1.getRangeStart)(node.location),
+            callsites: retrieveAllCallsites(input, node, recursiveCalls),
+            numberOfParameters: node.parameters.length,
+            returns: returnTypes,
+            length: {
+                lines: lexemeSplit?.length ?? -1,
+                characters: lexeme?.length ?? -1,
+                nonWhitespaceCharacters: lexeme?.replaceAll(/\s/g, '').length ?? 0
+            }
+        });
+    }, node => {
+        // drop again :D
+        if (node.type === "RFunctionDefinition" /* RType.FunctionDefinition */) {
+            definitionStack.pop();
+        }
+    });
+    info.total += allDefinitions.length;
+    (0, statistics_file_1.appendStatisticsFile)(exports.definedFunctions.name, exports.AllDefinitionsFileBase, allDefinitions, input.filepath);
+}
+exports.definedFunctions = {
+    name: 'Defined Functions',
+    description: 'All functions defined within the document',
+    process(existing, input) {
+        visitDefinitions(existing, input);
+        return existing;
+    },
+    initialValue: initialFunctionDefinitionInfo,
+    postProcess: post_process_1.postProcess
+};
+//# sourceMappingURL=defined-functions.js.map

package/statistics/features/supported/defined-functions/post-process.d.ts

@@ -0,0 +1,6 @@
+import type { FeatureStatisticsWithMeta } from '../../feature';
+import type { StatisticsSummarizerConfiguration } from '../../../summarizer/summarizer';
+/**
+ * Note: the summary does not contain a 0 for each function that is _not_ called by a file. Hence, the minimum can not be 0 (division for mean etc. will still be performed on total file count)
+ */
+export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;

package/statistics/features/supported/defined-functions/post-process.js

@@ -0,0 +1,169 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.postProcess = void 0;
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const defined_functions_1 = require("./defined-functions");
+const post_processing_1 = require("../../post-processing");
+const summarizer_1 = require("../../../../util/summarizer");
+const files_1 = require("../../../../util/files");
+const time_1 = require("../../../../util/time");
+const arrays_1 = require("../../../../util/arrays");
+function getFnDefCsv(idx, info) {
+    return `${JSON.stringify(idx)},${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.total.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.parameters.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.length.lines.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.length.chars.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.length.nonWhitespaceChars.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)([0]))}` /* this has to be changed once we have explicit returns again */
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)([0]))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.returns.onlyExplicit.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.returns.onlyImplicit.flat()))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.exitPointsLinePercentageInDef.flat(2)))}`
+        + `,${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(info.linePercentageInFile.flat()))}\n`;
+}
+function addToList(data, count, filepath, config) {
+    data.count.push(count);
+    if (count > 0) {
+        (0, post_processing_1.recordFilePath)(data, filepath, config);
+    }
+}
+function retrievePerFileDefinitionInformation(featureRoot, info, config, outputPath) {
+    /**
+     * maps fn-name (including namespace) to several definition information
+     * we use tuples to reduce the memory!
+     */
+    const definitionsPerFile = [];
+    const mergedSuperDefinitions = emptyFunctionDefinitionSummary();
+    // we collect only `all-calls`
+    (0, files_1.readLineByLineSync)(path_1.default.join(featureRoot, `${defined_functions_1.AllDefinitionsFileBase}.txt`), (line, lineNumber) => processNextLine(definitionsPerFile, lineNumber, info, JSON.parse(String(line))));
+    console.log(`    [${(0, time_1.date2string)(new Date())}] Defined functions process completed, start to write out function info`);
+    const fnOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-definitions.csv'));
+    const prefixes = ['total', 'params', 'length-lines', 'length-chars', 'length-non-ws-chars', 'return-explicit', 'return-implicit', 'return-only-explicit', 'return-only-implicit', 'exit-points-line-crac', 'def-line-frac'];
+    const others = prefixes.flatMap(summarizer_1.summarizedMeasurement2CsvHeader).join(',');
+    fnOutStream.write(`counter,${others}\n`);
+    for (const [idx, info] of definitionsPerFile.entries()) {
+        fnOutStream.write(getFnDefCsv(idx, info));
+        mergedSuperDefinitions.total.push(...info.total);
+        mergedSuperDefinitions.parameters.push(...info.parameters);
+        mergedSuperDefinitions.length.lines.push(...info.length.lines);
+        mergedSuperDefinitions.length.chars.push(...info.length.chars);
+        mergedSuperDefinitions.length.nonWhitespaceChars.push(...info.length.nonWhitespaceChars);
+        mergedSuperDefinitions.returns.onlyExplicit.push(...info.returns.onlyExplicit);
+        mergedSuperDefinitions.returns.onlyImplicit.push(...info.returns.onlyImplicit);
+        mergedSuperDefinitions.exitPointsLinePercentageInDef.push(...info.exitPointsLinePercentageInDef);
+        mergedSuperDefinitions.linePercentageInFile.push(...info.linePercentageInFile);
+        mergedSuperDefinitions.callsites.push(...info.callsites);
+    }
+    // now, write the ultimate summary at the end of the file
+    fnOutStream.write(getFnDefCsv('all', mergedSuperDefinitions));
+    fnOutStream.close();
+}
+function retrieveMetaInformation(info, config, outputPath) {
+    const data = {
+        total: (0, post_processing_1.emptySummarizedWithProject)(),
+        lambdasOnly: (0, post_processing_1.emptySummarizedWithProject)(),
+        assignedFunctions: (0, post_processing_1.emptySummarizedWithProject)(),
+        nestedFunctions: (0, post_processing_1.emptySummarizedWithProject)(),
+        recursive: (0, post_processing_1.emptySummarizedWithProject)(),
+        deepestNesting: (0, post_processing_1.emptySummarizedWithProject)()
+    };
+    for (const [filepath, meta] of info.entries()) {
+        const us = meta.definedFunctions;
+        addToList(data.total, us.total, filepath, config);
+        addToList(data.lambdasOnly, us.lambdasOnly, filepath, config);
+        addToList(data.assignedFunctions, us.assignedFunctions, filepath, config);
+        addToList(data.nestedFunctions, us.nestedFunctions, filepath, config);
+        addToList(data.recursive, us.recursive, filepath, config);
+        addToList(data.deepestNesting, us.deepestNesting, filepath, config);
+    }
+    console.log(`    [${(0, time_1.date2string)(new Date())}] Defined functions metadata reading completed, summarizing and writing to file`);
+    const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-definitions-meta.csv'));
+    out.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of Object.entries(data)) {
+        const data = val;
+        out.write(`${JSON.stringify(key)},${data.uniqueProjects.size},${data.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.count))}\n`);
+    }
+    out.close();
+}
+function retrieveAssignedFunctionNames(featureRoot, config, outputPath) {
+    const varNames = new Map();
+    (0, files_1.readLineByLineSync)(path_1.default.join(featureRoot, 'assignedFunctions.txt'), line => {
+        const parsed = JSON.parse(String(line));
+        const [hits, context] = parsed;
+        const countsForFile = (0, arrays_1.array2bag)(hits);
+        for (const [name, count] of countsForFile.entries()) {
+            let get = varNames.get(name);
+            if (!get) {
+                get = (0, post_processing_1.emptySummarizedWithProject)();
+                varNames.set(name, get);
+            }
+            addToList(get, count, context ?? '', config);
+        }
+    });
+    const varNamesOut = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'function-definitions-var-names.csv'));
+    varNamesOut.write(`name,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of varNames.entries()) {
+        varNamesOut.write(`${JSON.stringify(key)},${val.uniqueProjects.size},${val.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(val.count))}\n`);
+    }
+    varNamesOut.close();
+}
+/**
+ * Note: the summary does not contain a 0 for each function that is _not_ called by a file. Hence, the minimum can not be 0 (division for mean etc. will still be performed on total file count)
+ */
+function postProcess(featureRoot, info, outputPath, config) {
+    // each number[][] contains a 'number[]' per file
+    retrievePerFileDefinitionInformation(featureRoot, info, config, outputPath);
+    console.log(`    [${(0, time_1.date2string)(new Date())}] Defined functions reading completed, summarizing info...`);
+    retrieveMetaInformation(info, config, outputPath);
+    retrieveAssignedFunctionNames(featureRoot, config, outputPath);
+}
+exports.postProcess = postProcess;
+function emptyFunctionDefinitionSummary() {
+    return {
+        total: [],
+        parameters: [],
+        length: {
+            lines: [],
+            chars: [],
+            nonWhitespaceChars: []
+        },
+        returns: {
+            explicit: [],
+            implicit: [],
+            onlyExplicit: [],
+            onlyImplicit: []
+        },
+        exitPointsLinePercentageInDef: [],
+        linePercentageInFile: [],
+        callsites: []
+    };
+}
+function processNextLine(data, lineNumber, info, line) {
+    if (lineNumber % 2_500 === 0) {
+        console.log(`    [${(0, time_1.date2string)(new Date())}] Defined functions processed ${lineNumber} lines`);
+    }
+    const [hits, context] = line;
+    const forFile = emptyFunctionDefinitionSummary();
+    for (const { location, length, returns, numberOfParameters, callsites } of hits) {
+        const stats = info.get(context ?? '')?.stats.lines[0].length;
+        // we retrieve the first component fo the path
+        forFile.total.push(1);
+        forFile.parameters.push(numberOfParameters);
+        forFile.length.lines.push(length.lines);
+        forFile.length.chars.push(length.characters);
+        forFile.length.nonWhitespaceChars.push(length.nonWhitespaceCharacters);
+        forFile.exitPointsLinePercentageInDef.push(returns.map(r => r.location[0]).map(l => l / length.lines));
+        forFile.callsites.push(callsites.length);
+        if (stats) {
+            forFile.linePercentageInFile.push(location[0] / stats);
+        }
+    }
+    // push all of that to main :D
+    forFile.total = [forFile.total.length];
+    data.push(forFile);
+}
+//# sourceMappingURL=post-process.js.map

package/statistics/features/supported/expression-list/expression-list.d.ts

@@ -0,0 +1,9 @@
+import type { Writable } from 'ts-essentials';
+import type { Feature } from '../../feature';
+declare const initialExpressionListInfo: {
+    allExpressionLists: number;
+    deepestNesting: number;
+};
+export type ExpressionListInfo = Writable<typeof initialExpressionListInfo>;
+export declare const expressionList: Feature<ExpressionListInfo>;
+export {};

package/statistics/features/supported/expression-list/expression-list.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.expressionList = void 0;
+const post_process_1 = require("./post-process");
+const visitor_1 = require("../../../../r-bridge/lang-4.x/ast/model/processing/visitor");
+const initialExpressionListInfo = {
+    allExpressionLists: 0,
+    deepestNesting: 0
+};
+function visitLists(info, input) {
+    let nest = -1; // we start with nesting 0
+    let total = 0;
+    (0, visitor_1.visitAst)(input.normalizedRAst.ast, node => {
+        if (node.type === "RExpressionList" /* RType.ExpressionList */) {
+            nest++;
+            total++;
+            info.deepestNesting = Math.max(info.deepestNesting, nest);
+        }
+    }, node => {
+        if (node.type === "RExpressionList" /* RType.ExpressionList */) {
+            nest--;
+        }
+    });
+    info.allExpressionLists += total;
+}
+exports.expressionList = {
+    name: 'Expression Lists',
+    description: 'Counts expression list nestings',
+    process(existing, input) {
+        visitLists(existing, input);
+        return existing;
+    },
+    initialValue: initialExpressionListInfo,
+    postProcess: post_process_1.postProcess
+};
+//# sourceMappingURL=expression-list.js.map

package/statistics/features/supported/expression-list/post-process.d.ts

@@ -0,0 +1,3 @@
+import type { FeatureStatisticsWithMeta } from '../../feature';
+import type { StatisticsSummarizerConfiguration } from '../../../summarizer/summarizer';
+export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;

package/statistics/features/supported/expression-list/post-process.js

@@ -0,0 +1,43 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.postProcess = void 0;
+const post_processing_1 = require("../../post-processing");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const summarizer_1 = require("../../../../util/summarizer");
+function postProcess(featureRoot, info, outputPath, config) {
+    const collected = {};
+    const deepestNestingOut = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'deepest-nesting-per-file.csv'));
+    deepestNestingOut.write('file,deepest-nesting\n');
+    for (const [filepath, data] of info.entries()) {
+        const value = data.expressionList;
+        for (const [key, val] of Object.entries(value)) {
+            let get = collected[key];
+            if (!get) {
+                get = (0, post_processing_1.emptySummarizedWithProject)();
+                collected[key] = get;
+            }
+            if (key === 'deepestNesting') {
+                deepestNestingOut.write(`${JSON.stringify(filepath)},${val}\n`);
+            }
+            get.count.push(val);
+            if (val > 0) {
+                (0, post_processing_1.recordFilePath)(get, filepath, config);
+            }
+        }
+    }
+    deepestNestingOut.close();
+    const variablesOutStream = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'used-expression-lists.csv'));
+    variablesOutStream.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of Object.entries(collected)) {
+        const data = val;
+        const sum = (0, summarizer_1.summarizeMeasurement)(data.count);
+        variablesOutStream.write(`${JSON.stringify(key)},${data.uniqueProjects.size},${data.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)(sum)}\n`);
+    }
+    variablesOutStream.close();
+}
+exports.postProcess = postProcess;
+//# sourceMappingURL=post-process.js.map

package/statistics/features/supported/loops/loops.d.ts

@@ -0,0 +1,20 @@
+import type { Feature } from '../../feature';
+import type { Writable } from 'ts-essentials';
+declare const initialLoopInfo: {
+    forLoops: import("../../common-syntax-probability").CommonSyntaxTypeCounts<bigint>;
+    forLoopVar: import("../../common-syntax-probability").CommonSyntaxTypeCounts<bigint>;
+    forBody: import("../../common-syntax-probability").CommonSyntaxTypeCounts<bigint>;
+    whileLoops: import("../../common-syntax-probability").CommonSyntaxTypeCounts<bigint>;
+    whileBody: import("../../common-syntax-probability").CommonSyntaxTypeCounts<bigint>;
+    repeatLoops: bigint;
+    repeatBody: import("../../common-syntax-probability").CommonSyntaxTypeCounts<bigint>;
+    breakStatements: number;
+    nextStatements: number;
+    /** apply, tapply, lapply, ...*/
+    implicitLoops: number;
+    nestedExplicitLoops: number;
+    deepestExplicitNesting: number;
+};
+export type LoopInfo = Writable<typeof initialLoopInfo>;
+export declare const loops: Feature<LoopInfo>;
+export {};

package/statistics/features/supported/loops/loops.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loops = void 0;
+const common_syntax_probability_1 = require("../../common-syntax-probability");
+const post_process_1 = require("./post-process");
+const statistics_file_1 = require("../../../output/statistics-file");
+const visitor_1 = require("../../../../r-bridge/lang-4.x/ast/model/processing/visitor");
+const initialLoopInfo = {
+    forLoops: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(),
+    forLoopVar: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(),
+    forBody: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(),
+    whileLoops: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(),
+    whileBody: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(),
+    repeatLoops: 0n,
+    repeatBody: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(),
+    breakStatements: 0,
+    nextStatements: 0,
+    /** apply, tapply, lapply, ...*/
+    implicitLoops: 0,
+    nestedExplicitLoops: 0,
+    deepestExplicitNesting: 0
+};
+const isImplicitLoop = /[lsvmt]?apply/;
+function visitLoops(info, input) {
+    // holds number of loops and their nesting depths
+    const loopStack = [];
+    (0, visitor_1.visitAst)(input.normalizedRAst.ast, node => {
+        switch (node.type) {
+            case "RNext" /* RType.Next */:
+                info.nextStatements++;
+                return;
+            case "RBreak" /* RType.Break */:
+                info.breakStatements++;
+                return;
+            case "RFunctionCall" /* RType.FunctionCall */:
+                if (node.flavor === 'named' && isImplicitLoop.test(node.functionName.lexeme)) {
+                    info.implicitLoops++;
+                    (0, statistics_file_1.appendStatisticsFile)(exports.loops.name, 'implicit-loop', [node.functionName.info.fullLexeme ?? node.functionName.lexeme], input.filepath);
+                }
+                return;
+            case "RForLoop" /* RType.ForLoop */:
+                (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(info.forLoops, node.vector);
+                (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(info.forLoopVar, node.variable);
+                (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(info.forBody, ...node.body.children);
+                break;
+            case "RWhileLoop" /* RType.WhileLoop */:
+                (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(info.whileLoops, node.condition);
+                (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(info.whileBody, ...node.body.children);
+                break;
+            case "RRepeatLoop" /* RType.RepeatLoop */:
+                info.repeatLoops++;
+                (0, common_syntax_probability_1.updateCommonSyntaxTypeCounts)(info.repeatBody, ...node.body.children);
+                break;
+            default: return;
+        }
+        (0, statistics_file_1.appendStatisticsFile)(exports.loops.name, 'all-loops', [node.info.fullLexeme ?? node.lexeme], input.filepath);
+        if (loopStack.length > 0) {
+            info.nestedExplicitLoops++;
+            info.deepestExplicitNesting = Math.max(info.deepestExplicitNesting, loopStack.length);
+        }
+        loopStack.push(node);
+    }, node => {
+        // drop again :D
+        if (node.type === "RForLoop" /* RType.ForLoop */ || node.type === "RWhileLoop" /* RType.WhileLoop */ || node.type === "RRepeatLoop" /* RType.RepeatLoop */) {
+            loopStack.pop();
+        }
+    });
+}
+exports.loops = {
+    name: 'Loops',
+    description: 'All looping structures in the document',
+    process(existing, input) {
+        visitLoops(existing, input);
+        return existing;
+    },
+    initialValue: initialLoopInfo,
+    postProcess: post_process_1.postProcess
+};
+//# sourceMappingURL=loops.js.map

package/statistics/features/supported/loops/post-process.d.ts

@@ -0,0 +1,3 @@
+import type { FeatureStatisticsWithMeta } from '../../feature';
+import type { StatisticsSummarizerConfiguration } from '../../../summarizer/summarizer';
+export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;

package/statistics/features/supported/loops/post-process.js

@@ -0,0 +1,71 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.postProcess = void 0;
+const post_processing_1 = require("../../post-processing");
+const common_syntax_probability_1 = require("../../common-syntax-probability");
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const numbers_1 = require("../../../../util/numbers");
+const summarizer_1 = require("../../../../util/summarizer");
+function postProcess(featureRoot, info, outputPath, config) {
+    const collected = {
+        forLoops: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []),
+        forLoopVar: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []),
+        forBody: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []),
+        whileLoops: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []),
+        whileBody: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []),
+        repeatLoops: (0, post_processing_1.emptySummarizedWithProject)(),
+        repeatBody: (0, common_syntax_probability_1.emptyCommonSyntaxTypeCounts)(() => []),
+        breakStatements: (0, post_processing_1.emptySummarizedWithProject)(),
+        nextStatements: (0, post_processing_1.emptySummarizedWithProject)(),
+        implicitLoops: (0, post_processing_1.emptySummarizedWithProject)(),
+        nestedExplicitLoops: (0, post_processing_1.emptySummarizedWithProject)(),
+        deepestExplicitNesting: (0, post_processing_1.emptySummarizedWithProject)()
+    };
+    for (const [filepath, data] of info.entries()) {
+        const value = data.loops;
+        for (const [key, val] of Object.entries(value)) {
+            if (typeof val !== 'object') {
+                const get = collected[key];
+                get.count.push(typeof val === 'number' ? Number(val) : (0, numbers_1.bigint2number)(val));
+                if (val > 0) {
+                    (0, post_processing_1.recordFilePath)(get, filepath, config);
+                }
+            }
+            else {
+                (0, common_syntax_probability_1.appendCommonSyntaxTypeCounter)(collected[key], val);
+            }
+        }
+    }
+    const metaOut = fs_1.default.createWriteStream(path_1.default.join(outputPath, 'loops-meta.csv'));
+    metaOut.write(`kind,unique-projects,unique-files,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+    for (const [key, val] of Object.entries(collected)) {
+        const data = val;
+        if ('uniqueProjects' in data) {
+            metaOut.write(`${JSON.stringify(key)},${data.uniqueProjects.size},${data.uniqueFiles.size},${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(data.count))}\n`);
+        }
+        else {
+            // new file for each :D
+            const out = fs_1.default.createWriteStream(path_1.default.join(outputPath, `loops-type-${key}.csv`));
+            // name is for fields like number etc to allow to group multiple entries
+            out.write(`kind,name,${(0, summarizer_1.summarizedMeasurement2CsvHeader)()}\n`);
+            for (const [name, vals] of Object.entries(data)) {
+                if (Array.isArray(vals)) {
+                    out.write(`${JSON.stringify(name)},"",${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(vals.flat()))}\n`);
+                }
+                else {
+                    for (const [keyName, keyValue] of Object.entries(vals)) {
+                        out.write(`${JSON.stringify(name)},${JSON.stringify(keyName)},${(0, summarizer_1.summarizedMeasurement2Csv)((0, summarizer_1.summarizeMeasurement)(keyValue.flat()))}\n`);
+                    }
+                }
+            }
+            out.close();
+        }
+    }
+    metaOut.close();
+}
+exports.postProcess = postProcess;
+//# sourceMappingURL=post-process.js.map

package/statistics/features/supported/used-functions/post-process.d.ts

@@ -0,0 +1,6 @@
+import type { FeatureStatisticsWithMeta } from '../../feature';
+import type { StatisticsSummarizerConfiguration } from '../../../summarizer/summarizer';
+/**
+ * Note: the summary does not contain a 0 for each function that is _not_ called by a file. Hence, the minimum can not be 0 (division for mean etc. will still be performed on total file count)
+ */
+export declare function postProcess(featureRoot: string, info: Map<string, FeatureStatisticsWithMeta>, outputPath: string, config: StatisticsSummarizerConfiguration): void;