@eagleoutice/flowr 2.2.15 → 2.3.0

package/benchmark/summarizer/first-phase/process.js

@@ -53,6 +53,7 @@ const retriever_1 = require("../../../r-bridge/retriever");
 const visitor_1 = require("../../../r-bridge/lang-4.x/ast/model/processing/visitor");
 const type_1 = require("../../../r-bridge/lang-4.x/ast/model/type");
 const arrays_1 = require("../../../util/collections/arrays");
+const semantics_1 = require("../../../abstract-interpretation/data-frame/semantics");
 const tempfile = (() => {
     let _tempfile = undefined;
     return () => {
@@ -107,10 +108,10 @@ function calculateReductionForSlice(input, dataflow, perSlice, ignoreFluff) {
  * @see Slicer
  */
 async function summarizeSlicerStats(stats, report = () => {
-}) {
+}, engineConf) {
     const collect = new defaultmap_1.DefaultMap(() => []);
     const sizeOfSliceCriteria = [];
-    const reParseShellSession = new shell_1.RShell();
+    const reParseShellSession = new shell_1.RShell(engineConf);
     const sliceTimes = [];
     const reconstructTimes = [];
     const totalTimes = [];
@@ -250,9 +251,54 @@ async function summarizeSlicerStats(stats, report = () => {
                 normalizedTokensNoComments: (0, summarizer_1.summarizeMeasurement)(sliceSize.normalizedTokensNoComments),
                 dataflowNodes: (0, summarizer_1.summarizeMeasurement)(sliceSize.dataflowNodes)
             }
-        }
+        },
+        dataFrameShape: stats.dataFrameShape ? summarizeDfShapeStats(stats.dataFrameShape) : undefined
+    };
+}
+function summarizeDfShapeStats({ perNodeStats, ...stats }) {
+    const nodeStats = perNodeStats.values().toArray();
+    const isTop = (value) => value === 'top';
+    const isInfinite = (value) => value === 'infinite';
+    const isBottom = (value) => value === 'bottom';
+    const isValue = (value) => value !== undefined && !isTop(value) && !isInfinite(value) && !isBottom(value);
+    return {
+        ...stats,
+        numberOfEntriesPerNode: (0, summarizer_1.summarizeMeasurement)(nodeStats.map(s => s.numberOfEntries)),
+        numberOfOperations: (0, arrays_1.arraySum)(nodeStats.map(s => s.mappedOperations?.length).filter(assert_1.isNotUndefined)),
+        numberOfTotalValues: nodeStats.filter(s => isValue(s.inferredColNames) && isValue(s.inferredColCount) && isValue(s.inferredRowCount)).length,
+        numberOfTotalTop: nodeStats.filter(s => isTop(s.inferredColNames) && isTop(s.inferredColCount) && isTop(s.inferredRowCount)).length,
+        numberOfTotalBottom: nodeStats.filter(s => s.inferredColNames === 0 && isBottom(s.inferredColCount) && isBottom(s.inferredRowCount)).length,
+        inferredColNames: (0, summarizer_1.summarizeMeasurement)(nodeStats.map(s => s.inferredColNames).filter(isValue)),
+        numberOfColNamesValues: nodeStats.map(s => s.inferredColNames).filter(isValue).length,
+        numberOfColNamesTop: nodeStats.map(s => s.inferredColNames).filter(isTop).length,
+        numberOfColNamesBottom: nodeStats.map(s => s.inferredColNames).filter(number => number === 0).length,
+        inferredColCount: (0, summarizer_1.summarizeMeasurement)(nodeStats.map(s => s.inferredColCount).filter(isValue)),
+        numberOfColCountExact: nodeStats.map(s => s.approxRangeColCount).filter(range => range === 0).length,
+        numberOfColCountValues: nodeStats.map(s => s.inferredColCount).filter(isValue).length,
+        numberOfColCountTop: nodeStats.map(s => s.inferredColCount).filter(isTop).length,
+        numberOfColCountInfinite: nodeStats.map(s => s.inferredColCount).filter(isInfinite).length,
+        numberOfColCountBottom: nodeStats.map(s => s.inferredColCount).filter(isBottom).length,
+        approxRangeColCount: (0, summarizer_1.summarizeMeasurement)(nodeStats.map(s => s.approxRangeColCount).filter(assert_1.isNotUndefined).filter(isFinite)),
+        inferredRowCount: (0, summarizer_1.summarizeMeasurement)(nodeStats.map(s => s.inferredRowCount).filter(isValue)),
+        numberOfRowCountExact: nodeStats.map(s => s.approxRangeRowCount).filter(range => range === 0).length,
+        numberOfRowCountValues: nodeStats.map(s => s.inferredRowCount).filter(isValue).length,
+        numberOfRowCountTop: nodeStats.map(s => s.inferredRowCount).filter(isTop).length,
+        numberOfRowCountInfinite: nodeStats.map(s => s.inferredRowCount).filter(isInfinite).length,
+        numberOfRowCountBottom: nodeStats.map(s => s.inferredRowCount).filter(isBottom).length,
+        approxRangeRowCount: (0, summarizer_1.summarizeMeasurement)(nodeStats.map(s => s.approxRangeRowCount).filter(assert_1.isNotUndefined).filter(isFinite)),
+        perOperationNumber: summarizePerOperationStats(nodeStats),
     };
 }
+function summarizePerOperationStats(nodeStats) {
+    const perOperationNumber = new Map(semantics_1.DataFrameOperationNames.map(name => [name, 0]));
+    for (const stat of nodeStats) {
+        for (const operation of stat.mappedOperations ?? []) {
+            const value = perOperationNumber.get(operation) ?? 0;
+            perOperationNumber.set(operation, value + 1);
+        }
+    }
+    return perOperationNumber;
+}
 function summarizeSummarizedMeasurement(data) {
     data = data.filter(assert_1.isNotUndefined);
     const min = Math.min(...data.map(d => d.min).filter(assert_1.isNotUndefined));

package/benchmark/summarizer/second-phase/process.js

@@ -9,6 +9,8 @@ const defaultmap_1 = require("../../../util/collections/defaultmap");
 const summarizer_1 = require("../../../util/summarizer");
 const assert_1 = require("../../../util/assert");
 const stats_1 = require("../../stats/stats");
+const semantics_1 = require("../../../abstract-interpretation/data-frame/semantics");
+const arrays_1 = require("../../../util/collections/arrays");
 function summarizeAllSummarizedStats(stats) {
     const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
     const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
@@ -19,11 +21,14 @@ function summarizeAllSummarizedStats(stats) {
     const normalizeTimesPerToken = [];
     const dataflowTimesPerToken = [];
     const totalCommonTimesPerToken = [];
+    const controlFlowTimePerToken = [];
+    const dataFrameShapeTimePerToken = [];
     const memory = new defaultmap_1.DefaultMap(() => []);
     const reductions = [];
     const reductionsNoFluff = [];
     const inputs = [];
     const dataflows = [];
+    const dataFrameShapes = [];
     let failedToRepParse = 0;
     let timesHitThreshold = 0;
     let totalSlices = 0;
@@ -41,6 +46,12 @@ function summarizeAllSummarizedStats(stats) {
         normalizeTimesPerToken.push(stat.normalizeTimePerToken);
         dataflowTimesPerToken.push(stat.dataflowTimePerToken);
         totalCommonTimesPerToken.push(stat.totalCommonTimePerToken);
+        if (stat.controlFlowTimePerToken !== undefined) {
+            controlFlowTimePerToken.push(stat.controlFlowTimePerToken);
+        }
+        if (stat.dataFrameShapeTimePerToken !== undefined) {
+            dataFrameShapeTimePerToken.push(stat.dataFrameShapeTimePerToken);
+        }
         for (const [k, v] of stat.memory) {
             memory.get(k).push(v);
         }
@@ -48,6 +59,9 @@ function summarizeAllSummarizedStats(stats) {
         reductionsNoFluff.push(stat.perSliceMeasurements.reductionNoFluff);
         inputs.push(stat.input);
         dataflows.push(stat.dataflow);
+        if (stat.dataFrameShape !== undefined) {
+            dataFrameShapes.push(stat.dataFrameShape);
+        }
         failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
         totalSlices += stat.perSliceMeasurements.numberOfSlices;
         timesHitThreshold += stat.perSliceMeasurements.timesHitThreshold;
@@ -64,6 +78,8 @@ function summarizeAllSummarizedStats(stats) {
         normalizeTimePerToken: (0, process_1.summarizeTimePerToken)(normalizeTimesPerToken),
         dataflowTimePerToken: (0, process_1.summarizeTimePerToken)(dataflowTimesPerToken),
         totalCommonTimePerToken: (0, process_1.summarizeTimePerToken)(totalCommonTimesPerToken),
+        controlFlowTimePerToken: controlFlowTimePerToken.length > 0 ? (0, process_1.summarizeTimePerToken)(controlFlowTimePerToken) : undefined,
+        dataFrameShapeTimePerToken: dataFrameShapeTimePerToken.length > 0 ? (0, process_1.summarizeTimePerToken)(dataFrameShapeTimePerToken) : undefined,
         failedToRepParse,
         timesHitThreshold,
         reduction: (0, process_1.summarizeSummarizedReductions)(reductions),
@@ -89,7 +105,43 @@ function summarizeAllSummarizedStats(stats) {
             storedVertexIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.storedVertexIndices)),
             storedEnvIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.storedEnvIndices)),
             overwrittenIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.overwrittenIndices)),
-        }
+        },
+        dataFrameShape: stats.some(s => s.dataFrameShape !== undefined) ? {
+            numberOfDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfNonDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfNonDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfResultConstraints: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultConstraints).filter(assert_1.isNotUndefined)),
+            numberOfResultingValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingValues).filter(assert_1.isNotUndefined)),
+            numberOfResultingTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingTop).filter(assert_1.isNotUndefined)),
+            numberOfResultingBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingBottom).filter(assert_1.isNotUndefined)),
+            numberOfEmptyNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEmptyNodes).filter(assert_1.isNotUndefined)),
+            numberOfOperationNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperationNodes).filter(assert_1.isNotUndefined)),
+            numberOfValueNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfValueNodes).filter(assert_1.isNotUndefined)),
+            sizeOfInfo: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.sizeOfInfo).filter(assert_1.isNotUndefined)),
+            numberOfEntriesPerNode: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEntriesPerNode).filter(assert_1.isNotUndefined)),
+            numberOfOperations: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperations).filter(assert_1.isNotUndefined)),
+            numberOfTotalValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalValues).filter(assert_1.isNotUndefined)),
+            numberOfTotalTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalTop).filter(assert_1.isNotUndefined)),
+            numberOfTotalBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalBottom).filter(assert_1.isNotUndefined)),
+            inferredColNames: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColNames).filter(assert_1.isNotUndefined)),
+            numberOfColNamesValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesValues).filter(assert_1.isNotUndefined)),
+            numberOfColNamesTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesTop).filter(assert_1.isNotUndefined)),
+            numberOfColNamesBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesBottom).filter(assert_1.isNotUndefined)),
+            inferredColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColCount).filter(assert_1.isNotUndefined)),
+            numberOfColCountExact: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountExact).filter(assert_1.isNotUndefined)),
+            numberOfColCountValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountValues).filter(assert_1.isNotUndefined)),
+            numberOfColCountTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountTop).filter(assert_1.isNotUndefined)),
+            numberOfColCountInfinite: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfColCountBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeColCount).filter(assert_1.isNotUndefined)),
+            inferredRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredRowCount).filter(assert_1.isNotUndefined)),
+            numberOfRowCountExact: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountExact).filter(assert_1.isNotUndefined)),
+            numberOfRowCountValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountValues).filter(assert_1.isNotUndefined)),
+            numberOfRowCountTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountTop).filter(assert_1.isNotUndefined)),
+            numberOfRowCountInfinite: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfRowCountBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeRowCount).filter(assert_1.isNotUndefined)),
+            perOperationNumber: new Map(semantics_1.DataFrameOperationNames.map(n => [n, (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.perOperationNumber.get(n) ?? 0))]))
+        } : undefined
     };
 }
 function summarizeAllUltimateStats(stats) {
@@ -109,6 +161,8 @@ function summarizeAllUltimateStats(stats) {
         normalizeTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.normalizeTimePerToken)),
         dataflowTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataflowTimePerToken)),
         totalCommonTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.totalCommonTimePerToken)),
+        controlFlowTimePerToken: stats.some(s => s.controlFlowTimePerToken !== undefined) ? (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.controlFlowTimePerToken).filter(assert_1.isNotUndefined)) : undefined,
+        dataFrameShapeTimePerToken: stats.some(s => s.dataFrameShapeTimePerToken !== undefined) ? (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataFrameShapeTimePerToken).filter(assert_1.isNotUndefined)) : undefined,
         reduction: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reduction)),
         reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reductionNoFluff)),
         input: {
@@ -132,7 +186,43 @@ function summarizeAllUltimateStats(stats) {
             storedVertexIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.storedVertexIndices)),
             storedEnvIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.storedEnvIndices)),
             overwrittenIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.overwrittenIndices)),
-        }
+        },
+        dataFrameShape: stats.some(s => s.dataFrameShape !== undefined) ? {
+            numberOfDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfNonDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfNonDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfResultConstraints: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultConstraints).filter(assert_1.isNotUndefined)),
+            numberOfResultingValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingValues).filter(assert_1.isNotUndefined)),
+            numberOfResultingTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingTop).filter(assert_1.isNotUndefined)),
+            numberOfResultingBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingBottom).filter(assert_1.isNotUndefined)),
+            numberOfEmptyNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEmptyNodes).filter(assert_1.isNotUndefined)),
+            numberOfOperationNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperationNodes).filter(assert_1.isNotUndefined)),
+            numberOfValueNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfValueNodes).filter(assert_1.isNotUndefined)),
+            sizeOfInfo: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.sizeOfInfo).filter(assert_1.isNotUndefined)),
+            numberOfEntriesPerNode: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEntriesPerNode).filter(assert_1.isNotUndefined)),
+            numberOfOperations: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperations).filter(assert_1.isNotUndefined)),
+            numberOfTotalValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalValues).filter(assert_1.isNotUndefined)),
+            numberOfTotalTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalTop).filter(assert_1.isNotUndefined)),
+            numberOfTotalBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalBottom).filter(assert_1.isNotUndefined)),
+            inferredColNames: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColNames).filter(assert_1.isNotUndefined)),
+            numberOfColNamesValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesValues).filter(assert_1.isNotUndefined)),
+            numberOfColNamesTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesTop).filter(assert_1.isNotUndefined)),
+            numberOfColNamesBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesBottom).filter(assert_1.isNotUndefined)),
+            inferredColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColCount).filter(assert_1.isNotUndefined)),
+            numberOfColCountExact: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountExact).filter(assert_1.isNotUndefined)),
+            numberOfColCountValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountValues).filter(assert_1.isNotUndefined)),
+            numberOfColCountTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountTop).filter(assert_1.isNotUndefined)),
+            numberOfColCountInfinite: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfColCountBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeColCount).filter(assert_1.isNotUndefined)),
+            inferredRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredRowCount).filter(assert_1.isNotUndefined)),
+            numberOfRowCountExact: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountExact).filter(assert_1.isNotUndefined)),
+            numberOfRowCountValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountValues).filter(assert_1.isNotUndefined)),
+            numberOfRowCountTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountTop).filter(assert_1.isNotUndefined)),
+            numberOfRowCountInfinite: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfRowCountBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeRowCount).filter(assert_1.isNotUndefined)),
+            perOperationNumber: new Map(semantics_1.DataFrameOperationNames.map(n => [n, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.perOperationNumber.get(n)).filter(assert_1.isNotUndefined))]))
+        } : undefined
     };
 }
 function processNextSummary(line, allSummarized) {
@@ -152,7 +242,11 @@ function processNextSummary(line, allSummarized) {
                 ...got.summarize.perSliceMeasurements,
                 // restore maps
                 measurements: new Map(got.summarize.perSliceMeasurements.measurements),
-            }
+            },
+            dataFrameShape: got.summarize.dataFrameShape !== undefined ? {
+                ...got.summarize.dataFrameShape,
+                perOperationNumber: new Map(got.summarize.dataFrameShape.perOperationNumber)
+            } : undefined
         }
     };
     allSummarized.push(got.summarize);
@@ -164,6 +258,10 @@ function processNextUltimateSummary(line, allSummarized) {
         // restore maps
         commonMeasurements: new Map(got.commonMeasurements),
         perSliceMeasurements: new Map(got.perSliceMeasurements),
+        dataFrameShape: got.dataFrameShape !== undefined ? {
+            ...got.dataFrameShape,
+            perOperationNumber: new Map(got.dataFrameShape.perOperationNumber)
+        } : undefined
     };
     allSummarized.push(got);
 }

package/cli/benchmark-app.d.ts

@@ -8,7 +8,9 @@ export interface BenchmarkCliOptions {
     parallel: number;
     limit?: number;
     runs?: number;
+    seed?: string;
     parser: KnownParserName;
+    'dataframe-shape-inference': boolean;
     'enable-pointer-tracking': boolean;
     'max-file-slices': number;
     threshold?: number;

package/cli/benchmark-app.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
+const seedrandom_1 = __importDefault(require("seedrandom"));
 const assert_1 = require("../util/assert");
 const files_1 = require("../util/files");
 const log_1 = require("../util/log");
@@ -56,8 +57,9 @@ async function benchmark() {
     }
     if (options.limit) {
         log_1.log.info(`limiting to ${options.limit} files`);
+        const random = options.seed ? (0, seedrandom_1.default)(options.seed) : Math.random;
         // shuffle and limit
-        files.sort(() => Math.random() - 0.5);
+        files.sort(() => random() - 0.5);
     }
     const limit = options.limit ?? files.length;
     const verboseAdd = options.verbose ? ['--verbose'] : [];
@@ -67,10 +69,12 @@ async function benchmark() {
         '--output', path_1.default.join(options.output, path_1.default.relative(f.baseDir, `${f.request.content}.json`)),
         '--slice', options.slice, ...verboseAdd,
         '--parser', options.parser,
+        ...(options['dataframe-shape-inference'] ? ['--dataframe-shape-inference'] : []),
         ...(options['enable-pointer-tracking'] ? ['--enable-pointer-tracking'] : []),
         '--max-slices', `${options['max-file-slices']}`,
         ...(options.threshold ? ['--threshold', `${options.threshold}`] : []),
         '--sampling-strategy', options['sampling-strategy'],
+        ...(options.seed ? ['--seed', options.seed] : []),
     ]);
     const runs = options.runs ?? 1;
     for (let i = 1; i <= runs; i++) {

package/cli/benchmark-helper-app.d.ts

@@ -8,8 +8,10 @@ export interface SingleBenchmarkCliOptions {
     slice: string;
     output?: string;
     parser: KnownParserName;
+    'dataframe-shape-inference': boolean;
     'enable-pointer-tracking': boolean;
     'max-slices': number;
     threshold?: number;
     'sampling-strategy': string;
+    seed?: string;
 }

package/cli/benchmark-helper-app.js

@@ -39,12 +39,10 @@ async function benchmark() {
         fs_1.default.mkdirSync(directory, { recursive: true });
     }
     // Enable pointer analysis if requested, otherwise disable it
-    if (options['enable-pointer-tracking']) {
-        (0, config_1.amendConfig)({ solver: { ...(0, config_1.getConfig)().solver, pointerTracking: true, } });
-    }
-    else {
-        (0, config_1.amendConfig)({ solver: { ...(0, config_1.getConfig)().solver, pointerTracking: false, } });
-    }
+    const config = (0, config_1.amendConfig)((0, config_1.getConfig)(), c => {
+        c.solver.pointerTracking = options['enable-pointer-tracking'];
+        return c;
+    });
     // ensure the file exists
     const fileStat = fs_1.default.statSync(options.input);
     (0, assert_1.guard)(fileStat.isFile(), `File ${options.input} does not exist or is no file`);
@@ -52,7 +50,7 @@ async function benchmark() {
     const maxSlices = options['max-slices'] ?? -1;
     const slicer = new slicer_1.BenchmarkSlicer(options.parser);
     try {
-        await slicer.init(request, undefined, options.threshold);
+        await slicer.init(request, config, undefined, options.threshold);
         // ${escape}1F${escape}1G${escape}2K for line reset
         if (options.slice === 'all') {
             const count = await slicer.sliceForAll(all_variables_1.DefaultAllVariablesFilter, (i, total, arr) => console.log(`${prefix} Slicing ${i + 1}/${total} [${JSON.stringify(arr[i])}]`), { maxSliceCount: maxSlices });
@@ -66,11 +64,18 @@ async function benchmark() {
         else {
             const limit = parseInt(options.slice);
             console.log(`${prefix} Slicing up to ${limit} possible slices`);
-            const count = await slicer.sliceForAll(all_variables_1.DefaultAllVariablesFilter, (i, total, arr) => console.log(`${prefix} Slicing ${i + 1}/${total} [${JSON.stringify(arr[i])}]`), { sampleCount: limit, maxSliceCount: maxSlices, sampleStrategy: options['sampling-strategy'] });
+            const count = await slicer.sliceForAll(all_variables_1.DefaultAllVariablesFilter, (i, total, arr) => console.log(`${prefix} Slicing ${i + 1}/${total} [${JSON.stringify(arr[i])}]`), { sampleCount: limit, maxSliceCount: maxSlices, sampleStrategy: options['sampling-strategy'], seed: options.seed });
             console.log(`${prefix} Completed Slicing`);
             (0, assert_1.guard)(count >= 0, `Number of slices exceeded limit of ${maxSlices} with ${-count} slices, skipping in count`);
             (0, assert_1.guard)(count > 0, `No possible slices found for ${options.input}, skipping in count`);
         }
+        if (options['dataframe-shape-inference']) {
+            console.log(`${prefix} Extracting control flow graph for data frame shape inference`);
+            slicer.extractCFG();
+            console.log(`${prefix} Performing shape inference for data frames`);
+            slicer.inferDataFrameShapes();
+            console.log(`${prefix} Completed data frame shape inference`);
+        }
         const { stats } = slicer.finish();
         const output = {
             filename: options.input,

package/cli/common/options.js

@@ -16,11 +16,13 @@ exports.benchmarkOptions = [
     { name: 'help', alias: 'h', type: Boolean, description: 'Print this usage guide' },
     { name: 'limit', alias: 'l', type: Number, description: 'Limit the number of files to process (if given, this will choose these files randomly and add the chosen names to the output' },
     { name: 'runs', alias: 'r', type: Number, description: 'The amount of benchmark runs that should be done, out of which an average will be calculated' },
+    { name: 'seed', type: String, description: 'The random seed for sampling the files if a limit is set, and for sampling the slicing criteria if a maximum is set' },
     { name: 'input', alias: 'i', type: String, description: 'Pass a folder or file as src to read from. Alternatively, pass a single JSON file that contains a list of paths.', multiple: true, defaultOption: true, defaultValue: [], typeLabel: '{underline files/folders}' },
     { name: 'parallel', alias: 'p', type: String, description: 'Number of parallel executors (defaults to {italic max(cpu.count-1, 1)})', defaultValue: Math.max(os_1.default.cpus().length - 1, 1), typeLabel: '{underline number}' },
     { name: 'slice', alias: 's', type: String, description: 'Automatically slice for *all* variables (default) or *no* slicing and only parsing/dataflow construction. Numbers will indicate: sample X random slices from all.', defaultValue: 'all', typeLabel: '{underline all/no}' },
     { name: 'output', alias: 'o', type: String, description: `Folder to write all the measurements to in a per-file-basis (defaults to {italic benchmark-${StartTimeString}})`, defaultValue: `benchmark-${StartTimeString}`, typeLabel: '{underline folder}' },
     { name: 'parser', type: String, description: 'The parser to use for the benchmark', defaultValue: 'r-shell', typeLabel: '{underline parser}' },
+    { name: 'dataframe-shape-inference', type: Boolean, description: 'Infer the shape of data frames using abstract interpretation (includes control flow graph extraction)', defaultValue: false },
     { name: 'enable-pointer-tracking', type: Boolean, description: 'Run dataflow analysis with pointer tracking', defaultValue: false },
     { name: 'max-file-slices', type: Number, description: 'If file has more than passed number of slices, the file is not processed', defaultValue: -1, typeLabel: '{underline number}' },
     { name: 'threshold', alias: 't', type: Number, description: 'How many re-visits of the same node are ok?', defaultValue: undefined, typeLabel: '{underline number}' },
@@ -36,10 +38,12 @@ exports.benchmarkHelperOptions = [
     { name: 'slice', alias: 's', type: String, description: 'Automatically slice for *all* variables (default) or *no* slicing and only parsing/dataflow construction. Numbers will indicate: sample X random slices from all.', defaultValue: 'all', typeLabel: '{underline all/no}' },
     { name: 'output', alias: 'o', type: String, description: 'File to write the measurements to (appends a single line in JSON format)', typeLabel: '{underline file}' },
     { name: 'parser', type: String, description: 'The parser to use for the benchmark', defaultValue: 'r-shell', typeLabel: '{underline parser}' },
+    { name: 'dataframe-shape-inference', type: Boolean, description: 'Infer the shape of data frames using abstract interpretation (includes control flow graph extraction)', defaultValue: false },
     { name: 'enable-pointer-tracking', type: Boolean, description: 'Run dataflow analysis with pointer tracking', defaultValue: false },
     { name: 'max-slices', type: Number, description: 'If file has more than passed number of slices, the file is not processed', defaultValue: -1, typeLabel: '{underline number}' },
     { name: 'threshold', alias: 't', type: Number, description: 'How many re-visits of the same node are ok?', defaultValue: undefined, typeLabel: '{underline number}' },
     { name: 'sampling-strategy', type: String, description: 'Which strategy to use, when sampling is enabled', defaultValue: 'random', typeLabel: '{underline random/equidistant}' },
+    { name: 'seed', type: String, description: 'The random seed for sampling the slicing criteria if a maximum is set' },
 ];
 exports.exportQuadsOptions = [
     { name: 'verbose', alias: 'v', type: Boolean, description: 'Run with verbose logging' },

package/cli/export-quads-app.js

@@ -10,6 +10,7 @@ const files_1 = require("../util/files");
 const script_1 = require("./common/script");
 const shell_1 = require("../r-bridge/shell");
 const retriever_1 = require("../r-bridge/retriever");
+const config_1 = require("../config");
 const options = (0, script_1.processCommandLineArgs)('export-quads', [], {
     subtitle: 'Generate RDF N-Quads from the AST of a given R script',
     examples: [
@@ -17,7 +18,7 @@ const options = (0, script_1.processCommandLineArgs)('export-quads', [], {
         '{bold --help}'
     ]
 });
-const shell = new shell_1.RShell();
+const shell = new shell_1.RShell((0, config_1.getEngineConfig)((0, config_1.getConfig)(), 'r-shell'));
 async function writeQuadForSingleFile(request, output) {
     const normalized = await (0, retriever_1.retrieveNormalizedAstFromRCode)(request, shell);
     const serialized = (0, quads_1.serialize2quads)(normalized.ast, { context: request.content });

package/cli/flowr.js

@@ -52,45 +52,51 @@ if (options['no-ansi']) {
     log_1.log.info('disabling ansi colors');
     (0, ansi_1.setFormatter)(ansi_1.voidFormatter);
 }
-let usedConfig = false;
-if (options['config-json']) {
-    const config = (0, config_1.parseConfig)(options['config-json']);
-    if (config) {
-        log_1.log.info(`Using passed config ${JSON.stringify(config)}`);
-        (0, config_1.setConfig)(config);
-        usedConfig = true;
+function createConfig() {
+    let config;
+    if (options['config-json']) {
+        const passedConfig = (0, config_1.parseConfig)(options['config-json']);
+        if (passedConfig) {
+            log_1.log.info(`Using passed config ${JSON.stringify(passedConfig)}`);
+            config = passedConfig;
+        }
     }
-}
-if (!usedConfig) {
-    if (options['config-file']) {
-        // validate it exists
-        if (!fs_1.default.existsSync(path_1.default.resolve(options['config-file']))) {
-            log_1.log.error(`Config file '${options['config-file']}' does not exist`);
-            process.exit(1);
+    if (config == undefined) {
+        if (options['config-file']) {
+            // validate it exists
+            if (!fs_1.default.existsSync(path_1.default.resolve(options['config-file']))) {
+                log_1.log.error(`Config file '${options['config-file']}' does not exist`);
+                process.exit(1);
+            }
         }
+        config = (0, config_1.getConfig)(options['config-file'] ?? flowr_main_options_1.defaultConfigFile);
     }
-    (0, config_1.setConfigFile)(options['config-file'] ?? flowr_main_options_1.defaultConfigFile, undefined, true);
-}
-// for all options that we manually supply that have a config equivalent, set them in the config
-if (!options['engine.r-shell.disabled']) {
-    (0, config_1.amendConfig)({ engines: [{ type: 'r-shell', rPath: options['r-path'] || options['engine.r-shell.r-path'] }] });
-}
-if (!options['engine.tree-sitter.disabled']) {
-    (0, config_1.amendConfig)({ engines: [{
+    // for all options that we manually supply that have a config equivalent, set them in the config
+    config = (0, config_1.amendConfig)(config, c => {
+        c.engines ??= [];
+        if (!options['engine.r-shell.disabled']) {
+            c.engines.push({ type: 'r-shell', rPath: options['r-path'] || options['engine.r-shell.r-path'] });
+        }
+        if (!options['engine.tree-sitter.disabled']) {
+            c.engines.push({
                 type: 'tree-sitter',
                 wasmPath: options['engine.tree-sitter.wasm-path'],
                 treeSitterWasmPath: options['engine.tree-sitter.tree-sitter-wasm-path'],
                 lax: options['engine.tree-sitter.lax']
-            }] });
-}
-if (options['default-engine']) {
-    (0, config_1.amendConfig)({ defaultEngine: options['default-engine'] });
+            });
+        }
+        if (options['default-engine']) {
+            c.defaultEngine = options['default-engine'];
+        }
+        return c;
+    });
+    return config;
 }
-async function retrieveEngineInstances() {
+async function retrieveEngineInstances(config) {
     const engines = {};
-    if ((0, config_1.getEngineConfig)('r-shell')) {
+    if ((0, config_1.getEngineConfig)(config, 'r-shell')) {
         // we keep an active shell session to allow other parse investigations :)
-        engines['r-shell'] = new shell_1.RShell({
+        engines['r-shell'] = new shell_1.RShell((0, config_1.getEngineConfig)(config, 'r-shell'), {
             revive: 2 /* RShellReviveOptions.Always */,
             onRevive: (code, signal) => {
                 const signalText = signal == null ? '' : ` and signal ${signal}`;
@@ -99,11 +105,11 @@ async function retrieveEngineInstances() {
             }
         });
     }
-    if ((0, config_1.getEngineConfig)('tree-sitter')) {
-        await tree_sitter_executor_1.TreeSitterExecutor.initTreeSitter();
+    if ((0, config_1.getEngineConfig)(config, 'tree-sitter')) {
+        await tree_sitter_executor_1.TreeSitterExecutor.initTreeSitter((0, config_1.getEngineConfig)(config, 'tree-sitter'));
         engines['tree-sitter'] = new tree_sitter_executor_1.TreeSitterExecutor();
     }
-    let defaultEngine = (0, config_1.getConfig)().defaultEngine;
+    let defaultEngine = config.defaultEngine;
     if (!defaultEngine || !engines[defaultEngine]) {
         // if a default engine isn't specified, we just take the first one we have
         defaultEngine = Object.keys(engines)[0];
@@ -111,7 +117,19 @@ async function retrieveEngineInstances() {
     log_1.log.info(`Using engines ${Object.keys(engines).join(', ')} with default ${defaultEngine}`);
     return { engines, default: defaultEngine };
 }
+function hookSignalHandlers(engines) {
+    const end = () => {
+        if (options.execute === undefined) {
+            console.log(`\n${(0, ansi_1.italic)('Exiting...')}`);
+        }
+        Object.values(engines.engines).forEach(e => e?.close());
+        process.exit(0);
+    };
+    process.on('SIGINT', end);
+    process.on('SIGTERM', end);
+}
 async function mainRepl() {
+    const config = createConfig();
     if (options.script) {
         const target = scripts_info_1.scripts[options.script].target;
         (0, assert_1.guard)(target !== undefined, `Unknown script ${options.script}, pick one of ${(0, flowr_main_options_1.getScriptsText)()}.`);
@@ -124,7 +142,7 @@ async function mainRepl() {
         console.log((0, command_line_usage_1.default)(exports.optionHelp));
         process.exit(0);
     }
-    const engines = await retrieveEngineInstances();
+    const engines = await retrieveEngineInstances(config);
     const defaultEngine = engines.engines[engines.default];
     if (options.version) {
         for (const engine of Object.values(engines.engines)) {
@@ -133,39 +151,22 @@ async function mainRepl() {
         }
         process.exit(0);
     }
-    const end = () => {
-        if (options.execute === undefined) {
-            console.log(`\n${(0, ansi_1.italic)('Exiting...')}`);
-        }
-        Object.values(engines.engines).forEach(e => e?.close());
-        process.exit(0);
-    };
-    // hook some handlers
-    process.on('SIGINT', end);
-    process.on('SIGTERM', end);
+    hookSignalHandlers(engines);
     const allowRSessionAccess = options['r-session-access'] ?? false;
     if (options.execute) {
-        await (0, core_1.replProcessAnswer)(repl_main_1.standardReplOutput, options.execute, defaultEngine, allowRSessionAccess);
+        await (0, core_1.replProcessAnswer)(config, repl_main_1.standardReplOutput, options.execute, defaultEngine, allowRSessionAccess);
     }
     else {
         await (0, print_version_1.printVersionRepl)(defaultEngine);
-        await (0, core_1.repl)({ parser: defaultEngine, allowRSessionAccess });
+        await (0, core_1.repl)(config, { parser: defaultEngine, allowRSessionAccess });
     }
     process.exit(0);
 }
 async function mainServer(backend = new net_1.NetServer()) {
-    const engines = await retrieveEngineInstances();
-    const end = () => {
-        if (options.execute === undefined) {
-            console.log(`\n${(0, ansi_1.italic)('Exiting...')}`);
-        }
-        Object.values(engines.engines).forEach(e => e?.close());
-        process.exit(0);
-    };
-    // hook some handlers
-    process.on('SIGINT', end);
-    process.on('SIGTERM', end);
-    await new server_1.FlowRServer(engines.engines, engines.default, options['r-session-access'], backend).start(options.port);
+    const config = createConfig();
+    const engines = await retrieveEngineInstances(config);
+    hookSignalHandlers(engines);
+    await new server_1.FlowRServer(engines.engines, engines.default, options['r-session-access'], config, backend).start(options.port);
 }
 if (options.server) {
     void mainServer(options.ws ? new net_1.WebSocketServerWrapper() : new net_1.NetServer());