@eagleoutice/flowr 2.2.16 → 2.4.0

package/benchmark/summarizer/second-phase/process.js

@@ -9,6 +9,8 @@ const defaultmap_1 = require("../../../util/collections/defaultmap");
 const summarizer_1 = require("../../../util/summarizer");
 const assert_1 = require("../../../util/assert");
 const stats_1 = require("../../stats/stats");
+const semantics_1 = require("../../../abstract-interpretation/data-frame/semantics");
+const arrays_1 = require("../../../util/collections/arrays");
 function summarizeAllSummarizedStats(stats) {
     const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
     const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
@@ -19,11 +21,14 @@ function summarizeAllSummarizedStats(stats) {
     const normalizeTimesPerToken = [];
     const dataflowTimesPerToken = [];
     const totalCommonTimesPerToken = [];
+    const controlFlowTimePerToken = [];
+    const dataFrameShapeTimePerToken = [];
     const memory = new defaultmap_1.DefaultMap(() => []);
     const reductions = [];
     const reductionsNoFluff = [];
     const inputs = [];
     const dataflows = [];
+    const dataFrameShapes = [];
     let failedToRepParse = 0;
     let timesHitThreshold = 0;
     let totalSlices = 0;
@@ -41,6 +46,12 @@ function summarizeAllSummarizedStats(stats) {
         normalizeTimesPerToken.push(stat.normalizeTimePerToken);
         dataflowTimesPerToken.push(stat.dataflowTimePerToken);
         totalCommonTimesPerToken.push(stat.totalCommonTimePerToken);
+        if (stat.controlFlowTimePerToken !== undefined) {
+            controlFlowTimePerToken.push(stat.controlFlowTimePerToken);
+        }
+        if (stat.dataFrameShapeTimePerToken !== undefined) {
+            dataFrameShapeTimePerToken.push(stat.dataFrameShapeTimePerToken);
+        }
         for (const [k, v] of stat.memory) {
             memory.get(k).push(v);
         }
@@ -48,6 +59,9 @@ function summarizeAllSummarizedStats(stats) {
         reductionsNoFluff.push(stat.perSliceMeasurements.reductionNoFluff);
         inputs.push(stat.input);
         dataflows.push(stat.dataflow);
+        if (stat.dataFrameShape !== undefined) {
+            dataFrameShapes.push(stat.dataFrameShape);
+        }
         failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
         totalSlices += stat.perSliceMeasurements.numberOfSlices;
         timesHitThreshold += stat.perSliceMeasurements.timesHitThreshold;
@@ -64,6 +78,8 @@ function summarizeAllSummarizedStats(stats) {
         normalizeTimePerToken: (0, process_1.summarizeTimePerToken)(normalizeTimesPerToken),
         dataflowTimePerToken: (0, process_1.summarizeTimePerToken)(dataflowTimesPerToken),
         totalCommonTimePerToken: (0, process_1.summarizeTimePerToken)(totalCommonTimesPerToken),
+        controlFlowTimePerToken: controlFlowTimePerToken.length > 0 ? (0, process_1.summarizeTimePerToken)(controlFlowTimePerToken) : undefined,
+        dataFrameShapeTimePerToken: dataFrameShapeTimePerToken.length > 0 ? (0, process_1.summarizeTimePerToken)(dataFrameShapeTimePerToken) : undefined,
         failedToRepParse,
         timesHitThreshold,
         reduction: (0, process_1.summarizeSummarizedReductions)(reductions),
@@ -89,7 +105,43 @@ function summarizeAllSummarizedStats(stats) {
             storedVertexIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.storedVertexIndices)),
             storedEnvIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.storedEnvIndices)),
             overwrittenIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.overwrittenIndices)),
-        }
+        },
+        dataFrameShape: stats.some(s => s.dataFrameShape !== undefined) ? {
+            numberOfDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfNonDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfNonDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfResultConstraints: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultConstraints).filter(assert_1.isNotUndefined)),
+            numberOfResultingValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingValues).filter(assert_1.isNotUndefined)),
+            numberOfResultingTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingTop).filter(assert_1.isNotUndefined)),
+            numberOfResultingBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingBottom).filter(assert_1.isNotUndefined)),
+            numberOfEmptyNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEmptyNodes).filter(assert_1.isNotUndefined)),
+            numberOfOperationNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperationNodes).filter(assert_1.isNotUndefined)),
+            numberOfValueNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfValueNodes).filter(assert_1.isNotUndefined)),
+            sizeOfInfo: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.sizeOfInfo).filter(assert_1.isNotUndefined)),
+            numberOfEntriesPerNode: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEntriesPerNode).filter(assert_1.isNotUndefined)),
+            numberOfOperations: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperations).filter(assert_1.isNotUndefined)),
+            numberOfTotalValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalValues).filter(assert_1.isNotUndefined)),
+            numberOfTotalTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalTop).filter(assert_1.isNotUndefined)),
+            numberOfTotalBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalBottom).filter(assert_1.isNotUndefined)),
+            inferredColNames: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColNames).filter(assert_1.isNotUndefined)),
+            numberOfColNamesValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesValues).filter(assert_1.isNotUndefined)),
+            numberOfColNamesTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesTop).filter(assert_1.isNotUndefined)),
+            numberOfColNamesBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesBottom).filter(assert_1.isNotUndefined)),
+            inferredColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColCount).filter(assert_1.isNotUndefined)),
+            numberOfColCountExact: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountExact).filter(assert_1.isNotUndefined)),
+            numberOfColCountValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountValues).filter(assert_1.isNotUndefined)),
+            numberOfColCountTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountTop).filter(assert_1.isNotUndefined)),
+            numberOfColCountInfinite: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfColCountBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeColCount).filter(assert_1.isNotUndefined)),
+            inferredRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredRowCount).filter(assert_1.isNotUndefined)),
+            numberOfRowCountExact: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountExact).filter(assert_1.isNotUndefined)),
+            numberOfRowCountValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountValues).filter(assert_1.isNotUndefined)),
+            numberOfRowCountTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountTop).filter(assert_1.isNotUndefined)),
+            numberOfRowCountInfinite: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfRowCountBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeRowCount).filter(assert_1.isNotUndefined)),
+            perOperationNumber: new Map(semantics_1.DataFrameOperationNames.map(n => [n, (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.perOperationNumber.get(n) ?? 0))]))
+        } : undefined
     };
 }
 function summarizeAllUltimateStats(stats) {
@@ -100,7 +152,7 @@ function summarizeAllUltimateStats(stats) {
         failedToRepParse: Math.max(...stats.map(s => s.failedToRepParse)),
         timesHitThreshold: Math.max(...stats.map(s => s.timesHitThreshold)),
         // average out / summarize other measurements
-        commonMeasurements: new Map(stats_1.CommonSlicerMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.commonMeasurements.get(m)))])),
+        commonMeasurements: new Map(stats_1.CommonSlicerMeasurements.filter(m => stats.some(s => s.commonMeasurements.has(m))).map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.commonMeasurements.get(m)))])),
         perSliceMeasurements: new Map(stats_1.PerSliceMeasurements.map(m => [m, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.perSliceMeasurements.get(m)))])),
         sliceTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.sliceTimePerToken)),
         reconstructTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.reconstructTimePerToken)),
@@ -109,6 +161,8 @@ function summarizeAllUltimateStats(stats) {
         normalizeTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.normalizeTimePerToken)),
         dataflowTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataflowTimePerToken)),
         totalCommonTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.totalCommonTimePerToken)),
+        controlFlowTimePerToken: stats.some(s => s.controlFlowTimePerToken !== undefined) ? (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.controlFlowTimePerToken).filter(assert_1.isNotUndefined)) : undefined,
+        dataFrameShapeTimePerToken: stats.some(s => s.dataFrameShapeTimePerToken !== undefined) ? (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataFrameShapeTimePerToken).filter(assert_1.isNotUndefined)) : undefined,
         reduction: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reduction)),
         reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reductionNoFluff)),
         input: {
@@ -132,7 +186,43 @@ function summarizeAllUltimateStats(stats) {
             storedVertexIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.storedVertexIndices)),
             storedEnvIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.storedEnvIndices)),
             overwrittenIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.overwrittenIndices)),
-        }
+        },
+        dataFrameShape: stats.some(s => s.dataFrameShape !== undefined) ? {
+            numberOfDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfNonDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfNonDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfResultConstraints: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultConstraints).filter(assert_1.isNotUndefined)),
+            numberOfResultingValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingValues).filter(assert_1.isNotUndefined)),
+            numberOfResultingTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingTop).filter(assert_1.isNotUndefined)),
+            numberOfResultingBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingBottom).filter(assert_1.isNotUndefined)),
+            numberOfEmptyNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEmptyNodes).filter(assert_1.isNotUndefined)),
+            numberOfOperationNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperationNodes).filter(assert_1.isNotUndefined)),
+            numberOfValueNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfValueNodes).filter(assert_1.isNotUndefined)),
+            sizeOfInfo: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.sizeOfInfo).filter(assert_1.isNotUndefined)),
+            numberOfEntriesPerNode: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEntriesPerNode).filter(assert_1.isNotUndefined)),
+            numberOfOperations: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperations).filter(assert_1.isNotUndefined)),
+            numberOfTotalValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalValues).filter(assert_1.isNotUndefined)),
+            numberOfTotalTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalTop).filter(assert_1.isNotUndefined)),
+            numberOfTotalBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalBottom).filter(assert_1.isNotUndefined)),
+            inferredColNames: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColNames).filter(assert_1.isNotUndefined)),
+            numberOfColNamesValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesValues).filter(assert_1.isNotUndefined)),
+            numberOfColNamesTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesTop).filter(assert_1.isNotUndefined)),
+            numberOfColNamesBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesBottom).filter(assert_1.isNotUndefined)),
+            inferredColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColCount).filter(assert_1.isNotUndefined)),
+            numberOfColCountExact: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountExact).filter(assert_1.isNotUndefined)),
+            numberOfColCountValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountValues).filter(assert_1.isNotUndefined)),
+            numberOfColCountTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountTop).filter(assert_1.isNotUndefined)),
+            numberOfColCountInfinite: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfColCountBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeColCount).filter(assert_1.isNotUndefined)),
+            inferredRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredRowCount).filter(assert_1.isNotUndefined)),
+            numberOfRowCountExact: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountExact).filter(assert_1.isNotUndefined)),
+            numberOfRowCountValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountValues).filter(assert_1.isNotUndefined)),
+            numberOfRowCountTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountTop).filter(assert_1.isNotUndefined)),
+            numberOfRowCountInfinite: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfRowCountBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeRowCount).filter(assert_1.isNotUndefined)),
+            perOperationNumber: new Map(semantics_1.DataFrameOperationNames.map(n => [n, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.perOperationNumber.get(n)).filter(assert_1.isNotUndefined))]))
+        } : undefined
     };
 }
 function processNextSummary(line, allSummarized) {
@@ -152,7 +242,11 @@ function processNextSummary(line, allSummarized) {
                 ...got.summarize.perSliceMeasurements,
                 // restore maps
                 measurements: new Map(got.summarize.perSliceMeasurements.measurements),
-            }
+            },
+            dataFrameShape: got.summarize.dataFrameShape !== undefined ? {
+                ...got.summarize.dataFrameShape,
+                perOperationNumber: new Map(got.summarize.dataFrameShape.perOperationNumber)
+            } : undefined
         }
     };
     allSummarized.push(got.summarize);
@@ -164,6 +258,10 @@ function processNextUltimateSummary(line, allSummarized) {
         // restore maps
         commonMeasurements: new Map(got.commonMeasurements),
         perSliceMeasurements: new Map(got.perSliceMeasurements),
+        dataFrameShape: got.dataFrameShape !== undefined ? {
+            ...got.dataFrameShape,
+            perOperationNumber: new Map(got.dataFrameShape.perOperationNumber)
+        } : undefined
     };
     allSummarized.push(got);
 }

package/cli/benchmark-app.d.ts

@@ -10,9 +10,11 @@ export interface BenchmarkCliOptions {
     runs?: number;
     seed?: string;
     parser: KnownParserName;
+    'dataframe-shape-inference': boolean;
     'enable-pointer-tracking': boolean;
     'max-file-slices': number;
     threshold?: number;
     'per-file-time-limit'?: number;
     'sampling-strategy': string;
+    cfg?: boolean;
 }

package/cli/benchmark-app.js

@@ -69,11 +69,13 @@ async function benchmark() {
         '--output', path_1.default.join(options.output, path_1.default.relative(f.baseDir, `${f.request.content}.json`)),
         '--slice', options.slice, ...verboseAdd,
         '--parser', options.parser,
+        ...(options['dataframe-shape-inference'] ? ['--dataframe-shape-inference'] : []),
         ...(options['enable-pointer-tracking'] ? ['--enable-pointer-tracking'] : []),
         '--max-slices', `${options['max-file-slices']}`,
         ...(options.threshold ? ['--threshold', `${options.threshold}`] : []),
         '--sampling-strategy', options['sampling-strategy'],
         ...(options.seed ? ['--seed', options.seed] : []),
+        ...(options.cfg ? ['--cfg'] : []),
     ]);
     const runs = options.runs ?? 1;
     for (let i = 1; i <= runs; i++) {

package/cli/benchmark-helper-app.d.ts

@@ -8,8 +8,10 @@ export interface SingleBenchmarkCliOptions {
     slice: string;
     output?: string;
     parser: KnownParserName;
+    'dataframe-shape-inference': boolean;
     'enable-pointer-tracking': boolean;
     'max-slices': number;
+    'cfg': boolean;
     threshold?: number;
     'sampling-strategy': string;
     seed?: string;

package/cli/benchmark-helper-app.js

@@ -34,13 +34,12 @@ async function benchmark() {
     const prefix = `[${options.input}${options['file-id'] !== undefined ? ` (file ${options['file-id']}, run ${options['run-num']})` : ''}]`;
     console.log(`${prefix} Appending output to ${options.output}`);
     const directory = path_1.default.parse(options.output).dir;
-    // ensure the directory exists if path contains one
+    // ensure the directory exists if the path contains one
     if (directory !== '') {
         fs_1.default.mkdirSync(directory, { recursive: true });
     }
     // Enable pointer analysis if requested, otherwise disable it
-    const config = (0, config_1.getConfig)();
-    (0, config_1.amendConfig)(config, c => {
+    const config = (0, config_1.amendConfig)((0, config_1.getConfig)(), c => {
         c.solver.pointerTracking = options['enable-pointer-tracking'];
         return c;
     });
@@ -70,6 +69,14 @@ async function benchmark() {
             (0, assert_1.guard)(count >= 0, `Number of slices exceeded limit of ${maxSlices} with ${-count} slices, skipping in count`);
             (0, assert_1.guard)(count > 0, `No possible slices found for ${options.input}, skipping in count`);
         }
+        if (options['cfg'] || options['dataframe-shape-inference']) {
+            slicer.extractCFG();
+        }
+        if (options['dataframe-shape-inference']) {
+            console.log(`${prefix} Performing shape inference for data frames`);
+            slicer.inferDataFrameShapes();
+            console.log(`${prefix} Completed data frame shape inference`);
+        }
         const { stats } = slicer.finish();
         const output = {
             filename: options.input,

package/cli/common/options.js

@@ -22,11 +22,13 @@ exports.benchmarkOptions = [
     { name: 'slice', alias: 's', type: String, description: 'Automatically slice for *all* variables (default) or *no* slicing and only parsing/dataflow construction. Numbers will indicate: sample X random slices from all.', defaultValue: 'all', typeLabel: '{underline all/no}' },
     { name: 'output', alias: 'o', type: String, description: `Folder to write all the measurements to in a per-file-basis (defaults to {italic benchmark-${StartTimeString}})`, defaultValue: `benchmark-${StartTimeString}`, typeLabel: '{underline folder}' },
     { name: 'parser', type: String, description: 'The parser to use for the benchmark', defaultValue: 'r-shell', typeLabel: '{underline parser}' },
+    { name: 'dataframe-shape-inference', type: Boolean, description: 'Infer the shape of data frames using abstract interpretation (includes control flow graph extraction)', defaultValue: false },
     { name: 'enable-pointer-tracking', type: Boolean, description: 'Run dataflow analysis with pointer tracking', defaultValue: false },
     { name: 'max-file-slices', type: Number, description: 'If file has more than passed number of slices, the file is not processed', defaultValue: -1, typeLabel: '{underline number}' },
     { name: 'threshold', alias: 't', type: Number, description: 'How many re-visits of the same node are ok?', defaultValue: undefined, typeLabel: '{underline number}' },
     { name: 'per-file-time-limit', type: Number, description: 'Time limit in milliseconds to process single file (disabled by default)', defaultValue: undefined, typeLabel: '{underline number}' },
     { name: 'sampling-strategy', type: String, description: 'Which strategy to use, when sampling is enabled', defaultValue: 'random', typeLabel: '{underline random/equidistant}' },
+    { name: 'cfg', alias: 'c', type: Boolean, description: 'Extract the control flow graph of the file (benchmark it too)' }
 ];
 exports.benchmarkHelperOptions = [
     { name: 'verbose', alias: 'v', type: Boolean, description: 'Run with verbose logging [do not use for the real benchmark as this affects the time measurements, but only to find errors]' },
@@ -35,8 +37,10 @@ exports.benchmarkHelperOptions = [
     { name: 'file-id', alias: 'd', type: Number, description: 'A numeric file id that can be used to match an input and run-num to a file' },
     { name: 'run-num', alias: 'r', type: Number, description: 'The n-th time that the file with the given file-id is being benchmarked' },
     { name: 'slice', alias: 's', type: String, description: 'Automatically slice for *all* variables (default) or *no* slicing and only parsing/dataflow construction. Numbers will indicate: sample X random slices from all.', defaultValue: 'all', typeLabel: '{underline all/no}' },
+    { name: 'cfg', alias: 'c', type: Boolean, description: 'Extract the control flow graph of the file (benchmark it too)' },
     { name: 'output', alias: 'o', type: String, description: 'File to write the measurements to (appends a single line in JSON format)', typeLabel: '{underline file}' },
     { name: 'parser', type: String, description: 'The parser to use for the benchmark', defaultValue: 'r-shell', typeLabel: '{underline parser}' },
+    { name: 'dataframe-shape-inference', type: Boolean, description: 'Infer the shape of data frames using abstract interpretation (includes control flow graph extraction)', defaultValue: false },
     { name: 'enable-pointer-tracking', type: Boolean, description: 'Run dataflow analysis with pointer tracking', defaultValue: false },
     { name: 'max-slices', type: Number, description: 'If file has more than passed number of slices, the file is not processed', defaultValue: -1, typeLabel: '{underline number}' },
     { name: 'threshold', alias: 't', type: Number, description: 'How many re-visits of the same node are ok?', defaultValue: undefined, typeLabel: '{underline number}' },

package/cli/repl/commands/repl-query.js

@@ -59,7 +59,7 @@ async function processQueryArgs(line, parser, output, config) {
     }
     const processed = await getDataflow(config, parser, args.join(' '));
     return {
-        query: (0, query_1.executeQueries)({ dataflow: processed.dataflow, ast: processed.normalize, config: config }, parsedQuery),
+        query: await Promise.resolve((0, query_1.executeQueries)({ dataflow: processed.dataflow, ast: processed.normalize, config: config }, parsedQuery)),
         processed
     };
 }

package/cli/repl/server/connection.js

@@ -348,11 +348,20 @@ class FlowRServerConnection {
         const { dataflow: dfg, normalize: ast } = fileInformation.pipeline.getResults(true);
         (0, assert_1.guard)(dfg !== undefined, `Dataflow graph must be present (request: ${request.filetoken})`);
         (0, assert_1.guard)(ast !== undefined, `AST must be present (request: ${request.filetoken})`);
-        const results = (0, query_1.executeQueries)({ dataflow: dfg, ast, config: this.config }, request.query);
-        (0, send_1.sendMessage)(this.socket, {
-            type: 'response-query',
-            id: request.id,
-            results
+        void Promise.resolve((0, query_1.executeQueries)({ dataflow: dfg, ast, config: this.config }, request.query)).then(results => {
+            (0, send_1.sendMessage)(this.socket, {
+                type: 'response-query',
+                id: request.id,
+                results
+            });
+        }).catch(e => {
+            this.logger.error(`[${this.name}] Error while executing query: ${String(e)}`);
+            (0, send_1.sendMessage)(this.socket, {
+                id: request.id,
+                type: 'error',
+                fatal: false,
+                reason: `Error while executing query: ${String(e)}`
+            });
         });
     }
 }

package/config.d.ts

@@ -145,6 +145,37 @@ export interface FlowrConfigOptions extends MergeableRecord {
             readonly threshold?: number;
         };
     };
+    /**
+     * Configuration options for abstract interpretation
+     */
+    readonly abstractInterpretation: {
+        /**
+         * The configuration of the shape inference for data frames
+         */
+        readonly dataFrame: {
+            /**
+             * The maximum number of columns names to infer for data frames before over-approximating the column names to top
+             */
+            readonly maxColNames: number;
+            /**
+             * The threshold for the number of visitations of a node at which widening should be performed to ensure the termination of the fixpoint iteration
+             */
+            readonly wideningThreshold: number;
+            /**
+             * Configuration options for reading data frame shapes from loaded external data files, such as CSV files
+             */
+            readonly readLoadedData: {
+                /**
+                 * Whether data frame shapes should be extracted from loaded external data files, such as CSV files
+                 */
+                readonly readExternalFiles: boolean;
+                /**
+                 * The maximum number of lines to read when extracting data frame shapes from loaded files, such as CSV files
+                 */
+                readonly maxReadLines: number;
+            };
+        };
+    };
 }
 export interface TreeSitterEngineConfig extends MergeableRecord {
     readonly type: 'tree-sitter';

package/config.js

@@ -81,6 +81,16 @@ exports.defaultConfigOptions = {
         slicer: {
             threshold: 50
         }
+    },
+    abstractInterpretation: {
+        dataFrame: {
+            maxColNames: 50,
+            wideningThreshold: 4,
+            readLoadedData: {
+                readExternalFiles: true,
+                maxReadLines: 1e6
+            }
+        }
     }
 };
 exports.flowrConfigFileSchema = joi_1.default.object({
@@ -120,7 +130,17 @@ exports.flowrConfigFileSchema = joi_1.default.object({
         slicer: joi_1.default.object({
             threshold: joi_1.default.number().optional().description('The maximum number of iterations to perform on a single function call during slicing.')
         }).optional().description('The configuration for the slicer.')
-    }).description('How to resolve constants, constraints, cells, ...')
+    }).description('How to resolve constants, constraints, cells, ...'),
+    abstractInterpretation: joi_1.default.object({
+        dataFrame: joi_1.default.object({
+            maxColNames: joi_1.default.number().min(0).description('The maximum number of columns names to infer for data frames before over-approximating the column names to top.'),
+            wideningThreshold: joi_1.default.number().min(1).description('The threshold for the number of visitations of a node at which widening should be performed to ensure the termination of the fixpoint iteration.'),
+            readLoadedData: joi_1.default.object({
+                readExternalFiles: joi_1.default.boolean().description('Whether data frame shapes should be extracted from loaded external files, such as CSV files.'),
+                maxReadLines: joi_1.default.number().min(1).description('The maximum number of lines to read when extracting data frame shapes from loaded files, such as CSV files.')
+            }).description('Configuration options for reading data frame shapes from loaded external data files, such as CSV files.')
+        }).description('The configuration of the shape inference for data frames.')
+    }).description('The configuration options for abstract interpretation.')
 }).description('The configuration file format for flowR.');
 function parseConfig(jsonString) {
     try {

package/control-flow/basic-cfg-guided-visitor.d.ts

@@ -1,4 +1,4 @@
-import type { CfgBasicBlockVertex, CfgEndMarkerVertex, CfgExpressionVertex, CfgMidMarkerVertex, CfgSimpleVertex, CfgStatementVertex, ControlFlowInformation } from './control-flow-graph';
+import type { CfgBasicBlockVertex, CfgEndMarkerVertex, CfgExpressionVertex, CfgSimpleVertex, CfgStatementVertex, ControlFlowInformation } from './control-flow-graph';
 import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id';
 export interface BasicCfgGuidedVisitorConfiguration<ControlFlow extends ControlFlowInformation = ControlFlowInformation> {
     readonly controlFlow: ControlFlow;
@@ -34,6 +34,5 @@ export declare class BasicCfgGuidedVisitor<ControlFlow extends ControlFlowInform
     protected onBasicBlockNode(node: CfgBasicBlockVertex): void;
     protected onStatementNode(_node: CfgStatementVertex): void;
     protected onExpressionNode(_node: CfgExpressionVertex): void;
-    protected onMidMarkerNode(_node: CfgMidMarkerVertex): void;
     protected onEndMarkerNode(_node: CfgEndMarkerVertex): void;
 }

package/control-flow/basic-cfg-guided-visitor.js

@@ -72,9 +72,6 @@ class BasicCfgGuidedVisitor {
             case control_flow_graph_1.CfgVertexType.Expression:
                 this.onExpressionNode(vertex);
                 break;
-            case control_flow_graph_1.CfgVertexType.MidMarker:
-                this.onMidMarkerNode(vertex);
-                break;
             case control_flow_graph_1.CfgVertexType.EndMarker:
                 this.onEndMarkerNode(vertex);
                 break;
@@ -103,9 +100,6 @@ class BasicCfgGuidedVisitor {
     onExpressionNode(_node) {
         /* does nothing by default */
     }
-    onMidMarkerNode(_node) {
-        /* does nothing by default */
-    }
     onEndMarkerNode(_node) {
         /* does nothing by default */
     }

package/control-flow/cfg-simplification.d.ts

@@ -1,6 +1,7 @@
 import type { ControlFlowInformation } from './control-flow-graph';
 import type { NormalizedAst } from '../r-bridge/lang-4.x/ast/model/processing/decorate';
 import type { DataflowGraph } from '../dataflow/graph/graph';
+import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id';
 import { cfgAnalyzeDeadCode } from './cfg-dead-code';
 import type { FlowrConfigOptions } from '../config';
 export interface CfgPassInfo {
@@ -28,4 +29,9 @@ export declare function simplifyControlFlowInformation(cfg: ControlFlowInformati
 declare function cfgRemoveDeadCode(cfg: ControlFlowInformation, _info?: CfgPassInfo): ControlFlowInformation;
 declare function uniqueControlFlowSets(cfg: ControlFlowInformation, _info?: CfgPassInfo): ControlFlowInformation;
 declare function toBasicBlocks(cfg: ControlFlowInformation, _info?: CfgPassInfo): ControlFlowInformation;
+/**
+ * Uses {@link visitCfgInOrder} to find all nodes that are reachable from the control flow graph's {@link ControlFlowInformation.entryPoints} and returns them as a set.
+ * @param cfg - The control flow graph whose reachable nodes to find.
+ */
+export declare function cfgFindAllReachable(cfg: ControlFlowInformation): Set<NodeId>;
 export {};

package/control-flow/cfg-simplification.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DefaultCfgSimplificationOrder = exports.CfgSimplificationPasses = void 0;
 exports.simplifyControlFlowInformation = simplifyControlFlowInformation;
+exports.cfgFindAllReachable = cfgFindAllReachable;
 const cfg_to_basic_blocks_1 = require("./cfg-to-basic-blocks");
 const simple_visitor_1 = require("./simple-visitor");
 const cfg_dead_code_1 = require("./cfg-dead-code");
@@ -32,10 +33,7 @@ function simplifyControlFlowInformation(cfg, info, passes = exports.DefaultCfgSi
  */
 function cfgRemoveDeadCode(cfg, _info) {
     // remove every root level node and accompanying vertices that can not be reached from the entry points
-    const reachable = new Set();
-    (0, simple_visitor_1.visitCfgInOrder)(cfg.graph, cfg.entryPoints, node => {
-        reachable.add(node);
-    });
+    const reachable = cfgFindAllReachable(cfg);
     for (const id of cfg.graph.rootIds()) {
         if (!reachable.has(id)) {
             cfg.graph.removeVertex(id);
@@ -45,15 +43,26 @@ function cfgRemoveDeadCode(cfg, _info) {
 }
 function uniqueControlFlowSets(cfg, _info) {
     return {
-        returns: [...new Set(cfg.returns)],
-        entryPoints: [...new Set(cfg.entryPoints)],
-        exitPoints: [...new Set(cfg.exitPoints)],
-        breaks: [...new Set(cfg.breaks)],
-        nexts: [...new Set(cfg.nexts)],
+        returns: Array.from(new Set(cfg.returns)),
+        entryPoints: Array.from(new Set(cfg.entryPoints)),
+        exitPoints: Array.from(new Set(cfg.exitPoints)),
+        breaks: Array.from(new Set(cfg.breaks)),
+        nexts: Array.from(new Set(cfg.nexts)),
         graph: cfg.graph
     };
 }
 function toBasicBlocks(cfg, _info) {
     return (0, cfg_to_basic_blocks_1.convertCfgToBasicBlocks)(cfg);
 }
+/**
+ * Uses {@link visitCfgInOrder} to find all nodes that are reachable from the control flow graph's {@link ControlFlowInformation.entryPoints} and returns them as a set.
+ * @param cfg - The control flow graph whose reachable nodes to find.
+ */
+function cfgFindAllReachable(cfg) {
+    const reachable = new Set();
+    (0, simple_visitor_1.visitCfgInOrder)(cfg.graph, cfg.entryPoints, node => {
+        reachable.add(node);
+    });
+    return reachable;
+}
 //# sourceMappingURL=cfg-simplification.js.map

package/control-flow/control-flow-graph.d.ts

@@ -2,8 +2,6 @@ import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id';
 import type { MergeableRecord } from '../util/objects';
 import type { RFalse, RTrue } from '../r-bridge/lang-4.x/convert-values';
 export declare enum CfgVertexType {
-    /** Marks a break point in a construct (e.g., between the name and the value of an argument, or the formals and the body of a function)  */
-    MidMarker = "mid",
     /** The explicit exit-nodes to ensure the hammock property */
     EndMarker = "end",
     /** something like an if, assignment, ... even though in the classical sense of R they are still expressions */
@@ -50,10 +48,6 @@ export interface CfgWithRoot extends CfgBaseVertex {
     /** the vertex for which this is a marker */
     root: NodeId;
 }
-export interface CfgMidMarkerVertex extends CfgWithRoot {
-    type: CfgVertexType.MidMarker;
-    kind: string;
-}
 export interface CfgEndMarkerVertex extends CfgWithRoot {
     type: CfgVertexType.EndMarker;
 }
@@ -65,9 +59,10 @@ export interface CfgBasicBlockVertex extends CfgBaseVertex {
 /**
  * A vertex in the {@link ControlFlowGraph}.
  */
-export type CfgSimpleVertex = CfgStatementVertex | CfgExpressionVertex | CfgBasicBlockVertex | CfgMidMarkerVertex | CfgEndMarkerVertex;
+export type CfgSimpleVertex = CfgStatementVertex | CfgExpressionVertex | CfgBasicBlockVertex | CfgEndMarkerVertex;
 export declare function equalVertex(a: CfgSimpleVertex, b: CfgSimpleVertex): boolean;
-export declare function isMarkerVertex(vertex: CfgSimpleVertex): vertex is CfgMidMarkerVertex | CfgEndMarkerVertex;
+export declare function isMarkerVertex(vertex: CfgSimpleVertex): vertex is CfgEndMarkerVertex;
+export declare function getVertexRootId(vertex: CfgSimpleVertex): NodeId;
 interface CfgFlowDependencyEdge extends MergeableRecord {
     label: CfgEdgeType.Fd;
 }

package/control-flow/control-flow-graph.js

@@ -4,12 +4,11 @@ exports.ControlFlowGraph = exports.CfgVertexType = void 0;
 exports.edgeTypeToString = edgeTypeToString;
 exports.equalVertex = equalVertex;
 exports.isMarkerVertex = isMarkerVertex;
+exports.getVertexRootId = getVertexRootId;
 exports.emptyControlFlowInformation = emptyControlFlowInformation;
 const assert_1 = require("../util/assert");
 var CfgVertexType;
 (function (CfgVertexType) {
-    /** Marks a break point in a construct (e.g., between the name and the value of an argument, or the formals and the body of a function)  */
-    CfgVertexType["MidMarker"] = "mid";
     /** The explicit exit-nodes to ensure the hammock property */
     CfgVertexType["EndMarker"] = "end";
     /** something like an if, assignment, ... even though in the classical sense of R they are still expressions */
@@ -36,16 +35,16 @@ function equalVertex(a, b) {
     else if (a.type === CfgVertexType.Block && b.type === CfgVertexType.Block) {
         return a.elems.length === b.elems.length && a.elems.every((e, i) => e.id === b.elems[i].id);
     }
-    else if (a.type === CfgVertexType.MidMarker && b.type === CfgVertexType.MidMarker) {
-        return a.kind === b.kind && a.root === b.root;
-    }
     else if (a.type === CfgVertexType.EndMarker && b.type === CfgVertexType.EndMarker) {
         return a.root === b.root;
     }
     return true;
 }
 function isMarkerVertex(vertex) {
-    return vertex.type === CfgVertexType.MidMarker || vertex.type === CfgVertexType.EndMarker;
+    return vertex.type === CfgVertexType.EndMarker;
+}
+function getVertexRootId(vertex) {
+    return isMarkerVertex(vertex) ? vertex.root : vertex.id;
 }
 /**
  * This class represents the control flow graph of an R program.

package/control-flow/dfg-cfg-guided-visitor.js

@@ -36,7 +36,7 @@ class DataflowAwareCfgGuidedVisitor extends basic_cfg_guided_visitor_1.BasicCfgG
         }
     }
     visitDataflowNode(node) {
-        const dfgVertex = this.getDataflowGraph((0, control_flow_graph_1.isMarkerVertex)(node) ? node.root : node.id);
+        const dfgVertex = this.getDataflowGraph((0, control_flow_graph_1.getVertexRootId)(node));
         if (!dfgVertex) {
             this.visitUnknown(node);
             return;

package/control-flow/extract-cfg.d.ts

@@ -5,12 +5,12 @@ import type { ControlFlowInformation } from './control-flow-graph';
 import type { CfgSimplificationPassName } from './cfg-simplification';
 import type { FlowrConfigOptions } from '../config';
 /**
- * Given a normalized AST this approximates the control flow graph of the program.
+ * Given a normalized AST, this approximates the control flow graph of the program.
  * This view is different from the computation of the dataflow graph and may differ,
  * especially because it focuses on intra-procedural analysis.
  *
  * @param ast             - the normalized AST
- * @param config          - the Flowr config
+ * @param config          - the flowR config
  * @param graph           - additional dataflow facts to consider by the control flow extraction
  * @param simplifications - a list of simplification passes to apply to the control flow graph
  *