@eagleoutice/flowr 2.2.16 → 2.3.0

package/benchmark/summarizer/second-phase/process.js

@@ -9,6 +9,8 @@ const defaultmap_1 = require("../../../util/collections/defaultmap");
 const summarizer_1 = require("../../../util/summarizer");
 const assert_1 = require("../../../util/assert");
 const stats_1 = require("../../stats/stats");
+const semantics_1 = require("../../../abstract-interpretation/data-frame/semantics");
+const arrays_1 = require("../../../util/collections/arrays");
 function summarizeAllSummarizedStats(stats) {
     const commonMeasurements = new defaultmap_1.DefaultMap(() => []);
     const perSliceMeasurements = new defaultmap_1.DefaultMap(() => []);
@@ -19,11 +21,14 @@ function summarizeAllSummarizedStats(stats) {
     const normalizeTimesPerToken = [];
     const dataflowTimesPerToken = [];
     const totalCommonTimesPerToken = [];
+    const controlFlowTimePerToken = [];
+    const dataFrameShapeTimePerToken = [];
     const memory = new defaultmap_1.DefaultMap(() => []);
     const reductions = [];
     const reductionsNoFluff = [];
     const inputs = [];
     const dataflows = [];
+    const dataFrameShapes = [];
     let failedToRepParse = 0;
     let timesHitThreshold = 0;
     let totalSlices = 0;
@@ -41,6 +46,12 @@ function summarizeAllSummarizedStats(stats) {
         normalizeTimesPerToken.push(stat.normalizeTimePerToken);
         dataflowTimesPerToken.push(stat.dataflowTimePerToken);
         totalCommonTimesPerToken.push(stat.totalCommonTimePerToken);
+        if (stat.controlFlowTimePerToken !== undefined) {
+            controlFlowTimePerToken.push(stat.controlFlowTimePerToken);
+        }
+        if (stat.dataFrameShapeTimePerToken !== undefined) {
+            dataFrameShapeTimePerToken.push(stat.dataFrameShapeTimePerToken);
+        }
         for (const [k, v] of stat.memory) {
             memory.get(k).push(v);
         }
@@ -48,6 +59,9 @@ function summarizeAllSummarizedStats(stats) {
         reductionsNoFluff.push(stat.perSliceMeasurements.reductionNoFluff);
         inputs.push(stat.input);
         dataflows.push(stat.dataflow);
+        if (stat.dataFrameShape !== undefined) {
+            dataFrameShapes.push(stat.dataFrameShape);
+        }
         failedToRepParse += stat.perSliceMeasurements.failedToRepParse;
         totalSlices += stat.perSliceMeasurements.numberOfSlices;
         timesHitThreshold += stat.perSliceMeasurements.timesHitThreshold;
@@ -64,6 +78,8 @@ function summarizeAllSummarizedStats(stats) {
         normalizeTimePerToken: (0, process_1.summarizeTimePerToken)(normalizeTimesPerToken),
         dataflowTimePerToken: (0, process_1.summarizeTimePerToken)(dataflowTimesPerToken),
         totalCommonTimePerToken: (0, process_1.summarizeTimePerToken)(totalCommonTimesPerToken),
+        controlFlowTimePerToken: controlFlowTimePerToken.length > 0 ? (0, process_1.summarizeTimePerToken)(controlFlowTimePerToken) : undefined,
+        dataFrameShapeTimePerToken: dataFrameShapeTimePerToken.length > 0 ? (0, process_1.summarizeTimePerToken)(dataFrameShapeTimePerToken) : undefined,
         failedToRepParse,
         timesHitThreshold,
         reduction: (0, process_1.summarizeSummarizedReductions)(reductions),
@@ -89,7 +105,43 @@ function summarizeAllSummarizedStats(stats) {
             storedVertexIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.storedVertexIndices)),
             storedEnvIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.storedEnvIndices)),
             overwrittenIndices: (0, summarizer_1.summarizeMeasurement)(dataflows.map(d => d.overwrittenIndices)),
-        }
+        },
+        dataFrameShape: stats.some(s => s.dataFrameShape !== undefined) ? {
+            numberOfDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfNonDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfNonDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfResultConstraints: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultConstraints).filter(assert_1.isNotUndefined)),
+            numberOfResultingValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingValues).filter(assert_1.isNotUndefined)),
+            numberOfResultingTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingTop).filter(assert_1.isNotUndefined)),
+            numberOfResultingBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingBottom).filter(assert_1.isNotUndefined)),
+            numberOfEmptyNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEmptyNodes).filter(assert_1.isNotUndefined)),
+            numberOfOperationNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperationNodes).filter(assert_1.isNotUndefined)),
+            numberOfValueNodes: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfValueNodes).filter(assert_1.isNotUndefined)),
+            sizeOfInfo: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.sizeOfInfo).filter(assert_1.isNotUndefined)),
+            numberOfEntriesPerNode: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEntriesPerNode).filter(assert_1.isNotUndefined)),
+            numberOfOperations: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperations).filter(assert_1.isNotUndefined)),
+            numberOfTotalValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalValues).filter(assert_1.isNotUndefined)),
+            numberOfTotalTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalTop).filter(assert_1.isNotUndefined)),
+            numberOfTotalBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalBottom).filter(assert_1.isNotUndefined)),
+            inferredColNames: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColNames).filter(assert_1.isNotUndefined)),
+            numberOfColNamesValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesValues).filter(assert_1.isNotUndefined)),
+            numberOfColNamesTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesTop).filter(assert_1.isNotUndefined)),
+            numberOfColNamesBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesBottom).filter(assert_1.isNotUndefined)),
+            inferredColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColCount).filter(assert_1.isNotUndefined)),
+            numberOfColCountExact: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountExact).filter(assert_1.isNotUndefined)),
+            numberOfColCountValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountValues).filter(assert_1.isNotUndefined)),
+            numberOfColCountTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountTop).filter(assert_1.isNotUndefined)),
+            numberOfColCountInfinite: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfColCountBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeColCount).filter(assert_1.isNotUndefined)),
+            inferredRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredRowCount).filter(assert_1.isNotUndefined)),
+            numberOfRowCountExact: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountExact).filter(assert_1.isNotUndefined)),
+            numberOfRowCountValues: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountValues).filter(assert_1.isNotUndefined)),
+            numberOfRowCountTop: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountTop).filter(assert_1.isNotUndefined)),
+            numberOfRowCountInfinite: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfRowCountBottom: (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeRowCount).filter(assert_1.isNotUndefined)),
+            perOperationNumber: new Map(semantics_1.DataFrameOperationNames.map(n => [n, (0, summarizer_1.summarizeMeasurement)(stats.map(s => s.dataFrameShape?.perOperationNumber.get(n) ?? 0))]))
+        } : undefined
     };
 }
 function summarizeAllUltimateStats(stats) {
@@ -109,6 +161,8 @@ function summarizeAllUltimateStats(stats) {
         normalizeTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.normalizeTimePerToken)),
         dataflowTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataflowTimePerToken)),
         totalCommonTimePerToken: (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.totalCommonTimePerToken)),
+        controlFlowTimePerToken: stats.some(s => s.controlFlowTimePerToken !== undefined) ? (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.controlFlowTimePerToken).filter(assert_1.isNotUndefined)) : undefined,
+        dataFrameShapeTimePerToken: stats.some(s => s.dataFrameShapeTimePerToken !== undefined) ? (0, process_1.summarizeSummarizedTimePerToken)(stats.map(s => s.dataFrameShapeTimePerToken).filter(assert_1.isNotUndefined)) : undefined,
         reduction: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reduction)),
         reductionNoFluff: (0, process_1.summarizeSummarizedReductions)(stats.map(s => s.reductionNoFluff)),
         input: {
@@ -132,7 +186,43 @@ function summarizeAllUltimateStats(stats) {
             storedVertexIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.storedVertexIndices)),
             storedEnvIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.storedEnvIndices)),
             overwrittenIndices: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataflow.overwrittenIndices)),
-        }
+        },
+        dataFrameShape: stats.some(s => s.dataFrameShape !== undefined) ? {
+            numberOfDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfNonDataFrameFiles: (0, arrays_1.arraySum)(stats.map(s => s.dataFrameShape?.numberOfNonDataFrameFiles).filter(assert_1.isNotUndefined)),
+            numberOfResultConstraints: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultConstraints).filter(assert_1.isNotUndefined)),
+            numberOfResultingValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingValues).filter(assert_1.isNotUndefined)),
+            numberOfResultingTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingTop).filter(assert_1.isNotUndefined)),
+            numberOfResultingBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfResultingBottom).filter(assert_1.isNotUndefined)),
+            numberOfEmptyNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEmptyNodes).filter(assert_1.isNotUndefined)),
+            numberOfOperationNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperationNodes).filter(assert_1.isNotUndefined)),
+            numberOfValueNodes: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfValueNodes).filter(assert_1.isNotUndefined)),
+            sizeOfInfo: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.sizeOfInfo).filter(assert_1.isNotUndefined)),
+            numberOfEntriesPerNode: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfEntriesPerNode).filter(assert_1.isNotUndefined)),
+            numberOfOperations: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfOperations).filter(assert_1.isNotUndefined)),
+            numberOfTotalValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalValues).filter(assert_1.isNotUndefined)),
+            numberOfTotalTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalTop).filter(assert_1.isNotUndefined)),
+            numberOfTotalBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfTotalBottom).filter(assert_1.isNotUndefined)),
+            inferredColNames: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColNames).filter(assert_1.isNotUndefined)),
+            numberOfColNamesValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesValues).filter(assert_1.isNotUndefined)),
+            numberOfColNamesTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesTop).filter(assert_1.isNotUndefined)),
+            numberOfColNamesBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColNamesBottom).filter(assert_1.isNotUndefined)),
+            inferredColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredColCount).filter(assert_1.isNotUndefined)),
+            numberOfColCountExact: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountExact).filter(assert_1.isNotUndefined)),
+            numberOfColCountValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountValues).filter(assert_1.isNotUndefined)),
+            numberOfColCountTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountTop).filter(assert_1.isNotUndefined)),
+            numberOfColCountInfinite: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfColCountBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfColCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeColCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeColCount).filter(assert_1.isNotUndefined)),
+            inferredRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.inferredRowCount).filter(assert_1.isNotUndefined)),
+            numberOfRowCountExact: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountExact).filter(assert_1.isNotUndefined)),
+            numberOfRowCountValues: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountValues).filter(assert_1.isNotUndefined)),
+            numberOfRowCountTop: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountTop).filter(assert_1.isNotUndefined)),
+            numberOfRowCountInfinite: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountInfinite).filter(assert_1.isNotUndefined)),
+            numberOfRowCountBottom: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.numberOfRowCountBottom).filter(assert_1.isNotUndefined)),
+            approxRangeRowCount: (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.approxRangeRowCount).filter(assert_1.isNotUndefined)),
+            perOperationNumber: new Map(semantics_1.DataFrameOperationNames.map(n => [n, (0, process_1.summarizeSummarizedMeasurement)(stats.map(s => s.dataFrameShape?.perOperationNumber.get(n)).filter(assert_1.isNotUndefined))]))
+        } : undefined
     };
 }
 function processNextSummary(line, allSummarized) {
@@ -152,7 +242,11 @@ function processNextSummary(line, allSummarized) {
                 ...got.summarize.perSliceMeasurements,
                 // restore maps
                 measurements: new Map(got.summarize.perSliceMeasurements.measurements),
-            }
+            },
+            dataFrameShape: got.summarize.dataFrameShape !== undefined ? {
+                ...got.summarize.dataFrameShape,
+                perOperationNumber: new Map(got.summarize.dataFrameShape.perOperationNumber)
+            } : undefined
         }
     };
     allSummarized.push(got.summarize);
@@ -164,6 +258,10 @@ function processNextUltimateSummary(line, allSummarized) {
         // restore maps
         commonMeasurements: new Map(got.commonMeasurements),
         perSliceMeasurements: new Map(got.perSliceMeasurements),
+        dataFrameShape: got.dataFrameShape !== undefined ? {
+            ...got.dataFrameShape,
+            perOperationNumber: new Map(got.dataFrameShape.perOperationNumber)
+        } : undefined
     };
     allSummarized.push(got);
 }

package/cli/benchmark-app.d.ts

@@ -10,6 +10,7 @@ export interface BenchmarkCliOptions {
     runs?: number;
     seed?: string;
     parser: KnownParserName;
+    'dataframe-shape-inference': boolean;
     'enable-pointer-tracking': boolean;
     'max-file-slices': number;
     threshold?: number;

package/cli/benchmark-app.js

@@ -69,6 +69,7 @@ async function benchmark() {
         '--output', path_1.default.join(options.output, path_1.default.relative(f.baseDir, `${f.request.content}.json`)),
         '--slice', options.slice, ...verboseAdd,
         '--parser', options.parser,
+        ...(options['dataframe-shape-inference'] ? ['--dataframe-shape-inference'] : []),
         ...(options['enable-pointer-tracking'] ? ['--enable-pointer-tracking'] : []),
         '--max-slices', `${options['max-file-slices']}`,
         ...(options.threshold ? ['--threshold', `${options.threshold}`] : []),

package/cli/benchmark-helper-app.d.ts

@@ -8,6 +8,7 @@ export interface SingleBenchmarkCliOptions {
     slice: string;
     output?: string;
     parser: KnownParserName;
+    'dataframe-shape-inference': boolean;
     'enable-pointer-tracking': boolean;
     'max-slices': number;
     threshold?: number;

package/cli/benchmark-helper-app.js

@@ -39,8 +39,7 @@ async function benchmark() {
         fs_1.default.mkdirSync(directory, { recursive: true });
     }
     // Enable pointer analysis if requested, otherwise disable it
-    const config = (0, config_1.getConfig)();
-    (0, config_1.amendConfig)(config, c => {
+    const config = (0, config_1.amendConfig)((0, config_1.getConfig)(), c => {
         c.solver.pointerTracking = options['enable-pointer-tracking'];
         return c;
     });
@@ -70,6 +69,13 @@ async function benchmark() {
             (0, assert_1.guard)(count >= 0, `Number of slices exceeded limit of ${maxSlices} with ${-count} slices, skipping in count`);
             (0, assert_1.guard)(count > 0, `No possible slices found for ${options.input}, skipping in count`);
         }
+        if (options['dataframe-shape-inference']) {
+            console.log(`${prefix} Extracting control flow graph for data frame shape inference`);
+            slicer.extractCFG();
+            console.log(`${prefix} Performing shape inference for data frames`);
+            slicer.inferDataFrameShapes();
+            console.log(`${prefix} Completed data frame shape inference`);
+        }
         const { stats } = slicer.finish();
         const output = {
             filename: options.input,

package/cli/common/options.js

@@ -22,6 +22,7 @@ exports.benchmarkOptions = [
     { name: 'slice', alias: 's', type: String, description: 'Automatically slice for *all* variables (default) or *no* slicing and only parsing/dataflow construction. Numbers will indicate: sample X random slices from all.', defaultValue: 'all', typeLabel: '{underline all/no}' },
     { name: 'output', alias: 'o', type: String, description: `Folder to write all the measurements to in a per-file-basis (defaults to {italic benchmark-${StartTimeString}})`, defaultValue: `benchmark-${StartTimeString}`, typeLabel: '{underline folder}' },
     { name: 'parser', type: String, description: 'The parser to use for the benchmark', defaultValue: 'r-shell', typeLabel: '{underline parser}' },
+    { name: 'dataframe-shape-inference', type: Boolean, description: 'Infer the shape of data frames using abstract interpretation (includes control flow graph extraction)', defaultValue: false },
     { name: 'enable-pointer-tracking', type: Boolean, description: 'Run dataflow analysis with pointer tracking', defaultValue: false },
     { name: 'max-file-slices', type: Number, description: 'If file has more than passed number of slices, the file is not processed', defaultValue: -1, typeLabel: '{underline number}' },
     { name: 'threshold', alias: 't', type: Number, description: 'How many re-visits of the same node are ok?', defaultValue: undefined, typeLabel: '{underline number}' },
@@ -37,6 +38,7 @@ exports.benchmarkHelperOptions = [
     { name: 'slice', alias: 's', type: String, description: 'Automatically slice for *all* variables (default) or *no* slicing and only parsing/dataflow construction. Numbers will indicate: sample X random slices from all.', defaultValue: 'all', typeLabel: '{underline all/no}' },
     { name: 'output', alias: 'o', type: String, description: 'File to write the measurements to (appends a single line in JSON format)', typeLabel: '{underline file}' },
     { name: 'parser', type: String, description: 'The parser to use for the benchmark', defaultValue: 'r-shell', typeLabel: '{underline parser}' },
+    { name: 'dataframe-shape-inference', type: Boolean, description: 'Infer the shape of data frames using abstract interpretation (includes control flow graph extraction)', defaultValue: false },
     { name: 'enable-pointer-tracking', type: Boolean, description: 'Run dataflow analysis with pointer tracking', defaultValue: false },
     { name: 'max-slices', type: Number, description: 'If file has more than passed number of slices, the file is not processed', defaultValue: -1, typeLabel: '{underline number}' },
     { name: 'threshold', alias: 't', type: Number, description: 'How many re-visits of the same node are ok?', defaultValue: undefined, typeLabel: '{underline number}' },

package/config.d.ts

@@ -145,6 +145,37 @@ export interface FlowrConfigOptions extends MergeableRecord {
             readonly threshold?: number;
         };
     };
+    /**
+     * Configuration options for abstract interpretation
+     */
+    readonly abstractInterpretation: {
+        /**
+         * The configuration of the shape inference for data frames
+         */
+        readonly dataFrame: {
+            /**
+             * The maximum number of columns names to infer for data frames before over-approximating the column names to top
+             */
+            readonly maxColNames: number;
+            /**
+             * The threshold for the number of visitations of a node at which widening should be performed to ensure the termination of the fixpoint iteration
+             */
+            readonly wideningThreshold: number;
+            /**
+             * Configuration options for reading data frame shapes from loaded external data files, such as CSV files
+             */
+            readonly readLoadedData: {
+                /**
+                 * Whether data frame shapes should be extracted from loaded external data files, such as CSV files
+                 */
+                readonly readExternalFiles: boolean;
+                /**
+                 * The maximum number of lines to read when extracting data frame shapes from loaded files, such as CSV files
+                 */
+                readonly maxReadLines: number;
+            };
+        };
+    };
 }
 export interface TreeSitterEngineConfig extends MergeableRecord {
     readonly type: 'tree-sitter';

package/config.js

@@ -81,6 +81,16 @@ exports.defaultConfigOptions = {
         slicer: {
             threshold: 50
         }
+    },
+    abstractInterpretation: {
+        dataFrame: {
+            maxColNames: 50,
+            wideningThreshold: 4,
+            readLoadedData: {
+                readExternalFiles: true,
+                maxReadLines: 1e6
+            }
+        }
     }
 };
 exports.flowrConfigFileSchema = joi_1.default.object({
@@ -120,7 +130,17 @@ exports.flowrConfigFileSchema = joi_1.default.object({
         slicer: joi_1.default.object({
             threshold: joi_1.default.number().optional().description('The maximum number of iterations to perform on a single function call during slicing.')
         }).optional().description('The configuration for the slicer.')
-    }).description('How to resolve constants, constraints, cells, ...')
+    }).description('How to resolve constants, constraints, cells, ...'),
+    abstractInterpretation: joi_1.default.object({
+        dataFrame: joi_1.default.object({
+            maxColNames: joi_1.default.number().min(0).description('The maximum number of columns names to infer for data frames before over-approximating the column names to top.'),
+            wideningThreshold: joi_1.default.number().min(1).description('The threshold for the number of visitations of a node at which widening should be performed to ensure the termination of the fixpoint iteration.'),
+            readLoadedData: joi_1.default.object({
+                readExternalFiles: joi_1.default.boolean().description('Whether data frame shapes should be extracted from loaded external files, such as CSV files.'),
+                maxReadLines: joi_1.default.number().min(1).description('The maximum number of lines to read when extracting data frame shapes from loaded files, such as CSV files.')
+            }).description('Configuration options for reading data frame shapes from loaded external data files, such as CSV files.')
+        }).description('The configuration of the shape inference for data frames.')
+    }).description('The configuration options for abstract interpretation.')
 }).description('The configuration file format for flowR.');
 function parseConfig(jsonString) {
     try {

package/control-flow/control-flow-graph.d.ts

@@ -68,6 +68,7 @@ export interface CfgBasicBlockVertex extends CfgBaseVertex {
 export type CfgSimpleVertex = CfgStatementVertex | CfgExpressionVertex | CfgBasicBlockVertex | CfgMidMarkerVertex | CfgEndMarkerVertex;
 export declare function equalVertex(a: CfgSimpleVertex, b: CfgSimpleVertex): boolean;
 export declare function isMarkerVertex(vertex: CfgSimpleVertex): vertex is CfgMidMarkerVertex | CfgEndMarkerVertex;
+export declare function getVertexRootId(vertex: CfgSimpleVertex): NodeId;
 interface CfgFlowDependencyEdge extends MergeableRecord {
     label: CfgEdgeType.Fd;
 }

package/control-flow/control-flow-graph.js

@@ -4,6 +4,7 @@ exports.ControlFlowGraph = exports.CfgVertexType = void 0;
 exports.edgeTypeToString = edgeTypeToString;
 exports.equalVertex = equalVertex;
 exports.isMarkerVertex = isMarkerVertex;
+exports.getVertexRootId = getVertexRootId;
 exports.emptyControlFlowInformation = emptyControlFlowInformation;
 const assert_1 = require("../util/assert");
 var CfgVertexType;
@@ -47,6 +48,9 @@ function equalVertex(a, b) {
 function isMarkerVertex(vertex) {
     return vertex.type === CfgVertexType.MidMarker || vertex.type === CfgVertexType.EndMarker;
 }
+function getVertexRootId(vertex) {
+    return isMarkerVertex(vertex) ? vertex.root : vertex.id;
+}
 /**
  * This class represents the control flow graph of an R program.
  * The control flow may be hierarchical when confronted with function definitions (see {@link CfgSimpleVertex} and {@link CFG#rootVertexIds|rootVertexIds()}).

package/control-flow/dfg-cfg-guided-visitor.js

@@ -36,7 +36,7 @@ class DataflowAwareCfgGuidedVisitor extends basic_cfg_guided_visitor_1.BasicCfgG
         }
     }
     visitDataflowNode(node) {
-        const dfgVertex = this.getDataflowGraph((0, control_flow_graph_1.isMarkerVertex)(node) ? node.root : node.id);
+        const dfgVertex = this.getDataflowGraph((0, control_flow_graph_1.getVertexRootId)(node));
         if (!dfgVertex) {
             this.visitUnknown(node);
             return;

package/control-flow/semantic-cfg-guided-visitor.d.ts

@@ -45,7 +45,7 @@ export declare class SemanticCfgGuidedVisitor<OtherInfo = NoInfo, ControlFlow ex
     /**
      * A helper function to get the normalized AST node for the given id or fail if it does not exist.
      */
-    protected getNormalizedAst(id: NodeId): RNode<OtherInfo & ParentInformation> | undefined;
+    protected getNormalizedAst(id: NodeId | undefined): RNode<OtherInfo & ParentInformation> | undefined;
     /**
      * See {@link DataflowAwareCfgGuidedVisitor#visitValue} for the base implementation.
      * This now dispatches the value to the appropriate event handler based on its type.

package/control-flow/semantic-cfg-guided-visitor.js

@@ -35,7 +35,7 @@ class SemanticCfgGuidedVisitor extends dfg_cfg_guided_visitor_1.DataflowAwareCfg
      * A helper function to get the normalized AST node for the given id or fail if it does not exist.
      */
     getNormalizedAst(id) {
-        return this.config.normalizedAst.idMap.get(id);
+        return id !== undefined ? this.config.normalizedAst.idMap.get(id) : undefined;
     }
     /**
      * See {@link DataflowAwareCfgGuidedVisitor#visitValue} for the base implementation.

package/dataflow/environments/built-in.d.ts

@@ -30,7 +30,7 @@ import { processRm } from '../internal/process/functions/call/built-in/built-in-
 import { processEvalCall } from '../internal/process/functions/call/built-in/built-in-eval';
 import type { REnvironmentInformation } from './environment';
 import type { Value } from '../eval/values/r-value';
-import { resolveAsVector } from '../eval/resolve/resolve';
+import { resolveAsVector, resolveAsSeq, resolveAsMinus, resolveAsPlus } from '../eval/resolve/resolve';
 import type { DataflowGraph } from '../graph/graph';
 import type { VariableResolve } from '../../config';
 export type BuiltIn = `built-in:${string}`;
@@ -62,7 +62,7 @@ export interface DefaultBuiltInProcessorConfiguration extends ForceArguments {
      */
     readonly useAsProcessor?: UseAsProcessors;
 }
-export type BuiltInEvalHandler = (resolve: VariableResolve, a: RNodeWithParent, env: REnvironmentInformation, graph?: DataflowGraph, map?: AstIdMap) => Value;
+export type BuiltInEvalHandler = (resolve: VariableResolve, a: RNodeWithParent, env?: REnvironmentInformation, graph?: DataflowGraph, map?: AstIdMap) => Value;
 declare function defaultBuiltInProcessor<OtherInfo>(name: RSymbol<OtherInfo & ParentInformation>, args: readonly RFunctionArgument<OtherInfo & ParentInformation>[], rootId: NodeId, data: DataflowProcessorInformation<OtherInfo & ParentInformation>, config: DefaultBuiltInProcessorConfiguration): DataflowInformation;
 export declare function registerBuiltInFunctions<Config extends object, Proc extends BuiltInIdentifierProcessorWithConfig<Config>>(both: boolean, names: readonly Identifier[], processor: Proc, config: Config): void;
 export declare const BuiltInProcessorMapper: {
@@ -90,7 +90,9 @@ export declare const BuiltInProcessorMapper: {
 };
 export declare const BuiltInEvalHandlerMapper: {
     readonly 'built-in:c': typeof resolveAsVector;
-    readonly 'builtin:vector': typeof resolveAsVector;
+    readonly 'built-in::': typeof resolveAsSeq;
+    readonly 'built-in:+': typeof resolveAsPlus;
+    readonly 'built-in:-': typeof resolveAsMinus;
 };
 export type BuiltInMappingName = keyof typeof BuiltInProcessorMapper;
 export type ConfigOfBuiltInMappingName<N extends BuiltInMappingName> = Parameters<typeof BuiltInProcessorMapper[N]>[4];

package/dataflow/environments/built-in.js

@@ -144,7 +144,9 @@ exports.BuiltInProcessorMapper = {
 };
 exports.BuiltInEvalHandlerMapper = {
     'built-in:c': resolve_1.resolveAsVector,
-    'builtin:vector': resolve_1.resolveAsVector
+    'built-in::': resolve_1.resolveAsSeq,
+    'built-in:+': resolve_1.resolveAsPlus,
+    'built-in:-': resolve_1.resolveAsMinus
 };
 exports.BuiltInMemory = new Map();
 exports.EmptyBuiltInMemory = new Map();

package/dataflow/eval/resolve/alias-tracking.js

@@ -139,6 +139,8 @@ function resolveIdToValue(id, { environment, graph, idMap, full = true, resolve
                 return r_value_1.Top;
             }
         case type_1.RType.FunctionCall:
+        case type_1.RType.BinaryOp:
+        case type_1.RType.UnaryOp:
             return (0, set_constants_1.setFrom)((0, resolve_1.resolveNode)(resolve, node, environment, graph, idMap));
         case type_1.RType.String:
         case type_1.RType.Number:
@@ -261,8 +263,6 @@ function isNestedInLoop(node, ast) {
 function trackAliasesInGraph(id, graph, idMap) {
     idMap ??= graph.idMap;
     (0, assert_1.guard)(idMap !== undefined, 'The ID map is required to get the lineage of a node');
-    const start = graph.getVertex(id);
-    (0, assert_1.guard)(start !== undefined, 'Unable to find start for alias tracking');
     const queue = new visiting_queue_1.VisitingQueue(25);
     const clean = (0, environment_1.initializeCleanEnvironments)();
     const cleanFingerprint = (0, fingerprint_1.envFingerprint)(clean);

package/dataflow/eval/resolve/resolve.d.ts

@@ -1,7 +1,8 @@
 import type { AstIdMap, RNodeWithParent } from '../../../r-bridge/lang-4.x/ast/model/processing/decorate';
 import type { REnvironmentInformation } from '../../environments/environment';
 import type { DataflowGraph } from '../../graph/graph';
-import type { Value } from '../values/r-value';
+import type { Lift, Value, ValueNumber, ValueVector } from '../values/r-value';
+import { Top } from '../values/r-value';
 import type { VariableResolve } from '../../../config';
 /**
  * Helper function used by {@link resolveIdToValue}, please use that instead, if
@@ -22,16 +23,59 @@ export declare function resolveNode(resolve: VariableResolve, a: RNodeWithParent
  * Helper function used by {@link resolveIdToValue}, please use that instead, if
  * you want to resolve the value of an identifier / node
  *
- * This function converts an r-node to a Value Vector {@link vectorFrom}
- * It also recursively resolves any symbols, values, function calls (only c), in
- * order to construct the value of the vector to resolve by calling {@link resolveIdToValue}
- * or {@link resolveNode}
+ * This function resolves a vector function call `c` to a {@link ValueVector}
+ * by recursively resolving the values of the arguments by calling {@link resolveIdToValue}
  *
- * @param a       - Node of the vector to resolve
- * @param env     - Environment to use
  * @param resolve - Variable resolve mode
+ * @param node    - Node of the vector function to resolve
+ * @param env     - Environment to use
  * @param graph   - Dataflow graph
- * @param map     - Idmap of Dataflow Graph
+ * @param map     - Id map of the dataflow graph
  * @returns ValueVector or Top
  */
-export declare function resolveAsVector(resolve: VariableResolve, a: RNodeWithParent, env: REnvironmentInformation, graph?: DataflowGraph, map?: AstIdMap): Value;
+export declare function resolveAsVector(resolve: VariableResolve, node: RNodeWithParent, environment?: REnvironmentInformation, graph?: DataflowGraph, idMap?: AstIdMap): ValueVector<Lift<Value[]>> | typeof Top;
+/**
+ * Helper function used by {@link resolveIdToValue}, please use that instead, if
+ * you want to resolve the value of an identifier / node
+ *
+ * This function resolves a binary sequence operator `:` to a {@link ValueVector} of {@link ValueNumber}s
+ * by recursively resolving the values of the arguments by calling {@link resolveIdToValue}
+ *
+ * @param resolve  - Variable resolve mode
+ * @param operator - Node of the sequence operator to resolve
+ * @param env      - Environment to use
+ * @param graph    - Dataflow graph
+ * @param map      - Id map of the dataflow graph
+ * @returns ValueVector of ValueNumbers or Top
+ */
+export declare function resolveAsSeq(resolve: VariableResolve, operator: RNodeWithParent, environment?: REnvironmentInformation, graph?: DataflowGraph, idMap?: AstIdMap): ValueVector<Lift<ValueNumber[]>> | typeof Top;
+/**
+ * Helper function used by {@link resolveIdToValue}, please use that instead, if
+ * you want to resolve the value of an identifier / node
+ *
+ * This function resolves a unary plus operator `+` to a {@link ValueNumber} or {@link ValueVector} of ValueNumbers
+ * by recursively resolving the values of the arguments by calling {@link resolveIdToValue}
+ *
+ * @param resolve  - Variable resolve mode
+ * @param operator - Node of the plus operator to resolve
+ * @param env      - Environment to use
+ * @param graph    - Dataflow graph
+ * @param map      - Id map of the dataflow graph
+ * @returns ValueNumber, ValueVector of ValueNumbers, or Top
+ */
+export declare function resolveAsPlus(resolve: VariableResolve, operator: RNodeWithParent, environment?: REnvironmentInformation, graph?: DataflowGraph, idMap?: AstIdMap): ValueNumber | ValueVector<Lift<ValueNumber[]>> | typeof Top;
+/**
+ * Helper function used by {@link resolveIdToValue}, please use that instead, if
+ * you want to resolve the value of an identifier / node
+ *
+ * This function resolves a unary minus operator `-` to a {@link ValueNumber} or {@link ValueVector} of ValueNumbers
+ * by recursively resolving the values of the arguments by calling {@link resolveIdToValue}
+ *
+ * @param resolve  - Variable resolve mode
+ * @param operator - Node of the minus operator to resolve
+ * @param env      - Environment to use
+ * @param graph    - Dataflow graph
+ * @param map      - Id map of the dataflow graph
+ * @returns ValueNumber, ValueVector of ValueNumbers, or Top
+ */
+export declare function resolveAsMinus(resolve: VariableResolve, operator: RNodeWithParent, environment?: REnvironmentInformation, graph?: DataflowGraph, idMap?: AstIdMap): ValueNumber | ValueVector<Lift<ValueNumber[]>> | typeof Top;