@eagleoutice/flowr 1.4.1 → 1.5.0

package/dataflow/extractor.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.processBinaryOp = exports.produceDataFlowGraph = void 0;
+const r_bridge_1 = require("../r-bridge");
 const processor_1 = require("./processor");
 const uninteresting_leaf_1 = require("./internal/process/uninteresting-leaf");
 const symbol_1 = require("./internal/process/symbol");
@@ -44,8 +45,15 @@ const processors = {
     ["RArgument" /* RType.Argument */]: argument_1.processFunctionArgument,
     ["RExpressionList" /* RType.ExpressionList */]: expression_list_1.processExpressionList,
 };
-function produceDataFlowGraph(ast, initialScope = scopes_1.LocalScope) {
-    return (0, processor_1.processDataflowFor)(ast.ast, { completeAst: ast, activeScope: initialScope, environments: (0, environments_1.initializeCleanEnvironments)(), processors: processors });
+function produceDataFlowGraph(request, ast, initialScope = scopes_1.LocalScope) {
+    return (0, processor_1.processDataflowFor)(ast.ast, {
+        completeAst: ast,
+        activeScope: initialScope,
+        environments: (0, environments_1.initializeCleanEnvironments)(),
+        processors: processors,
+        currentRequest: request,
+        referenceChain: [(0, r_bridge_1.requestFingerprint)(request)]
+    });
 }
 exports.produceDataFlowGraph = produceDataFlowGraph;
 function processBinaryOp(node, data) {

package/dataflow/internal/process/functions/function-call.js

@@ -7,6 +7,7 @@ const assert_1 = require("../../../../util/assert");
 const index_1 = require("../../../index");
 const linker_1 = require("../../linker");
 const scopes_1 = require("../../../environments/scopes");
+const source_1 = require("./source");
 exports.UnnamedFunctionCallPrefix = 'unnamed-function-call-';
 function processFunctionCall(functionCall, data) {
     const named = functionCall.flavor === 'named';
@@ -86,7 +87,7 @@ function processFunctionCall(functionCall, data) {
         // push the called function to the ids:
         inIds.push(...functionName.in, ...functionName.unknownReferences);
     }
-    return {
+    let info = {
         unknownReferences: [],
         in: inIds,
         out: functionName.out, // we do not keep argument out as it has been linked by the function
@@ -94,6 +95,11 @@ function processFunctionCall(functionCall, data) {
         environments: finalEnv,
         scope: data.activeScope
     };
+    // parse a source call and analyze the referenced code
+    if ((0, source_1.isSourceCall)(functionCallName, data.activeScope, finalEnv)) {
+        info = (0, source_1.processSourceCall)(functionCall, data, info);
+    }
+    return info;
 }
 exports.processFunctionCall = processFunctionCall;
 //# sourceMappingURL=function-call.js.map

package/dataflow/internal/process/functions/source.d.ts

@@ -0,0 +1,8 @@
+import type { IdGenerator, NoInfo, RParseRequest, RParseRequestProvider, ParentInformation, RFunctionCall } from '../../../../r-bridge';
+import { type DataflowProcessorInformation } from '../../../processor';
+import { type DataflowScopeName, type Identifier, type REnvironmentInformation } from '../../../environments';
+import type { DataflowInformation } from '../../info';
+export declare function setSourceProvider(provider: RParseRequestProvider): void;
+export declare function isSourceCall(name: Identifier, scope: DataflowScopeName, environments: REnvironmentInformation): boolean;
+export declare function processSourceCall<OtherInfo>(functionCall: RFunctionCall<OtherInfo & ParentInformation>, data: DataflowProcessorInformation<OtherInfo & ParentInformation>, information: DataflowInformation): DataflowInformation;
+export declare function sourceRequest<OtherInfo>(request: RParseRequest, data: DataflowProcessorInformation<OtherInfo & ParentInformation>, information: DataflowInformation, getId: IdGenerator<NoInfo>): DataflowInformation;

package/dataflow/internal/process/functions/source.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sourceRequest = exports.processSourceCall = exports.isSourceCall = exports.setSourceProvider = void 0;
+const r_bridge_1 = require("../../../../r-bridge");
+const shell_executor_1 = require("../../../../r-bridge/shell-executor");
+const core_1 = require("../../../../core");
+const processor_1 = require("../../../processor");
+const environments_1 = require("../../../environments");
+const index_1 = require("../../../index");
+const config_1 = require("../../../../config");
+let sourceProvider = (0, r_bridge_1.requestProviderFromFile)();
+function setSourceProvider(provider) {
+    sourceProvider = provider;
+}
+exports.setSourceProvider = setSourceProvider;
+function isSourceCall(name, scope, environments) {
+    const definitions = (0, environments_1.resolveByName)(name, scope, environments);
+    if (definitions === undefined) {
+        return false;
+    }
+    // fail if there are multiple definitions because then we must treat the complete import as a maybe because it might do something different
+    if (definitions.length !== 1) {
+        return false;
+    }
+    const def = definitions[0];
+    return def.name == 'source' && def.kind == 'built-in-function';
+}
+exports.isSourceCall = isSourceCall;
+function processSourceCall(functionCall, data, information) {
+    const sourceFile = functionCall.arguments[0];
+    if ((0, config_1.getConfig)().ignoreSourceCalls) {
+        index_1.dataflowLogger.info(`Skipping source call ${JSON.stringify(sourceFile)} (disabled in config file)`);
+        return information;
+    }
+    if (sourceFile?.value?.type == "RString" /* RType.String */) {
+        const path = (0, r_bridge_1.removeRQuotes)(sourceFile.lexeme);
+        const request = sourceProvider.createRequest(path);
+        // check if the sourced file has already been dataflow analyzed, and if so, skip it
+        if (data.referenceChain.includes((0, r_bridge_1.requestFingerprint)(request))) {
+            index_1.dataflowLogger.info(`Found loop in dataflow analysis for ${JSON.stringify(request)}: ${JSON.stringify(data.referenceChain)}, skipping further dataflow analysis`);
+            return information;
+        }
+        return sourceRequest(request, data, information, (0, r_bridge_1.sourcedDeterministicCountingIdGenerator)(path, functionCall.location));
+    }
+    else {
+        index_1.dataflowLogger.info(`Non-constant argument ${JSON.stringify(sourceFile)} for source is currently not supported, skipping`);
+        return information;
+    }
+}
+exports.processSourceCall = processSourceCall;
+function sourceRequest(request, data, information, getId) {
+    const executor = new shell_executor_1.RShellExecutor();
+    // parse, normalize and dataflow the sourced file
+    let normalized;
+    let dataflow;
+    try {
+        const parsed = (0, core_1.executeSingleSubStep)('parse', request, executor);
+        normalized = (0, core_1.executeSingleSubStep)('normalize', parsed, undefined, getId);
+        dataflow = (0, processor_1.processDataflowFor)(normalized.ast, {
+            ...data,
+            currentRequest: request,
+            environments: information.environments,
+            referenceChain: [...data.referenceChain, (0, r_bridge_1.requestFingerprint)(request)]
+        });
+    }
+    catch (e) {
+        index_1.dataflowLogger.warn(`Failed to analyze sourced file ${JSON.stringify(request)}, skipping: ${e.message}`);
+        return information;
+    }
+    // update our graph with the sourced file's information
+    const newInformation = { ...information };
+    newInformation.environments = (0, environments_1.overwriteEnvironments)(information.environments, dataflow.environments);
+    newInformation.graph.mergeWith(dataflow.graph);
+    // this can be improved, see issue #628
+    for (const [k, v] of normalized.idMap) {
+        data.completeAst.idMap.set(k, v);
+    }
+    return newInformation;
+}
+exports.sourceRequest = sourceRequest;
+//# sourceMappingURL=source.js.map

package/dataflow/processor.d.ts

@@ -1,7 +1,7 @@
 /**
  * Based on a two-way fold, this processor will automatically supply scope information
  */
-import type { NormalizedAst, ParentInformation, RNode, RNodeWithParent } from '../r-bridge';
+import type { NormalizedAst, ParentInformation, RNode, RNodeWithParent, RParseRequest } from '../r-bridge';
 import type { DataflowInformation } from './internal/info';
 import type { DataflowScopeName, REnvironmentInformation } from './environments';
 export interface DataflowProcessorInformation<OtherInfo> {
@@ -22,6 +22,15 @@ export interface DataflowProcessorInformation<OtherInfo> {
    * Other processors to be called by the given functions
    */
     readonly processors: DataflowProcessors<OtherInfo>;
+    /**
+     * The {@link RParseRequest} that is currently being parsed
+     */
+    readonly currentRequest: RParseRequest;
+    /**
+     * The chain of {@link RParseRequest} fingerprints ({@link requestFingerprint}) that lead to the {@link currentRequest}.
+     * The most recent (last) entry is expected to always be the {@link currentRequest}.
+     */
+    readonly referenceChain: string[];
 }
 export type DataflowProcessor<OtherInfo, NodeType extends RNodeWithParent<OtherInfo>> = (node: NodeType, data: DataflowProcessorInformation<OtherInfo>) => DataflowInformation;
 type NodeWithKey<OtherInfo, Node extends RNode<OtherInfo & ParentInformation>, TypeKey> = Node['type'] extends TypeKey ? Node : never;

package/index.d.ts

@@ -2,5 +2,3 @@ export * from './core';
 export * from './slicing';
 export * from './dataflow';
 export * from './r-bridge';
-export * from './benchmark';
-export * from './statistics';

package/index.js

@@ -18,6 +18,4 @@ __exportStar(require("./core"), exports);
 __exportStar(require("./slicing"), exports);
 __exportStar(require("./dataflow"), exports);
 __exportStar(require("./r-bridge"), exports);
-__exportStar(require("./benchmark"), exports);
-__exportStar(require("./statistics"), exports);
 //# sourceMappingURL=index.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eagleoutice/flowr",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "Static Dataflow Analyzer and Program Slicer for the R Programming Language",
   "types": "dist/src/index.d.ts",
   "repository": {
@@ -13,22 +13,14 @@
   },
   "scripts": {
     "publish-library": "cp .npmignore package.json README.md LICENSE dist/src/ && cd dist/src && npm publish --access public",
-    "main": "ts-node src/flowr.ts",
-    "stats": "ts-node src/cli/statistics-app.ts",
-    "stats-helper": "ts-node src/cli/statistics-helper-app.ts",
-    "slicer": "ts-node src/cli/slicer-app.ts",
     "release": "release-it --ci",
-    "benchmark-helper": "ts-node src/cli/benchmark-helper-app.ts",
-    "benchmark": "ts-node src/cli/benchmark-app.ts",
-    "summarizer": "ts-node src/cli/summarizer-app.ts",
-    "export-quads": "ts-node src/cli/export-quads-app.ts",
     "build": "tsc --project .",
     "lint-local": "eslint src/ test/ --rule \"no-warning-comments: off\"",
     "lint": "npm run license-compat -- --summary && eslint src/ test/",
     "license-compat": "license-checker --onlyAllow 'MIT;MIT OR X11;GPLv2;LGPL;GNUGPL;ISC;Apache-2.0;FreeBSD;BSD-2-Clause;clearbsd;ModifiedBSD;BSD-3-Clause;Python-2.0;Unlicense;WTFPL;CC-BY-4.0;CC-BY-3.0;CC0-1.0;0BSD'",
     "doc": "typedoc",
-    "test": "nyc --no-clean mocha --require ts-node/register --timeout 60000 \"test/**/*.spec.ts\"",
-    "performance-test": "func() { cd test/performance/ && bash run-all-suites.sh $1 $2; cd ../../; }; func",
+    "test": "nyc --no-clean mocha",
+    "performance-test": "func() { cd test/performance/ && bash run-all-suites.sh $1 $2 $3; cd ../../; }; func",
     "test-full": "npm run test -- --test-installation"
   },
   "keywords": [
@@ -41,6 +33,11 @@
   ],
   "author": "Florian Sihler",
   "license": "ISC",
+  "mocha": {
+    "require": "ts-node/register",
+    "timeout": 60000,
+    "spec": "test/**/*.spec.ts"
+  },
   "nyc": {
     "all": true,
     "per-file": true,
@@ -58,6 +55,69 @@
       "cobertura"
     ]
   },
+  "eslintConfig": {
+    "settings": {
+      "import/resolver": {
+        "node": {
+          "extensions": [
+            ".ts"
+          ]
+        }
+      }
+    },
+    "extends": [
+      "@eagleoutice/eslint-config-flowr"
+    ],
+    "rules": {
+      "import/no-restricted-paths": [
+        "error",
+        {
+          "zones": [
+            {
+              "target": "benchmark",
+              "from": [
+                "cli",
+                "src",
+                "statistics"
+              ]
+            },
+            {
+              "target": "cli",
+              "from": [
+                "benchmark",
+                "src",
+                "statistics"
+              ]
+            },
+            {
+              "target": "statistics",
+              "from": [
+                "benchmark",
+                "cli",
+                "src"
+              ]
+            },
+            {
+              "target": "src",
+              "from": [
+                "benchmark",
+                "cli",
+                "statistics"
+              ]
+            },
+            {
+              "target": "test",
+              "from": [
+                "benchmark",
+                "cli",
+                "statistics"
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  },
   "typedocOptions": {
     "includeVersion": true,
     "plugin": [
@@ -109,195 +169,6 @@
     },
     "treatValidationWarningsAsErrors": false
   },
-  "eslintConfig": {
-    "root": true,
-    "env": {
-      "browser": false,
-      "node": true,
-      "es2022": true
-    },
-    "extends": [
-      "eslint:recommended",
-      "plugin:@typescript-eslint/recommended-requiring-type-checking",
-      "plugin:@typescript-eslint/strict",
-      "plugin:@typescript-eslint/recommended"
-    ],
-    "overrides": [],
-    "parserOptions": {
-      "ecmaVersion": "latest",
-      "sourceType": "module",
-      "project": "./tsconfig.json"
-    },
-    "parser": "@typescript-eslint/parser",
-    "plugins": [
-      "@typescript-eslint",
-      "eslint-plugin-tsdoc",
-      "check-file"
-    ],
-    "rules": {
-      "indent": "off",
-      "@typescript-eslint/indent": [
-        "error",
-        "tab",
-        {
-          "FunctionDeclaration": {
-            "parameters": "first"
-          },
-          "ObjectExpression": 1,
-          "SwitchCase": 1
-        }
-      ],
-      "quotes": [
-        "error",
-        "single",
-        {
-          "avoidEscape": true
-        }
-      ],
-      "no-mixed-spaces-and-tabs": [
-        "error",
-        "smart-tabs"
-      ],
-      "no-warning-comments": [
-        "error",
-        {
-          "terms": [
-            "todo",
-            "fixme",
-            "xxx"
-          ],
-          "location": "anywhere"
-        }
-      ],
-      "@typescript-eslint/non-nullable-type-assertion-style": "off",
-      "@typescript-eslint/no-unsafe-enum-comparison": "off",
-      "@typescript-eslint/no-redundant-type-constituents": "off",
-      "@typescript-eslint/consistent-type-assertions": [
-        "error",
-        {
-          "assertionStyle": "as"
-        }
-      ],
-      "@typescript-eslint/key-spacing": [
-        "error",
-        {
-          "align": "value"
-        }
-      ],
-      "semi": [
-        "error",
-        "never"
-      ],
-      "space-before-function-paren": [
-        "error",
-        "never"
-      ],
-      "keyword-spacing": "off",
-      "check-file/filename-naming-convention": [
-        "error",
-        {
-          "**/*.ts": "?([A-Z])+([a-z])*((-|.)?([A-Z])+([a-z]))"
-        }
-      ],
-      "check-file/folder-match-with-fex": [
-        "error",
-        {
-          "*.spec.{js,jsx,ts,tsx}": "test/**"
-        }
-      ],
-      "@typescript-eslint/keyword-spacing": [
-        "error",
-        {
-          "before": true,
-          "after": true,
-          "overrides": {
-            "if": {
-              "after": false
-            },
-            "for": {
-              "after": false
-            },
-            "while": {
-              "after": false
-            },
-            "do": {
-              "after": false
-            },
-            "catch": {
-              "after": false
-            },
-            "switch": {
-              "after": false
-            },
-            "default": {
-              "after": false
-            },
-            "throw": {
-              "after": false
-            }
-          }
-        }
-      ],
-      "@typescript-eslint/space-before-function-paren": [
-        "error",
-        "never"
-      ],
-      "@typescript-eslint/no-unused-vars": [
-        "error",
-        {
-          "argsIgnorePattern": "^_",
-          "destructuredArrayIgnorePattern": "^_",
-          "varsIgnorePattern": "^_"
-        }
-      ],
-      "tsdoc/syntax": "error",
-      "@typescript-eslint/naming-convention": [
-        "error",
-        {
-          "selector": "variable",
-          "modifiers": [
-            "const",
-            "global",
-            "exported"
-          ],
-          "format": [
-            "camelCase",
-            "PascalCase",
-            "UPPER_CASE"
-          ],
-          "leadingUnderscore": "allow",
-          "trailingUnderscore": "allow"
-        },
-        {
-          "selector": "variable",
-          "modifiers": [
-            "const"
-          ],
-          "format": [
-            "camelCase",
-            "PascalCase"
-          ],
-          "leadingUnderscore": "allow",
-          "trailingUnderscore": "allow"
-        },
-        {
-          "selector": "enumMember",
-          "format": [
-            "StrictPascalCase"
-          ],
-          "leadingUnderscore": "forbid",
-          "trailingUnderscore": "forbid"
-        },
-        {
-          "selector": "typeLike",
-          "format": [
-            "PascalCase"
-          ]
-        }
-      ],
-      "@typescript-eslint/consistent-type-imports": "error"
-    }
-  },
   "release-it": {
     "git": {
       "commitMessage": "[skip ci] Release v${version}",
@@ -335,6 +206,10 @@
   "devDependencies": {
     "@commitlint/cli": "^18.4.3",
     "@commitlint/config-angular": "^18.4.3",
+    "@eagleoutice/eslint-config-flowr": "^1.0.2",
+    "@stylistic/eslint-plugin": "^1.6.2",
+    "@stylistic/eslint-plugin-plus": "^1.6.2",
+    "@stylistic/eslint-plugin-ts": "^1.6.2",
     "@types/chai": "^4.3.11",
     "@types/chai-as-promised": "^7.1.8",
     "@types/command-line-args": "^5.2.3",
@@ -344,7 +219,6 @@
     "@types/n3": "^1.16.4",
     "@types/object-hash": "^3.0.6",
     "@types/tmp": "^0.2.6",
-    "@types/xml2js": "^0.4.14",
     "@typescript-eslint/eslint-plugin": "^6.13.2",
     "chai": "^4.3.10",
     "chai-as-promised": "^7.1.1",
@@ -386,7 +260,6 @@
     "ts-essentials": "^9.4.1",
     "tslog": "^4.9.2",
     "ws": "^8.16.0",
-    "xml2js": "^0.6.2",
     "xpath-ts2": "^1.4.2"
   }
 }

package/r-bridge/lang-4.x/ast/index.d.ts

@@ -1,2 +1,3 @@
 export * from './model';
 export * from './parser/xml';
+export { parseLog } from './parser/json/parser';

package/r-bridge/lang-4.x/ast/index.js

@@ -14,6 +14,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseLog = void 0;
 __exportStar(require("./model"), exports);
 __exportStar(require("./parser/xml"), exports);
+var parser_1 = require("./parser/json/parser");
+Object.defineProperty(exports, "parseLog", { enumerable: true, get: function () { return parser_1.parseLog; } });
 //# sourceMappingURL=index.js.map

package/r-bridge/lang-4.x/ast/model/processing/decorate.d.ts

@@ -9,6 +9,7 @@
  * @module
  */
 import type { NoInfo, RNode } from '../model';
+import type { SourceRange } from '../../../../../util/range';
 import { BiMap } from '../../../../../util/bimap';
 import type { MergeableRecord } from '../../../../../util/objects';
 import { RoleInParent } from './role';
@@ -27,6 +28,7 @@ export type IdGenerator<OtherInfo> = (data: RNode<OtherInfo>) => NodeId;
  * The simplest id generator which just increments a number on each call.
  */
 export declare function deterministicCountingIdGenerator(start?: number): () => NodeId;
+export declare function sourcedDeterministicCountingIdGenerator(path: string, location: SourceRange, start?: number): () => NodeId;
 /**
  * Generates the location id, used by {@link deterministicLocationIdGenerator}.
  *

package/r-bridge/lang-4.x/ast/model/processing/decorate.js

@@ -10,7 +10,7 @@
  * @module
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.decorateAst = exports.deterministicLocationIdGenerator = exports.nodeToLocationId = exports.deterministicCountingIdGenerator = void 0;
+exports.decorateAst = exports.deterministicLocationIdGenerator = exports.nodeToLocationId = exports.sourcedDeterministicCountingIdGenerator = exports.deterministicCountingIdGenerator = void 0;
 const assert_1 = require("../../../../../util/assert");
 const bimap_1 = require("../../../../../util/bimap");
 const fold_1 = require("./fold");
@@ -22,6 +22,11 @@ function deterministicCountingIdGenerator(start = 0) {
     return () => `${id++}`;
 }
 exports.deterministicCountingIdGenerator = deterministicCountingIdGenerator;
+function sourcedDeterministicCountingIdGenerator(path, location, start = 0) {
+    let id = start;
+    return () => `${path}-${loc2Id(location)}-${id++}`;
+}
+exports.sourcedDeterministicCountingIdGenerator = sourcedDeterministicCountingIdGenerator;
 function loc2Id(loc) {
     return `${loc.start.line}:${loc.start.column}-${loc.end.line}:${loc.end.column}`;
 }

package/r-bridge/lang-4.x/ast/parser/json/format.d.ts

@@ -0,0 +1,14 @@
+export declare const RootId = 0;
+export interface Entry extends Record<string, unknown> {
+    line1: number;
+    col1: number;
+    line2: number;
+    col2: number;
+    id: number;
+    parent: number;
+    token: string;
+    terminal: boolean;
+    text: string;
+    children?: Entry[];
+}
+export declare function prepareParsedData(data: string): Map<number, Entry>;

package/r-bridge/lang-4.x/ast/parser/json/format.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prepareParsedData = exports.RootId = void 0;
+const retriever_1 = require("../../../../retriever");
+const assert_1 = require("../../../../../util/assert");
+exports.RootId = 0;
+function prepareParsedData(data) {
+    const json = JSON.parse(data);
+    (0, assert_1.guard)(Array.isArray(json), () => `Expected ${data} to be an array but was not`);
+    const ret = new Map(json.map(([line1, col1, line2, col2, id, parent, token, terminal, text]) => {
+        return [id, { line1, col1, line2, col2, id, parent, token: (0, retriever_1.removeRQuotes)(token), terminal, text }];
+    }));
+    // iterate a second time to set parent-child relations (since they may be out of order in the csv)
+    for (const entry of ret.values()) {
+        if (entry.parent != exports.RootId) {
+            const parent = ret.get(entry.parent);
+            if (parent) {
+                parent.children ??= [];
+                parent.children.push(entry);
+            }
+        }
+    }
+    return ret;
+}
+exports.prepareParsedData = prepareParsedData;
+//# sourceMappingURL=format.js.map

package/r-bridge/lang-4.x/ast/parser/json/parser.d.ts

@@ -0,0 +1,7 @@
+import type { DeepPartial } from 'ts-essentials';
+import type { XmlBasedJson, XmlParserHooks } from '../xml';
+import type { IdGenerator, NoInfo, NormalizedAst } from '../../model';
+import type { Entry } from './format';
+export declare const parseLog: import("tslog").Logger<import("tslog").ILogObj>;
+export declare function normalize(jsonString: string, hooks?: DeepPartial<XmlParserHooks>, getId?: IdGenerator<NoInfo>): NormalizedAst;
+export declare function convertPreparedParsedData(valueMapping: Map<number, Entry>): XmlBasedJson;