@eagleoutice/flowr 2.2.1 → 2.2.3

package/documentation/print-capabilities-markdown.js

@@ -1,18 +1,81 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const data_1 = require("../r-bridge/data/data");
-const version_1 = require("../util/version");
 const log_1 = require("../../test/functionality/_helper/log");
+const doc_auto_gen_1 = require("./doc-util/doc-auto-gen");
+const strings_1 = require("../util/strings");
+const doc_general_1 = require("./doc-util/doc-general");
+const tree_sitter_executor_1 = require("../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor");
+const fs_1 = __importDefault(require("fs"));
+const doc_structure_1 = require("./doc-util/doc-structure");
+const detailedInfoFile = 'coverage/flowr-test-details.json';
+function obtainDetailedInfos() {
+    if (!fs_1.default.existsSync(detailedInfoFile)) {
+        return undefined;
+    }
+    const content = fs_1.default.readFileSync(detailedInfoFile).toString();
+    const base = JSON.parse(content);
+    const out = new Map();
+    for (const [key, values] of base) {
+        out.set(key, values.map(v => ({
+            id: v.id,
+            name: v.name,
+            capabilities: new Set(v.capabilities),
+            context: new Set(v.context)
+        })));
+    }
+    return out;
+}
 const supportedSymbolMap = new Map([
-    ['not', ':red_circle:'],
-    ['partially', ':large_orange_diamond:'],
-    ['fully', ':green_square:']
+    ['not', '🔴'],
+    ['partially', '🔶'],
+    ['fully', '🟩']
 ]);
-function printSingleCapability(depth, index, capability) {
+function getTestDetails(info, capability) {
+    if (!info.info) {
+        return '';
+    }
+    const totalTests = info.info.get(capability.id);
+    const uniqueTests = totalTests?.filter((v, i, a) => a.findIndex(t => t.id === v.id) === i);
+    if (!uniqueTests || uniqueTests.length === 0) {
+        return '';
+    }
+    const grouped = new Map();
+    for (const { context } of uniqueTests) {
+        for (const c of context) {
+            grouped.set(c, (grouped.get(c) ?? 0) + 1);
+        }
+    }
+    if (grouped.get('desugar-tree-sitter') !== undefined && grouped.get('desugar-tree-sitter') === grouped.get('desugar-shell')) {
+        grouped.set('desugar', grouped.get('desugar-tree-sitter') ?? 0);
+        grouped.delete('desugar-shell');
+        grouped.delete('desugar-tree-sitter');
+    }
+    grouped.delete('other'); // opinionated view on the categories
+    const output = grouped.get('output');
+    grouped.delete('output');
+    const testString = [`${uniqueTests.length} test${uniqueTests.length !== 1 ? 's' : ''}`];
+    // sort by count
+    const sorted = [...grouped.entries()].sort((a, b) => b[1] - a[1]);
+    for (const [context, count] of sorted) {
+        testString.push(`${context}: ${count}`);
+    }
+    if (output) {
+        testString.push(`and backed with output: ${output}`);
+    }
+    return ` (${testString.join(', ')})`;
+}
+function escapeId(id) {
+    return id.replace(/[^a-zA-Z0-9]/g, '_');
+}
+async function printSingleCapability(info, depth, index, capability) {
     const indent = '    '.repeat(depth);
     const indexStr = index.toString().padStart(2, ' ');
     const nextLineIndent = '  '.repeat(depth + indexStr.length);
-    const mainLine = `${indent}${indexStr}. **${capability.name}** (<a id='${capability.id}'>\`${capability.id}\`</a>)`;
+    const mainLine = `${indent}${indexStr}. <a id='${capability.id}'></a>**${capability.name}** <a href="#${escapeId(capability.id)}">🔗</a>${getTestDetails(info, capability)}`;
     let nextLine = '';
     if (capability.supported) {
         nextLine += `${supportedSymbolMap.get(capability.supported)} `;
@@ -20,43 +83,66 @@ function printSingleCapability(depth, index, capability) {
     if (capability.description) {
         nextLine += capability.description;
     }
-    if (capability.note) {
-        nextLine += `\\\n${nextLineIndent}_${capability.note}_`;
+    if (capability.url) {
+        nextLine += '\\\nSee ' + (0, strings_1.joinWithLast)(capability.url.map(({ name, href }) => `[${name}](${href})`)) + ' for more info.';
+    }
+    nextLine += ' Internal ID: `' + capability.id + '`';
+    if (capability.example) {
+        nextLine += `\n${(0, doc_general_1.prefixLines)(typeof capability.example === 'string' ? capability.example : await capability.example(info.parser), nextLineIndent + '> ')}`;
     }
     return nextLine ? `${mainLine}\\\n${nextLineIndent}${nextLine}` : mainLine;
 }
-function printAsMarkdown(capabilities, depth = 0, lines = []) {
+async function printAsMarkdown(info, capabilities, depth = 0, lines = []) {
     for (let i = 0; i < capabilities.length; i++) {
         const capability = capabilities[i];
-        const result = printSingleCapability(depth, i + 1, capability);
+        const result = await printSingleCapability(info, depth, i + 1, capability);
         lines.push(result);
         if (capability.capabilities) {
-            printAsMarkdown(capability.capabilities, depth + 1, lines);
+            await printAsMarkdown(info, capability.capabilities, depth + 1, lines);
         }
     }
     return lines.join('\n');
 }
 function getPreamble() {
-    const currentDateAndTime = new Date().toISOString().replace('T', ', ').replace(/\.\d+Z$/, ' UTC');
-    const shortenFilename = module.filename.replace(/^.*src\//, 'src/');
-    return `_This document was generated from '${shortenFilename}' on ${currentDateAndTime} summarizing flowR's current capabilities (v${(0, version_1.flowrVersion)().format()})._
+    return `${(0, doc_auto_gen_1.autoGenHeader)({ filename: module.filename, purpose: 'current capabilities' })}
 
-The code-font behind each capability name is a link to the capability's id. This id can be used to reference the capability in a labeled test within flowR.
+Each capability has an id that can be used to link to it (use the link symbol to get a direct link to the capability).
+The internal id is also mentioned in the capability description. This id can be used to reference the capability in a labeled test within flowR.
 Besides, we use colored bullets like this:
 
 | <!-- -->               | <!-- -->                                              |
 | ---------------------- | ----------------------------------------------------- |
-| :green_square:         | _flowR_ is capable of handling this feature fully     |
-| :large_orange_diamond: | _flowR_ is capable of handling this feature partially |
-| :red_circle:           | _flowR_ is not capable of handling this feature       |
+| ${supportedSymbolMap.get('fully')} | _flowR_ is capable of handling this feature _fully_     |
+| ${supportedSymbolMap.get('partially')} | _flowR_ is capable of handling this feature _partially_ |
+| ${supportedSymbolMap.get('not')} | _flowR_ is _not_ capable of handling this feature     |
 
 :cloud: This could be a feature diagram... :cloud:
 
+${(0, doc_structure_1.block)({
+        type: 'NOTE',
+        content: `
+The capabilities are a qualitative measure of the features that flowR can handle.
+Statements like "flowR can fully handle 50/80 capabilities" are discouraged as the capabilities may have a vastly different granularity.
+Please prefer using a statement like "flowR has only partial support for feature 'XY'" (or simply reference this document) within the flowR sources.
+	`
+    })}
 `;
 }
+async function print(parser) {
+    /* check if the detailed test data is available */
+    if (!fs_1.default.existsSync(detailedInfoFile)) {
+        console.warn('\x1b[31mNo detailed test data available. Run the full tests (npm run test-full) to generate it.\x1b[m');
+    }
+    return getPreamble() + await printAsMarkdown({ parser, info: obtainDetailedInfos() }, data_1.flowrCapabilities.capabilities);
+}
 /** if we run this script, we want a Markdown representation of the capabilities */
 if (require.main === module) {
     (0, log_1.setMinLevelOfAllLogs)(6 /* LogLevel.Fatal */);
-    console.log(getPreamble() + printAsMarkdown(data_1.flowrCapabilities.capabilities));
+    void tree_sitter_executor_1.TreeSitterExecutor.initTreeSitter().then(() => {
+        const parser = new tree_sitter_executor_1.TreeSitterExecutor();
+        void print(parser).then(str => {
+            console.log(str);
+        });
+    });
 }
 //# sourceMappingURL=print-capabilities-markdown.js.map

package/documentation/print-core-wiki.d.ts

@@ -0,0 +1 @@
+export {};

package/documentation/print-core-wiki.js

@@ -0,0 +1,406 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const shell_1 = require("../r-bridge/shell");
+const log_1 = require("../../test/functionality/_helper/log");
+const log_2 = require("../util/log");
+const doc_auto_gen_1 = require("./doc-util/doc-auto-gen");
+const doc_structure_1 = require("./doc-util/doc-structure");
+const doc_files_1 = require("./doc-util/doc-files");
+const doc_cli_option_1 = require("./doc-util/doc-cli-option");
+const doc_types_1 = require("./doc-util/doc-types");
+const path_1 = __importDefault(require("path"));
+const doc_code_1 = require("./doc-util/doc-code");
+const extractor_1 = require("../dataflow/extractor");
+const parser_1 = require("../r-bridge/parser");
+const parser_2 = require("../r-bridge/lang-4.x/ast/parser/json/parser");
+const doc_repl_1 = require("./doc-util/doc-repl");
+const doc_dfg_1 = require("./doc-util/doc-dfg");
+const doc_normalized_ast_1 = require("./doc-util/doc-normalized-ast");
+const init_1 = require("../r-bridge/init");
+const format_1 = require("../r-bridge/lang-4.x/ast/parser/json/format");
+const normalize_root_1 = require("../r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-root");
+const decorate_1 = require("../r-bridge/lang-4.x/ast/model/processing/decorate");
+const process_uninteresting_leaf_1 = require("../dataflow/internal/process/process-uninteresting-leaf");
+const built_in_access_1 = require("../dataflow/internal/process/functions/call/built-in/built-in-access");
+const built_in_for_loop_1 = require("../dataflow/internal/process/functions/call/built-in/built-in-for-loop");
+const built_in_repeat_loop_1 = require("../dataflow/internal/process/functions/call/built-in/built-in-repeat-loop");
+const linker_1 = require("../dataflow/internal/linker");
+const static_slicer_1 = require("../slicing/static/static-slicer");
+const info_1 = require("../dataflow/info");
+const processor_1 = require("../dataflow/processor");
+const default_pipelines_1 = require("../core/steps/pipeline/default-pipelines");
+const tree_sitter_executor_1 = require("../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor");
+const retriever_1 = require("../r-bridge/retriever");
+const json_1 = require("../util/json");
+const stateful_fold_1 = require("../r-bridge/lang-4.x/ast/model/processing/stateful-fold");
+const normalize_single_node_1 = require("../r-bridge/lang-4.x/ast/parser/main/internal/structure/normalize-single-node");
+const normalize_if_then_1 = require("../r-bridge/lang-4.x/ast/parser/main/internal/control/normalize-if-then");
+const normalize_for_1 = require("../r-bridge/lang-4.x/ast/parser/main/internal/loops/normalize-for");
+const doc_issue_1 = require("./doc-util/doc-issue");
+const pipeline_executor_1 = require("../core/pipeline-executor");
+const pipeline_1 = require("../core/steps/pipeline/pipeline");
+async function getText(shell) {
+    const rversion = (await shell.usedRVersion())?.format() ?? 'unknown';
+    const sampleCode = 'x <- 1; print(x)';
+    const { info, program } = (0, doc_types_1.getTypesFromFolderAsMermaid)({
+        rootFolder: path_1.default.resolve('./src'),
+        typeName: shell_1.RShell.name,
+        inlineTypes: doc_types_1.mermaidHide
+    });
+    return `${(0, doc_auto_gen_1.autoGenHeader)({ filename: module.filename, purpose: 'core', rVersion: rversion })}
+
+This wiki page provides an overview of the inner workings of _flowR_.
+It is mostly intended for developers that want to extend the capabilities of _flowR_
+and assumes knowledge of [TypeScript](https://www.typescriptlang.org/) and [R](https://www.r-project.org/).
+If you think parts of the wiki are missing, wrong, or outdated, please do not hesitate to [open a new issue](${doc_issue_1.NewIssueUrl})!
+In case you are new and want to develop for flowR, please check out the relevant [Setup](${doc_files_1.FlowrWikiBaseRef}/Setup#-developing-for-flowr) wiki page
+and the [Contributing Guidelines](${doc_files_1.RemoteFlowrFilePathBaseRef}/.github/CONTRIBUTING.md).
+
+${(0, doc_structure_1.block)({
+        type: 'NOTE',
+        content: `
+Essentially every step we explain here can be explored directly from flowR's REPL in an interactive fashion (see the [Interface](${doc_files_1.FlowrWikiBaseRef}/Interface#using-the-repl) wiki page).
+We recommend to use commands like ${(0, doc_cli_option_1.getReplCommand)('parse')} or ${(0, doc_cli_option_1.getReplCommand)('dataflow*')} to explore the output of flowR using your own samples.
+As a quickstart you may use:
+
+${await (0, doc_repl_1.documentReplSession)(shell, [{
+                command: `:parse "${sampleCode}"`,
+                description: `Retrieves the AST from the ${(0, doc_types_1.shortLink)(shell_1.RShell.name, info)}.`
+            }])}
+	
+If you are brave (or desperate) enough, you can also try to use the ${(0, doc_cli_option_1.getCliLongOptionOf)('flowr', 'verbose')} option to be dumped with information about flowR's internals (please, never use this for benchmarking).
+See the [Getting flowR to Talk](#getting-flowr-to-talk) section below for more information.
+`
+    })}
+	
+* [Pipelines and their Execution](#pipelines-and-their-execution)
+* [How flowR Produces Dataflow Graphs](#how-flowr-produces-dataflow-graphs)
+  * [Overview](#overview)
+  * [Parsing](#parsing)
+  * [Normalization](#normalization)
+  * [Dataflow Graph Generation](#dataflow-graph-generation)
+* [Beyond the Dataflow Graph](#beyond-the-dataflow-graph)
+  * [Static Backward Slicing](#static-backward-slicing)
+* [Getting flowR to Talk](#getting-flowr-to-talk)
+	
+## Pipelines and their Execution
+
+At the core of every analysis by flowR is the ${(0, doc_types_1.shortLink)(pipeline_executor_1.PipelineExecutor.name, info)} class which takes a sequence of analysis steps (in the form of a ${(0, doc_types_1.shortLink)('Pipeline', info)}) and executes it
+on a given input. In general, these pipeline steps are analysis agnostic and may use arbitrary input and ordering. However, two important and predefined pipelines, 
+the ${(0, doc_types_1.shortLink)('DEFAULT_DATAFLOW_PIPELINE', info)} and the ${(0, doc_types_1.shortLink)('TREE_SITTER_DATAFLOW_PIPELINE', info)} adequately cover the most common analysis steps 
+(differentiated only by the [Engine](${doc_files_1.FlowrWikiBaseRef}/Engines) used).
+
+${(0, doc_structure_1.block)({
+        type: 'TIP',
+        content: `
+	You can hover over most links within these wiki pages to get access to the tsdoc comment of the respective element. 
+	The links should direct you to the up-to-date implementation.
+`
+    })}
+	
+Using the [\`tree-sitter\` engine](${doc_files_1.FlowrWikiBaseRef}/Engines) you can request a dataflow analysis of a sample piece of R code like the following:
+
+${(0, doc_code_1.codeBlock)('typescript', `
+const executor = new PipelineExecutor(TREE_SITTER_DATAFLOW_PIPELINE, {
+	parser:  new TreeSitterExecutor(),
+	request: requestFromInput('x <- 1; y <- x; print(y);')
+});
+const result = await executor.allRemainingSteps();
+`)}
+
+This is, roughly, what the ${(0, doc_types_1.shortLink)('replGetDataflow', info)} function does for the ${(0, doc_cli_option_1.getReplCommand)('dataflow')} REPL command when using the [\`tree-sitter\` engine](${doc_files_1.FlowrWikiBaseRef}/Engines).
+We create a new ${(0, doc_types_1.shortLink)(pipeline_executor_1.PipelineExecutor.name, info)} with the ${(0, doc_types_1.shortLink)('TREE_SITTER_DATAFLOW_PIPELINE', info)} and then use ${(0, doc_types_1.shortLink)(`${pipeline_executor_1.PipelineExecutor.name}::${new pipeline_executor_1.PipelineExecutor(default_pipelines_1.TREE_SITTER_PARSE_PIPELINE, { parser: new tree_sitter_executor_1.TreeSitterExecutor(), request: (0, retriever_1.requestFromInput)('') }).allRemainingSteps.name}`, info)} 
+to cause the execution of all contained steps (in general, pipelines can be executed step-by-step, but this is usually not required if you just want the result).
+${(0, doc_types_1.shortLink)(retriever_1.requestFromInput.name, info)} is merely a convenience function to create a request object from a code string.
+
+In general, however, most flowR-internal functions which are tasked with generating dataflow prefer the use of ${(0, doc_types_1.shortLink)(default_pipelines_1.createDataflowPipeline.name, info)} as this function
+automatically selects the correct pipeline based on the engine used.
+
+### Understanding Pipeline Steps
+
+Everything that complies to the ${(0, doc_types_1.shortLink)('IPipelineStep', info)} interface can be used as a step in a pipeline, with the most important definition being the
+\`processor\` function, which refers to the actual work performed by the step.
+For example, the ${(0, doc_types_1.shortLink)('STATIC_DATAFLOW', info)} step ultimately relies on the ${(0, doc_types_1.shortLink)(extractor_1.produceDataFlowGraph.name, info)} function to create a [dataflow graph](${doc_files_1.FlowrWikiBaseRef}/Dataflow-Graph) 
+using the [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST) of the program.
+
+### Shape of a Pipeline Step
+
+Using code, you can provide an arbitrary pipeline step to the executor, as long as it implements the ${(0, doc_types_1.shortLink)('IPipelineStep', info)} interface:
+
+${(0, doc_types_1.printHierarchy)({ program, info, root: 'IPipelineStep', maxDepth: 0 })}
+
+Every step may specify required inputs, ways of visualizing the output, and its dependencies using the ${(0, doc_types_1.shortLink)('IPipelineStepOrder', info)} interface.
+As the types may seem to be somewhat confusing or over-complicated, we recommend you to look at some existing steps, like 
+the ${(0, doc_types_1.shortLink)('PARSE_WITH_R_SHELL_STEP', info)} or the ${(0, doc_types_1.shortLink)('STATIC_DATAFLOW', info)} step.
+The pipeline executor should do a good job of scheduling these steps (usually using a topological sort), and inferring the required inputs in the type system (have a look at the ${(0, doc_types_1.shortLink)(pipeline_1.createPipeline.name, info)} function if you want to know more).
+
+${(0, doc_structure_1.block)({
+        type: 'NOTE',
+        content: `
+Under the hood there is a step-subtype called a decoration. Such a step can be added to a pipeline to decorate the output of another one (e.g., making it more precise, re-adding debug info, ...).
+To mark a step as a decoration, you can use the \`decorates\` field in the ${(0, doc_types_1.shortLink)('IPipelineStepOrder', info)} interface.
+However, as such steps are currently not relevant for any of flowR's core analyses we will not go into detail here. It suffices to know how "real" steps work.
+`
+    })}
+	
+## How flowR Produces Dataflow Graphs
+
+This section focuses on the generation of a [dataflow graph](${doc_files_1.FlowrWikiBaseRef}/Dataflow-Graph) from a given R program, using the [RShell Engine](${doc_files_1.FlowrWikiBaseRef}/Engines) and hence the 
+${(0, doc_types_1.shortLink)('DEFAULT_DATAFLOW_PIPELINE', info)}. The [\`tree-sitter\` engine](${doc_files_1.FlowrWikiBaseRef}/Engines) uses the ${(0, doc_types_1.shortLink)('TREE_SITTER_DATAFLOW_PIPELINE', info)}), 
+which replaces the parser with the integrated tree-sitter parser and hence uses a slightly adapted normalization step to produce a similar [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST).
+The [dataflow graph](${doc_files_1.FlowrWikiBaseRef}/Dataflow-Graph) should be the same for both engines (although [\`tree-sitter\`](${doc_files_1.FlowrWikiBaseRef}/Engines) is faster and may be able to parse more files).
+
+### Overview
+
+Let's have a look at the definition of the pipeline:
+
+${(0, doc_types_1.printHierarchy)({ program, info, root: 'DEFAULT_DATAFLOW_PIPELINE', maxDepth: 0 })}
+
+We can see that it relies on three steps:
+
+1. **${(0, doc_types_1.shortLink)('PARSE_WITH_R_SHELL_STEP', info, false)}** ([parsing](#parsing)): Uses the ${(0, doc_types_1.shortLink)(shell_1.RShell.name, info)} to parse the input program.\\
+   _Its main function linked as the processor is the ${(0, doc_types_1.shortLink)(parser_1.parseRequests.name, info, false)} function._
+2. **${(0, doc_types_1.shortLink)('NORMALIZE', info, false)}** ([normalization](#normalization)):  Normalizes the AST produced by the parser (to create a [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST)).\\
+   _Its main function linked as the processor is the ${(0, doc_types_1.shortLink)(parser_2.normalize.name, info, false)} function._
+3. **${(0, doc_types_1.shortLink)('STATIC_DATAFLOW', info, false)}** ([dataflow](#dataflow-graph-generation)): Produces the actual [dataflow graph](${doc_files_1.FlowrWikiBaseRef}/Dataflow-Graph) from the normalized AST.\\
+   _Its main function linked as the processor is the ${(0, doc_types_1.shortLink)(extractor_1.produceDataFlowGraph.name, info, false)} function._
+
+To explore these steps, let's use the REPL with the (very simple and contrived) R code: \`${sampleCode}\`.
+
+${await (0, doc_repl_1.documentReplSession)(shell, [{
+            command: `:parse "${sampleCode}"`,
+            description: `This shows the ASCII-Art representation of the parse-tree of the R code \`${sampleCode}\`, as it is provided by the ${(0, doc_types_1.shortLink)(shell_1.RShell.name, info)}. See the ${(0, doc_types_1.shortLink)(init_1.initCommand.name, info)} function for more information on how we request a parse.`
+        },
+        {
+            command: `:normalize* "${sampleCode}"`,
+            description: `Following the link output should show the following:\n${await (0, doc_normalized_ast_1.printNormalizedAstForCode)(shell, sampleCode, { showCode: false })}`
+        },
+        {
+            command: `:dataflow* "${sampleCode}"`,
+            description: `Following the link output should show the following:\n${await (0, doc_dfg_1.printDfGraphForCode)(shell, sampleCode, { showCode: false })}`
+        }
+    ], { openOutput: false })}
+	
+Especially when you are just starting with flowR, we recommend to use the REPL to explore the output of the different steps.
+
+${(0, doc_structure_1.block)({
+        type: 'NOTE',
+        content: 'Maybe you are left with the question: What is tree-sitter doing differently? Expand the following to get more information!\n\n' + (0, doc_structure_1.details)('And what changes with tree-sitter?', `
+
+Essentially not much (from a user perspective, it does essentially everything and all differently under the hood)! Have a look at the [Engines](${doc_files_1.FlowrWikiBaseRef}/Engines) wiki page for more information on the differences between the engines.
+Below you can see the Repl commands for the tree-sitter engine (using ${(0, doc_cli_option_1.getCliLongOptionOf)('flowr', 'default-engine')} to set the engine to tree-sitter):
+
+${await (async () => {
+            const exec = new tree_sitter_executor_1.TreeSitterExecutor();
+            return await (0, doc_repl_1.documentReplSession)(exec, [{
+                    command: `:parse "${sampleCode}"`,
+                    description: `This shows the ASCII-Art representation of the parse-tree of the R code \`${sampleCode}\`, as it is provided by the ${(0, doc_types_1.shortLink)(tree_sitter_executor_1.TreeSitterExecutor.name, info)}. See the [Engines](${doc_files_1.FlowrWikiBaseRef}/Engines) wiki page for more information on the differences between the engines.`
+                },
+                {
+                    command: `:normalize* "${sampleCode}"`,
+                    description: `Following the link output should show the following:\n${await (0, doc_normalized_ast_1.printNormalizedAstForCode)(exec, sampleCode, { showCode: false })}`
+                },
+                {
+                    command: `:dataflow* "${sampleCode}"`,
+                    description: `Following the link output should show the following:\n${await (0, doc_dfg_1.printDfGraphForCode)(exec, sampleCode, { showCode: false })}`
+                }], { openOutput: false, args: '--default-engine tree-sitter' });
+        })()}
+`)
+    })}
+
+### Parsing
+
+The parsing step uses the ${(0, doc_types_1.shortLink)(shell_1.RShell.name, info)} to parse the input program (or, of course, the ${(0, doc_types_1.shortLink)(tree_sitter_executor_1.TreeSitterExecutor.name, info)} when using the [\`tree-sitter\` engine](${doc_files_1.FlowrWikiBaseRef}/Engines)).
+To speed up the process, we use the ${(0, doc_types_1.shortLink)(init_1.initCommand.name, info)} function to compile the parsing function and rely on a 
+custom serialization, which outputs the information in a CSV-like format.
+This means, that the ${(0, doc_cli_option_1.getReplCommand)('parse')} command actually kind-of lies to you, as it does pretty print the serialized version which looks more like the following (this uses the ${(0, doc_types_1.shortLink)(retriever_1.retrieveParseDataFromRCode.name, info)} function with the sample code \`${sampleCode}\`):
+
+${(0, doc_structure_1.details)(`Raw parse output for <code>${sampleCode}</code>`, `For the code \`${sampleCode}\`:\n\n` + (0, doc_code_1.codeBlock)('csv', await (0, retriever_1.retrieveParseDataFromRCode)((0, retriever_1.requestFromInput)(sampleCode), shell)))}
+
+Beautiful, right? I thought so too! In fact, the output is a little bit nicer, when we put it into a table-format and add the appropriate headers:
+
+<details open>
+<summary>Parse output in table format</summary>
+
+For the code \`${sampleCode}\`:
+
+| line-start | col-start | line-end | col-end | id | parent | token type | terminal | text |
+| ---------: | --------: | -------: | ------: | -: | -----: | ---------- | -------- | ---- |
+${await (0, retriever_1.retrieveParseDataFromRCode)((0, retriever_1.requestFromInput)(sampleCode), shell).then(data => JSON.parse('[' + data + ']').map(([line1, col1, line2, col2, id, parent, type, terminal, text]) => `| ${line1} | ${col1} | ${line2} | ${col2} | ${id} | ${parent} | \`${type}\` | ${terminal} | ${text} |`).join('\n'))}
+
+</details>
+
+In fact, this data is merely what R's [\`base::parse\`](https://stat.ethz.ch/R-manual/R-devel/library/base/html/parse.html) and [\`utils::getParseData\`](https://stat.ethz.ch/R-manual/R-devel/library/utils/html/getParseData.html) functions provide.
+We then use this data in the [normalization](#normalization) step to create a [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST).
+
+If you are interested in the raw token types that we may encounter, have a look at the ${(0, doc_types_1.shortLink)('RawRType', info)} enum.
+
+### Normalization
+
+The normalization function ${(0, doc_types_1.shortLink)(parser_2.normalize.name, info)} takes the output from the previous steps and uses the ${(0, doc_types_1.shortLink)(format_1.prepareParsedData.name, info)} and 
+${(0, doc_types_1.shortLink)(format_1.convertPreparedParsedData.name, info)} functions to first transform the serialized parsing output to an object. 
+Next, ${(0, doc_types_1.shortLink)(normalize_root_1.normalizeRootObjToAst.name, info)} transforms this object to a normalized AST and ${(0, doc_types_1.shortLink)(decorate_1.decorateAst.name, info)} adds additional information to the AST (like roles, ids, depth, etc.).
+While looking at the mermaid visualization of such an AST is nice and usually sufficient, looking at the objects themselves shows you the full range of information the AST provides (all encompassed within the ${(0, doc_types_1.shortLink)('RNode', info)} type).
+
+Let's have a look at the normalized AST for the sample code \`${sampleCode}\` (please refer to the [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST) wiki page for more information):
+
+${(0, doc_structure_1.details)('Normalized AST for <code>x <- 1; print(x)</code>', (0, doc_code_1.codeBlock)('json', JSON.stringify((await (0, default_pipelines_1.createNormalizePipeline)(shell, { request: (0, retriever_1.requestFromInput)(sampleCode) }).allRemainingSteps()).normalize.ast, json_1.jsonReplacer, 4)))}
+
+This is… a lot! We get the type from the ${(0, doc_types_1.shortLink)('RType', info)} enum, the lexeme, location information, an id, the children of the node, and their parents.
+While the [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST) wiki page provides you with information on how to interpret this data, we will focus on how we get it from the
+table provided by the [parsing](#parsing) step.
+
+There are two important functions: ${(0, doc_types_1.shortLink)(normalize_root_1.normalizeRootObjToAst.name, info)}, which operates on the parse-output already transformed into a tree-like structure,
+and ${(0, doc_types_1.shortLink)(decorate_1.decorateAst.name, info)}, which adds additional information to the AST.
+Both follow a [fold](https://en.wikipedia.org/wiki/Fold_(higher-order_function)) pattern.
+The fold is explicit for ${(0, doc_types_1.shortLink)(decorate_1.decorateAst.name, info)}, which directly relies on the ${(0, doc_types_1.shortLink)(stateful_fold_1.foldAstStateful.name, info)} function,
+while ${(0, doc_types_1.shortLink)(normalize_root_1.normalizeRootObjToAst.name, info)} uses the fold-idiom but deviates in cases in which (for example) we require more information on other nodes to know what it should be normalized too.
+
+#### Normalizing the Object
+
+We have a handler for everything. For example ${(0, doc_types_1.shortLink)(normalize_if_then_1.tryNormalizeIfThen.name, info)} or ${(0, doc_types_1.shortLink)(normalize_for_1.tryNormalizeFor.name, info)} to handle \`if(x) y\` or \`for(i in 1:10) x\` constructs.
+All of these handlers contain many sanity checks to be sure that we talk to an ${(0, doc_types_1.shortLink)('RShell', info)} which we can handle (as assumptions may break with newer versions).
+These functions contain the keyword \`try\` as they may fail. For example, whenever they notice late into normalization that they should actually be a different construct (R is great).
+For single nodes, we use ${(0, doc_types_1.shortLink)(normalize_single_node_1.normalizeSingleNode.name, info)} which contains a catch-all for some edge-cases in the R grammar.
+
+The output of just this pass is listed below (using the ${(0, doc_types_1.shortLink)(parser_2.normalizeButNotDecorated.name, info)} function):
+
+${(0, doc_structure_1.details)('Ast for <code>x <- 1; print(x)</code> after the first normalization', (0, doc_code_1.codeBlock)('json', JSON.stringify((0, parser_2.normalizeButNotDecorated)((await (0, default_pipelines_1.createParsePipeline)(shell, { request: (0, retriever_1.requestFromInput)(sampleCode) }).allRemainingSteps()).parse), json_1.jsonReplacer, 4)))}
+
+
+#### Decorating the AST
+
+The decoration is comparatively trivial. We take the AST throw it into the ${(0, doc_types_1.shortLink)(decorate_1.decorateAst.name, info)} function (which again, handles each normalized node type) and
+get:
+
+1. The AST with ids, roles, and depth information (see the [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST) wiki page for more information).
+2. A mapping of ids to nodes in the form of a ${(0, doc_types_1.shortLink)('AstIdMap', info)} object. This allows us to quickly access nodes by their id.
+
+The ids used for the AST generation are arbitrary (usually created by the ${(0, doc_types_1.shortLink)(decorate_1.deterministicCountingIdGenerator.name, info)}) function) but unique and intentionally
+separated from the ids used by the R&nbsp;parser. For one, this detaches us from the [Engine](${doc_files_1.FlowrWikiBaseRef}/Engines) used, and secondly, it allows for much easier
+extension of the AST (e.g., when R&nbsp;files use [\`base::source\`](https://stat.ethz.ch/R-manual/R-devel/library/base/html/source.html) to include other R&nbsp;files).
+All ids conform to the ${(0, doc_types_1.shortLink)('NodeId', info)} type.
+
+### Dataflow Graph Generation
+
+The core of the dataflow graph generation works as a "stateful [fold](https://en.wikipedia.org/wiki/Fold_(higher-order_function))", 
+which uses the tree-like structure of the AST to combine the dataflow information of the children, while tracking the currently active variables and control flow 
+information as a “backpack” (state).	
+We use the ${(0, doc_types_1.shortLink)(extractor_1.produceDataFlowGraph.name, info)} function as an entry point to the dataflow generation (the actual fold entry is in ${(0, doc_types_1.shortLink)(processor_1.processDataflowFor.name, info)}).
+The function is mainly backed by its ${(0, doc_types_1.shortLink)('processors', info)} object which maps each type in the normalized AST to an appropriate handler ("fold-function").
+
+To understand these handlers, let's start with the simplest one, ${(0, doc_types_1.shortLink)(process_uninteresting_leaf_1.processUninterestingLeaf.name, info)} signals that 
+we do not care about this node and just produce an empty dataflow information (using ${(0, doc_types_1.shortLink)(info_1.initializeCleanDataflowInformation.name, info)}). 
+Looking at the function showcases the general structure of a processor:
+
+${(0, doc_types_1.printHierarchy)({ program, info, root: 'processUninterestingLeaf', maxDepth: 2, openTop: true })}
+
+Every processor has the same shape. It takes the normalized node (see the [normalized AST](${doc_files_1.FlowrWikiBaseRef}/Normalized-AST) for more information),
+and a ${(0, doc_types_1.shortLink)('DataflowProcessorInformation', info)} object which, as some kind of "backpack" carries global information
+to every handler. 
+This information is to be used to create a ${(0, doc_types_1.shortLink)('DataflowInformation', info)}:
+
+${(0, doc_types_1.printHierarchy)({ program, info, root: 'DataflowInformation', maxDepth: 2 })}
+
+Essentially, these processors should use the dataflow information from their children combined with their own semantics
+to produce a new dataflow information to pass upwards in the fold. The ${(0, doc_types_1.shortLink)('DataflowInformation', info)} contains:
+
+* the ${(0, doc_types_1.shortLink)('DataflowGraph', info)} of the current subtree 
+* the currently active ${(0, doc_types_1.shortLink)('REnvironmentInformation', info)} as an abstraction of all active definitions linking to potential definition locations (see [Advanced R::Environments](https://adv-r.hadley.nz/environments.html))
+* control flow information in ${(0, doc_types_1.shortLink)('DataflowCfgInformation', info)} which is used to enrich the dataflow information with control flow information
+* and sets of currently ingoing (read), outgoing (write) and unknown ${(0, doc_types_1.shortLink)('IdentifierReference', info)}s.
+
+While all of them are essentially empty when processing an “uninteresting leaf”, handling a constant is slightly more interesting with ${(0, doc_types_1.shortLink)('processValue', info)}:
+
+${(0, doc_types_1.printHierarchy)({ program, info, root: 'processValue', maxDepth: 2, openTop: true })}
+
+Please note, that we add the [value vertex](${doc_files_1.FlowrWikiBaseRef}/Dataflow-Graph#value-vertex) to the newly created dataflow graph,
+which holds a reference to the constant. If you are confused with the use of the ${(0, doc_types_1.shortLink)('ParentInformation', info)} type, 
+this stems from the [AST decoration](#normalization) and signals that we have a decorated ${(0, doc_types_1.shortLink)('RNode', info)} (which may have additional information in \`OtherInfo\`).
+
+Yet again, this is not very interesting. When looking at the ${(0, doc_types_1.shortLink)('processors', info)} object you may be confused by
+many lines just mapping the node to the ${(0, doc_types_1.shortLink)('processAsNamedCall', info)} function.
+This is because during the dataflow analysis we actually "desugar" the AST, and treat syntax constructs like binary operators (e.g., \`x + y\`) as function calls (e.g. \`\` \`+\`(x, y) \`\`).
+We do this, because R does it the same way, and allows to even overwrite these operators (including \`if\`, \`<-\`, etc.) by their name.
+By treating them like R, as function calls, we get support for these overwrites for free, courtesy of flowR's call resolution.
+
+But where are all the interesting things handled then? 
+For that, we want to have a look at the built-in environment, which can be freely configured using flowR's [configuration system](${doc_files_1.FlowrWikiBaseRef}/Interface#configuring-flowr).
+FlowR's heart and soul resides in the ${(0, doc_types_1.shortLink)('DefaultBuiltinConfig', info)} object, which is used to configure the built-in environment
+by mapping function names to ${(0, doc_types_1.shortLink)('BuiltInProcessorMapper', info)} functions.
+There you can find functions like ${(0, doc_types_1.shortLink)(built_in_access_1.processAccess.name, info)} which handles the (subset) access to a variable, 
+or ${(0, doc_types_1.shortLink)(built_in_for_loop_1.processForLoop.name, info)} which handles the primitive for loop construct (whenever it is not overwritten).
+
+Just as an example, we want to have a look at the ${(0, doc_types_1.shortLink)(built_in_repeat_loop_1.processRepeatLoop.name, info)} function, as it is one of the simplest built-in processors
+we have:
+
+${(0, doc_types_1.printHierarchy)({ program, info, root: 'processRepeatLoop', maxDepth: 2, openTop: true })}
+
+Similar to any other built-in processor, we get the name of the function call which caused us to land here,
+as well as the passed arguments. The \`rootId\` refers to what caused the call to happen (and is usually just the function call),
+while \`data\` is our good old backpack, carrying all the information we need to produce a dataflow graph.
+
+After a couple of common sanity checks at the beginning which we use to check whether the repeat loop is used in a way that we expect,
+we start by issuing the fold continuation by processing its arguments. Given we expect \`repeat <body>\`, we expect only a single argument.
+During the processing we make sure to stitch in the correct control dependencies, adding the repeat loop to the mix.
+For just the repeat loop the stitching is actually not necessary, but this way the handling is consistent for all looping constructs.
+
+Afterward, we take the \`processedArguments\`, perform another round of sanity checks and then use two special functions to apply the
+semantic effects of the repeat loop. We first use one of flowR's linkers to
+${(0, doc_types_1.shortLink)(linker_1.linkCircularRedefinitionsWithinALoop.name, info)} and then retrieve the active exit points with ${(0, doc_types_1.shortLink)(info_1.filterOutLoopExitPoints.name, info)}.
+
+Feel free to have a look around and explore the other handlers for now. Each of them uses the results of its children alongside the active backpack 
+to produce a new dataflow information.
+
+## Beyond the Dataflow Graph
+
+Given the [dataflow graph](${doc_files_1.FlowrWikiBaseRef}/Dataflow-Graph), you can do a lot more!
+You can issue [queries](${doc_files_1.FlowrWikiBaseRef}/Query-API) to explore the graph, [search](${doc_files_1.FlowrWikiBaseRef}/Search-API) for specific elements, or, for example, request a [static backward slice](#static-backward-slicing).
+Of course, all of these endeavors work not just with the ${(0, doc_types_1.shortLink)(shell_1.RShell.name, info)} but also with the [\`tree-sitter\` engine](${doc_files_1.FlowrWikiBaseRef}/Engines). 
+
+### Static Backward Slicing
+
+The slicing is available as an extra step as you can see by inspecting he ${(0, doc_types_1.shortLink)('DEFAULT_SLICING_PIPELINE', info)}.
+Besides ${(0, doc_types_1.shortLink)('STATIC_SLICE', info)} it contains a ${(0, doc_types_1.shortLink)('NAIVE_RECONSTRUCT', info)} to print the slice as (executable) R code.
+
+Your main point of interesting here is the ${(0, doc_types_1.shortLink)(static_slicer_1.staticSlicing.name, info)} function which relies on a modified
+breadth-first search to collect all nodes which are part of the slice. 
+For more information on how the slicing works, please refer to the [tool demonstration (Section 3.2)](https://doi.org/10.1145/3691620.3695359),
+or the [original master's thesis (Chapter 4)](https://doi.org/10.18725/OPARU-50107).
+
+You can explore the slicing using the REPL with the ${(0, doc_cli_option_1.getReplCommand)('slicer')} command:
+
+${await (0, doc_repl_1.documentReplSession)(shell, [{
+            command: ':slicer test/testfiles/example.R --criterion "12@product"',
+            description: 'Slice for the example file for the variable "prod" in line 12.'
+        }], { openOutput: true })}
+
+## Helpful Things
+
+### Getting flowR to Talk
+
+When using flowR from the CLI, you can use the ${(0, doc_cli_option_1.getCliLongOptionOf)('flowr', 'verbose')} option to get more information about what flowR is doing.
+While coding, however, you can use the ${log_1.setMinLevelOfAllLogs.name} function to set the minimum level of logs to be displayed (this works with the ${(0, doc_types_1.shortLink)(log_2.FlowrLogger.name, info)} abstraction).
+In general, you can configure the levels of individual logs, such as the general \`log\` (obtained with ${(0, doc_types_1.shortLink)('getActiveLog', info)}) or the ${(0, doc_types_1.shortLink)('parseLog', info)}.
+Please note that flowR makes no guarantees that log outputs are persistent across versions, and it is up to the implementors to provide sensible logging.
+If you are an implementor and want to add logging, please make sure that there are no larger runtime impliciations when logging is disabled. 
+Have a look at the ${(0, doc_types_1.shortLink)(log_2.expensiveTrace.name, info)} function for example, which uses a function to generate the log message only when the log level is reached.
+
+`;
+}
+/** if we run this script, we want a Markdown representation of the capabilities */
+if (require.main === module) {
+    void tree_sitter_executor_1.TreeSitterExecutor.initTreeSitter().then(() => {
+        (0, log_1.setMinLevelOfAllLogs)(6 /* LogLevel.Fatal */);
+        const shell = new shell_1.RShell();
+        void getText(shell).then(str => {
+            console.log(str);
+        }).finally(() => {
+            shell.close();
+        });
+    });
+}
+//# sourceMappingURL=print-core-wiki.js.map