@eagleoutice/flowr 2.1.11 → 2.2.0

package/core/steps/pipeline/default-pipelines.js

@@ -1,19 +1,63 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_PARSE_PIPELINE = exports.DEFAULT_NORMALIZE_PIPELINE = exports.DEFAULT_DATAFLOW_PIPELINE = exports.DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE = exports.DEFAULT_SLICE_AND_RECONSTRUCT_PIPELINE = exports.DEFAULT_SLICING_PIPELINE = void 0;
-/**
- * Contains the default pipeline for working with flowr
- */
+exports.TREE_SITTER_PARSE_PIPELINE = exports.DEFAULT_PARSE_PIPELINE = exports.TREE_SITTER_NORMALIZE_PIPELINE = exports.DEFAULT_NORMALIZE_PIPELINE = exports.TREE_SITTER_DATAFLOW_PIPELINE = exports.DEFAULT_DATAFLOW_PIPELINE = exports.TREE_SITTER_SLICE_WITHOUT_RECONSTRUCT_PIPELINE = exports.TREE_SITTER_SLICE_AND_RECONSTRUCT_PIPELINE = exports.TREE_SITTER_SLICING_PIPELINE = exports.DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE = exports.DEFAULT_SLICE_AND_RECONSTRUCT_PIPELINE = exports.DEFAULT_SLICING_PIPELINE = void 0;
+exports.createParsePipeline = createParsePipeline;
+exports.createSlicePipeline = createSlicePipeline;
+exports.createNormalizePipeline = createNormalizePipeline;
+exports.createDataflowPipeline = createDataflowPipeline;
 const pipeline_1 = require("./pipeline");
 const _00_parse_1 = require("../all/core/00-parse");
 const _10_normalize_1 = require("../all/core/10-normalize");
 const _20_dataflow_1 = require("../all/core/20-dataflow");
 const _00_slice_1 = require("../all/static-slicing/00-slice");
 const _10_reconstruct_1 = require("../all/static-slicing/10-reconstruct");
+const _01_parse_tree_sitter_1 = require("../all/core/01-parse-tree-sitter");
+const _11_normalize_tree_sitter_1 = require("../all/core/11-normalize-tree-sitter");
+const pipeline_executor_1 = require("../../pipeline-executor");
 exports.DEFAULT_SLICING_PIPELINE = (0, pipeline_1.createPipeline)(_00_parse_1.PARSE_WITH_R_SHELL_STEP, _10_normalize_1.NORMALIZE, _20_dataflow_1.STATIC_DATAFLOW, _00_slice_1.STATIC_SLICE, _10_reconstruct_1.NAIVE_RECONSTRUCT);
 exports.DEFAULT_SLICE_AND_RECONSTRUCT_PIPELINE = exports.DEFAULT_SLICING_PIPELINE;
 exports.DEFAULT_SLICE_WITHOUT_RECONSTRUCT_PIPELINE = (0, pipeline_1.createPipeline)(_00_parse_1.PARSE_WITH_R_SHELL_STEP, _10_normalize_1.NORMALIZE, _20_dataflow_1.STATIC_DATAFLOW, _00_slice_1.STATIC_SLICE);
+exports.TREE_SITTER_SLICING_PIPELINE = (0, pipeline_1.createPipeline)(_01_parse_tree_sitter_1.PARSE_WITH_TREE_SITTER_STEP, _11_normalize_tree_sitter_1.NORMALIZE_TREE_SITTER, _20_dataflow_1.STATIC_DATAFLOW, _00_slice_1.STATIC_SLICE, _10_reconstruct_1.NAIVE_RECONSTRUCT);
+exports.TREE_SITTER_SLICE_AND_RECONSTRUCT_PIPELINE = exports.TREE_SITTER_SLICING_PIPELINE;
+exports.TREE_SITTER_SLICE_WITHOUT_RECONSTRUCT_PIPELINE = (0, pipeline_1.createPipeline)(_01_parse_tree_sitter_1.PARSE_WITH_TREE_SITTER_STEP, _11_normalize_tree_sitter_1.NORMALIZE_TREE_SITTER, _20_dataflow_1.STATIC_DATAFLOW, _00_slice_1.STATIC_SLICE);
+/**
+ * The default pipeline for working with flowr, including the dataflow step,
+ * see the {@link DEFAULT_NORMALIZE_PIPELINE} for the pipeline without the dataflow step,
+ * and the {@link DEFAULT_SLICE_AND_RECONSTRUCT_PIPELINE} for the pipeline with slicing and reconstructing steps
+ */
 exports.DEFAULT_DATAFLOW_PIPELINE = (0, pipeline_1.createPipeline)(_00_parse_1.PARSE_WITH_R_SHELL_STEP, _10_normalize_1.NORMALIZE, _20_dataflow_1.STATIC_DATAFLOW);
+exports.TREE_SITTER_DATAFLOW_PIPELINE = (0, pipeline_1.createPipeline)(_01_parse_tree_sitter_1.PARSE_WITH_TREE_SITTER_STEP, _11_normalize_tree_sitter_1.NORMALIZE_TREE_SITTER, _20_dataflow_1.STATIC_DATAFLOW);
+/** The pipeline to use when you want to parse and normalize your R file, see {@link DEFAULT_DATAFLOW_PIPELINE} for the additional `dataflow` step */
 exports.DEFAULT_NORMALIZE_PIPELINE = (0, pipeline_1.createPipeline)(_00_parse_1.PARSE_WITH_R_SHELL_STEP, _10_normalize_1.NORMALIZE);
+exports.TREE_SITTER_NORMALIZE_PIPELINE = (0, pipeline_1.createPipeline)(_01_parse_tree_sitter_1.PARSE_WITH_TREE_SITTER_STEP, _11_normalize_tree_sitter_1.NORMALIZE_TREE_SITTER);
 exports.DEFAULT_PARSE_PIPELINE = (0, pipeline_1.createPipeline)(_00_parse_1.PARSE_WITH_R_SHELL_STEP);
+exports.TREE_SITTER_PARSE_PIPELINE = (0, pipeline_1.createPipeline)(_01_parse_tree_sitter_1.PARSE_WITH_TREE_SITTER_STEP);
+function createParsePipeline(parser, inputs) {
+    const base = parser.name === 'tree-sitter' ? exports.TREE_SITTER_PARSE_PIPELINE : exports.DEFAULT_PARSE_PIPELINE;
+    return new pipeline_executor_1.PipelineExecutor(base, {
+        parser: parser,
+        ...inputs
+    });
+}
+function createSlicePipeline(parser, inputs) {
+    const base = parser.name === 'tree-sitter' ? exports.TREE_SITTER_SLICING_PIPELINE : exports.DEFAULT_SLICING_PIPELINE;
+    return new pipeline_executor_1.PipelineExecutor(base, {
+        parser: parser,
+        ...inputs
+    });
+}
+function createNormalizePipeline(parser, inputs) {
+    const base = parser.name === 'tree-sitter' ? exports.TREE_SITTER_NORMALIZE_PIPELINE : exports.DEFAULT_NORMALIZE_PIPELINE;
+    return new pipeline_executor_1.PipelineExecutor(base, {
+        parser: parser,
+        ...inputs
+    });
+}
+function createDataflowPipeline(parser, inputs) {
+    const base = parser.name === 'tree-sitter' ? exports.TREE_SITTER_DATAFLOW_PIPELINE : exports.DEFAULT_DATAFLOW_PIPELINE;
+    return new pipeline_executor_1.PipelineExecutor(base, {
+        parser: parser,
+        ...inputs
+    });
+}
 //# sourceMappingURL=default-pipelines.js.map

package/dataflow/extractor.d.ts

@@ -2,5 +2,6 @@ import type { DataflowInformation } from './info';
 import type { DataflowProcessors } from './processor';
 import type { NormalizedAst, ParentInformation } from '../r-bridge/lang-4.x/ast/model/processing/decorate';
 import type { RParseRequests } from '../r-bridge/retriever';
+import type { KnownParserType, Parser } from '../r-bridge/parser';
 export declare const processors: DataflowProcessors<ParentInformation>;
-export declare function produceDataFlowGraph<OtherInfo>(request: RParseRequests, ast: NormalizedAst<OtherInfo & ParentInformation>): DataflowInformation;
+export declare function produceDataFlowGraph<OtherInfo>(parser: Parser<KnownParserType>, request: RParseRequests, ast: NormalizedAst<OtherInfo & ParentInformation>): DataflowInformation;

package/dataflow/extractor.js

@@ -68,7 +68,7 @@ function resolveLinkToSideEffects(ast, graph) {
         }
     }
 }
-function produceDataFlowGraph(request, ast) {
+function produceDataFlowGraph(parser, request, ast) {
     const multifile = Array.isArray(request);
     let firstRequest;
     if (multifile) {
@@ -78,6 +78,7 @@ function produceDataFlowGraph(request, ast) {
         firstRequest = request;
     }
     const dfData = {
+        parser: parser,
         completeAst: ast,
         environment: (0, environment_1.initializeCleanEnvironments)(),
         processors: exports.processors,

package/dataflow/graph/vertex.d.ts

@@ -10,6 +10,10 @@ export declare enum VertexType {
     VariableDefinition = "variable-definition",
     FunctionDefinition = "function-definition"
 }
+export declare const ValidVertexTypes: Set<string>;
+export declare const ValidVertexTypeReverse: {
+    [k: string]: string;
+};
 /**
  * A single index of a container, which is not a container itself.
  *

package/dataflow/graph/vertex.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VertexType = void 0;
+exports.ValidVertexTypeReverse = exports.ValidVertexTypes = exports.VertexType = void 0;
 exports.isParentContainerIndex = isParentContainerIndex;
 exports.isValueVertex = isValueVertex;
 exports.isUseVertex = isUseVertex;
@@ -15,6 +15,8 @@ var VertexType;
     VertexType["VariableDefinition"] = "variable-definition";
     VertexType["FunctionDefinition"] = "function-definition";
 })(VertexType || (exports.VertexType = VertexType = {}));
+exports.ValidVertexTypes = new Set(Object.values(VertexType));
+exports.ValidVertexTypeReverse = Object.fromEntries(Object.entries(VertexType).map(([k, v]) => [v, k]));
 function isParentContainerIndex(index) {
     return 'subIndices' in index;
 }

package/dataflow/internal/process/functions/call/built-in/built-in-source.js

@@ -7,11 +7,9 @@ exports.setSourceProvider = setSourceProvider;
 exports.processSourceCall = processSourceCall;
 exports.sourceRequest = sourceRequest;
 exports.standaloneSourceFile = standaloneSourceFile;
-const shell_executor_1 = require("../../../../../../r-bridge/shell-executor");
 const processor_1 = require("../../../../../processor");
 const info_1 = require("../../../../../info");
 const config_1 = require("../../../../../../config");
-const parser_1 = require("../../../../../../r-bridge/lang-4.x/ast/parser/json/parser");
 const known_call_handling_1 = require("../known-call-handling");
 const retriever_1 = require("../../../../../../r-bridge/retriever");
 const decorate_1 = require("../../../../../../r-bridge/lang-4.x/ast/model/processing/decorate");
@@ -21,6 +19,8 @@ const type_1 = require("../../../../../../r-bridge/lang-4.x/ast/model/type");
 const overwrite_1 = require("../../../../../environments/overwrite");
 const log_1 = require("../../../../../../util/log");
 const fs_1 = __importDefault(require("fs"));
+const parser_1 = require("../../../../../../r-bridge/lang-4.x/ast/parser/json/parser");
+const shell_executor_1 = require("../../../../../../r-bridge/shell-executor");
 let sourceProvider = (0, retriever_1.requestProviderFromFile)();
 function setSourceProvider(provider) {
     sourceProvider = provider;
@@ -61,13 +61,14 @@ function sourceRequest(rootId, request, data, information, getId) {
             return information;
         }
     }
-    const executor = new shell_executor_1.RShellExecutor();
     // parse, normalize and dataflow the sourced file
     let normalized;
     let dataflow;
     try {
-        const parsed = (0, retriever_1.retrieveParseDataFromRCode)(request, executor);
-        normalized = (0, parser_1.normalize)({ parsed }, getId, request.request === 'file' ? request.content : undefined);
+        const file = request.request === 'file' ? request.content : undefined;
+        const parsed = (!data.parser.async ? data.parser : new shell_executor_1.RShellExecutor()).parse(request);
+        normalized = (typeof parsed !== 'string' ?
+            (0, parser_1.normalizeTreeSitter)({ parsed }, getId, file) : (0, parser_1.normalize)({ parsed }, getId, file));
         dataflow = (0, processor_1.processDataflowFor)(normalized.ast, {
             ...data,
             currentRequest: request,

package/dataflow/processor.d.ts

@@ -6,7 +6,9 @@ import type { NormalizedAst, ParentInformation, RNodeWithParent } from '../r-bri
 import type { REnvironmentInformation } from './environments/environment';
 import type { RParseRequest } from '../r-bridge/retriever';
 import type { RNode } from '../r-bridge/lang-4.x/ast/model/model';
+import type { KnownParserType, Parser } from '../r-bridge/parser';
 export interface DataflowProcessorInformation<OtherInfo> {
+    readonly parser: Parser<KnownParserType>;
     /**
     * Initial and frozen ast-information
     */

package/documentation/doc-util/doc-auto-gen.js

@@ -2,8 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.autoGenHeader = autoGenHeader;
 const version_1 = require("../../util/version");
+const doc_files_1 = require("./doc-files");
 function autoGenHeader({ rVersion, filename, purpose, currentDateAndTime = new Date().toISOString().replace('T', ', ').replace(/\.\d+Z$/, ' UTC') }) {
     const shortenFilename = filename.replace(/^.*src\//, 'src/');
-    return `_This document was generated from '${shortenFilename}' on ${currentDateAndTime} presenting an overview of flowR's ${purpose} (v${(0, version_1.flowrVersion)().format()}${rVersion ? ', using R v' + rVersion : ''})._`;
+    return `_This document was generated from '${(0, doc_files_1.linkFlowRSourceFile)(shortenFilename)}' on ${currentDateAndTime} presenting an overview of flowR's ${purpose} (v${(0, version_1.flowrVersion)().format()}${rVersion ? ', using R v' + rVersion : ''}). Please do not edit this file/wiki page directly._`;
 }
 //# sourceMappingURL=doc-auto-gen.js.map

package/documentation/doc-util/doc-cfg.js

@@ -7,7 +7,7 @@ const default_pipelines_1 = require("../../core/steps/pipeline/default-pipelines
 const retriever_1 = require("../../r-bridge/retriever");
 async function getCfg(shell, code) {
     const steps = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_NORMALIZE_PIPELINE, {
-        shell,
+        parser: shell,
         request: (0, retriever_1.requestFromInput)(code)
     }).allRemainingSteps();
     return {

package/documentation/doc-util/doc-dfg.js

@@ -35,14 +35,14 @@ function formatSideEffect(ef) {
 async function printDfGraphForCode(shell, code, { mark, showCode = true, codeOpen = false, exposeResult, switchCodeAndGraph = false } = {}) {
     const now = performance.now();
     const result = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_DATAFLOW_PIPELINE, {
-        shell,
+        parser: shell,
         request: (0, retriever_1.requestFromInput)(code)
     }).allRemainingSteps();
     const duration = performance.now() - now;
     if (switchCodeAndGraph) {
         (0, assert_1.guard)(showCode, 'can not switch code and graph if code is not shown');
     }
-    const metaInfo = `The analysis required _${(0, time_1.printAsMs)(duration)}_ (incl. parse and normalize) within the generation environment.`;
+    const metaInfo = `The analysis required _${(0, time_1.printAsMs)(duration)}_ (including parse and normalize) within the generation environment.`;
     const dfGraph = printDfGraph(result.dataflow.graph, mark);
     let resultText = '\n\n';
     if (showCode) {
@@ -74,7 +74,7 @@ ${switchCodeAndGraph ? dfGraph : codeText}
 async function verifyExpectedSubgraph(shell, code, expectedSubgraph) {
     /* we verify that we get what we want first! */
     const info = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_DATAFLOW_PIPELINE, {
-        shell,
+        parser: shell,
         request: (0, retriever_1.requestFromInput)(code),
         getId: (0, decorate_1.deterministicCountingIdGenerator)(0)
     }).allRemainingSteps();

package/documentation/doc-util/doc-files.d.ts

@@ -7,3 +7,4 @@ export declare const FlowrDockerRef = "https://hub.docker.com/r/eagleoutice/flow
 export declare const FlowrCodecovRef = "https://app.codecov.io/gh/flowr-analysis/flowr";
 export declare function getFilePathMd(path: string): string;
 export declare function getFileContentFromRoot(path: string): string;
+export declare function linkFlowRSourceFile(path: string): string;

package/documentation/doc-util/doc-files.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FlowrCodecovRef = exports.FlowrDockerRef = exports.FlowrNpmRef = exports.FlowrWikiBaseRef = exports.RemoteFlowrFilePathBaseRef = exports.FlowrSiteBaseRef = exports.FlowrGithubBaseRef = void 0;
 exports.getFilePathMd = getFilePathMd;
 exports.getFileContentFromRoot = getFileContentFromRoot;
+exports.linkFlowRSourceFile = linkFlowRSourceFile;
 const fs_1 = __importDefault(require("fs"));
 exports.FlowrGithubBaseRef = 'https://github.com/flowr-analysis';
 exports.FlowrSiteBaseRef = 'https://flowr-analysis.github.io/flowr';
@@ -27,4 +28,7 @@ function getFileContentFromRoot(path) {
     const fullpath = require.resolve('../../../' + path);
     return fs_1.default.readFileSync(fullpath, 'utf-8');
 }
+function linkFlowRSourceFile(path) {
+    return `[${path}](${exports.RemoteFlowrFilePathBaseRef}/${path})`;
+}
 //# sourceMappingURL=doc-files.js.map

package/documentation/doc-util/doc-normalized-ast.js

@@ -22,7 +22,7 @@ ${(0, ast_1.normalizedAstToMermaid)(ast, prefix)}
 async function printNormalizedAstForCode(shell, code, { showCode = true, prefix = 'flowchart TD\n' } = {}) {
     const now = performance.now();
     const result = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_NORMALIZE_PIPELINE, {
-        shell,
+        parser: shell,
         request: (0, retriever_1.requestFromInput)(code)
     }).allRemainingSteps();
     const duration = performance.now() - now;
@@ -55,8 +55,7 @@ ${(0, ast_1.normalizedAstToMermaid)(result.normalize.ast, prefix)}
 /** returns resolved expected df graph */
 async function verifyExpectedSubgraph(shell, code, expectedSubgraph) {
     /* we verify that we get what we want first! */
-    const info = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_DATAFLOW_PIPELINE, {
-        shell,
+    const info = await (0, default_pipelines_1.createDataflowPipeline)(shell, {
         request: (0, retriever_1.requestFromInput)(code),
         getId: (0, decorate_1.deterministicCountingIdGenerator)(0)
     }).allRemainingSteps();

package/documentation/doc-util/doc-query.js

@@ -20,10 +20,10 @@ const query_print_1 = require("../../queries/query-print");
 async function showQuery(shell, code, queries, { showCode, collapseResult, collapseQuery } = {}) {
     const now = performance.now();
     const analysis = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_DATAFLOW_PIPELINE, {
-        shell,
+        parser: shell,
         request: (0, retriever_1.requestFromInput)(code)
     }).allRemainingSteps();
-    const results = (0, query_1.executeQueries)({ graph: analysis.dataflow.graph, ast: analysis.normalize }, queries);
+    const results = (0, query_1.executeQueries)({ dataflow: analysis.dataflow, ast: analysis.normalize }, queries);
     const duration = performance.now() - now;
     const metaInfo = `
 The analysis required _${(0, time_1.printAsMs)(duration)}_ (including parsing and normalization and the query) within the generation environment.

package/documentation/doc-util/doc-search.d.ts

@@ -0,0 +1,25 @@
+import type { RShell } from '../../r-bridge/shell';
+import type { SupportedQueryTypes } from '../../queries/query';
+import type { SupportedVirtualQueryTypes } from '../../queries/virtual-query/virtual-queries';
+import type { FlowrSearchLike } from '../../search/flowr-search-builder';
+export interface ShowSearchOptions {
+    readonly showCode?: boolean;
+    readonly collapseResult?: boolean;
+}
+export declare function showSearch(shell: RShell, code: string, search: FlowrSearchLike, { collapseResult }?: ShowSearchOptions): Promise<string>;
+export interface QueryDocumentation {
+    readonly name: string;
+    readonly type: 'virtual' | 'active';
+    readonly shortDescription: string;
+    readonly functionName: string;
+    readonly functionFile: string;
+    readonly buildExplanation: (shell: RShell) => Promise<string>;
+}
+export declare const RegisteredQueries: {
+    active: Map<string, QueryDocumentation>;
+    virtual: Map<string, QueryDocumentation>;
+};
+export declare function registerQueryDocumentation(query: SupportedQueryTypes | SupportedVirtualQueryTypes, doc: QueryDocumentation): void;
+export declare function linkToQueryOfName(id: SupportedQueryTypes | SupportedVirtualQueryTypes): string;
+export declare function tocForQueryType(type: 'active' | 'virtual'): string;
+export declare function explainQueries(shell: RShell, type: 'active' | 'virtual'): Promise<string>;

package/documentation/doc-util/doc-search.js

@@ -0,0 +1,121 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RegisteredQueries = void 0;
+exports.showSearch = showSearch;
+exports.registerQueryDocumentation = registerQueryDocumentation;
+exports.linkToQueryOfName = linkToQueryOfName;
+exports.tocForQueryType = tocForQueryType;
+exports.explainQueries = explainQueries;
+const pipeline_executor_1 = require("../../core/pipeline-executor");
+const default_pipelines_1 = require("../../core/steps/pipeline/default-pipelines");
+const retriever_1 = require("../../r-bridge/retriever");
+const doc_files_1 = require("./doc-files");
+const doc_dfg_1 = require("./doc-dfg");
+const doc_code_1 = require("./doc-code");
+const time_1 = require("../../util/time");
+const flowr_search_executor_1 = require("../../search/flowr-search-executor");
+const flowr_search_printer_1 = require("../../search/flowr-search-printer");
+const node_id_1 = require("../../r-bridge/lang-4.x/ast/model/processing/node-id");
+const dfg_1 = require("../../util/mermaid/dfg");
+async function showSearch(shell, code, search, { collapseResult = true } = {}) {
+    const now = performance.now();
+    const analysis = await new pipeline_executor_1.PipelineExecutor(default_pipelines_1.DEFAULT_DATAFLOW_PIPELINE, {
+        parser: shell,
+        request: (0, retriever_1.requestFromInput)(code)
+    }).allRemainingSteps();
+    const result = (0, flowr_search_executor_1.runSearch)(search, analysis);
+    const duration = performance.now() - now;
+    const metaInfo = `
+The search required _${(0, time_1.printAsMs)(duration)}_ (including parsing and normalization and the query) within the generation environment.
+	`.trim();
+    return `
+
+${(0, doc_code_1.codeBlock)('ts', (0, flowr_search_printer_1.flowrSearchToCode)(search))}
+
+<details style="color:gray"> <summary>Search Visualization</summary>
+
+${(0, doc_code_1.codeBlock)('mermaid', (0, flowr_search_printer_1.flowrSearchToMermaid)(search))}
+
+In the code:
+
+${(0, doc_code_1.codeBlock)('r', code)}
+
+<details style="color:gray"> <summary>JSON Representation</summary>
+
+${(0, doc_code_1.codeBlock)('json', JSON.stringify(search, null, 2))}
+
+</details>
+
+</details>
+
+
+${collapseResult ? ' <details> <summary style="color:gray">Show Results</summary>' : ''}
+
+The query returns the following vetices (all references to \`x\` in the code):
+${result.map(({ node }) => `<b>${node.info.id} ('${(0, node_id_1.recoverContent)(node.info.id, analysis.dataflow.graph)}')</b> at L${(0, dfg_1.formatRange)(node.location)}`).join(', ')}
+
+${metaInfo}	
+
+The returned results are highlighted thick and blue within the dataflow graph:
+
+${await (0, doc_dfg_1.printDfGraphForCode)(shell, code, { showCode: false, switchCodeAndGraph: false, mark: new Set(result.map(({ node }) => node.info.id)) })}
+
+
+${collapseResult ? '</details>' : ''}
+
+	`;
+}
+exports.RegisteredQueries = {
+    'active': new Map(),
+    'virtual': new Map()
+};
+function registerQueryDocumentation(query, doc) {
+    const map = exports.RegisteredQueries[doc.type];
+    if (map.has(query)) {
+        throw new Error(`Query ${query} already registered`);
+    }
+    map.set(query, doc);
+}
+function linkify(name) {
+    return name.toLowerCase().replace(/ /g, '-');
+}
+function linkToQueryOfName(id) {
+    const query = exports.RegisteredQueries.active.get(id) ?? exports.RegisteredQueries.virtual.get(id);
+    if (!query) {
+        throw new Error(`Query ${id} not found`);
+    }
+    return `[${query.name}](#${linkify(query.name)})`;
+}
+function tocForQueryType(type) {
+    const queries = [...exports.RegisteredQueries[type].entries()].sort(([, { name: a }], [, { name: b }]) => a.localeCompare(b));
+    const result = [];
+    for (const [id, { name, shortDescription }] of queries) {
+        result.push(`1. [${name}](#${linkify(name)}) (\`${id}\`):\\\n    ${shortDescription}`);
+    }
+    return result.join('\n');
+}
+async function explainQuery(shell, { name, functionName, functionFile, buildExplanation }) {
+    return `
+### ${name}
+
+${await buildExplanation(shell)}
+
+<details> 
+
+<summary style="color:gray">Implementation Details</summary>
+
+Responsible for the execution of the ${name} query is \`${functionName}\` in ${(0, doc_files_1.getFilePathMd)(functionFile)}.
+
+</details>	
+
+`;
+}
+async function explainQueries(shell, type) {
+    const queries = [...exports.RegisteredQueries[type].entries()].sort(([, { name: a }], [, { name: b }]) => a.localeCompare(b));
+    const result = [];
+    for (const [, doc] of queries) {
+        result.push(await explainQuery(shell, doc));
+    }
+    return result.join(`\n${'-'.repeat(5)}\n\n`);
+}
+//# sourceMappingURL=doc-search.js.map

package/documentation/doc-util/doc-types.d.ts

@@ -2,7 +2,7 @@ import ts from 'typescript';
 export interface TypeElementInSource {
     name: string;
     node: ts.Node;
-    kind: 'interface' | 'type' | 'enum' | 'class';
+    kind: 'interface' | 'type' | 'enum' | 'class' | 'variable';
     extends: string[];
     generics: string[];
     filePath: string;
@@ -25,7 +25,7 @@ export interface MermaidTypeReport {
     program: ts.Program;
 }
 export declare function getTypesFromFolderAsMermaid(options: GetTypesAsMermaidOption): MermaidTypeReport;
-export declare function implSnippet(node: TypeElementInSource | undefined, program: ts.Program, nesting?: number): string;
+export declare function implSnippet(node: TypeElementInSource | undefined, program: ts.Program, showName?: boolean, nesting?: number): string;
 export interface PrintHierarchyArguments {
     readonly program: ts.Program;
     readonly hierarchy: TypeElementInSource[];
@@ -36,3 +36,11 @@ export interface PrintHierarchyArguments {
 }
 export declare const mermaidHide: string[];
 export declare function printHierarchy({ program, hierarchy, root, collapseFromNesting, initialNesting, maxDepth }: PrintHierarchyArguments): string;
+/**
+ * Create a short link to a type in the documentation
+ * @param name      - The name of the type, e.g. `MyType`, may include a container, e.g. `MyContainer::MyType` (this works with function nestings too)
+ * @param hierarchy - The hierarchy of types to search in
+ * @param codeStyle - Whether to use code style for the link
+ */
+export declare function shortLink(name: string, hierarchy: TypeElementInSource[], codeStyle?: boolean): string;
+export declare function getDocumentationForType(name: string, hierarchy: TypeElementInSource[]): string;

package/documentation/doc-util/doc-types.js

@@ -10,6 +10,8 @@ exports.getTypePathLink = getTypePathLink;
 exports.getTypesFromFolderAsMermaid = getTypesFromFolderAsMermaid;
 exports.implSnippet = implSnippet;
 exports.printHierarchy = printHierarchy;
+exports.shortLink = shortLink;
+exports.getDocumentationForType = getDocumentationForType;
 const typescript_1 = __importDefault(require("typescript"));
 const assert_1 = require("../../util/assert");
 const doc_files_1 = require("./doc-files");
@@ -18,6 +20,7 @@ const path_1 = __importDefault(require("path"));
 const mermaid_1 = require("../../util/mermaid/mermaid");
 const doc_code_1 = require("./doc-code");
 const doc_structure_1 = require("./doc-structure");
+const html_hover_over_1 = require("../../util/html-hover-over");
 function getSourceFiles(fileNames) {
     try {
         const program = typescript_1.default.createProgram(fileNames, { target: typescript_1.default.ScriptTarget.ESNext });
@@ -171,6 +174,42 @@ function collectHierarchyInformation(sourceFiles, options) {
                 }),
             });
         }
+        else if (typescript_1.default.isVariableDeclaration(node) || typescript_1.default.isExportDeclaration(node) || typescript_1.default.isExportAssignment(node) || typescript_1.default.isDeclarationStatement(node)) {
+            const name = node.name?.getText(sourceFile) ?? '';
+            const comments = getTextualComments(node);
+            hierarchyList.push({
+                name: dropGenericsFromType(name),
+                node,
+                kind: 'variable',
+                extends: [],
+                comments,
+                generics: [],
+                filePath: sourceFile.fileName,
+                lineNumber: getStartLine(node, sourceFile),
+            });
+        }
+        else if (typescript_1.default.isPropertyAssignment(node) || typescript_1.default.isPropertyDeclaration(node) || typescript_1.default.isPropertySignature(node)
+            || typescript_1.default.isMethodDeclaration(node) || typescript_1.default.isMethodSignature(node) || typescript_1.default.isFunctionDeclaration(node)) {
+            const name = node.name?.getText(sourceFile) ?? '';
+            // get the name of the object/enclosing type
+            let parent = node.parent;
+            while (typeof parent === 'object' && parent !== undefined && !('name' in parent)) {
+                parent = parent.parent;
+            }
+            if (typeof parent === 'object' && 'name' in parent) {
+                const comments = getTextualComments(node);
+                hierarchyList.push({
+                    name: dropGenericsFromType(name),
+                    node,
+                    kind: 'variable',
+                    extends: [parent.name?.getText(sourceFile) ?? ''],
+                    comments,
+                    generics: [],
+                    filePath: sourceFile.fileName,
+                    lineNumber: getStartLine(node, sourceFile),
+                });
+            }
+        }
         typescript_1.default.forEachChild(node, child => visit(child, sourceFile));
     };
     sourceFiles.forEach(sourceFile => {
@@ -266,7 +305,7 @@ function getTypesFromFolderAsMermaid(options) {
     }
     return getTypesFromFileAsMermaid(files, options);
 }
-function implSnippet(node, program, nesting = 0) {
+function implSnippet(node, program, showName = true, nesting = 0) {
     (0, assert_1.guard)(node !== undefined, 'Node must be defined => invalid change of type name?');
     const indent = ' '.repeat(nesting * 2);
     const bold = node.kind === 'interface' || node.kind === 'enum' ? '**' : '';
@@ -274,7 +313,8 @@ function implSnippet(node, program, nesting = 0) {
     let text = node.comments?.join('\n') ?? '';
     const code = node.node.getFullText(program.getSourceFile(node.node.getSourceFile().fileName));
     text += `\n<details><summary style="color:gray">Defined at <a href="${getTypePathLink(node)}">${getTypePathLink(node, '.')}</a></summary>\n\n${(0, doc_code_1.codeBlock)('ts', code)}\n\n</details>\n`;
-    return `${indent} * ${bold}[${node.name}](${getTypePathLink(node)})${bold} ${sep}${indent}   ${text.replaceAll('\t', '    ').split(/\n/g).join(`\n${indent}   `)}`;
+    const init = showName ? ` * ${bold}[${node.name}](${getTypePathLink(node)})${bold} ${sep}${indent}` : '';
+    return ` ${indent} ${showName ? init : ''} ${text.replaceAll('\t', '    ').split(/\n/g).join(`\n${indent}   `)}`;
 }
 exports.mermaidHide = ['Leaf', 'Location', 'Namespace', 'Base', 'WithChildren', 'Partial', 'RAccessBase'];
 function printHierarchy({ program, hierarchy, root, collapseFromNesting = 1, initialNesting = 0, maxDepth = 20 }) {
@@ -285,7 +325,7 @@ function printHierarchy({ program, hierarchy, root, collapseFromNesting = 1, ini
     if (!node) {
         return '';
     }
-    const thisLine = implSnippet(node, program, initialNesting);
+    const thisLine = implSnippet(node, program, true, initialNesting);
     const result = [];
     for (const baseType of node.extends) {
         if (exports.mermaidHide.includes(baseType)) {
@@ -302,4 +342,42 @@ function printHierarchy({ program, hierarchy, root, collapseFromNesting = 1, ini
         return thisLine + (out ? '\n' + out : '');
     }
 }
+function retrieveNode(name, hierarchy) {
+    let container = undefined;
+    if (name.includes('::')) {
+        [container, name] = name.split('::');
+    }
+    const node = hierarchy.find(e => e.name === name);
+    if (!node) {
+        return undefined;
+    }
+    else if (container && !node.extends.includes(container)) {
+        return undefined;
+    }
+    return [container, name, node];
+}
+/**
+ * Create a short link to a type in the documentation
+ * @param name      - The name of the type, e.g. `MyType`, may include a container, e.g. `MyContainer::MyType` (this works with function nestings too)
+ * @param hierarchy - The hierarchy of types to search in
+ * @param codeStyle - Whether to use code style for the link
+ */
+function shortLink(name, hierarchy, codeStyle = true) {
+    const res = retrieveNode(name, hierarchy);
+    if (!res) {
+        return '';
+    }
+    const [pkg, mainName, node] = res;
+    const comments = node.comments?.join('\n').replace(/\\?\n|```[a-zA-Z]*|\s\s*/g, ' ').replace(/<\/?code>|`/g, '').replace(/"/g, '\'') ?? '';
+    return `[${codeStyle ? '<code>' : ''}${(node.comments?.length ?? 0) > 0 ?
+        (0, html_hover_over_1.textWithTooltip)(pkg ? `${pkg}::<b>${mainName}</b>` : mainName, comments.length > 400 ? comments.slice(0, 400) + '...' : comments) : node.name}${codeStyle ? '</code>' : ''}](${getTypePathLink(node)})`;
+}
+function getDocumentationForType(name, hierarchy) {
+    const res = retrieveNode(name, hierarchy);
+    if (!res) {
+        return '';
+    }
+    const [, , node] = res;
+    return node.comments?.join('\n') ?? '';
+}
 //# sourceMappingURL=doc-types.js.map