@eagleoutice/flowr 1.3.11

package/r-bridge/shell.d.ts

@@ -0,0 +1,147 @@
+/// <reference types="node" />
+import { type MergeableRecord } from '../util/objects';
+import { SemVer } from 'semver';
+import { TokenMap } from './retriever';
+export type OutputStreamSelector = 'stdout' | 'stderr' | 'both';
+export interface CollectorTimeout extends MergeableRecord {
+    /**
+   * number of milliseconds to wait for the collection to finish
+   */
+    ms: number;
+    /**
+   * if true, the timeout will reset whenever we receive new data
+   */
+    resetOnNewData: boolean;
+}
+/**
+ * Configuration for the internal output collector used by the {@link RShell}
+ * The defaults are configured with {@link DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION}
+ */
+export interface OutputCollectorConfiguration extends MergeableRecord {
+    /** the streams to use to collect the output from */
+    from: OutputStreamSelector;
+    /**
+   * a string marker to signal that the command was executed successfully.
+   * must not appear as a standalone line in the output. this is our hacky way of ensuring that we are done.
+   */
+    postamble: string;
+    /** internal timeout configuration to use (see {@link CollectorTimeout}) */
+    timeout: CollectorTimeout;
+    /** should the postamble be included in the result? */
+    keepPostamble: boolean;
+    /** automatically trim all lines in the output (useful to ignore trailing whitespace etc.) */
+    automaticallyTrimOutput: boolean;
+}
+export declare const DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION: OutputCollectorConfiguration;
+export interface RShellSessionOptions extends MergeableRecord {
+    /** The path to the R executable, can be only the executable if it is to be found on the PATH. */
+    readonly pathToRExecutable: string;
+    /** Command line options to use when starting the R session. */
+    readonly commandLineOptions: readonly string[];
+    /** The current working directory of the R session. */
+    readonly cwd: string;
+    /** The character to use to mark the end of a line. Is probably always `\n` (even on windows). */
+    readonly eol: string;
+    /** The environment variables available in the R session. */
+    readonly env: NodeJS.ProcessEnv;
+    /** If set, the R session will be restarted if it exits due to an error */
+    readonly revive: 'never' | 'on-error' | 'always';
+    /** Called when the R session is restarted, this makes only sense if `revive` is not set to `'never'` */
+    readonly onRevive: (code: number, signal: string | null) => void;
+    /** The path to the library directory, use undefined to let R figure that out for itself */
+    readonly homeLibPath: string | undefined;
+}
+/**
+ * Configuration of an {@link RShell} instance.
+ * See {@link DEFAULT_R_SHELL_OPTIONS} for the default values used by {@link RShell}.
+ */
+export interface RShellOptions extends RShellSessionOptions {
+    readonly sessionName: string;
+}
+export declare const DEFAULT_R_SHELL_OPTIONS: RShellOptions;
+/**
+ * The `RShell` represents an interactive session with the R interpreter.
+ * You can configure it by {@link RShellOptions}.
+ *
+ * At the moment we are using a live R session (and not networking etc.) to communicate with R easily,
+ * which allows us to install packages etc. However, this might and probably will change in the future (leaving this
+ * as a legacy mode :D)
+ */
+export declare class RShell {
+    readonly options: Readonly<RShellOptions>;
+    private session;
+    private readonly log;
+    private versionCache;
+    private tokenMapCache;
+    private tempDirs;
+    constructor(options?: Partial<RShellOptions>);
+    private revive;
+    /**
+   * sends the given command directly to the current R session
+   * will not do anything to alter input markers!
+   */
+    sendCommand(command: string): void;
+    usedRVersion(): Promise<SemVer | null>;
+    /**
+     * Retrieve the token map of the xmlparsedata package.
+     *
+     * @note For multiple calls, this makes use of caching
+     */
+    tokenMap(): Promise<TokenMap>;
+    private retrieveTokenMap;
+    /**
+   * Send a command and collect the output
+   *
+   * @param command     - The R command to execute (similar to {@link sendCommand})
+   * @param addonConfig - Further configuration on how and what to collect: see {@link OutputCollectorConfiguration},
+   *                      defaults are set in {@link DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION}
+   */
+    sendCommandWithOutput(command: string, addonConfig?: Partial<OutputCollectorConfiguration>): Promise<string[]>;
+    /**
+   * execute multiple commands in order
+   *
+   * @see sendCommand
+   */
+    sendCommands(...commands: string[]): void;
+    /**
+   * clears the R environment using the `rm` command.
+   */
+    clearEnvironment(): void;
+    /**
+   * usually R will stop execution on errors, with this the R session will try to
+   * continue working!
+   */
+    continueOnError(): void;
+    injectLibPaths(...paths: string[]): void;
+    tryToInjectHomeLibPath(): void;
+    /**
+   * checks if a given package is already installed on the system!
+   */
+    isPackageInstalled(packageName: string): Promise<boolean>;
+    allInstalledPackages(): Promise<string[]>;
+    /**
+   * Installs the package using a temporary location
+   *
+   * @param packageName - The package to install
+   * @param autoload    - If true, the package will be loaded after installation
+   * @param force       - If true, the package will be installed no if it is already on the system and ready to be loaded
+   */
+    ensurePackageInstalled(packageName: string, autoload?: boolean, force?: boolean): Promise<{
+        packageName: string;
+        packageExistedAlready: boolean;
+        /** the temporary directory used for the installation, undefined if none was used */
+        libraryLocation?: string;
+    }>;
+    /**
+     * Obtain the temporary directory used by R.
+     * Additionally, this marks the directory for removal when the shell exits.
+     */
+    obtainTmpDir(): Promise<string>;
+    /**
+   * Close the current R session, makes the object effectively invalid (can no longer be reopened etc.)
+   *
+   * @returns true if the operation succeeds, false otherwise
+   */
+    close(): boolean;
+    private _sendCommand;
+}

package/r-bridge/shell.js

@@ -0,0 +1,423 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RShell = exports.DEFAULT_R_SHELL_OPTIONS = exports.DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION = void 0;
+const child_process_1 = require("child_process");
+const objects_1 = require("../util/objects");
+const readline = __importStar(require("node:readline"));
+const lang_4_x_1 = require("./lang-4.x");
+const log_1 = require("../util/log");
+const preload_1 = __importDefault(require("semver/preload"));
+const os_1 = require("../util/os");
+const fs_1 = __importDefault(require("fs"));
+const retriever_1 = require("./retriever");
+exports.DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION = {
+    from: 'stdout',
+    postamble: `🐧${'-'.repeat(5)}🐧`,
+    timeout: {
+        ms: 750_000,
+        resetOnNewData: true
+    },
+    keepPostamble: false,
+    automaticallyTrimOutput: true,
+    errorStopsWaiting: true
+};
+exports.DEFAULT_R_SHELL_OPTIONS = {
+    sessionName: 'default',
+    pathToRExecutable: (0, os_1.getPlatform)() === 'windows' ? 'R.exe' : 'R',
+    commandLineOptions: ['--vanilla', '--quiet', '--no-echo', '--no-save'],
+    cwd: process.cwd(),
+    env: process.env,
+    eol: '\n',
+    homeLibPath: (0, os_1.getPlatform)() === 'windows' ? undefined : '~/.r-libs',
+    revive: 'never',
+    onRevive: () => { }
+};
+/**
+ * The `RShell` represents an interactive session with the R interpreter.
+ * You can configure it by {@link RShellOptions}.
+ *
+ * At the moment we are using a live R session (and not networking etc.) to communicate with R easily,
+ * which allows us to install packages etc. However, this might and probably will change in the future (leaving this
+ * as a legacy mode :D)
+ */
+class RShell {
+    options;
+    session;
+    log;
+    versionCache = null;
+    tokenMapCache = null;
+    // should never be more than one, but let's be sure
+    tempDirs = new Set();
+    constructor(options) {
+        this.options = (0, objects_1.deepMergeObject)(exports.DEFAULT_R_SHELL_OPTIONS, options);
+        this.log = log_1.log.getSubLogger({ name: this.options.sessionName });
+        this.session = new RShellSession(this.options, this.log);
+        this.revive();
+    }
+    revive() {
+        if (this.options.revive === 'never') {
+            return;
+        }
+        this.session.onExit((code, signal) => {
+            if (this.options.revive === 'always' || (this.options.revive === 'on-error' && code !== 0)) {
+                this.log.warn(`R session exited with code ${code}, reviving!`);
+                this.options.onRevive(code, signal);
+                this.session = new RShellSession(this.options, this.log);
+                this.revive();
+            }
+        });
+    }
+    /**
+   * sends the given command directly to the current R session
+   * will not do anything to alter input markers!
+   */
+    sendCommand(command) {
+        if (this.log.settings.minLevel >= 1 /* LogLevel.Trace */) {
+            this.log.trace(`> ${JSON.stringify(command)}`);
+        }
+        this._sendCommand(command);
+    }
+    async usedRVersion() {
+        if (this.versionCache !== null) {
+            return this.versionCache;
+        }
+        // retrieve raw version:
+        const result = await this.sendCommandWithOutput(`cat(paste0(R.version$major,".",R.version$minor), ${(0, lang_4_x_1.ts2r)(this.options.eol)})`);
+        this.log.trace(`raw version: ${JSON.stringify(result)}`);
+        this.versionCache = preload_1.default.coerce(result[0]);
+        return result.length === 1 ? this.versionCache : null;
+    }
+    /**
+     * Retrieve the token map of the xmlparsedata package.
+     *
+     * @note For multiple calls, this makes use of caching
+     */
+    async tokenMap() {
+        if (this.tokenMapCache === null) {
+            this.tokenMapCache = await this.retrieveTokenMap();
+        }
+        return this.tokenMapCache;
+    }
+    async retrieveTokenMap() {
+        await this.ensurePackageInstalled('xmlparsedata', true /* use some kind of environment in the future */);
+        // we invert the token map to get a mapping back from the replacement
+        const parsed = (0, lang_4_x_1.parseCSV)(await this.sendCommandWithOutput('write.table(xmlparsedata::xml_parse_token_map,sep=",",col.names=FALSE)'));
+        if (parsed.some(s => s.length !== 2)) {
+            throw new Error(`Expected two columns in token map, but got ${JSON.stringify(parsed)}`);
+        }
+        // we swap key and value to get the other direction, furthermore we remove quotes from keys if they are quoted
+        const cache = {};
+        for (const [key, value] of parsed) {
+            cache[value] = (0, retriever_1.removeTokenMapQuotationMarks)(key);
+        }
+        return cache;
+    }
+    /**
+   * Send a command and collect the output
+   *
+   * @param command     - The R command to execute (similar to {@link sendCommand})
+   * @param addonConfig - Further configuration on how and what to collect: see {@link OutputCollectorConfiguration},
+   *                      defaults are set in {@link DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION}
+   */
+    async sendCommandWithOutput(command, addonConfig) {
+        const config = (0, objects_1.deepMergeObject)(exports.DEFAULT_OUTPUT_COLLECTOR_CONFIGURATION, addonConfig);
+        if (this.log.settings.minLevel >= 1 /* LogLevel.Trace */) {
+            this.log.trace(`> ${JSON.stringify(command)}`);
+        }
+        const output = await this.session.collectLinesUntil(config.from, {
+            predicate: data => data === config.postamble,
+            includeInResult: config.keepPostamble // we do not want the postamble
+        }, config.timeout, () => {
+            this._sendCommand(command);
+            if (config.from === 'stderr') {
+                this._sendCommand(`cat("${config.postamble}${this.options.eol}", file=stderr())`);
+            }
+            else {
+                this._sendCommand(`cat("${config.postamble}${this.options.eol}")`);
+            }
+        });
+        if (config.automaticallyTrimOutput) {
+            return output.map(line => line.trim());
+        }
+        else {
+            return output;
+        }
+    }
+    /**
+   * execute multiple commands in order
+   *
+   * @see sendCommand
+   */
+    sendCommands(...commands) {
+        for (const element of commands) {
+            this.sendCommand(element);
+        }
+    }
+    /**
+   * clears the R environment using the `rm` command.
+   */
+    clearEnvironment() {
+        this.log.debug('clearing environment');
+        this._sendCommand('rm(list=ls())');
+    }
+    /**
+   * usually R will stop execution on errors, with this the R session will try to
+   * continue working!
+   */
+    continueOnError() {
+        this.log.info('continue in case of Errors');
+        this._sendCommand('options(error=function() {})');
+    }
+    injectLibPaths(...paths) {
+        this.log.debug(`injecting lib paths ${JSON.stringify(paths)}`);
+        this._sendCommand(`.libPaths(c(.libPaths(), ${paths.map(lang_4_x_1.ts2r).join(',')}))`);
+    }
+    tryToInjectHomeLibPath() {
+        // ensure the path exists first
+        if (this.options.homeLibPath === undefined) {
+            this.log.debug('ensuring home lib path exists (automatic inject)');
+            this.sendCommand('if(!dir.exists(Sys.getenv("R_LIBS_USER"))) { dir.create(path=Sys.getenv("R_LIBS_USER"),showWarnings=FALSE,recursive=TRUE) }');
+            this.sendCommand('.libPaths(c(.libPaths(), Sys.getenv("R_LIBS_USER")))');
+        }
+        else {
+            this.injectLibPaths(this.options.homeLibPath);
+        }
+    }
+    /**
+   * checks if a given package is already installed on the system!
+   */
+    async isPackageInstalled(packageName) {
+        this.log.debug(`checking if package "${packageName}" is installed`);
+        const result = await this.sendCommandWithOutput(`cat(system.file(package="${packageName}")!="","${this.options.eol}")`);
+        return result.length === 1 && result[0] === 'TRUE';
+    }
+    async allInstalledPackages() {
+        this.log.debug('getting all installed packages');
+        const [packages] = await this.sendCommandWithOutput(`cat(paste0(installed.packages()[,1], collapse=","),"${this.options.eol}")`);
+        return packages.split(',');
+    }
+    /**
+   * Installs the package using a temporary location
+   *
+   * @param packageName - The package to install
+   * @param autoload    - If true, the package will be loaded after installation
+   * @param force       - If true, the package will be installed no if it is already on the system and ready to be loaded
+   */
+    async ensurePackageInstalled(packageName, autoload = false, force = false) {
+        const packageExistedAlready = await this.isPackageInstalled(packageName);
+        if (!force && packageExistedAlready) {
+            this.log.info(`package "${packageName}" is already installed`);
+            if (autoload) {
+                this.sendCommand(`library(${(0, lang_4_x_1.ts2r)(packageName)})`);
+            }
+            return {
+                packageName,
+                packageExistedAlready: true
+            };
+        }
+        // obtain a temporary directory
+        const tempdir = await this.obtainTmpDir();
+        this.log.debug(`using temporary directory: "${tempdir}" to install package "${packageName}"`);
+        await this.sendCommandWithOutput(`install.packages(${(0, lang_4_x_1.ts2r)(packageName)},repos="https://cloud.r-project.org/",quiet=FALSE,lib=temp)`, {
+            ms: 750_000,
+            resetOnNewData: true
+        });
+        if (autoload) {
+            this.sendCommand(`library(${(0, lang_4_x_1.ts2r)(packageName)},lib.loc=temp)`);
+        }
+        return {
+            packageName,
+            libraryLocation: tempdir,
+            packageExistedAlready
+        };
+    }
+    /**
+     * Obtain the temporary directory used by R.
+     * Additionally, this marks the directory for removal when the shell exits.
+     */
+    async obtainTmpDir() {
+        this.sendCommand('temp <- tempdir()');
+        const [tempdir] = await this.sendCommandWithOutput(`cat(temp, ${(0, lang_4_x_1.ts2r)(this.options.eol)})`);
+        this.tempDirs.add(tempdir);
+        return tempdir;
+    }
+    /**
+   * Close the current R session, makes the object effectively invalid (can no longer be reopened etc.)
+   *
+   * @returns true if the operation succeeds, false otherwise
+   */
+    close() {
+        return this.session.end([...this.tempDirs]);
+    }
+    _sendCommand(command) {
+        this.session.writeLine(command);
+    }
+}
+exports.RShell = RShell;
+/**
+ * Used to deal with the underlying input-output streams of the R process
+ */
+class RShellSession {
+    bareSession;
+    sessionStdOut;
+    sessionStdErr;
+    options;
+    log;
+    collectionTimeout;
+    constructor(options, log) {
+        this.bareSession = (0, child_process_1.spawn)(options.pathToRExecutable, options.commandLineOptions, {
+            env: options.env,
+            cwd: options.cwd,
+            windowsHide: true
+        });
+        this.sessionStdOut = readline.createInterface({
+            input: this.bareSession.stdout,
+            terminal: false
+        });
+        this.sessionStdErr = readline.createInterface({
+            input: this.bareSession.stderr,
+            terminal: false
+        });
+        this.onExit(() => { this.end(); });
+        this.options = options;
+        this.log = log;
+        this.setupRSessionLoggers();
+    }
+    write(data) {
+        this.bareSession.stdin.write(data);
+    }
+    writeLine(data) {
+        this.write(`${data}${this.options.eol}`);
+    }
+    /**
+   * Collect lines from the selected streams until the given condition is met or the timeout is reached
+   *
+   * This method does allow other listeners to consume the same input
+   *
+   * @param from        - The stream(s) to collect the information from
+   * @param until       - If the predicate returns true, this will stop the collection and resolve the promise
+   * @param timeout     - Configuration for how and when to timeout
+   * @param action      - Event to be performed after all listeners are installed, this might be the action that triggers the output you want to collect
+   */
+    async collectLinesUntil(from, until, timeout, action) {
+        const result = [];
+        let handler;
+        let error;
+        return await new Promise((resolve, reject) => {
+            const makeTimer = () => setTimeout(() => {
+                reject(new Error(`timeout of ${timeout.ms}ms reached (${JSON.stringify(result)})`));
+            }, timeout.ms);
+            this.collectionTimeout = makeTimer();
+            handler = (data) => {
+                const end = until.predicate(data);
+                if (!end || until.includeInResult) {
+                    result.push(data);
+                }
+                if (end) {
+                    clearTimeout(this.collectionTimeout);
+                    resolve(result);
+                }
+                else if (timeout.resetOnNewData) {
+                    clearTimeout(this.collectionTimeout);
+                    this.collectionTimeout = makeTimer();
+                }
+            };
+            error = () => { resolve(result); };
+            this.onExit(error);
+            this.on(from, 'line', handler);
+            action?.();
+        }).finally(() => {
+            this.removeListener(from, 'line', handler);
+            this.bareSession.removeListener('exit', error);
+            this.bareSession.stdin.removeListener('error', error);
+        });
+    }
+    /**
+   * close the current R session, makes the object effectively invalid (can no longer be reopened etc.)
+   *
+     * @param filesToUnlink - If set, these files will be unlinked before closing the session (e.g., to clean up tempfiles)
+     *
+   * @returns true if the kill succeeds, false otherwise
+   * @see RShell#close
+   */
+    end(filesToUnlink) {
+        if (filesToUnlink !== undefined) {
+            log_1.log.info(`unlinking ${filesToUnlink.length} files (${JSON.stringify(filesToUnlink)})`);
+            for (const f of filesToUnlink) {
+                fs_1.default.rmSync(f, { recursive: true, force: true });
+            }
+        }
+        const killResult = this.bareSession.kill();
+        if (this.collectionTimeout !== undefined) {
+            clearTimeout(this.collectionTimeout);
+        }
+        this.sessionStdOut.close();
+        this.sessionStdErr.close();
+        log_1.log.info(`killed R session with pid ${this.bareSession.pid ?? '<unknown>'} and result ${killResult ? 'successful' : 'failed'} (including streams)`);
+        return killResult;
+    }
+    setupRSessionLoggers() {
+        if (this.log.settings.minLevel >= 1 /* LogLevel.Trace */) {
+            this.bareSession.stdout.on('data', (data) => {
+                this.log.trace(`< ${data.toString()}`);
+            });
+            this.bareSession.on('close', (code) => {
+                this.log.trace(`session exited with code ${code}`);
+            });
+        }
+        this.bareSession.stderr.on('data', (data) => {
+            this.log.warn(`< ${data}`);
+        });
+    }
+    onExit(callback) {
+        this.bareSession.on('exit', callback);
+        this.bareSession.stdin.on('error', callback);
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    on(from, event, listener) {
+        const both = from === 'both';
+        if (both || from === 'stdout') {
+            this.sessionStdOut.on(event, listener);
+        }
+        if (both || from === 'stderr') {
+            this.sessionStdErr.on(event, listener);
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    removeListener(from, event, listener) {
+        const both = from === 'both';
+        if (both || from === 'stdout') {
+            this.sessionStdOut.removeListener(event, listener);
+        }
+        if (both || from === 'stderr') {
+            this.sessionStdErr.removeListener(event, listener);
+        }
+    }
+}
+//# sourceMappingURL=shell.js.map

package/reconstruct/reconstruct.d.ts

@@ -0,0 +1,28 @@
+/**
+ * This module has one goal (and is to be rewritten soon to achieve that goal,
+ * as the file itself is way too long). See {@link reconstructToCode}.
+ * @module
+ */
+import { NormalizedAst, NodeId, ParentInformation, RNode } from '../r-bridge';
+type Selection = Set<NodeId>;
+export declare const reconstructLogger: import("tslog").Logger<import("tslog").ILogObj>;
+/** The structure of the predicate that should be used to determine if a given normalized node should be included in the reconstructed code independent of if it is selected by the slice or not */
+export type AutoSelectPredicate = (node: RNode<ParentInformation>) => boolean;
+export declare function doNotAutoSelect(_node: RNode<ParentInformation>): boolean;
+export declare function autoSelectLibrary(node: RNode<ParentInformation>): boolean;
+export interface ReconstructionResult {
+    code: string;
+    /** number of nodes that triggered the `autoSelectIf` predicate {@link reconstructToCode} */
+    autoSelected: number;
+}
+/**
+ * Reconstructs parts of a normalized R ast into R code on an expression basis.
+ *
+ * @param ast          - The {@link NormalizedAst|normalized ast} to be used as a basis for reconstruction
+ * @param selection    - The selection of nodes to be reconstructed (probably the {@link NodeId|NodeIds} identified by the slicer)
+ * @param autoSelectIf - A predicate that can be used to force the reconstruction of a node (for example to reconstruct library call statements, see {@link autoSelectLibrary}, {@link doNotAutoSelect})
+ *
+ * @returns The number of times `autoSelectIf` triggered, as well as the reconstructed code itself.
+ */
+export declare function reconstructToCode<Info>(ast: NormalizedAst<Info>, selection: Selection, autoSelectIf?: AutoSelectPredicate): ReconstructionResult;
+export {};