@fynjs/run 1.0.1 → 1.1.1

package/dist/lib/xqtor.js

@@ -0,0 +1,704 @@
+import { createRequire } from "node:module";
+import assert from "assert";
+import defaults from "./defaults.js";
+import Insync from "insync";
+import XQItem from "./xqitem.js";
+import { exec as exec } from "xsh";
+import parseArray from "./util/parse-array.js";
+import childProc from "child_process";
+import updateEnv from "./util/update-env.js";
+import { NixClap } from "@fynjs/cli-args";
+import { getTaskOptionSpec } from "./util/task-options.js";
+//
+// Kept lazy rather than a static import: a static import is hoisted to module load, which is
+// exactly what the call site below avoids so fynpo bootstrap can build unwrap-npm-cmd first.
+//
+const lazyRequire = createRequire(import.meta.url);
+const STAGE_FINALLY = "finally";
+const isSerial = x => defaults.SERIAL_SIG.includes(x[0]);
+const isConcurrent = x => defaults.CONCURRENT_SIG.includes(x[0]);
+const isReadableStream = x => Boolean(x && x.pipe && x.on && x._readableState);
+/*
+ * Task executor (XQtor) - After all the fuss, it comes down to this -- the core of xrun
+ *                         that handles the execution of tasks.
+ *
+ * Tasks are added to a stack using an array.
+ *
+ * - Each concurrent tasks array is executed with a new XQtor and Insync.parallel.
+ * - Serial tasks are all added to the top of the "stack" in reverse order
+ * - Some items are actions that cause more items to be created and pushed
+ *   into the stack.  Like looking up the task of a task name.
+ * - As each task is completed, the XQtor's next is invoked to pop items
+ *   from the stack for processing.
+ * - A mark item is added to the stack before each task so when a mark
+ *   is seen, error is checked and done event is emitted.
+ *
+ */
+class XQtor {
+    constructor(options) {
+        this._tasks = options.tasks;
+        this._done = options.done;
+        this._xrun = options.xrun;
+        this.xqItems = [];
+    }
+    next(err, xqId) {
+        process.nextTick(() => this._next(err, xqId), 0);
+    }
+    _next(err, xqId) {
+        if (err) {
+            const qItem = this._xrun.xqTree.item(xqId);
+            qItem.err = err;
+            this._xrun.fail(err);
+        }
+        if (this.xqItems.length > 0) {
+            this.execute();
+        }
+        else {
+            this._done(this._xrun.failed);
+        }
+    }
+    execute() {
+        assert(this.xqItems.length > 0, "no next task");
+        const qItem = this.popItem();
+        if (qItem.mark) {
+            return this._processMark(qItem);
+        }
+        if (this._xrun._isStop || (this._xrun.failed && !qItem.isFinally && this._xrun.stopOnError)) {
+            return this.next(null, qItem.id);
+        }
+        const value = qItem.value();
+        if (value === defaults.STOP_SYM) {
+            this._xrun.actStop();
+            return this.next(null, qItem.id);
+        }
+        const vtype = value.constructor.name;
+        if (vtype === "XTaskSpec") {
+            if (value.type === "exec") {
+                return this._shellXer(qItem, value, true);
+            }
+            else if (value.type === "env") {
+                return this._envXer(qItem, value);
+            }
+            this._xrun.fail(new Error(`Unable to process XTaskSpec type ${value.type} for task ${qItem.name}`));
+        }
+        else if (vtype === "String") {
+            if (this._isAnonShell(value)) {
+                return this._shellXer(qItem, this._parseAnonShell(value), true);
+            }
+            return this._processLookup(qItem);
+        }
+        else if (vtype === "Function" || vtype === "AsyncFunction") {
+            return this._functionXer(qItem, value);
+        }
+        else if (vtype === "Array") {
+            return this._processArray(qItem, value);
+        }
+        else if (value.item) {
+            const dep = value.item.dep;
+            if (dep && !qItem.xqDep) {
+                return this._processDep(qItem, dep);
+            }
+            else {
+                return this._processTaskObject(qItem);
+            }
+        }
+        else {
+            this._xrun.fail(new Error(`Unable to process task ${qItem.name} \
+because value type ${vtype} is unknown and no value.item`));
+        }
+        return this.next(null, qItem.id);
+    }
+    pushMarkItem(qItem, callback) {
+        const x = this._makeXqMarkItem(qItem, callback);
+        this.pushItem(x);
+        return x;
+    }
+    pushItem(qItem) {
+        this.xqItems.push(qItem);
+    }
+    popItem() {
+        return this.xqItems.pop();
+    }
+    _isAnonShell(value) {
+        let sig;
+        if ((sig = defaults.ANON_SHELL_OPT_SIG.find(x => value.startsWith(x)))) {
+            return { sig, opt: true };
+        }
+        if ((sig = defaults.ANON_SHELL_SIG.find(x => value.startsWith(x)))) {
+            return { sig, opt: false };
+        }
+        return false;
+    }
+    _parseShellFlags(flags, value) {
+        const unknowns = [];
+        flags = flags.reduce((a, o) => {
+            const f = o.trim().toLowerCase();
+            if (defaults.SHELL_FLAGS.indexOf(f) < 0) {
+                unknowns.push(o);
+            }
+            else {
+                a[f] = true;
+            }
+            return a;
+        }, {});
+        if (unknowns.length) {
+            throw new Error(`Unknown flag ${unknowns.join(",")} in shell task: ${value}`);
+        }
+        return flags;
+    }
+    _parseAnonShell(value) {
+        if (typeof value !== "string")
+            return value;
+        let cmd = value;
+        let flags = {};
+        let error;
+        const anon = this._isAnonShell(value);
+        if (anon) {
+            const { sig, opt } = anon;
+            // support options like ~(tty,spawn,sync)$
+            if (opt) {
+                let ix;
+                const closeSig = defaults.ANON_SHELL_OPT_CLOSE_SIG.find(x => {
+                    ix = value.indexOf(x, sig.length);
+                    return ix > sig.length;
+                });
+                if (closeSig) {
+                    const so = value.substring(sig.length, ix);
+                    try {
+                        flags = this._parseShellFlags(so.split(","), value);
+                    }
+                    catch (err) {
+                        error = err;
+                    }
+                    cmd = value.substring(ix + closeSig.length);
+                }
+                else {
+                    error = new Error(`Missing ${defaults.ANON_SHELL_OPT_CLOSE_SIG[0]} in shell task: ${value}`);
+                }
+            }
+            else {
+                cmd = value.substring(sig.length);
+            }
+        }
+        return { flags, cmd, error };
+    }
+    _isStrArray(value) {
+        return value.startsWith(defaults.STR_ARRAY_SIG);
+    }
+    _parseStrArray(value) {
+        if (this._isStrArray(value)) {
+            return parseArray(value.substring(defaults.STR_ARRAY_SIG.length - 1));
+        }
+        return undefined;
+    }
+    _processLookup(qItem) {
+        this._emit("execute", {
+            type: "lookup",
+            qItem
+        });
+        // lookup is sync; should not insert mark for it
+        try {
+            const found = qItem.lookup(this._tasks);
+            this.pushItem(qItem);
+            this.next(null, qItem.id);
+            if (found.search) {
+                this._xrun.emit("search", { qItem, found });
+                qItem.setNamespace(found.ns);
+            }
+        }
+        catch (error) {
+            if (error.optional) {
+                this._xrun.emit("not-found", error);
+                this.next();
+            }
+            else {
+                this.next(error, qItem.id);
+            }
+        }
+    }
+    _processDep(qItem, dep) {
+        this._emit("dep", {
+            qItem
+        });
+        qItem.xqDep = true; // item's dep has been processed
+        this.pushItem(qItem);
+        const di = this._createQItem({
+            name: qItem.name,
+            value: { top: true, depDee: qItem, item: { task: dep } }
+        }, this._xrun.xqTree.parent(qItem));
+        this.pushItem(di);
+        this.next(null, qItem.id);
+    }
+    _processTaskObject(qItem) {
+        const value = qItem.value();
+        const stage = qItem.stage || "task";
+        const itemTask = value.item[stage] || value.item;
+        const type = itemTask.constructor.name;
+        if (type === "Array") {
+            return this._processArray(qItem, itemTask, value.top);
+        }
+        else if (type === "Function" || type === "AsyncFunction") {
+            return this._functionXer(qItem, itemTask);
+        }
+        else if (type === "String") {
+            const parsedArray = this._parseStrArray(itemTask);
+            if (parsedArray)
+                return this._processArray(qItem, parsedArray, value.top);
+            return this._shellXer(qItem, itemTask);
+        }
+        else if (type === "XTaskSpec") {
+            if (itemTask.type === "exec") {
+                return this._shellXer(qItem, itemTask);
+            }
+            else if (itemTask.type === "env") {
+                return this._envXer(qItem, itemTask);
+            }
+            this._xrun.fail(new Error(`Unable to process XTaskSpec type ${itemTask.type} for task ${qItem.name}`));
+        }
+        else {
+            this._xrun.fail(new Error(`Task ${qItem.name} has unrecognize task value type ${type}`));
+        }
+        return this.next(null, qItem.id);
+    }
+    _processArray(qItem, tasks, top) {
+        // check first element for concurrent signature
+        // top level task arrays are automatically serial
+        // or check first element for serial signature
+        if (isConcurrent(tasks) || (!top && !isSerial(tasks))) {
+            this._concurrentArrayXer(qItem, tasks);
+        }
+        else {
+            this._serialArrayXer(qItem, tasks);
+        }
+    }
+    _serialArrayXer(qItem, tasks) {
+        if (isSerial(tasks)) {
+            tasks = tasks.slice(1);
+        }
+        this._emit("execute", {
+            type: "serial-arr",
+            qItem,
+            array: tasks
+        });
+        this.pushMarkItem(qItem);
+        this.xqItems = this.xqItems.concat(tasks
+            .map(value => {
+            return this._createQItem(this._resolveValueToQItemOptions(qItem.ns, `${qItem.name}.S`, value, "serial_child"), qItem);
+        })
+            .reverse());
+        this.next(null, qItem.id);
+    }
+    _resolveValueToQItemOptions(ns, name, value, type) {
+        if (typeof value === "string" && !this._isAnonShell(value)) {
+            name = value;
+            value = undefined;
+        }
+        const anon = typeof value === "function";
+        return {
+            ns,
+            name,
+            value,
+            anon,
+            type
+        };
+    }
+    _concurrentArrayXer(qItem, tasks) {
+        if (isConcurrent(tasks)) {
+            tasks = tasks.slice(1);
+        }
+        this._emit("execute", {
+            type: "concurrent-arr",
+            qItem,
+            array: tasks
+        });
+        this.pushMarkItem(qItem);
+        Insync.parallel(tasks.map(value => cb => {
+            this._emit("spawn-async", { name: qItem.name });
+            const xqtor = new XQtor({
+                tasks: this._tasks,
+                xrun: this._xrun,
+                done: () => {
+                    this._emit("done-async");
+                    process.nextTick(() => cb());
+                }
+            });
+            const options = this._resolveValueToQItemOptions(qItem.ns, `${qItem.name}.C`, value, "concurrent_child");
+            const item = this._createQItem(options, qItem);
+            if (options.value === undefined) {
+                xqtor._processLookup(item);
+            }
+            else {
+                xqtor.pushItem(item);
+                xqtor.next();
+            }
+        }), () => this.next());
+    }
+    _envXer(qItem, cmdVal) {
+        this._emit("execute", {
+            type: "env",
+            qItem,
+            cmdVal
+        });
+        updateEnv(cmdVal.options.env, process.env, cmdVal.options.override);
+        return this.next();
+    }
+    _shellXer(qItem, cmdVal, anon) {
+        cmdVal = this._parseAnonShell(cmdVal);
+        if (cmdVal.error) {
+            setTimeout(() => this.next(cmdVal.error, qItem.id), 0);
+            return;
+        }
+        const { cmd } = cmdVal;
+        const xrunOptions = Object.assign({ delayRunMs: 0 }, cmdVal.xrun || cmdVal.xclap);
+        const options = Object.assign({}, cmdVal.options);
+        let { flags } = cmdVal;
+        if (typeof flags === "string") {
+            flags = this._parseShellFlags(flags.split(","), `XTaskSpec "${flags}"`);
+        }
+        else if (Array.isArray(flags)) {
+            flags = this._parseShellFlags(flags);
+        }
+        const item = qItem.value().item || {};
+        Object.assign(flags, item.flags);
+        // Access CLI context through runner
+        const cliContext = this._xrun.getCliContext();
+        const itemArgv = qItem.argv;
+        const cliArgv = cliContext.getTaskArgv(qItem.name);
+        const remainingArgv = (cliContext.isLastTask(qItem.name) && cliContext.getRemainingArgs()) || [];
+        // Build command with task-specific argv
+        const cmd2 = []
+            .concat(itemArgv.slice(1), cliArgv.slice(1), remainingArgv)
+            .map(x => (x && x.includes(" ") ? `"${x}"` : x))
+            .join(" ");
+        setTimeout(() => {
+            const execData = {
+                type: "shell",
+                anon,
+                qItem,
+                cmd: cmd + ((cmd2 && " " + cmd2) || ""),
+                cmd2,
+                flags,
+                cmdVal,
+                options,
+                item
+            };
+            this._emit("execute", execData);
+            this._doShellXer(execData);
+        }, xrunOptions.delayRunMs);
+    }
+    _doShellXer({ /*anon,*/ qItem, cmd, flags, /*cmdVal,*/ options, item }) {
+        const itemOptions = Object.assign({}, item.options);
+        this.pushMarkItem(qItem);
+        const env = Object.assign(flags.noenv ? {} : Object.assign({}, process.env), itemOptions.env, options.env);
+        if (qItem.err) {
+            env.XCLAP_ERR = env.XRUN_ERR = qItem.err.message || "true";
+        }
+        if (this._xrun.failed || qItem.err) {
+            env.XCLAP_FAILED = env.XRUN_FAILED = "true";
+        }
+        let child;
+        const watch = { finish: false };
+        const done = (err) => {
+            // even if there's error, but if it's due to child being terminated with
+            // SIGTERM, then treat that as a normal exit.
+            if (this._isChildSigTerm(err, child) || this._isChildTerminated(child)) {
+                err = null;
+            }
+            //
+            // A spawned child can report both `error` and `close`, so `done` can be called twice.
+            // Guarding it is the point; reproducing that race on demand in a test is not something
+            // this suite can do deterministically.
+            //
+            /* istanbul ignore next */
+            if (watch.finish)
+                return 0;
+            watch.finish = true;
+            return this.next(err, qItem.id);
+        };
+        let { tty, spawn, sync, npm } = flags;
+        if (npm) {
+            tty = spawn = true;
+        }
+        // Lazy require unwrap-npm-cmd to avoid requiring it at module load time
+        // This ensures it's built by fynpo bootstrap before it's required
+        const { unwrapNpmCmd } = lazyRequire("unwrap-npm-cmd");
+        const cmd2 = unwrapNpmCmd(cmd, { path: env.PATH });
+        if (tty || spawn) {
+            const spawnOpts = { shell: true };
+            Object.assign(spawnOpts, itemOptions, options, { env });
+            if (tty)
+                spawnOpts.stdio = "inherit";
+            if (sync) {
+                child = childProc.spawnSync(cmd2, spawnOpts);
+                if (child.error) {
+                    done(child.error);
+                }
+                else if (child.status === 0) {
+                    done();
+                }
+                else {
+                    done(new Error(`cmd "${cmd}" exit code ${child.status}`));
+                }
+            }
+            else {
+                child = childProc.spawn(cmd2, spawnOpts);
+                child.on("close", (code, signal) => {
+                    if (code === 0 || signal === "SIGTERM") {
+                        return done();
+                    }
+                    return done(new Error(`cmd "${cmd}" exit code ${code}`));
+                });
+            }
+        }
+        else {
+            child = exec(Object.assign({ silent: false }, itemOptions, options, { env }), cmd2, done);
+        }
+        if (!watch.finish) {
+            this._handleChildTask(child, cmd);
+            watch.cancel = () => {
+                this._xrun.killChildProcess(child);
+                done();
+            };
+            this._watchFailure(watch);
+        }
+    }
+    _handleChildTask(child, cmd) {
+        if (child && child instanceof childProc.ChildProcess) {
+            const sym = Symbol(cmd);
+            this._xrun.addTaskChild(child, sym);
+            child.on("exit", () => {
+                this._xrun.removeTaskChild(child, sym);
+            });
+            return child;
+        }
+    }
+    _processMoreFromFn(qItem, value, callback) {
+        let tof = typeof value;
+        if (tof === "string") {
+            const parsedArray = this._parseStrArray(value);
+            if (parsedArray)
+                value = parsedArray;
+            tof = typeof value;
+        }
+        if (!value ||
+            (tof !== "string" &&
+                tof !== "function" &&
+                !Array.isArray(value) &&
+                value.constructor.name !== "XTaskSpec" &&
+                value !== defaults.STOP_SYM)) {
+            return;
+        }
+        const qi = this._createQItem(this._resolveValueToQItemOptions(qItem.ns, `${qItem.name}.fR`, value, "func_returned_child"), qItem);
+        if (callback) {
+            assert(typeof callback === "function", `${qItem.name} callback from function calling run is not a function`);
+            this.pushMarkItem(qItem, callback);
+        }
+        this.pushItem(qi);
+    }
+    _isChildSigTerm(err, child) {
+        return err && err.code === null && child && child.signalCode === "SIGTERM";
+    }
+    _isChildTerminated(child) {
+        return child && child[defaults.STOP_SYM];
+    }
+    _handleStream(val) {
+        if (!isReadableStream(val)) {
+            return val;
+        }
+        return new Promise((resolve, reject) => {
+            const handle = err => {
+                val.removeListener("end", handle);
+                val.removeListener("error", handle);
+                return err ? reject(err) : resolve(undefined);
+            };
+            val.once("close", handle);
+            val.once("error", handle);
+        });
+    }
+    _functionXer(qItem, fn) {
+        this._emit("execute", {
+            type: "function",
+            anon: qItem.anon,
+            qItem
+        });
+        const watch = { finish: false, qItem };
+        const done = (err, value) => {
+            if (watch.finish)
+                return;
+            watch.finish = true;
+            if (value) {
+                this._processMoreFromFn(qItem, value);
+            }
+            this.next(err, qItem.id);
+        };
+        watch.cancel = done;
+        try {
+            this.pushMarkItem(qItem);
+            const run = (value, cb) => this._processMoreFromFn(qItem, value, cb);
+            const task = qItem.value();
+            // Access CLI context through runner
+            const cliContext = this._xrun.getCliContext();
+            let cliCmd = cliContext.getTaskCommand(qItem.name);
+            const { options, commands, allowUnknownOption } = getTaskOptionSpec(task);
+            const itemArgv = qItem.argv;
+            const cliArgv = cliCmd.argv || [];
+            const hasOptionSpec = Object.keys(options).length > 0 || Object.keys(commands).length > 0;
+            if (hasOptionSpec || itemArgv.length > 1 || cliArgv.length > 1) {
+                const argv = [qItem.name].concat(itemArgv.slice(1), cliArgv.slice(1));
+                const config = {
+                    name: qItem.name,
+                    allowUnknownOption: allowUnknownOption !== undefined ? allowUnknownOption : true,
+                    allowUnknownCommand: false,
+                    noDefaultHandlers: true
+                };
+                const argp = new NixClap(config)
+                    .init(options, commands)
+                    .parse(argv, 1);
+                if (argp.errorNodes.length > 0) {
+                    const unknownOptions = [];
+                    const otherErrors = [];
+                    for (const node of argp.errorNodes) {
+                        for (const e of node.errors) {
+                            if (e.message.includes("unknown CLI option")) {
+                                unknownOptions.push(e.data?.name);
+                            }
+                            else {
+                                otherErrors.push(e.message);
+                            }
+                        }
+                    }
+                    if (unknownOptions.length > 0) {
+                        return done(new Error(`Unknown options for task ${qItem.name}: ${unknownOptions.join(", ")}`));
+                    }
+                    if (otherErrors.length > 0) {
+                        return done(new Error(`Error parsing options for task ${qItem.name}: ${otherErrors.join("; ")}`));
+                    }
+                }
+                cliCmd = argp.command;
+            }
+            const argv = cliCmd.argv || [];
+            const context = {
+                run,
+                argv,
+                cliCmd,
+                argp: cliCmd.jsonMeta,
+                args: cliCmd.argsList,
+                argOpts: cliCmd.opts,
+                err: qItem.err,
+                failed: this._xrun.failed
+            };
+            if (fn.length > 1) {
+                // takes two params, pass in context and done callback
+                this._watchFailure(watch);
+                return fn.call(context, context, done);
+            }
+            if (fn.constructor.name !== "AsyncFunction" && fn.length > 0) {
+                // non async function that takes 1 param, check if it wants the context
+                // or a callback.
+                const fnStr = fn.toString().replace(/\s/g, "");
+                const matchParam = x => {
+                    return (fnStr.startsWith(`${x}=>`) ||
+                        fnStr.match(new RegExp(`\\(${x}[^\\)]*\\)=>`)) ||
+                        fnStr.match(new RegExp(`function[^\\(]*\\(${x}[^\\)]*\\){`)) ||
+                        fnStr.match(new RegExp(`[^(]*\\(${x}[^\\)]*\\){`)));
+                };
+                const takeContext = ["ctx", "context"].find(matchParam);
+                if (!takeContext) {
+                    // non async task function takes callback, expect async behavior
+                    this._watchFailure(watch);
+                    return fn.call(context, done);
+                }
+                // function takes context as first param
+                return done(null, fn.call(context, context));
+            }
+            const x = this._handleStream(fn.call(context, context));
+            // handling a exec/spawn child from a function
+            const cmd = `ChildProcess of function task ${qItem.name}`;
+            const child = this._handleChildTask(x && x.child, cmd) || this._handleChildTask(x, cmd);
+            if (!x || typeof x.then !== "function") {
+                // assume no async behavior in task function
+                return done(null, x);
+            }
+            // x.then is a function, assume task function returned a promise
+            this._watchFailure(watch);
+            return x.then(v => done(null, v), err => {
+                // treat child process that got SIGTERM as exited normally
+                if (this._isChildSigTerm(err, child)) {
+                    done();
+                }
+                else {
+                    done(err);
+                }
+            });
+        }
+        catch (err) {
+            return done(err);
+        }
+    }
+    _watchFailure(watch) {
+        if (this._xrun.stopOnError !== "full")
+            return undefined;
+        return setTimeout(() => {
+            if (watch.finish)
+                return 0;
+            if (this._xrun.failed) {
+                this._xrun.emit("fail-cancel", watch);
+                return watch.cancel();
+            }
+            return this._watchFailure(watch);
+        }, 50).unref();
+    }
+    _processMark(qItem) {
+        const value = qItem.value();
+        const xqItem = this._xrun.xqTree.item(value.xqId);
+        const endTime = Date.now();
+        const data = {
+            qItem,
+            xqItem,
+            elapse: endTime - value.startTime,
+            startTime: value.startTime,
+            endTime,
+            hrStartTime: value.hrStartTime,
+            hrElapse: process.hrtime(value.hrStartTime)
+        };
+        if (value.callback) {
+            value.callback(this._xrun.failed, data);
+        }
+        else {
+            this._emit("done-item", data);
+        }
+        const xqValue = xqItem.value();
+        if (xqValue.item && xqValue.item[STAGE_FINALLY] && !xqItem.isFinally) {
+            xqItem.stage = STAGE_FINALLY;
+            xqItem.isFinally = true;
+            this.pushItem(xqItem);
+        }
+        this.next();
+    }
+    _makeXqMarkItem(qItem, callback) {
+        // do not add marking item to tree
+        const mark = new XQItem({
+            name: `mark_${qItem.name}`,
+            value: {
+                startTime: Date.now(),
+                hrStartTime: process.hrtime(),
+                callback,
+                xqId: qItem.id
+            }
+        });
+        mark.mark = true;
+        return mark;
+    }
+    _emit(event, data) {
+        try {
+            this._xrun.emit(event, data);
+        }
+        catch (err) {
+            this._xrun.fail(err);
+        }
+    }
+    _createQItem(options, parent) {
+        return this._xrun.xqTree.create(options, parent);
+    }
+}
+export default XQtor;