@fynjs/run 1.0.0

package/lib/xqtor.js

@@ -0,0 +1,814 @@
+import { createRequire } from "node:module";
+import assert from "assert";
+import defaults from "./defaults.js";
+import Insync from "insync";
+import XQItem from "./xqitem.js";
+import { exec as exec } from "xsh";
+import parseArray from "./util/parse-array.js";
+import childProc from "child_process";
+import updateEnv from "./util/update-env.js";
+import { NixClap } from "@fynjs/cli-args";
+
+//
+// Kept lazy rather than a static import: a static import is hoisted to module load, which is
+// exactly what the call site below avoids so fynpo bootstrap can build unwrap-npm-cmd first.
+//
+const lazyRequire = createRequire(import.meta.url);
+
+const STAGE_FINALLY = "finally";
+
+const isSerial = x => defaults.SERIAL_SIG.includes(x[0]);
+const isConcurrent = x => defaults.CONCURRENT_SIG.includes(x[0]);
+
+const isReadableStream = x => Boolean(x && x.pipe && x.on && x._readableState);
+
+/*
+ * Task executor (XQtor) - After all the fuss, it comes down to this -- the core of xrun
+ *                         that handles the execution of tasks.
+ *
+ * Tasks are added to a stack using an array.
+ *
+ * - Each concurrent tasks array is executed with a new XQtor and Insync.parallel.
+ * - Serial tasks are all added to the top of the "stack" in reverse order
+ * - Some items are actions that cause more items to be created and pushed
+ *   into the stack.  Like looking up the task of a task name.
+ * - As each task is completed, the XQtor's next is invoked to pop items
+ *   from the stack for processing.
+ * - A mark item is added to the stack before each task so when a mark
+ *   is seen, error is checked and done event is emitted.
+ *
+ */
+class XQtor {
+  constructor(options) {
+    this._tasks = options.tasks;
+    this._done = options.done;
+    this._xrun = options.xrun;
+    this.xqItems = [];
+  }
+
+  next(err, xqId) {
+    process.nextTick(() => this._next(err, xqId), 0);
+  }
+
+  _next(err, xqId) {
+    if (err) {
+      const qItem = this._xrun.xqTree.item(xqId);
+      qItem.err = err;
+      this._xrun.fail(err);
+    }
+
+    if (this.xqItems.length > 0) {
+      this.execute();
+    } else {
+      this._done(this._xrun.failed);
+    }
+  }
+
+  execute() {
+    assert(this.xqItems.length > 0, "no next task");
+
+    const qItem = this.popItem();
+
+    if (qItem.mark) {
+      return this._processMark(qItem);
+    }
+
+    if (this._xrun._isStop || (this._xrun.failed && !qItem.isFinally && this._xrun.stopOnError)) {
+      return this.next(null, qItem.id);
+    }
+
+    const value = qItem.value();
+    if (value === defaults.STOP_SYM) {
+      this._xrun.actStop();
+      return this.next(null, qItem.id);
+    }
+
+    const vtype = value.constructor.name;
+    if (vtype === "XTaskSpec") {
+      if (value.type === "exec") {
+        return this._shellXer(qItem, value, true);
+      } else if (value.type === "env") {
+        return this._envXer(qItem, value);
+      }
+      this._xrun.fail(
+        new Error(`Unable to process XTaskSpec type ${value.type} for task ${qItem.name}`)
+      );
+    } else if (vtype === "String") {
+      if (this._isAnonShell(value)) {
+        return this._shellXer(qItem, this._parseAnonShell(value), true);
+      }
+
+      return this._processLookup(qItem);
+    } else if (vtype === "Function" || vtype === "AsyncFunction") {
+      return this._functionXer(qItem, value);
+    } else if (vtype === "Array") {
+      return this._processArray(qItem, value);
+    } else if (value.item) {
+      const dep = value.item.dep;
+      if (dep && !qItem.xqDep) {
+        return this._processDep(qItem, dep);
+      } else {
+        return this._processTaskObject(qItem);
+      }
+    } else {
+      this._xrun.fail(
+        new Error(`Unable to process task ${qItem.name} \
+because value type ${vtype} is unknown and no value.item`)
+      );
+    }
+
+    return this.next(null, qItem.id);
+  }
+
+  pushMarkItem(qItem, callback) {
+    const x = this._makeXqMarkItem(qItem, callback);
+    this.pushItem(x);
+    return x;
+  }
+
+  pushItem(qItem) {
+    this.xqItems.push(qItem);
+  }
+
+  popItem() {
+    return this.xqItems.pop();
+  }
+
+  _isAnonShell(value) {
+    let sig;
+    if ((sig = defaults.ANON_SHELL_OPT_SIG.find(x => value.startsWith(x)))) {
+      return { sig, opt: true };
+    }
+    if ((sig = defaults.ANON_SHELL_SIG.find(x => value.startsWith(x)))) {
+      return { sig, opt: false };
+    }
+    return false;
+  }
+
+  _parseShellFlags(flags, value) {
+    const unknowns = [];
+
+    flags = flags.reduce((a, o) => {
+      const f = o.trim().toLowerCase();
+      if (defaults.SHELL_FLAGS.indexOf(f) < 0) {
+        unknowns.push(o);
+      } else {
+        a[f] = true;
+      }
+      return a;
+    }, {});
+
+    if (unknowns.length) {
+      throw new Error(`Unknown flag ${unknowns.join(",")} in shell task: ${value}`);
+    }
+
+    return flags;
+  }
+
+  _parseAnonShell(value) {
+    if (typeof value !== "string") return value;
+
+    let cmd = value;
+    let flags = {};
+    let error;
+    const anon = this._isAnonShell(value);
+    if (anon) {
+      const { sig, opt } = anon;
+      // support options like ~(tty,spawn,sync)$
+
+      if (opt) {
+        let ix;
+        const closeSig = defaults.ANON_SHELL_OPT_CLOSE_SIG.find(x => {
+          ix = value.indexOf(x, sig.length);
+          return ix > sig.length;
+        });
+
+        if (closeSig) {
+          const so = value.substring(sig.length, ix);
+          try {
+            flags = this._parseShellFlags(so.split(","), value);
+          } catch (err) {
+            error = err;
+          }
+          cmd = value.substr(ix + closeSig.length);
+        } else {
+          error = new Error(
+            `Missing ${defaults.ANON_SHELL_OPT_CLOSE_SIG[0]} in shell task: ${value}`
+          );
+        }
+      } else {
+        cmd = value.substr(sig.length);
+      }
+    }
+
+    return { flags, cmd, error };
+  }
+
+  _isStrArray(value) {
+    return value.startsWith(defaults.STR_ARRAY_SIG);
+  }
+
+  _parseStrArray(value) {
+    if (this._isStrArray(value)) {
+      return parseArray(value.substr(defaults.STR_ARRAY_SIG.length - 1));
+    }
+    return undefined;
+  }
+
+  _processLookup(qItem) {
+    this._emit("execute", {
+      type: "lookup",
+      qItem
+    });
+    // lookup is sync; should not insert mark for it
+    try {
+      const found = qItem.lookup(this._tasks);
+      this.pushItem(qItem);
+      this.next(null, qItem.id);
+      if (found.search) {
+        this._xrun.emit("search", { qItem, found });
+        qItem.setNamespace(found.ns);
+      }
+    } catch (error) {
+      if (error.optional) {
+        this._xrun.emit("not-found", error);
+        this.next();
+      } else {
+        this.next(error, qItem.id);
+      }
+    }
+  }
+
+  _processDep(qItem, dep) {
+    this._emit("dep", {
+      qItem
+    });
+    qItem.xqDep = true; // item's dep has been processed
+    this.pushItem(qItem);
+    const di = this._createQItem(
+      {
+        name: qItem.name,
+        value: { top: true, depDee: qItem, item: { task: dep } }
+      },
+      this._xrun.xqTree.parent(qItem)
+    );
+
+    this.pushItem(di);
+
+    this.next(null, qItem.id);
+  }
+
+  _processTaskObject(qItem) {
+    const value = qItem.value();
+    const stage = qItem.stage || "task";
+    const itemTask = value.item[stage] || value.item;
+    const type = itemTask.constructor.name;
+    if (type === "Array") {
+      return this._processArray(qItem, itemTask, value.top);
+    } else if (type === "Function" || type === "AsyncFunction") {
+      return this._functionXer(qItem, itemTask);
+    } else if (type === "String") {
+      const parsedArray = this._parseStrArray(itemTask);
+      if (parsedArray) return this._processArray(qItem, parsedArray, value.top);
+      return this._shellXer(qItem, itemTask);
+    } else if (type === "XTaskSpec") {
+      if (itemTask.type === "exec") {
+        return this._shellXer(qItem, itemTask);
+      } else if (itemTask.type === "env") {
+        return this._envXer(qItem, itemTask);
+      }
+      this._xrun.fail(
+        new Error(`Unable to process XTaskSpec type ${itemTask.type} for task ${qItem.name}`)
+      );
+    } else {
+      this._xrun.fail(new Error(`Task ${qItem.name} has unrecognize task value type ${type}`));
+    }
+
+    return this.next(null, qItem.id);
+  }
+
+  _processArray(qItem, tasks, top) {
+    // check first element for concurrent signature
+    // top level task arrays are automatically serial
+    // or check first element for serial signature
+    if (isConcurrent(tasks) || (!top && !isSerial(tasks))) {
+      this._concurrentArrayXer(qItem, tasks);
+    } else {
+      this._serialArrayXer(qItem, tasks);
+    }
+  }
+
+  _serialArrayXer(qItem, tasks) {
+    if (isSerial(tasks)) {
+      tasks = tasks.slice(1);
+    }
+
+    this._emit("execute", {
+      type: "serial-arr",
+      qItem,
+      array: tasks
+    });
+
+    this.pushMarkItem(qItem);
+    this.xqItems = this.xqItems.concat(
+      tasks
+        .map(value => {
+          return this._createQItem(
+            this._resolveValueToQItemOptions(qItem.ns, `${qItem.name}.S`, value, "serial_child"),
+            qItem
+          );
+        })
+        .reverse()
+    );
+    this.next(null, qItem.id);
+  }
+
+  _resolveValueToQItemOptions(ns, name, value, type) {
+    if (typeof value === "string" && !this._isAnonShell(value)) {
+      name = value;
+      value = undefined;
+    }
+
+    const anon = typeof value === "function";
+    return {
+      ns,
+      name,
+      value,
+      anon,
+      type
+    };
+  }
+
+  _concurrentArrayXer(qItem, tasks) {
+    if (isConcurrent(tasks)) {
+      tasks = tasks.slice(1);
+    }
+
+    this._emit("execute", {
+      type: "concurrent-arr",
+      qItem,
+      array: tasks
+    });
+
+    this.pushMarkItem(qItem);
+    Insync.parallel(
+      tasks.map(value => cb => {
+        this._emit("spawn-async", { name: qItem.name });
+        const xqtor = new XQtor({
+          tasks: this._tasks,
+          xrun: this._xrun,
+          done: () => {
+            this._emit("done-async");
+            process.nextTick(() => cb());
+          }
+        });
+        const options = this._resolveValueToQItemOptions(
+          qItem.ns,
+          `${qItem.name}.C`,
+          value,
+          "concurrent_child"
+        );
+        const item = this._createQItem(options, qItem);
+        if (options.value === undefined) {
+          xqtor._processLookup(item);
+        } else {
+          xqtor.pushItem(item);
+          xqtor.next();
+        }
+      }),
+      () => this.next()
+    );
+  }
+
+  _envXer(qItem, cmdVal) {
+    this._emit("execute", {
+      type: "env",
+      qItem,
+      cmdVal
+    });
+
+    updateEnv(cmdVal.options.env, process.env, cmdVal.options.override);
+    return this.next();
+  }
+
+  _shellXer(qItem, cmdVal, anon) {
+    cmdVal = this._parseAnonShell(cmdVal);
+
+    if (cmdVal.error) {
+      setTimeout(() => this.next(cmdVal.error, qItem.id), 0);
+      return;
+    }
+
+    const { cmd } = cmdVal;
+    const xrunOptions = Object.assign({ delayRunMs: 0 }, cmdVal.xrun || cmdVal.xclap);
+    const options = Object.assign({}, cmdVal.options);
+
+    let { flags } = cmdVal;
+
+    if (typeof flags === "string") {
+      flags = this._parseShellFlags(flags.split(","), `XTaskSpec "${flags}"`);
+    } else if (Array.isArray(flags)) {
+      flags = this._parseShellFlags(flags);
+    }
+
+    const item = qItem.value().item || {};
+
+    Object.assign(flags, item.flags);
+
+    // Access CLI context through runner
+    const cliContext = this._xrun.getCliContext();
+    const itemArgv = qItem.argv;
+    const cliArgv = cliContext.getTaskArgv(qItem.name);
+    const remainingArgv =
+      (cliContext.isLastTask(qItem.name) && cliContext.getRemainingArgs()) || [];
+
+    // Build command with task-specific argv
+    const cmd2 = []
+      .concat(itemArgv.slice(1), cliArgv.slice(1), remainingArgv)
+      .map(x => (x && x.includes(" ") ? `"${x}"` : x))
+      .join(" ");
+
+    setTimeout(() => {
+      const execData = {
+        type: "shell",
+        anon,
+        qItem,
+        cmd: cmd + ((cmd2 && " " + cmd2) || ""),
+        cmd2,
+        flags,
+        cmdVal,
+        options,
+        item
+      };
+
+      this._emit("execute", execData);
+      this._doShellXer(execData);
+    }, xrunOptions.delayRunMs);
+  }
+
+  _doShellXer({ /*anon,*/ qItem, cmd, flags, /*cmdVal,*/ options, item }) {
+    const itemOptions = Object.assign({}, item.options);
+
+    this.pushMarkItem(qItem);
+    const env = Object.assign(
+      flags.noenv ? {} : Object.assign({}, process.env),
+      itemOptions.env,
+      options.env
+    );
+
+    if (qItem.err) {
+      env.XCLAP_ERR = env.XRUN_ERR = qItem.err.message || "true";
+    }
+
+    if (this._xrun.failed || qItem.err) {
+      env.XCLAP_FAILED = env.XRUN_FAILED = "true";
+    }
+
+    let child;
+
+    const watch = { finish: false };
+    const done = err => {
+      // even if there's error, but if it's due to child being terminated with
+      // SIGTERM, then treat that as a normal exit.
+      if (this._isChildSigTerm(err, child) || this._isChildTerminated(child)) {
+        err = null;
+      }
+      //
+      // A spawned child can report both `error` and `close`, so `done` can be called twice.
+      // Guarding it is the point; reproducing that race on demand in a test is not something
+      // this suite can do deterministically.
+      //
+      /* istanbul ignore next */
+      if (watch.finish) return 0;
+      watch.finish = true;
+      return this.next(err, qItem.id);
+    };
+
+    let { tty, spawn, sync, npm } = flags;
+    if (npm) {
+      tty = spawn = true;
+    }
+
+    // Lazy require unwrap-npm-cmd to avoid requiring it at module load time
+    // This ensures it's built by fynpo bootstrap before it's required
+    const { unwrapNpmCmd } = lazyRequire("unwrap-npm-cmd");
+    const cmd2 = unwrapNpmCmd(cmd, { path: env.PATH });
+
+    if (tty || spawn) {
+      const spawnOpts = { shell: true };
+      Object.assign(spawnOpts, itemOptions, options, { env });
+      if (tty) spawnOpts.stdio = "inherit";
+
+      if (sync) {
+        child = childProc.spawnSync(cmd2, spawnOpts);
+        if (child.error) {
+          done(child.error);
+        } else if (child.status === 0) {
+          done();
+        } else {
+          done(new Error(`cmd "${cmd}" exit code ${child.status}`));
+        }
+      } else {
+        child = childProc.spawn(cmd2, spawnOpts);
+        child.on("close", (code, signal) => {
+          if (code === 0 || signal === "SIGTERM") {
+            return done();
+          }
+          return done(new Error(`cmd "${cmd}" exit code ${code}`));
+        });
+      }
+    } else {
+      child = exec(Object.assign({ silent: false }, itemOptions, options, { env }), cmd2, done);
+    }
+
+    if (!watch.finish) {
+      this._handleChildTask(child, cmd);
+      watch.cancel = () => {
+        this._xrun.killChildProcess(child);
+        done();
+      };
+      this._watchFailure(watch);
+    }
+  }
+
+  _handleChildTask(child, cmd) {
+    if (child && child instanceof childProc.ChildProcess) {
+      const sym = Symbol(cmd);
+      this._xrun.addTaskChild(child, sym);
+      child.on("exit", () => {
+        this._xrun.removeTaskChild(child, sym);
+      });
+      return child;
+    }
+  }
+
+  _processMoreFromFn(qItem, value, callback) {
+    let tof = typeof value;
+
+    if (tof === "string") {
+      const parsedArray = this._parseStrArray(value);
+      if (parsedArray) value = parsedArray;
+      tof = typeof value;
+    }
+
+    if (
+      !value ||
+      (tof !== "string" &&
+        tof !== "function" &&
+        !Array.isArray(value) &&
+        value.constructor.name !== "XTaskSpec" &&
+        value !== defaults.STOP_SYM)
+    ) {
+      return;
+    }
+
+    const qi = this._createQItem(
+      this._resolveValueToQItemOptions(qItem.ns, `${qItem.name}.fR`, value, "func_returned_child"),
+      qItem
+    );
+    if (callback) {
+      assert(
+        typeof callback === "function",
+        `${qItem.name} callback from function calling run is not a function`
+      );
+      this.pushMarkItem(qItem, callback);
+    }
+    this.pushItem(qi);
+  }
+
+  _isChildSigTerm(err, child) {
+    return err && err.code === null && child && child.signalCode === "SIGTERM";
+  }
+
+  _isChildTerminated(child) {
+    return child && child[defaults.STOP_SYM];
+  }
+
+  _handleStream(val) {
+    if (!isReadableStream(val)) {
+      return val;
+    }
+
+    return new Promise((resolve, reject) => {
+      const handle = err => {
+        val.removeListener("end", handle);
+        val.removeListener("error", handle);
+        return err ? reject(err) : resolve();
+      };
+
+      val.once("close", handle);
+      val.once("error", handle);
+    });
+  }
+
+  _functionXer(qItem, fn) {
+    this._emit("execute", {
+      type: "function",
+      anon: qItem.anon,
+      qItem
+    });
+
+    const watch = { finish: false, qItem };
+    const done = (err, value) => {
+      if (watch.finish) return;
+      watch.finish = true;
+      if (value) {
+        this._processMoreFromFn(qItem, value);
+      }
+      this.next(err, qItem.id);
+    };
+
+    watch.cancel = done;
+
+    try {
+      this.pushMarkItem(qItem);
+      const run = (value, cb) => this._processMoreFromFn(qItem, value, cb);
+      const task = qItem.value();
+
+      // Access CLI context through runner
+      const cliContext = this._xrun.getCliContext();
+      let cliCmd = cliContext.getTaskCommand(qItem.name);
+
+      const cliParser = task.item?.cliParser || task.cliParser || {};
+      const itemArgv = qItem.argv;
+      const cliArgv = cliCmd.argv || [];
+
+      if (itemArgv.length > 1 || (cliParser && cliArgv.length > 1)) {
+        const argv = [qItem.name].concat(itemArgv.slice(1), cliArgv.slice(1));
+        const config = {
+          name: qItem.name,
+          allowUnknownOption:
+            cliParser.allowUnknownOption !== undefined ? cliParser.allowUnknownOption : true,
+          allowUnknownCommand: false,
+          noDefaultHandlers: true
+        };
+
+        const argp = new NixClap(config)
+          .init(cliParser.options || {}, cliParser.commands || {})
+          .parse(argv, 1);
+
+        if (argp.errorNodes.length > 0) {
+          const unknownOptions = argp.errorNodes.reduce((acc, node) => {
+            acc.push(
+              ...node.errors
+                .filter(e => e.message.includes("unknown CLI option"))
+                .map(e => e.data.name)
+            );
+            return acc;
+          }, []);
+
+          if (unknownOptions.length > 0) {
+            return done(
+              new Error(`Unknown options for task ${qItem.name}: ${unknownOptions.join(", ")}`)
+            );
+          }
+        }
+
+        cliCmd = argp.command;
+      }
+
+      const argv = cliCmd.argv || [];
+
+      const context = {
+        run,
+        argv,
+        cliCmd,
+        argp: cliCmd.jsonMeta,
+        args: cliCmd.argsList,
+        argOpts: cliCmd.opts,
+        err: qItem.err,
+        failed: this._xrun.failed
+      };
+
+      if (fn.length > 1) {
+        // takes two params, pass in context and done callback
+        this._watchFailure(watch);
+        return fn.call(context, context, done);
+      }
+
+      if (fn.constructor.name !== "AsyncFunction" && fn.length > 0) {
+        // non async function that takes 1 param, check if it wants the context
+        // or a callback.
+        const fnStr = fn.toString().replace(/\s/g, "");
+        const matchParam = x => {
+          return (
+            fnStr.startsWith(`${x}=>`) ||
+            fnStr.match(new RegExp(`\\(${x}[^\\)]*\\)=>`)) ||
+            fnStr.match(new RegExp(`function[^\\(]*\\(${x}[^\\)]*\\){`)) ||
+            fnStr.match(new RegExp(`[^(]*\\(${x}[^\\)]*\\){`))
+          );
+        };
+
+        const takeContext = ["ctx", "context"].find(matchParam);
+
+        if (!takeContext) {
+          // non async task function takes callback, expect async behavior
+          this._watchFailure(watch);
+          return fn.call(context, done);
+        }
+
+        // function takes context as first param
+        return done(null, fn.call(context, context));
+      }
+
+      const x = this._handleStream(fn.call(context, context));
+
+      // handling a exec/spawn child from a function
+      const cmd = `ChildProcess of function task ${qItem.name}`;
+      const child = this._handleChildTask(x && x.child, cmd) || this._handleChildTask(x, cmd);
+
+      if (!x || typeof x.then !== "function") {
+        // assume no async behavior in task function
+        return done(null, x);
+      }
+
+      // x.then is a function, assume task function returned a promise
+      this._watchFailure(watch);
+      return x.then(
+        v => done(null, v),
+        err => {
+          // treat child process that got SIGTERM as exited normally
+          if (this._isChildSigTerm(err, child)) {
+            done();
+          } else {
+            done(err);
+          }
+        }
+      );
+    } catch (err) {
+      return done(err);
+    }
+  }
+
+  _watchFailure(watch) {
+    if (this._xrun.stopOnError !== "full") return undefined;
+
+    return setTimeout(() => {
+      if (watch.finish) return 0;
+      if (this._xrun.failed) {
+        this._xrun.emit("fail-cancel", watch);
+        return watch.cancel();
+      }
+      return this._watchFailure(watch);
+    }, 50).unref();
+  }
+
+  _processMark(qItem) {
+    const value = qItem.value();
+    const xqItem = this._xrun.xqTree.item(value.xqId);
+    const endTime = Date.now();
+    const data = {
+      qItem,
+      xqItem,
+      elapse: endTime - value.startTime,
+      startTime: value.startTime,
+      endTime,
+      hrStartTime: value.hrStartTime,
+      hrElapse: process.hrtime(value.hrStartTime)
+    };
+    if (value.callback) {
+      value.callback(this._xrun.failed, data);
+    } else {
+      this._emit("done-item", data);
+    }
+
+    const xqValue = xqItem.value();
+
+    if (xqValue.item && xqValue.item[STAGE_FINALLY] && !xqItem.isFinally) {
+      xqItem.stage = STAGE_FINALLY;
+      xqItem.isFinally = true;
+      this.pushItem(xqItem);
+    }
+
+    this.next();
+  }
+
+  _makeXqMarkItem(qItem, callback) {
+    // do not add marking item to tree
+    const mark = new XQItem({
+      name: `mark_${qItem.name}`,
+      value: {
+        startTime: Date.now(),
+        hrStartTime: process.hrtime(),
+        callback,
+        xqId: qItem.id
+      }
+    });
+    mark.mark = true;
+    return mark;
+  }
+
+  _emit(event, data) {
+    try {
+      this._xrun.emit(event, data);
+    } catch (err) {
+      this._xrun.fail(err);
+    }
+  }
+
+  _createQItem(options, parent) {
+    return this._xrun.xqTree.create(options, parent);
+  }
+}
+
+export default XQtor;