clyops-jobs 0.2.0 → 0.3.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Dan VerWeire
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -6,9 +6,13 @@ artifacts, and each run gets a job record. A small in-memory queue runs jobs wit
 limit, for servers that hand out job ids. Built on [clyops-tools](../tools).
 
 ```sh
-npm install clyops-jobs
+pnpm add clyops-jobs
 ```
 
+**Single file.** `clyops-jobs.cjs` (in this package, and on npm) is the whole library as one dependency-free
+CommonJS file: copy or link it into a project and `require()` it on Node 20+, with no build step
+and no `node_modules`. It includes clyops-tools. `require('clyops-jobs')` loads it; `import` loads the ESM build. It is generated from the sources and checked in CI.
+
 ## Function config
 
 ```json
@@ -27,6 +31,7 @@ npm install clyops-jobs
 - `key` names the options block (default: the function name). Options are
   `deepMerge(defaults[key], config[key])`, keyed like the tool's options (`out_dir`, `out-dir`)
   and mapped onto its command line through `--help-json-schema` ([spec §13](../../spec/SPEC.md#13-json-input-toargv)).
+- `positionals_order` (optional) is `first` (default) or `last`, for wrappers needing a specific order. `--` is only emitted for positional values starting with `-`.
 - `positionals` (optional) lists the tool's positional arguments explicitly; otherwise they're
   options keyed by argument name.
 - String values may use `${dot.path}` or `{{dot.path}}` templates against a context you build
@@ -84,3 +89,7 @@ queue.get(job.record.job_id)?.cancel();
 ```
 
 ESM only, Node 20+.
+
+A missing or unreadable configured stdin file rejects `runScriptFunction()`
+with its input-file error. It fails that job and stops its tool; the host worker
+stays alive.

package/clyops-jobs.cjs

@@ -0,0 +1,835 @@
+// Generated by tools/bundle-js.mjs from packages/jobs/src/index.ts. Do not edit.
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// packages/jobs/src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  JobQueue: () => JobQueue,
+  artifactIncludePatterns: () => artifactIncludePatterns,
+  artifactRequirement: () => artifactRequirement,
+  buildFunctionCommand: () => buildFunctionCommand,
+  buildResultRecord: () => buildResultRecord,
+  configureLogging: () => configureLogging,
+  copyConfiguredArtifacts: () => copyConfiguredArtifacts,
+  copyFileEnsuringDir: () => copyFileEnsuringDir,
+  deepMerge: () => deepMerge,
+  ensureDir: () => ensureDir,
+  findFilesByArtifactPattern: () => findFilesByArtifactPattern,
+  globToRegex: () => globToRegex,
+  isDirectoryPath: () => isDirectoryPath,
+  isEnabled: () => isEnabled,
+  isFilePath: () => isFilePath,
+  listFilesRecursive: () => listFilesRecursive,
+  lookupTemplateValue: () => lookupTemplateValue,
+  newJobId: () => newJobId,
+  newJobRecord: () => newJobRecord,
+  objectValue: () => objectValue,
+  pathParts: () => pathParts,
+  readJson: () => readJson,
+  renderConfigValue: () => renderConfigValue,
+  renderResultMap: () => renderResultMap,
+  renderResultValue: () => renderResultValue,
+  renderTemplateString: () => renderTemplateString,
+  resolveConfigPath: () => resolveConfigPath,
+  resolveFunctionConfig: () => resolveFunctionConfig,
+  resolveScriptPath: () => resolveScriptPath,
+  runScriptFunction: () => runScriptFunction,
+  shellQuote: () => shellQuote,
+  statFollowingSymlinks: () => statFollowingSymlinks,
+  timestampIso: () => timestampIso,
+  transitionJobRecord: () => transitionJobRecord,
+  truthy: () => truthy,
+  writeJson: () => writeJson
+});
+module.exports = __toCommonJS(index_exports);
+
+// packages/jobs/src/util.ts
+var import_node_fs = require("node:fs");
+var import_node_path = require("node:path");
+function timestampIso() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+function isEnabled(value) {
+  return value !== "" && value !== "disabled" && value !== "false";
+}
+function truthy(value) {
+  return /^(true|1|yes|on)$/i.test(String(value));
+}
+function ensureDir(dir) {
+  (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+}
+function readJson(file) {
+  return JSON.parse((0, import_node_fs.readFileSync)(file, "utf8"));
+}
+function writeJson(file, value) {
+  (0, import_node_fs.writeFileSync)(file, `${JSON.stringify(value, null, 2)}
+`, "utf8");
+}
+function statFollowingSymlinks(file) {
+  try {
+    return (0, import_node_fs.statSync)(file);
+  } catch {
+    return null;
+  }
+}
+function isDirectoryPath(file) {
+  return Boolean(statFollowingSymlinks(file)?.isDirectory());
+}
+function isFilePath(file) {
+  return Boolean(statFollowingSymlinks(file)?.isFile());
+}
+function copyFileEnsuringDir(source, destination) {
+  ensureDir((0, import_node_path.dirname)(destination));
+  (0, import_node_fs.copyFileSync)(source, destination);
+}
+function pathParts(file) {
+  const name = (0, import_node_path.basename)(file);
+  const ext = (0, import_node_path.extname)(name);
+  return { path: file, dir: (0, import_node_path.dirname)(file), name, basename: ext ? name.slice(0, -ext.length) : name, ext: ext.replace(/^\./, "") };
+}
+function objectValue(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+function deepMerge(base, override) {
+  if (!base || typeof base !== "object" || Array.isArray(base)) return override;
+  if (!override || typeof override !== "object" || Array.isArray(override)) return override;
+  const out = { ...base };
+  for (const [key, value] of Object.entries(override)) out[key] = key in out ? deepMerge(out[key], value) : value;
+  return out;
+}
+function resolveConfigPath(root, value) {
+  if (!value || value === "disabled" || value === "false" || value === "-") return value || "";
+  if ((0, import_node_path.isAbsolute)(value) || /^[a-z][a-z0-9+.-]*:\/\//i.test(value)) return value;
+  return (0, import_node_path.resolve)(root, value);
+}
+var resolveScriptPath = resolveConfigPath;
+
+// packages/jobs/src/template.ts
+function lookupTemplateValue(context, expression) {
+  let current = context;
+  for (const part of String(expression).split(".").filter(Boolean)) {
+    if (!current || typeof current !== "object" || !Object.prototype.hasOwnProperty.call(current, part)) {
+      throw new Error(`unknown template value: ${expression}`);
+    }
+    current = current[part];
+  }
+  if (current === void 0 || current === null) return "";
+  return typeof current === "object" ? JSON.stringify(current) : String(current);
+}
+function renderTemplateString(value, context) {
+  return String(value).replace(
+    /\$\{\s*([A-Za-z0-9_.-]+)\s*\}|\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g,
+    (_, dollar, brace) => lookupTemplateValue(context, dollar || brace)
+  );
+}
+function renderConfigValue(value, context) {
+  if (!context) return value;
+  if (typeof value === "string") return renderTemplateString(value, context);
+  if (Array.isArray(value)) return value.map((item) => renderConfigValue(item, context));
+  return value;
+}
+function renderResultValue(value, context) {
+  if (typeof value === "string") return renderTemplateString(value, context);
+  if (Array.isArray(value)) return value.map((item) => renderResultValue(item, context));
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, renderResultValue(v, context)]));
+  }
+  return value;
+}
+function renderResultMap(resultMap, context) {
+  if (!resultMap || typeof resultMap !== "object" || Array.isArray(resultMap)) return {};
+  return renderResultValue(resultMap, context);
+}
+
+// packages/tools/src/schema.ts
+var import_node_child_process = require("node:child_process");
+var import_node_fs2 = require("node:fs");
+function stamp(file) {
+  const s = (0, import_node_fs2.statSync)(file);
+  return `${s.mtimeMs}:${s.size}`;
+}
+var cache = /* @__PURE__ */ new Map();
+function loadSchema(file, opts = {}) {
+  const key = stamp(file);
+  const hit = cache.get(file);
+  if (hit && hit.stamp === key) return hit.schema;
+  const schema = new Promise((resolve5, reject) => {
+    (0, import_node_child_process.execFile)(file, ["--help-json-schema"], { cwd: opts.cwd, timeout: opts.timeoutMs ?? 5e3, maxBuffer: 16 << 20 }, (err, stdout) => {
+      if (err) return reject(new Error(`${file} does not provide --help-json-schema: ${err.message}`));
+      const start2 = stdout.indexOf("{");
+      const end = stdout.lastIndexOf("}");
+      if (start2 < 0 || end < start2) return reject(new Error(`${file} produced no JSON schema`));
+      try {
+        const parsed = JSON.parse(stdout.slice(start2, end + 1));
+        if (parsed.clyops !== 1 || !Array.isArray(parsed.options) || !Array.isArray(parsed.arguments) || typeof parsed.description !== "string") {
+          throw new Error("expected a clyops: 1 schema with description, options and arguments");
+        }
+        resolve5(parsed);
+      } catch (e) {
+        reject(new Error(`${file}: invalid JSON schema: ${e.message}`));
+      }
+    });
+  });
+  schema.catch(() => cache.delete(file));
+  cache.set(file, { stamp: key, schema });
+  return schema;
+}
+
+// packages/tools/src/argv.ts
+var import_node_fs3 = require("node:fs");
+var import_node_path2 = require("node:path");
+var InputError = class extends Error {
+  status = 400;
+};
+function inputKey(name) {
+  return name.replace(/-/g, "_");
+}
+function inputKeys(item) {
+  const variable = "variableName" in item ? item.variableName : item.name;
+  return [.../* @__PURE__ */ new Set([inputKey(item.name), item.name, variable.toLowerCase()])].filter(Boolean);
+}
+function isBareFlag(option) {
+  const choices = [...option.choices].sort().join(",");
+  return option.isFlag && choices !== "false,true";
+}
+function isPathValued(validation, type) {
+  return type === "path" || validation === "path" || validation.startsWith("file:") || validation.startsWith("dir:");
+}
+var TRUE = /^(true|1|yes|on)$/i;
+var FALSE = /^(false|0|no|off)$/i;
+function toArgv(schema, input, opts = {}) {
+  const used = /* @__PURE__ */ new Set();
+  const out = { argv: [], env: {}, unknown: [], controlled: [] };
+  const pick = (item) => {
+    for (const key of inputKeys(item)) {
+      if (Object.prototype.hasOwnProperty.call(input, key)) {
+        used.add(key);
+        return { key, value: input[key] };
+      }
+    }
+    return void 0;
+  };
+  const text = (value, pathValued, name = "") => {
+    let s = typeof value === "object" ? JSON.stringify(value) : String(value);
+    if (opts.render) s = opts.render(s);
+    if (pathValued && opts.base && !["", "-", "disabled", "false"].includes(s) && !(0, import_node_path2.isAbsolute)(s) && !/^[a-z][a-z0-9+.-]*:\/\//i.test(s)) {
+      s = (0, import_node_path2.resolve)(opts.base, s);
+    }
+    if (pathValued && opts.within) checkWithin(name, s, opts.within, opts.base ?? opts.cwd ?? process.cwd());
+    return s;
+  };
+  for (const option of schema.options) {
+    const found = pick(option);
+    if (!found) continue;
+    if (opts.controlled?.includes(option.name)) {
+      out.controlled.push(found.key);
+      continue;
+    }
+    if (found.value === null || found.value === void 0) continue;
+    if (isBareFlag(option)) {
+      if (!["boolean", "string", "number"].includes(typeof found.value)) {
+        throw new InputError(`--${option.name} must be a boolean (true/false, yes/no, 1/0, on/off)`);
+      }
+      const value = text(found.value, false);
+      if (!TRUE.test(value) && !FALSE.test(value)) {
+        throw new InputError(`--${option.name} must be a boolean (true/false, yes/no, 1/0, on/off), got '${value}'`);
+      }
+      out.argv.push(TRUE.test(value) ? `--${option.name}` : `--no-${option.name}`);
+      continue;
+    }
+    if (opts.secretEnv && option.secret && !option.isArray) {
+      out.env[option.variableName] = text(found.value, isPathValued(option.validation, option.type), `--${option.name}`);
+      continue;
+    }
+    for (const item of Array.isArray(found.value) ? found.value : [found.value]) {
+      out.argv.push(`--${option.name}`, text(item, isPathValued(option.validation, option.type), `--${option.name}`));
+    }
+  }
+  const explicit = Array.isArray(opts.positionals) ? opts.positionals : void 0;
+  const order = typeof opts.positionals === "string" ? opts.positionals : opts.positionalsOrder ?? "first";
+  if (order !== "first" && order !== "last") throw new InputError("positionals order must be first or last");
+  const positionals = [...explicit ?? []];
+  if (!explicit) {
+    let skipped = [];
+    for (const argument of schema.arguments) {
+      const found = pick(argument);
+      if (!found || found.value === null || found.value === void 0) {
+        skipped.push(argument);
+        continue;
+      }
+      for (const gap of skipped) {
+        if (!gap.default) throw new Error(`${argument.name} is given, so ${gap.name} must be too`);
+        positionals.push(text(gap.default, false));
+      }
+      skipped = [];
+      const values = Array.isArray(found.value) ? found.value : [found.value];
+      positionals.push(...values.map((v) => text(v, isPathValued(argument.validation), argument.name)));
+    }
+  }
+  if (positionals.some((p) => p.startsWith("-"))) out.argv.push("--", ...positionals);
+  else if (order === "last") out.argv.push(...positionals);
+  else out.argv.unshift(...positionals);
+  out.unknown = Object.keys(input).filter((k) => !used.has(k));
+  return out;
+}
+function realish(p) {
+  let head = p;
+  const tail2 = [];
+  while (!(0, import_node_fs3.existsSync)(head) && (0, import_node_path2.dirname)(head) !== head) {
+    tail2.unshift(head.slice((0, import_node_path2.dirname)(head).length).replace(/^[\\/]/, ""));
+    head = (0, import_node_path2.dirname)(head);
+  }
+  return (0, import_node_path2.resolve)((0, import_node_fs3.existsSync)(head) ? (0, import_node_fs3.realpathSync)(head) : head, ...tail2);
+}
+function checkWithin(name, value, dirs, base) {
+  if (["", "-", "disabled", "false"].includes(value)) return;
+  if (/^[a-z][a-z0-9+.-]+:/i.test(value) && !(0, import_node_path2.isAbsolute)(value)) {
+    throw new InputError(`${name}: ${value} is not a path inside the allowed directories`);
+  }
+  const target = realish((0, import_node_path2.resolve)(base, value));
+  const inside = dirs.some((dir) => {
+    const rel = (0, import_node_path2.relative)(realish((0, import_node_path2.resolve)(dir)), target);
+    return rel === "" || !rel.startsWith("..") && !(0, import_node_path2.isAbsolute)(rel);
+  });
+  if (!inside) throw new InputError(`${name}: ${value} is outside the allowed directories`);
+}
+function redactArgv(schema, argv) {
+  const secret = new Set(schema.options.filter((o) => o.secret).map((o) => `--${o.name}`));
+  const out = [...argv];
+  for (let i = 0; i < out.length; i++) {
+    if (out[i] === "--") break;
+    const eq = out[i].indexOf("=");
+    if (eq > 0 && secret.has(out[i].slice(0, eq))) out[i] = `${out[i].slice(0, eq)}=***`;
+    else if (secret.has(out[i]) && i + 1 < out.length) out[++i] = "***";
+  }
+  return out;
+}
+
+// packages/tools/src/run.ts
+var import_node_child_process2 = require("node:child_process");
+var import_node_stream = require("node:stream");
+var import_node_string_decoder = require("node:string_decoder");
+var ABORT_GRACE_MS = 200;
+var STOP_WAIT_MS = 1e3;
+function terminateTree(child, signal) {
+  const pid = child.pid;
+  if (!pid || pid <= 0) return Promise.resolve();
+  if (process.platform === "win32") {
+    return new Promise((resolve5) => {
+      (0, import_node_child_process2.execFile)("taskkill", ["/PID", String(pid), "/T", "/F"], { windowsHide: true, timeout: 500 }, (err) => {
+        if (err && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
+        resolve5();
+      });
+    });
+  }
+  try {
+    process.kill(-pid, signal);
+  } catch (err) {
+    if (err.code !== "ESRCH") child.kill(signal);
+  }
+  return Promise.resolve();
+}
+function start(file, argv, opts = {}) {
+  const started = Date.now();
+  const child = (0, import_node_child_process2.spawn)(file, argv, {
+    cwd: opts.cwd,
+    env: opts.env ?? process.env,
+    stdio: [opts.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe"],
+    // POSIX detached children lead a new session/group, still with owned pipes.
+    detached: process.platform !== "win32",
+    windowsHide: true
+  });
+  const limit = opts.maxOutput || Infinity;
+  let truncated = false;
+  const collect = (stream, onChunk) => {
+    const chunks = [];
+    const decoder = new import_node_string_decoder.StringDecoder("utf8");
+    let size = 0;
+    stream.on("data", (chunk) => {
+      if (onChunk) onChunk(decoder.write(chunk));
+      if (size >= limit) return void (truncated = true);
+      if (size + chunk.length > limit) {
+        chunk = chunk.subarray(0, limit - size);
+        truncated = true;
+      }
+      chunks.push(chunk);
+      size += chunk.length;
+    });
+    return () => Buffer.concat(chunks);
+  };
+  const [childOut, childErr] = [child.stdout, child.stderr];
+  const mode = opts.stdout ?? "text";
+  const stdout = mode === "stream" ? () => Buffer.alloc(0) : collect(childOut, opts.onStdout);
+  const stderr = collect(childErr, opts.onStderr);
+  const result = new Promise((resolve5, reject) => {
+    let timedOut = false;
+    let failure;
+    let exited = false;
+    let settled = false;
+    let stopping;
+    let timer;
+    let escalation;
+    let deadline;
+    let treeStopped;
+    let inputClosed = false;
+    const source = opts.stdin instanceof import_node_stream.Readable ? opts.stdin : void 0;
+    const forgetSource = () => {
+      source?.removeListener("error", inputError);
+      source?.removeListener("close", forgetSource);
+    };
+    const closeInput = () => {
+      if (inputClosed) return;
+      inputClosed = true;
+      if (source) {
+        source.unpipe(child.stdin ?? void 0);
+        if (source.closed) forgetSource();
+        else {
+          source.once("close", forgetSource);
+          source.destroy();
+        }
+      }
+      child.stdin?.destroy();
+    };
+    const finish = async (exitCode, signal) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      clearTimeout(escalation);
+      clearTimeout(deadline);
+      opts.signal?.removeEventListener("abort", abort);
+      closeInput();
+      if (stopping && process.platform !== "win32") treeStopped = terminateTree(child, "SIGKILL");
+      await treeStopped;
+      if (failure) {
+        reject(failure);
+        return;
+      }
+      const out = stdout();
+      resolve5({
+        command: [file, ...argv],
+        exitCode,
+        signal: signal ?? stopping ?? null,
+        timedOut,
+        stdout: mode === "text" ? out.toString("utf8") : "",
+        ...mode === "buffer" ? { stdoutBuffer: out } : {},
+        stderr: stderr().toString("utf8"),
+        ...truncated ? { truncated } : {},
+        durationMs: Date.now() - started
+      });
+    };
+    const stop = (signal) => {
+      if (settled || stopping) return;
+      stopping = signal;
+      clearTimeout(timer);
+      treeStopped = terminateTree(child, signal);
+      closeInput();
+      if (signal === "SIGTERM") {
+        escalation = setTimeout(() => {
+          treeStopped = terminateTree(child, "SIGKILL");
+        }, ABORT_GRACE_MS);
+      }
+      deadline = setTimeout(() => {
+        treeStopped = terminateTree(child, "SIGKILL");
+        for (const stream of [childOut, childErr]) {
+          if (!stream.destroyed) {
+            stream.once("end", () => stream.destroy());
+            stream.push(null);
+            stream.resume();
+          }
+        }
+        void finish(child.exitCode, child.signalCode);
+      }, STOP_WAIT_MS);
+    };
+    const abort = () => stop("SIGTERM");
+    const inputError = (err) => {
+      if (exited || failure || settled || stopping) return;
+      failure = err;
+      stop("SIGKILL");
+    };
+    child.on("error", (err) => {
+      failure ??= err;
+      stop("SIGKILL");
+    });
+    child.once("exit", () => {
+      exited = true;
+      closeInput();
+    });
+    child.once("close", (exitCode, signal) => {
+      void finish(exitCode, signal);
+    });
+    timer = opts.timeoutMs ? setTimeout(() => {
+      timedOut = true;
+      stop("SIGKILL");
+    }, opts.timeoutMs) : void 0;
+    opts.signal?.addEventListener("abort", abort, { once: true });
+    if (opts.stdin !== void 0 && child.stdin) {
+      child.stdin.on("error", closeInput);
+      child.stdin.once("close", () => child.stdin?.removeListener("error", closeInput));
+      if (source) {
+        source.on("error", inputError);
+        if (source.errored) inputError(source.errored);
+        else source.pipe(child.stdin);
+      } else child.stdin.end(opts.stdin);
+    }
+    if (opts.signal?.aborted) abort();
+  });
+  return { child, stdout: childOut, result };
+}
+function tail(text, lines = 40) {
+  return text.replace(/\x1b\[[0-9;]*m/g, "").split("\n").filter((l, i, all) => l || i < all.length - 1).slice(-lines);
+}
+function shellQuote(args) {
+  return args.map((a) => /^[A-Za-z0-9_@%+=:,./-]+$/.test(a) ? a : `'${a.replace(/'/g, "'\\''")}'`).join(" ");
+}
+
+// packages/jobs/src/functions.ts
+var import_node_fs4 = require("node:fs");
+var import_promises = require("node:stream/promises");
+var import_node_path3 = require("node:path");
+var LOGGER = {
+  module: "clyops-jobs",
+  warn: (..._args) => {
+  },
+  verbose: (..._args) => {
+  }
+};
+function configureLogging(opts) {
+  Object.assign(LOGGER, Object.fromEntries(Object.entries(opts).filter(([, v]) => v)));
+}
+function resolveFunctionConfig(scriptRoot, routeConfig, functionName) {
+  const definition = objectValue(objectValue(routeConfig.functions)[functionName]);
+  if (Object.keys(definition).length === 0) return null;
+  const key = String(definition.key || functionName);
+  const script = String(definition.script || "");
+  if (!script) throw new Error(`function ${functionName} does not define a script`);
+  const config = deepMerge(objectValue(objectValue(routeConfig.defaults)[key]), objectValue(routeConfig[key]));
+  return { functionName, key, script: resolveScriptPath(scriptRoot, script), config, definition };
+}
+function explicitPositionals(fn, context) {
+  if (!Array.isArray(fn.definition.positionals)) return void 0;
+  return fn.definition.positionals.map((item) => String(renderConfigValue(item, context))).filter(Boolean);
+}
+async function buildFunctionCommand(fn, opts, schema) {
+  schema ??= await loadSchema(fn.script);
+  const config = { ...fn.config };
+  const first = schema.arguments[0];
+  if (opts.defaultPositional && first && !Array.isArray(fn.definition.positionals) && !inputKeys(first).some((k) => k in config)) {
+    config[inputKeys(first)[0]] = opts.defaultPositional;
+  }
+  const context = opts.context;
+  const { argv, unknown, controlled } = toArgv(schema, config, {
+    base: opts.configRoot,
+    controlled: opts.controlled ?? ["help"],
+    render: context ? (s) => renderTemplateString(s, context) : void 0,
+    positionals: explicitPositionals(fn, context),
+    positionalsOrder: fn.definition.positionals_order
+  });
+  const name = (0, import_node_path3.basename)(fn.script);
+  for (const key of controlled) LOGGER.warn("[%s] ignoring config key %s; it is controlled by %s", name, key, LOGGER.module);
+  for (const key of unknown) LOGGER.warn("[%s] ignoring config key %s; no matching CLI option in schema", name, key);
+  return [fn.script, ...argv];
+}
+function streamPath(template, opts, base) {
+  if (!template) return void 0;
+  const rendered = opts.context ? renderTemplateString(template, opts.context) : template;
+  return (0, import_node_path3.isAbsolute)(rendered) ? rendered : (0, import_node_path3.resolve)(base, rendered);
+}
+async function runScriptFunction(fn, opts) {
+  const schema = await loadSchema(fn.script);
+  const [file, ...argv] = await buildFunctionCommand(fn, opts, schema);
+  const command = [file, ...redactArgv(schema, argv)];
+  LOGGER.verbose("command: %s", shellQuote(command));
+  const workDir = opts.context?.paths?.work_dir;
+  const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
+  const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+  const started = start(file, argv, {
+    cwd: opts.cwd,
+    env: opts.env,
+    timeoutMs: opts.timeoutMs,
+    signal: opts.signal,
+    onStderr: opts.onStderr,
+    stdin: stdinPath ? (0, import_node_fs4.createReadStream)(stdinPath) : void 0,
+    stdout: stdoutPath ? "stream" : "text"
+  });
+  let written;
+  if (stdoutPath) {
+    ensureDir((0, import_node_path3.dirname)(stdoutPath));
+    written = (0, import_promises.finished)(started.stdout.pipe((0, import_node_fs4.createWriteStream)(stdoutPath)));
+  }
+  const [result] = await Promise.all([started.result, written]);
+  return { ...result, command, stderrTail: tail(result.stderr) };
+}
+function buildResultRecord(p) {
+  return {
+    job_id: p.jobId,
+    ...p.fields,
+    function: p.fn.functionName,
+    function_key: p.fn.key,
+    script: p.fn.script,
+    artifact_keys: p.artifactKeys,
+    completed_at: p.now,
+    config: p.fn.config,
+    ...renderResultMap(p.fn.definition.result, p.context)
+  };
+}
+
+// packages/jobs/src/artifacts.ts
+var import_node_fs5 = require("node:fs");
+var import_node_path4 = require("node:path");
+function globToRegex(pattern) {
+  const source = String(pattern).replace(/\\/g, "/");
+  let out = "^";
+  for (let i = 0; i < source.length; i += 1) {
+    const char = source[i];
+    if (char === "*") {
+      if (source[i + 1] === "*" && source[i + 2] === "/") {
+        out += "(?:.*/)?";
+        i += 2;
+      } else if (source[i + 1] === "*") {
+        out += ".*";
+        i += 1;
+      } else out += "[^/]*";
+    } else if (char === "?") out += "[^/]";
+    else out += char.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+  }
+  return new RegExp(`${out}$`);
+}
+function globSearchRoot(pattern) {
+  const normalized = String(pattern).replace(/\\/g, "/");
+  const wildcard = normalized.search(/[*?]/);
+  if (wildcard < 0) return (0, import_node_path4.dirname)(pattern);
+  const prefix = normalized.slice(0, wildcard);
+  return (0, import_node_path4.normalize)(prefix.slice(0, prefix.lastIndexOf("/")) || (0, import_node_path4.parse)(pattern).root || ".");
+}
+function listFilesRecursive(dir, visited = /* @__PURE__ */ new Set()) {
+  if (!isDirectoryPath(dir)) return [];
+  const real = (0, import_node_fs5.realpathSync)(dir);
+  if (visited.has(real)) return [];
+  visited.add(real);
+  const files = [];
+  for (const name of (0, import_node_fs5.readdirSync)(dir)) {
+    const full = (0, import_node_path4.join)(dir, name);
+    const stat = statFollowingSymlinks(full);
+    if (stat?.isDirectory()) files.push(...listFilesRecursive(full, visited));
+    else if (stat?.isFile()) files.push(full);
+  }
+  return files;
+}
+function findFilesByArtifactPattern(pattern, context) {
+  const rendered = renderTemplateString(pattern, context);
+  const absolute = (0, import_node_path4.isAbsolute)(rendered) ? rendered : (0, import_node_path4.resolve)(context.paths.work_dir, rendered);
+  if (!/[*?]/.test(absolute)) return isFilePath(absolute) ? [absolute] : [];
+  const regex = globToRegex(absolute);
+  return listFilesRecursive(globSearchRoot(absolute)).filter((file) => regex.test(file.replace(/\\/g, "/"))).sort();
+}
+function artifactIncludePatterns(fn) {
+  const stdout = fn.definition.stdout ? [String(fn.definition.stdout)] : [];
+  const artifacts = fn.definition.artifacts;
+  if (!artifacts) return stdout;
+  if (typeof artifacts === "string") return [artifacts, ...stdout];
+  if (Array.isArray(artifacts)) return [...artifacts.map(String), ...stdout];
+  if (typeof artifacts === "object") {
+    const a = artifacts;
+    const include = a.include || a.includes || a.patterns;
+    if (typeof include === "string") return [include, ...stdout];
+    if (Array.isArray(include)) return [...include.map(String), ...stdout];
+  }
+  return stdout;
+}
+function artifactRequirement(fn) {
+  const artifacts = fn.definition.artifacts;
+  if (!artifacts || typeof artifacts !== "object" || Array.isArray(artifacts)) return 0;
+  const min = Number(artifacts.min);
+  if (Number.isFinite(min) && min > 0) return Math.floor(min);
+  return artifacts.required === true || truthy(artifacts.required) ? 1 : 0;
+}
+function copyConfiguredArtifacts(fn, context, opts = {}) {
+  const patterns = artifactIncludePatterns(fn);
+  const min = artifactRequirement(fn);
+  if (patterns.length === 0) {
+    if (min > 0) throw new Error(`artifacts require at least ${min} file(s) but no include patterns are configured`);
+    return [];
+  }
+  const dir = context.paths.artifacts_dir;
+  if ((0, import_node_fs5.existsSync)(dir)) throw new Error(`artifact directory already exists: ${dir}`);
+  const seen = /* @__PURE__ */ new Set();
+  const copied = [];
+  for (const pattern of patterns) {
+    for (const source of findFilesByArtifactPattern(pattern, context)) {
+      if (seen.has(source)) continue;
+      seen.add(source);
+      const destination = (0, import_node_path4.join)(dir, (0, import_node_path4.basename)(source));
+      if ((0, import_node_fs5.existsSync)(destination)) throw new Error(`duplicate artifact filename: ${(0, import_node_path4.basename)(source)}`);
+      copyFileEnsuringDir(source, destination);
+      copied.push({ source, key: `${opts.keyPrefix ?? ""}${(0, import_node_path4.basename)(dir)}/${(0, import_node_path4.basename)(destination)}` });
+    }
+  }
+  if (copied.length < min) throw new Error(`expected at least ${min} artifact file(s) but matched ${copied.length}`);
+  return copied;
+}
+
+// packages/jobs/src/records.ts
+var import_node_crypto = require("node:crypto");
+function newJobId(seed, prefix = "job") {
+  const material = `${seed}/${Date.now()}/${process.pid}/${Math.random()}`;
+  return `${prefix}-${(0, import_node_crypto.createHash)("md5").update(material).digest("hex").slice(0, 12)}`;
+}
+function newJobRecord(p) {
+  const now = p.now ?? timestampIso();
+  return {
+    job_id: p.jobId,
+    ...p.fields,
+    function: p.functionName,
+    status: "pending",
+    stage: null,
+    started_at: now,
+    updated_at: now,
+    completed_at: null,
+    error: null,
+    result_key: null,
+    history: []
+  };
+}
+function transitionJobRecord(record, change) {
+  return {
+    ...record,
+    ...change.extra,
+    history: [{ status: record.status, stage: record.stage, updated_at: record.updated_at }, ...record.history || []],
+    status: change.status,
+    stage: change.stage ?? null,
+    updated_at: change.now ?? timestampIso(),
+    completed_at: change.completedAt || null,
+    error: change.error || null,
+    result_key: change.resultKey || null
+  };
+}
+
+// packages/jobs/src/queue.ts
+var import_node_os = require("node:os");
+var JobQueue = class {
+  constructor(opts = {}) {
+    this.opts = opts;
+  }
+  opts;
+  jobs = /* @__PURE__ */ new Map();
+  waiting = [];
+  running = 0;
+  finished = [];
+  /**
+   * Queue `task` as a job of `functionName`. The task gets an AbortSignal and
+   * a `stage` setter for progress; its return value is the job's result.
+   */
+  add(functionName, task, fields = {}) {
+    const id = newJobId(functionName, this.opts.prefix ?? "job");
+    const abort = new AbortController();
+    const job = { record: newJobRecord({ jobId: id, functionName, fields }) };
+    const update = (change) => {
+      job.record = transitionJobRecord(job.record, change);
+      this.opts.onChange?.(job.record);
+    };
+    job.cancel = () => abort.abort();
+    job.done = (async () => {
+      await this.slot();
+      try {
+        abort.signal.throwIfAborted();
+        update({ status: "processing" });
+        const result = await task({ signal: abort.signal, stage: (stage) => update({ status: "processing", stage }) });
+        job.result = result;
+        abort.signal.throwIfAborted();
+        update({ status: "done", completedAt: timestampIso() });
+        return result;
+      } catch (err) {
+        update({ status: "error", error: abort.signal.aborted ? "cancelled" : err.message, completedAt: timestampIso() });
+        throw err;
+      } finally {
+        this.release(id);
+      }
+    })();
+    job.done.catch(() => {
+    });
+    this.jobs.set(id, job);
+    this.opts.onChange?.(job.record);
+    return job;
+  }
+  get(id) {
+    return this.jobs.get(id);
+  }
+  list() {
+    return [...this.jobs.values()].map((j) => j.record);
+  }
+  slot() {
+    if (this.running < (this.opts.concurrency ?? (0, import_node_os.availableParallelism)())) {
+      this.running += 1;
+      return Promise.resolve();
+    }
+    return new Promise((resolve5) => this.waiting.push(resolve5));
+  }
+  release(id) {
+    const next = this.waiting.shift();
+    if (next) next();
+    else this.running -= 1;
+    this.finished.push(id);
+    while (this.finished.length > (this.opts.keep ?? 1e3)) {
+      const dropped = this.finished.shift();
+      const job = this.jobs.get(dropped);
+      this.jobs.delete(dropped);
+      if (job) this.opts.onDrop?.(job.record);
+    }
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  JobQueue,
+  artifactIncludePatterns,
+  artifactRequirement,
+  buildFunctionCommand,
+  buildResultRecord,
+  configureLogging,
+  copyConfiguredArtifacts,
+  copyFileEnsuringDir,
+  deepMerge,
+  ensureDir,
+  findFilesByArtifactPattern,
+  globToRegex,
+  isDirectoryPath,
+  isEnabled,
+  isFilePath,
+  listFilesRecursive,
+  lookupTemplateValue,
+  newJobId,
+  newJobRecord,
+  objectValue,
+  pathParts,
+  readJson,
+  renderConfigValue,
+  renderResultMap,
+  renderResultValue,
+  renderTemplateString,
+  resolveConfigPath,
+  resolveFunctionConfig,
+  resolveScriptPath,
+  runScriptFunction,
+  shellQuote,
+  statFollowingSymlinks,
+  timestampIso,
+  transitionJobRecord,
+  truthy,
+  writeJson
+});

package/dist/functions.d.ts

@@ -15,6 +15,8 @@ export interface FunctionDefinition {
     key?: string;
     script?: string;
     positionals?: unknown[];
+    /** Positionals before or after options (default: first). */
+    positionals_order?: 'first' | 'last';
     /** A template for a file fed to the tool's stdin, e.g. '${media.path}'. */
     stdin?: string;
     /** A template for the file the tool's stdout is written to; it also counts as an artifact. */

package/dist/functions.js

@@ -66,6 +66,7 @@ export async function buildFunctionCommand(fn, opts, schema) {
         controlled: opts.controlled ?? ['help'],
         render: context ? (s) => renderTemplateString(s, context) : undefined,
         positionals: explicitPositionals(fn, context),
+        positionalsOrder: fn.definition.positionals_order,
     });
     const name = basename(fn.script);
     for (const key of controlled)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clyops-jobs",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Run clyops tools as jobs: config-bound functions, templated invocation, artifacts, job records and a queue.",
   "license": "MIT",
   "repository": {
@@ -24,24 +24,25 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
-      "default": "./dist/index.js"
+      "import": "./dist/index.js",
+      "require": "./clyops-jobs.cjs"
     }
   },
   "files": [
     "dist",
-    "src"
+    "src",
+    "clyops-jobs.cjs"
   ],
-  "scripts": {
-    "build": "tsc -p tsconfig.json",
-    "typecheck": "tsc -p tsconfig.json --noEmit",
-    "test": "node --test test/*.test.mjs",
-    "prepublishOnly": "npm run build && npm test"
-  },
   "devDependencies": {
     "@types/node": "^22.0.0",
     "typescript": "^5.6.0"
   },
   "dependencies": {
-    "clyops-tools": "0.2.0"
+    "clyops-tools": "0.3.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "node --test test/*.test.mjs"
   }
-}
+}

package/src/functions.ts

@@ -34,6 +34,8 @@ export interface FunctionDefinition {
   key?: string;
   script?: string;
   positionals?: unknown[];
+  /** Positionals before or after options (default: first). */
+  positionals_order?: 'first' | 'last';
   /** A template for a file fed to the tool's stdin, e.g. '${media.path}'. */
   stdin?: string;
   /** A template for the file the tool's stdout is written to; it also counts as an artifact. */
@@ -105,6 +107,7 @@ export async function buildFunctionCommand(fn: ResolvedFunction, opts: CommandOp
     controlled: opts.controlled ?? ['help'],
     render: context ? (s) => renderTemplateString(s, context) : undefined,
     positionals: explicitPositionals(fn, context),
+    positionalsOrder: fn.definition.positionals_order,
   });
   const name = basename(fn.script);
   for (const key of controlled) LOGGER.warn('[%s] ignoring config key %s; it is controlled by %s', name, key, LOGGER.module);