clyops-jobs 0.3.0 → 0.5.0

package/README.md

@@ -30,7 +30,7 @@ and no `node_modules`. It includes clyops-tools. `require('clyops-jobs')` loads
 - `functions.NAME.script` is the tool, relative to a script root.
 - `key` names the options block (default: the function name). Options are
   `deepMerge(defaults[key], config[key])`, keyed like the tool's options (`out_dir`, `out-dir`)
-  and mapped onto its command line through `--help-json-schema` ([spec §13](../../spec/SPEC.md#13-json-input-toargv)).
+  and mapped onto its command line through `--help-json-schema` ([spec §13](https://github.com/wankdanker/clyops/blob/main/spec/SPEC.md#13-json-input-toargv)).
 - `positionals_order` (optional) is `first` (default) or `last`, for wrappers needing a specific order. `--` is only emitted for positional values starting with `-`.
 - `positionals` (optional) lists the tool's positional arguments explicitly; otherwise they're
   options keyed by argument name.
@@ -63,14 +63,24 @@ record = transitionJobRecord(record, { status: 'done', completedAt: timestampIso
 `runScriptFunction` options: `configRoot` (relative path options resolve against it),
 `context`, `controlled` (options the trigger sets itself, default `['help']`), `defaultPositional`
 (a template for the first argument when the config gives none, e.g. `'${media.path}'`),
-`timeoutMs`, `env`, `cwd`, `signal`, `onStderr`. Its `command` shows secret options as `***`.
+`timeoutMs`, `env`, `cwd`, `signal`, `onStderr`, `onProgress`, `onLog`. Its `command` shows secret options as `***`.
+
+When the tool declares the well-known [roles](https://github.com/wankdanker/clyops/blob/main/spec/SPEC.md#14-roles-on-file-descriptors),
+the run wires them up: each `progress` object goes to `onProgress` (and the last one is the run's
+`progress`), `log` lines to `onLog`, and the `meta` document is parsed into the run's `meta`.
+`updateJobProgress(record, update)` puts an update on a record as `progress` (and its `stage`, if
+any) without a history entry; the queue's tasks get it as `progress(update)`:
+
+```js
+queue.add('transcribe', ({ signal, progress }) => runScriptFunction(fn, { configRoot, context, signal, onProgress: progress }));
+```
 `configureLogging({module, warn, verbose})` routes warnings about config keys that match no option.
 
 ## Job records
 
 ```
 { job_id, ...fields, function, status, stage, started_at, updated_at, completed_at, error,
-  result_key, history }
+  result_key, progress?, history }
 ```
 
 `status` moves `pending` → `processing` → `done` | `error`. `newJobId(seed, prefix = 'job')`,
@@ -93,3 +103,9 @@ ESM only, Node 20+.
 A missing or unreadable configured stdin file rejects `runScriptFunction()`
 with its input-file error. It fails that job and stops its tool; the host worker
 stays alive.
+
+## Documentation
+
+Read the [Node guide](https://clyops.io/node-tools.html) and
+[complete public API reference](https://clyops.io/reference/node/index.html).
+Type declarations and documentation comments are included in the package.

package/clyops-jobs.cjs

@@ -41,6 +41,7 @@ __export(index_exports, {
   newJobId: () => newJobId,
   newJobRecord: () => newJobRecord,
   objectValue: () => objectValue,
+  parseRoleJson: () => parseRoleJson,
   pathParts: () => pathParts,
   readJson: () => readJson,
   renderConfigValue: () => renderConfigValue,
@@ -56,6 +57,7 @@ __export(index_exports, {
   timestampIso: () => timestampIso,
   transitionJobRecord: () => transitionJobRecord,
   truthy: () => truthy,
+  updateJobProgress: () => updateJobProgress,
   writeJson: () => writeJson
 });
 module.exports = __toCommonJS(index_exports);
@@ -102,7 +104,18 @@ function copyFileEnsuringDir(source, destination) {
 function pathParts(file) {
   const name = (0, import_node_path.basename)(file);
   const ext = (0, import_node_path.extname)(name);
-  return { path: file, dir: (0, import_node_path.dirname)(file), name, basename: ext ? name.slice(0, -ext.length) : name, ext: ext.replace(/^\./, "") };
+  return {
+    /** Resolved filesystem path or HTTP endpoint path, according to the containing type. */
+    path: file,
+    /** Directory represented by this group or specified in .clyops settings. */
+    dir: (0, import_node_path.dirname)(file),
+    /** Program, command, option, or argument name as declared in its containing type. */
+    name,
+    /** Filename without its extension. */
+    basename: ext ? name.slice(0, -ext.length) : name,
+    /** Filename extension including the leading dot. */
+    ext: ext.replace(/^\./, "")
+  };
 }
 function objectValue(value) {
   return value && typeof value === "object" && !Array.isArray(value) ? value : {};
@@ -195,9 +208,8 @@ function loadSchema(file, opts = {}) {
 // packages/tools/src/argv.ts
 var import_node_fs3 = require("node:fs");
 var import_node_path2 = require("node:path");
-var InputError = class extends Error {
-  status = 400;
-};
+
+// packages/schema/src/names.ts
 function inputKey(name) {
   return name.replace(/-/g, "_");
 }
@@ -209,6 +221,12 @@ function isBareFlag(option) {
   const choices = [...option.choices].sort().join(",");
   return option.isFlag && choices !== "false,true";
 }
+
+// packages/tools/src/argv.ts
+var InputError = class extends Error {
+  /** HTTP status used by adapters to report invalid caller input. */
+  status = 400;
+};
 function isPathValued(validation, type) {
   return type === "path" || validation === "path" || validation.startsWith("file:") || validation.startsWith("dir:");
 }
@@ -326,6 +344,7 @@ function redactArgv(schema, argv) {
 var import_node_child_process2 = require("node:child_process");
 var import_node_stream = require("node:stream");
 var import_node_string_decoder = require("node:string_decoder");
+var ROLE_NAME = /^[A-Za-z][A-Za-z0-9_.-]*$/;
 var ABORT_GRACE_MS = 200;
 var STOP_WAIT_MS = 1e3;
 function terminateTree(child, signal) {
@@ -348,10 +367,15 @@ function terminateTree(child, signal) {
 }
 function start(file, argv, opts = {}) {
   const started = Date.now();
+  const roles = process.platform === "win32" ? [] : Object.entries(opts.roles ?? {});
+  for (const [name] of roles) if (!ROLE_NAME.test(name)) throw new Error(`Invalid role name '${name}'`);
+  const env = { ...opts.env ?? process.env };
+  delete env.CLYOPS_FDS;
+  if (roles.length) env.CLYOPS_FDS = roles.map(([name], i) => `${name}:${i + 3}`).join(",");
   const child = (0, import_node_child_process2.spawn)(file, argv, {
     cwd: opts.cwd,
-    env: opts.env ?? process.env,
-    stdio: [opts.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe"],
+    env,
+    stdio: [opts.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe", ...roles.map(() => "pipe")],
     // POSIX detached children lead a new session/group, still with owned pipes.
     detached: process.platform !== "win32",
     windowsHide: true
@@ -378,6 +402,26 @@ function start(file, argv, opts = {}) {
   const mode = opts.stdout ?? "text";
   const stdout = mode === "stream" ? () => Buffer.alloc(0) : collect(childOut, opts.onStdout);
   const stderr = collect(childErr, opts.onStderr);
+  const roleOut = [];
+  const roleIn = [];
+  roles.forEach(([name, role], i) => {
+    const stream = child.stdio[i + 3];
+    if (role.input !== void 0) {
+      roleIn.push({ stream, input: role.input });
+      return;
+    }
+    let partial = "";
+    const lines = role.onLine && ((chunk) => {
+      const parts = (partial + chunk).split("\n");
+      partial = parts.pop() ?? "";
+      for (const line of parts) role.onLine?.(line);
+    });
+    stream.once("end", () => {
+      if (partial) role.onLine?.(partial);
+      partial = "";
+    });
+    roleOut.push({ name, stream, text: collect(stream, lines || void 0) });
+  });
   const result = new Promise((resolve5, reject) => {
     let timedOut = false;
     let failure;
@@ -406,6 +450,10 @@ function start(file, argv, opts = {}) {
         }
       }
       child.stdin?.destroy();
+      for (const { stream, input } of roleIn) {
+        if (input instanceof import_node_stream.Readable) input.unpipe(stream);
+        stream.destroy();
+      }
     };
     const finish = async (exitCode, signal) => {
       if (settled) return;
@@ -431,6 +479,7 @@ function start(file, argv, opts = {}) {
         ...mode === "buffer" ? { stdoutBuffer: out } : {},
         stderr: stderr().toString("utf8"),
         ...truncated ? { truncated } : {},
+        ...roleOut.length ? { roles: Object.fromEntries(roleOut.map((r) => [r.name, r.text().toString("utf8")])) } : {},
         durationMs: Date.now() - started
       });
     };
@@ -447,7 +496,7 @@ function start(file, argv, opts = {}) {
       }
       deadline = setTimeout(() => {
         treeStopped = terminateTree(child, "SIGKILL");
-        for (const stream of [childOut, childErr]) {
+        for (const stream of [childOut, childErr, ...roleOut.map((r) => r.stream)]) {
           if (!stream.destroyed) {
             stream.once("end", () => stream.destroy());
             stream.push(null);
@@ -488,6 +537,13 @@ function start(file, argv, opts = {}) {
         else source.pipe(child.stdin);
       } else child.stdin.end(opts.stdin);
     }
+    for (const { stream, input } of roleIn) {
+      stream.on("error", () => void 0).resume();
+      if (input instanceof import_node_stream.Readable) {
+        input.on("error", inputError);
+        input.pipe(stream);
+      } else stream.end(input);
+    }
     if (opts.signal?.aborted) abort();
   });
   return { child, stdout: childOut, result };
@@ -499,6 +555,12 @@ function shellQuote(args) {
   return args.map((a) => /^[A-Za-z0-9_@%+=:,./-]+$/.test(a) ? a : `'${a.replace(/'/g, "'\\''")}'`).join(" ");
 }
 
+// packages/tools/src/tools.ts
+function declaredRoles(tool, requested) {
+  const declared = new Map((tool.schema.roles ?? []).map((r) => [r.name, r.direction]));
+  return Object.fromEntries(Object.entries(requested).filter(([name, role]) => declared.get(name) === (role.input === void 0 ? "out" : "in")));
+}
+
 // packages/jobs/src/functions.ts
 var import_node_fs4 = require("node:fs");
 var import_promises = require("node:stream/promises");
@@ -546,6 +608,14 @@ async function buildFunctionCommand(fn, opts, schema) {
   for (const key of unknown) LOGGER.warn("[%s] ignoring config key %s; no matching CLI option in schema", name, key);
   return [fn.script, ...argv];
 }
+function parseRoleJson(text) {
+  if (!text?.trim()) return void 0;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return void 0;
+  }
+}
 function streamPath(template, opts, base) {
   if (!template) return void 0;
   const rendered = opts.context ? renderTemplateString(template, opts.context) : template;
@@ -559,6 +629,19 @@ async function runScriptFunction(fn, opts) {
   const workDir = opts.context?.paths?.work_dir;
   const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
   const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+  let progress;
+  const roles = {
+    progress: {
+      onLine: (line) => {
+        const update = parseRoleJson(line);
+        if (!update || typeof update !== "object" || Array.isArray(update)) return;
+        progress = update;
+        opts.onProgress?.(progress);
+      }
+    },
+    meta: {},
+    log: { onLine: opts.onLog }
+  };
   const started = start(file, argv, {
     cwd: opts.cwd,
     env: opts.env,
@@ -566,7 +649,8 @@ async function runScriptFunction(fn, opts) {
     signal: opts.signal,
     onStderr: opts.onStderr,
     stdin: stdinPath ? (0, import_node_fs4.createReadStream)(stdinPath) : void 0,
-    stdout: stdoutPath ? "stream" : "text"
+    stdout: stdoutPath ? "stream" : "text",
+    roles: declaredRoles({ schema }, roles)
   });
   let written;
   if (stdoutPath) {
@@ -574,7 +658,8 @@ async function runScriptFunction(fn, opts) {
     written = (0, import_promises.finished)(started.stdout.pipe((0, import_node_fs4.createWriteStream)(stdoutPath)));
   }
   const [result] = await Promise.all([started.result, written]);
-  return { ...result, command, stderrTail: tail(result.stderr) };
+  const meta = parseRoleJson(result.roles?.meta);
+  return { ...result, command, stderrTail: tail(result.stderr), ...meta !== void 0 ? { meta } : {}, ...progress ? { progress } : {} };
 }
 function buildResultRecord(p) {
   return {
@@ -720,10 +805,15 @@ function transitionJobRecord(record, change) {
     result_key: change.resultKey || null
   };
 }
+function updateJobProgress(record, progress, now) {
+  const stage = typeof progress.stage === "string" ? progress.stage : record.stage;
+  return { ...record, progress, stage, updated_at: now ?? timestampIso() };
+}
 
 // packages/jobs/src/queue.ts
 var import_node_os = require("node:os");
 var JobQueue = class {
+  /** Create an instance with the supplied configuration; omitted fields use the documented defaults. */
   constructor(opts = {}) {
     this.opts = opts;
   }
@@ -750,7 +840,15 @@ var JobQueue = class {
       try {
         abort.signal.throwIfAborted();
         update({ status: "processing" });
-        const result = await task({ signal: abort.signal, stage: (stage) => update({ status: "processing", stage }) });
+        const result = await task({
+          signal: abort.signal,
+          stage: (stage) => update({ status: "processing", stage }),
+          progress: (p) => {
+            if (job.record.status !== "processing") return;
+            job.record = updateJobProgress(job.record, p);
+            this.opts.onChange?.(job.record);
+          }
+        });
         job.result = result;
         abort.signal.throwIfAborted();
         update({ status: "done", completedAt: timestampIso() });
@@ -768,9 +866,11 @@ var JobQueue = class {
     this.opts.onChange?.(job.record);
     return job;
   }
+  /** Look up a retained job by ID; returns undefined when unknown or dropped. */
   get(id) {
     return this.jobs.get(id);
   }
+  /** Return the current records for all retained jobs, including pending and finished jobs. */
   list() {
     return [...this.jobs.values()].map((j) => j.record);
   }
@@ -816,6 +916,7 @@ var JobQueue = class {
   newJobId,
   newJobRecord,
   objectValue,
+  parseRoleJson,
   pathParts,
   readJson,
   renderConfigValue,
@@ -831,5 +932,6 @@ var JobQueue = class {
   timestampIso,
   transitionJobRecord,
   truthy,
+  updateJobProgress,
   writeJson
 });

package/dist/artifacts.d.ts

@@ -1,5 +1,6 @@
 import type { ResolvedFunction } from './functions.js';
 import { type TemplateContext } from './template.js';
+/** Compile an artifact glob into an anchored regular expression. */
 export declare function globToRegex(pattern: string): RegExp;
 /** Every file below `dir`, following symlinks without looping. */
 export declare function listFilesRecursive(dir: string, visited?: Set<string>): string[];
@@ -13,7 +14,9 @@ export declare function findFilesByArtifactPattern(pattern: string, context: Tem
 export declare function artifactIncludePatterns(fn: ResolvedFunction): string[];
 /** How many artifacts a function must produce: `min`, or 1 for `required: true`. */
 export declare function artifactRequirement(fn: ResolvedFunction): number;
+/** An artifact copied to its destination, recording the original source path. */
 export interface CopiedArtifact {
+    /** Original source path of the copied artifact. */
     source: string;
     /** `<keyPrefix><artifacts dir name>/<file name>`. */
     key: string;
@@ -31,3 +34,4 @@ export declare function copyConfiguredArtifacts(fn: ResolvedFunction, context: T
 }, opts?: {
     keyPrefix?: string;
 }): CopiedArtifact[];
+//# sourceMappingURL=artifacts.d.ts.map

package/dist/artifacts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"artifacts.d.ts","sourceRoot":"","sources":["../src/artifacts.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAwB,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAK3E,oEAAoE;AACpE,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAiBnD;AAWD,kEAAkE;AAClE,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,cAAoB,GAAG,MAAM,EAAE,CAarF;AAED,oFAAoF;AACpF,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAAG,MAAM,EAAE,CAQhI;AAED,sFAAsF;AACtF,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAatE;AAED,oFAAoF;AACpF,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,gBAAgB,GAAG,MAAM,CAMhE;AAED,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,gBAAgB,EACpB,OAAO,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,EACjF,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAChC,cAAc,EAAE,CAwBlB"}

package/dist/artifacts.js

@@ -5,6 +5,7 @@ import { renderTemplateString } from './template.js';
 import { copyFileEnsuringDir, isDirectoryPath, isFilePath, statFollowingSymlinks, truthy } from './util.js';
 // A small glob subset as a regex: `*`, `?`, `**` and `**/` (which also matches no
 // directory), with `/` separators.
+/** Compile an artifact glob into an anchored regular expression. */
 export function globToRegex(pattern) {
     const source = String(pattern).replace(/\\/g, '/');
     let out = '^';

package/dist/functions.d.ts

@@ -7,13 +7,17 @@ import { type Json } from './util.js';
  * controlled by media-dropoff".
  */
 export declare function configureLogging(opts: {
-    module?: string;
+    module?: string; /** Write a timestamped warning to stderr; supports printf-style arguments. */
     warn?: (...args: unknown[]) => void;
     verbose?: (...args: unknown[]) => void;
 }): void;
+/** A configured binding to an executable, optional input/output files, artifacts, and result templates. */
 export interface FunctionDefinition {
+    /** Config key holding function options; defaults to the function name. */
     key?: string;
+    /** Executable path; configured paths are resolved against the script root. */
     script?: string;
+    /** Explicit positional templates; an empty array explicitly supplies no positionals. */
     positionals?: unknown[];
     /** Positionals before or after options (default: first). */
     positionals_order?: 'first' | 'last';
@@ -21,11 +25,15 @@ export interface FunctionDefinition {
     stdin?: string;
     /** A template for the file the tool's stdout is written to; it also counts as an artifact. */
     stdout?: string;
+    /** Artifact configuration consumed by the job engine after execution. */
     artifacts?: unknown;
+    /** Task result or configured result templates, according to the containing interface. */
     result?: unknown;
     [key: string]: unknown;
 }
+/** A configured function with resolved executable and merged defaults and overrides. */
 export interface ResolvedFunction {
+    /** Name of the configured function to invoke. */
     functionName: string;
     /** The config key holding its options (default: the function name). */
     key: string;
@@ -33,6 +41,7 @@ export interface ResolvedFunction {
     script: string;
     /** `deepMerge(defaults[key], config[key])`. */
     config: Json;
+    /** Original function binding, including positional, stream, artifact, and result templates. */
     definition: FunctionDefinition;
 }
 /**
@@ -40,9 +49,11 @@ export interface ResolvedFunction {
  * has no such function. A function without a `script` is an error.
  */
 export declare function resolveFunctionConfig(scriptRoot: string, routeConfig: Json, functionName: string): ResolvedFunction | null;
+/** Path base, template context, and controlled arguments used when constructing a job command. */
 export interface CommandOptions {
     /** Directory relative path options resolve against. */
     configRoot: string;
+    /** Values available to dotted-path templates; null disables context-based rendering. */
     context?: TemplateContext | null;
     /** Options the trigger sets itself; config keys for them are ignored with a warning. Default: ['help']. */
     controlled?: string[];
@@ -54,10 +65,17 @@ export interface CommandOptions {
 }
 /** The full command line for a function: [script, ...argv]. */
 export declare function buildFunctionCommand(fn: ResolvedFunction, opts: CommandOptions, schema?: Schema): Promise<string[]>;
+/** The completed tool execution with a color-stripped stderr tail. */
 export interface FunctionRun extends RunResult {
     /** The last 40 lines of stderr without color codes. */
     stderrTail: string[];
+    /** The tool's `meta` role (spec section 14), parsed, when it declares one and wrote JSON. */
+    meta?: unknown;
+    /** The last update on its `progress` role. */
+    progress?: Record<string, unknown>;
 }
+/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
+export declare function parseRoleJson(text: string | undefined): unknown;
 /**
  * Run a function's tool. Its stdin is the function's `stdin` file, if any;
  * stdout goes to its `stdout` file, else is collected (a tool may print its
@@ -65,10 +83,12 @@ export interface FunctionRun extends RunResult {
  */
 export declare function runScriptFunction(fn: ResolvedFunction, opts: CommandOptions & {
     timeoutMs?: number;
-    env?: NodeJS.ProcessEnv;
-    cwd?: string;
-    signal?: AbortSignal;
-    onStderr?: (chunk: string) => void;
+    /** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
+    /** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
+    /** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
+    /** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
+    /** Called with each object the tool writes on its `progress` role (spec section 14). */ onProgress?: (update: Record<string, unknown>) => void;
+    /** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
 }): Promise<FunctionRun>;
 /** The result record every function shares, plus its rendered `result` map. */
 export declare function buildResultRecord(p: {
@@ -76,6 +96,8 @@ export declare function buildResultRecord(p: {
     fields?: Json;
     fn: ResolvedFunction;
     artifactKeys: string[];
+    /** Values available to dotted-path templates; null disables context-based rendering. */
     context: TemplateContext;
     now: string;
 }): Json;
+//# sourceMappingURL=functions.d.ts.map

package/dist/functions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"functions.d.ts","sourceRoot":"","sources":["../src/functions.ts"],"names":[],"mappings":"AASA,OAAO,EAAsG,KAAK,SAAS,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAI/J,OAAO,EAA4D,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAC/G,OAAO,EAAwD,KAAK,IAAI,EAAE,MAAM,WAAW,CAAC;AAS5F;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,8EAA8E;IAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;CAAE,GAAG,IAAI,CAE5N;AAED,2GAA2G;AAC3G,MAAM,WAAW,kBAAkB;IACjC,0EAA0E;IAC1E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wFAAwF;IACxF,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC;IACxB,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACrC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yFAAyF;IACzF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,wFAAwF;AACxF,MAAM,WAAW,gBAAgB;IAC/B,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,+FAA+F;IAC/F,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAQ1H;AAED,kGAAkG;AAClG,MAAM,WAAW,cAAc;IAC7B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,2GAA2G;IAC3G,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAYD,+DAA+D;AAC/D,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAmBzH;AAED,sEAAsE;AACtE,MAAM,WAAW,WAAY,SAAQ,SAAS;IAC5C,uDAAuD;IACvD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,oFAAoF;AACpF,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAO/D;AASD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,gBAAgB,EACpB,IAAI,EAAE,cAAc,GAAG;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uEAAuE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChG,8EAA8E,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAC5F,gFAAgF,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IACtG,4CAA4C,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChF,wFAAwF,CAAC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAChJ,+DAA+D,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAChG,GACA,OAAO,CAAC,WAAW,CAAC,CAqCtB;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,CAAC,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,EAAE,EAAE,gBAAgB,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wFAAwF;IACxF,OAAO,EAAE,eAAe,CAAC;IACzB,GAAG,EAAE,MAAM,CAAC;CACb,GAAG,IAAI,CAYP"}

package/dist/functions.js

@@ -7,7 +7,7 @@
 // Tool options are mapped onto the tool's command line through its
 // --help-json-schema (spec section 13), after `${dot.path}` / `{{dot.path}}`
 // templates are rendered against a context the trigger builds.
-import { inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv } from 'clyops-tools';
+import { declaredRoles, inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv } from 'clyops-tools';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { finished } from 'node:stream/promises';
 import { basename, dirname, isAbsolute, resolve } from 'node:path';
@@ -75,6 +75,17 @@ export async function buildFunctionCommand(fn, opts, schema) {
         LOGGER.warn('[%s] ignoring config key %s; no matching CLI option in schema', name, key);
     return [fn.script, ...argv];
 }
+/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
+export function parseRoleJson(text) {
+    if (!text?.trim())
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
 /** A function's `stdin` or `stdout` template rendered, a relative one against `base`. */
 function streamPath(template, opts, base) {
     if (!template)
@@ -96,10 +107,26 @@ export async function runScriptFunction(fn, opts) {
     const workDir = opts.context?.paths?.work_dir;
     const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
     const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+    // Wire the well-known roles the tool declares: progress onto the record, meta into the run.
+    let progress;
+    const roles = {
+        progress: {
+            onLine: (line) => {
+                const update = parseRoleJson(line);
+                if (!update || typeof update !== 'object' || Array.isArray(update))
+                    return;
+                progress = update;
+                opts.onProgress?.(progress);
+            },
+        },
+        meta: {},
+        log: { onLine: opts.onLog },
+    };
     const started = start(file, argv, {
         cwd: opts.cwd, env: opts.env, timeoutMs: opts.timeoutMs, signal: opts.signal, onStderr: opts.onStderr,
         stdin: stdinPath ? createReadStream(stdinPath) : undefined,
         stdout: stdoutPath ? 'stream' : 'text',
+        roles: declaredRoles({ schema }, roles),
     });
     let written;
     if (stdoutPath) {
@@ -107,7 +134,8 @@ export async function runScriptFunction(fn, opts) {
         written = finished(started.stdout.pipe(createWriteStream(stdoutPath)));
     }
     const [result] = await Promise.all([started.result, written]);
-    return { ...result, command, stderrTail: tail(result.stderr) };
+    const meta = parseRoleJson(result.roles?.meta);
+    return { ...result, command, stderrTail: tail(result.stderr), ...(meta !== undefined ? { meta } : {}), ...(progress ? { progress } : {}) };
 }
 /** The result record every function shares, plus its rendered `result` map. */
 export function buildResultRecord(p) {

package/dist/index.d.ts

@@ -5,3 +5,4 @@ export * from './artifacts.js';
 export * from './records.js';
 export * from './queue.js';
 export { shellQuote } from 'clyops-tools';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC"}

package/dist/queue.d.ts

@@ -1,12 +1,16 @@
 import { type JobRecord } from './records.js';
+/** A queued task, its current record, completion promise, optional result, and cancellation method. */
 export interface Job<R> {
+    /** Current serializable job state, updated as the task progresses. */
     record: JobRecord;
+    /** Task result or configured result templates, according to the containing interface. */
     result?: R;
     /** Resolves with the result when the job is done (rejects when it fails). */
     done: Promise<R>;
     /** Abort a pending or running job. */
     cancel(): void;
 }
+/** Concurrency, retention, ID prefix, and lifecycle callbacks for the in-memory queue. */
 export interface QueueOptions {
     /** Jobs run at the same time (default: the number of CPUs). */
     concurrency?: number;
@@ -19,23 +23,29 @@ export interface QueueOptions {
     /** Called when a finished job is dropped (beyond `keep`), e.g. to remove its files. */
     onDrop?: (record: JobRecord) => void;
 }
+/** An in-memory queue with bounded concurrency and retention. Await each job.done to observe its result. */
 export declare class JobQueue<R = unknown> {
     private opts;
     private jobs;
     private waiting;
     private running;
     private finished;
+    /** Create an instance with the supplied configuration; omitted fields use the documented defaults. */
     constructor(opts?: QueueOptions);
     /**
      * Queue `task` as a job of `functionName`. The task gets an AbortSignal and
      * a `stage` setter for progress; its return value is the job's result.
      */
     add(functionName: string, task: (ctx: {
-        signal: AbortSignal;
-        stage: (stage: string) => void;
+        /** Termination signal, or null for an ordinary exit. RunOptions uses an AbortSignal for cancellation. */ signal: AbortSignal;
+        /** Current progress stage, or null when unspecified. */ stage: (stage: string) => void;
+        /** Record a progress update, e.g. runScriptFunction's onProgress (no history entry). */ progress: (update: Record<string, unknown>) => void;
     }) => Promise<R>, fields?: Record<string, unknown>): Job<R>;
+    /** Look up a retained job by ID; returns undefined when unknown or dropped. */
     get(id: string): Job<R> | undefined;
+    /** Return the current records for all retained jobs, including pending and finished jobs. */
     list(): JobRecord[];
     private slot;
     private release;
 }
+//# sourceMappingURL=queue.d.ts.map

package/dist/queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9G,uGAAuG;AACvG,MAAM,WAAW,GAAG,CAAC,CAAC;IACpB,sEAAsE;IACtE,MAAM,EAAE,SAAS,CAAC;IAClB,yFAAyF;IACzF,MAAM,CAAC,EAAE,CAAC,CAAC;IACX,6EAA6E;IAC7E,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACjB,sCAAsC;IACtC,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,YAAY;IAC3B,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACvC,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;CACtC;AAED,4GAA4G;AAC5G,qBAAa,QAAQ,CAAC,CAAC,GAAG,OAAO;IAOnB,OAAO,CAAC,IAAI;IANxB,OAAO,CAAC,IAAI,CAA6B;IACzC,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,QAAQ,CAAgB;IAEhC,sGAAsG;gBAClF,IAAI,GAAE,YAAiB;IAE3C;;;OAGG;IACH,GAAG,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE;QACpC,yGAAyG,CAAC,MAAM,EAAE,WAAW,CAAC;QAC9H,wDAAwD,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxF,wFAAwF,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;KAC9I,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAAG,GAAG,CAAC,CAAC,CAAC;IAyC/D,+EAA+E;IAC/E,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,SAAS;IAInC,6FAA6F;IAC7F,IAAI,IAAI,SAAS,EAAE;IAInB,OAAO,CAAC,IAAI;IAQZ,OAAO,CAAC,OAAO;CAYhB"}

package/dist/queue.js

@@ -1,14 +1,16 @@
 // An in-memory job queue: run tasks with a concurrency limit and keep their
 // records and results, for a server that hands out job ids.
 import { availableParallelism } from 'node:os';
-import { newJobId, newJobRecord, transitionJobRecord } from './records.js';
+import { newJobId, newJobRecord, transitionJobRecord, updateJobProgress } from './records.js';
 import { timestampIso } from './util.js';
+/** An in-memory queue with bounded concurrency and retention. Await each job.done to observe its result. */
 export class JobQueue {
     opts;
     jobs = new Map();
     waiting = [];
     running = 0;
     finished = [];
+    /** Create an instance with the supplied configuration; omitted fields use the documented defaults. */
     constructor(opts = {}) {
         this.opts = opts;
     }
@@ -30,7 +32,16 @@ export class JobQueue {
             try {
                 abort.signal.throwIfAborted();
                 update({ status: 'processing' });
-                const result = await task({ signal: abort.signal, stage: (stage) => update({ status: 'processing', stage }) });
+                const result = await task({
+                    signal: abort.signal,
+                    stage: (stage) => update({ status: 'processing', stage }),
+                    progress: (p) => {
+                        if (job.record.status !== 'processing')
+                            return;
+                        job.record = updateJobProgress(job.record, p);
+                        this.opts.onChange?.(job.record);
+                    },
+                });
                 job.result = result;
                 // A cancelled task that still returns (a killed tool) is cancelled all the same.
                 abort.signal.throwIfAborted();
@@ -50,9 +61,11 @@ export class JobQueue {
         this.opts.onChange?.(job.record);
         return job;
     }
+    /** Look up a retained job by ID; returns undefined when unknown or dropped. */
     get(id) {
         return this.jobs.get(id);
     }
+    /** Return the current records for all retained jobs, including pending and finished jobs. */
     list() {
         return [...this.jobs.values()].map((j) => j.record);
     }

package/dist/records.d.ts

@@ -1,17 +1,31 @@
+/** Job lifecycle: pending, processing, done, or error. */
 export type JobStatus = 'pending' | 'processing' | 'done' | 'error';
+/** A serializable job state with lifecycle timestamps, error information, and history. */
 export interface JobRecord {
+    /** Unique identifier for the job. */
     job_id: string;
+    /** Configured function associated with the job. */
     function: string;
+    /** Lifecycle state or HTTP input-error status, according to the containing type. */
     status: JobStatus;
+    /** Current progress stage, or null when unspecified. */
     stage: string | null;
+    /** ISO timestamp when the record was created. */
     started_at: string;
+    /** ISO timestamp of the last transition. */
     updated_at: string;
+    /** ISO timestamp when execution finished, or null while unfinished. */
     completed_at: string | null;
+    /** Failure message, or null while no error has been recorded. */
     error: string | null;
+    /** Storage key of the result, or null when absent. */
     result_key: string | null;
+    /** The latest update from the tool's `progress` role (spec section 14), when it has one. */
+    progress?: Record<string, unknown> | null;
+    /** Earlier job states, newest first. */
     history: {
-        status: JobStatus;
-        stage: string | null;
+        status: JobStatus; /** Current progress stage, or null when unspecified. */
+        stage: string | null; /** ISO timestamp of the last transition. */
         updated_at: string;
     }[];
     [field: string]: unknown;
@@ -20,18 +34,24 @@ export interface JobRecord {
 export declare function newJobId(seed: string, prefix?: string): string;
 /** A new record in the `pending` state; `fields` are flattened onto it. */
 export declare function newJobRecord(p: {
-    jobId: string;
+    jobId: string; /** Name of the configured function to invoke. */
     functionName: string;
     fields?: Record<string, unknown>;
     now?: string;
 }): JobRecord;
 /** Move a record on, pushing its current state onto the front of `history`. Returns a new record. */
 export declare function transitionJobRecord(record: JobRecord, change: {
-    status: JobStatus;
-    stage?: string | null;
+    status: JobStatus; /** Current progress stage, or null when unspecified. */
+    stage?: string | null; /** Write a timestamped error to stderr; supports printf-style arguments. */
     error?: string | null;
     resultKey?: string | null;
     completedAt?: string | null;
     now?: string;
     extra?: Record<string, unknown>;
 }): JobRecord;
+/**
+ * Record a progress update (spec section 14) without a history entry, so frequent updates don't
+ * grow it: `progress` is the update, and its `stage`, when it has one, becomes the record's.
+ */
+export declare function updateJobProgress(record: JobRecord, progress: Record<string, unknown>, now?: string): JobRecord;
+//# sourceMappingURL=records.d.ts.map

package/dist/records.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAOA,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpE,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,MAAM,EAAE,SAAS,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,iDAAiD;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,iEAAiE;IACjE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,sDAAsD;IACtD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,wCAAwC;IACxC,OAAO,EAAE;QAAuF,MAAM,EAAE,SAAS,CAAC,CAAC,wDAAwD;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,4CAA4C;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,gGAAgG;AAChG,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,SAAQ,GAAG,MAAM,CAG7D;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC,CAAC,iDAAiD;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAepL;AAED,qGAAqG;AACrG,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE;IAAuF,MAAM,EAAE,SAAS,CAAC,CAAC,wDAAwD;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,4EAA4E;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAC5Y,SAAS,CAYX;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAG/G"}

package/dist/records.js

@@ -40,3 +40,11 @@ export function transitionJobRecord(record, change) {
         result_key: change.resultKey || null,
     };
 }
+/**
+ * Record a progress update (spec section 14) without a history entry, so frequent updates don't
+ * grow it: `progress` is the update, and its `stage`, when it has one, becomes the record's.
+ */
+export function updateJobProgress(record, progress, now) {
+    const stage = typeof progress.stage === 'string' ? progress.stage : record.stage;
+    return { ...record, progress, stage, updated_at: now ?? timestampIso() };
+}

package/dist/template.d.ts

@@ -1,8 +1,10 @@
+/** Values available to dotted-path template expressions. */
 export type TemplateContext = {
     [key: string]: unknown;
 };
 /** A dot-path value from the context; unknown paths are errors, objects become JSON. */
 export declare function lookupTemplateValue(context: TemplateContext, expression: string): string;
+/** Replace dotted-path ${expression} and {{expression}} templates using the context. */
 export declare function renderTemplateString(value: string, context: TemplateContext): string;
 /** Render strings and the strings in arrays; other scalars keep their type. */
 export declare function renderConfigValue(value: unknown, context?: TemplateContext | null): unknown;
@@ -12,3 +14,4 @@ export declare function renderResultValue(value: unknown, context: TemplateConte
 export declare function renderResultMap(resultMap: unknown, context: TemplateContext): {
     [key: string]: unknown;
 };
+//# sourceMappingURL=template.d.ts.map

package/dist/template.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"template.d.ts","sourceRoot":"","sources":["../src/template.ts"],"names":[],"mappings":"AACA,4DAA4D;AAC5D,MAAM,MAAM,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEzD,wFAAwF;AACxF,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAUxF;AAED,wFAAwF;AACxF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAIpF;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,GAAG,OAAO,CAK3F;AAED,4EAA4E;AAC5E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAOnF;AAED,0EAA0E;AAC1E,wBAAgB,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAGxG"}

package/dist/template.js

@@ -11,6 +11,7 @@ export function lookupTemplateValue(context, expression) {
         return '';
     return typeof current === 'object' ? JSON.stringify(current) : String(current);
 }
+/** Replace dotted-path ${expression} and {{expression}} templates using the context. */
 export function renderTemplateString(value, context) {
     return String(value).replace(/\$\{\s*([A-Za-z0-9_.-]+)\s*\}|\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g, (_, dollar, brace) => lookupTemplateValue(context, dollar || brace));
 }

package/dist/util.d.ts

@@ -5,14 +5,19 @@ export declare function timestampIso(): string;
 export declare function isEnabled(value: unknown): boolean;
 /** Human-friendly booleans used in config files. */
 export declare function truthy(value: unknown): boolean;
+/** Create the directory and missing ancestors; filesystem failures throw. */
 export declare function ensureDir(dir: string): void;
+/** Read and parse a UTF-8 JSON file; malformed JSON and filesystem failures throw. */
 export declare function readJson<T = unknown>(file: string): T;
 /** Stable pretty-printed JSON with a trailing newline. */
 export declare function writeJson(file: string, value: unknown): void;
 /** stat() following symlinks; null when missing or inaccessible. */
 export declare function statFollowingSymlinks(file: string): Stats | null;
+/** Whether the path exists and refers to a directory, following symlinks. */
 export declare function isDirectoryPath(file: string): boolean;
+/** Whether the path exists and refers to a regular file, following symlinks. */
 export declare function isFilePath(file: string): boolean;
+/** Create destination parents and copy the file; filesystem failures throw. */
 export declare function copyFileEnsuringDir(source: string, destination: string): void;
 /** Path components for templates: `{path, dir, name, basename, ext}`. */
 export declare function pathParts(file: string): {
@@ -22,6 +27,7 @@ export declare function pathParts(file: string): {
     basename: string;
     ext: string;
 };
+/** A JSON object used for job configuration. */
 export type Json = {
     [key: string]: unknown;
 };
@@ -36,3 +42,4 @@ export declare function deepMerge(base: unknown, override: unknown): unknown;
 export declare function resolveConfigPath(root: string, value: string): string;
 /** Same rules as resolveConfigPath, for script paths against the script root. */
 export declare const resolveScriptPath: typeof resolveConfigPath;
+//# sourceMappingURL=util.d.ts.map

package/dist/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AACA,OAAO,EAAmD,KAAK,KAAK,EAAiB,MAAM,SAAS,CAAC;AAGrG,8DAA8D;AAC9D,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,qEAAqE;AACrE,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEjD;AAED,oDAAoD;AACpD,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAE9C;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE3C;AAED,sFAAsF;AACtF,wBAAgB,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAErD;AAED,0DAA0D;AAC1D,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAE5D;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,IAAI,CAMhE;AAED,6EAA6E;AAC7E,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,gFAAgF;AAChF,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEhD;AAED,+EAA+E;AAC/E,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,CAG7E;AAED,yEAAyE;AACzE,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM;;;;;;EAIrC;AAED,gDAAgD;AAChD,MAAM,MAAM,IAAI,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAE9C,4DAA4D;AAC5D,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAEhD;AAED,2EAA2E;AAC3E,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAMnE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAIrE;AAED,iFAAiF;AACjF,eAAO,MAAM,iBAAiB,0BAAoB,CAAC"}

package/dist/util.js

@@ -13,9 +13,11 @@ export function isEnabled(value) {
 export function truthy(value) {
     return /^(true|1|yes|on)$/i.test(String(value));
 }
+/** Create the directory and missing ancestors; filesystem failures throw. */
 export function ensureDir(dir) {
     mkdirSync(dir, { recursive: true });
 }
+/** Read and parse a UTF-8 JSON file; malformed JSON and filesystem failures throw. */
 export function readJson(file) {
     return JSON.parse(readFileSync(file, 'utf8'));
 }
@@ -32,12 +34,15 @@ export function statFollowingSymlinks(file) {
         return null;
     }
 }
+/** Whether the path exists and refers to a directory, following symlinks. */
 export function isDirectoryPath(file) {
     return Boolean(statFollowingSymlinks(file)?.isDirectory());
 }
+/** Whether the path exists and refers to a regular file, following symlinks. */
 export function isFilePath(file) {
     return Boolean(statFollowingSymlinks(file)?.isFile());
 }
+/** Create destination parents and copy the file; filesystem failures throw. */
 export function copyFileEnsuringDir(source, destination) {
     ensureDir(dirname(destination));
     copyFileSync(source, destination);
@@ -46,7 +51,7 @@ export function copyFileEnsuringDir(source, destination) {
 export function pathParts(file) {
     const name = basename(file);
     const ext = extname(name);
-    return { path: file, dir: dirname(file), name, basename: ext ? name.slice(0, -ext.length) : name, ext: ext.replace(/^\./, '') };
+    return { /** Resolved filesystem path or HTTP endpoint path, according to the containing type. */ path: file, /** Directory represented by this group or specified in .clyops settings. */ dir: dirname(file), /** Program, command, option, or argument name as declared in its containing type. */ name, /** Filename without its extension. */ basename: ext ? name.slice(0, -ext.length) : name, /** Filename extension including the leading dot. */ ext: ext.replace(/^\./, '') };
 }
 /** A plain object, or `{}` for scalars, arrays and null. */
 export function objectValue(value) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clyops-jobs",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Run clyops tools as jobs: config-bound functions, templated invocation, artifacts, job records and a queue.",
   "license": "MIT",
   "repository": {
@@ -38,8 +38,9 @@
     "typescript": "^5.6.0"
   },
   "dependencies": {
-    "clyops-tools": "0.3.0"
+    "clyops-tools": "0.5.0"
   },
+  "homepage": "https://clyops.io/node-tools.html",
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",

package/src/artifacts.ts

@@ -7,6 +7,7 @@ import { copyFileEnsuringDir, isDirectoryPath, isFilePath, statFollowingSymlinks
 
 // A small glob subset as a regex: `*`, `?`, `**` and `**/` (which also matches no
 // directory), with `/` separators.
+/** Compile an artifact glob into an anchored regular expression. */
 export function globToRegex(pattern: string): RegExp {
   const source = String(pattern).replace(/\\/g, '/');
   let out = '^';
@@ -87,7 +88,9 @@ export function artifactRequirement(fn: ResolvedFunction): number {
   return artifacts.required === true || truthy(artifacts.required) ? 1 : 0;
 }
 
+/** An artifact copied to its destination, recording the original source path. */
 export interface CopiedArtifact {
+  /** Original source path of the copied artifact. */
   source: string;
   /** `<keyPrefix><artifacts dir name>/<file name>`. */
   key: string;

package/src/functions.ts

@@ -7,7 +7,7 @@
 // Tool options are mapped onto the tool's command line through its
 // --help-json-schema (spec section 13), after `${dot.path}` / `{{dot.path}}`
 // templates are rendered against a context the trigger builds.
-import { inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv, type RunResult, type Schema } from 'clyops-tools';
+import { declaredRoles, inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv, type RoleOption, type RunResult, type Schema } from 'clyops-tools';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { finished } from 'node:stream/promises';
 import { basename, dirname, isAbsolute, resolve } from 'node:path';
@@ -26,13 +26,17 @@ const LOGGER = {
  * arguments; `module` names the trigger in messages such as "--help is
  * controlled by media-dropoff".
  */
-export function configureLogging(opts: { module?: string; warn?: (...args: unknown[]) => void; verbose?: (...args: unknown[]) => void }): void {
+export function configureLogging(opts: { module?: string; /** Write a timestamped warning to stderr; supports printf-style arguments. */ warn?: (...args: unknown[]) => void; verbose?: (...args: unknown[]) => void }): void {
   Object.assign(LOGGER, Object.fromEntries(Object.entries(opts).filter(([, v]) => v)));
 }
 
+/** A configured binding to an executable, optional input/output files, artifacts, and result templates. */
 export interface FunctionDefinition {
+  /** Config key holding function options; defaults to the function name. */
   key?: string;
+  /** Executable path; configured paths are resolved against the script root. */
   script?: string;
+  /** Explicit positional templates; an empty array explicitly supplies no positionals. */
   positionals?: unknown[];
   /** Positionals before or after options (default: first). */
   positionals_order?: 'first' | 'last';
@@ -40,12 +44,16 @@ export interface FunctionDefinition {
   stdin?: string;
   /** A template for the file the tool's stdout is written to; it also counts as an artifact. */
   stdout?: string;
+  /** Artifact configuration consumed by the job engine after execution. */
   artifacts?: unknown;
+  /** Task result or configured result templates, according to the containing interface. */
   result?: unknown;
   [key: string]: unknown;
 }
 
+/** A configured function with resolved executable and merged defaults and overrides. */
 export interface ResolvedFunction {
+  /** Name of the configured function to invoke. */
   functionName: string;
   /** The config key holding its options (default: the function name). */
   key: string;
@@ -53,6 +61,7 @@ export interface ResolvedFunction {
   script: string;
   /** `deepMerge(defaults[key], config[key])`. */
   config: Json;
+  /** Original function binding, including positional, stream, artifact, and result templates. */
   definition: FunctionDefinition;
 }
 
@@ -70,9 +79,11 @@ export function resolveFunctionConfig(scriptRoot: string, routeConfig: Json, fun
   return { functionName, key, script: resolveScriptPath(scriptRoot, script), config, definition };
 }
 
+/** Path base, template context, and controlled arguments used when constructing a job command. */
 export interface CommandOptions {
   /** Directory relative path options resolve against. */
   configRoot: string;
+  /** Values available to dotted-path templates; null disables context-based rendering. */
   context?: TemplateContext | null;
   /** Options the trigger sets itself; config keys for them are ignored with a warning. Default: ['help']. */
   controlled?: string[];
@@ -115,9 +126,24 @@ export async function buildFunctionCommand(fn: ResolvedFunction, opts: CommandOp
   return [fn.script, ...argv];
 }
 
+/** The completed tool execution with a color-stripped stderr tail. */
 export interface FunctionRun extends RunResult {
   /** The last 40 lines of stderr without color codes. */
   stderrTail: string[];
+  /** The tool's `meta` role (spec section 14), parsed, when it declares one and wrote JSON. */
+  meta?: unknown;
+  /** The last update on its `progress` role. */
+  progress?: Record<string, unknown>;
+}
+
+/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
+export function parseRoleJson(text: string | undefined): unknown {
+  if (!text?.trim()) return undefined;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
 }
 
 /** A function's `stdin` or `stdout` template rendered, a relative one against `base`. */
@@ -134,7 +160,15 @@ function streamPath(template: string | undefined, opts: CommandOptions, base: st
  */
 export async function runScriptFunction(
   fn: ResolvedFunction,
-  opts: CommandOptions & { timeoutMs?: number; env?: NodeJS.ProcessEnv; cwd?: string; signal?: AbortSignal; onStderr?: (chunk: string) => void },
+  opts: CommandOptions & {
+    timeoutMs?: number;
+    /** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
+    /** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
+    /** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
+    /** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
+    /** Called with each object the tool writes on its `progress` role (spec section 14). */ onProgress?: (update: Record<string, unknown>) => void;
+    /** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
+  },
 ): Promise<FunctionRun> {
   const schema = await loadSchema(fn.script);
   const [file, ...argv] = await buildFunctionCommand(fn, opts, schema);
@@ -144,10 +178,25 @@ export async function runScriptFunction(
   const workDir = (opts.context?.paths as { work_dir?: string } | undefined)?.work_dir;
   const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
   const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+  // Wire the well-known roles the tool declares: progress onto the record, meta into the run.
+  let progress: Record<string, unknown> | undefined;
+  const roles: Record<string, RoleOption> = {
+    progress: {
+      onLine: (line) => {
+        const update = parseRoleJson(line);
+        if (!update || typeof update !== 'object' || Array.isArray(update)) return;
+        progress = update as Record<string, unknown>;
+        opts.onProgress?.(progress);
+      },
+    },
+    meta: {},
+    log: { onLine: opts.onLog },
+  };
   const started = start(file, argv, {
     cwd: opts.cwd, env: opts.env, timeoutMs: opts.timeoutMs, signal: opts.signal, onStderr: opts.onStderr,
     stdin: stdinPath ? createReadStream(stdinPath) : undefined,
     stdout: stdoutPath ? 'stream' : 'text',
+    roles: declaredRoles({ schema }, roles),
   });
   let written: Promise<void> | undefined;
   if (stdoutPath) {
@@ -155,7 +204,8 @@ export async function runScriptFunction(
     written = finished(started.stdout.pipe(createWriteStream(stdoutPath)));
   }
   const [result] = await Promise.all([started.result, written]);
-  return { ...result, command, stderrTail: tail(result.stderr) };
+  const meta = parseRoleJson(result.roles?.meta);
+  return { ...result, command, stderrTail: tail(result.stderr), ...(meta !== undefined ? { meta } : {}), ...(progress ? { progress } : {}) };
 }
 
 /** The result record every function shares, plus its rendered `result` map. */
@@ -164,6 +214,7 @@ export function buildResultRecord(p: {
   fields?: Json;
   fn: ResolvedFunction;
   artifactKeys: string[];
+  /** Values available to dotted-path templates; null disables context-based rendering. */
   context: TemplateContext;
   now: string;
 }): Json {

package/src/queue.ts

@@ -1,11 +1,14 @@
 // An in-memory job queue: run tasks with a concurrency limit and keep their
 // records and results, for a server that hands out job ids.
 import { availableParallelism } from 'node:os';
-import { newJobId, newJobRecord, transitionJobRecord, type JobRecord } from './records.js';
+import { newJobId, newJobRecord, transitionJobRecord, updateJobProgress, type JobRecord } from './records.js';
 import { timestampIso } from './util.js';
 
+/** A queued task, its current record, completion promise, optional result, and cancellation method. */
 export interface Job<R> {
+  /** Current serializable job state, updated as the task progresses. */
   record: JobRecord;
+  /** Task result or configured result templates, according to the containing interface. */
   result?: R;
   /** Resolves with the result when the job is done (rejects when it fails). */
   done: Promise<R>;
@@ -13,6 +16,7 @@ export interface Job<R> {
   cancel(): void;
 }
 
+/** Concurrency, retention, ID prefix, and lifecycle callbacks for the in-memory queue. */
 export interface QueueOptions {
   /** Jobs run at the same time (default: the number of CPUs). */
   concurrency?: number;
@@ -26,19 +30,25 @@ export interface QueueOptions {
   onDrop?: (record: JobRecord) => void;
 }
 
+/** An in-memory queue with bounded concurrency and retention. Await each job.done to observe its result. */
 export class JobQueue<R = unknown> {
   private jobs = new Map<string, Job<R>>();
   private waiting: (() => void)[] = [];
   private running = 0;
   private finished: string[] = [];
 
+  /** Create an instance with the supplied configuration; omitted fields use the documented defaults. */
   constructor(private opts: QueueOptions = {}) {}
 
   /**
    * Queue `task` as a job of `functionName`. The task gets an AbortSignal and
    * a `stage` setter for progress; its return value is the job's result.
    */
-  add(functionName: string, task: (ctx: { signal: AbortSignal; stage: (stage: string) => void }) => Promise<R>, fields: Record<string, unknown> = {}): Job<R> {
+  add(functionName: string, task: (ctx: {
+    /** Termination signal, or null for an ordinary exit. RunOptions uses an AbortSignal for cancellation. */ signal: AbortSignal;
+    /** Current progress stage, or null when unspecified. */ stage: (stage: string) => void;
+    /** Record a progress update, e.g. runScriptFunction's onProgress (no history entry). */ progress: (update: Record<string, unknown>) => void;
+  }) => Promise<R>, fields: Record<string, unknown> = {}): Job<R> {
     const id = newJobId(functionName, this.opts.prefix ?? 'job');
     const abort = new AbortController();
     const job = { record: newJobRecord({ jobId: id, functionName, fields }) } as Job<R>;
@@ -52,7 +62,15 @@ export class JobQueue<R = unknown> {
       try {
         abort.signal.throwIfAborted();
         update({ status: 'processing' });
-        const result = await task({ signal: abort.signal, stage: (stage) => update({ status: 'processing', stage }) });
+        const result = await task({
+          signal: abort.signal,
+          stage: (stage) => update({ status: 'processing', stage }),
+          progress: (p) => {
+            if (job.record.status !== 'processing') return;
+            job.record = updateJobProgress(job.record, p);
+            this.opts.onChange?.(job.record);
+          },
+        });
         job.result = result;
         // A cancelled task that still returns (a killed tool) is cancelled all the same.
         abort.signal.throwIfAborted();
@@ -71,10 +89,12 @@ export class JobQueue<R = unknown> {
     return job;
   }
 
+  /** Look up a retained job by ID; returns undefined when unknown or dropped. */
   get(id: string): Job<R> | undefined {
     return this.jobs.get(id);
   }
 
+  /** Return the current records for all retained jobs, including pending and finished jobs. */
   list(): JobRecord[] {
     return [...this.jobs.values()].map((j) => j.record);
   }

package/src/records.ts

@@ -5,19 +5,33 @@
 import { createHash } from 'node:crypto';
 import { timestampIso } from './util.js';
 
+/** Job lifecycle: pending, processing, done, or error. */
 export type JobStatus = 'pending' | 'processing' | 'done' | 'error';
 
+/** A serializable job state with lifecycle timestamps, error information, and history. */
 export interface JobRecord {
+  /** Unique identifier for the job. */
   job_id: string;
+  /** Configured function associated with the job. */
   function: string;
+  /** Lifecycle state or HTTP input-error status, according to the containing type. */
   status: JobStatus;
+  /** Current progress stage, or null when unspecified. */
   stage: string | null;
+  /** ISO timestamp when the record was created. */
   started_at: string;
+  /** ISO timestamp of the last transition. */
   updated_at: string;
+  /** ISO timestamp when execution finished, or null while unfinished. */
   completed_at: string | null;
+  /** Failure message, or null while no error has been recorded. */
   error: string | null;
+  /** Storage key of the result, or null when absent. */
   result_key: string | null;
-  history: { status: JobStatus; stage: string | null; updated_at: string }[];
+  /** The latest update from the tool's `progress` role (spec section 14), when it has one. */
+  progress?: Record<string, unknown> | null;
+  /** Earlier job states, newest first. */
+  history: { /** Lifecycle state or HTTP input-error status, according to the containing type. */ status: JobStatus; /** Current progress stage, or null when unspecified. */ stage: string | null; /** ISO timestamp of the last transition. */ updated_at: string }[];
   [field: string]: unknown;
 }
 
@@ -28,7 +42,7 @@ export function newJobId(seed: string, prefix = 'job'): string {
 }
 
 /** A new record in the `pending` state; `fields` are flattened onto it. */
-export function newJobRecord(p: { jobId: string; functionName: string; fields?: Record<string, unknown>; now?: string }): JobRecord {
+export function newJobRecord(p: { jobId: string; /** Name of the configured function to invoke. */ functionName: string; fields?: Record<string, unknown>; now?: string }): JobRecord {
   const now = p.now ?? timestampIso();
   return {
     job_id: p.jobId,
@@ -48,7 +62,7 @@ export function newJobRecord(p: { jobId: string; functionName: string; fields?:
 /** Move a record on, pushing its current state onto the front of `history`. Returns a new record. */
 export function transitionJobRecord(
   record: JobRecord,
-  change: { status: JobStatus; stage?: string | null; error?: string | null; resultKey?: string | null; completedAt?: string | null; now?: string; extra?: Record<string, unknown> },
+  change: { /** Lifecycle state or HTTP input-error status, according to the containing type. */ status: JobStatus; /** Current progress stage, or null when unspecified. */ stage?: string | null; /** Write a timestamped error to stderr; supports printf-style arguments. */ error?: string | null; resultKey?: string | null; completedAt?: string | null; now?: string; extra?: Record<string, unknown> },
 ): JobRecord {
   return {
     ...record,
@@ -62,3 +76,12 @@ export function transitionJobRecord(
     result_key: change.resultKey || null,
   };
 }
+
+/**
+ * Record a progress update (spec section 14) without a history entry, so frequent updates don't
+ * grow it: `progress` is the update, and its `stage`, when it has one, becomes the record's.
+ */
+export function updateJobProgress(record: JobRecord, progress: Record<string, unknown>, now?: string): JobRecord {
+  const stage = typeof progress.stage === 'string' ? progress.stage : record.stage;
+  return { ...record, progress, stage, updated_at: now ?? timestampIso() };
+}

package/src/template.ts

@@ -1,4 +1,5 @@
 // `${dot.path}` and `{{dot.path}}` templates in config values.
+/** Values available to dotted-path template expressions. */
 export type TemplateContext = { [key: string]: unknown };
 
 /** A dot-path value from the context; unknown paths are errors, objects become JSON. */
@@ -14,6 +15,7 @@ export function lookupTemplateValue(context: TemplateContext, expression: string
   return typeof current === 'object' ? JSON.stringify(current) : String(current);
 }
 
+/** Replace dotted-path ${expression} and {{expression}} templates using the context. */
 export function renderTemplateString(value: string, context: TemplateContext): string {
   return String(value).replace(/\$\{\s*([A-Za-z0-9_.-]+)\s*\}|\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}/g, (_, dollar, brace) =>
     lookupTemplateValue(context, dollar || brace),

package/src/util.ts

@@ -17,10 +17,12 @@ export function truthy(value: unknown): boolean {
   return /^(true|1|yes|on)$/i.test(String(value));
 }
 
+/** Create the directory and missing ancestors; filesystem failures throw. */
 export function ensureDir(dir: string): void {
   mkdirSync(dir, { recursive: true });
 }
 
+/** Read and parse a UTF-8 JSON file; malformed JSON and filesystem failures throw. */
 export function readJson<T = unknown>(file: string): T {
   return JSON.parse(readFileSync(file, 'utf8')) as T;
 }
@@ -39,14 +41,17 @@ export function statFollowingSymlinks(file: string): Stats | null {
   }
 }
 
+/** Whether the path exists and refers to a directory, following symlinks. */
 export function isDirectoryPath(file: string): boolean {
   return Boolean(statFollowingSymlinks(file)?.isDirectory());
 }
 
+/** Whether the path exists and refers to a regular file, following symlinks. */
 export function isFilePath(file: string): boolean {
   return Boolean(statFollowingSymlinks(file)?.isFile());
 }
 
+/** Create destination parents and copy the file; filesystem failures throw. */
 export function copyFileEnsuringDir(source: string, destination: string): void {
   ensureDir(dirname(destination));
   copyFileSync(source, destination);
@@ -56,9 +61,10 @@ export function copyFileEnsuringDir(source: string, destination: string): void {
 export function pathParts(file: string) {
   const name = basename(file);
   const ext = extname(name);
-  return { path: file, dir: dirname(file), name, basename: ext ? name.slice(0, -ext.length) : name, ext: ext.replace(/^\./, '') };
+  return { /** Resolved filesystem path or HTTP endpoint path, according to the containing type. */ path: file, /** Directory represented by this group or specified in .clyops settings. */ dir: dirname(file), /** Program, command, option, or argument name as declared in its containing type. */ name, /** Filename without its extension. */ basename: ext ? name.slice(0, -ext.length) : name, /** Filename extension including the leading dot. */ ext: ext.replace(/^\./, '') };
 }
 
+/** A JSON object used for job configuration. */
 export type Json = { [key: string]: unknown };
 
 /** A plain object, or `{}` for scalars, arrays and null. */