clyops-jobs 0.4.0 → 0.6.0

package/README.md

@@ -63,14 +63,24 @@ record = transitionJobRecord(record, { status: 'done', completedAt: timestampIso
 `runScriptFunction` options: `configRoot` (relative path options resolve against it),
 `context`, `controlled` (options the trigger sets itself, default `['help']`), `defaultPositional`
 (a template for the first argument when the config gives none, e.g. `'${media.path}'`),
-`timeoutMs`, `env`, `cwd`, `signal`, `onStderr`. Its `command` shows secret options as `***`.
+`timeoutMs`, `env`, `cwd`, `signal`, `onStderr`, `onProgress`, `onLog`. Its `command` shows secret options as `***`.
+
+When the tool declares the well-known [roles](https://github.com/wankdanker/clyops/blob/main/spec/SPEC.md#14-roles-on-file-descriptors),
+the run wires them up: each `progress` object goes to `onProgress` (and the last one is the run's
+`progress`), `log` lines to `onLog`, and the `meta` document is parsed into the run's `meta`.
+`updateJobProgress(record, update)` puts an update on a record as `progress` (and its `stage`, if
+any) without a history entry; the queue's tasks get it as `progress(update)`:
+
+```js
+queue.add('transcribe', ({ signal, progress }) => runScriptFunction(fn, { configRoot, context, signal, onProgress: progress }));
+```
 `configureLogging({module, warn, verbose})` routes warnings about config keys that match no option.
 
 ## Job records
 
 ```
 { job_id, ...fields, function, status, stage, started_at, updated_at, completed_at, error,
-  result_key, history }
+  result_key, progress?, history }
 ```
 
 `status` moves `pending` → `processing` → `done` | `error`. `newJobId(seed, prefix = 'job')`,

package/clyops-jobs.cjs

@@ -41,6 +41,7 @@ __export(index_exports, {
   newJobId: () => newJobId,
   newJobRecord: () => newJobRecord,
   objectValue: () => objectValue,
+  parseRoleJson: () => parseRoleJson,
   pathParts: () => pathParts,
   readJson: () => readJson,
   renderConfigValue: () => renderConfigValue,
@@ -56,6 +57,7 @@ __export(index_exports, {
   timestampIso: () => timestampIso,
   transitionJobRecord: () => transitionJobRecord,
   truthy: () => truthy,
+  updateJobProgress: () => updateJobProgress,
   writeJson: () => writeJson
 });
 module.exports = __toCommonJS(index_exports);
@@ -342,6 +344,7 @@ function redactArgv(schema, argv) {
 var import_node_child_process2 = require("node:child_process");
 var import_node_stream = require("node:stream");
 var import_node_string_decoder = require("node:string_decoder");
+var ROLE_NAME = /^[A-Za-z][A-Za-z0-9_.-]*$/;
 var ABORT_GRACE_MS = 200;
 var STOP_WAIT_MS = 1e3;
 function terminateTree(child, signal) {
@@ -364,10 +367,15 @@ function terminateTree(child, signal) {
 }
 function start(file, argv, opts = {}) {
   const started = Date.now();
+  const roles = process.platform === "win32" ? [] : Object.entries(opts.roles ?? {});
+  for (const [name] of roles) if (!ROLE_NAME.test(name)) throw new Error(`Invalid role name '${name}'`);
+  const env = { ...opts.env ?? process.env };
+  delete env.CLYOPS_FDS;
+  if (roles.length) env.CLYOPS_FDS = roles.map(([name], i) => `${name}:${i + 3}`).join(",");
   const child = (0, import_node_child_process2.spawn)(file, argv, {
     cwd: opts.cwd,
-    env: opts.env ?? process.env,
-    stdio: [opts.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe"],
+    env,
+    stdio: [opts.stdin === void 0 ? "ignore" : "pipe", "pipe", "pipe", ...roles.map(() => "pipe")],
     // POSIX detached children lead a new session/group, still with owned pipes.
     detached: process.platform !== "win32",
     windowsHide: true
@@ -394,6 +402,26 @@ function start(file, argv, opts = {}) {
   const mode = opts.stdout ?? "text";
   const stdout = mode === "stream" ? () => Buffer.alloc(0) : collect(childOut, opts.onStdout);
   const stderr = collect(childErr, opts.onStderr);
+  const roleOut = [];
+  const roleIn = [];
+  roles.forEach(([name, role], i) => {
+    const stream = child.stdio[i + 3];
+    if (role.input !== void 0) {
+      roleIn.push({ stream, input: role.input });
+      return;
+    }
+    let partial = "";
+    const lines = role.onLine && ((chunk) => {
+      const parts = (partial + chunk).split("\n");
+      partial = parts.pop() ?? "";
+      for (const line of parts) role.onLine?.(line);
+    });
+    stream.once("end", () => {
+      if (partial) role.onLine?.(partial);
+      partial = "";
+    });
+    roleOut.push({ name, stream, text: collect(stream, lines || void 0) });
+  });
   const result = new Promise((resolve5, reject) => {
     let timedOut = false;
     let failure;
@@ -422,6 +450,10 @@ function start(file, argv, opts = {}) {
         }
       }
       child.stdin?.destroy();
+      for (const { stream, input } of roleIn) {
+        if (input instanceof import_node_stream.Readable) input.unpipe(stream);
+        stream.destroy();
+      }
     };
     const finish = async (exitCode, signal) => {
       if (settled) return;
@@ -447,6 +479,7 @@ function start(file, argv, opts = {}) {
         ...mode === "buffer" ? { stdoutBuffer: out } : {},
         stderr: stderr().toString("utf8"),
         ...truncated ? { truncated } : {},
+        ...roleOut.length ? { roles: Object.fromEntries(roleOut.map((r) => [r.name, r.text().toString("utf8")])) } : {},
         durationMs: Date.now() - started
       });
     };
@@ -463,7 +496,7 @@ function start(file, argv, opts = {}) {
       }
       deadline = setTimeout(() => {
         treeStopped = terminateTree(child, "SIGKILL");
-        for (const stream of [childOut, childErr]) {
+        for (const stream of [childOut, childErr, ...roleOut.map((r) => r.stream)]) {
           if (!stream.destroyed) {
             stream.once("end", () => stream.destroy());
             stream.push(null);
@@ -504,6 +537,13 @@ function start(file, argv, opts = {}) {
         else source.pipe(child.stdin);
       } else child.stdin.end(opts.stdin);
     }
+    for (const { stream, input } of roleIn) {
+      stream.on("error", () => void 0).resume();
+      if (input instanceof import_node_stream.Readable) {
+        input.on("error", inputError);
+        input.pipe(stream);
+      } else stream.end(input);
+    }
     if (opts.signal?.aborted) abort();
   });
   return { child, stdout: childOut, result };
@@ -515,6 +555,12 @@ function shellQuote(args) {
   return args.map((a) => /^[A-Za-z0-9_@%+=:,./-]+$/.test(a) ? a : `'${a.replace(/'/g, "'\\''")}'`).join(" ");
 }
 
+// packages/tools/src/tools.ts
+function declaredRoles(tool, requested) {
+  const declared = new Map((tool.schema.roles ?? []).map((r) => [r.name, r.direction]));
+  return Object.fromEntries(Object.entries(requested).filter(([name, role]) => declared.get(name) === (role.input === void 0 ? "out" : "in")));
+}
+
 // packages/jobs/src/functions.ts
 var import_node_fs4 = require("node:fs");
 var import_promises = require("node:stream/promises");
@@ -562,6 +608,14 @@ async function buildFunctionCommand(fn, opts, schema) {
   for (const key of unknown) LOGGER.warn("[%s] ignoring config key %s; no matching CLI option in schema", name, key);
   return [fn.script, ...argv];
 }
+function parseRoleJson(text) {
+  if (!text?.trim()) return void 0;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return void 0;
+  }
+}
 function streamPath(template, opts, base) {
   if (!template) return void 0;
   const rendered = opts.context ? renderTemplateString(template, opts.context) : template;
@@ -575,6 +629,19 @@ async function runScriptFunction(fn, opts) {
   const workDir = opts.context?.paths?.work_dir;
   const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
   const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+  let progress;
+  const roles = {
+    progress: {
+      onLine: (line) => {
+        const update = parseRoleJson(line);
+        if (!update || typeof update !== "object" || Array.isArray(update)) return;
+        progress = update;
+        opts.onProgress?.(progress);
+      }
+    },
+    meta: {},
+    log: { onLine: opts.onLog }
+  };
   const started = start(file, argv, {
     cwd: opts.cwd,
     env: opts.env,
@@ -582,7 +649,8 @@ async function runScriptFunction(fn, opts) {
     signal: opts.signal,
     onStderr: opts.onStderr,
     stdin: stdinPath ? (0, import_node_fs4.createReadStream)(stdinPath) : void 0,
-    stdout: stdoutPath ? "stream" : "text"
+    stdout: stdoutPath ? "stream" : "text",
+    roles: declaredRoles({ schema }, roles)
   });
   let written;
   if (stdoutPath) {
@@ -590,7 +658,8 @@ async function runScriptFunction(fn, opts) {
     written = (0, import_promises.finished)(started.stdout.pipe((0, import_node_fs4.createWriteStream)(stdoutPath)));
   }
   const [result] = await Promise.all([started.result, written]);
-  return { ...result, command, stderrTail: tail(result.stderr) };
+  const meta = parseRoleJson(result.roles?.meta);
+  return { ...result, command, stderrTail: tail(result.stderr), ...meta !== void 0 ? { meta } : {}, ...progress ? { progress } : {} };
 }
 function buildResultRecord(p) {
   return {
@@ -736,6 +805,10 @@ function transitionJobRecord(record, change) {
     result_key: change.resultKey || null
   };
 }
+function updateJobProgress(record, progress, now) {
+  const stage = typeof progress.stage === "string" ? progress.stage : record.stage;
+  return { ...record, progress, stage, updated_at: now ?? timestampIso() };
+}
 
 // packages/jobs/src/queue.ts
 var import_node_os = require("node:os");
@@ -767,7 +840,15 @@ var JobQueue = class {
       try {
         abort.signal.throwIfAborted();
         update({ status: "processing" });
-        const result = await task({ signal: abort.signal, stage: (stage) => update({ status: "processing", stage }) });
+        const result = await task({
+          signal: abort.signal,
+          stage: (stage) => update({ status: "processing", stage }),
+          progress: (p) => {
+            if (job.record.status !== "processing") return;
+            job.record = updateJobProgress(job.record, p);
+            this.opts.onChange?.(job.record);
+          }
+        });
         job.result = result;
         abort.signal.throwIfAborted();
         update({ status: "done", completedAt: timestampIso() });
@@ -835,6 +916,7 @@ var JobQueue = class {
   newJobId,
   newJobRecord,
   objectValue,
+  parseRoleJson,
   pathParts,
   readJson,
   renderConfigValue,
@@ -850,5 +932,6 @@ var JobQueue = class {
   timestampIso,
   transitionJobRecord,
   truthy,
+  updateJobProgress,
   writeJson
 });

package/dist/functions.d.ts

@@ -69,18 +69,26 @@ export declare function buildFunctionCommand(fn: ResolvedFunction, opts: Command
 export interface FunctionRun extends RunResult {
     /** The last 40 lines of stderr without color codes. */
     stderrTail: string[];
+    /** The tool's `meta` role (spec section 14), parsed, when it declares one and wrote JSON. */
+    meta?: unknown;
+    /** The last update on its `progress` role. */
+    progress?: Record<string, unknown>;
 }
+/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
+export declare function parseRoleJson(text: string | undefined): unknown;
 /**
  * Run a function's tool. Its stdin is the function's `stdin` file, if any;
  * stdout goes to its `stdout` file, else is collected (a tool may print its
  * result), and stderr is collected. Secret options are `***` in `command`.
  */
 export declare function runScriptFunction(fn: ResolvedFunction, opts: CommandOptions & {
-    timeoutMs?: number; /** Environment supplied to the process or read for option defaults. */
-    env?: NodeJS.ProcessEnv; /** Working directory used to resolve command-line paths and execute tools. */
-    cwd?: string; /** AbortSignal used to cancel execution and terminate the tool process tree. */
-    signal?: AbortSignal; /** Called for each decoded stderr chunk. */
-    onStderr?: (chunk: string) => void;
+    timeoutMs?: number;
+    /** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
+    /** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
+    /** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
+    /** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
+    /** Called with each object the tool writes on its `progress` role (spec section 14). */ onProgress?: (update: Record<string, unknown>) => void;
+    /** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
 }): Promise<FunctionRun>;
 /** The result record every function shares, plus its rendered `result` map. */
 export declare function buildResultRecord(p: {

package/dist/functions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"functions.d.ts","sourceRoot":"","sources":["../src/functions.ts"],"names":[],"mappings":"AASA,OAAO,EAAsE,KAAK,SAAS,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAI/H,OAAO,EAA4D,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAC/G,OAAO,EAAwD,KAAK,IAAI,EAAE,MAAM,WAAW,CAAC;AAS5F;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,8EAA8E;IAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;CAAE,GAAG,IAAI,CAE5N;AAED,2GAA2G;AAC3G,MAAM,WAAW,kBAAkB;IACjC,0EAA0E;IAC1E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wFAAwF;IACxF,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC;IACxB,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACrC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yFAAyF;IACzF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,wFAAwF;AACxF,MAAM,WAAW,gBAAgB;IAC/B,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,+FAA+F;IAC/F,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAQ1H;AAED,kGAAkG;AAClG,MAAM,WAAW,cAAc;IAC7B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,2GAA2G;IAC3G,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAYD,+DAA+D;AAC/D,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAmBzH;AAED,sEAAsE;AACtE,MAAM,WAAW,WAAY,SAAQ,SAAS;IAC5C,uDAAuD;IACvD,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AASD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,gBAAgB,EACpB,IAAI,EAAE,cAAc,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,uEAAuE;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,8EAA8E;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,gFAAgF;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC,CAAC,4CAA4C;IAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,GACla,OAAO,CAAC,WAAW,CAAC,CAqBtB;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,CAAC,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,EAAE,EAAE,gBAAgB,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wFAAwF;IACxF,OAAO,EAAE,eAAe,CAAC;IACzB,GAAG,EAAE,MAAM,CAAC;CACb,GAAG,IAAI,CAYP"}
+{"version":3,"file":"functions.d.ts","sourceRoot":"","sources":["../src/functions.ts"],"names":[],"mappings":"AASA,OAAO,EAAsG,KAAK,SAAS,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAI/J,OAAO,EAA4D,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAC/G,OAAO,EAAwD,KAAK,IAAI,EAAE,MAAM,WAAW,CAAC;AAS5F;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,8EAA8E;IAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;CAAE,GAAG,IAAI,CAE5N;AAED,2GAA2G;AAC3G,MAAM,WAAW,kBAAkB;IACjC,0EAA0E;IAC1E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wFAAwF;IACxF,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC;IACxB,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACrC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yFAAyF;IACzF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,wFAAwF;AACxF,MAAM,WAAW,gBAAgB;IAC/B,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,+FAA+F;IAC/F,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAQ1H;AAED,kGAAkG;AAClG,MAAM,WAAW,cAAc;IAC7B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,wFAAwF;IACxF,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,2GAA2G;IAC3G,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAYD,+DAA+D;AAC/D,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAmBzH;AAED,sEAAsE;AACtE,MAAM,WAAW,WAAY,SAAQ,SAAS;IAC5C,uDAAuD;IACvD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,oFAAoF;AACpF,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAO/D;AASD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,gBAAgB,EACpB,IAAI,EAAE,cAAc,GAAG;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uEAAuE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChG,8EAA8E,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAC5F,gFAAgF,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IACtG,4CAA4C,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChF,wFAAwF,CAAC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAChJ,+DAA+D,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAChG,GACA,OAAO,CAAC,WAAW,CAAC,CAqCtB;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,CAAC,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,EAAE,EAAE,gBAAgB,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wFAAwF;IACxF,OAAO,EAAE,eAAe,CAAC;IACzB,GAAG,EAAE,MAAM,CAAC;CACb,GAAG,IAAI,CAYP"}

package/dist/functions.js

@@ -7,7 +7,7 @@
 // Tool options are mapped onto the tool's command line through its
 // --help-json-schema (spec section 13), after `${dot.path}` / `{{dot.path}}`
 // templates are rendered against a context the trigger builds.
-import { inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv } from 'clyops-tools';
+import { declaredRoles, inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv } from 'clyops-tools';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { finished } from 'node:stream/promises';
 import { basename, dirname, isAbsolute, resolve } from 'node:path';
@@ -75,6 +75,17 @@ export async function buildFunctionCommand(fn, opts, schema) {
         LOGGER.warn('[%s] ignoring config key %s; no matching CLI option in schema', name, key);
     return [fn.script, ...argv];
 }
+/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
+export function parseRoleJson(text) {
+    if (!text?.trim())
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
 /** A function's `stdin` or `stdout` template rendered, a relative one against `base`. */
 function streamPath(template, opts, base) {
     if (!template)
@@ -96,10 +107,26 @@ export async function runScriptFunction(fn, opts) {
     const workDir = opts.context?.paths?.work_dir;
     const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
     const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+    // Wire the well-known roles the tool declares: progress onto the record, meta into the run.
+    let progress;
+    const roles = {
+        progress: {
+            onLine: (line) => {
+                const update = parseRoleJson(line);
+                if (!update || typeof update !== 'object' || Array.isArray(update))
+                    return;
+                progress = update;
+                opts.onProgress?.(progress);
+            },
+        },
+        meta: {},
+        log: { onLine: opts.onLog },
+    };
     const started = start(file, argv, {
         cwd: opts.cwd, env: opts.env, timeoutMs: opts.timeoutMs, signal: opts.signal, onStderr: opts.onStderr,
         stdin: stdinPath ? createReadStream(stdinPath) : undefined,
         stdout: stdoutPath ? 'stream' : 'text',
+        roles: declaredRoles({ schema }, roles),
     });
     let written;
     if (stdoutPath) {
@@ -107,7 +134,8 @@ export async function runScriptFunction(fn, opts) {
         written = finished(started.stdout.pipe(createWriteStream(stdoutPath)));
     }
     const [result] = await Promise.all([started.result, written]);
-    return { ...result, command, stderrTail: tail(result.stderr) };
+    const meta = parseRoleJson(result.roles?.meta);
+    return { ...result, command, stderrTail: tail(result.stderr), ...(meta !== undefined ? { meta } : {}), ...(progress ? { progress } : {}) };
 }
 /** The result record every function shares, plus its rendered `result` map. */
 export function buildResultRecord(p) {

package/dist/queue.d.ts

@@ -37,8 +37,9 @@ export declare class JobQueue<R = unknown> {
      * a `stage` setter for progress; its return value is the job's result.
      */
     add(functionName: string, task: (ctx: {
-        signal: AbortSignal; /** Current progress stage, or null when unspecified. */
-        stage: (stage: string) => void;
+        /** Termination signal, or null for an ordinary exit. RunOptions uses an AbortSignal for cancellation. */ signal: AbortSignal;
+        /** Current progress stage, or null when unspecified. */ stage: (stage: string) => void;
+        /** Record a progress update, e.g. runScriptFunction's onProgress (no history entry). */ progress: (update: Record<string, unknown>) => void;
     }) => Promise<R>, fields?: Record<string, unknown>): Job<R>;
     /** Look up a retained job by ID; returns undefined when unknown or dropped. */
     get(id: string): Job<R> | undefined;

package/dist/queue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAGA,OAAO,EAA+C,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAG3F,uGAAuG;AACvG,MAAM,WAAW,GAAG,CAAC,CAAC;IACpB,sEAAsE;IACtE,MAAM,EAAE,SAAS,CAAC;IAClB,yFAAyF;IACzF,MAAM,CAAC,EAAE,CAAC,CAAC;IACX,6EAA6E;IAC7E,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACjB,sCAAsC;IACtC,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,YAAY;IAC3B,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACvC,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;CACtC;AAED,4GAA4G;AAC5G,qBAAa,QAAQ,CAAC,CAAC,GAAG,OAAO;IAOnB,OAAO,CAAC,IAAI;IANxB,OAAO,CAAC,IAAI,CAA6B;IACzC,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,QAAQ,CAAgB;IAEhC,sGAAsG;gBAClF,IAAI,GAAE,YAAiB;IAE3C;;;OAGG;IACH,GAAG,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE;QAA4G,MAAM,EAAE,WAAW,CAAC,CAAC,wDAAwD;QAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAAG,GAAG,CAAC,CAAC,CAAC;IAiC9T,+EAA+E;IAC/E,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,SAAS;IAInC,6FAA6F;IAC7F,IAAI,IAAI,SAAS,EAAE;IAInB,OAAO,CAAC,IAAI;IAQZ,OAAO,CAAC,OAAO;CAYhB"}
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9G,uGAAuG;AACvG,MAAM,WAAW,GAAG,CAAC,CAAC;IACpB,sEAAsE;IACtE,MAAM,EAAE,SAAS,CAAC;IAClB,yFAAyF;IACzF,MAAM,CAAC,EAAE,CAAC,CAAC;IACX,6EAA6E;IAC7E,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACjB,sCAAsC;IACtC,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,YAAY;IAC3B,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACvC,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;CACtC;AAED,4GAA4G;AAC5G,qBAAa,QAAQ,CAAC,CAAC,GAAG,OAAO;IAOnB,OAAO,CAAC,IAAI;IANxB,OAAO,CAAC,IAAI,CAA6B;IACzC,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,QAAQ,CAAgB;IAEhC,sGAAsG;gBAClF,IAAI,GAAE,YAAiB;IAE3C;;;OAGG;IACH,GAAG,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,EAAE;QACpC,yGAAyG,CAAC,MAAM,EAAE,WAAW,CAAC;QAC9H,wDAAwD,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxF,wFAAwF,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;KAC9I,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAAG,GAAG,CAAC,CAAC,CAAC;IAyC/D,+EAA+E;IAC/E,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,SAAS;IAInC,6FAA6F;IAC7F,IAAI,IAAI,SAAS,EAAE;IAInB,OAAO,CAAC,IAAI;IAQZ,OAAO,CAAC,OAAO;CAYhB"}

package/dist/queue.js

@@ -1,7 +1,7 @@
 // An in-memory job queue: run tasks with a concurrency limit and keep their
 // records and results, for a server that hands out job ids.
 import { availableParallelism } from 'node:os';
-import { newJobId, newJobRecord, transitionJobRecord } from './records.js';
+import { newJobId, newJobRecord, transitionJobRecord, updateJobProgress } from './records.js';
 import { timestampIso } from './util.js';
 /** An in-memory queue with bounded concurrency and retention. Await each job.done to observe its result. */
 export class JobQueue {
@@ -32,7 +32,16 @@ export class JobQueue {
             try {
                 abort.signal.throwIfAborted();
                 update({ status: 'processing' });
-                const result = await task({ signal: abort.signal, stage: (stage) => update({ status: 'processing', stage }) });
+                const result = await task({
+                    signal: abort.signal,
+                    stage: (stage) => update({ status: 'processing', stage }),
+                    progress: (p) => {
+                        if (job.record.status !== 'processing')
+                            return;
+                        job.record = updateJobProgress(job.record, p);
+                        this.opts.onChange?.(job.record);
+                    },
+                });
                 job.result = result;
                 // A cancelled task that still returns (a killed tool) is cancelled all the same.
                 abort.signal.throwIfAborted();

package/dist/records.d.ts

@@ -20,6 +20,8 @@ export interface JobRecord {
     error: string | null;
     /** Storage key of the result, or null when absent. */
     result_key: string | null;
+    /** The latest update from the tool's `progress` role (spec section 14), when it has one. */
+    progress?: Record<string, unknown> | null;
     /** Earlier job states, newest first. */
     history: {
         status: JobStatus; /** Current progress stage, or null when unspecified. */
@@ -47,4 +49,9 @@ export declare function transitionJobRecord(record: JobRecord, change: {
     now?: string;
     extra?: Record<string, unknown>;
 }): JobRecord;
+/**
+ * Record a progress update (spec section 14) without a history entry, so frequent updates don't
+ * grow it: `progress` is the update, and its `stage`, when it has one, becomes the record's.
+ */
+export declare function updateJobProgress(record: JobRecord, progress: Record<string, unknown>, now?: string): JobRecord;
 //# sourceMappingURL=records.d.ts.map

package/dist/records.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAOA,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpE,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,MAAM,EAAE,SAAS,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,iDAAiD;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,iEAAiE;IACjE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,sDAAsD;IACtD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,wCAAwC;IACxC,OAAO,EAAE;QAAuF,MAAM,EAAE,SAAS,CAAC,CAAC,wDAAwD;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,4CAA4C;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,gGAAgG;AAChG,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,SAAQ,GAAG,MAAM,CAG7D;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC,CAAC,iDAAiD;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAepL;AAED,qGAAqG;AACrG,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE;IAAuF,MAAM,EAAE,SAAS,CAAC,CAAC,wDAAwD;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,4EAA4E;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAC5Y,SAAS,CAYX"}
+{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAOA,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpE,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,MAAM,EAAE,SAAS,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,iDAAiD;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,iEAAiE;IACjE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,sDAAsD;IACtD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,wCAAwC;IACxC,OAAO,EAAE;QAAuF,MAAM,EAAE,SAAS,CAAC,CAAC,wDAAwD;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,4CAA4C;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,gGAAgG;AAChG,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,SAAQ,GAAG,MAAM,CAG7D;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC,CAAC,iDAAiD;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAepL;AAED,qGAAqG;AACrG,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE;IAAuF,MAAM,EAAE,SAAS,CAAC,CAAC,wDAAwD;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,4EAA4E;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAC5Y,SAAS,CAYX;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAG/G"}

package/dist/records.js

@@ -40,3 +40,11 @@ export function transitionJobRecord(record, change) {
         result_key: change.resultKey || null,
     };
 }
+/**
+ * Record a progress update (spec section 14) without a history entry, so frequent updates don't
+ * grow it: `progress` is the update, and its `stage`, when it has one, becomes the record's.
+ */
+export function updateJobProgress(record, progress, now) {
+    const stage = typeof progress.stage === 'string' ? progress.stage : record.stage;
+    return { ...record, progress, stage, updated_at: now ?? timestampIso() };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clyops-jobs",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Run clyops tools as jobs: config-bound functions, templated invocation, artifacts, job records and a queue.",
   "license": "MIT",
   "repository": {
@@ -38,7 +38,7 @@
     "typescript": "^5.6.0"
   },
   "dependencies": {
-    "clyops-tools": "0.4.0"
+    "clyops-tools": "0.6.0"
   },
   "homepage": "https://clyops.io/node-tools.html",
   "scripts": {

package/src/functions.ts

@@ -7,7 +7,7 @@
 // Tool options are mapped onto the tool's command line through its
 // --help-json-schema (spec section 13), after `${dot.path}` / `{{dot.path}}`
 // templates are rendered against a context the trigger builds.
-import { inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv, type RunResult, type Schema } from 'clyops-tools';
+import { declaredRoles, inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv, type RoleOption, type RunResult, type Schema } from 'clyops-tools';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { finished } from 'node:stream/promises';
 import { basename, dirname, isAbsolute, resolve } from 'node:path';
@@ -130,6 +130,20 @@ export async function buildFunctionCommand(fn: ResolvedFunction, opts: CommandOp
 export interface FunctionRun extends RunResult {
   /** The last 40 lines of stderr without color codes. */
   stderrTail: string[];
+  /** The tool's `meta` role (spec section 14), parsed, when it declares one and wrote JSON. */
+  meta?: unknown;
+  /** The last update on its `progress` role. */
+  progress?: Record<string, unknown>;
+}
+
+/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
+export function parseRoleJson(text: string | undefined): unknown {
+  if (!text?.trim()) return undefined;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
 }
 
 /** A function's `stdin` or `stdout` template rendered, a relative one against `base`. */
@@ -146,7 +160,15 @@ function streamPath(template: string | undefined, opts: CommandOptions, base: st
  */
 export async function runScriptFunction(
   fn: ResolvedFunction,
-  opts: CommandOptions & { timeoutMs?: number; /** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv; /** Working directory used to resolve command-line paths and execute tools. */ cwd?: string; /** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal; /** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void },
+  opts: CommandOptions & {
+    timeoutMs?: number;
+    /** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
+    /** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
+    /** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
+    /** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
+    /** Called with each object the tool writes on its `progress` role (spec section 14). */ onProgress?: (update: Record<string, unknown>) => void;
+    /** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
+  },
 ): Promise<FunctionRun> {
   const schema = await loadSchema(fn.script);
   const [file, ...argv] = await buildFunctionCommand(fn, opts, schema);
@@ -156,10 +178,25 @@ export async function runScriptFunction(
   const workDir = (opts.context?.paths as { work_dir?: string } | undefined)?.work_dir;
   const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot);
   const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot);
+  // Wire the well-known roles the tool declares: progress onto the record, meta into the run.
+  let progress: Record<string, unknown> | undefined;
+  const roles: Record<string, RoleOption> = {
+    progress: {
+      onLine: (line) => {
+        const update = parseRoleJson(line);
+        if (!update || typeof update !== 'object' || Array.isArray(update)) return;
+        progress = update as Record<string, unknown>;
+        opts.onProgress?.(progress);
+      },
+    },
+    meta: {},
+    log: { onLine: opts.onLog },
+  };
   const started = start(file, argv, {
     cwd: opts.cwd, env: opts.env, timeoutMs: opts.timeoutMs, signal: opts.signal, onStderr: opts.onStderr,
     stdin: stdinPath ? createReadStream(stdinPath) : undefined,
     stdout: stdoutPath ? 'stream' : 'text',
+    roles: declaredRoles({ schema }, roles),
   });
   let written: Promise<void> | undefined;
   if (stdoutPath) {
@@ -167,7 +204,8 @@ export async function runScriptFunction(
     written = finished(started.stdout.pipe(createWriteStream(stdoutPath)));
   }
   const [result] = await Promise.all([started.result, written]);
-  return { ...result, command, stderrTail: tail(result.stderr) };
+  const meta = parseRoleJson(result.roles?.meta);
+  return { ...result, command, stderrTail: tail(result.stderr), ...(meta !== undefined ? { meta } : {}), ...(progress ? { progress } : {}) };
 }
 
 /** The result record every function shares, plus its rendered `result` map. */

package/src/queue.ts

@@ -1,7 +1,7 @@
 // An in-memory job queue: run tasks with a concurrency limit and keep their
 // records and results, for a server that hands out job ids.
 import { availableParallelism } from 'node:os';
-import { newJobId, newJobRecord, transitionJobRecord, type JobRecord } from './records.js';
+import { newJobId, newJobRecord, transitionJobRecord, updateJobProgress, type JobRecord } from './records.js';
 import { timestampIso } from './util.js';
 
 /** A queued task, its current record, completion promise, optional result, and cancellation method. */
@@ -44,7 +44,11 @@ export class JobQueue<R = unknown> {
    * Queue `task` as a job of `functionName`. The task gets an AbortSignal and
    * a `stage` setter for progress; its return value is the job's result.
    */
-  add(functionName: string, task: (ctx: { /** Termination signal, or null for an ordinary exit. RunOptions uses an AbortSignal for cancellation. */ signal: AbortSignal; /** Current progress stage, or null when unspecified. */ stage: (stage: string) => void }) => Promise<R>, fields: Record<string, unknown> = {}): Job<R> {
+  add(functionName: string, task: (ctx: {
+    /** Termination signal, or null for an ordinary exit. RunOptions uses an AbortSignal for cancellation. */ signal: AbortSignal;
+    /** Current progress stage, or null when unspecified. */ stage: (stage: string) => void;
+    /** Record a progress update, e.g. runScriptFunction's onProgress (no history entry). */ progress: (update: Record<string, unknown>) => void;
+  }) => Promise<R>, fields: Record<string, unknown> = {}): Job<R> {
     const id = newJobId(functionName, this.opts.prefix ?? 'job');
     const abort = new AbortController();
     const job = { record: newJobRecord({ jobId: id, functionName, fields }) } as Job<R>;
@@ -58,7 +62,15 @@ export class JobQueue<R = unknown> {
       try {
         abort.signal.throwIfAborted();
         update({ status: 'processing' });
-        const result = await task({ signal: abort.signal, stage: (stage) => update({ status: 'processing', stage }) });
+        const result = await task({
+          signal: abort.signal,
+          stage: (stage) => update({ status: 'processing', stage }),
+          progress: (p) => {
+            if (job.record.status !== 'processing') return;
+            job.record = updateJobProgress(job.record, p);
+            this.opts.onChange?.(job.record);
+          },
+        });
         job.result = result;
         // A cancelled task that still returns (a killed tool) is cancelled all the same.
         abort.signal.throwIfAborted();

package/src/records.ts

@@ -28,6 +28,8 @@ export interface JobRecord {
   error: string | null;
   /** Storage key of the result, or null when absent. */
   result_key: string | null;
+  /** The latest update from the tool's `progress` role (spec section 14), when it has one. */
+  progress?: Record<string, unknown> | null;
   /** Earlier job states, newest first. */
   history: { /** Lifecycle state or HTTP input-error status, according to the containing type. */ status: JobStatus; /** Current progress stage, or null when unspecified. */ stage: string | null; /** ISO timestamp of the last transition. */ updated_at: string }[];
   [field: string]: unknown;
@@ -74,3 +76,12 @@ export function transitionJobRecord(
     result_key: change.resultKey || null,
   };
 }
+
+/**
+ * Record a progress update (spec section 14) without a history entry, so frequent updates don't
+ * grow it: `progress` is the update, and its `stage`, when it has one, becomes the record's.
+ */
+export function updateJobProgress(record: JobRecord, progress: Record<string, unknown>, now?: string): JobRecord {
+  const stage = typeof progress.stage === 'string' ? progress.stage : record.stage;
+  return { ...record, progress, stage, updated_at: now ?? timestampIso() };
+}