clyops-jobs 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -0
- package/clyops-jobs.cjs +124 -3
- package/config.schema.json +113 -0
- package/dist/functions.d.ts +86 -11
- package/dist/functions.d.ts.map +1 -1
- package/dist/functions.js +146 -6
- package/dist/template.d.ts +2 -0
- package/dist/template.d.ts.map +1 -1
- package/dist/template.js +17 -0
- package/package.json +8 -5
- package/src/functions.ts +208 -20
- package/src/template.ts +12 -0
package/README.md
CHANGED
|
@@ -27,6 +27,19 @@ and no `node_modules`. It includes clyops-tools. `require('clyops-jobs')` loads
|
|
|
27
27
|
}
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
+
**Editor support.** The config has a JSON Schema, so editors complete and check it. Name it in
|
|
31
|
+
the file, from the site or from the installed package:
|
|
32
|
+
|
|
33
|
+
```json
|
|
34
|
+
{ "$schema": "https://clyops.io/schemas/clyops-jobs.json", "functions": {} }
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
It's `config.schema.json` in the package (`clyops-jobs/config.schema.json`, e.g.
|
|
38
|
+
`"./node_modules/clyops-jobs/config.schema.json"`). It catches typos in function and step
|
|
39
|
+
fields (`x-` fields are free for your own notes), `script` and `steps` together, and bad step
|
|
40
|
+
names; that a branch names a later step is checked when the function is resolved. Options
|
|
41
|
+
blocks are any object, since their keys come from each tool's schema.
|
|
42
|
+
|
|
30
43
|
- `functions.NAME.script` is the tool, relative to a script root.
|
|
31
44
|
- `key` names the options block (default: the function name). Options are
|
|
32
45
|
`deepMerge(defaults[key], config[key])`, keyed like the tool's options (`out_dir`, `out-dir`)
|
|
@@ -48,6 +61,73 @@ and no `node_modules`. It includes clyops-tools. `require('clyops-jobs')` loads
|
|
|
48
61
|
(`"${paths.work_dir}/out.mp3"`; relative to `paths.work_dir`), which also counts as an artifact.
|
|
49
62
|
- `result`: a template map added to the result record.
|
|
50
63
|
|
|
64
|
+
### Steps
|
|
65
|
+
|
|
66
|
+
A function can run several tools in order instead of one `script`, so a definition chains
|
|
67
|
+
tools that already exist (fetch some JSON, download what it names, process the file):
|
|
68
|
+
|
|
69
|
+
```json
|
|
70
|
+
{
|
|
71
|
+
"functions": {
|
|
72
|
+
"fingerprint": {
|
|
73
|
+
"steps": [
|
|
74
|
+
{ "name": "fetch", "script": "tools/http-get-json" },
|
|
75
|
+
{ "name": "download", "script": "tools/download", "stdout": "${job.id}/media" },
|
|
76
|
+
{ "name": "fp", "script": "tools/media-to-fp", "stdin": "${steps.download.stdout}" }
|
|
77
|
+
],
|
|
78
|
+
"artifacts": { "include": ["${job.id}/*.fp"], "required": true },
|
|
79
|
+
"result": { "size": "${steps.fetch.result.size}" }
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
"fetch": { "url": "https://media.example.com/recording/${params.uuid}", "token": "${secrets.token}" },
|
|
83
|
+
"download": { "url": "${steps.fetch.result.signedUrl}" }
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
- Each step is a binding like a function: `name` (letters, digits, `_`, `-`), `script`, and
|
|
88
|
+
optionally `key` (its options block, default the step name), `positionals`,
|
|
89
|
+
`positionals_order`, `stdin`, `stdout` and `strict_params` (default the function's).
|
|
90
|
+
`artifacts` and `result` belong to the function, as do the caller-input rules: a caller
|
|
91
|
+
reaches a step only through `params` fields its options name.
|
|
92
|
+
- Every step shares the context, `paths.work_dir` included, so files pass by path. A step's
|
|
93
|
+
outputs are `steps.NAME` in the templates after it: `exit_code`; `stdout`, the path of its
|
|
94
|
+
`stdout` file or else its collected output without trailing newlines; `result`, its
|
|
95
|
+
`result` role or else its collected stdout parsed as JSON; `meta`; `progress`. A
|
|
96
|
+
`${steps...}` path that isn't there, such as a later step or a field the JSON lacks, is an
|
|
97
|
+
error naming the step.
|
|
98
|
+
- `previous` is the step that ran just before, on whichever path led here, with the same
|
|
99
|
+
fields plus its `name`: a handler several branches share reads `${previous.name}`,
|
|
100
|
+
`${previous.exit_code}`, `${previous.result.field}` or, after an `error` branch,
|
|
101
|
+
`${previous.error}`. The first step has none, so naming it there is an error.
|
|
102
|
+
- After a step that succeeds the chain goes on to the next step, or ends after the last one.
|
|
103
|
+
A step that exits nonzero, times out or is stopped ends the chain; the run is that step's,
|
|
104
|
+
with `failedStep` naming it. A step that can't start (a template, stdin or schema error)
|
|
105
|
+
rejects the run. `continue_on_exit: [2]` on a step makes those codes a success, and the
|
|
106
|
+
chain then exits with the first of them (a partial result).
|
|
107
|
+
- Branches change where a step leads: `succeed`, `fail` (a nonzero exit or timeout) and
|
|
108
|
+
`error` (it couldn't start) each name a later step, and `succeed: "$end"` ends the chain.
|
|
109
|
+
Branches only go forward, so a chain always ends. A step reached only through a branch,
|
|
110
|
+
such as a fallback or a cleanup, still sits in the list, so the step before it usually
|
|
111
|
+
needs a `succeed`. The chain's exit code is its last step's: a fallback that succeeds
|
|
112
|
+
makes the job succeed, and a cleanup that should leave it failed exits nonzero. A step
|
|
113
|
+
that took its `error` branch is `steps.NAME.error` (the message), with `exit_code: null`.
|
|
114
|
+
A cancelled chain, or one out of time, takes no branch.
|
|
115
|
+
|
|
116
|
+
```json
|
|
117
|
+
"steps": [
|
|
118
|
+
{ "name": "fetch", "script": "tools/http-get-json", "fail": "mirror", "succeed": "process" },
|
|
119
|
+
{ "name": "mirror", "script": "tools/http-get-json" },
|
|
120
|
+
{ "name": "process", "script": "tools/media-to-fp" }
|
|
121
|
+
]
|
|
122
|
+
```
|
|
123
|
+
- `timeoutMs` limits the whole chain and a step's `timeout_ms` that step; `signal` stops the
|
|
124
|
+
running step. `onProgress` gets each step's updates with `step` and a default `stage`.
|
|
125
|
+
- The run is the last step's, plus `steps` (the outputs by name), `previous` (the last step)
|
|
126
|
+
and `stepRuns`. Render artifacts and the result with them through `runContext(context,
|
|
127
|
+
run)`, so a result map can use `${previous...}` whichever branch the chain ended on.
|
|
128
|
+
- `functionParamNames(fn)` lists the `params` fields a function's templates use, steps
|
|
129
|
+
included, for describing its caller inputs.
|
|
130
|
+
|
|
51
131
|
```js
|
|
52
132
|
import { resolveFunctionConfig, runScriptFunction, copyConfiguredArtifacts, buildResultRecord,
|
|
53
133
|
newJobId, newJobRecord, transitionJobRecord, timestampIso } from 'clyops-jobs';
|
|
@@ -73,6 +153,7 @@ record = transitionJobRecord(record, { status: 'done', completedAt: timestampIso
|
|
|
73
153
|
When the tool declares the well-known [roles](https://github.com/wankdanker/clyops/blob/main/spec/SPEC.md#14-roles-on-file-descriptors),
|
|
74
154
|
the run wires them up: each `progress` object goes to `onProgress` (and the last one is the run's
|
|
75
155
|
`progress`), `log` lines to `onLog`, and the `meta` document is parsed into the run's `meta`.
|
|
156
|
+
The run's `result` is the `result` role parsed as JSON, or else the collected stdout when it is JSON.
|
|
76
157
|
`updateJobProgress(record, update)` puts an update on a record as `progress` (and its `stage`, if
|
|
77
158
|
any) without a history entry; the queue's tasks get it as `progress(update)`:
|
|
78
159
|
|
package/clyops-jobs.cjs
CHANGED
|
@@ -32,6 +32,7 @@ __export(index_exports, {
|
|
|
32
32
|
deepMerge: () => deepMerge,
|
|
33
33
|
ensureDir: () => ensureDir,
|
|
34
34
|
findFilesByArtifactPattern: () => findFilesByArtifactPattern,
|
|
35
|
+
functionParamNames: () => functionParamNames,
|
|
35
36
|
functionTemplateOptions: () => functionTemplateOptions,
|
|
36
37
|
globToRegex: () => globToRegex,
|
|
37
38
|
isDirectoryPath: () => isDirectoryPath,
|
|
@@ -53,9 +54,11 @@ __export(index_exports, {
|
|
|
53
54
|
resolveConfigPath: () => resolveConfigPath,
|
|
54
55
|
resolveFunctionConfig: () => resolveFunctionConfig,
|
|
55
56
|
resolveScriptPath: () => resolveScriptPath,
|
|
57
|
+
runContext: () => runContext,
|
|
56
58
|
runScriptFunction: () => runScriptFunction,
|
|
57
59
|
shellQuote: () => shellQuote,
|
|
58
60
|
statFollowingSymlinks: () => statFollowingSymlinks,
|
|
61
|
+
templateParamNames: () => templateParamNames,
|
|
59
62
|
timestampIso: () => timestampIso,
|
|
60
63
|
transitionJobRecord: () => transitionJobRecord,
|
|
61
64
|
truthy: () => truthy,
|
|
@@ -170,6 +173,16 @@ function isUnsetTemplate(value, context, opts) {
|
|
|
170
173
|
}
|
|
171
174
|
return false;
|
|
172
175
|
}
|
|
176
|
+
function templateParamNames(value, names = /* @__PURE__ */ new Set()) {
|
|
177
|
+
if (typeof value === "string") {
|
|
178
|
+
for (const [, dollar, brace] of value.matchAll(TEMPLATE)) {
|
|
179
|
+
const [root, field] = String(dollar || brace).split(".").filter(Boolean);
|
|
180
|
+
if (OPTIONAL_ROOTS.includes(root ?? "") && field) names.add(field);
|
|
181
|
+
}
|
|
182
|
+
} else if (Array.isArray(value)) for (const item of value) templateParamNames(item, names);
|
|
183
|
+
else if (value && typeof value === "object") for (const item of Object.values(value)) templateParamNames(item, names);
|
|
184
|
+
return names;
|
|
185
|
+
}
|
|
173
186
|
function renderTemplateString(value, context, opts) {
|
|
174
187
|
return String(value).replace(TEMPLATE, (_, dollar, brace) => lookupTemplateValue(context, dollar || brace, opts));
|
|
175
188
|
}
|
|
@@ -615,15 +628,51 @@ var LOGGER = {
|
|
|
615
628
|
function configureLogging(opts) {
|
|
616
629
|
Object.assign(LOGGER, Object.fromEntries(Object.entries(opts).filter(([, v]) => v)));
|
|
617
630
|
}
|
|
631
|
+
var STEP_NAME = /^[A-Za-z0-9_-]+$/;
|
|
632
|
+
var END = "$end";
|
|
618
633
|
function resolveFunctionConfig(scriptRoot, routeConfig, functionName) {
|
|
619
634
|
const definition = objectValue(objectValue(routeConfig.functions)[functionName]);
|
|
620
635
|
if (Object.keys(definition).length === 0) return null;
|
|
621
636
|
const key = String(definition.key || functionName);
|
|
637
|
+
if (definition.steps === void 0) return resolveBinding(scriptRoot, routeConfig, functionName, key, definition, `function ${functionName}`);
|
|
638
|
+
if (definition.script) throw new Error(`function ${functionName} defines both script and steps`);
|
|
639
|
+
if (!Array.isArray(definition.steps) || definition.steps.length === 0) throw new Error(`function ${functionName} defines no steps`);
|
|
640
|
+
const steps = [];
|
|
641
|
+
for (const [i, item] of definition.steps.entries()) {
|
|
642
|
+
const step = { strict_params: definition.strict_params, ...objectValue(item) };
|
|
643
|
+
const name = String(step.name ?? "");
|
|
644
|
+
if (!STEP_NAME.test(name)) throw new Error(`function ${functionName} step ${i + 1} needs a name of letters, digits, _ or -`);
|
|
645
|
+
if (steps.some((s) => s.stepName === name)) throw new Error(`function ${functionName} has two steps named ${name}`);
|
|
646
|
+
const where = `function ${functionName} step ${name}`;
|
|
647
|
+
steps.push({ ...resolveBinding(scriptRoot, routeConfig, functionName, String(step.key || name), step, where), stepName: name, definition: step });
|
|
648
|
+
}
|
|
649
|
+
for (const [i, step] of steps.entries()) {
|
|
650
|
+
for (const outcome of ["succeed", "fail", "error"]) {
|
|
651
|
+
const target = step.definition[outcome];
|
|
652
|
+
if (target === void 0 || outcome === "succeed" && target === END) continue;
|
|
653
|
+
const at = steps.findIndex((s) => s.stepName === target);
|
|
654
|
+
if (at < 0) throw new Error(`function ${functionName} step ${step.stepName}: ${outcome} names no step ${String(target)}`);
|
|
655
|
+
if (at <= i) throw new Error(`function ${functionName} step ${step.stepName}: ${outcome} must name a later step, not ${String(target)}`);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
const config = Object.fromEntries(steps.map((s) => [s.stepName, s.config]));
|
|
659
|
+
return { functionName, key, script: steps[steps.length - 1].script, config, definition, steps };
|
|
660
|
+
}
|
|
661
|
+
function resolveBinding(scriptRoot, routeConfig, functionName, key, definition, where) {
|
|
622
662
|
const script = String(definition.script || "");
|
|
623
|
-
if (!script) throw new Error(
|
|
663
|
+
if (!script) throw new Error(`${where} does not define a script`);
|
|
624
664
|
const config = deepMerge(objectValue(objectValue(routeConfig.defaults)[key]), objectValue(routeConfig[key]));
|
|
625
665
|
return { functionName, key, script: resolveScriptPath(scriptRoot, script), config, definition };
|
|
626
666
|
}
|
|
667
|
+
function functionParamNames(fn) {
|
|
668
|
+
const names = /* @__PURE__ */ new Set();
|
|
669
|
+
for (const part of fn.steps ?? [fn]) {
|
|
670
|
+
const { positionals, stdin, stdout } = part.definition;
|
|
671
|
+
templateParamNames([part.config, positionals, stdin, stdout], names);
|
|
672
|
+
}
|
|
673
|
+
templateParamNames([fn.definition.artifacts, fn.definition.result], names);
|
|
674
|
+
return [...names].sort();
|
|
675
|
+
}
|
|
627
676
|
function functionTemplateOptions(fn) {
|
|
628
677
|
return { strict: fn.definition.strict_params === true };
|
|
629
678
|
}
|
|
@@ -641,6 +690,7 @@ function explicitPositionals(fn, context) {
|
|
|
641
690
|
return fn.definition.positionals.filter((item) => !context || !isUnsetTemplate(item, context, templates)).map((item) => String(renderConfigValue(item, context, templates))).filter(Boolean);
|
|
642
691
|
}
|
|
643
692
|
async function buildFunctionCommand(fn, opts, schema) {
|
|
693
|
+
if (fn.steps) throw new Error(`function ${fn.functionName} runs steps; build each step's command`);
|
|
644
694
|
schema ??= await loadSchema(fn.script);
|
|
645
695
|
const context = opts.context;
|
|
646
696
|
const templates = functionTemplateOptions(fn);
|
|
@@ -675,6 +725,9 @@ function streamPath(template, opts, base, templates) {
|
|
|
675
725
|
return (0, import_node_path3.isAbsolute)(rendered) ? rendered : (0, import_node_path3.resolve)(base, rendered);
|
|
676
726
|
}
|
|
677
727
|
async function runScriptFunction(fn, opts) {
|
|
728
|
+
return fn.steps ? runSteps(fn.steps, opts) : (await runTool(fn, opts)).run;
|
|
729
|
+
}
|
|
730
|
+
async function runTool(fn, opts) {
|
|
678
731
|
const schema = await loadSchema(fn.script);
|
|
679
732
|
const [file, ...argv] = await buildFunctionCommand(fn, opts, schema);
|
|
680
733
|
const command = [file, ...redactArgv(schema, argv)];
|
|
@@ -693,8 +746,10 @@ async function runScriptFunction(fn, opts) {
|
|
|
693
746
|
}
|
|
694
747
|
},
|
|
695
748
|
meta: {},
|
|
749
|
+
result: {},
|
|
696
750
|
log: { onLine: opts.onLog }
|
|
697
751
|
};
|
|
752
|
+
const stdin = stdinPath ? (0, import_node_fs4.createReadStream)(stdinPath, { fd: (0, import_node_fs4.openSync)(stdinPath, "r") }) : void 0;
|
|
698
753
|
const started = start(file, argv, {
|
|
699
754
|
cwd: opts.cwd,
|
|
700
755
|
env: opts.env,
|
|
@@ -702,7 +757,7 @@ async function runScriptFunction(fn, opts) {
|
|
|
702
757
|
killGraceMs: opts.killGraceMs,
|
|
703
758
|
signal: opts.signal,
|
|
704
759
|
onStderr: opts.onStderr,
|
|
705
|
-
stdin
|
|
760
|
+
stdin,
|
|
706
761
|
stdout: stdoutPath ? "stream" : "text",
|
|
707
762
|
roles: declaredRoles({ schema }, roles)
|
|
708
763
|
});
|
|
@@ -713,7 +768,70 @@ async function runScriptFunction(fn, opts) {
|
|
|
713
768
|
}
|
|
714
769
|
const [result] = await Promise.all([started.result, written]);
|
|
715
770
|
const meta = parseRoleJson(result.roles?.meta);
|
|
716
|
-
|
|
771
|
+
const output = parseRoleJson(result.roles?.result) ?? (stdoutPath ? void 0 : parseRoleJson(result.stdout));
|
|
772
|
+
const run = {
|
|
773
|
+
...result,
|
|
774
|
+
command,
|
|
775
|
+
stderrTail: tail(result.stderr),
|
|
776
|
+
...meta !== void 0 ? { meta } : {},
|
|
777
|
+
...progress ? { progress } : {},
|
|
778
|
+
...output !== void 0 ? { result: output } : {}
|
|
779
|
+
};
|
|
780
|
+
return { run, stdoutPath };
|
|
781
|
+
}
|
|
782
|
+
async function runSteps(steps, opts) {
|
|
783
|
+
const deadline = opts.timeoutMs ? Date.now() + opts.timeoutMs : void 0;
|
|
784
|
+
const outputs = {};
|
|
785
|
+
const stepRuns = [];
|
|
786
|
+
const byName = new Map(steps.map((step, i) => [step.stepName, i]));
|
|
787
|
+
const after = (i, branch, fallthrough) => branch === END ? void 0 : branch ? byName.get(branch) : fallthrough && i + 1 < steps.length ? i + 1 : void 0;
|
|
788
|
+
let continued;
|
|
789
|
+
let last;
|
|
790
|
+
let previous;
|
|
791
|
+
for (let i = 0; i !== void 0; ) {
|
|
792
|
+
const step = steps[i];
|
|
793
|
+
const name = step.stepName;
|
|
794
|
+
const limits = [step.definition.timeout_ms, deadline && Math.max(1, deadline - Date.now())].filter((ms) => Number(ms) > 0);
|
|
795
|
+
const context = { ...opts.context, steps: { ...outputs }, ...previous ? { previous } : {} };
|
|
796
|
+
const onProgress = opts.onProgress && ((update) => opts.onProgress?.({ stage: name, ...update, step: name }));
|
|
797
|
+
let ran;
|
|
798
|
+
try {
|
|
799
|
+
ran = await runTool(step, { ...opts, context, onProgress, timeoutMs: limits.length ? Math.min(...limits) : void 0 });
|
|
800
|
+
} catch (error) {
|
|
801
|
+
const message = `step ${name}: ${error.message}`;
|
|
802
|
+
if (!step.definition.error) throw new Error(message, { cause: error });
|
|
803
|
+
outputs[name] = { exit_code: null, stdout: "", error: message };
|
|
804
|
+
previous = { name, ...outputs[name] };
|
|
805
|
+
i = byName.get(step.definition.error);
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
const { run, stdoutPath } = ran;
|
|
809
|
+
stepRuns.push({ ...run, step: name });
|
|
810
|
+
outputs[name] = {
|
|
811
|
+
exit_code: run.exitCode,
|
|
812
|
+
stdout: stdoutPath ?? run.stdout.replace(/\n+$/, ""),
|
|
813
|
+
...run.result !== void 0 ? { result: run.result } : {},
|
|
814
|
+
...run.meta !== void 0 ? { meta: run.meta } : {},
|
|
815
|
+
...run.progress ? { progress: run.progress } : {}
|
|
816
|
+
};
|
|
817
|
+
previous = { name, ...outputs[name] };
|
|
818
|
+
last = { ...run, steps: outputs, previous, stepRuns, durationMs: stepRuns.reduce((ms, r) => ms + r.durationMs, 0) };
|
|
819
|
+
const ok = !run.timedOut && (run.exitCode === 0 || run.exitCode !== null && (step.definition.continue_on_exit ?? []).includes(run.exitCode));
|
|
820
|
+
if (ok) {
|
|
821
|
+
if (run.exitCode !== 0) continued ??= run.exitCode ?? void 0;
|
|
822
|
+
i = after(i, step.definition.succeed, true);
|
|
823
|
+
if (i === void 0) return { ...last, exitCode: run.exitCode || continued || 0 };
|
|
824
|
+
continue;
|
|
825
|
+
}
|
|
826
|
+
const stopped = opts.signal?.aborted || deadline !== void 0 && Date.now() >= deadline;
|
|
827
|
+
i = stopped ? void 0 : after(i, step.definition.fail, false);
|
|
828
|
+
if (i === void 0) return { ...last, failedStep: name };
|
|
829
|
+
}
|
|
830
|
+
if (!last) throw new Error("unreachable: a chain ends after a step that ran");
|
|
831
|
+
return last;
|
|
832
|
+
}
|
|
833
|
+
function runContext(context, run) {
|
|
834
|
+
return { ...context, ...run.steps ? { steps: run.steps } : {}, ...run.previous ? { previous: run.previous } : {} };
|
|
717
835
|
}
|
|
718
836
|
function buildResultRecord(p) {
|
|
719
837
|
return {
|
|
@@ -961,6 +1079,7 @@ var JobQueue = class {
|
|
|
961
1079
|
deepMerge,
|
|
962
1080
|
ensureDir,
|
|
963
1081
|
findFilesByArtifactPattern,
|
|
1082
|
+
functionParamNames,
|
|
964
1083
|
functionTemplateOptions,
|
|
965
1084
|
globToRegex,
|
|
966
1085
|
isDirectoryPath,
|
|
@@ -982,9 +1101,11 @@ var JobQueue = class {
|
|
|
982
1101
|
resolveConfigPath,
|
|
983
1102
|
resolveFunctionConfig,
|
|
984
1103
|
resolveScriptPath,
|
|
1104
|
+
runContext,
|
|
985
1105
|
runScriptFunction,
|
|
986
1106
|
shellQuote,
|
|
987
1107
|
statFollowingSymlinks,
|
|
1108
|
+
templateParamNames,
|
|
988
1109
|
timestampIso,
|
|
989
1110
|
transitionJobRecord,
|
|
990
1111
|
truthy,
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://clyops.io/schemas/clyops-jobs.json",
|
|
4
|
+
"title": "clyops-jobs config",
|
|
5
|
+
"description": "Functions bound to clyops tools, their default options and their option blocks. Every other top-level key is an options block, named by a function's or step's `key`.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"properties": {
|
|
8
|
+
"$schema": { "type": "string", "description": "This schema's URL, for editors: https://clyops.io/schemas/clyops-jobs.json" },
|
|
9
|
+
"functions": {
|
|
10
|
+
"type": "object",
|
|
11
|
+
"description": "The functions a trigger can run, by name.",
|
|
12
|
+
"additionalProperties": { "$ref": "#/$defs/function" }
|
|
13
|
+
},
|
|
14
|
+
"defaults": {
|
|
15
|
+
"type": "object",
|
|
16
|
+
"description": "Default options by block key, merged under the block of the same key.",
|
|
17
|
+
"additionalProperties": { "$ref": "#/$defs/options" }
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"additionalProperties": { "$ref": "#/$defs/options" },
|
|
21
|
+
"$defs": {
|
|
22
|
+
"template": {
|
|
23
|
+
"type": "string",
|
|
24
|
+
"description": "Text with `${dot.path}` or `{{dot.path}}` templates rendered against the trigger's context (`job`, `paths`, `params`, `steps`, ...)."
|
|
25
|
+
},
|
|
26
|
+
"stepName": { "type": "string", "pattern": "^[A-Za-z0-9_-]+$" },
|
|
27
|
+
"options": {
|
|
28
|
+
"type": "object",
|
|
29
|
+
"description": "Tool options keyed like the tool's options (`out_dir` or `out-dir`) or arguments, mapped onto its command line through --help-json-schema. String values are templates; an option naming a missing `params` field is left unset."
|
|
30
|
+
},
|
|
31
|
+
"binding": {
|
|
32
|
+
"type": "object",
|
|
33
|
+
"properties": {
|
|
34
|
+
"key": { "type": "string", "description": "The options block (a top-level key and a `defaults` key); default: the function's or step's name." },
|
|
35
|
+
"script": { "type": "string", "description": "The tool, relative to the script root." },
|
|
36
|
+
"positionals": {
|
|
37
|
+
"type": "array",
|
|
38
|
+
"description": "The tool's positional arguments as templates, in order; `[]` for none. Without it, arguments are options keyed by argument name.",
|
|
39
|
+
"items": { "type": ["string", "number", "boolean"] }
|
|
40
|
+
},
|
|
41
|
+
"positionals_order": { "enum": ["first", "last"], "description": "Positionals before (default) or after the options." },
|
|
42
|
+
"stdin": { "$ref": "#/$defs/template", "description": "A template naming the file fed to the tool's stdin, relative to the config root." },
|
|
43
|
+
"stdout": { "$ref": "#/$defs/template", "description": "A template naming the file the tool's stdout is written to, relative to `paths.work_dir`." },
|
|
44
|
+
"strict_params": { "type": "boolean", "description": "A missing `params` field is an error instead of leaving the option unset (default false)." }
|
|
45
|
+
},
|
|
46
|
+
"patternProperties": { "^x-": { "description": "Free for triggers and comments; clyops-jobs ignores it." } }
|
|
47
|
+
},
|
|
48
|
+
"function": {
|
|
49
|
+
"description": "A function runs one tool (`script`) or several in order (`steps`).",
|
|
50
|
+
"type": "object",
|
|
51
|
+
"allOf": [{ "$ref": "#/$defs/binding" }],
|
|
52
|
+
"properties": {
|
|
53
|
+
"key": true, "script": true, "positionals": true, "positionals_order": true, "stdin": true, "stdout": true, "strict_params": true,
|
|
54
|
+
"steps": {
|
|
55
|
+
"type": "array",
|
|
56
|
+
"minItems": 1,
|
|
57
|
+
"description": "Tools run in order with one context; each step's outputs are `steps.NAME` (`exit_code`, `stdout`, `result`, `meta`, `progress`, `error`) in the templates after it, and the step that ran just before is `previous` (the same fields plus `name`).",
|
|
58
|
+
"items": { "$ref": "#/$defs/step" }
|
|
59
|
+
},
|
|
60
|
+
"artifacts": { "$ref": "#/$defs/artifacts" },
|
|
61
|
+
"result": { "type": "object", "description": "Templates added to the result record; nested objects and arrays are rendered too." }
|
|
62
|
+
},
|
|
63
|
+
"patternProperties": { "^x-": true },
|
|
64
|
+
"additionalProperties": false,
|
|
65
|
+
"oneOf": [
|
|
66
|
+
{ "required": ["script"], "not": { "required": ["steps"] } },
|
|
67
|
+
{ "required": ["steps"], "not": { "required": ["script"] } }
|
|
68
|
+
]
|
|
69
|
+
},
|
|
70
|
+
"step": {
|
|
71
|
+
"description": "One tool in a function's `steps`.",
|
|
72
|
+
"type": "object",
|
|
73
|
+
"allOf": [{ "$ref": "#/$defs/binding" }],
|
|
74
|
+
"required": ["name", "script"],
|
|
75
|
+
"properties": {
|
|
76
|
+
"key": true, "script": true, "positionals": true, "positionals_order": true, "stdin": true, "stdout": true, "strict_params": true,
|
|
77
|
+
"name": { "$ref": "#/$defs/stepName", "description": "Names the step in `${steps.NAME...}` templates, branches and errors." },
|
|
78
|
+
"timeout_ms": { "type": "integer", "exclusiveMinimum": 0, "description": "This step's time limit; the whole chain also stays within the run's timeoutMs." },
|
|
79
|
+
"continue_on_exit": {
|
|
80
|
+
"type": "array",
|
|
81
|
+
"items": { "type": "integer" },
|
|
82
|
+
"description": "Exit codes besides 0 that count as success; the chain then reports the first of them, e.g. [2] for a partial result."
|
|
83
|
+
},
|
|
84
|
+
"succeed": {
|
|
85
|
+
"anyOf": [{ "$ref": "#/$defs/stepName" }, { "const": "$end" }],
|
|
86
|
+
"description": "The later step to run after this one succeeds, instead of the next; `$end` ends the chain."
|
|
87
|
+
},
|
|
88
|
+
"fail": { "$ref": "#/$defs/stepName", "description": "The later step to run when this one exits nonzero or times out, instead of ending the chain." },
|
|
89
|
+
"error": { "$ref": "#/$defs/stepName", "description": "The later step to run when this one can't start (a template, stdin or schema error), instead of failing the run." }
|
|
90
|
+
},
|
|
91
|
+
"patternProperties": { "^x-": true },
|
|
92
|
+
"additionalProperties": false
|
|
93
|
+
},
|
|
94
|
+
"artifacts": {
|
|
95
|
+
"description": "Files copied from `paths.work_dir` into `paths.artifacts_dir` after the run: glob patterns (`*`, `?`, `**`), or an object with them and a requirement.",
|
|
96
|
+
"anyOf": [
|
|
97
|
+
{ "$ref": "#/$defs/template" },
|
|
98
|
+
{ "type": "array", "items": { "$ref": "#/$defs/template" } },
|
|
99
|
+
{
|
|
100
|
+
"type": "object",
|
|
101
|
+
"properties": {
|
|
102
|
+
"include": { "anyOf": [{ "$ref": "#/$defs/template" }, { "type": "array", "items": { "$ref": "#/$defs/template" } }], "description": "Glob patterns relative to `paths.work_dir`." },
|
|
103
|
+
"includes": { "$ref": "#/$defs/artifacts/anyOf/2/properties/include", "description": "Same as include." },
|
|
104
|
+
"patterns": { "$ref": "#/$defs/artifacts/anyOf/2/properties/include", "description": "Same as include." },
|
|
105
|
+
"required": { "type": ["boolean", "string"], "description": "At least one file must match." },
|
|
106
|
+
"min": { "type": "integer", "minimum": 0, "description": "At least this many files must match." }
|
|
107
|
+
},
|
|
108
|
+
"additionalProperties": false
|
|
109
|
+
}
|
|
110
|
+
]
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
package/dist/functions.d.ts
CHANGED
|
@@ -31,8 +31,27 @@ export interface FunctionDefinition {
|
|
|
31
31
|
result?: unknown;
|
|
32
32
|
/** A missing `params` field in a template is an error instead of unset (default false). */
|
|
33
33
|
strict_params?: boolean;
|
|
34
|
+
/** Tools run in order instead of one `script`; each step's outputs are `steps.NAME` in later templates. */
|
|
35
|
+
steps?: StepDefinition[];
|
|
34
36
|
[key: string]: unknown;
|
|
35
37
|
}
|
|
38
|
+
/** One tool in a function's `steps`: a function binding with a name, without artifacts or a result map. */
|
|
39
|
+
export interface StepDefinition extends FunctionDefinition {
|
|
40
|
+
/** Names the step in `${steps.NAME...}` templates and errors; letters, digits, `_` and `-`. */
|
|
41
|
+
name: string;
|
|
42
|
+
/** Config key holding the step's options; defaults to the step name. */
|
|
43
|
+
key?: string;
|
|
44
|
+
/** This step's time limit; the whole chain also stays within the run's `timeoutMs`. */
|
|
45
|
+
timeout_ms?: number;
|
|
46
|
+
/** Exit codes besides 0 that count as success; the chain then reports the first of them. */
|
|
47
|
+
continue_on_exit?: number[];
|
|
48
|
+
/** The later step to run after this one succeeds instead of the next, or `$end` to end the chain there. */
|
|
49
|
+
succeed?: string;
|
|
50
|
+
/** The later step to run when this one exits nonzero or times out, instead of ending the chain. */
|
|
51
|
+
fail?: string;
|
|
52
|
+
/** The later step to run when this one can't start (a template, stdin or schema error), instead of throwing. */
|
|
53
|
+
error?: string;
|
|
54
|
+
}
|
|
36
55
|
/** A configured function with resolved executable and merged defaults and overrides. */
|
|
37
56
|
export interface ResolvedFunction {
|
|
38
57
|
/** Name of the configured function to invoke. */
|
|
@@ -45,17 +64,32 @@ export interface ResolvedFunction {
|
|
|
45
64
|
config: Json;
|
|
46
65
|
/** Original function binding, including positional, stream, artifact, and result templates. */
|
|
47
66
|
definition: FunctionDefinition;
|
|
67
|
+
/** A chain's steps, in order; its `script` is then the last step's and `config` is keyed by step name. */
|
|
68
|
+
steps?: ResolvedStep[];
|
|
69
|
+
}
|
|
70
|
+
/** A resolved step of a chain. */
|
|
71
|
+
export interface ResolvedStep extends ResolvedFunction {
|
|
72
|
+
/** The step's `name`. */
|
|
73
|
+
stepName: string;
|
|
74
|
+
/** The step's binding. */
|
|
75
|
+
definition: StepDefinition;
|
|
48
76
|
}
|
|
49
77
|
/**
|
|
50
78
|
* The binding for `functionName` in a merged config, or null when the config
|
|
51
79
|
* has no such function. A function without a `script` is an error.
|
|
52
80
|
*/
|
|
53
81
|
export declare function resolveFunctionConfig(scriptRoot: string, routeConfig: Json, functionName: string): ResolvedFunction | null;
|
|
82
|
+
/**
|
|
83
|
+
* The caller fields (`params.NAME`) a function's templates use, sorted: in its
|
|
84
|
+
* options and its or its steps' positionals, `stdin`, `stdout`, artifacts and
|
|
85
|
+
* result. A trigger can describe a function's inputs with it.
|
|
86
|
+
*/
|
|
87
|
+
export declare function functionParamNames(fn: ResolvedFunction): string[];
|
|
54
88
|
/** Path base, template context, and controlled arguments used when constructing a job command. */
|
|
55
89
|
export interface CommandOptions {
|
|
56
90
|
/** Directory relative path options resolve against. */
|
|
57
91
|
configRoot: string;
|
|
58
|
-
/** Values available to dotted-path templates; null disables context-based rendering. */
|
|
92
|
+
/** Values available to dotted-path templates; null disables context-based rendering, except in a chain's steps. */
|
|
59
93
|
context?: TemplateContext | null;
|
|
60
94
|
/** Options the trigger sets itself; config keys for them are ignored with a warning. Default: ['help']. */
|
|
61
95
|
controlled?: string[];
|
|
@@ -77,24 +111,65 @@ export interface FunctionRun extends RunResult {
|
|
|
77
111
|
meta?: unknown;
|
|
78
112
|
/** The last update on its `progress` role. */
|
|
79
113
|
progress?: Record<string, unknown>;
|
|
114
|
+
/** Its `result` role parsed, else its collected stdout parsed, when either is JSON. */
|
|
115
|
+
result?: unknown;
|
|
116
|
+
/** A chain's outputs by step name, for templates after it (see runContext). */
|
|
117
|
+
steps?: Record<string, StepOutput>;
|
|
118
|
+
/** A chain's last step that ran or took its `error` branch, with its name: `previous` in templates after the run. */
|
|
119
|
+
previous?: StepOutput & {
|
|
120
|
+
name: string;
|
|
121
|
+
};
|
|
122
|
+
/** A chain's runs, one per step that ran, in order. */
|
|
123
|
+
stepRuns?: Array<FunctionRun & {
|
|
124
|
+
step: string;
|
|
125
|
+
}>;
|
|
126
|
+
/** The step that stopped a chain: a nonzero exit (not in its `continue_on_exit`) or timeout with no `fail` branch, or a signal. */
|
|
127
|
+
failedStep?: string;
|
|
80
128
|
}
|
|
129
|
+
/** What a step leaves for later templates as `steps.NAME`. */
|
|
130
|
+
export interface StepOutput {
|
|
131
|
+
/** Its exit code, or null when a signal stopped it. */
|
|
132
|
+
exit_code: number | null;
|
|
133
|
+
/** The path of its `stdout` file, else its collected stdout without trailing newlines. */
|
|
134
|
+
stdout: string;
|
|
135
|
+
/** Its `result` role or stdout as JSON; absent when neither is. */
|
|
136
|
+
result?: unknown;
|
|
137
|
+
/** Its `meta` role as JSON. */
|
|
138
|
+
meta?: unknown;
|
|
139
|
+
/** Its last `progress` update. */
|
|
140
|
+
progress?: Record<string, unknown>;
|
|
141
|
+
/** Why it couldn't start, when it has an `error` branch. */
|
|
142
|
+
error?: string;
|
|
143
|
+
}
|
|
144
|
+
/** Options for running a function's tool, or each tool of a chain. */
|
|
145
|
+
export type RunFunctionOptions = CommandOptions & {
|
|
146
|
+
/** For a chain, the whole chain's limit; a step's `timeout_ms` limits that step too. */ timeoutMs?: number;
|
|
147
|
+
/** How long a stopped tool (abort or timeout) has after SIGTERM before SIGKILL; default 200. */ killGraceMs?: number;
|
|
148
|
+
/** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
|
|
149
|
+
/** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
|
|
150
|
+
/** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
|
|
151
|
+
/** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
|
|
152
|
+
/** Called with each object the tool writes on its `progress` role (spec section 14); a chain's carry `step` and a default `stage`. */ onProgress?: (update: Record<string, unknown>) => void;
|
|
153
|
+
/** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
|
|
154
|
+
};
|
|
81
155
|
/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
|
|
82
156
|
export declare function parseRoleJson(text: string | undefined): unknown;
|
|
83
157
|
/**
|
|
84
158
|
* Run a function's tool. Its stdin is the function's `stdin` file, if any;
|
|
85
159
|
* stdout goes to its `stdout` file, else is collected (a tool may print its
|
|
86
160
|
* result), and stderr is collected. Secret options are `***` in `command`.
|
|
161
|
+
*
|
|
162
|
+
* A function with `steps` runs them in order with one context, each seeing the
|
|
163
|
+
* earlier ones as `steps.NAME`. A step's `succeed`, `fail` and `error` name a
|
|
164
|
+
* later step to go to instead. A failure with no `fail` branch stops the chain
|
|
165
|
+
* and is its `failedStep`; the run is that of the last step that ran.
|
|
87
166
|
*/
|
|
88
|
-
export declare function runScriptFunction(fn: ResolvedFunction, opts:
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
/** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
|
|
95
|
-
/** Called with each object the tool writes on its `progress` role (spec section 14). */ onProgress?: (update: Record<string, unknown>) => void;
|
|
96
|
-
/** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
|
|
97
|
-
}): Promise<FunctionRun>;
|
|
167
|
+
export declare function runScriptFunction(fn: ResolvedFunction, opts: RunFunctionOptions): Promise<FunctionRun>;
|
|
168
|
+
/**
|
|
169
|
+
* The context to render a function's artifacts and result map with after a run:
|
|
170
|
+
* a chain's `steps` and `previous` (its last step) added to the run's context.
|
|
171
|
+
*/
|
|
172
|
+
export declare function runContext<T extends TemplateContext>(context: T, run: FunctionRun): T & Pick<FunctionRun, 'steps' | 'previous'>;
|
|
98
173
|
/** The result record every function shares, plus its rendered `result` map. */
|
|
99
174
|
export declare function buildResultRecord(p: {
|
|
100
175
|
jobId: string;
|
package/dist/functions.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"functions.d.ts","sourceRoot":"","sources":["../src/functions.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"functions.d.ts","sourceRoot":"","sources":["../src/functions.ts"],"names":[],"mappings":"AAUA,OAAO,EAAsG,KAAK,SAAS,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAI/J,OAAO,EAAiG,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAC1K,OAAO,EAAwD,KAAK,IAAI,EAAE,MAAM,WAAW,CAAC;AAS5F;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,8EAA8E;IAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;CAAE,GAAG,IAAI,CAE5N;AAED,2GAA2G;AAC3G,MAAM,WAAW,kBAAkB;IACjC,0EAA0E;IAC1E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wFAAwF;IACxF,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC;IACxB,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACrC,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yEAAyE;IACzE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yFAAyF;IACzF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2FAA2F;IAC3F,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,2GAA2G;IAC3G,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,2GAA2G;AAC3G,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,+FAA+F;IAC/F,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,uFAAuF;IACvF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4FAA4F;IAC5F,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,2GAA2G;IAC3G,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mGAAmG;IACnG,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gHAAgH;IAChH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wFAAwF;AACxF,MAAM,WAAW,gBAAgB;IAC/B,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,GAAG,EAAE,MAAM,CAAC;IACZ,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,+FAA+F;IAC/F,UAAU,EAAE,kBAAkB,CAAC;IAC/B,0GAA0G;IAC1G,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,kCAAkC;AAClC,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACpD,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,UAAU,EAAE,cAAc,CAAC;CAC5B;AAMD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CA4B1H;AASD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAQjE;AAED,kGAAkG;AAClG,MAAM,WAAW,cAAc;IAC7B,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,mHAAmH;IACnH,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,2GAA2G;IAC3G,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,6DAA6D;AAC7D,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,gBAAgB,GAAG,eAAe,CAE7E;AA8BD,+DAA+D;AAC/D,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAsBzH;AAED,sEAAsE;AACtE,MAAM,WAAW,WAAY,SAAQ,SAAS;IAC5C,uDAAuD;IACvD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,uFAAuF;IACvF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+EAA+E;IAC/E,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACnC,qHAAqH;IACrH,QAAQ,CAAC,EAAE,UAAU,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,uDAAuD;IACvD,QAAQ,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,mIAAmI;IACnI,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8DAA8D;AAC9D,MAAM,WAAW,UAAU;IACzB,uDAAuD;IACvD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0FAA0F;IAC1F,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,sEAAsE;AACtE,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG;IAChD,wFAAwF,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5G,gGAAgG,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IACtH,uEAAuE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChG,8EAA8E,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAC5F,gFAAgF,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IACtG,4CAA4C,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAChF,sIAAsI,CAAC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAC9L,+DAA+D,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAChG,CAAC;AAEF,oFAAoF;AACpF,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAO/D;AASD;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC,CAE5G;AA4GD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,CAAC,SAAS,eAAe,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,GAAG,UAAU,CAAC,CAE/H;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,CAAC,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,EAAE,EAAE,gBAAgB,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wFAAwF;IACxF,OAAO,EAAE,eAAe,CAAC;IACzB,GAAG,EAAE,MAAM,CAAC;CACb,GAAG,IAAI,CAYP"}
|
package/dist/functions.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Config-bound functions: which tool a function runs and with what options.
|
|
2
2
|
//
|
|
3
3
|
// { "functions": { "<name>": { "key", "script", "positionals"?, "stdin"?, "stdout"?, "artifacts"?, "result"? } },
|
|
4
|
+
// { "<name>": { "steps": [{ "name", "key"?, "script", ... }], "artifacts"?, "result"? } },
|
|
4
5
|
// "defaults": { "<key>": { <tool options> } },
|
|
5
6
|
// "<key>": { <overrides> } }
|
|
6
7
|
//
|
|
@@ -8,10 +9,10 @@
|
|
|
8
9
|
// --help-json-schema (spec section 13), after `${dot.path}` / `{{dot.path}}`
|
|
9
10
|
// templates are rendered against a context the trigger builds.
|
|
10
11
|
import { declaredRoles, inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv } from 'clyops-tools';
|
|
11
|
-
import { createReadStream, createWriteStream } from 'node:fs';
|
|
12
|
+
import { createReadStream, createWriteStream, openSync } from 'node:fs';
|
|
12
13
|
import { finished } from 'node:stream/promises';
|
|
13
14
|
import { basename, dirname, isAbsolute, resolve } from 'node:path';
|
|
14
|
-
import { isUnsetTemplate, renderConfigValue, renderResultMap, renderTemplateString } from './template.js';
|
|
15
|
+
import { isUnsetTemplate, renderConfigValue, renderResultMap, renderTemplateString, templateParamNames } from './template.js';
|
|
15
16
|
import { deepMerge, ensureDir, objectValue, resolveScriptPath } from './util.js';
|
|
16
17
|
/** Where engine diagnostics go; a trigger installs its own with configureLogging. */
|
|
17
18
|
const LOGGER = {
|
|
@@ -27,6 +28,9 @@ const LOGGER = {
|
|
|
27
28
|
export function configureLogging(opts) {
|
|
28
29
|
Object.assign(LOGGER, Object.fromEntries(Object.entries(opts).filter(([, v]) => v)));
|
|
29
30
|
}
|
|
31
|
+
const STEP_NAME = /^[A-Za-z0-9_-]+$/;
|
|
32
|
+
// The `succeed` target that ends a chain; `$` can't start a step name.
|
|
33
|
+
const END = '$end';
|
|
30
34
|
/**
|
|
31
35
|
* The binding for `functionName` in a merged config, or null when the config
|
|
32
36
|
* has no such function. A function without a `script` is an error.
|
|
@@ -36,12 +40,60 @@ export function resolveFunctionConfig(scriptRoot, routeConfig, functionName) {
|
|
|
36
40
|
if (Object.keys(definition).length === 0)
|
|
37
41
|
return null;
|
|
38
42
|
const key = String(definition.key || functionName);
|
|
43
|
+
if (definition.steps === undefined)
|
|
44
|
+
return resolveBinding(scriptRoot, routeConfig, functionName, key, definition, `function ${functionName}`);
|
|
45
|
+
if (definition.script)
|
|
46
|
+
throw new Error(`function ${functionName} defines both script and steps`);
|
|
47
|
+
if (!Array.isArray(definition.steps) || definition.steps.length === 0)
|
|
48
|
+
throw new Error(`function ${functionName} defines no steps`);
|
|
49
|
+
const steps = [];
|
|
50
|
+
for (const [i, item] of definition.steps.entries()) {
|
|
51
|
+
const step = { strict_params: definition.strict_params, ...objectValue(item) };
|
|
52
|
+
const name = String(step.name ?? '');
|
|
53
|
+
if (!STEP_NAME.test(name))
|
|
54
|
+
throw new Error(`function ${functionName} step ${i + 1} needs a name of letters, digits, _ or -`);
|
|
55
|
+
if (steps.some((s) => s.stepName === name))
|
|
56
|
+
throw new Error(`function ${functionName} has two steps named ${name}`);
|
|
57
|
+
const where = `function ${functionName} step ${name}`;
|
|
58
|
+
steps.push({ ...resolveBinding(scriptRoot, routeConfig, functionName, String(step.key || name), step, where), stepName: name, definition: step });
|
|
59
|
+
}
|
|
60
|
+
// Branches go forward to a step that exists, so a chain always ends.
|
|
61
|
+
for (const [i, step] of steps.entries()) {
|
|
62
|
+
for (const outcome of ['succeed', 'fail', 'error']) {
|
|
63
|
+
const target = step.definition[outcome];
|
|
64
|
+
if (target === undefined || (outcome === 'succeed' && target === END))
|
|
65
|
+
continue;
|
|
66
|
+
const at = steps.findIndex((s) => s.stepName === target);
|
|
67
|
+
if (at < 0)
|
|
68
|
+
throw new Error(`function ${functionName} step ${step.stepName}: ${outcome} names no step ${String(target)}`);
|
|
69
|
+
if (at <= i)
|
|
70
|
+
throw new Error(`function ${functionName} step ${step.stepName}: ${outcome} must name a later step, not ${String(target)}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const config = Object.fromEntries(steps.map((s) => [s.stepName, s.config]));
|
|
74
|
+
return { functionName, key, script: steps[steps.length - 1].script, config, definition, steps };
|
|
75
|
+
}
|
|
76
|
+
function resolveBinding(scriptRoot, routeConfig, functionName, key, definition, where) {
|
|
39
77
|
const script = String(definition.script || '');
|
|
40
78
|
if (!script)
|
|
41
|
-
throw new Error(
|
|
79
|
+
throw new Error(`${where} does not define a script`);
|
|
42
80
|
const config = deepMerge(objectValue(objectValue(routeConfig.defaults)[key]), objectValue(routeConfig[key]));
|
|
43
81
|
return { functionName, key, script: resolveScriptPath(scriptRoot, script), config, definition };
|
|
44
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* The caller fields (`params.NAME`) a function's templates use, sorted: in its
|
|
85
|
+
* options and its or its steps' positionals, `stdin`, `stdout`, artifacts and
|
|
86
|
+
* result. A trigger can describe a function's inputs with it.
|
|
87
|
+
*/
|
|
88
|
+
export function functionParamNames(fn) {
|
|
89
|
+
const names = new Set();
|
|
90
|
+
for (const part of fn.steps ?? [fn]) {
|
|
91
|
+
const { positionals, stdin, stdout } = part.definition;
|
|
92
|
+
templateParamNames([part.config, positionals, stdin, stdout], names);
|
|
93
|
+
}
|
|
94
|
+
templateParamNames([fn.definition.artifacts, fn.definition.result], names);
|
|
95
|
+
return [...names].sort();
|
|
96
|
+
}
|
|
45
97
|
/** The template options a function's definition asks for. */
|
|
46
98
|
export function functionTemplateOptions(fn) {
|
|
47
99
|
return { strict: fn.definition.strict_params === true };
|
|
@@ -76,6 +128,8 @@ function explicitPositionals(fn, context) {
|
|
|
76
128
|
}
|
|
77
129
|
/** The full command line for a function: [script, ...argv]. */
|
|
78
130
|
export async function buildFunctionCommand(fn, opts, schema) {
|
|
131
|
+
if (fn.steps)
|
|
132
|
+
throw new Error(`function ${fn.functionName} runs steps; build each step's command`);
|
|
79
133
|
schema ??= await loadSchema(fn.script);
|
|
80
134
|
const context = opts.context;
|
|
81
135
|
const templates = functionTemplateOptions(fn);
|
|
@@ -121,8 +175,16 @@ function streamPath(template, opts, base, templates) {
|
|
|
121
175
|
* Run a function's tool. Its stdin is the function's `stdin` file, if any;
|
|
122
176
|
* stdout goes to its `stdout` file, else is collected (a tool may print its
|
|
123
177
|
* result), and stderr is collected. Secret options are `***` in `command`.
|
|
178
|
+
*
|
|
179
|
+
* A function with `steps` runs them in order with one context, each seeing the
|
|
180
|
+
* earlier ones as `steps.NAME`. A step's `succeed`, `fail` and `error` name a
|
|
181
|
+
* later step to go to instead. A failure with no `fail` branch stops the chain
|
|
182
|
+
* and is its `failedStep`; the run is that of the last step that ran.
|
|
124
183
|
*/
|
|
125
184
|
export async function runScriptFunction(fn, opts) {
|
|
185
|
+
return fn.steps ? runSteps(fn.steps, opts) : (await runTool(fn, opts)).run;
|
|
186
|
+
}
|
|
187
|
+
async function runTool(fn, opts) {
|
|
126
188
|
const schema = await loadSchema(fn.script);
|
|
127
189
|
const [file, ...argv] = await buildFunctionCommand(fn, opts, schema);
|
|
128
190
|
const command = [file, ...redactArgv(schema, argv)];
|
|
@@ -131,7 +193,7 @@ export async function runScriptFunction(fn, opts) {
|
|
|
131
193
|
const workDir = opts.context?.paths?.work_dir;
|
|
132
194
|
const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot, functionTemplateOptions(fn));
|
|
133
195
|
const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot, functionTemplateOptions(fn));
|
|
134
|
-
// Wire the well-known roles the tool declares: progress onto the record, meta into the run.
|
|
196
|
+
// Wire the well-known roles the tool declares: progress onto the record, meta and result into the run.
|
|
135
197
|
let progress;
|
|
136
198
|
const roles = {
|
|
137
199
|
progress: {
|
|
@@ -144,11 +206,15 @@ export async function runScriptFunction(fn, opts) {
|
|
|
144
206
|
},
|
|
145
207
|
},
|
|
146
208
|
meta: {},
|
|
209
|
+
result: {},
|
|
147
210
|
log: { onLine: opts.onLog },
|
|
148
211
|
};
|
|
212
|
+
// Opened before the tool starts, so a missing or unreadable file always fails the run: a lazy
|
|
213
|
+
// stream reports it only after the tool may have exited without reading its input.
|
|
214
|
+
const stdin = stdinPath ? createReadStream(stdinPath, { fd: openSync(stdinPath, 'r') }) : undefined;
|
|
149
215
|
const started = start(file, argv, {
|
|
150
216
|
cwd: opts.cwd, env: opts.env, timeoutMs: opts.timeoutMs, killGraceMs: opts.killGraceMs, signal: opts.signal, onStderr: opts.onStderr,
|
|
151
|
-
stdin
|
|
217
|
+
stdin,
|
|
152
218
|
stdout: stdoutPath ? 'stream' : 'text',
|
|
153
219
|
roles: declaredRoles({ schema }, roles),
|
|
154
220
|
});
|
|
@@ -159,7 +225,81 @@ export async function runScriptFunction(fn, opts) {
|
|
|
159
225
|
}
|
|
160
226
|
const [result] = await Promise.all([started.result, written]);
|
|
161
227
|
const meta = parseRoleJson(result.roles?.meta);
|
|
162
|
-
|
|
228
|
+
const output = parseRoleJson(result.roles?.result) ?? (stdoutPath ? undefined : parseRoleJson(result.stdout));
|
|
229
|
+
const run = {
|
|
230
|
+
...result, command, stderrTail: tail(result.stderr),
|
|
231
|
+
...(meta !== undefined ? { meta } : {}), ...(progress ? { progress } : {}), ...(output !== undefined ? { result: output } : {}),
|
|
232
|
+
};
|
|
233
|
+
return { run, stdoutPath };
|
|
234
|
+
}
|
|
235
|
+
async function runSteps(steps, opts) {
|
|
236
|
+
const deadline = opts.timeoutMs ? Date.now() + opts.timeoutMs : undefined;
|
|
237
|
+
const outputs = {};
|
|
238
|
+
const stepRuns = [];
|
|
239
|
+
const byName = new Map(steps.map((step, i) => [step.stepName, i]));
|
|
240
|
+
// The step after `i` on an outcome: its branch when set ($end ends the chain), else the next step on success.
|
|
241
|
+
const after = (i, branch, fallthrough) => branch === END ? undefined : branch ? byName.get(branch) : fallthrough && i + 1 < steps.length ? i + 1 : undefined;
|
|
242
|
+
let continued;
|
|
243
|
+
let last;
|
|
244
|
+
let previous;
|
|
245
|
+
for (let i = 0; i !== undefined;) {
|
|
246
|
+
const step = steps[i];
|
|
247
|
+
const name = step.stepName;
|
|
248
|
+
const limits = [step.definition.timeout_ms, deadline && Math.max(1, deadline - Date.now())].filter((ms) => Number(ms) > 0);
|
|
249
|
+
// A chain always renders: its steps reach each other through `steps` (and the one before
|
|
250
|
+
// through `previous`) even without a caller context.
|
|
251
|
+
const context = { ...opts.context, steps: { ...outputs }, ...(previous ? { previous } : {}) };
|
|
252
|
+
const onProgress = opts.onProgress && ((update) => opts.onProgress?.({ stage: name, ...update, step: name }));
|
|
253
|
+
let ran;
|
|
254
|
+
try {
|
|
255
|
+
ran = await runTool(step, { ...opts, context, onProgress, timeoutMs: limits.length ? Math.min(...limits) : undefined });
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
const message = `step ${name}: ${error.message}`;
|
|
259
|
+
if (!step.definition.error)
|
|
260
|
+
throw new Error(message, { cause: error });
|
|
261
|
+
outputs[name] = { exit_code: null, stdout: '', error: message };
|
|
262
|
+
previous = { name, ...outputs[name] };
|
|
263
|
+
i = byName.get(step.definition.error);
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
const { run, stdoutPath } = ran;
|
|
267
|
+
stepRuns.push({ ...run, step: name });
|
|
268
|
+
outputs[name] = {
|
|
269
|
+
exit_code: run.exitCode,
|
|
270
|
+
stdout: stdoutPath ?? run.stdout.replace(/\n+$/, ''),
|
|
271
|
+
...(run.result !== undefined ? { result: run.result } : {}),
|
|
272
|
+
...(run.meta !== undefined ? { meta: run.meta } : {}),
|
|
273
|
+
...(run.progress ? { progress: run.progress } : {}),
|
|
274
|
+
};
|
|
275
|
+
previous = { name, ...outputs[name] };
|
|
276
|
+
last = { ...run, steps: outputs, previous, stepRuns, durationMs: stepRuns.reduce((ms, r) => ms + r.durationMs, 0) };
|
|
277
|
+
// A timed-out step failed, even when it exits 0 on SIGTERM.
|
|
278
|
+
const ok = !run.timedOut && (run.exitCode === 0 || (run.exitCode !== null && (step.definition.continue_on_exit ?? []).includes(run.exitCode)));
|
|
279
|
+
if (ok) {
|
|
280
|
+
if (run.exitCode !== 0)
|
|
281
|
+
continued ??= run.exitCode ?? undefined;
|
|
282
|
+
i = after(i, step.definition.succeed, true);
|
|
283
|
+
if (i === undefined)
|
|
284
|
+
return { ...last, exitCode: run.exitCode || continued || 0 };
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
// A cancelled or out-of-time chain stops here, whatever the step's `fail` says.
|
|
288
|
+
const stopped = opts.signal?.aborted || (deadline !== undefined && Date.now() >= deadline);
|
|
289
|
+
i = stopped ? undefined : after(i, step.definition.fail, false);
|
|
290
|
+
if (i === undefined)
|
|
291
|
+
return { ...last, failedStep: name };
|
|
292
|
+
}
|
|
293
|
+
if (!last)
|
|
294
|
+
throw new Error('unreachable: a chain ends after a step that ran');
|
|
295
|
+
return last;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* The context to render a function's artifacts and result map with after a run:
|
|
299
|
+
* a chain's `steps` and `previous` (its last step) added to the run's context.
|
|
300
|
+
*/
|
|
301
|
+
export function runContext(context, run) {
|
|
302
|
+
return { ...context, ...(run.steps ? { steps: run.steps } : {}), ...(run.previous ? { previous: run.previous } : {}) };
|
|
163
303
|
}
|
|
164
304
|
/** The result record every function shares, plus its rendered `result` map. */
|
|
165
305
|
export function buildResultRecord(p) {
|
package/dist/template.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export declare function lookupTemplateValue(context: TemplateContext, expression
|
|
|
19
19
|
* or positional set to it is left unset. Always false with `opts.strict`.
|
|
20
20
|
*/
|
|
21
21
|
export declare function isUnsetTemplate(value: unknown, context: TemplateContext, opts?: TemplateOptions): boolean;
|
|
22
|
+
/** The `params` fields a value's templates name (the first part after `params.`), in strings, arrays and objects. */
|
|
23
|
+
export declare function templateParamNames(value: unknown, names?: Set<string>): Set<string>;
|
|
22
24
|
/** Replace dotted-path ${expression} and {{expression}} templates using the context. */
|
|
23
25
|
export declare function renderTemplateString(value: string, context: TemplateContext, opts?: TemplateOptions): string;
|
|
24
26
|
/** Render strings and the strings in arrays; other scalars keep their type. */
|
package/dist/template.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template.d.ts","sourceRoot":"","sources":["../src/template.ts"],"names":[],"mappings":"AACA,4DAA4D;AAC5D,MAAM,MAAM,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEzD,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,8IAA8I;IAC9I,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAqBD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAQhH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CASzG;AAED,wFAAwF;AACxF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAE5G;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAKnH;AAED,4EAA4E;AAC5E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAO3G;AAED,0EAA0E;AAC1E,wBAAgB,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAGhI"}
|
|
1
|
+
{"version":3,"file":"template.d.ts","sourceRoot":"","sources":["../src/template.ts"],"names":[],"mappings":"AACA,4DAA4D;AAC5D,MAAM,MAAM,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC;AAEzD,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,8IAA8I;IAC9I,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAqBD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAQhH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CASzG;AAED,qHAAqH;AACrH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,cAAoB,GAAG,GAAG,CAAC,MAAM,CAAC,CASzF;AAED,wFAAwF;AACxF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAE5G;AAED,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,IAAI,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAKnH;AAED,4EAA4E;AAC5E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAO3G;AAED,0EAA0E;AAC1E,wBAAgB,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAGhI"}
|
package/dist/template.js
CHANGED
|
@@ -49,6 +49,23 @@ export function isUnsetTemplate(value, context, opts) {
|
|
|
49
49
|
}
|
|
50
50
|
return false;
|
|
51
51
|
}
|
|
52
|
+
/** The `params` fields a value's templates name (the first part after `params.`), in strings, arrays and objects. */
|
|
53
|
+
export function templateParamNames(value, names = new Set()) {
|
|
54
|
+
if (typeof value === 'string') {
|
|
55
|
+
for (const [, dollar, brace] of value.matchAll(TEMPLATE)) {
|
|
56
|
+
const [root, field] = String(dollar || brace).split('.').filter(Boolean);
|
|
57
|
+
if (OPTIONAL_ROOTS.includes(root ?? '') && field)
|
|
58
|
+
names.add(field);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
else if (Array.isArray(value))
|
|
62
|
+
for (const item of value)
|
|
63
|
+
templateParamNames(item, names);
|
|
64
|
+
else if (value && typeof value === 'object')
|
|
65
|
+
for (const item of Object.values(value))
|
|
66
|
+
templateParamNames(item, names);
|
|
67
|
+
return names;
|
|
68
|
+
}
|
|
52
69
|
/** Replace dotted-path ${expression} and {{expression}} templates using the context. */
|
|
53
70
|
export function renderTemplateString(value, context, opts) {
|
|
54
71
|
return String(value).replace(TEMPLATE, (_, dollar, brace) => lookupTemplateValue(context, dollar || brace, opts));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "clyops-jobs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"description": "Run clyops tools as jobs: config-bound functions, templated invocation, artifacts, job records and a queue.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -26,19 +26,22 @@
|
|
|
26
26
|
"types": "./dist/index.d.ts",
|
|
27
27
|
"import": "./dist/index.js",
|
|
28
28
|
"require": "./clyops-jobs.cjs"
|
|
29
|
-
}
|
|
29
|
+
},
|
|
30
|
+
"./config.schema.json": "./config.schema.json"
|
|
30
31
|
},
|
|
31
32
|
"files": [
|
|
32
33
|
"dist",
|
|
33
34
|
"src",
|
|
34
|
-
"clyops-jobs.cjs"
|
|
35
|
+
"clyops-jobs.cjs",
|
|
36
|
+
"config.schema.json"
|
|
35
37
|
],
|
|
36
38
|
"devDependencies": {
|
|
37
39
|
"@types/node": "^22.0.0",
|
|
38
|
-
"typescript": "^5.6.0"
|
|
40
|
+
"typescript": "^5.6.0",
|
|
41
|
+
"ajv": "^8.20.0"
|
|
39
42
|
},
|
|
40
43
|
"dependencies": {
|
|
41
|
-
"clyops-tools": "0.
|
|
44
|
+
"clyops-tools": "0.12.0"
|
|
42
45
|
},
|
|
43
46
|
"homepage": "https://clyops.io/node-tools.html",
|
|
44
47
|
"scripts": {
|
package/src/functions.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Config-bound functions: which tool a function runs and with what options.
|
|
2
2
|
//
|
|
3
3
|
// { "functions": { "<name>": { "key", "script", "positionals"?, "stdin"?, "stdout"?, "artifacts"?, "result"? } },
|
|
4
|
+
// { "<name>": { "steps": [{ "name", "key"?, "script", ... }], "artifacts"?, "result"? } },
|
|
4
5
|
// "defaults": { "<key>": { <tool options> } },
|
|
5
6
|
// "<key>": { <overrides> } }
|
|
6
7
|
//
|
|
@@ -8,10 +9,10 @@
|
|
|
8
9
|
// --help-json-schema (spec section 13), after `${dot.path}` / `{{dot.path}}`
|
|
9
10
|
// templates are rendered against a context the trigger builds.
|
|
10
11
|
import { declaredRoles, inputKeys, loadSchema, redactArgv, start, shellQuote, tail, toArgv, type RoleOption, type RunResult, type Schema } from 'clyops-tools';
|
|
11
|
-
import { createReadStream, createWriteStream } from 'node:fs';
|
|
12
|
+
import { createReadStream, createWriteStream, openSync } from 'node:fs';
|
|
12
13
|
import { finished } from 'node:stream/promises';
|
|
13
14
|
import { basename, dirname, isAbsolute, resolve } from 'node:path';
|
|
14
|
-
import { isUnsetTemplate, renderConfigValue, renderResultMap, renderTemplateString, type TemplateContext, type TemplateOptions } from './template.js';
|
|
15
|
+
import { isUnsetTemplate, renderConfigValue, renderResultMap, renderTemplateString, templateParamNames, type TemplateContext, type TemplateOptions } from './template.js';
|
|
15
16
|
import { deepMerge, ensureDir, objectValue, resolveScriptPath, type Json } from './util.js';
|
|
16
17
|
|
|
17
18
|
/** Where engine diagnostics go; a trigger installs its own with configureLogging. */
|
|
@@ -50,9 +51,29 @@ export interface FunctionDefinition {
|
|
|
50
51
|
result?: unknown;
|
|
51
52
|
/** A missing `params` field in a template is an error instead of unset (default false). */
|
|
52
53
|
strict_params?: boolean;
|
|
54
|
+
/** Tools run in order instead of one `script`; each step's outputs are `steps.NAME` in later templates. */
|
|
55
|
+
steps?: StepDefinition[];
|
|
53
56
|
[key: string]: unknown;
|
|
54
57
|
}
|
|
55
58
|
|
|
59
|
+
/** One tool in a function's `steps`: a function binding with a name, without artifacts or a result map. */
|
|
60
|
+
export interface StepDefinition extends FunctionDefinition {
|
|
61
|
+
/** Names the step in `${steps.NAME...}` templates and errors; letters, digits, `_` and `-`. */
|
|
62
|
+
name: string;
|
|
63
|
+
/** Config key holding the step's options; defaults to the step name. */
|
|
64
|
+
key?: string;
|
|
65
|
+
/** This step's time limit; the whole chain also stays within the run's `timeoutMs`. */
|
|
66
|
+
timeout_ms?: number;
|
|
67
|
+
/** Exit codes besides 0 that count as success; the chain then reports the first of them. */
|
|
68
|
+
continue_on_exit?: number[];
|
|
69
|
+
/** The later step to run after this one succeeds instead of the next, or `$end` to end the chain there. */
|
|
70
|
+
succeed?: string;
|
|
71
|
+
/** The later step to run when this one exits nonzero or times out, instead of ending the chain. */
|
|
72
|
+
fail?: string;
|
|
73
|
+
/** The later step to run when this one can't start (a template, stdin or schema error), instead of throwing. */
|
|
74
|
+
error?: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
56
77
|
/** A configured function with resolved executable and merged defaults and overrides. */
|
|
57
78
|
export interface ResolvedFunction {
|
|
58
79
|
/** Name of the configured function to invoke. */
|
|
@@ -65,8 +86,22 @@ export interface ResolvedFunction {
|
|
|
65
86
|
config: Json;
|
|
66
87
|
/** Original function binding, including positional, stream, artifact, and result templates. */
|
|
67
88
|
definition: FunctionDefinition;
|
|
89
|
+
/** A chain's steps, in order; its `script` is then the last step's and `config` is keyed by step name. */
|
|
90
|
+
steps?: ResolvedStep[];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** A resolved step of a chain. */
|
|
94
|
+
export interface ResolvedStep extends ResolvedFunction {
|
|
95
|
+
/** The step's `name`. */
|
|
96
|
+
stepName: string;
|
|
97
|
+
/** The step's binding. */
|
|
98
|
+
definition: StepDefinition;
|
|
68
99
|
}
|
|
69
100
|
|
|
101
|
+
const STEP_NAME = /^[A-Za-z0-9_-]+$/;
|
|
102
|
+
// The `succeed` target that ends a chain; `$` can't start a step name.
|
|
103
|
+
const END = '$end';
|
|
104
|
+
|
|
70
105
|
/**
|
|
71
106
|
* The binding for `functionName` in a merged config, or null when the config
|
|
72
107
|
* has no such function. A function without a `script` is an error.
|
|
@@ -75,17 +110,59 @@ export function resolveFunctionConfig(scriptRoot: string, routeConfig: Json, fun
|
|
|
75
110
|
const definition = objectValue(objectValue(routeConfig.functions)[functionName]) as FunctionDefinition;
|
|
76
111
|
if (Object.keys(definition).length === 0) return null;
|
|
77
112
|
const key = String(definition.key || functionName);
|
|
113
|
+
if (definition.steps === undefined) return resolveBinding(scriptRoot, routeConfig, functionName, key, definition, `function ${functionName}`);
|
|
114
|
+
if (definition.script) throw new Error(`function ${functionName} defines both script and steps`);
|
|
115
|
+
if (!Array.isArray(definition.steps) || definition.steps.length === 0) throw new Error(`function ${functionName} defines no steps`);
|
|
116
|
+
const steps: ResolvedStep[] = [];
|
|
117
|
+
for (const [i, item] of definition.steps.entries()) {
|
|
118
|
+
const step = { strict_params: definition.strict_params, ...objectValue(item) } as StepDefinition;
|
|
119
|
+
const name = String(step.name ?? '');
|
|
120
|
+
if (!STEP_NAME.test(name)) throw new Error(`function ${functionName} step ${i + 1} needs a name of letters, digits, _ or -`);
|
|
121
|
+
if (steps.some((s) => s.stepName === name)) throw new Error(`function ${functionName} has two steps named ${name}`);
|
|
122
|
+
const where = `function ${functionName} step ${name}`;
|
|
123
|
+
steps.push({ ...resolveBinding(scriptRoot, routeConfig, functionName, String(step.key || name), step, where), stepName: name, definition: step });
|
|
124
|
+
}
|
|
125
|
+
// Branches go forward to a step that exists, so a chain always ends.
|
|
126
|
+
for (const [i, step] of steps.entries()) {
|
|
127
|
+
for (const outcome of ['succeed', 'fail', 'error'] as const) {
|
|
128
|
+
const target = step.definition[outcome];
|
|
129
|
+
if (target === undefined || (outcome === 'succeed' && target === END)) continue;
|
|
130
|
+
const at = steps.findIndex((s) => s.stepName === target);
|
|
131
|
+
if (at < 0) throw new Error(`function ${functionName} step ${step.stepName}: ${outcome} names no step ${String(target)}`);
|
|
132
|
+
if (at <= i) throw new Error(`function ${functionName} step ${step.stepName}: ${outcome} must name a later step, not ${String(target)}`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const config = Object.fromEntries(steps.map((s) => [s.stepName, s.config]));
|
|
136
|
+
return { functionName, key, script: steps[steps.length - 1].script, config, definition, steps };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function resolveBinding(scriptRoot: string, routeConfig: Json, functionName: string, key: string, definition: FunctionDefinition, where: string): ResolvedFunction {
|
|
78
140
|
const script = String(definition.script || '');
|
|
79
|
-
if (!script) throw new Error(
|
|
141
|
+
if (!script) throw new Error(`${where} does not define a script`);
|
|
80
142
|
const config = deepMerge(objectValue(objectValue(routeConfig.defaults)[key]), objectValue(routeConfig[key])) as Json;
|
|
81
143
|
return { functionName, key, script: resolveScriptPath(scriptRoot, script), config, definition };
|
|
82
144
|
}
|
|
83
145
|
|
|
146
|
+
/**
|
|
147
|
+
* The caller fields (`params.NAME`) a function's templates use, sorted: in its
|
|
148
|
+
* options and its or its steps' positionals, `stdin`, `stdout`, artifacts and
|
|
149
|
+
* result. A trigger can describe a function's inputs with it.
|
|
150
|
+
*/
|
|
151
|
+
export function functionParamNames(fn: ResolvedFunction): string[] {
|
|
152
|
+
const names = new Set<string>();
|
|
153
|
+
for (const part of fn.steps ?? [fn]) {
|
|
154
|
+
const { positionals, stdin, stdout } = part.definition;
|
|
155
|
+
templateParamNames([part.config, positionals, stdin, stdout], names);
|
|
156
|
+
}
|
|
157
|
+
templateParamNames([fn.definition.artifacts, fn.definition.result], names);
|
|
158
|
+
return [...names].sort();
|
|
159
|
+
}
|
|
160
|
+
|
|
84
161
|
/** Path base, template context, and controlled arguments used when constructing a job command. */
|
|
85
162
|
export interface CommandOptions {
|
|
86
163
|
/** Directory relative path options resolve against. */
|
|
87
164
|
configRoot: string;
|
|
88
|
-
/** Values available to dotted-path templates; null disables context-based rendering. */
|
|
165
|
+
/** Values available to dotted-path templates; null disables context-based rendering, except in a chain's steps. */
|
|
89
166
|
context?: TemplateContext | null;
|
|
90
167
|
/** Options the trigger sets itself; config keys for them are ignored with a warning. Default: ['help']. */
|
|
91
168
|
controlled?: string[];
|
|
@@ -131,6 +208,7 @@ function explicitPositionals(fn: ResolvedFunction, context?: TemplateContext | n
|
|
|
131
208
|
|
|
132
209
|
/** The full command line for a function: [script, ...argv]. */
|
|
133
210
|
export async function buildFunctionCommand(fn: ResolvedFunction, opts: CommandOptions, schema?: Schema): Promise<string[]> {
|
|
211
|
+
if (fn.steps) throw new Error(`function ${fn.functionName} runs steps; build each step's command`);
|
|
134
212
|
schema ??= await loadSchema(fn.script);
|
|
135
213
|
const context = opts.context;
|
|
136
214
|
const templates = functionTemplateOptions(fn);
|
|
@@ -161,8 +239,46 @@ export interface FunctionRun extends RunResult {
|
|
|
161
239
|
meta?: unknown;
|
|
162
240
|
/** The last update on its `progress` role. */
|
|
163
241
|
progress?: Record<string, unknown>;
|
|
242
|
+
/** Its `result` role parsed, else its collected stdout parsed, when either is JSON. */
|
|
243
|
+
result?: unknown;
|
|
244
|
+
/** A chain's outputs by step name, for templates after it (see runContext). */
|
|
245
|
+
steps?: Record<string, StepOutput>;
|
|
246
|
+
/** A chain's last step that ran or took its `error` branch, with its name: `previous` in templates after the run. */
|
|
247
|
+
previous?: StepOutput & { name: string };
|
|
248
|
+
/** A chain's runs, one per step that ran, in order. */
|
|
249
|
+
stepRuns?: Array<FunctionRun & { step: string }>;
|
|
250
|
+
/** The step that stopped a chain: a nonzero exit (not in its `continue_on_exit`) or timeout with no `fail` branch, or a signal. */
|
|
251
|
+
failedStep?: string;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** What a step leaves for later templates as `steps.NAME`. */
|
|
255
|
+
export interface StepOutput {
|
|
256
|
+
/** Its exit code, or null when a signal stopped it. */
|
|
257
|
+
exit_code: number | null;
|
|
258
|
+
/** The path of its `stdout` file, else its collected stdout without trailing newlines. */
|
|
259
|
+
stdout: string;
|
|
260
|
+
/** Its `result` role or stdout as JSON; absent when neither is. */
|
|
261
|
+
result?: unknown;
|
|
262
|
+
/** Its `meta` role as JSON. */
|
|
263
|
+
meta?: unknown;
|
|
264
|
+
/** Its last `progress` update. */
|
|
265
|
+
progress?: Record<string, unknown>;
|
|
266
|
+
/** Why it couldn't start, when it has an `error` branch. */
|
|
267
|
+
error?: string;
|
|
164
268
|
}
|
|
165
269
|
|
|
270
|
+
/** Options for running a function's tool, or each tool of a chain. */
|
|
271
|
+
export type RunFunctionOptions = CommandOptions & {
|
|
272
|
+
/** For a chain, the whole chain's limit; a step's `timeout_ms` limits that step too. */ timeoutMs?: number;
|
|
273
|
+
/** How long a stopped tool (abort or timeout) has after SIGTERM before SIGKILL; default 200. */ killGraceMs?: number;
|
|
274
|
+
/** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
|
|
275
|
+
/** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
|
|
276
|
+
/** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
|
|
277
|
+
/** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
|
|
278
|
+
/** Called with each object the tool writes on its `progress` role (spec section 14); a chain's carry `step` and a default `stage`. */ onProgress?: (update: Record<string, unknown>) => void;
|
|
279
|
+
/** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
|
|
280
|
+
};
|
|
281
|
+
|
|
166
282
|
/** A role's collected text as JSON, or undefined when it is empty or isn't JSON. */
|
|
167
283
|
export function parseRoleJson(text: string | undefined): unknown {
|
|
168
284
|
if (!text?.trim()) return undefined;
|
|
@@ -184,20 +300,17 @@ function streamPath(template: string | undefined, opts: CommandOptions, base: st
|
|
|
184
300
|
* Run a function's tool. Its stdin is the function's `stdin` file, if any;
|
|
185
301
|
* stdout goes to its `stdout` file, else is collected (a tool may print its
|
|
186
302
|
* result), and stderr is collected. Secret options are `***` in `command`.
|
|
303
|
+
*
|
|
304
|
+
* A function with `steps` runs them in order with one context, each seeing the
|
|
305
|
+
* earlier ones as `steps.NAME`. A step's `succeed`, `fail` and `error` name a
|
|
306
|
+
* later step to go to instead. A failure with no `fail` branch stops the chain
|
|
307
|
+
* and is its `failedStep`; the run is that of the last step that ran.
|
|
187
308
|
*/
|
|
188
|
-
export async function runScriptFunction(
|
|
189
|
-
fn:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
/** Environment supplied to the process or read for option defaults. */ env?: NodeJS.ProcessEnv;
|
|
194
|
-
/** Working directory used to resolve command-line paths and execute tools. */ cwd?: string;
|
|
195
|
-
/** AbortSignal used to cancel execution and terminate the tool process tree. */ signal?: AbortSignal;
|
|
196
|
-
/** Called for each decoded stderr chunk. */ onStderr?: (chunk: string) => void;
|
|
197
|
-
/** Called with each object the tool writes on its `progress` role (spec section 14). */ onProgress?: (update: Record<string, unknown>) => void;
|
|
198
|
-
/** Called with each line the tool writes on its `log` role. */ onLog?: (line: string) => void;
|
|
199
|
-
},
|
|
200
|
-
): Promise<FunctionRun> {
|
|
309
|
+
export async function runScriptFunction(fn: ResolvedFunction, opts: RunFunctionOptions): Promise<FunctionRun> {
|
|
310
|
+
return fn.steps ? runSteps(fn.steps, opts) : (await runTool(fn, opts)).run;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
async function runTool(fn: ResolvedFunction, opts: RunFunctionOptions): Promise<{ run: FunctionRun; stdoutPath?: string }> {
|
|
201
314
|
const schema = await loadSchema(fn.script);
|
|
202
315
|
const [file, ...argv] = await buildFunctionCommand(fn, opts, schema);
|
|
203
316
|
const command = [file, ...redactArgv(schema, argv)];
|
|
@@ -206,7 +319,7 @@ export async function runScriptFunction(
|
|
|
206
319
|
const workDir = (opts.context?.paths as { work_dir?: string } | undefined)?.work_dir;
|
|
207
320
|
const stdinPath = streamPath(fn.definition.stdin, opts, opts.configRoot, functionTemplateOptions(fn));
|
|
208
321
|
const stdoutPath = streamPath(fn.definition.stdout, opts, workDir ?? opts.configRoot, functionTemplateOptions(fn));
|
|
209
|
-
// Wire the well-known roles the tool declares: progress onto the record, meta into the run.
|
|
322
|
+
// Wire the well-known roles the tool declares: progress onto the record, meta and result into the run.
|
|
210
323
|
let progress: Record<string, unknown> | undefined;
|
|
211
324
|
const roles: Record<string, RoleOption> = {
|
|
212
325
|
progress: {
|
|
@@ -218,11 +331,15 @@ export async function runScriptFunction(
|
|
|
218
331
|
},
|
|
219
332
|
},
|
|
220
333
|
meta: {},
|
|
334
|
+
result: {},
|
|
221
335
|
log: { onLine: opts.onLog },
|
|
222
336
|
};
|
|
337
|
+
// Opened before the tool starts, so a missing or unreadable file always fails the run: a lazy
|
|
338
|
+
// stream reports it only after the tool may have exited without reading its input.
|
|
339
|
+
const stdin = stdinPath ? createReadStream(stdinPath, { fd: openSync(stdinPath, 'r') }) : undefined;
|
|
223
340
|
const started = start(file, argv, {
|
|
224
341
|
cwd: opts.cwd, env: opts.env, timeoutMs: opts.timeoutMs, killGraceMs: opts.killGraceMs, signal: opts.signal, onStderr: opts.onStderr,
|
|
225
|
-
stdin
|
|
342
|
+
stdin,
|
|
226
343
|
stdout: stdoutPath ? 'stream' : 'text',
|
|
227
344
|
roles: declaredRoles({ schema }, roles),
|
|
228
345
|
});
|
|
@@ -233,7 +350,78 @@ export async function runScriptFunction(
|
|
|
233
350
|
}
|
|
234
351
|
const [result] = await Promise.all([started.result, written]);
|
|
235
352
|
const meta = parseRoleJson(result.roles?.meta);
|
|
236
|
-
|
|
353
|
+
const output = parseRoleJson(result.roles?.result) ?? (stdoutPath ? undefined : parseRoleJson(result.stdout));
|
|
354
|
+
const run: FunctionRun = {
|
|
355
|
+
...result, command, stderrTail: tail(result.stderr),
|
|
356
|
+
...(meta !== undefined ? { meta } : {}), ...(progress ? { progress } : {}), ...(output !== undefined ? { result: output } : {}),
|
|
357
|
+
};
|
|
358
|
+
return { run, stdoutPath };
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function runSteps(steps: ResolvedStep[], opts: RunFunctionOptions): Promise<FunctionRun> {
|
|
362
|
+
const deadline = opts.timeoutMs ? Date.now() + opts.timeoutMs : undefined;
|
|
363
|
+
const outputs: Record<string, StepOutput> = {};
|
|
364
|
+
const stepRuns: Array<FunctionRun & { step: string }> = [];
|
|
365
|
+
const byName = new Map(steps.map((step, i) => [step.stepName, i]));
|
|
366
|
+
// The step after `i` on an outcome: its branch when set ($end ends the chain), else the next step on success.
|
|
367
|
+
const after = (i: number, branch: string | undefined, fallthrough: boolean): number | undefined =>
|
|
368
|
+
branch === END ? undefined : branch ? byName.get(branch) : fallthrough && i + 1 < steps.length ? i + 1 : undefined;
|
|
369
|
+
let continued: number | undefined;
|
|
370
|
+
let last: FunctionRun | undefined;
|
|
371
|
+
let previous: (StepOutput & { name: string }) | undefined;
|
|
372
|
+
for (let i: number | undefined = 0; i !== undefined;) {
|
|
373
|
+
const step = steps[i];
|
|
374
|
+
const name = step.stepName;
|
|
375
|
+
const limits = [step.definition.timeout_ms, deadline && Math.max(1, deadline - Date.now())].filter((ms): ms is number => Number(ms) > 0);
|
|
376
|
+
// A chain always renders: its steps reach each other through `steps` (and the one before
|
|
377
|
+
// through `previous`) even without a caller context.
|
|
378
|
+
const context = { ...opts.context, steps: { ...outputs }, ...(previous ? { previous } : {}) };
|
|
379
|
+
const onProgress = opts.onProgress && ((update: Record<string, unknown>) => opts.onProgress?.({ stage: name, ...update, step: name }));
|
|
380
|
+
let ran: Awaited<ReturnType<typeof runTool>>;
|
|
381
|
+
try {
|
|
382
|
+
ran = await runTool(step, { ...opts, context, onProgress, timeoutMs: limits.length ? Math.min(...limits) : undefined });
|
|
383
|
+
} catch (error) {
|
|
384
|
+
const message = `step ${name}: ${(error as Error).message}`;
|
|
385
|
+
if (!step.definition.error) throw new Error(message, { cause: error });
|
|
386
|
+
outputs[name] = { exit_code: null, stdout: '', error: message };
|
|
387
|
+
previous = { name, ...outputs[name] };
|
|
388
|
+
i = byName.get(step.definition.error);
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
const { run, stdoutPath } = ran;
|
|
392
|
+
stepRuns.push({ ...run, step: name });
|
|
393
|
+
outputs[name] = {
|
|
394
|
+
exit_code: run.exitCode,
|
|
395
|
+
stdout: stdoutPath ?? run.stdout.replace(/\n+$/, ''),
|
|
396
|
+
...(run.result !== undefined ? { result: run.result } : {}),
|
|
397
|
+
...(run.meta !== undefined ? { meta: run.meta } : {}),
|
|
398
|
+
...(run.progress ? { progress: run.progress } : {}),
|
|
399
|
+
};
|
|
400
|
+
previous = { name, ...outputs[name] };
|
|
401
|
+
last = { ...run, steps: outputs, previous, stepRuns, durationMs: stepRuns.reduce((ms, r) => ms + r.durationMs, 0) };
|
|
402
|
+
// A timed-out step failed, even when it exits 0 on SIGTERM.
|
|
403
|
+
const ok = !run.timedOut && (run.exitCode === 0 || (run.exitCode !== null && (step.definition.continue_on_exit ?? []).includes(run.exitCode)));
|
|
404
|
+
if (ok) {
|
|
405
|
+
if (run.exitCode !== 0) continued ??= run.exitCode ?? undefined;
|
|
406
|
+
i = after(i, step.definition.succeed, true);
|
|
407
|
+
if (i === undefined) return { ...last, exitCode: run.exitCode || continued || 0 };
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
410
|
+
// A cancelled or out-of-time chain stops here, whatever the step's `fail` says.
|
|
411
|
+
const stopped = opts.signal?.aborted || (deadline !== undefined && Date.now() >= deadline);
|
|
412
|
+
i = stopped ? undefined : after(i, step.definition.fail, false);
|
|
413
|
+
if (i === undefined) return { ...last, failedStep: name };
|
|
414
|
+
}
|
|
415
|
+
if (!last) throw new Error('unreachable: a chain ends after a step that ran');
|
|
416
|
+
return last;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* The context to render a function's artifacts and result map with after a run:
|
|
421
|
+
* a chain's `steps` and `previous` (its last step) added to the run's context.
|
|
422
|
+
*/
|
|
423
|
+
export function runContext<T extends TemplateContext>(context: T, run: FunctionRun): T & Pick<FunctionRun, 'steps' | 'previous'> {
|
|
424
|
+
return { ...context, ...(run.steps ? { steps: run.steps } : {}), ...(run.previous ? { previous: run.previous } : {}) };
|
|
237
425
|
}
|
|
238
426
|
|
|
239
427
|
/** The result record every function shares, plus its rendered `result` map. */
|
package/src/template.ts
CHANGED
|
@@ -58,6 +58,18 @@ export function isUnsetTemplate(value: unknown, context: TemplateContext, opts?:
|
|
|
58
58
|
return false;
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
/** The `params` fields a value's templates name (the first part after `params.`), in strings, arrays and objects. */
|
|
62
|
+
export function templateParamNames(value: unknown, names = new Set<string>()): Set<string> {
|
|
63
|
+
if (typeof value === 'string') {
|
|
64
|
+
for (const [, dollar, brace] of value.matchAll(TEMPLATE)) {
|
|
65
|
+
const [root, field] = String(dollar || brace).split('.').filter(Boolean);
|
|
66
|
+
if (OPTIONAL_ROOTS.includes(root ?? '') && field) names.add(field);
|
|
67
|
+
}
|
|
68
|
+
} else if (Array.isArray(value)) for (const item of value) templateParamNames(item, names);
|
|
69
|
+
else if (value && typeof value === 'object') for (const item of Object.values(value)) templateParamNames(item, names);
|
|
70
|
+
return names;
|
|
71
|
+
}
|
|
72
|
+
|
|
61
73
|
/** Replace dotted-path ${expression} and {{expression}} templates using the context. */
|
|
62
74
|
export function renderTemplateString(value: string, context: TemplateContext, opts?: TemplateOptions): string {
|
|
63
75
|
return String(value).replace(TEMPLATE, (_, dollar, brace) => lookupTemplateValue(context, dollar || brace, opts));
|