@withone/cli 1.26.0 → 1.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5,6 +5,7 @@ import crypto from "crypto";
|
|
|
5
5
|
|
|
6
6
|
// src/lib/flow-engine.ts
|
|
7
7
|
import fs from "fs";
|
|
8
|
+
import os from "os";
|
|
8
9
|
import path from "path";
|
|
9
10
|
import { exec, spawn } from "child_process";
|
|
10
11
|
import { promisify } from "util";
|
|
@@ -463,6 +464,27 @@ function withTimeout(promise, timeoutMs, stepId) {
|
|
|
463
464
|
if (timer) clearTimeout(timer);
|
|
464
465
|
});
|
|
465
466
|
}
|
|
467
|
+
function shouldRetryError(err, onError) {
|
|
468
|
+
if (!onError) return { retry: false };
|
|
469
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
470
|
+
const errorCode = err?.errorCode;
|
|
471
|
+
const matches = (entry) => {
|
|
472
|
+
if (typeof entry === "number") {
|
|
473
|
+
const re = new RegExp(`\\b${entry}\\b`);
|
|
474
|
+
return re.test(message);
|
|
475
|
+
}
|
|
476
|
+
if (errorCode && entry === errorCode) return true;
|
|
477
|
+
return message.toLowerCase().includes(entry.toLowerCase());
|
|
478
|
+
};
|
|
479
|
+
if (Array.isArray(onError.failFastOn) && onError.failFastOn.some(matches)) {
|
|
480
|
+
return { retry: false, reason: "failFastOn" };
|
|
481
|
+
}
|
|
482
|
+
if (Array.isArray(onError.retryOn)) {
|
|
483
|
+
if (onError.retryOn.some(matches)) return { retry: true, reason: "retryOn" };
|
|
484
|
+
return { retry: false, reason: "no-retryOn-match" };
|
|
485
|
+
}
|
|
486
|
+
return { retry: true };
|
|
487
|
+
}
|
|
466
488
|
function computeRetryDelay(onError, attempt) {
|
|
467
489
|
const base = onError.retryDelayMs ?? 1e3;
|
|
468
490
|
const max = onError.maxDelayMs ?? 3e4;
|
|
@@ -509,27 +531,64 @@ function resolveSelector(selectorPath, context) {
|
|
|
509
531
|
function shellQuote(value) {
|
|
510
532
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
511
533
|
}
|
|
534
|
+
function applyHandlebarsPipe(value, pipe) {
|
|
535
|
+
switch (pipe) {
|
|
536
|
+
case "json":
|
|
537
|
+
return JSON.stringify(value ?? null);
|
|
538
|
+
case "shell": {
|
|
539
|
+
if (value === void 0 || value === null) return `''`;
|
|
540
|
+
const str = typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
541
|
+
return shellQuote(str);
|
|
542
|
+
}
|
|
543
|
+
case "url": {
|
|
544
|
+
if (value === void 0 || value === null) return "";
|
|
545
|
+
const str = typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
546
|
+
return encodeURIComponent(str);
|
|
547
|
+
}
|
|
548
|
+
case "md": {
|
|
549
|
+
if (value === void 0 || value === null) return "";
|
|
550
|
+
const str = typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
551
|
+
return str.replace(/([\\`*_{}\[\]()#+\-!|])/g, "\\$1");
|
|
552
|
+
}
|
|
553
|
+
case "html": {
|
|
554
|
+
if (value === void 0 || value === null) return "";
|
|
555
|
+
const str = typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
556
|
+
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
557
|
+
}
|
|
558
|
+
default:
|
|
559
|
+
throw new Error(`Unknown Handlebars pipe: "${pipe}". Supported: json, shell, url, md, html.`);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
512
562
|
function interpolateString(str, context) {
|
|
513
|
-
return str.replace(
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
563
|
+
return str.replace(
|
|
564
|
+
/\{\{\s*(q\s+)?(\$\.[^}\s|]+)(?:\s*\|\s*([a-zA-Z]+))?\s*\}\}/g,
|
|
565
|
+
(_match, qFlag, selector, pipe) => {
|
|
566
|
+
const value = resolveSelector(selector, context);
|
|
567
|
+
if (pipe) {
|
|
568
|
+
if (qFlag) {
|
|
569
|
+
throw new Error(`Handlebars expression "{{q ${selector} | ${pipe}}}" combines the legacy "q" prefix with a pipe \u2014 pick one (prefer the pipe form).`);
|
|
570
|
+
}
|
|
571
|
+
return applyHandlebarsPipe(value, pipe);
|
|
572
|
+
}
|
|
573
|
+
if (value === void 0 || value === null) return qFlag ? `''` : "";
|
|
574
|
+
if (typeof value === "object") {
|
|
575
|
+
console.warn(
|
|
576
|
+
`[flow] WARNING: Handlebars expression "{{${qFlag ? "q " : ""}${selector}}}" resolved to ${Array.isArray(value) ? "an array" : "an object"} and was stringified as JSON. To pass objects/arrays as native values, use a direct selector without {{ }}: "${selector}"`
|
|
577
|
+
);
|
|
578
|
+
const json = JSON.stringify(value);
|
|
579
|
+
return qFlag ? shellQuote(json) : json;
|
|
580
|
+
}
|
|
581
|
+
const str2 = String(value);
|
|
582
|
+
return qFlag ? shellQuote(str2) : str2;
|
|
522
583
|
}
|
|
523
|
-
|
|
524
|
-
return qFlag ? shellQuote(str2) : str2;
|
|
525
|
-
});
|
|
584
|
+
);
|
|
526
585
|
}
|
|
527
586
|
function resolveValue(value, context) {
|
|
528
587
|
if (typeof value === "string") {
|
|
529
588
|
if (value.startsWith("$.") && !value.includes("{{")) {
|
|
530
589
|
return resolveSelector(value, context);
|
|
531
590
|
}
|
|
532
|
-
if (
|
|
591
|
+
if (/\{\{\s*(q\s+)?\$\./.test(value)) {
|
|
533
592
|
return interpolateString(value, context);
|
|
534
593
|
}
|
|
535
594
|
return value;
|
|
@@ -914,7 +973,7 @@ async function executeSubflowStep(step, context, api, permissions, allowedAction
|
|
|
914
973
|
if (flowStack.includes(resolvedKey)) {
|
|
915
974
|
throw new Error(`Circular flow detected: ${[...flowStack, resolvedKey].join(" \u2192 ")}`);
|
|
916
975
|
}
|
|
917
|
-
const { loadFlowWithMeta: loadFlowWithMeta2 } = await import("./flow-runner-
|
|
976
|
+
const { loadFlowWithMeta: loadFlowWithMeta2 } = await import("./flow-runner-AK5W4GLF.js");
|
|
918
977
|
const { flow: subFlow, rootDir: subRootDir } = loadFlowWithMeta2(resolvedKey);
|
|
919
978
|
const subContext = await executeFlow(
|
|
920
979
|
subFlow,
|
|
@@ -1006,6 +1065,39 @@ async function executePaginateStep(step, context, api, permissions, allowedActio
|
|
|
1006
1065
|
response: { pages, totalResults: allResults.length, results: allResults }
|
|
1007
1066
|
};
|
|
1008
1067
|
}
|
|
1068
|
+
function resolveBashEnv(envConfig, context, stepId) {
|
|
1069
|
+
const out = {};
|
|
1070
|
+
const tempFiles = [];
|
|
1071
|
+
if (!envConfig) return { env: out, tempFiles };
|
|
1072
|
+
for (const [key, raw] of Object.entries(envConfig)) {
|
|
1073
|
+
if (raw === void 0 || raw === null) continue;
|
|
1074
|
+
if (typeof raw === "object" && !Array.isArray(raw)) {
|
|
1075
|
+
const obj = raw;
|
|
1076
|
+
if ("json" in obj) {
|
|
1077
|
+
const resolved2 = resolveValue(obj.json, context);
|
|
1078
|
+
const json = JSON.stringify(resolved2 ?? null);
|
|
1079
|
+
const tmp = path.join(
|
|
1080
|
+
os.tmpdir(),
|
|
1081
|
+
`one-flow-${stepId}-${key}-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`
|
|
1082
|
+
);
|
|
1083
|
+
fs.writeFileSync(tmp, json, { encoding: "utf-8" });
|
|
1084
|
+
tempFiles.push(tmp);
|
|
1085
|
+
out[key] = tmp;
|
|
1086
|
+
continue;
|
|
1087
|
+
}
|
|
1088
|
+
if ("shell" in obj) {
|
|
1089
|
+
const resolved2 = resolveValue(obj.shell, context);
|
|
1090
|
+
out[key] = resolved2 === void 0 || resolved2 === null ? "" : typeof resolved2 === "object" ? JSON.stringify(resolved2) : String(resolved2);
|
|
1091
|
+
continue;
|
|
1092
|
+
}
|
|
1093
|
+
out[key] = JSON.stringify(resolveValue(raw, context));
|
|
1094
|
+
continue;
|
|
1095
|
+
}
|
|
1096
|
+
const resolved = resolveValue(raw, context);
|
|
1097
|
+
out[key] = resolved === void 0 || resolved === null ? "" : typeof resolved === "object" ? JSON.stringify(resolved) : String(resolved);
|
|
1098
|
+
}
|
|
1099
|
+
return { env: out, tempFiles };
|
|
1100
|
+
}
|
|
1009
1101
|
async function executeBashStep(step, context, options) {
|
|
1010
1102
|
if (!options.allowBash) {
|
|
1011
1103
|
throw new Error("Bash steps require --allow-bash flag for security");
|
|
@@ -1013,19 +1105,33 @@ async function executeBashStep(step, context, options) {
|
|
|
1013
1105
|
const config = step.bash;
|
|
1014
1106
|
const command = resolveValue(config.command, context);
|
|
1015
1107
|
const cwd = config.cwd ? resolveValue(config.cwd, context) : process.cwd();
|
|
1016
|
-
const
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1108
|
+
const { env: resolvedEnv, tempFiles } = resolveBashEnv(
|
|
1109
|
+
config.env,
|
|
1110
|
+
context,
|
|
1111
|
+
step.id
|
|
1112
|
+
);
|
|
1113
|
+
const env = config.env ? { ...process.env, ...resolvedEnv } : process.env;
|
|
1114
|
+
try {
|
|
1115
|
+
const { stdout, stderr } = await execAsync(command, {
|
|
1116
|
+
timeout: config.timeout || 3e4,
|
|
1117
|
+
cwd,
|
|
1118
|
+
env,
|
|
1119
|
+
maxBuffer: 10 * 1024 * 1024
|
|
1120
|
+
});
|
|
1121
|
+
const output = config.parseJson ? JSON.parse(stripCodeFences(stdout)) : stdout.trim();
|
|
1122
|
+
return {
|
|
1123
|
+
status: "success",
|
|
1124
|
+
output,
|
|
1125
|
+
response: { stdout: stdout.trim(), stderr: stderr.trim(), exitCode: 0 }
|
|
1126
|
+
};
|
|
1127
|
+
} finally {
|
|
1128
|
+
for (const tmp of tempFiles) {
|
|
1129
|
+
try {
|
|
1130
|
+
fs.unlinkSync(tmp);
|
|
1131
|
+
} catch {
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1029
1135
|
}
|
|
1030
1136
|
function describe(value) {
|
|
1031
1137
|
if (value === null) return "null";
|
|
@@ -1155,6 +1261,18 @@ async function executeSingleStep(step, context, api, permissions, allowedActionI
|
|
|
1155
1261
|
if (attempt === maxAttempts) {
|
|
1156
1262
|
break;
|
|
1157
1263
|
}
|
|
1264
|
+
if (step.onError?.strategy === "retry" && (step.onError.retryOn || step.onError.failFastOn)) {
|
|
1265
|
+
const decision = shouldRetryError(lastError, step.onError);
|
|
1266
|
+
if (!decision.retry) {
|
|
1267
|
+
options.onEvent?.({
|
|
1268
|
+
event: "step:retry-skip",
|
|
1269
|
+
stepId: step.id,
|
|
1270
|
+
reason: decision.reason ?? "no-match",
|
|
1271
|
+
error: lastError.message
|
|
1272
|
+
});
|
|
1273
|
+
break;
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1158
1276
|
}
|
|
1159
1277
|
}
|
|
1160
1278
|
const errorMessage = lastError?.message || "Unknown error";
|
|
@@ -1368,7 +1486,8 @@ var FLOW_SCHEMA = {
|
|
|
1368
1486
|
if: { type: "string", required: false, description: "JS expression \u2014 skip step if falsy" },
|
|
1369
1487
|
unless: { type: "string", required: false, description: "JS expression \u2014 skip step if truthy" },
|
|
1370
1488
|
timeoutMs: { type: "number", required: false, description: 'Wall-clock timeout (ms). On expiry the step fails with errorCode:"TIMEOUT"; with onError:continue the result gets status:"timeout".' },
|
|
1371
|
-
requires: { type: "array", required: false, description: "Presence preconditions: array of $.input.X or $.steps.X.output... selectors that must resolve to a non-empty value before the step runs. Failures honor onError." }
|
|
1489
|
+
requires: { type: "array", required: false, description: "Presence preconditions: array of $.input.X or $.steps.X.output... selectors that must resolve to a non-empty value before the step runs. Failures honor onError." },
|
|
1490
|
+
outputSchema: { type: "object", required: false, description: `Optional declaration of the shape this step's output produces. When set, the validator checks that downstream $.steps.<this.id>.output.<field> references point at declared fields. Format: { fieldName: "string"|"number"|"boolean"|"object"|"array"|"unknown" } \u2014 nested objects are supported.` }
|
|
1372
1491
|
},
|
|
1373
1492
|
stepTypes: [
|
|
1374
1493
|
{
|
|
@@ -1808,7 +1927,7 @@ Every step MUST have \`id\`, \`name\`, and \`type\`. The \`type\` determines whi
|
|
|
1808
1927
|
for (const [name, fd] of Object.entries(FLOW_SCHEMA.stepCommonFields)) {
|
|
1809
1928
|
sections.push(`| \`${name}\` | ${fd.type} | ${fd.required ? "yes" : "no"} | ${fd.description} |`);
|
|
1810
1929
|
}
|
|
1811
|
-
sections.push(`| \`onError\` | object | no | Error handling: \`{ "strategy": "${FLOW_SCHEMA.errorStrategies.join(" | ")}", "retries": 3, "retryDelayMs": 1000, "backoff": "fixed \\| exponential \\| exponential-jitter", "maxDelayMs": 30000 }\` |`);
|
|
1930
|
+
sections.push(`| \`onError\` | object | no | Error handling: \`{ "strategy": "${FLOW_SCHEMA.errorStrategies.join(" | ")}", "retries": 3, "retryDelayMs": 1000, "backoff": "fixed \\| exponential \\| exponential-jitter", "maxDelayMs": 30000, "retryOn": [429, 502, "ETIMEDOUT"], "failFastOn": [401, 403, 404] }\`. \`retryOn\`/\`failFastOn\` (cli#53) make retries conditional on the error code/status \u2014 \`failFastOn\` matches skip the retry entirely. |`);
|
|
1812
1931
|
sections.push(`
|
|
1813
1932
|
## Step Types
|
|
1814
1933
|
|
|
@@ -1853,6 +1972,20 @@ ${JSON.stringify(st.example, null, 2)}
|
|
|
1853
1972
|
| \`$.loop.item\` / \`$.loop.i\` | Loop iteration |
|
|
1854
1973
|
| \`"Hello {{$.steps.getUser.response.name}}"\` | String interpolation |
|
|
1855
1974
|
|
|
1975
|
+
### Context-aware escape pipes (cli#53)
|
|
1976
|
+
|
|
1977
|
+
Handlebars interpolations support pipe-based escaping for safe embedding into shell commands, JSON, URLs, markdown, or HTML:
|
|
1978
|
+
|
|
1979
|
+
| Pipe | Effect |
|
|
1980
|
+
|------|--------|
|
|
1981
|
+
| \`{{ $.x \\| json }}\` | \`JSON.stringify\` (handles quotes, newlines, unicode) |
|
|
1982
|
+
| \`{{ $.x \\| shell }}\` | POSIX-shell-quote \u2014 safe inside bash arguments |
|
|
1983
|
+
| \`{{ $.x \\| url }}\` | \`encodeURIComponent\` |
|
|
1984
|
+
| \`{{ $.x \\| md }}\` | Escape markdown structural characters |
|
|
1985
|
+
| \`{{ $.x \\| html }}\` | Entity-escape \`& < > " '\` |
|
|
1986
|
+
|
|
1987
|
+
Pipes can be applied to any value (objects/arrays are JSON-stringified first for shell/url/md/html). An unknown pipe name throws at runtime. The legacy \`{{q $.x}}\` shell-quote helper still works but new flows should prefer \`{{$.x | shell}}\`.
|
|
1988
|
+
|
|
1856
1989
|
### When to use bare selectors vs \`{{...}}\` interpolation
|
|
1857
1990
|
|
|
1858
1991
|
- **Bare selectors** (\`$.input.x\`): Use for fields the engine resolves directly \u2014 \`connectionKey\`, \`over\`, \`path\`, \`expression\`, \`condition\`, and any field where the entire value is a single selector. The resolved value keeps its original type (object, array, number).
|
|
@@ -1917,6 +2050,68 @@ If a sub-step id collides with a flattened field name, the flattened field wins
|
|
|
1917
2050
|
|
|
1918
2051
|
Strategies: \`${FLOW_SCHEMA.errorStrategies.join("`, `")}\`
|
|
1919
2052
|
|
|
2053
|
+
**Conditional retry (cli#53):** add \`retryOn\` and/or \`failFastOn\` to discriminate transient errors from permanent ones. \`failFastOn\` takes precedence; \`retryOn\` (when set) requires a match for the retry to happen. Numbers match against any 3-digit substring of the error message (HTTP statuses); strings match against \`error.errorCode\` exactly OR as a case-insensitive substring of the message.
|
|
2054
|
+
|
|
2055
|
+
\`\`\`json
|
|
2056
|
+
{
|
|
2057
|
+
"onError": {
|
|
2058
|
+
"strategy": "retry",
|
|
2059
|
+
"retries": 4,
|
|
2060
|
+
"backoff": "exponential",
|
|
2061
|
+
"retryOn": [429, 502, 503, "ETIMEDOUT", "ECONNRESET"],
|
|
2062
|
+
"failFastOn": [401, 403, 404]
|
|
2063
|
+
}
|
|
2064
|
+
}
|
|
2065
|
+
\`\`\`
|
|
2066
|
+
|
|
2067
|
+
## Step Output Contracts (\`outputSchema\`, cli#59)
|
|
2068
|
+
|
|
2069
|
+
Declare a step's output shape so the validator catches downstream field-name typos at flow load time:
|
|
2070
|
+
|
|
2071
|
+
\`\`\`json
|
|
2072
|
+
{
|
|
2073
|
+
"id": "research",
|
|
2074
|
+
"type": "flow",
|
|
2075
|
+
"flow": { "key": "company-research" },
|
|
2076
|
+
"outputSchema": {
|
|
2077
|
+
"company": "string",
|
|
2078
|
+
"charCount": "number",
|
|
2079
|
+
"quality": { "confidence": "string", "score": "number" }
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
\`\`\`
|
|
2083
|
+
|
|
2084
|
+
Field types: \`string\`, \`number\`, \`boolean\`, \`object\`, \`array\`, \`unknown\`. Nested objects describe sub-fields. Any \`$.steps.<id>.output.<field>\` reference from a downstream step is checked against the schema; unknown fields fail validation. The runtime engine does not enforce the schema \u2014 it's a documentation / wiring-bug aid.
|
|
2085
|
+
|
|
2086
|
+
## Bash structured env vars (cli#54)
|
|
2087
|
+
|
|
2088
|
+
A \`bash\` step's \`env\` map accepts two structured forms in addition to plain strings:
|
|
2089
|
+
|
|
2090
|
+
\`\`\`json
|
|
2091
|
+
{
|
|
2092
|
+
"type": "bash",
|
|
2093
|
+
"bash": {
|
|
2094
|
+
"env": {
|
|
2095
|
+
"PAYLOAD_FILE": { "json": "$.steps.buildConfig.output" },
|
|
2096
|
+
"COMPANY": { "shell": "$.input.companyName" }
|
|
2097
|
+
},
|
|
2098
|
+
"command": "curl -X POST $ENDPOINT -d @$PAYLOAD_FILE && echo \\"$COMPANY\\""
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
\`\`\`
|
|
2102
|
+
|
|
2103
|
+
- \`{ "json": <selector|value> }\` \u2014 JSON-serialized to a temp file; the env var holds the temp file path. Auto-cleaned after the step runs (success or failure).
|
|
2104
|
+
- \`{ "shell": <selector|value> }\` \u2014 exposed as a plain string env var; reference inside bash double quotes (\`"$VAR"\`).
|
|
2105
|
+
- A plain string is the legacy form (interpolated as-is, caller is responsible for escaping).
|
|
2106
|
+
|
|
2107
|
+
## Dynamic sub-flow dispatch (cli#61)
|
|
2108
|
+
|
|
2109
|
+
A \`flow\` step's \`flow.key\` accepts selectors and Handlebars interpolations, so a single orchestrator can route to different sub-flows at runtime:
|
|
2110
|
+
|
|
2111
|
+
\`\`\`json
|
|
2112
|
+
{ "type": "flow", "flow": { "key": "{{$.input.target}}", "inputs": { "company": "$.input.company" } } }
|
|
2113
|
+
\`\`\`
|
|
2114
|
+
|
|
1920
2115
|
Conditional execution: \`"if": "$.steps.prev.response.data.length > 0"\`
|
|
1921
2116
|
|
|
1922
2117
|
## Input Connection Auto-Resolution
|
package/dist/index.js
CHANGED
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
loadFlowWithMeta,
|
|
17
17
|
resolveFlowPath,
|
|
18
18
|
saveFlow
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-AZV4EGKT.js";
|
|
20
20
|
|
|
21
21
|
// src/index.ts
|
|
22
22
|
import { createRequire as createRequire2 } from "module";
|
|
@@ -2605,6 +2605,144 @@ function validateSelectorReferences(flow2) {
|
|
|
2605
2605
|
});
|
|
2606
2606
|
return errors;
|
|
2607
2607
|
}
|
|
2608
|
+
var VALID_OUTPUT_SCHEMA_TYPES = /* @__PURE__ */ new Set(["string", "number", "boolean", "object", "array", "unknown"]);
|
|
2609
|
+
function isOutputSchemaObject(v) {
|
|
2610
|
+
return !!v && typeof v === "object" && !Array.isArray(v);
|
|
2611
|
+
}
|
|
2612
|
+
function walkOutputSchema(schema, path8) {
|
|
2613
|
+
let current = schema;
|
|
2614
|
+
for (let i = 0; i < path8.length; i++) {
|
|
2615
|
+
const seg = path8[i];
|
|
2616
|
+
if (typeof current === "string") {
|
|
2617
|
+
return current === "unknown" || current === "object" || current === "array" ? "opaque" : "opaque";
|
|
2618
|
+
}
|
|
2619
|
+
if (!(seg in current)) return "unknown-field";
|
|
2620
|
+
const next = current[seg];
|
|
2621
|
+
if (typeof next === "string") {
|
|
2622
|
+
if (!VALID_OUTPUT_SCHEMA_TYPES.has(next)) return "unknown-field";
|
|
2623
|
+
current = next;
|
|
2624
|
+
} else if (isOutputSchemaObject(next)) {
|
|
2625
|
+
current = next;
|
|
2626
|
+
} else {
|
|
2627
|
+
return "unknown-field";
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
return "ok";
|
|
2631
|
+
}
|
|
2632
|
+
function collectOutputSchemas(flow2) {
|
|
2633
|
+
const out = /* @__PURE__ */ new Map();
|
|
2634
|
+
const nestedKeys = getNestedStepsKeys();
|
|
2635
|
+
function walk(steps) {
|
|
2636
|
+
for (const step of steps) {
|
|
2637
|
+
if (step.outputSchema && isOutputSchemaObject(step.outputSchema)) {
|
|
2638
|
+
out.set(step.id, step.outputSchema);
|
|
2639
|
+
}
|
|
2640
|
+
for (const { configKey, fieldName } of nestedKeys) {
|
|
2641
|
+
const config2 = step[configKey];
|
|
2642
|
+
if (config2 && Array.isArray(config2[fieldName])) {
|
|
2643
|
+
walk(config2[fieldName]);
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
walk(flow2.steps);
|
|
2649
|
+
return out;
|
|
2650
|
+
}
|
|
2651
|
+
function validateOutputSchemas(flow2) {
|
|
2652
|
+
const errors = [];
|
|
2653
|
+
const schemas = collectOutputSchemas(flow2);
|
|
2654
|
+
if (schemas.size === 0) return errors;
|
|
2655
|
+
for (const [stepId, schema] of schemas) {
|
|
2656
|
+
const schemaErrors = validateOutputSchemaShape(schema, `step "${stepId}".outputSchema`);
|
|
2657
|
+
errors.push(...schemaErrors);
|
|
2658
|
+
}
|
|
2659
|
+
const SELECTOR_RE = /\$\.steps\.([a-zA-Z_][\w-]*)\.output((?:\.[a-zA-Z_][\w-]*)+)/g;
|
|
2660
|
+
function checkText(text4, location) {
|
|
2661
|
+
if (typeof text4 !== "string") return;
|
|
2662
|
+
for (const m of text4.matchAll(SELECTOR_RE)) {
|
|
2663
|
+
const stepId = m[1];
|
|
2664
|
+
const tail = m[2].slice(1).split(".");
|
|
2665
|
+
const schema = schemas.get(stepId);
|
|
2666
|
+
if (!schema) continue;
|
|
2667
|
+
const result = walkOutputSchema(schema, tail);
|
|
2668
|
+
if (result === "unknown-field") {
|
|
2669
|
+
errors.push({
|
|
2670
|
+
path: location,
|
|
2671
|
+
message: `Selector "${m[0]}" references field "${tail.join(".")}" which is not declared in step "${stepId}".outputSchema. Either fix the field name or update the outputSchema declaration.`
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
function walkValue(value, location) {
|
|
2677
|
+
if (typeof value === "string") {
|
|
2678
|
+
checkText(value, location);
|
|
2679
|
+
} else if (Array.isArray(value)) {
|
|
2680
|
+
value.forEach((v, i) => walkValue(v, `${location}[${i}]`));
|
|
2681
|
+
} else if (value && typeof value === "object") {
|
|
2682
|
+
for (const [k, v] of Object.entries(value)) {
|
|
2683
|
+
walkValue(v, `${location}.${k}`);
|
|
2684
|
+
}
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
const nestedKeys = getNestedStepsKeys();
|
|
2688
|
+
const nestedFieldSet = new Set(nestedKeys.map((k) => `${k.configKey}.${k.fieldName}`));
|
|
2689
|
+
function walkConfig(config2, configKey, pathPrefix) {
|
|
2690
|
+
if (!config2 || typeof config2 !== "object" || Array.isArray(config2)) {
|
|
2691
|
+
walkValue(config2, pathPrefix);
|
|
2692
|
+
return;
|
|
2693
|
+
}
|
|
2694
|
+
for (const [k, v] of Object.entries(config2)) {
|
|
2695
|
+
if (nestedFieldSet.has(`${configKey}.${k}`)) continue;
|
|
2696
|
+
walkValue(v, `${pathPrefix}.${k}`);
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
function walkSteps(steps, pathPrefix) {
|
|
2700
|
+
steps.forEach((step, i) => {
|
|
2701
|
+
const stepPath = `${pathPrefix}[${i}]`;
|
|
2702
|
+
if (step.if) checkText(step.if, `${stepPath}.if`);
|
|
2703
|
+
if (step.unless) checkText(step.unless, `${stepPath}.unless`);
|
|
2704
|
+
if (Array.isArray(step.requires)) {
|
|
2705
|
+
step.requires.forEach((s, ri) => checkText(s, `${stepPath}.requires[${ri}]`));
|
|
2706
|
+
}
|
|
2707
|
+
const descriptor = getStepTypeDescriptor(step.type);
|
|
2708
|
+
if (descriptor) {
|
|
2709
|
+
const config2 = step[descriptor.configKey];
|
|
2710
|
+
if (config2) walkConfig(config2, descriptor.configKey, `${stepPath}.${descriptor.configKey}`);
|
|
2711
|
+
for (const { configKey, fieldName } of nestedKeys) {
|
|
2712
|
+
const c = step[configKey];
|
|
2713
|
+
if (c && Array.isArray(c[fieldName])) {
|
|
2714
|
+
walkSteps(c[fieldName], `${stepPath}.${configKey}.${fieldName}`);
|
|
2715
|
+
}
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
});
|
|
2719
|
+
}
|
|
2720
|
+
walkSteps(flow2.steps, "steps");
|
|
2721
|
+
return errors;
|
|
2722
|
+
}
|
|
2723
|
+
function validateOutputSchemaShape(schema, location) {
|
|
2724
|
+
const errors = [];
|
|
2725
|
+
if (!isOutputSchemaObject(schema)) {
|
|
2726
|
+
errors.push({ path: location, message: "outputSchema must be an object" });
|
|
2727
|
+
return errors;
|
|
2728
|
+
}
|
|
2729
|
+
for (const [key, val] of Object.entries(schema)) {
|
|
2730
|
+
const where = `${location}.${key}`;
|
|
2731
|
+
if (typeof val === "string") {
|
|
2732
|
+
if (!VALID_OUTPUT_SCHEMA_TYPES.has(val)) {
|
|
2733
|
+
errors.push({
|
|
2734
|
+
path: where,
|
|
2735
|
+
message: `outputSchema field "${key}" has unknown type "${val}". Allowed: ${[...VALID_OUTPUT_SCHEMA_TYPES].join(", ")}.`
|
|
2736
|
+
});
|
|
2737
|
+
}
|
|
2738
|
+
} else if (isOutputSchemaObject(val)) {
|
|
2739
|
+
errors.push(...validateOutputSchemaShape(val, where));
|
|
2740
|
+
} else {
|
|
2741
|
+
errors.push({ path: where, message: `outputSchema field "${key}" must be a type string or a nested object` });
|
|
2742
|
+
}
|
|
2743
|
+
}
|
|
2744
|
+
return errors;
|
|
2745
|
+
}
|
|
2608
2746
|
function validateFlow(flow2, rootDir) {
|
|
2609
2747
|
const schemaErrors = validateFlowSchema(flow2);
|
|
2610
2748
|
if (schemaErrors.length > 0) return schemaErrors;
|
|
@@ -2612,6 +2750,7 @@ function validateFlow(flow2, rootDir) {
|
|
|
2612
2750
|
return [
|
|
2613
2751
|
...validateStepIds(f),
|
|
2614
2752
|
...validateSelectorReferences(f),
|
|
2753
|
+
...validateOutputSchemas(f),
|
|
2615
2754
|
...rootDir ? validateCodeModules(f, rootDir) : []
|
|
2616
2755
|
];
|
|
2617
2756
|
}
|
package/package.json
CHANGED
|
@@ -234,6 +234,29 @@ A pure `$.xxx` value resolves to the raw type. A string containing `{{$.xxx}}` d
|
|
|
234
234
|
"files": "$.steps.extract.output.allFiles"
|
|
235
235
|
```
|
|
236
236
|
|
|
237
|
+
### Context-aware escape pipes (cli#53)
|
|
238
|
+
|
|
239
|
+
Handlebars interpolations support pipe-based escaping so user-controlled values are safe to embed in shell commands, JSON payloads, URLs, markdown, or HTML without writing per-call escapers:
|
|
240
|
+
|
|
241
|
+
```json
|
|
242
|
+
{
|
|
243
|
+
"command": "curl -d {{$.input.payload | json}} https://example.com/{{$.input.slug | url}}",
|
|
244
|
+
"env": { "GREETING": { "shell": "$.input.name" } }
|
|
245
|
+
}
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Available pipes:
|
|
249
|
+
|
|
250
|
+
| Pipe | Effect | Example |
|
|
251
|
+
|------|--------|---------|
|
|
252
|
+
| `json` | `JSON.stringify` (handles quotes, newlines, unicode) | `{{$.x \| json}}` → `"O'Brien & Co."` |
|
|
253
|
+
| `shell` | POSIX-shell-quote (apostrophes use the `'\''` close-reopen trick) | `{{$.x \| shell}}` → `'O'\''Brien & Co.'` |
|
|
254
|
+
| `url` | `encodeURIComponent` | `{{$.x \| url}}` → `O'Brien%20%26%20Co.` |
|
|
255
|
+
| `md` | Escape markdown structural characters (` ` ` * _ { } [ ] ( ) # + - ! \| `) | `{{x \| md}}` |
|
|
256
|
+
| `html` | Entity-escape `& < > " '` | `{{x \| html}}` → `<b>` |
|
|
257
|
+
|
|
258
|
+
Pipes can be applied to numbers, booleans, objects, and `null`/`undefined` (which become empty string for shell/url/md/html, `null` for json). An unknown pipe name throws a clear error at flow execution time. Pipes cannot be combined with the legacy `q` prefix — use the pipe form (`{{$.x | shell}}` instead of `{{q $.x}}`).
|
|
259
|
+
|
|
237
260
|
### Selectors vs expressions
|
|
238
261
|
|
|
239
262
|
Selectors in data fields (`data`, `queryParams`, `pathVars`, `connectionKey`) are **dot-path lookups only** — they do not support JavaScript operators like `||` or `&&`. For default values, use the `default` field on the input definition:
|
|
@@ -390,6 +413,8 @@ After a parallel step, access each substep's output by its `id`: `$.steps.fetchE
|
|
|
390
413
|
}
|
|
391
414
|
```
|
|
392
415
|
|
|
416
|
+
**Dynamic dispatch (cli#61).** `flow.key` accepts a selector (`"$.input.target"`) or Handlebars interpolation (`"{{$.input.prefix}}-{{$.input.suffix}}"`). The resolved key is loaded at runtime, so you can write a single orchestrator that picks among multiple sub-flows based on inputs or upstream results — no bash workaround required. If the resolved key does not exist, the step fails with the standard "flow not found" error.
|
|
417
|
+
|
|
393
418
|
A sub-flow step exposes the sub-flow's **step results map** at both `.output` and `.response` (they are aliases — pick whichever reads better). Access a specific sub-step's data with:
|
|
394
419
|
|
|
395
420
|
```
|
|
@@ -444,6 +469,27 @@ Alternatively, pass values as environment variables (also shell-safe) and refere
|
|
|
444
469
|
}
|
|
445
470
|
```
|
|
446
471
|
|
|
472
|
+
**Structured env vars (cli#54).** A bash step's `env` map also accepts two structured forms that handle JSON and shell escaping safely without writing temp files by hand:
|
|
473
|
+
|
|
474
|
+
```json
|
|
475
|
+
{
|
|
476
|
+
"type": "bash",
|
|
477
|
+
"bash": {
|
|
478
|
+
"env": {
|
|
479
|
+
"PAYLOAD_FILE": { "json": "$.steps.buildConfig.output" },
|
|
480
|
+
"COMPANY": { "shell": "$.input.companyName" }
|
|
481
|
+
},
|
|
482
|
+
"command": "curl -X POST $ENDPOINT -H 'Content-Type: application/json' -d @$PAYLOAD_FILE && echo \"$COMPANY\""
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
- `{ "json": <selector|value> }` — the resolved value is JSON-serialized, written to a temp file, and the env var is set to the temp file's path. Use it with `curl -d @$VAR` or `cat $VAR`. The temp file is cleaned up automatically after the step finishes (success OR failure).
|
|
488
|
+
- `{ "shell": <selector|value> }` — the resolved value is exposed as a plain string env var. Reference it inside bash double quotes (`"$VAR"`) so bash itself handles word-splitting.
|
|
489
|
+
- A plain string value (`"COMPANY": "$.input.companyName"`) is the legacy form — interpolated as-is.
|
|
490
|
+
|
|
491
|
+
This is the recommended way to pass structured payloads to `curl`, `claude --print`, or any CLI that expects a JSON file. It eliminates the older `file-write → bash` two-step workaround.
|
|
492
|
+
|
|
447
493
|
## Step Input Contracts (`requires`)
|
|
448
494
|
|
|
449
495
|
Declare the data a step depends on so the engine fails fast — with a useful error — when an upstream value is missing. Without `requires`, a skipped or failed upstream step silently leaves `undefined` in the context and the consumer either crashes deep in user code or burns an LLM call on empty input.
|
|
@@ -473,6 +519,34 @@ The "because…" suffix tells you exactly why — skipped, failed, or timed out
|
|
|
473
519
|
|
|
474
520
|
Forward references are caught at flow load time: if `requires` points at a step declared after the current step, validation rejects the flow.
|
|
475
521
|
|
|
522
|
+
## Step Output Contracts (`outputSchema`)
|
|
523
|
+
|
|
524
|
+
Declare the shape of a step's `output` so the validator can catch field-name typos in downstream selectors at flow load time — long before a misspelled `$.steps.research.output.charCount` silently resolves to `undefined` at runtime:
|
|
525
|
+
|
|
526
|
+
```json
|
|
527
|
+
{
|
|
528
|
+
"id": "research",
|
|
529
|
+
"type": "flow",
|
|
530
|
+
"flow": { "key": "company-research" },
|
|
531
|
+
"outputSchema": {
|
|
532
|
+
"company": "string",
|
|
533
|
+
"research": "string",
|
|
534
|
+
"charCount": "number",
|
|
535
|
+
"quality": { "confidence": "string", "coverageScore": "number" }
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
Field types: `"string"`, `"number"`, `"boolean"`, `"object"`, `"array"`, `"unknown"`. Nest objects to describe sub-fields (`quality.coverageScore` above). Anything not declared is rejected when referenced via `$.steps.<id>.output.<field>` from a downstream step:
|
|
541
|
+
|
|
542
|
+
```
|
|
543
|
+
Selector "$.steps.research.output.chars" references field "chars" which is not
|
|
544
|
+
declared in step "research".outputSchema. Either fix the field name or update
|
|
545
|
+
the outputSchema declaration.
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
`outputSchema` is purely a documentation / validation aid — the engine does **not** enforce the shape at runtime, so a code step that returns an unexpected field still works (it just won't be discoverable from typed selectors). Schemas declared on a step apply to all references from anywhere in the flow tree (including inside loops, conditions, parallel blocks, code, and transform expressions).
|
|
549
|
+
|
|
476
550
|
## Error Handling
|
|
477
551
|
|
|
478
552
|
```json
|
|
@@ -497,6 +571,26 @@ Strategies: `fail` (default), `continue`, `retry`, `fallback`.
|
|
|
497
571
|
|
|
498
572
|
`exponential` waits `retryDelayMs * 2^(retryIndex)` (1s, 2s, 4s, 8s…) capped at `maxDelayMs`. `exponential-jitter` multiplies each wait by a random factor in [0.5, 1.0) so concurrent retries spread out.
|
|
499
573
|
|
|
574
|
+
**Conditional retry (cli#53).** By default a `retry` strategy retries every error. To distinguish transient failures (rate-limits, timeouts) from permanent ones (auth errors, 404s) add `retryOn` and/or `failFastOn`:
|
|
575
|
+
|
|
576
|
+
```json
|
|
577
|
+
{
|
|
578
|
+
"onError": {
|
|
579
|
+
"strategy": "retry",
|
|
580
|
+
"retries": 4,
|
|
581
|
+
"backoff": "exponential",
|
|
582
|
+
"retryOn": [429, 502, 503, "ETIMEDOUT", "ECONNRESET"],
|
|
583
|
+
"failFastOn": [401, 403, 404]
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
```
|
|
587
|
+
|
|
588
|
+
- `failFastOn` takes precedence: if the error matches any entry, the step fails immediately without consuming retries.
|
|
589
|
+
- `retryOn` (when set): the error must match an entry to be retried; non-matching errors fail immediately.
|
|
590
|
+
- If neither is set the legacy "retry every error" behavior applies.
|
|
591
|
+
|
|
592
|
+
Match rules: number entries are compared against any 3-digit token in the error message (covers `HTTP 429`, `status 502`, etc.); string entries match `error.errorCode` exactly OR appear as a case-insensitive substring of the message (covers Node error codes like `ETIMEDOUT` and our own `TIMEOUT`).
|
|
593
|
+
|
|
500
594
|
**Inspecting retry outcomes.** Every retried step exposes how it ended on its `StepResult`:
|
|
501
595
|
|
|
502
596
|
- `$.steps.<id>.status` — `"success"` or `"failed"`
|