@automatalabs/workflows 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -7
- package/dist/cli.js +58 -5
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/validate.d.ts +39 -0
- package/dist/validate.d.ts.map +1 -1
- package/dist/validate.js +512 -21
- package/package.json +4 -6
package/README.md
CHANGED
|
@@ -482,16 +482,56 @@ failure, `2` dry-run failure, `3` usage error.
|
|
|
482
482
|
Flags: `--args <json>` / `--args-file <path>`, `--workflows-dir <dir>` (repeatable — validate by
|
|
483
483
|
NAME and resolve nested `workflow("<name>")` calls from your folder), `--parse-only`,
|
|
484
484
|
`--cwd <dir>`, `--token-budget <n>` (exercise `budget`-guarded paths; the mock reports 1000
|
|
485
|
-
tokens per call), `--max-agents <n>`, `--timeout-ms <n>`, `--json
|
|
486
|
-
|
|
487
|
-
|
|
485
|
+
tokens per call), `--max-agents <n>`, `--timeout-ms <n>`, `--mock-answers <json>`,
|
|
486
|
+
`--mock-answers-file <path>`, `--json`. The two mock-answer flags are mutually exclusive.
|
|
487
|
+
|
|
488
|
+
Mock answers select the final resolved agent label with case-sensitive, whole-label globs: `*`
|
|
489
|
+
matches any characters (including `:` and `/`), `?` matches one character, and `\` escapes the
|
|
490
|
+
next character. Rules are captured once in object-member order and the **last matching rule wins**,
|
|
491
|
+
so put broad defaults before narrow exceptions. Raw canonical array-index property names (`"0"`
|
|
492
|
+
through `"4294967294"`, with no leading zero) are rejected because JavaScript reorders them;
|
|
493
|
+
escape a digit to match a numeric label, for example the JSON key `"\\10"` matches label `10`.
|
|
494
|
+
`"01"` and `"4294967295"` are ordinary keys.
|
|
495
|
+
|
|
496
|
+
A rule is either one reusable JSON answer or `{ "$sequence": [...] }`, a finite sequence consumed
|
|
497
|
+
only when that rule wins. A raw JSON array is one array-valued answer. Each schema-bearing answer
|
|
498
|
+
deep-merges over a fresh `fabricateFromSchema()` base: objects merge recursively, while arrays,
|
|
499
|
+
`null`, scalars, and falsy primitives replace. The merged value is checked without coercion. Any
|
|
500
|
+
answer-caused schema violation fails with `SCHEMA_NONCOMPLIANCE`; an identical failure inherited
|
|
501
|
+
from an untouched limitation of the simple fabricator may be accepted with a grouped warning.
|
|
502
|
+
Schema-less answers must be nonblank strings. Sequences never repeat or fall back: exhaustion fails
|
|
503
|
+
the dry run, while unconsumed singles/items remain non-fatal and appear in structured `unused`
|
|
504
|
+
records plus grouped warnings. Supplying mock answers serializes dry-run agent service for stable
|
|
505
|
+
FIFO sequence allocation, so this mode is not a concurrency/load simulator and its token-budget
|
|
506
|
+
admission timing can differ from an unscripted concurrent dry run.
|
|
507
|
+
|
|
508
|
+
Fixture input is capped at 256 KiB (raw CLI UTF-8 and canonical programmatic JSON), 256 rules,
|
|
509
|
+
256 UTF-16 code units per glob, 256 sequence entries, and answer nesting depth 32. Values must be
|
|
510
|
+
plain JSON data. Reports and fixture errors expose only globs, counters, positions, and schema
|
|
511
|
+
diagnostics—not answer bodies. A fixture is still handed to workflow code like a real agent result,
|
|
512
|
+
so the script can deliberately expose it through `log()` or its return value; do not put credentials
|
|
513
|
+
or production data in mock-answer files.
|
|
514
|
+
|
|
515
|
+
The same check is available programmatically. Invalid workflow scripts still resolve to reports;
|
|
516
|
+
an invalid `mockAnswers` option is an option-contract error and throws `TypeError` before parsing:
|
|
488
517
|
|
|
489
518
|
```ts
|
|
490
|
-
import { validateWorkflowScript } from "@automatalabs/workflows";
|
|
491
|
-
|
|
492
|
-
const
|
|
519
|
+
import { validateWorkflowScript, type MockAnswers } from "@automatalabs/workflows";
|
|
520
|
+
|
|
521
|
+
const mockAnswers: MockAnswers = {
|
|
522
|
+
"*": { approved: true },
|
|
523
|
+
"refute:*": { real: false },
|
|
524
|
+
"quality:review": {
|
|
525
|
+
$sequence: [
|
|
526
|
+
{ ok: false, feedback: "exercise the revision path" },
|
|
527
|
+
{ ok: true },
|
|
528
|
+
],
|
|
529
|
+
},
|
|
530
|
+
};
|
|
531
|
+
const report = await validateWorkflowScript(script, { args: { target: "src/" }, mockAnswers });
|
|
493
532
|
report.ok; // parse ok AND dry run completed
|
|
494
|
-
report.dryRun?.agentCalls; //
|
|
533
|
+
report.dryRun?.agentCalls; // calls include mockAnswer: { glob, sequenceIndex?, sequenceLength? }
|
|
534
|
+
report.dryRun?.mockAnswers;// normalized rule counters + item-level unused records
|
|
495
535
|
report.warnings; // approval reminders, phase mismatches, headless-abort checkpoints, …
|
|
496
536
|
```
|
|
497
537
|
|
|
@@ -593,6 +633,8 @@ AGENTPRISM_PERSISTENCE_ROOT_ENV,
|
|
|
593
633
|
|
|
594
634
|
// ── Types ──
|
|
595
635
|
RunDynamicWorkflowOptions, WorkflowRunOptions, AgentOptions, ExecOptions,
|
|
636
|
+
MockAnswerJson, MockAnswerSequence, MockAnswerRule, MockAnswers,
|
|
637
|
+
ValidatedMockAnswerUse, ValidatedMockAnswerRule, UnusedMockAnswer, ValidatedMockAnswers,
|
|
596
638
|
ValidateWorkflowOptions, ValidateWorkflowReport, ValidatedAgentCall, ValidatedCheckpoint,
|
|
597
639
|
WorkflowManagerOptions, CheckpointOptions, WorkflowRunResult, WorkflowSnapshot,
|
|
598
640
|
WorkflowPathOptions, RunPersistence, RunPersistenceOptions,
|
package/dist/cli.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// syntax, determinism blocklist), then a dry run over an in-process mock AgentRunner
|
|
8
8
|
// that fabricates schema-conforming results — no ACP process is spawned. See
|
|
9
9
|
// ./validate.ts for the programmatic API (`validateWorkflowScript`).
|
|
10
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
10
|
+
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { resolve } from "node:path";
|
|
12
12
|
import { openWorkflowDir } from "@automatalabs/workflow-engine";
|
|
13
13
|
import { validateWorkflowScript, formatValidateReport } from "./validate.js";
|
|
@@ -22,6 +22,8 @@ Validates an AgentPrism workflow script without spending tokens:
|
|
|
22
22
|
Options:
|
|
23
23
|
--args <json> the script's \`args\` global for the dry run (a JSON value)
|
|
24
24
|
--args-file <path> read the args JSON from a file instead
|
|
25
|
+
--mock-answers <json> label-glob mock answers for dry-run agent calls
|
|
26
|
+
--mock-answers-file <path> read the label-glob mock answers JSON from a UTF-8 file
|
|
25
27
|
--workflows-dir <dir> a directory of workflow scripts (repeatable; precedence in
|
|
26
28
|
the order given). Enables validating by NAME (filename stem)
|
|
27
29
|
and resolves nested workflow("<name>") calls in the dry run
|
|
@@ -44,7 +46,7 @@ Notes:
|
|
|
44
46
|
|
|
45
47
|
Exit codes: 0 valid · 1 parse/static failure · 2 dry-run failure · 3 usage error`;
|
|
46
48
|
function fail(message) {
|
|
47
|
-
process.stderr.
|
|
49
|
+
writeFileSync(process.stderr.fd, `${message}\n\nRun \`agentprism-workflows validate --help\` for usage.\n`);
|
|
48
50
|
process.exit(3);
|
|
49
51
|
}
|
|
50
52
|
function parseIntFlag(name, raw) {
|
|
@@ -63,6 +65,7 @@ async function main(argv) {
|
|
|
63
65
|
fail(`unknown command "${command}" — the only command is: validate`);
|
|
64
66
|
let file;
|
|
65
67
|
let json = false;
|
|
68
|
+
let mockAnswersFlag;
|
|
66
69
|
const workflowDirs = [];
|
|
67
70
|
const options = {};
|
|
68
71
|
for (let i = 0; i < rest.length; i++) {
|
|
@@ -95,6 +98,47 @@ async function main(argv) {
|
|
|
95
98
|
fail(`--args-file: ${error instanceof Error ? error.message : String(error)}`);
|
|
96
99
|
}
|
|
97
100
|
break;
|
|
101
|
+
case "--mock-answers": {
|
|
102
|
+
if (mockAnswersFlag === arg)
|
|
103
|
+
fail(`${arg} may appear at most once`);
|
|
104
|
+
if (mockAnswersFlag)
|
|
105
|
+
fail("--mock-answers and --mock-answers-file are mutually exclusive");
|
|
106
|
+
mockAnswersFlag = arg;
|
|
107
|
+
const source = rest[++i];
|
|
108
|
+
if (source === undefined)
|
|
109
|
+
fail("--mock-answers expects a JSON object (quote it for your shell)");
|
|
110
|
+
if (Buffer.byteLength(source, "utf8") > 256 * 1024)
|
|
111
|
+
fail("--mock-answers exceeds the 256 KiB source limit");
|
|
112
|
+
try {
|
|
113
|
+
options.mockAnswers = JSON.parse(source);
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
fail("--mock-answers expects a JSON object (quote it for your shell)");
|
|
117
|
+
}
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
case "--mock-answers-file": {
|
|
121
|
+
if (mockAnswersFlag === arg)
|
|
122
|
+
fail(`${arg} may appear at most once`);
|
|
123
|
+
if (mockAnswersFlag)
|
|
124
|
+
fail("--mock-answers and --mock-answers-file are mutually exclusive");
|
|
125
|
+
mockAnswersFlag = arg;
|
|
126
|
+
const rawPath = rest[++i];
|
|
127
|
+
if (rawPath === undefined)
|
|
128
|
+
fail("--mock-answers-file expects a path");
|
|
129
|
+
const path = resolve(rawPath);
|
|
130
|
+
try {
|
|
131
|
+
if (statSync(path).size > 256 * 1024)
|
|
132
|
+
fail("--mock-answers-file exceeds the 256 KiB source limit");
|
|
133
|
+
options.mockAnswers = JSON.parse(readFileSync(path, "utf8"));
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
if (error instanceof SyntaxError)
|
|
137
|
+
fail(`--mock-answers-file: ${error.message}`);
|
|
138
|
+
fail(`--mock-answers-file: ${error instanceof Error ? error.message : String(error)}`);
|
|
139
|
+
}
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
98
142
|
case "--workflows-dir":
|
|
99
143
|
workflowDirs.push(resolve(rest[++i] ?? fail("--workflows-dir expects a directory")));
|
|
100
144
|
break;
|
|
@@ -144,9 +188,18 @@ async function main(argv) {
|
|
|
144
188
|
else {
|
|
145
189
|
fail(`cannot read ${file}: no such file (pass --workflows-dir to validate by name)`);
|
|
146
190
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
191
|
+
let report;
|
|
192
|
+
try {
|
|
193
|
+
report = await validateWorkflowScript(script, options);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (mockAnswersFlag && error instanceof TypeError) {
|
|
197
|
+
fail(`${mockAnswersFlag}: ${error.message}`);
|
|
198
|
+
}
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
writeFileSync(process.stdout.fd, json ? `${JSON.stringify(report, null, 2)}\n` : `${formatValidateReport(report)}\n`);
|
|
202
|
+
process.exitCode = report.exitCode;
|
|
150
203
|
}
|
|
151
204
|
main(process.argv.slice(2)).catch((error) => {
|
|
152
205
|
process.stderr.write(`validate crashed: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`);
|
package/dist/index.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import type { AgentRunner, WorkflowBackendConfig, WorkflowRunResult } from "@aut
|
|
|
5
5
|
export { runWorkflow, parseWorkflowScript, redactText, truncateUtf8 } from "@automatalabs/workflow-engine";
|
|
6
6
|
export { openWorkflowDir, type WorkflowDir, type WorkflowDirEntry, type OpenWorkflowDirOptions, } from "@automatalabs/workflow-engine";
|
|
7
7
|
export { validateWorkflowScript, fabricateFromSchema, formatValidateReport, MOCK_TOKENS_PER_AGENT } from "./validate.js";
|
|
8
|
-
export type { ValidateWorkflowOptions, ValidateWorkflowReport, ValidatedAgentCall, ValidatedCheckpoint, } from "./validate.js";
|
|
8
|
+
export type { MockAnswerJson, MockAnswerRule, MockAnswers, MockAnswerSequence, UnusedMockAnswer, ValidateWorkflowOptions, ValidateWorkflowReport, ValidatedAgentCall, ValidatedCheckpoint, ValidatedMockAnswerRule, ValidatedMockAnswers, ValidatedMockAnswerUse, } from "./validate.js";
|
|
9
9
|
export type { WorkflowRunOptions, AgentOptions, ExecOptions, WorkflowManagerOptions, CheckpointOptions, WorkflowRunResult, WorkflowSnapshot, WorkflowPathOptions, RunPersistence, RunPersistenceOptions, PersistedRunState, PersistedAgentState, WorkflowLogTail, WorkflowRunCallStatus, WorkflowRunInspectionOptions, WorkflowRunStatus, WorkflowRunStatusTruncation, } from "@automatalabs/workflow-engine";
|
|
10
10
|
export { AGENTPRISM_PERSISTENCE_ROOT_ENV, WorkflowError, WorkflowErrorCode, isWorkflowError, isProviderUsageLimit, isAuthRequired, } from "@automatalabs/workflow-engine";
|
|
11
11
|
export { createAcpRunner, AcpAgentRunner, InteractiveSession, selectBackend, ClaudeBackend, CodexBackend, CustomAcpBackend, AGENT_METHODS, CLIENT_METHODS, AGENT_METHOD_COVERAGE, CLIENT_METHOD_COVERAGE, ACP_AUTH_REQUIRED_ERROR_CODE, clientCapabilitiesFor, adaptPromptContent, resolveBackendRegistry, BACKENDS_ENV, toJsonSchema, toStrictJsonSchema, } from "@automatalabs/acp-agents";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAKL,eAAe,IAAI,qBAAqB,EACzC,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAoB,YAAY,EAAE,iBAAiB,EAAiB,MAAM,0BAA0B,CAAC;AACjH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACtG,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAMxG,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAO3G,OAAO,EACL,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,GAC5B,MAAM,+BAA+B,CAAC;AAIvC,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACzH,YAAY,EACV,uBAAuB,EACvB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAKL,eAAe,IAAI,qBAAqB,EACzC,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAoB,YAAY,EAAE,iBAAiB,EAAiB,MAAM,0BAA0B,CAAC;AACjH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACtG,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAMxG,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAO3G,OAAO,EACL,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,GAC5B,MAAM,+BAA+B,CAAC;AAIvC,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACzH,YAAY,EACV,cAAc,EACd,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,uBAAuB,EACvB,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,qBAAqB,EACrB,4BAA4B,EAC5B,iBAAiB,EACjB,2BAA2B,GAC5B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,+BAA+B,EAC/B,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,cAAc,GACf,MAAM,+BAA+B,CAAC;AAQvC,OAAO,EACL,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,aAAa,EACb,cAAc,EACd,qBAAqB,EACrB,sBAAsB,EACtB,4BAA4B,EAC5B,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACZ,YAAY,EACZ,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,yBAAyB,EACzB,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,cAAc,EACd,UAAU,EACV,WAAW,EACX,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,0BAA0B,EAC1B,6BAA6B,EAC7B,gBAAgB,EAChB,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,UAAU,EACV,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,WAAW,EACX,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,cAAc,EACd,eAAe,EACf,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EACrB,UAAU,EACV,YAAY,EACZ,qBAAqB,EACrB,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,uBAAuB,EACvB,+BAA+B,EAC/B,+BAA+B,EAC/B,wBAAwB,EACxB,yBAAyB,EACzB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,2BAA2B,EAC3B,mBAAmB,EACnB,aAAa,EACb,yBAAyB,EACzB,uBAAuB,EACvB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,EAC1B,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,WAAW,EACX,kBAAkB,EAClB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,0BAA0B,CAAC;AAOlC,YAAY,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAMtF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EACV,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,2BAA2B,EAC3B,mBAAmB,EACnB,0BAA0B,EAC1B,yBAAyB,EACzB,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,0BAA0B,CAAC;AAIlC,YAAY,EACV,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,UAAU,EACV,WAAW,EACX,UAAU,GACX,MAAM,4BAA4B,CAAC;AACpC,YAAY,EACV,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,YAAY,EACZ,qBAAqB,EACrB,YAAY,GACb,MAAM,4BAA4B,CAAC;AAwBpC,KAAK,eAAe,CAAC,CAAC,EAAE,CAAC,SAAS,WAAW,IAAI,CAAC,SAAS,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAClF,KAAK,uBAAuB,CAAC,CAAC,EAAE,CAAC,SAAS,WAAW,IAAI,CAAC,SAAS,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;AAE9F;;6FAE6F;AAC7F,KAAK,oBAAoB,GAAG;KACzB,CAAC,IAAI,YAAY,GAAG;QACnB,IAAI,EAAE,CAAC,CAAC;QACR,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAC5B,SAAS,EAAE,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;KAC/D,GAAG,CAAC,WAAW,SAAS,MAAM,iBAAiB,CAAC,CAAC,CAAC,GAC/C;QAAE,SAAS,EAAE,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;KAAE,GACjE;QAAE,SAAS,CAAC,EAAE,SAAS,CAAA;KAAE,CAAC,GAAG;QAC7B,KAAK,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC/D,KAAK,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;KAChE;CACJ,CAAC;AACF,MAAM,MAAM,iBAAiB,CAAC,CAAC,SAAS,YAAY,GAAG,YAAY,IAAI,oBAAoB,CAAC,CAAC,CAAC,CAAC;AAE/F;;;;;;;;;GASG;AACH,qBAAa,eAAgB,SAAQ,qBAAqB;IACxD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgD;gBAE/D,OAAO,GAAE,sBAA2B;IAKvC,iBAAiB,CACxB,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,OAAO,EACd,IAAI,GAAE,WAAgB,GACrB;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAA;KAAE;IAY1C,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAS3F,kBAAkB,CAC/B,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,WAAgB,GACrB,OAAO,CACN;QAAE,QAAQ,EAAE,KAAK,CAAC;QAAC,OAAO,CAAC,EAAE,SAAS,CAAA;KAAE,GACxC;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAA;KAAE,CAC1D;IAgBc,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,OAAO,CAAC;IAK9E;8FAC0F;IAC1F,OAAO,IAAI,IAAI;IAOf,8EAA8E;IAC9E,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,sBAAsB;CAiC/B;AAoCD;;;;;;;;GAQG;AACH,MAAM,MAAM,qBAAqB,GAC7B,OAAO,GACP,CAAC,CAAC,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,qBAAqB,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAExF,8CAA8C;AAC9C,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,kGAAkG;IAClG,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,+FAA+F;IAC/F,mBAAmB,CAAC,EAAE,qBAAqB,CAAC;IAC5C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC;CAC7C;AAED;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,yBAA8B,GACnC,OAAO,CAAC,iBAAiB,CAAC,CAmC5B"}
|
package/dist/validate.d.ts
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
import type { WorkflowDir } from "@automatalabs/workflow-engine";
|
|
2
2
|
import type { WorkflowMeta } from "@automatalabs/shared-types";
|
|
3
|
+
export type MockAnswerJson = null | boolean | number | string | MockAnswerJson[] | {
|
|
4
|
+
[key: string]: MockAnswerJson;
|
|
5
|
+
};
|
|
6
|
+
export interface MockAnswerSequence {
|
|
7
|
+
readonly $sequence: readonly MockAnswerJson[];
|
|
8
|
+
}
|
|
9
|
+
export type MockAnswerRule = MockAnswerJson | MockAnswerSequence;
|
|
10
|
+
/** Label glob -> one reusable answer or one finite answer sequence. */
|
|
11
|
+
export type MockAnswers = Readonly<Record<string, MockAnswerRule>>;
|
|
3
12
|
export interface ValidateWorkflowOptions {
|
|
4
13
|
/** The `args` global handed to the script during the dry run. */
|
|
5
14
|
args?: unknown;
|
|
@@ -18,6 +27,34 @@ export interface ValidateWorkflowOptions {
|
|
|
18
27
|
maxAgents?: number;
|
|
19
28
|
/** Dry-run wall-clock limit. Default 30_000 ms. */
|
|
20
29
|
timeoutMs?: number;
|
|
30
|
+
/** Dry-run answers selected by the resolved agent label. */
|
|
31
|
+
mockAnswers?: MockAnswers;
|
|
32
|
+
}
|
|
33
|
+
export interface ValidatedMockAnswerUse {
|
|
34
|
+
glob: string;
|
|
35
|
+
/** Zero-based in the machine report; absent for a reusable single answer. */
|
|
36
|
+
sequenceIndex?: number;
|
|
37
|
+
sequenceLength?: number;
|
|
38
|
+
}
|
|
39
|
+
export interface ValidatedMockAnswerRule {
|
|
40
|
+
glob: string;
|
|
41
|
+
kind: "single" | "sequence";
|
|
42
|
+
/** Reached calls whose labels matched this glob, including calls won by a later glob. */
|
|
43
|
+
matchingCalls: number;
|
|
44
|
+
/** Calls for which this rule won and reserved an answer, including fixture-validation failures. */
|
|
45
|
+
consumedCalls: number;
|
|
46
|
+
sequenceLength?: number;
|
|
47
|
+
}
|
|
48
|
+
export interface UnusedMockAnswer {
|
|
49
|
+
glob: string;
|
|
50
|
+
/** Zero-based sequence item; absent for a reusable single answer. */
|
|
51
|
+
sequenceIndex?: number;
|
|
52
|
+
reason: "no-match" | "shadowed" | "not-reached";
|
|
53
|
+
}
|
|
54
|
+
export interface ValidatedMockAnswers {
|
|
55
|
+
/** Captured normalized rule order, which also documents last-match precedence. */
|
|
56
|
+
rules: ValidatedMockAnswerRule[];
|
|
57
|
+
unused: UnusedMockAnswer[];
|
|
21
58
|
}
|
|
22
59
|
/** One agent() call observed during the dry run, with its backend attribution. */
|
|
23
60
|
export interface ValidatedAgentCall {
|
|
@@ -32,6 +69,7 @@ export interface ValidatedAgentCall {
|
|
|
32
69
|
backend: string;
|
|
33
70
|
/** True when the call requested structured output. */
|
|
34
71
|
schema: boolean;
|
|
72
|
+
mockAnswer?: ValidatedMockAnswerUse;
|
|
35
73
|
}
|
|
36
74
|
export interface ValidatedCheckpoint {
|
|
37
75
|
prompt: string;
|
|
@@ -62,6 +100,7 @@ export interface ValidateWorkflowReport {
|
|
|
62
100
|
durationMs: number;
|
|
63
101
|
/** The script's return value, composed from fabricated agent results. */
|
|
64
102
|
result?: unknown;
|
|
103
|
+
mockAnswers?: ValidatedMockAnswers;
|
|
65
104
|
};
|
|
66
105
|
warnings: string[];
|
|
67
106
|
}
|
package/dist/validate.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAA2B,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAGxF,MAAM,MAAM,cAAc,GACtB,IAAI,GACJ,OAAO,GACP,MAAM,GACN,MAAM,GACN,cAAc,EAAE,GAChB;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAAA;CAAE,CAAC;AAEtC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;CAC/C;AAED,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,kBAAkB,CAAC;AAEjE,uEAAuE;AACvE,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;AAEnE,MAAM,WAAW,uBAAuB;IACtC,iEAAiE;IACjE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;2FACuF;IACvF,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC;IAC5C;6EACyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;8CAC0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAC;IAC5B,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;IACtB,mGAAmG;IACnG,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,aAAa,CAAC;CACjD;AAED,MAAM,WAAW,oBAAoB;IACnC,kFAAkF;IAClF,KAAK,EAAE,uBAAuB,EAAE,CAAC;IACjC,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC5B;AAED,kFAAkF;AAClF,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;6FACyF;IACzF,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,sBAAsB,CAAC;CACrC;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,8FAA8F;IAC9F,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,8EAA8E;IAC9E,EAAE,EAAE,OAAO,CAAC;IACZ,gEAAgE;IAChE,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpB,KAAK,EAAE;QACL,EAAE,EAAE,OAAO,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,YAAY,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,EAAE,EAAE,OAAO,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0EAA0E;QAC1E,QAAQ,EAAE,OAAO,CAAC;QAClB,UAAU,EAAE,kBAAkB,EAAE,CAAC;QACjC,WAAW,EAAE,mBAAmB,EAAE,CAAC;QACnC,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,yEAAyE;QACzE,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,oBAAoB,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAwfD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,SAAU,EAAE,KAAK,SAAI,GAAG,OAAO,CA6DvF;AAcD;qDACqD;AACrD,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAqB1C;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,sBAAsB,CAAC,CAkNjC;AAMD,wEAAwE;AACxE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,CAsC3E"}
|
package/dist/validate.js
CHANGED
|
@@ -9,8 +9,463 @@
|
|
|
9
9
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
10
10
|
import { tmpdir } from "node:os";
|
|
11
11
|
import { join } from "node:path";
|
|
12
|
-
import { openWorkflowDir, WorkflowManager, parseWorkflowScript } from "@automatalabs/workflow-engine";
|
|
12
|
+
import { openWorkflowDir, WorkflowError, WorkflowErrorCode, WorkflowManager, parseWorkflowScript, redactText, } from "@automatalabs/workflow-engine";
|
|
13
13
|
import { resolveBackendRegistry, selectBackend } from "@automatalabs/acp-agents";
|
|
14
|
+
import { Check, Errors } from "typebox/value";
|
|
15
|
+
const MAX_MOCK_ANSWERS_BYTES = 256 * 1024;
|
|
16
|
+
const MAX_MOCK_ANSWER_RULES = 256;
|
|
17
|
+
const MAX_MOCK_ANSWER_GLOB_LENGTH = 256;
|
|
18
|
+
const MAX_MOCK_ANSWER_SEQUENCE_LENGTH = 256;
|
|
19
|
+
const MAX_MOCK_ANSWER_DEPTH = 32;
|
|
20
|
+
const MAX_FIXTURE_REASON_LENGTH = 1024;
|
|
21
|
+
function defineDataProperty(target, key, value) {
|
|
22
|
+
Object.defineProperty(target, key, {
|
|
23
|
+
value,
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
function isJsonRecord(value) {
|
|
30
|
+
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
31
|
+
return false;
|
|
32
|
+
const prototype = Object.getPrototypeOf(value);
|
|
33
|
+
return prototype === Object.prototype || prototype === null;
|
|
34
|
+
}
|
|
35
|
+
function validateRecordContainer(value, path) {
|
|
36
|
+
if (!isJsonRecord(value))
|
|
37
|
+
throw new TypeError(`${path} must be an ordinary JSON object`);
|
|
38
|
+
for (const key of Reflect.ownKeys(value)) {
|
|
39
|
+
if (typeof key !== "string")
|
|
40
|
+
throw new TypeError(`${path} must not contain symbol keys`);
|
|
41
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
42
|
+
if (!descriptor || !descriptor.enumerable || !("value" in descriptor)) {
|
|
43
|
+
throw new TypeError(`${path} must contain only enumerable string-keyed data properties`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function validateArrayContainer(value, path) {
|
|
48
|
+
if (!Array.isArray(value))
|
|
49
|
+
throw new TypeError(`${path} must be an array`);
|
|
50
|
+
for (const key of Reflect.ownKeys(value)) {
|
|
51
|
+
if (key === "length")
|
|
52
|
+
continue;
|
|
53
|
+
if (typeof key !== "string" || !/^(0|[1-9]\d*)$/.test(key) || Number(key) >= value.length) {
|
|
54
|
+
throw new TypeError(`${path} arrays must contain only indexed JSON data`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
for (let index = 0; index < value.length; index++) {
|
|
58
|
+
if (!Object.prototype.hasOwnProperty.call(value, index))
|
|
59
|
+
throw new TypeError(`${path} must not contain array holes`);
|
|
60
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
|
|
61
|
+
if (!descriptor || !descriptor.enumerable || !("value" in descriptor)) {
|
|
62
|
+
throw new TypeError(`${path} must contain only enumerable data properties`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function validateJsonGraph(root, rootPath) {
|
|
67
|
+
const active = new Set();
|
|
68
|
+
const stack = [{ value: root, path: rootPath }];
|
|
69
|
+
while (stack.length > 0) {
|
|
70
|
+
const item = stack.pop();
|
|
71
|
+
const value = item.value;
|
|
72
|
+
if (item.exit) {
|
|
73
|
+
active.delete(value);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (value === null || typeof value === "string" || typeof value === "boolean")
|
|
77
|
+
continue;
|
|
78
|
+
if (typeof value === "number") {
|
|
79
|
+
if (!Number.isFinite(value))
|
|
80
|
+
throw new TypeError(`${item.path} must contain only finite JSON numbers`);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (typeof value !== "object")
|
|
84
|
+
throw new TypeError(`${item.path} must contain only JSON data`);
|
|
85
|
+
if (active.has(value))
|
|
86
|
+
throw new TypeError(`${item.path} must not contain cycles`);
|
|
87
|
+
active.add(value);
|
|
88
|
+
stack.push({ value, path: item.path, exit: true });
|
|
89
|
+
if (Array.isArray(value)) {
|
|
90
|
+
validateArrayContainer(value, item.path);
|
|
91
|
+
for (let index = value.length - 1; index >= 0; index--) {
|
|
92
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
|
|
93
|
+
if (!descriptor || !("value" in descriptor))
|
|
94
|
+
throw new TypeError(`${item.path} contains invalid array data`);
|
|
95
|
+
stack.push({ value: descriptor.value, path: `${item.path}[${index}]` });
|
|
96
|
+
}
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
validateRecordContainer(value, item.path);
|
|
100
|
+
const keys = Reflect.ownKeys(value);
|
|
101
|
+
for (let index = keys.length - 1; index >= 0; index--) {
|
|
102
|
+
const key = keys[index];
|
|
103
|
+
if (typeof key !== "string")
|
|
104
|
+
throw new TypeError(`${item.path} contains an invalid symbol key`);
|
|
105
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
106
|
+
if (!descriptor || !("value" in descriptor))
|
|
107
|
+
throw new TypeError(`${item.path} contains an invalid data property`);
|
|
108
|
+
stack.push({ value: descriptor.value, path: `${item.path}.${key}` });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function validateJsonData(value, path, depth, ancestors) {
|
|
113
|
+
if (value === null || typeof value === "string" || typeof value === "boolean")
|
|
114
|
+
return;
|
|
115
|
+
if (typeof value === "number") {
|
|
116
|
+
if (!Number.isFinite(value))
|
|
117
|
+
throw new TypeError(`${path} must contain only finite JSON numbers`);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (typeof value !== "object")
|
|
121
|
+
throw new TypeError(`${path} must contain only JSON data`);
|
|
122
|
+
if (ancestors.has(value))
|
|
123
|
+
throw new TypeError(`${path} must not contain cycles`);
|
|
124
|
+
if (depth > MAX_MOCK_ANSWER_DEPTH) {
|
|
125
|
+
throw new TypeError(`${path} exceeds the maximum answer nesting depth of ${MAX_MOCK_ANSWER_DEPTH}`);
|
|
126
|
+
}
|
|
127
|
+
ancestors.add(value);
|
|
128
|
+
try {
|
|
129
|
+
if (Array.isArray(value)) {
|
|
130
|
+
validateArrayContainer(value, path);
|
|
131
|
+
for (let index = 0; index < value.length; index++) {
|
|
132
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
|
|
133
|
+
if (!descriptor || !("value" in descriptor))
|
|
134
|
+
throw new TypeError(`${path} contains invalid array data`);
|
|
135
|
+
validateJsonData(descriptor.value, `${path}[${index}]`, depth + 1, ancestors);
|
|
136
|
+
}
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
validateRecordContainer(value, path);
|
|
140
|
+
for (const key of Reflect.ownKeys(value)) {
|
|
141
|
+
if (typeof key !== "string")
|
|
142
|
+
throw new TypeError(`${path} contains an invalid symbol key`);
|
|
143
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
144
|
+
if (!descriptor || !("value" in descriptor))
|
|
145
|
+
throw new TypeError(`${path} contains an invalid data property`);
|
|
146
|
+
validateJsonData(descriptor.value, `${path}.${key}`, depth + 1, ancestors);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
finally {
|
|
150
|
+
ancestors.delete(value);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
function cloneJsonData(value) {
|
|
154
|
+
if (value === null || typeof value !== "object")
|
|
155
|
+
return value;
|
|
156
|
+
if (Array.isArray(value))
|
|
157
|
+
return Object.freeze(value.map((item) => cloneJsonData(item)));
|
|
158
|
+
const output = Object.create(Object.getPrototypeOf(value) === null ? null : Object.prototype);
|
|
159
|
+
for (const key of Object.keys(value))
|
|
160
|
+
defineDataProperty(output, key, cloneJsonData(value[key]));
|
|
161
|
+
return Object.freeze(output);
|
|
162
|
+
}
|
|
163
|
+
function isCanonicalArrayIndex(key) {
|
|
164
|
+
if (!/^(0|[1-9]\d*)$/.test(key))
|
|
165
|
+
return false;
|
|
166
|
+
return BigInt(key) <= 4294967294n;
|
|
167
|
+
}
|
|
168
|
+
function compileMockAnswerGlob(glob) {
|
|
169
|
+
if (glob.length === 0)
|
|
170
|
+
throw new TypeError("mock-answer globs must not be empty");
|
|
171
|
+
if (glob.length > MAX_MOCK_ANSWER_GLOB_LENGTH) {
|
|
172
|
+
throw new TypeError(`mock-answer glob ${JSON.stringify(glob)} exceeds ${MAX_MOCK_ANSWER_GLOB_LENGTH} UTF-16 code units`);
|
|
173
|
+
}
|
|
174
|
+
const points = Array.from(glob);
|
|
175
|
+
const tokens = [];
|
|
176
|
+
for (let index = 0; index < points.length; index++) {
|
|
177
|
+
const point = points[index];
|
|
178
|
+
if (point === "\\") {
|
|
179
|
+
const escaped = points[++index];
|
|
180
|
+
if (escaped === undefined)
|
|
181
|
+
throw new TypeError(`mock-answer glob ${JSON.stringify(glob)} has a trailing escape`);
|
|
182
|
+
tokens.push(Object.freeze({ kind: "literal", value: escaped }));
|
|
183
|
+
}
|
|
184
|
+
else if (point === "*") {
|
|
185
|
+
if (tokens.at(-1)?.kind !== "many")
|
|
186
|
+
tokens.push(Object.freeze({ kind: "many" }));
|
|
187
|
+
}
|
|
188
|
+
else if (point === "?") {
|
|
189
|
+
tokens.push(Object.freeze({ kind: "one" }));
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
tokens.push(Object.freeze({ kind: "literal", value: point }));
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return Object.freeze(tokens);
|
|
196
|
+
}
|
|
197
|
+
function matchesMockAnswerGlob(tokens, label) {
|
|
198
|
+
const points = Array.from(label);
|
|
199
|
+
let current = new Array(points.length + 1).fill(false);
|
|
200
|
+
current[0] = true;
|
|
201
|
+
for (const token of tokens) {
|
|
202
|
+
const next = new Array(points.length + 1).fill(false);
|
|
203
|
+
if (token.kind === "many") {
|
|
204
|
+
next[0] = current[0];
|
|
205
|
+
for (let index = 1; index <= points.length; index++)
|
|
206
|
+
next[index] = current[index] || next[index - 1];
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
for (let index = 1; index <= points.length; index++) {
|
|
210
|
+
next[index] = current[index - 1] && (token.kind === "one" || token.value === points[index - 1]);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
current = next;
|
|
214
|
+
}
|
|
215
|
+
return current[points.length];
|
|
216
|
+
}
|
|
217
|
+
function normalizeMockAnswers(value) {
|
|
218
|
+
if (!isJsonRecord(value))
|
|
219
|
+
throw new TypeError("mockAnswers must be an object mapping label globs to answers");
|
|
220
|
+
validateJsonGraph(value, "mockAnswers");
|
|
221
|
+
const globs = Object.keys(value);
|
|
222
|
+
if (globs.length > MAX_MOCK_ANSWER_RULES) {
|
|
223
|
+
throw new TypeError(`mockAnswers supports at most ${MAX_MOCK_ANSWER_RULES} rules`);
|
|
224
|
+
}
|
|
225
|
+
const rules = globs.map((glob) => {
|
|
226
|
+
if (isCanonicalArrayIndex(glob)) {
|
|
227
|
+
throw new TypeError(`mock-answer glob ${JSON.stringify(glob)} is a reserved canonical array-index key; escape a digit to match a numeric label`);
|
|
228
|
+
}
|
|
229
|
+
const tokens = compileMockAnswerGlob(glob);
|
|
230
|
+
const rawRule = value[glob];
|
|
231
|
+
if (isJsonRecord(rawRule) && Object.prototype.hasOwnProperty.call(rawRule, "$sequence")) {
|
|
232
|
+
validateRecordContainer(rawRule, `mockAnswers.${glob}`);
|
|
233
|
+
const keys = Object.keys(rawRule);
|
|
234
|
+
if (keys.length !== 1) {
|
|
235
|
+
throw new TypeError(`mock-answer sequence ${JSON.stringify(glob)} must contain only the top-level $sequence property`);
|
|
236
|
+
}
|
|
237
|
+
const sequence = rawRule.$sequence;
|
|
238
|
+
if (!Array.isArray(sequence) || sequence.length === 0) {
|
|
239
|
+
throw new TypeError(`mock-answer sequence ${JSON.stringify(glob)} must be a non-empty array`);
|
|
240
|
+
}
|
|
241
|
+
validateArrayContainer(sequence, `mockAnswers.${glob}.$sequence`);
|
|
242
|
+
if (sequence.length > MAX_MOCK_ANSWER_SEQUENCE_LENGTH) {
|
|
243
|
+
throw new TypeError(`mock-answer sequence ${JSON.stringify(glob)} supports at most ${MAX_MOCK_ANSWER_SEQUENCE_LENGTH} entries`);
|
|
244
|
+
}
|
|
245
|
+
sequence.forEach((answer, index) => validateJsonData(answer, `mockAnswers.${glob}.$sequence[${index}]`, 1, new Set()));
|
|
246
|
+
return Object.freeze({
|
|
247
|
+
glob,
|
|
248
|
+
tokens,
|
|
249
|
+
kind: "sequence",
|
|
250
|
+
answers: Object.freeze(sequence.map((answer) => cloneJsonData(answer))),
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
validateJsonData(rawRule, `mockAnswers.${glob}`, 1, new Set());
|
|
254
|
+
return Object.freeze({
|
|
255
|
+
glob,
|
|
256
|
+
tokens,
|
|
257
|
+
kind: "single",
|
|
258
|
+
answers: Object.freeze([cloneJsonData(rawRule)]),
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
const canonical = JSON.stringify(value);
|
|
262
|
+
if (Buffer.byteLength(canonical, "utf8") > MAX_MOCK_ANSWERS_BYTES) {
|
|
263
|
+
throw new TypeError(`mockAnswers exceeds the maximum canonical JSON size of ${MAX_MOCK_ANSWERS_BYTES} bytes`);
|
|
264
|
+
}
|
|
265
|
+
return {
|
|
266
|
+
rules: Object.freeze(rules),
|
|
267
|
+
counters: rules.map(() => ({ matchingCalls: 0, consumedCalls: 0 })),
|
|
268
|
+
inheritedWarnings: new Map(),
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
function reserveMockAnswer(state, label) {
|
|
272
|
+
let winningIndex = -1;
|
|
273
|
+
for (let index = 0; index < state.rules.length; index++) {
|
|
274
|
+
if (matchesMockAnswerGlob(state.rules[index].tokens, label)) {
|
|
275
|
+
state.counters[index].matchingCalls++;
|
|
276
|
+
winningIndex = index;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
if (winningIndex < 0)
|
|
280
|
+
return undefined;
|
|
281
|
+
const rule = state.rules[winningIndex];
|
|
282
|
+
const counter = state.counters[winningIndex];
|
|
283
|
+
if (rule.kind === "sequence" && counter.consumedCalls >= rule.answers.length) {
|
|
284
|
+
const message = redactText(`Mock answer sequence exhausted for agent ${JSON.stringify(label)} using glob ${JSON.stringify(rule.glob)}: ` +
|
|
285
|
+
`sequence length ${rule.answers.length}, ${counter.consumedCalls} already consumed.`).value;
|
|
286
|
+
throw new WorkflowError(truncate(message, MAX_FIXTURE_REASON_LENGTH), WorkflowErrorCode.SCRIPT_VALIDATION_ERROR, {
|
|
287
|
+
recoverable: false,
|
|
288
|
+
agentLabel: label,
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
const sequenceIndex = rule.kind === "sequence" ? counter.consumedCalls : undefined;
|
|
292
|
+
const answer = rule.answers[sequenceIndex ?? 0];
|
|
293
|
+
counter.consumedCalls++;
|
|
294
|
+
return {
|
|
295
|
+
ruleIndex: winningIndex,
|
|
296
|
+
rule,
|
|
297
|
+
answer,
|
|
298
|
+
use: {
|
|
299
|
+
glob: rule.glob,
|
|
300
|
+
...(sequenceIndex === undefined
|
|
301
|
+
? {}
|
|
302
|
+
: { sequenceIndex, sequenceLength: rule.answers.length }),
|
|
303
|
+
},
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
function cloneMergeValue(value) {
|
|
307
|
+
if (value === null || typeof value !== "object")
|
|
308
|
+
return value;
|
|
309
|
+
if (Array.isArray(value))
|
|
310
|
+
return value.map((item) => cloneMergeValue(item));
|
|
311
|
+
const output = Object.create(Object.getPrototypeOf(value) === null ? null : Object.prototype);
|
|
312
|
+
for (const key of Object.keys(value))
|
|
313
|
+
defineDataProperty(output, key, cloneMergeValue(value[key]));
|
|
314
|
+
return output;
|
|
315
|
+
}
|
|
316
|
+
function mergeMockAnswer(base, override, path, replacedPaths) {
|
|
317
|
+
if (isJsonRecord(base) && isJsonRecord(override)) {
|
|
318
|
+
const output = Object.create(Object.getPrototypeOf(base) === null ? null : Object.prototype);
|
|
319
|
+
for (const key of Object.keys(base))
|
|
320
|
+
defineDataProperty(output, key, cloneMergeValue(base[key]));
|
|
321
|
+
for (const key of Object.keys(override)) {
|
|
322
|
+
if (Object.prototype.hasOwnProperty.call(base, key)) {
|
|
323
|
+
defineDataProperty(output, key, mergeMockAnswer(base[key], override[key], [...path, key], replacedPaths));
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
replacedPaths.push([...path, key]);
|
|
327
|
+
defineDataProperty(output, key, cloneMergeValue(override[key]));
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return output;
|
|
331
|
+
}
|
|
332
|
+
replacedPaths.push(path);
|
|
333
|
+
return cloneMergeValue(override);
|
|
334
|
+
}
|
|
335
|
+
function parseJsonPointer(pointer) {
|
|
336
|
+
if (pointer === "")
|
|
337
|
+
return [];
|
|
338
|
+
return pointer
|
|
339
|
+
.slice(1)
|
|
340
|
+
.split("/")
|
|
341
|
+
.map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~"));
|
|
342
|
+
}
|
|
343
|
+
function renderJsonPointer(tokens) {
|
|
344
|
+
if (tokens.length === 0)
|
|
345
|
+
return "/";
|
|
346
|
+
return `/${tokens.map((token) => token.replace(/~/g, "~0").replace(/\//g, "~1")).join("/")}`;
|
|
347
|
+
}
|
|
348
|
+
function relatedPaths(left, right) {
|
|
349
|
+
const length = Math.min(left.length, right.length);
|
|
350
|
+
for (let index = 0; index < length; index++) {
|
|
351
|
+
if (left[index] !== right[index])
|
|
352
|
+
return false;
|
|
353
|
+
}
|
|
354
|
+
return true;
|
|
355
|
+
}
|
|
356
|
+
function normalizedSchemaErrors(schema, value) {
|
|
357
|
+
return Errors(schema, value).map((error) => {
|
|
358
|
+
const tokens = parseJsonPointer(error.instancePath);
|
|
359
|
+
return { path: error.instancePath, tokens, message: error.message };
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
function schemaFailure(label, reservation, errors) {
|
|
363
|
+
const position = reservation.use.sequenceIndex === undefined
|
|
364
|
+
? ""
|
|
365
|
+
: ` sequence ${reservation.use.sequenceIndex + 1}/${reservation.use.sequenceLength}`;
|
|
366
|
+
const diagnostics = errors.length === 0
|
|
367
|
+
? "schema validation could not compare the fabricated base and scripted answer"
|
|
368
|
+
: errors
|
|
369
|
+
.slice(0, 3)
|
|
370
|
+
.map((error) => `${renderJsonPointer(error.tokens)} ${error.message}`)
|
|
371
|
+
.join("; ");
|
|
372
|
+
const message = redactText(`Mock answer for agent ${JSON.stringify(label)} from glob ${JSON.stringify(reservation.rule.glob)}${position} ` +
|
|
373
|
+
`failed schema validation: ${diagnostics}`).value;
|
|
374
|
+
return new WorkflowError(truncate(message, MAX_FIXTURE_REASON_LENGTH), WorkflowErrorCode.SCHEMA_NONCOMPLIANCE, {
|
|
375
|
+
recoverable: false,
|
|
376
|
+
agentLabel: label,
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
function applyStructuredMockAnswer(schema, base, reservation, label, state) {
|
|
380
|
+
const replacedPaths = [];
|
|
381
|
+
const merged = mergeMockAnswer(base, reservation.answer, [], replacedPaths);
|
|
382
|
+
try {
|
|
383
|
+
if (Check(schema, merged))
|
|
384
|
+
return merged;
|
|
385
|
+
Check(schema, base);
|
|
386
|
+
const mergedErrors = normalizedSchemaErrors(schema, merged);
|
|
387
|
+
const baseErrors = normalizedSchemaErrors(schema, base);
|
|
388
|
+
const baseFingerprints = new Set(baseErrors.map((error) => `${error.path}\u0000${error.message}`));
|
|
389
|
+
const introduced = mergedErrors.filter((error) => !baseFingerprints.has(`${error.path}\u0000${error.message}`) ||
|
|
390
|
+
replacedPaths.some((replaced) => relatedPaths(error.tokens, replaced)));
|
|
391
|
+
if (introduced.length > 0)
|
|
392
|
+
throw schemaFailure(label, reservation, introduced);
|
|
393
|
+
const paths = [...new Set(mergedErrors.map((error) => renderJsonPointer(error.tokens)))];
|
|
394
|
+
const warningKey = `${reservation.ruleIndex}\u0000${label}\u0000${paths.join("\u0000")}`;
|
|
395
|
+
const previous = state.inheritedWarnings.get(warningKey);
|
|
396
|
+
if (previous)
|
|
397
|
+
previous.count++;
|
|
398
|
+
else
|
|
399
|
+
state.inheritedWarnings.set(warningKey, { ruleIndex: reservation.ruleIndex, label, paths, count: 1 });
|
|
400
|
+
return merged;
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
if (error instanceof WorkflowError)
|
|
404
|
+
throw error;
|
|
405
|
+
throw schemaFailure(label, reservation, []);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
function applyTextMockAnswer(reservation, label) {
|
|
409
|
+
if (typeof reservation.answer === "string" && reservation.answer.trim().length > 0)
|
|
410
|
+
return reservation.answer;
|
|
411
|
+
throw schemaFailure(label, reservation, [
|
|
412
|
+
{ path: "", tokens: [], message: "Expected a non-blank string for a schema-less agent call" },
|
|
413
|
+
]);
|
|
414
|
+
}
|
|
415
|
+
function buildMockAnswersReport(state) {
|
|
416
|
+
const rules = state.rules.map((rule, index) => ({
|
|
417
|
+
glob: rule.glob,
|
|
418
|
+
kind: rule.kind,
|
|
419
|
+
matchingCalls: state.counters[index].matchingCalls,
|
|
420
|
+
consumedCalls: state.counters[index].consumedCalls,
|
|
421
|
+
...(rule.kind === "sequence" ? { sequenceLength: rule.answers.length } : {}),
|
|
422
|
+
}));
|
|
423
|
+
const unused = [];
|
|
424
|
+
for (let ruleIndex = 0; ruleIndex < state.rules.length; ruleIndex++) {
|
|
425
|
+
const rule = state.rules[ruleIndex];
|
|
426
|
+
const counter = state.counters[ruleIndex];
|
|
427
|
+
if (rule.kind === "single") {
|
|
428
|
+
if (counter.consumedCalls === 0) {
|
|
429
|
+
unused.push({
|
|
430
|
+
glob: rule.glob,
|
|
431
|
+
reason: counter.matchingCalls === 0 ? "no-match" : "shadowed",
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
for (let sequenceIndex = counter.consumedCalls; sequenceIndex < rule.answers.length; sequenceIndex++) {
|
|
437
|
+
unused.push({
|
|
438
|
+
glob: rule.glob,
|
|
439
|
+
sequenceIndex,
|
|
440
|
+
reason: counter.matchingCalls === 0
|
|
441
|
+
? "no-match"
|
|
442
|
+
: counter.consumedCalls === 0
|
|
443
|
+
? "shadowed"
|
|
444
|
+
: "not-reached",
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
return { rules, unused };
|
|
449
|
+
}
|
|
450
|
+
function appendMockAnswerWarnings(state, report, warnings) {
|
|
451
|
+
for (let ruleIndex = 0; ruleIndex < state.rules.length; ruleIndex++) {
|
|
452
|
+
for (const incident of state.inheritedWarnings.values()) {
|
|
453
|
+
if (incident.ruleIndex !== ruleIndex)
|
|
454
|
+
continue;
|
|
455
|
+
const shownPaths = incident.paths.slice(0, 3);
|
|
456
|
+
const more = incident.paths.length > shownPaths.length ? ` (+${incident.paths.length - shownPaths.length} more)` : "";
|
|
457
|
+
warnings.push(`mock-answer rule ${JSON.stringify(state.rules[ruleIndex].glob)} for agent ${JSON.stringify(incident.label)} ` +
|
|
458
|
+
`was accepted with pre-existing fabricated-default limitations at ${shownPaths.join(", ")}${more} ` +
|
|
459
|
+
`(${incident.count} occurrence${incident.count === 1 ? "" : "s"})`);
|
|
460
|
+
}
|
|
461
|
+
const unused = report.unused.filter((entry) => entry.glob === state.rules[ruleIndex].glob);
|
|
462
|
+
if (unused.length > 0) {
|
|
463
|
+
const reasons = [...new Set(unused.map((entry) => entry.reason))].join(", ");
|
|
464
|
+
warnings.push(`mock-answer rule ${JSON.stringify(state.rules[ruleIndex].glob)} has ${unused.length} unused answer` +
|
|
465
|
+
`${unused.length === 1 ? "" : "s"} (${reasons})`);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
}
|
|
14
469
|
/**
|
|
15
470
|
* Fabricate a value that structurally satisfies a JSON Schema — the dry run's stand-in
|
|
16
471
|
* for a real agent's structured output. Deterministic and intentionally simple: first
|
|
@@ -46,11 +501,12 @@ export function fabricateFromSchema(schema, hint = "value", depth = 0) {
|
|
|
46
501
|
case "object": {
|
|
47
502
|
const out = {};
|
|
48
503
|
const props = (s.properties ?? {});
|
|
49
|
-
for (const [name, sub] of Object.entries(props))
|
|
50
|
-
out
|
|
504
|
+
for (const [name, sub] of Object.entries(props)) {
|
|
505
|
+
defineDataProperty(out, name, fabricateFromSchema(sub, name, depth + 1));
|
|
506
|
+
}
|
|
51
507
|
for (const name of Array.isArray(s.required) ? s.required : []) {
|
|
52
|
-
if (!(name
|
|
53
|
-
out
|
|
508
|
+
if (!Object.prototype.hasOwnProperty.call(out, name))
|
|
509
|
+
defineDataProperty(out, name, `mock-${name}`);
|
|
54
510
|
}
|
|
55
511
|
return out;
|
|
56
512
|
}
|
|
@@ -115,6 +571,7 @@ function attributeBackend(model, tier, declared) {
|
|
|
115
571
|
* AgentRunner. Never throws for an invalid script — read `report.ok` / `report.exitCode`.
|
|
116
572
|
*/
|
|
117
573
|
export async function validateWorkflowScript(script, options = {}) {
|
|
574
|
+
const mockAnswerState = options.mockAnswers === undefined ? undefined : normalizeMockAnswers(options.mockAnswers);
|
|
118
575
|
const warnings = [];
|
|
119
576
|
let meta;
|
|
120
577
|
try {
|
|
@@ -141,15 +598,30 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
141
598
|
const ownedCwd = options.cwd === undefined;
|
|
142
599
|
const baseCwd = options.cwd ?? mkdtempSync(join(tmpdir(), "agentprism-validate-"));
|
|
143
600
|
const persistenceRoot = mkdtempSync(join(tmpdir(), "agentprism-validate-state-"));
|
|
601
|
+
const agentCalls = [];
|
|
602
|
+
const pendingAgentCalls = [];
|
|
603
|
+
const checkpoints = [];
|
|
144
604
|
const mockMeta = new Map();
|
|
145
605
|
const runner = {
|
|
146
606
|
async run(_prompt, runOptions = {}) {
|
|
147
607
|
const label = runOptions.label ?? "";
|
|
148
|
-
|
|
608
|
+
const metadata = {
|
|
149
609
|
tier: runOptions.tier,
|
|
150
610
|
mode: runOptions.mode,
|
|
151
611
|
schema: runOptions.schema !== undefined,
|
|
152
|
-
}
|
|
612
|
+
};
|
|
613
|
+
mockMeta.set(label, metadata);
|
|
614
|
+
const pendingCall = mockAnswerState ? pendingAgentCalls.shift() : undefined;
|
|
615
|
+
if (pendingCall) {
|
|
616
|
+
pendingCall.tier = metadata.tier;
|
|
617
|
+
pendingCall.mode = metadata.mode;
|
|
618
|
+
pendingCall.schema = metadata.schema;
|
|
619
|
+
pendingCall.backend = attributeBackend(pendingCall.model, metadata.tier, declaredBackends);
|
|
620
|
+
}
|
|
621
|
+
const base = runOptions.schema === undefined ? undefined : fabricateFromSchema(runOptions.schema);
|
|
622
|
+
const reservation = mockAnswerState ? reserveMockAnswer(mockAnswerState, label) : undefined;
|
|
623
|
+
if (reservation && pendingCall)
|
|
624
|
+
pendingCall.mockAnswer = reservation.use;
|
|
153
625
|
runOptions.onUsage?.({
|
|
154
626
|
input: MOCK_TOKENS_PER_AGENT - 250,
|
|
155
627
|
output: 250,
|
|
@@ -158,13 +630,16 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
158
630
|
total: MOCK_TOKENS_PER_AGENT,
|
|
159
631
|
cost: 0,
|
|
160
632
|
});
|
|
161
|
-
if (runOptions.schema !== undefined)
|
|
162
|
-
|
|
633
|
+
if (runOptions.schema !== undefined) {
|
|
634
|
+
if (!reservation)
|
|
635
|
+
return base;
|
|
636
|
+
return applyStructuredMockAnswer(runOptions.schema, base, reservation, label, mockAnswerState);
|
|
637
|
+
}
|
|
638
|
+
if (reservation)
|
|
639
|
+
return applyTextMockAnswer(reservation, label);
|
|
163
640
|
return `[dry-run] mock output for ${runOptions.label ?? "agent"}`;
|
|
164
641
|
},
|
|
165
642
|
};
|
|
166
|
-
const agentCalls = [];
|
|
167
|
-
const checkpoints = [];
|
|
168
643
|
const controller = new AbortController();
|
|
169
644
|
let timedOut = false;
|
|
170
645
|
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
@@ -181,13 +656,14 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
181
656
|
const manager = new WorkflowManager({
|
|
182
657
|
agent: runner,
|
|
183
658
|
cwd: baseCwd,
|
|
659
|
+
...(mockAnswerState ? { concurrency: 1 } : {}),
|
|
184
660
|
journaling: false,
|
|
185
661
|
persistenceRoot,
|
|
186
662
|
loadSavedWorkflow: flows?.resolve,
|
|
187
663
|
});
|
|
188
664
|
manager.on("agentStart", (event) => {
|
|
189
665
|
const extra = mockMeta.get(event.label) ?? mockMeta.get("") ?? { schema: false };
|
|
190
|
-
|
|
666
|
+
const call = {
|
|
191
667
|
label: event.label,
|
|
192
668
|
phase: event.phase,
|
|
193
669
|
model: event.model,
|
|
@@ -195,7 +671,10 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
195
671
|
mode: extra.mode,
|
|
196
672
|
backend: attributeBackend(event.model, extra.tier, declaredBackends),
|
|
197
673
|
schema: extra.schema,
|
|
198
|
-
}
|
|
674
|
+
};
|
|
675
|
+
agentCalls.push(call);
|
|
676
|
+
if (mockAnswerState)
|
|
677
|
+
pendingAgentCalls.push(call);
|
|
199
678
|
});
|
|
200
679
|
try {
|
|
201
680
|
const run = await manager.runSync(script, options.args, {
|
|
@@ -217,13 +696,15 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
217
696
|
});
|
|
218
697
|
// agentStart fires BEFORE the mock records its options, so backfill attribution for
|
|
219
698
|
// any call whose mock metadata arrived after the event (same tick ordering).
|
|
220
|
-
|
|
221
|
-
const
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
699
|
+
if (!mockAnswerState) {
|
|
700
|
+
for (const call of agentCalls) {
|
|
701
|
+
const extra = mockMeta.get(call.label);
|
|
702
|
+
if (extra) {
|
|
703
|
+
call.tier = extra.tier;
|
|
704
|
+
call.mode = extra.mode;
|
|
705
|
+
call.schema = extra.schema;
|
|
706
|
+
call.backend = attributeBackend(call.model, extra.tier, declaredBackends);
|
|
707
|
+
}
|
|
227
708
|
}
|
|
228
709
|
}
|
|
229
710
|
const ok = run.status === "completed";
|
|
@@ -249,6 +730,9 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
249
730
|
}
|
|
250
731
|
}
|
|
251
732
|
}
|
|
733
|
+
const mockAnswers = mockAnswerState ? buildMockAnswersReport(mockAnswerState) : undefined;
|
|
734
|
+
if (mockAnswerState && mockAnswers)
|
|
735
|
+
appendMockAnswerWarnings(mockAnswerState, mockAnswers, warnings);
|
|
252
736
|
return {
|
|
253
737
|
ok,
|
|
254
738
|
exitCode: ok ? 0 : 2,
|
|
@@ -264,6 +748,7 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
264
748
|
logs: run.logs ?? [],
|
|
265
749
|
durationMs: run.durationMs,
|
|
266
750
|
result: run.result,
|
|
751
|
+
...(mockAnswers ? { mockAnswers } : {}),
|
|
267
752
|
},
|
|
268
753
|
warnings,
|
|
269
754
|
};
|
|
@@ -307,7 +792,13 @@ export function formatValidateReport(report) {
|
|
|
307
792
|
lines.push(dry.ok ? `✓ dry run completed — ${summary}` : `✗ dry run ${dry.status} — ${dry.reason ?? "unknown failure"} (${summary})`);
|
|
308
793
|
for (const call of dry.agentCalls) {
|
|
309
794
|
const spec = call.model ?? (call.tier ? `tier=${call.tier}` : "(default model)");
|
|
310
|
-
const
|
|
795
|
+
const mock = call.mockAnswer
|
|
796
|
+
? `mock=${JSON.stringify(call.mockAnswer.glob)}` +
|
|
797
|
+
(call.mockAnswer.sequenceIndex === undefined
|
|
798
|
+
? ""
|
|
799
|
+
: `[${call.mockAnswer.sequenceIndex + 1}/${call.mockAnswer.sequenceLength}]`)
|
|
800
|
+
: undefined;
|
|
801
|
+
const bits = [call.phase ? `[${call.phase}]` : undefined, spec, `→ ${call.backend}`, call.schema ? "(schema)" : undefined, call.mode ? `mode=${call.mode}` : undefined, mock]
|
|
311
802
|
.filter(Boolean)
|
|
312
803
|
.join(" ");
|
|
313
804
|
lines.push(` • ${call.label} ${bits}`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@automatalabs/workflows",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.29.0",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22"
|
|
@@ -30,12 +30,10 @@
|
|
|
30
30
|
"access": "public"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"
|
|
33
|
+
"typebox": "1.3.2",
|
|
34
|
+
"@automatalabs/workflow-engine": "0.18.0",
|
|
34
35
|
"@automatalabs/acp-agents": "0.24.8",
|
|
35
|
-
"@automatalabs/
|
|
36
|
-
},
|
|
37
|
-
"devDependencies": {
|
|
38
|
-
"typebox": "1.3.2"
|
|
36
|
+
"@automatalabs/shared-types": "0.17.0"
|
|
39
37
|
},
|
|
40
38
|
"scripts": {
|
|
41
39
|
"build": "tsc -b",
|