@restormel/testing-runner 0.1.0 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/ac-agent-loop.d.ts +21 -0
- package/dist/ac-agent-loop.d.ts.map +1 -0
- package/dist/ac-agent-loop.js +174 -0
- package/dist/ac-agent-loop.js.map +1 -0
- package/dist/ac-judge.d.ts +12 -0
- package/dist/ac-judge.d.ts.map +1 -0
- package/dist/ac-judge.js +116 -0
- package/dist/ac-judge.js.map +1 -0
- package/dist/ac-llm.d.ts +19 -0
- package/dist/ac-llm.d.ts.map +1 -0
- package/dist/ac-llm.js +47 -0
- package/dist/ac-llm.js.map +1 -0
- package/dist/ac-post-checks.d.ts +18 -0
- package/dist/ac-post-checks.d.ts.map +1 -0
- package/dist/ac-post-checks.js +101 -0
- package/dist/ac-post-checks.js.map +1 -0
- package/dist/browser-goal.d.ts +2 -0
- package/dist/browser-goal.d.ts.map +1 -1
- package/dist/browser-goal.js +20 -5
- package/dist/browser-goal.js.map +1 -1
- package/dist/evaluate-criteria.d.ts.map +1 -1
- package/dist/evaluate-criteria.js +159 -19
- package/dist/evaluate-criteria.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/lighthouse-structured-check.d.ts +28 -0
- package/dist/lighthouse-structured-check.d.ts.map +1 -0
- package/dist/lighthouse-structured-check.js +179 -0
- package/dist/lighthouse-structured-check.js.map +1 -0
- package/dist/post-mission-goal.d.ts +7 -0
- package/dist/post-mission-goal.d.ts.map +1 -0
- package/dist/post-mission-goal.js +23 -0
- package/dist/post-mission-goal.js.map +1 -0
- package/dist/run-ac-sequence-goal.d.ts +13 -0
- package/dist/run-ac-sequence-goal.d.ts.map +1 -0
- package/dist/run-ac-sequence-goal.js +430 -0
- package/dist/run-ac-sequence-goal.js.map +1 -0
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +306 -33
- package/dist/run-suite.js.map +1 -1
- package/dist/shell-hooks.d.ts +31 -0
- package/dist/shell-hooks.d.ts.map +1 -0
- package/dist/shell-hooks.js +93 -0
- package/dist/shell-hooks.js.map +1 -0
- package/dist/storage-state.d.ts.map +1 -1
- package/dist/storage-state.js +15 -4
- package/dist/storage-state.js.map +1 -1
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -9
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* After `mission_executor` exits 0, browser evaluation uses this goal: agent-only fields removed,
|
|
3
|
+
* navigation path is `after_agent.start_path` when set, otherwise the goal's top-level `start_path`.
|
|
4
|
+
*/
|
|
5
|
+
export function toPostMissionObserveGoal(goal) {
|
|
6
|
+
const startPath = goal.afterAgent?.startPath ?? goal.startPath;
|
|
7
|
+
return {
|
|
8
|
+
id: goal.id,
|
|
9
|
+
type: goal.type,
|
|
10
|
+
description: goal.description,
|
|
11
|
+
successCriteria: goal.successCriteria,
|
|
12
|
+
startPath,
|
|
13
|
+
executionMode: "observe",
|
|
14
|
+
preconditions: goal.preconditions,
|
|
15
|
+
cleanup: goal.cleanup,
|
|
16
|
+
exclusiveWith: goal.exclusiveWith,
|
|
17
|
+
tags: goal.tags,
|
|
18
|
+
...(goal.acceptanceCriterionIds !== undefined && goal.acceptanceCriterionIds.length > 0
|
|
19
|
+
? { acceptanceCriterionIds: [...goal.acceptanceCriterionIds] }
|
|
20
|
+
: {}),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=post-mission-goal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"post-mission-goal.js","sourceRoot":"","sources":["../src/post-mission-goal.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAc;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;IAC/D,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,SAAS;QACT,aAAa,EAAE,SAAS;QACxB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,GAAG,CAAC,IAAI,CAAC,sBAAsB,KAAK,SAAS,IAAI,IAAI,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC;YACrF,CAAC,CAAC,EAAE,sBAAsB,EAAE,CAAC,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE;YAC9D,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { AcceptanceCriterionDefinition } from "@restormel/testing-core";
|
|
2
|
+
import { type RunBrowserGoalOptions, type RunBrowserGoalResult } from "./browser-goal.js";
|
|
3
|
+
export type RunAcSequenceBrowserGoalOptions = RunBrowserGoalOptions & {
|
|
4
|
+
suiteUserStory?: string;
|
|
5
|
+
acceptanceCriteria: AcceptanceCriterionDefinition[];
|
|
6
|
+
hookCwd: string;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* `execution_mode: ac_sequence` — walk acceptance criteria with the built-in LLM browser agent (R-BA-4),
|
|
10
|
+
* optional per-AC judge JSON with `ac_id` (R-BA-5), and HTTP / DOM / shell post-checks (R-BA-6).
|
|
11
|
+
*/
|
|
12
|
+
export declare function runAcSequenceBrowserGoal(options: RunAcSequenceBrowserGoalOptions): Promise<RunBrowserGoalResult>;
|
|
13
|
+
//# sourceMappingURL=run-ac-sequence-goal.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-ac-sequence-goal.d.ts","sourceRoot":"","sources":["../src/run-ac-sequence-goal.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,6BAA6B,EAO9B,MAAM,yBAAyB,CAAC;AAUjC,OAAO,EAAgB,KAAK,qBAAqB,EAAE,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAsExG,MAAM,MAAM,+BAA+B,GAAG,qBAAqB,GAAG;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,6BAA6B,EAAE,CAAC;IACpD,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;GAGG;AACH,wBAAsB,wBAAwB,CAAC,OAAO,EAAE,+BAA+B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAoYtH"}
|
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { browserTracesToCoreEvents, createPlaywrightTestingSession } from "@restormel/testing-browser-playwright";
|
|
5
|
+
import { sanitizePathSegment } from "@restormel/testing-core";
|
|
6
|
+
import { resolveModel } from "@restormel/testing-keys-adapter";
|
|
7
|
+
import { runBuiltInAcAgentLoop } from "./ac-agent-loop.js";
|
|
8
|
+
import { runAcShapedJudgeRubric } from "./ac-judge.js";
|
|
9
|
+
import { runAcPostCheck } from "./ac-post-checks.js";
|
|
10
|
+
import { evaluateBrowserSuccessCriteria } from "./evaluate-criteria.js";
|
|
11
|
+
import { judgeLogicalRefForCriteria } from "./judge-ref.js";
|
|
12
|
+
import { goalEntryUrl } from "./browser-goal.js";
|
|
13
|
+
import { runMissionExecutorCommand } from "./shell-hooks.js";
|
|
14
|
+
import { runGoalAttempts } from "./retries.js";
|
|
15
|
+
import { TimeoutError, withTimeout } from "./timeout.js";
|
|
16
|
+
function nowIso() {
|
|
17
|
+
return new Date().toISOString();
|
|
18
|
+
}
|
|
19
|
+
function worstVerdict(a, b) {
|
|
20
|
+
const rank = { passed: 0, indeterminate: 1, failed: 2 };
|
|
21
|
+
return rank[a] >= rank[b] ? a : b;
|
|
22
|
+
}
|
|
23
|
+
function aggregateStepVerdicts(steps) {
|
|
24
|
+
let v = "passed";
|
|
25
|
+
for (const s of steps) {
|
|
26
|
+
v = worstVerdict(v, s.verdict);
|
|
27
|
+
}
|
|
28
|
+
return v;
|
|
29
|
+
}
|
|
30
|
+
function toKeysModelMeta(model, invocations) {
|
|
31
|
+
return {
|
|
32
|
+
logicalRef: model.meta.logicalRef,
|
|
33
|
+
provider: model.meta.provider,
|
|
34
|
+
model: model.meta.model,
|
|
35
|
+
resolutionSource: model.meta.resolutionSource,
|
|
36
|
+
invocationCount: invocations,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function refForAgent(goal, resolvedKeys) {
|
|
40
|
+
const mr = goal.acSequence?.builtInAgent.modelRef?.trim();
|
|
41
|
+
if (mr)
|
|
42
|
+
return mr;
|
|
43
|
+
return (resolvedKeys.llm_primary ??
|
|
44
|
+
resolvedKeys.llmPrimary ??
|
|
45
|
+
resolvedKeys.judge ??
|
|
46
|
+
resolvedKeys.llm_judge ??
|
|
47
|
+
resolvedKeys.llmJudge);
|
|
48
|
+
}
|
|
49
|
+
async function captureStepScreenshot(session, artifactDir, goalId, acId, attemptIndex) {
|
|
50
|
+
const safeGoal = sanitizePathSegment(goalId);
|
|
51
|
+
const safeAc = sanitizePathSegment(acId) || "ac";
|
|
52
|
+
const dir = join(artifactDir, "goals", safeGoal, "ac-sequence");
|
|
53
|
+
await mkdir(dir, { recursive: true });
|
|
54
|
+
const name = `attempt-${attemptIndex}-${safeAc}.png`;
|
|
55
|
+
const abs = join(dir, name);
|
|
56
|
+
try {
|
|
57
|
+
await session.screenshot(abs);
|
|
58
|
+
return join("goals", safeGoal, "ac-sequence", name);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function acIdsFragment(goal) {
|
|
65
|
+
if (goal.acceptanceCriterionIds === undefined || goal.acceptanceCriterionIds.length === 0)
|
|
66
|
+
return {};
|
|
67
|
+
return { acceptanceCriterionIds: [...goal.acceptanceCriterionIds] };
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* `execution_mode: ac_sequence` — walk acceptance criteria with the built-in LLM browser agent (R-BA-4),
|
|
71
|
+
* optional per-AC judge JSON with `ac_id` (R-BA-5), and HTTP / DOM / shell post-checks (R-BA-6).
|
|
72
|
+
*/
|
|
73
|
+
export async function runAcSequenceBrowserGoal(options) {
|
|
74
|
+
const { goal, runId, acceptanceCriteria } = options;
|
|
75
|
+
const cfg = goal.acSequence;
|
|
76
|
+
if (!cfg) {
|
|
77
|
+
throw new Error("runAcSequenceBrowserGoal: goal.acSequence missing");
|
|
78
|
+
}
|
|
79
|
+
const warnings = [];
|
|
80
|
+
const keysModelMetaFragments = [];
|
|
81
|
+
const allTraces = [];
|
|
82
|
+
let stepCursor = options.startingStepIndex;
|
|
83
|
+
const pushTrace = (partial) => {
|
|
84
|
+
allTraces.push({
|
|
85
|
+
id: randomUUID(),
|
|
86
|
+
runId,
|
|
87
|
+
goalId: goal.id,
|
|
88
|
+
stepIndex: stepCursor++,
|
|
89
|
+
timestamp: nowIso(),
|
|
90
|
+
...partial,
|
|
91
|
+
});
|
|
92
|
+
};
|
|
93
|
+
const agentRef = refForAgent(goal, options.resolvedKeys);
|
|
94
|
+
if (!agentRef) {
|
|
95
|
+
const gr = {
|
|
96
|
+
goalId: goal.id,
|
|
97
|
+
verdict: "failed",
|
|
98
|
+
reasonCode: "AC_AGENT_NO_MODEL_REF",
|
|
99
|
+
summary: "ac_sequence requires built_in_agent.model_ref or llm_primary / judge in environment keys",
|
|
100
|
+
retriesUsed: 0,
|
|
101
|
+
evidenceRefs: [],
|
|
102
|
+
...acIdsFragment(goal),
|
|
103
|
+
};
|
|
104
|
+
return { goalRecord: gr, traces: allTraces, nextStepIndex: stepCursor, warnings, keysModelMetaFragments };
|
|
105
|
+
}
|
|
106
|
+
const agentResolve = await resolveModel(agentRef, options.keysAdapterOptions ?? {});
|
|
107
|
+
if (!agentResolve.ok) {
|
|
108
|
+
const gr = {
|
|
109
|
+
goalId: goal.id,
|
|
110
|
+
verdict: "failed",
|
|
111
|
+
reasonCode: "AC_AGENT_MODEL_RESOLVE_FAILED",
|
|
112
|
+
summary: agentResolve.error.message,
|
|
113
|
+
retriesUsed: 0,
|
|
114
|
+
evidenceRefs: [],
|
|
115
|
+
...acIdsFragment(goal),
|
|
116
|
+
};
|
|
117
|
+
warnings.push(`Goal "${goal.id}": AC agent model resolution failed (${agentResolve.error.code})`);
|
|
118
|
+
return { goalRecord: gr, traces: allTraces, nextStepIndex: stepCursor, warnings, keysModelMetaFragments };
|
|
119
|
+
}
|
|
120
|
+
const agentModel = agentResolve.model;
|
|
121
|
+
warnings.push(...agentResolve.warnings);
|
|
122
|
+
const sessionFactory = options.createBrowserSession ?? createPlaywrightTestingSession;
|
|
123
|
+
const maxRounds = cfg.builtInAgent.maxRoundsPerCriterion ?? 12;
|
|
124
|
+
let lastSteps = [];
|
|
125
|
+
const attemptResult = await runGoalAttempts({
|
|
126
|
+
maxRetries: options.retryPolicy.maxRetries,
|
|
127
|
+
backoffMs: options.retryPolicy.backoffMs,
|
|
128
|
+
runAttempt: async (attemptIndex) => {
|
|
129
|
+
pushTrace({
|
|
130
|
+
kind: "observation",
|
|
131
|
+
summary: `AC sequence attempt ${attemptIndex + 1} start`,
|
|
132
|
+
metadata: { phase: "ac_sequence_attempt", attemptIndex },
|
|
133
|
+
});
|
|
134
|
+
let session;
|
|
135
|
+
try {
|
|
136
|
+
session = await sessionFactory({
|
|
137
|
+
headless: options.headless ?? true,
|
|
138
|
+
timeoutMs: options.timeoutMs,
|
|
139
|
+
...(options.storageStatePath !== undefined && options.storageStatePath.length > 0
|
|
140
|
+
? { storageState: options.storageStatePath }
|
|
141
|
+
: {}),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
catch (e) {
|
|
145
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
146
|
+
pushTrace({ kind: "error", summary: `Browser session failed: ${msg}`, metadata: { attemptIndex } });
|
|
147
|
+
return {
|
|
148
|
+
kind: "stop",
|
|
149
|
+
verdict: "failed",
|
|
150
|
+
reasonCode: "ADAPTER_ERROR",
|
|
151
|
+
summary: `Browser adapter could not start a session: ${msg}`,
|
|
152
|
+
retryable: false,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
const steps = [];
|
|
156
|
+
let chatInvocations = 0;
|
|
157
|
+
try {
|
|
158
|
+
const entryUrl = goalEntryUrl(options.baseUrl, goal.startPath);
|
|
159
|
+
await withTimeout(session.navigate(entryUrl, { timeoutMs: options.timeoutMs, waitUntil: "load" }), options.timeoutMs + 2000, "navigation");
|
|
160
|
+
pushTrace({
|
|
161
|
+
kind: "observation",
|
|
162
|
+
summary: `Navigated to ${entryUrl}`,
|
|
163
|
+
metadata: { phase: "ac_sequence_entry", attemptIndex },
|
|
164
|
+
});
|
|
165
|
+
for (let ci = 0; ci < acceptanceCriteria.length; ci++) {
|
|
166
|
+
const ac = acceptanceCriteria[ci];
|
|
167
|
+
let stepVerdict = "passed";
|
|
168
|
+
const notes = [];
|
|
169
|
+
const stepEvidence = [];
|
|
170
|
+
let agentRounds = 0;
|
|
171
|
+
let agentOut = {
|
|
172
|
+
ok: true,
|
|
173
|
+
roundsUsed: 0,
|
|
174
|
+
finished: "done",
|
|
175
|
+
};
|
|
176
|
+
pushTrace({
|
|
177
|
+
kind: "observation",
|
|
178
|
+
summary: `AC ${ac.id}: start`,
|
|
179
|
+
metadata: { phase: "ac_start", acId: ac.id, attemptIndex },
|
|
180
|
+
});
|
|
181
|
+
const baseExtra = {
|
|
182
|
+
RESTORMEL_TESTING_AC_ID: ac.id,
|
|
183
|
+
RESTORMEL_TESTING_AC_TEXT: ac.text,
|
|
184
|
+
RESTORMEL_TESTING_AC_INDEX: String(ci),
|
|
185
|
+
RESTORMEL_TESTING_BASE_URL: options.baseUrl,
|
|
186
|
+
RESTORMEL_TESTING_GOAL_ID: goal.id,
|
|
187
|
+
RESTORMEL_TESTING_RUN_ID: runId,
|
|
188
|
+
RESTORMEL_TESTING_USER_STORY: options.suiteUserStory ?? "",
|
|
189
|
+
};
|
|
190
|
+
if (options.artifactDir !== undefined && options.artifactDir.trim() !== "") {
|
|
191
|
+
baseExtra.RESTORMEL_TESTING_ARTIFACT_DIR = options.artifactDir;
|
|
192
|
+
}
|
|
193
|
+
if (cfg.criterionExecutor !== undefined && cfg.criterionExecutor.trim() !== "") {
|
|
194
|
+
const cr = await runMissionExecutorCommand(cfg.criterionExecutor.trim(), {
|
|
195
|
+
cwd: options.hookCwd,
|
|
196
|
+
label: `ac_sequence criterion_executor (${goal.id} / ${ac.id})`,
|
|
197
|
+
extraEnv: baseExtra,
|
|
198
|
+
});
|
|
199
|
+
if (!cr.ok) {
|
|
200
|
+
stepVerdict = "failed";
|
|
201
|
+
notes.push(`criterion_executor: ${cr.message}`);
|
|
202
|
+
if (options.captureScreenshotOnFailure && options.artifactDir) {
|
|
203
|
+
const shot = await captureStepScreenshot(session, options.artifactDir, goal.id, ac.id, attemptIndex);
|
|
204
|
+
if (shot)
|
|
205
|
+
stepEvidence.push(shot);
|
|
206
|
+
}
|
|
207
|
+
steps.push({
|
|
208
|
+
criterionId: ac.id,
|
|
209
|
+
verdict: "failed",
|
|
210
|
+
reasonCode: "AC_CRITERION_EXECUTOR_FAILED",
|
|
211
|
+
summary: notes.join(" | "),
|
|
212
|
+
agentRoundsUsed: 0,
|
|
213
|
+
evidenceRefs: stepEvidence,
|
|
214
|
+
});
|
|
215
|
+
pushTrace({
|
|
216
|
+
kind: "assertion",
|
|
217
|
+
summary: `AC ${ac.id}: failed`,
|
|
218
|
+
metadata: { acId: ac.id, verdict: "failed", attemptIndex },
|
|
219
|
+
});
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const perAcBudget = options.timeoutMs * 4 + maxRounds * 5000;
|
|
224
|
+
try {
|
|
225
|
+
agentOut = await withTimeout(runBuiltInAcAgentLoop(session.page, ac, agentModel, options.baseUrl, {
|
|
226
|
+
maxRounds,
|
|
227
|
+
instructions: cfg.builtInAgent.instructions,
|
|
228
|
+
}), perAcBudget, "ac_agent_loop");
|
|
229
|
+
}
|
|
230
|
+
catch (e) {
|
|
231
|
+
if (e instanceof TimeoutError) {
|
|
232
|
+
agentOut = {
|
|
233
|
+
ok: false,
|
|
234
|
+
roundsUsed: maxRounds,
|
|
235
|
+
reasonCode: "AC_AGENT_TIMEOUT",
|
|
236
|
+
summary: e.message,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
throw e;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
chatInvocations += agentOut.roundsUsed;
|
|
244
|
+
agentRounds = agentOut.roundsUsed;
|
|
245
|
+
if (!agentOut.ok) {
|
|
246
|
+
stepVerdict = worstVerdict(stepVerdict, "failed");
|
|
247
|
+
notes.push(`agent: ${agentOut.summary}`);
|
|
248
|
+
}
|
|
249
|
+
const scMap = cfg.criterionSuccess?.[ac.id];
|
|
250
|
+
if (scMap !== undefined) {
|
|
251
|
+
const jRef = judgeLogicalRefForCriteria(scMap, options.resolvedKeys);
|
|
252
|
+
let judgeM;
|
|
253
|
+
if (jRef) {
|
|
254
|
+
const jr = await resolveModel(jRef, options.keysAdapterOptions ?? {});
|
|
255
|
+
if (jr.ok) {
|
|
256
|
+
judgeM = jr.model;
|
|
257
|
+
warnings.push(...jr.warnings);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
const det = await evaluateBrowserSuccessCriteria(session.page, scMap, { judgeModel: judgeM });
|
|
261
|
+
if (det.judgeModelInvocations && det.judgeModelInvocations > 0) {
|
|
262
|
+
if (judgeM) {
|
|
263
|
+
keysModelMetaFragments.push(toKeysModelMeta(judgeM, det.judgeModelInvocations));
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
stepVerdict = worstVerdict(stepVerdict, det.verdict);
|
|
267
|
+
if (det.verdict !== "passed")
|
|
268
|
+
notes.push(`criteria: ${det.summary}`);
|
|
269
|
+
}
|
|
270
|
+
const rub = cfg.criterionRubrics?.[ac.id];
|
|
271
|
+
if (rub !== undefined) {
|
|
272
|
+
const rref = rub.modelRef?.trim() ||
|
|
273
|
+
options.resolvedKeys.llm_primary ||
|
|
274
|
+
options.resolvedKeys.llmPrimary ||
|
|
275
|
+
options.resolvedKeys.judge;
|
|
276
|
+
if (!rref) {
|
|
277
|
+
stepVerdict = worstVerdict(stepVerdict, "indeterminate");
|
|
278
|
+
notes.push("criterion_rubrics: no model_ref / llm_primary");
|
|
279
|
+
}
|
|
280
|
+
else {
|
|
281
|
+
const rr = await resolveModel(rref, options.keysAdapterOptions ?? {});
|
|
282
|
+
if (!rr.ok) {
|
|
283
|
+
stepVerdict = worstVerdict(stepVerdict, "indeterminate");
|
|
284
|
+
notes.push(`criterion_rubrics resolve: ${rr.error.message}`);
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
warnings.push(...rr.warnings);
|
|
288
|
+
const jr = await runAcShapedJudgeRubric(session.page, rub, rr.model, { id: ac.id, text: ac.text });
|
|
289
|
+
if (jr.judgeModelInvocations && jr.judgeModelInvocations > 0) {
|
|
290
|
+
keysModelMetaFragments.push(toKeysModelMeta(rr.model, jr.judgeModelInvocations));
|
|
291
|
+
}
|
|
292
|
+
stepVerdict = worstVerdict(stepVerdict, jr.verdict);
|
|
293
|
+
if (jr.verdict !== "passed")
|
|
294
|
+
notes.push(`judge: ${jr.summary}`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
for (const pc of (cfg.postChecks ?? []).filter((p) => p.acId === ac.id)) {
|
|
299
|
+
const pr = await runAcPostCheck(session.page, pc, {
|
|
300
|
+
baseUrl: options.baseUrl,
|
|
301
|
+
hookCwd: options.hookCwd,
|
|
302
|
+
extraEnv: baseExtra,
|
|
303
|
+
});
|
|
304
|
+
if (!pr.ok) {
|
|
305
|
+
stepVerdict = worstVerdict(stepVerdict, "failed");
|
|
306
|
+
notes.push(`post_check: ${pr.summary}`);
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (stepVerdict !== "passed" && options.captureScreenshotOnFailure && options.artifactDir) {
|
|
311
|
+
const shot = await captureStepScreenshot(session, options.artifactDir, goal.id, ac.id, attemptIndex);
|
|
312
|
+
if (shot)
|
|
313
|
+
stepEvidence.push(shot);
|
|
314
|
+
}
|
|
315
|
+
const reasonCode = stepVerdict === "passed"
|
|
316
|
+
? "AC_STEP_OK"
|
|
317
|
+
: !agentOut.ok
|
|
318
|
+
? agentOut.reasonCode
|
|
319
|
+
: "AC_STEP_FAILED";
|
|
320
|
+
steps.push({
|
|
321
|
+
criterionId: ac.id,
|
|
322
|
+
verdict: stepVerdict,
|
|
323
|
+
reasonCode,
|
|
324
|
+
summary: notes.length > 0 ? notes.join(" | ") : "OK",
|
|
325
|
+
agentRoundsUsed: agentRounds,
|
|
326
|
+
evidenceRefs: stepEvidence,
|
|
327
|
+
});
|
|
328
|
+
pushTrace({
|
|
329
|
+
kind: "assertion",
|
|
330
|
+
summary: `AC ${ac.id}: ${stepVerdict}`,
|
|
331
|
+
metadata: { acId: ac.id, verdict: stepVerdict, attemptIndex },
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
const drained = session.drainTraceEntries();
|
|
335
|
+
const mapped = browserTracesToCoreEvents(drained, {
|
|
336
|
+
runId,
|
|
337
|
+
goalId: goal.id,
|
|
338
|
+
startingStepIndex: stepCursor,
|
|
339
|
+
});
|
|
340
|
+
stepCursor += mapped.length;
|
|
341
|
+
allTraces.push(...mapped);
|
|
342
|
+
if (chatInvocations > 0) {
|
|
343
|
+
keysModelMetaFragments.push(toKeysModelMeta(agentModel, chatInvocations));
|
|
344
|
+
}
|
|
345
|
+
lastSteps = steps;
|
|
346
|
+
const overall = aggregateStepVerdicts(steps);
|
|
347
|
+
const failedCount = steps.filter((s) => s.verdict !== "passed").length;
|
|
348
|
+
const summary = failedCount === 0
|
|
349
|
+
? "All acceptance criteria satisfied"
|
|
350
|
+
: `${failedCount} acceptance criterion/criteria not satisfied`;
|
|
351
|
+
if (overall === "passed") {
|
|
352
|
+
return {
|
|
353
|
+
kind: "stop",
|
|
354
|
+
verdict: "passed",
|
|
355
|
+
reasonCode: "AC_SEQUENCE_OK",
|
|
356
|
+
summary,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
if (overall === "indeterminate") {
|
|
360
|
+
return {
|
|
361
|
+
kind: "stop",
|
|
362
|
+
verdict: "indeterminate",
|
|
363
|
+
reasonCode: "AC_SEQUENCE_INDETERMINATE",
|
|
364
|
+
summary,
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
return {
|
|
368
|
+
kind: "retry",
|
|
369
|
+
verdict: "failed",
|
|
370
|
+
reasonCode: "AC_SEQUENCE_FAILED",
|
|
371
|
+
summary,
|
|
372
|
+
retryable: true,
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
catch (e) {
|
|
376
|
+
const drained = session.drainTraceEntries();
|
|
377
|
+
const mapped = browserTracesToCoreEvents(drained, {
|
|
378
|
+
runId,
|
|
379
|
+
goalId: goal.id,
|
|
380
|
+
startingStepIndex: stepCursor,
|
|
381
|
+
});
|
|
382
|
+
stepCursor += mapped.length;
|
|
383
|
+
allTraces.push(...mapped);
|
|
384
|
+
if (e instanceof TimeoutError) {
|
|
385
|
+
pushTrace({ kind: "error", summary: e.message, metadata: { attemptIndex, code: e.code } });
|
|
386
|
+
lastSteps = steps;
|
|
387
|
+
return {
|
|
388
|
+
kind: "retry",
|
|
389
|
+
verdict: "failed",
|
|
390
|
+
reasonCode: "TIMEOUT",
|
|
391
|
+
summary: e.message,
|
|
392
|
+
retryable: true,
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
396
|
+
pushTrace({ kind: "error", summary: msg, metadata: { attemptIndex } });
|
|
397
|
+
lastSteps = steps;
|
|
398
|
+
return {
|
|
399
|
+
kind: "retry",
|
|
400
|
+
verdict: "failed",
|
|
401
|
+
reasonCode: "BROWSER_ERROR",
|
|
402
|
+
summary: msg,
|
|
403
|
+
retryable: true,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
finally {
|
|
407
|
+
await session.dispose().catch(() => undefined);
|
|
408
|
+
}
|
|
409
|
+
},
|
|
410
|
+
});
|
|
411
|
+
const allEvidence = lastSteps.flatMap((s) => s.evidenceRefs);
|
|
412
|
+
const goalRecord = {
|
|
413
|
+
goalId: goal.id,
|
|
414
|
+
verdict: attemptResult.verdict,
|
|
415
|
+
reasonCode: attemptResult.reasonCode,
|
|
416
|
+
summary: attemptResult.summary,
|
|
417
|
+
retriesUsed: attemptResult.retriesUsed,
|
|
418
|
+
evidenceRefs: [...new Set(allEvidence)],
|
|
419
|
+
acSequenceSteps: lastSteps.length > 0 ? lastSteps : undefined,
|
|
420
|
+
...acIdsFragment(goal),
|
|
421
|
+
};
|
|
422
|
+
return {
|
|
423
|
+
goalRecord,
|
|
424
|
+
traces: allTraces,
|
|
425
|
+
nextStepIndex: stepCursor,
|
|
426
|
+
warnings,
|
|
427
|
+
keysModelMetaFragments,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
//# sourceMappingURL=run-ac-sequence-goal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-ac-sequence-goal.js","sourceRoot":"","sources":["../src/run-ac-sequence-goal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,8BAA8B,EAAE,MAAM,uCAAuC,CAAC;AAUlH,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAG9D,OAAO,EAAE,YAAY,EAAsB,MAAM,iCAAiC,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,8BAA8B,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAyD,MAAM,mBAAmB,CAAC;AACxG,OAAO,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAuB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzD,SAAS,MAAM;IACb,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,YAAY,CAAC,CAAU,EAAE,CAAU;IAC1C,MAAM,IAAI,GAA4B,EAAE,MAAM,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACjF,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,qBAAqB,CAAC,KAA6B;IAC1D,IAAI,CAAC,GAAY,QAAQ,CAAC;IAC1B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,CAAC,GAAG,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,eAAe,CAAC,KAAoB,EAAE,WAAmB;IAChE,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU;QACjC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;QAC7B,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;QACvB,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB;QAC7C,eAAe,EAAE,WAAW;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAc,EAAE,YAAoC;IACvE,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC1D,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAClB,OAAO,CACL,YAAY,CAAC,WAAW;QACxB,YAAY,CAAC,UAAU;QACvB,YAAY,CAAC,KAAK;QAClB,YAAY,CAAC,SAAS;QACtB,YAAY,CAAC,QAAQ,CACtB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,OAA8B,EAC9B,WAAmB,EACnB,MAAc,EACd,IAAY,EACZ,YAAoB;IAEpB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAChE,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,WAAW,YAAY,IAAI,MAAM,MAAM,CAAC;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAc;IACnC,IAAI,IAAI,CAAC,sBAAsB,KAAK,SAAS,IAAI,IAAI,CAAC,sBAAsB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrG,OAAO,EAAE,sBAAsB,EAAE,CAAC,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC;AACtE,CAAC;AAQD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAAwC;IACrF,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IACpD,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;IAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,sBAAsB,GAAoB,EAAE,CAAC;IACnD,MAAM,SAAS,GAAiB,EAAE,CAAC;IACnC,IAAI,UAAU,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAE3C,MAAM,SAAS,GAAG,CAAC,OAAgF,EAAE,EAAE;QACrG,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,UAAU,EAAE;YAChB,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,UAAU,EAAE;YACvB,SAAS,EAAE,MAAM,EAAE;YACnB,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,EAAE,GAAkB;YACxB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,OAAO,EAAE,QAAQ;YACjB,UAAU,EAAE,uBAAuB;YACnC,OAAO,EAAE,0FAA0F;YACnG,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,EAAE;YAChB,GAAG,aAAa,CAAC,IAAI,CAAC;SACvB,CAAC;QACF,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAC5G,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;IACpF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;QACrB,MAAM,EAAE,GAAkB;YACxB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,OAAO,EAAE,QAAQ;YACjB,UAAU,EAAE,+BAA+B;YAC3C,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,OAAO;YACnC,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,EAAE;YAChB,GAAG,aAAa,CAAC,IAAI,CAAC;SACvB,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,wCAAwC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;QAClG,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IAC5G,CAAC;IACD,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC;IACtC,QAAQ,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAExC,MAAM,cAAc,GAAG,OAAO,CAAC,oBAAoB,IAAI,8BAA8B,CAAC;IACtF,MAAM,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC,qBAAqB,IAAI,EAAE,CAAC;IAE/D,IAAI,SAAS,GAA2B,EAAE,CAAC;IAE3C,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC;QAC1C,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC,UAAU;QAC1C,SAAS,EAAE,OAAO,CAAC,WAAW,CAAC,SAAS;QACxC,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE;YACjC,SAAS,CAAC;gBACR,IAAI,EAAE,aAAa;gBACnB,OAAO,EAAE,uBAAuB,YAAY,GAAG,CAAC,QAAQ;gBACxD,QAAQ,EAAE,EAAE,KAAK,EAAE,qBAAqB,EAAE,YAAY,EAAE;aACzD,CAAC,CAAC;YAEH,IAAI,OAA0C,CAAC;YAC/C,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,cAAc,CAAC;oBAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;oBAClC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,IAAI,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;wBAC/E,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,gBAAgB,EAAE;wBAC5C,CAAC,CAAC,EAAE,CAAC;iBACR,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,2BAA2B,GAAG,EAAE,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;gBACpG,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,eAAe;oBAC3B,OAAO,EAAE,8CAA8C,GAAG,EAAE;oBAC5D,SAAS,EAAE,KAAK;iBACQ,CAAC;YAC7B,CAAC;YAED,MAAM,KAAK,GAA2B,EAAE,CAAC;YACzC,IAAI,eAAe,GAAG,CAAC,CAAC;YAExB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC/D,MAAM,WAAW,CACf,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAC/E,OAAO,CAAC,SAAS,GAAG,IAAI,EACxB,YAAY,CACb,CAAC;gBACF,SAAS,CAAC;oBACR,IAAI,EAAE,aAAa;oBACnB,OAAO,EAAE,gBAAgB,QAAQ,EAAE;oBACnC,QAAQ,EAAE,EAAE,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE;iBACvD,CAAC,CAAC;gBAEH,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,kBAAkB,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;oBACtD,MAAM,EAAE,GAAG,kBAAkB,CAAC,EAAE,CAAE,CAAC;oBACnC,IAAI,WAAW,GAAY,QAAQ,CAAC;oBACpC,MAAM,KAAK,GAAa,EAAE,CAAC;oBAC3B,MAAM,YAAY,GAAa,EAAE,CAAC;oBAClC,IAAI,WAAW,GAAG,CAAC,CAAC;oBACpB,IAAI,QAAQ,GAAsB;wBAChC,EAAE,EAAE,IAAI;wBACR,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,MAAM;qBACjB,CAAC;oBAEF,SAAS,CAAC;wBACR,IAAI,EAAE,aAAa;wBACnB,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS;wBAC7B,QAAQ,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,YAAY,EAAE;qBAC3D,CAAC,CAAC;oBAEH,MAAM,SAAS,GAAuC;wBACpD,uBAAuB,EAAE,EAAE,CAAC,EAAE;wBAC9B,yBAAyB,EAAE,EAAE,CAAC,IAAI;wBAClC,0BAA0B,EAAE,MAAM,CAAC,EAAE,CAAC;wBACtC,0BAA0B,EAAE,OAAO,CAAC,OAAO;wBAC3C,yBAAyB,EAAE,IAAI,CAAC,EAAE;wBAClC,wBAAwB,EAAE,KAAK;wBAC/B,4BAA4B,EAAE,OAAO,CAAC,cAAc,IAAI,EAAE;qBAC3D,CAAC;oBACF,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;wBAC3E,SAAS,CAAC,8BAA8B,GAAG,OAAO,CAAC,WAAW,CAAC;oBACjE,CAAC;oBAED,IAAI,GAAG,CAAC,iBAAiB,KAAK,SAAS,IAAI,GAAG,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;wBAC/E,MAAM,EAAE,GAAG,MAAM,yBAAyB,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,EAAE,EAAE;4BACvE,GAAG,EAAE,OAAO,CAAC,OAAO;4BACpB,KAAK,EAAE,mCAAmC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;4BAC/D,QAAQ,EAAE,SAAS;yBACpB,CAAC,CAAC;wBACH,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;4BACX,WAAW,GAAG,QAAQ,CAAC;4BACvB,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;4BAChD,IAAI,OAAO,CAAC,0BAA0B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;gCAC9D,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;gCACrG,IAAI,IAAI;oCAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACpC,CAAC;4BACD,KAAK,CAAC,IAAI,CAAC;gCACT,WAAW,EAAE,EAAE,CAAC,EAAE;gCAClB,OAAO,EAAE,QAAQ;gCACjB,UAAU,EAAE,8BAA8B;gCAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gCAC1B,eAAe,EAAE,CAAC;gCAClB,YAAY,EAAE,YAAY;6BAC3B,CAAC,CAAC;4BACH,SAAS,CAAC;gCACR,IAAI,EAAE,WAAW;gCACjB,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU;gCAC9B,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE;6BAC3D,CAAC,CAAC;4BACH,SAAS;wBACX,CAAC;oBACH,CAAC;oBAED,MAAM,WAAW,GAAG,OAAO,CAAC,SAAS,GAAG,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC;oBAC7D,IAAI,CAAC;wBACH,QAAQ,GAAG,MAAM,WAAW,CAC1B,qBAAqB,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,OAAO,EAAE;4BACnE,SAAS;4BACT,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,YAAY;yBAC5C,CAAC,EACF,WAAW,EACX,eAAe,CAChB,CAAC;oBACJ,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;4BAC9B,QAAQ,GAAG;gCACT,EAAE,EAAE,KAAK;gCACT,UAAU,EAAE,SAAS;gCACrB,UAAU,EAAE,kBAAkB;gCAC9B,OAAO,EAAE,CAAC,CAAC,OAAO;6BACnB,CAAC;wBACJ,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,CAAC;wBACV,CAAC;oBACH,CAAC;oBAED,eAAe,IAAI,QAAQ,CAAC,UAAU,CAAC;oBACvC,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC;oBAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACjB,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;wBAClD,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC3C,CAAC;oBAED,MAAM,KAAK,GAAG,GAAG,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACxB,MAAM,IAAI,GAAG,0BAA0B,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;wBACrE,IAAI,MAAiC,CAAC;wBACtC,IAAI,IAAI,EAAE,CAAC;4BACT,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;4BACtE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;gCACV,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC;gCAClB,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;4BAChC,CAAC;wBACH,CAAC;wBACD,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;wBAC9F,IAAI,GAAG,CAAC,qBAAqB,IAAI,GAAG,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;4BAC/D,IAAI,MAAM,EAAE,CAAC;gCACX,sBAAsB,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAC,CAAC;4BAClF,CAAC;wBACH,CAAC;wBACD,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;wBACrD,IAAI,GAAG,CAAC,OAAO,KAAK,QAAQ;4BAAE,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;oBACvE,CAAC;oBAED,MAAM,GAAG,GAAG,GAAG,CAAC,gBAAgB,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC1C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;wBACtB,MAAM,IAAI,GACR,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE;4BACpB,OAAO,CAAC,YAAY,CAAC,WAAW;4BAChC,OAAO,CAAC,YAAY,CAAC,UAAU;4BAC/B,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC;wBAC7B,IAAI,CAAC,IAAI,EAAE,CAAC;4BACV,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;4BACzD,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;wBAC9D,CAAC;6BAAM,CAAC;4BACN,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;4BACtE,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gCACX,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;gCACzD,KAAK,CAAC,IAAI,CAAC,8BAA8B,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;4BAC/D,CAAC;iCAAM,CAAC;gCACN,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;gCAC9B,MAAM,EAAE,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gCACnG,IAAI,EAAE,CAAC,qBAAqB,IAAI,EAAE,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;oCAC7D,sBAAsB,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,qBAAqB,CAAC,CAAC,CAAC;gCACnF,CAAC;gCACD,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;gCACpD,IAAI,EAAE,CAAC,OAAO,KAAK,QAAQ;oCAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;4BAClE,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;wBACxE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE;4BAChD,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,QAAQ,EAAE,SAAS;yBACpB,CAAC,CAAC;wBACH,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;4BACX,WAAW,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;4BAClD,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;4BACxC,MAAM;wBACR,CAAC;oBACH,CAAC;oBAED,IAAI,WAAW,KAAK,QAAQ,IAAI,OAAO,CAAC,0BAA0B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;wBAC1F,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;wBACrG,IAAI,IAAI;4BAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACpC,CAAC;oBAED,MAAM,UAAU,GACd,WAAW,KAAK,QAAQ;wBACtB,CAAC,CAAC,YAAY;wBACd,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE;4BACZ,CAAC,CAAC,QAAQ,CAAC,UAAU;4BACrB,CAAC,CAAC,gBAAgB,CAAC;oBAEzB,KAAK,CAAC,IAAI,CAAC;wBACT,WAAW,EAAE,EAAE,CAAC,EAAE;wBAClB,OAAO,EAAE,WAAW;wBACpB,UAAU;wBACV,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;wBACpD,eAAe,EAAE,WAAW;wBAC5B,YAAY,EAAE,YAAY;qBAC3B,CAAC,CAAC;oBAEH,SAAS,CAAC;wBACR,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,WAAW,EAAE;wBACtC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;qBAC9D,CAAC,CAAC;gBACL,CAAC;gBAED,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,EAAE;oBAChD,KAAK;oBACL,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,iBAAiB,EAAE,UAAU;iBAC9B,CAAC,CAAC;gBACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC5B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;oBACxB,sBAAsB,CAAC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC5E,CAAC;gBAED,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;gBACvE,MAAM,OAAO,GACX,WAAW,KAAK,CAAC;oBACf,CAAC,CAAC,mCAAmC;oBACrC,CAAC,CAAC,GAAG,WAAW,8CAA8C,CAAC;gBAEnE,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACzB,OAAO;wBACL,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,QAAQ;wBACjB,UAAU,EAAE,gBAAgB;wBAC5B,OAAO;qBACiB,CAAC;gBAC7B,CAAC;gBACD,IAAI,OAAO,KAAK,eAAe,EAAE,CAAC;oBAChC,OAAO;wBACL,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,eAAe;wBACxB,UAAU,EAAE,2BAA2B;wBACvC,OAAO;qBACiB,CAAC;gBAC7B,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,oBAAoB;oBAChC,OAAO;oBACP,SAAS,EAAE,IAAI;iBACS,CAAC;YAC7B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,EAAE;oBAChD,KAAK;oBACL,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,iBAAiB,EAAE,UAAU;iBAC9B,CAAC,CAAC;gBACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC5B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAE1B,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;oBAC9B,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC3F,SAAS,GAAG,KAAK,CAAC;oBAClB,OAAO;wBACL,IAAI,EAAE,OAAO;wBACb,OAAO,EAAE,QAAQ;wBACjB,UAAU,EAAE,SAAS;wBACrB,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,SAAS,EAAE,IAAI;qBACS,CAAC;gBAC7B,CAAC;gBAED,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;gBACvE,SAAS,GAAG,KAAK,CAAC;gBAClB,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,eAAe;oBAC3B,OAAO,EAAE,GAAG;oBACZ,SAAS,EAAE,IAAI;iBACS,CAAC;YAC7B,CAAC;oBAAS,CAAC;gBACT,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAkB;QAChC,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,UAAU,EAAE,aAAa,CAAC,UAAU;QACpC,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,eAAe,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC7D,GAAG,aAAa,CAAC,IAAI,CAAC;KACvB,CAAC;IAEF,OAAO;QACL,UAAU;QACV,MAAM,EAAE,SAAS;QACjB,aAAa,EAAE,UAAU;QACzB,QAAQ;QACR,sBAAsB;KACvB,CAAC;AACJ,CAAC"}
|
package/dist/run-suite.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"AAyBA,OAAO,KAAK,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AA0CjG;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC,CAmYnG;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CA2B1F"}
|