@restormel/testing-runner 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Allotment Technology Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/browser-goal.d.ts

@@ -0,0 +1,29 @@
+import type { PlaywrightTestingSessionOptions, TestingBrowserSession } from "@restormel/testing-browser-playwright";
+import type { GoalRunRecord, KeysModelMeta, TestGoal, TraceEvent } from "@restormel/testing-core";
+import type { RetryPolicy } from "@restormel/testing-core";
+import { type KeysModelAdapterOptions } from "@restormel/testing-keys-adapter";
+export interface RunBrowserGoalOptions {
+    runId: string;
+    goal: TestGoal;
+    baseUrl: string;
+    timeoutMs: number;
+    retryPolicy: RetryPolicy;
+    artifactDir?: string;
+    captureScreenshotOnFailure: boolean;
+    headless?: boolean;
+    /** Playwright storage state JSON path (from environment auth_ref). */
+    storageStatePath?: string;
+    createBrowserSession?: (opts?: PlaywrightTestingSessionOptions) => Promise<TestingBrowserSession>;
+    resolvedKeys: Record<string, string>;
+    keysAdapterOptions?: KeysModelAdapterOptions;
+    startingStepIndex: number;
+}
+export interface RunBrowserGoalResult {
+    goalRecord: GoalRunRecord;
+    traces: TraceEvent[];
+    nextStepIndex: number;
+    warnings: string[];
+    keysModelMetaFragments: KeysModelMeta[];
+}
+export declare function runBrowserGoal(options: RunBrowserGoalOptions): Promise<RunBrowserGoalResult>;
+//# sourceMappingURL=browser-goal.d.ts.map

package/dist/browser-goal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-goal.d.ts","sourceRoot":"","sources":["../src/browser-goal.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,+BAA+B,EAC/B,qBAAqB,EACtB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAClG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAgB,KAAK,uBAAuB,EAAsB,MAAM,iCAAiC,CAAC;AAMjH,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,WAAW,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B,EAAE,OAAO,CAAC;IACpC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,+BAA+B,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAClG,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,kBAAkB,CAAC,EAAE,uBAAuB,CAAC;IAC7C,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,aAAa,CAAC;IAC1B,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,sBAAsB,EAAE,aAAa,EAAE,CAAC;CACzC;AAkCD,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAgOlG"}

package/dist/browser-goal.js

@@ -0,0 +1,240 @@
+import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { browserTracesToCoreEvents, createPlaywrightTestingSession } from "@restormel/testing-browser-playwright";
+import { resolveModel } from "@restormel/testing-keys-adapter";
+import { evaluateBrowserSuccessCriteria } from "./evaluate-criteria.js";
+import { judgeLogicalRefForCriteria } from "./judge-ref.js";
+import { runGoalAttempts } from "./retries.js";
+import { TimeoutError, withTimeout } from "./timeout.js";
+function nowIso() {
+    return new Date().toISOString();
+}
+function toKeysModelMeta(model, invocations) {
+    return {
+        logicalRef: model.meta.logicalRef,
+        provider: model.meta.provider,
+        model: model.meta.model,
+        resolutionSource: model.meta.resolutionSource,
+        invocationCount: invocations,
+    };
+}
+async function captureScreenshot(session, artifactDir, goalId, attemptIndex) {
+    const dir = join(artifactDir, "goals", goalId);
+    await mkdir(dir, { recursive: true });
+    const name = `attempt-${attemptIndex}.png`;
+    const abs = join(dir, name);
+    try {
+        await session.screenshot(abs);
+        return join("goals", goalId, name);
+    }
+    catch {
+        return undefined;
+    }
+}
+export async function runBrowserGoal(options) {
+    const { goal, runId } = options;
+    const warnings = [];
+    const keysModelMetaFragments = [];
+    const allTraces = [];
+    const evidenceRefs = [];
+    let stepCursor = options.startingStepIndex;
+    const pushTrace = (partial) => {
+        allTraces.push({
+            id: randomUUID(),
+            runId,
+            goalId: goal.id,
+            stepIndex: stepCursor++,
+            timestamp: nowIso(),
+            ...partial,
+        });
+    };
+    let judgeModel;
+    if (goal.successCriteria.judgeRubric) {
+        const ref = judgeLogicalRefForCriteria(goal.successCriteria, options.resolvedKeys);
+        if (!ref) {
+            warnings.push(`Goal "${goal.id}": judge_rubric has no model_ref and no llm_primary / judge key in environment`);
+        }
+        else {
+            const res = await resolveModel(ref, options.keysAdapterOptions ?? {});
+            if (res.ok) {
+                warnings.push(...res.warnings);
+                judgeModel = res.model;
+            }
+            else {
+                warnings.push(`Goal "${goal.id}": model resolution failed (${res.error.code}): ${res.error.message}`);
+            }
+        }
+    }
+    const sessionFactory = options.createBrowserSession ?? createPlaywrightTestingSession;
+    const attemptResult = await runGoalAttempts({
+        maxRetries: options.retryPolicy.maxRetries,
+        backoffMs: options.retryPolicy.backoffMs,
+        runAttempt: async (attemptIndex) => {
+            pushTrace({
+                kind: "observation",
+                summary: `Attempt ${attemptIndex + 1} start`,
+                metadata: { phase: "attempt_start", attemptIndex },
+            });
+            let session;
+            try {
+                session = await sessionFactory({
+                    headless: options.headless ?? true,
+                    timeoutMs: options.timeoutMs,
+                    ...(options.storageStatePath !== undefined && options.storageStatePath.length > 0
+                        ? { storageState: options.storageStatePath }
+                        : {}),
+                });
+            }
+            catch (e) {
+                const msg = e instanceof Error ? e.message : String(e);
+                pushTrace({
+                    kind: "error",
+                    summary: `Browser session failed: ${msg}`,
+                    metadata: { attemptIndex },
+                });
+                return {
+                    kind: "stop",
+                    verdict: "failed",
+                    reasonCode: "ADAPTER_ERROR",
+                    summary: `Browser adapter could not start a session: ${msg}`,
+                    retryable: false,
+                };
+            }
+            try {
+                await withTimeout(session.navigate(options.baseUrl, { timeoutMs: options.timeoutMs, waitUntil: "load" }), options.timeoutMs + 2000, "navigation");
+                pushTrace({
+                    kind: "observation",
+                    summary: `Navigated to ${options.baseUrl}`,
+                    metadata: { phase: "post_navigate", attemptIndex },
+                });
+                const evalResult = await withTimeout(evaluateBrowserSuccessCriteria(session.page, goal.successCriteria, {
+                    judgeModel,
+                }), options.timeoutMs + 2000, "evaluation");
+                const drained = session.drainTraceEntries();
+                const mapped = browserTracesToCoreEvents(drained, {
+                    runId,
+                    goalId: goal.id,
+                    startingStepIndex: stepCursor,
+                });
+                stepCursor += mapped.length;
+                allTraces.push(...mapped);
+                if (judgeModel && (evalResult.judgeModelInvocations ?? 0) > 0) {
+                    keysModelMetaFragments.push(toKeysModelMeta(judgeModel, evalResult.judgeModelInvocations ?? 1));
+                }
+                pushTrace({
+                    kind: "assertion",
+                    summary: evalResult.summary,
+                    metadata: {
+                        verdict: evalResult.verdict,
+                        reasonCode: evalResult.reasonCode,
+                        attemptIndex,
+                    },
+                });
+                if (evalResult.verdict === "passed") {
+                    return {
+                        kind: "stop",
+                        verdict: "passed",
+                        reasonCode: evalResult.reasonCode,
+                        summary: evalResult.summary,
+                    };
+                }
+                if (evalResult.verdict === "indeterminate") {
+                    if (options.captureScreenshotOnFailure &&
+                        options.artifactDir &&
+                        (evalResult.reasonCode === "JUDGE_UNCERTAIN" ||
+                            evalResult.reasonCode === "JUDGE_HTTP_ERROR" ||
+                            evalResult.reasonCode === "JUDGE_PARSE_ERROR")) {
+                        const rel = await captureScreenshot(session, options.artifactDir, goal.id, attemptIndex);
+                        if (rel)
+                            evidenceRefs.push(rel);
+                    }
+                    return {
+                        kind: "stop",
+                        verdict: "indeterminate",
+                        reasonCode: evalResult.reasonCode,
+                        summary: evalResult.summary,
+                    };
+                }
+                if (options.captureScreenshotOnFailure && options.artifactDir) {
+                    const rel = await captureScreenshot(session, options.artifactDir, goal.id, attemptIndex);
+                    if (rel)
+                        evidenceRefs.push(rel);
+                }
+                return {
+                    kind: "retry",
+                    verdict: "failed",
+                    reasonCode: evalResult.reasonCode,
+                    summary: evalResult.summary,
+                    retryable: true,
+                };
+            }
+            catch (e) {
+                const drained = session.drainTraceEntries();
+                const mapped = browserTracesToCoreEvents(drained, {
+                    runId,
+                    goalId: goal.id,
+                    startingStepIndex: stepCursor,
+                });
+                stepCursor += mapped.length;
+                allTraces.push(...mapped);
+                if (e instanceof TimeoutError) {
+                    pushTrace({
+                        kind: "error",
+                        summary: e.message,
+                        metadata: { attemptIndex, code: e.code },
+                    });
+                    if (options.captureScreenshotOnFailure && options.artifactDir) {
+                        const rel = await captureScreenshot(session, options.artifactDir, goal.id, attemptIndex);
+                        if (rel)
+                            evidenceRefs.push(rel);
+                    }
+                    return {
+                        kind: "retry",
+                        verdict: "failed",
+                        reasonCode: "TIMEOUT",
+                        summary: e.message,
+                        retryable: true,
+                    };
+                }
+                const msg = e instanceof Error ? e.message : String(e);
+                pushTrace({
+                    kind: "error",
+                    summary: msg,
+                    metadata: { attemptIndex },
+                });
+                if (options.captureScreenshotOnFailure && options.artifactDir) {
+                    const rel = await captureScreenshot(session, options.artifactDir, goal.id, attemptIndex);
+                    if (rel)
+                        evidenceRefs.push(rel);
+                }
+                return {
+                    kind: "retry",
+                    verdict: "failed",
+                    reasonCode: "BROWSER_ERROR",
+                    summary: msg,
+                    retryable: true,
+                };
+            }
+            finally {
+                await session.dispose().catch(() => undefined);
+            }
+        },
+    });
+    const goalRecord = {
+        goalId: goal.id,
+        verdict: attemptResult.verdict,
+        reasonCode: attemptResult.reasonCode,
+        summary: attemptResult.summary,
+        retriesUsed: attemptResult.retriesUsed,
+        evidenceRefs: [...evidenceRefs],
+    };
+    return {
+        goalRecord,
+        traces: allTraces,
+        nextStepIndex: stepCursor,
+        warnings,
+        keysModelMetaFragments,
+    };
+}
+//# sourceMappingURL=browser-goal.js.map

package/dist/browser-goal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-goal.js","sourceRoot":"","sources":["../src/browser-goal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,8BAA8B,EAAE,MAAM,uCAAuC,CAAC;AAOlH,OAAO,EAAE,YAAY,EAAoD,MAAM,iCAAiC,CAAC;AACjH,OAAO,EAAE,8BAA8B,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAuB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AA2BzD,SAAS,MAAM;IACb,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,eAAe,CAAC,KAAoB,EAAE,WAAmB;IAChE,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU;QACjC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;QAC7B,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;QACvB,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB;QAC7C,eAAe,EAAE,WAAW;KAC7B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,OAA8B,EAC9B,WAAmB,EACnB,MAAc,EACd,YAAoB;IAEpB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,WAAW,YAAY,MAAM,CAAC;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IACjE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAChC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,sBAAsB,GAAoB,EAAE,CAAC;IACnD,MAAM,SAAS,GAAiB,EAAE,CAAC;IACnC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAE3C,MAAM,SAAS,GAAG,CAAC,OAAgF,EAAE,EAAE;QACrG,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,UAAU,EAAE;YAChB,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,SAAS,EAAE,UAAU,EAAE;YACvB,SAAS,EAAE,MAAM,EAAE;YACnB,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,IAAI,UAAqC,CAAC;IAC1C,IAAI,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,0BAA0B,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QACnF,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,gFAAgF,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;YACtE,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC/B,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,EAAE,+BAA+B,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,oBAAoB,IAAI,8BAA8B,CAAC;IAEtF,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC;QAC1C,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC,UAAU;QAC1C,SAAS,EAAE,OAAO,CAAC,WAAW,CAAC,SAAS;QACxC,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE;YACjC,SAAS,CAAC;gBACR,IAAI,EAAE,aAAa;gBACnB,OAAO,EAAE,WAAW,YAAY,GAAG,CAAC,QAAQ;gBAC5C,QAAQ,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE;aACnD,CAAC,CAAC;YAEH,IAAI,OAA0C,CAAC;YAC/C,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,cAAc,CAAC;oBAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;oBAClC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,IAAI,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;wBAC/E,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,gBAAgB,EAAE;wBAC5C,CAAC,CAAC,EAAE,CAAC;iBACR,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,SAAS,CAAC;oBACR,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,2BAA2B,GAAG,EAAE;oBACzC,QAAQ,EAAE,EAAE,YAAY,EAAE;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,eAAe;oBAC3B,OAAO,EAAE,8CAA8C,GAAG,EAAE;oBAC5D,SAAS,EAAE,KAAK;iBACQ,CAAC;YAC7B,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,WAAW,CACf,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EACtF,OAAO,CAAC,SAAS,GAAG,IAAI,EACxB,YAAY,CACb,CAAC;gBAEF,SAAS,CAAC;oBACR,IAAI,EAAE,aAAa;oBACnB,OAAO,EAAE,gBAAgB,OAAO,CAAC,OAAO,EAAE;oBAC1C,QAAQ,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE;iBACnD,CAAC,CAAC;gBAEH,MAAM,UAAU,GAAG,MAAM,WAAW,CAClC,8BAA8B,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE;oBACjE,UAAU;iBACX,CAAC,EACF,OAAO,CAAC,SAAS,GAAG,IAAI,EACxB,YAAY,CACb,CAAC;gBAEF,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,EAAE;oBAChD,KAAK;oBACL,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,iBAAiB,EAAE,UAAU;iBAC9B,CAAC,CAAC;gBACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC5B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAE1B,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,qBAAqB,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9D,sBAAsB,CAAC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU,CAAC,qBAAqB,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClG,CAAC;gBAED,SAAS,CAAC;oBACR,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,UAAU,CAAC,OAAO;oBAC3B,QAAQ,EAAE;wBACR,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,UAAU,EAAE,UAAU,CAAC,UAAU;wBACjC,YAAY;qBACb;iBACF,CAAC,CAAC;gBAEH,IAAI,UAAU,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACpC,OAAO;wBACL,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,QAAQ;wBACjB,UAAU,EAAE,UAAU,CAAC,UAAU;wBACjC,OAAO,EAAE,UAAU,CAAC,OAAO;qBACH,CAAC;gBAC7B,CAAC;gBAED,IAAI,UAAU,CAAC,OAAO,KAAK,eAAe,EAAE,CAAC;oBAC3C,IACE,OAAO,CAAC,0BAA0B;wBAClC,OAAO,CAAC,WAAW;wBACnB,CAAC,UAAU,CAAC,UAAU,KAAK,iBAAiB;4BAC1C,UAAU,CAAC,UAAU,KAAK,kBAAkB;4BAC5C,UAAU,CAAC,UAAU,KAAK,mBAAmB,CAAC,EAChD,CAAC;wBACD,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;wBACzF,IAAI,GAAG;4BAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAClC,CAAC;oBACD,OAAO;wBACL,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,eAAe;wBACxB,UAAU,EAAE,UAAU,CAAC,UAAU;wBACjC,OAAO,EAAE,UAAU,CAAC,OAAO;qBACH,CAAC;gBAC7B,CAAC;gBAED,IAAI,OAAO,CAAC,0BAA0B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;oBAC9D,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;oBACzF,IAAI,GAAG;wBAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClC,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,UAAU,CAAC,UAAU;oBACjC,OAAO,EAAE,UAAU,CAAC,OAAO;oBAC3B,SAAS,EAAE,IAAI;iBACS,CAAC;YAC7B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,EAAE;oBAChD,KAAK;oBACL,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,iBAAiB,EAAE,UAAU;iBAC9B,CAAC,CAAC;gBACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC5B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAE1B,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;oBAC9B,SAAS,CAAC;wBACR,IAAI,EAAE,OAAO;wBACb,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,QAAQ,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;qBACzC,CAAC,CAAC;oBACH,IAAI,OAAO,CAAC,0BAA0B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;wBAC9D,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;wBACzF,IAAI,GAAG;4BAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAClC,CAAC;oBACD,OAAO;wBACL,IAAI,EAAE,OAAO;wBACb,OAAO,EAAE,QAAQ;wBACjB,UAAU,EAAE,SAAS;wBACrB,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,SAAS,EAAE,IAAI;qBACS,CAAC;gBAC7B,CAAC;gBAED,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvD,SAAS,CAAC;oBACR,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,GAAG;oBACZ,QAAQ,EAAE,EAAE,YAAY,EAAE;iBAC3B,CAAC,CAAC;gBACH,IAAI,OAAO,CAAC,0BAA0B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;oBAC9D,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;oBACzF,IAAI,GAAG;wBAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,eAAe;oBAC3B,OAAO,EAAE,GAAG;oBACZ,SAAS,EAAE,IAAI;iBACS,CAAC;YAC7B,CAAC;oBAAS,CAAC;gBACT,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAkB;QAChC,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,UAAU,EAAE,aAAa,CAAC,UAAU;QACpC,OAAO,EAAE,aAAa,CAAC,OAAO;QAC9B,WAAW,EAAE,aAAa,CAAC,WAAW;QACtC,YAAY,EAAE,CAAC,GAAG,YAAY,CAAC;KAChC,CAAC;IAEF,OAAO;QACL,UAAU;QACV,MAAM,EAAE,SAAS;QACjB,aAAa,EAAE,UAAU;QACzB,QAAQ;QACR,sBAAsB;KACvB,CAAC;AACJ,CAAC"}

package/dist/evaluate-criteria.d.ts

@@ -0,0 +1,18 @@
+import type { Page } from "playwright";
+import type { SuccessCriteria, Verdict } from "@restormel/testing-core";
+import type { ResolvedModel } from "@restormel/testing-keys-adapter";
+export interface CriteriaEvaluation {
+    verdict: Verdict;
+    reasonCode: string;
+    summary: string;
+    /** Set when an OpenAI-compatible judge request was sent (for Keys model meta). */
+    judgeModelInvocations?: number;
+}
+/**
+ * Deterministic checks: URL, DOM visibility, text present/absent, minimal structured checks.
+ * Judge rubric (optional) runs after deterministics all pass.
+ */
+export declare function evaluateBrowserSuccessCriteria(page: Page, criteria: SuccessCriteria, options?: {
+    judgeModel?: ResolvedModel;
+}): Promise<CriteriaEvaluation>;
+//# sourceMappingURL=evaluate-criteria.d.ts.map

package/dist/evaluate-criteria.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate-criteria.d.ts","sourceRoot":"","sources":["../src/evaluate-criteria.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAe,eAAe,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAErE,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,kFAAkF;IAClF,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAUD;;;GAGG;AACH,wBAAsB,8BAA8B,CAClD,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,eAAe,EACzB,OAAO,CAAC,EAAE;IACR,UAAU,CAAC,EAAE,aAAa,CAAC;CAC5B,GACA,OAAO,CAAC,kBAAkB,CAAC,CA8G7B"}

package/dist/evaluate-criteria.js

@@ -0,0 +1,243 @@
+function normalizeUrl(u) {
+    try {
+        return new URL(u).href;
+    }
+    catch {
+        return u;
+    }
+}
+/**
+ * Deterministic checks: URL, DOM visibility, text present/absent, minimal structured checks.
+ * Judge rubric (optional) runs after deterministics all pass.
+ */
+export async function evaluateBrowserSuccessCriteria(page, criteria, options) {
+    const url = page.url();
+    if (criteria.urlMatches !== undefined) {
+        const patterns = Array.isArray(criteria.urlMatches) ? criteria.urlMatches : [criteria.urlMatches];
+        const nu = normalizeUrl(url);
+        for (const p of patterns) {
+            if (!nu.includes(p) && !url.includes(p)) {
+                return {
+                    verdict: "failed",
+                    reasonCode: "URL_MISMATCH",
+                    summary: `Expected URL to match ${JSON.stringify(p)}; got ${truncate(url, 200)}`,
+                };
+            }
+        }
+    }
+    if (criteria.domSignals !== undefined && criteria.domSignals.length > 0) {
+        for (const sel of criteria.domSignals) {
+            const loc = page.locator(sel).first();
+            const n = await locatorCountSafe(loc);
+            if (n === 0) {
+                return {
+                    verdict: "failed",
+                    reasonCode: "DOM_SIGNAL_MISSING",
+                    summary: `Expected element for selector ${JSON.stringify(sel)}`,
+                };
+            }
+            const visible = await loc.isVisible().catch(() => false);
+            if (!visible) {
+                return {
+                    verdict: "failed",
+                    reasonCode: "DOM_NOT_VISIBLE",
+                    summary: `Selector ${JSON.stringify(sel)} exists but is not visible`,
+                };
+            }
+        }
+    }
+    const bodyText = await page.locator("body").innerText().catch(() => "");
+    const hay = bodyText.toLowerCase();
+    if (criteria.textPresent !== undefined && criteria.textPresent.length > 0) {
+        for (const t of criteria.textPresent) {
+            if (!hay.includes(t.toLowerCase())) {
+                return {
+                    verdict: "failed",
+                    reasonCode: "TEXT_NOT_FOUND",
+                    summary: `Expected text ${JSON.stringify(truncate(t, 80))} on page`,
+                };
+            }
+        }
+    }
+    if (criteria.textAbsent !== undefined && criteria.textAbsent.length > 0) {
+        for (const t of criteria.textAbsent) {
+            if (hay.includes(t.toLowerCase())) {
+                return {
+                    verdict: "failed",
+                    reasonCode: "TEXT_UNEXPECTED",
+                    summary: `Did not expect text ${JSON.stringify(truncate(t, 80))}`,
+                };
+            }
+        }
+    }
+    if (criteria.structuredChecks !== undefined && criteria.structuredChecks.length > 0) {
+        for (const check of criteria.structuredChecks) {
+            const parsed = parseStructuredPath(check.path);
+            if (!parsed) {
+                return {
+                    verdict: "indeterminate",
+                    reasonCode: "STRUCTURED_PATH_UNKNOWN",
+                    summary: `Unsupported structured check path ${JSON.stringify(truncate(check.path, 120))}`,
+                };
+            }
+            if (parsed.kind === "css") {
+                const text = await page.locator(parsed.selector).first().innerText().catch(() => "");
+                if (!structuredExpectOk(text, check.expect)) {
+                    return {
+                        verdict: "failed",
+                        reasonCode: "STRUCTURED_MISMATCH",
+                        summary: `Structured check ${check.id ?? parsed.selector} did not match expected value`,
+                    };
+                }
+            }
+        }
+    }
+    if (criteria.judgeRubric !== undefined) {
+        if (!options?.judgeModel) {
+            return {
+                verdict: "indeterminate",
+                reasonCode: "JUDGE_NO_MODEL",
+                summary: "judge_rubric present but model could not be resolved via Keys",
+            };
+        }
+        const j = await runJudgeRubric(page, criteria, options.judgeModel);
+        return j;
+    }
+    if (!hasAnyCriterion(criteria)) {
+        return {
+            verdict: "indeterminate",
+            reasonCode: "NO_CRITERIA",
+            summary: "No evaluable success criteria",
+        };
+    }
+    return { verdict: "passed", reasonCode: "OK", summary: "All criteria satisfied" };
+}
+function hasAnyCriterion(c) {
+    return (c.urlMatches != null ||
+        (c.domSignals != null && c.domSignals.length > 0) ||
+        (c.textPresent != null && c.textPresent.length > 0) ||
+        (c.textAbsent != null && c.textAbsent.length > 0) ||
+        (c.structuredChecks != null && c.structuredChecks.length > 0) ||
+        c.judgeRubric != null);
+}
+function parseStructuredPath(path) {
+    if (path.startsWith("css:")) {
+        return { kind: "css", selector: path.slice("css:".length).trim() };
+    }
+    return null;
+}
+function structuredExpectOk(text, expect) {
+    if (expect === undefined)
+        return text.trim().length > 0;
+    return text.trim() === String(expect);
+}
+async function locatorCountSafe(loc) {
+    try {
+        return await loc.count();
+    }
+    catch {
+        return 0;
+    }
+}
+function truncate(s, n) {
+    return s.length <= n ? s : `${s.slice(0, n)}…`;
+}
+const MAX_JUDGE_SAMPLE_CHARS = 8000;
+async function sampleTextForJudge(page, rubric) {
+    if (rubric.contextSelector !== undefined && rubric.contextSelector.trim() !== "") {
+        const t = await page.locator(rubric.contextSelector).first().innerText().catch(() => "");
+        return truncate(t.trim(), MAX_JUDGE_SAMPLE_CHARS);
+    }
+    const mainText = await page.locator("main").first().innerText().catch(() => "");
+    if (mainText.trim().length > 0) {
+        return truncate(mainText.trim(), MAX_JUDGE_SAMPLE_CHARS);
+    }
+    const bodyText = await page.locator("body").innerText().catch(() => "");
+    return truncate(bodyText.trim(), MAX_JUDGE_SAMPLE_CHARS);
+}
+function redactForLog(s) {
+    return s.replace(/\bBearer\s+[\w-_.]+\b/gi, "Bearer [redacted]").replace(/\bsk-[a-zA-Z0-9]{10,}\b/g, "sk-[redacted]");
+}
+async function runJudgeRubric(page, criteria, model) {
+    const rubric = criteria.judgeRubric;
+    if (!rubric) {
+        return { verdict: "passed", reasonCode: "OK", summary: "No rubric" };
+    }
+    const sample = await sampleTextForJudge(page, rubric);
+    const base = model.providerBaseUrl?.replace(/\/?$/, "") ?? "https://api.openai.com/v1";
+    const url = `${base}/chat/completions`;
+    const system = 'You are a test oracle. Reply with a single JSON object only: {"verdict":"pass"|"fail"|"uncertain"} matching whether the page satisfies the rubric.';
+    const user = `Rubric id: ${rubric.id}\nSummary: ${rubric.summary ?? "(none)"}\n\nPage text:\n${sample}`;
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                authorization: `Bearer ${model.credentials.apiKey}`,
+            },
+            body: JSON.stringify({
+                model: model.modelId,
+                messages: [
+                    { role: "system", content: system },
+                    { role: "user", content: user },
+                ],
+                max_tokens: 80,
+                response_format: { type: "json_object" },
+            }),
+        });
+        if (!res.ok) {
+            const t = await res.text().catch(() => "");
+            return {
+                verdict: "indeterminate",
+                reasonCode: "JUDGE_HTTP_ERROR",
+                summary: `Judge request failed: HTTP ${res.status} ${truncate(redactForLog(t), 80)}`,
+                judgeModelInvocations: 1,
+            };
+        }
+        const data = (await res.json());
+        const raw = data.choices?.[0]?.message?.content ?? "{}";
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            return {
+                verdict: "indeterminate",
+                reasonCode: "JUDGE_PARSE_ERROR",
+                summary: "Judge response was not valid JSON",
+                judgeModelInvocations: 1,
+            };
+        }
+        const v = (parsed.verdict ?? "").toLowerCase();
+        if (v === "pass") {
+            return {
+                verdict: "passed",
+                reasonCode: "JUDGE_PASS",
+                summary: "Judge rubric passed",
+                judgeModelInvocations: 1,
+            };
+        }
+        if (v === "fail") {
+            return {
+                verdict: "failed",
+                reasonCode: "JUDGE_FAIL",
+                summary: "Judge rubric failed",
+                judgeModelInvocations: 1,
+            };
+        }
+        return {
+            verdict: "indeterminate",
+            reasonCode: "JUDGE_UNCERTAIN",
+            summary: "Judge returned uncertain",
+            judgeModelInvocations: 1,
+        };
+    }
+    catch (e) {
+        return {
+            verdict: "indeterminate",
+            reasonCode: "JUDGE_ERROR",
+            summary: `Judge error: ${e instanceof Error ? e.message : String(e)}`,
+        };
+    }
+}
+//# sourceMappingURL=evaluate-criteria.js.map

package/dist/evaluate-criteria.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate-criteria.js","sourceRoot":"","sources":["../src/evaluate-criteria.ts"],"names":[],"mappings":"AAYA,SAAS,YAAY,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,IAAU,EACV,QAAyB,EACzB,OAEC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxC,OAAO;oBACL,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,cAAc;oBAC1B,OAAO,EAAE,yBAAyB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;iBACjF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACZ,OAAO;oBACL,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,oBAAoB;oBAChC,OAAO,EAAE,iCAAiC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE;iBAChE,CAAC;YACJ,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO;oBACL,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,iBAAiB;oBAC7B,OAAO,EAAE,YAAY,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4BAA4B;iBACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAEnC,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1E,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,gBAAgB;oBAC5B,OAAO,EAAE,iBAAiB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU;iBACpE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBAClC,OAAO;oBACL,OAAO,EAAE,QAAQ;oBACjB,UAAU,EAAE,iBAAiB;oBAC7B,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE;iBAClE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,gBAAgB,KAAK,SAAS,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpF,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,OAAO,EAAE,eAAe;oBACxB,UAAU,EAAE,yBAAyB;oBACrC,OAAO,EAAE,qCAAqC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE;iBAC1F,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrF,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC5C,OAAO;wBACL,OAAO,EAAE,QAAQ;wBACjB,UAAU,EAAE,qBAAqB;wBACjC,OAAO,EAAE,oBAAoB,KAAK,CAAC,EAAE,IAAI,MAAM,CAAC,QAAQ,+BAA+B;qBACxF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,OAAO;gBACL,OAAO,EAAE,eAAe;gBACxB,UAAU,EAAE,gBAAgB;gBAC5B,OAAO,EAAE,+DAA+D;aACzE,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACnE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,OAAO;YACL,OAAO,EAAE,eAAe;YACxB,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE,+BAA+B;SACzC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;AACpF,CAAC;AAED,SAAS,eAAe,CAAC,CAAkB;IACzC,OAAO,CACL,CAAC,CAAC,UAAU,IAAI,IAAI;QACpB,CAAC,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC,gBAAgB,IAAI,IAAI,IAAI,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7D,CAAC,CAAC,WAAW,IAAI,IAAI,CACtB,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IACrE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,MAAgB;IACxD,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAgC;IAC9D,IAAI,CAAC;QACH,OAAO,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC;AACjD,CAAC;AAED,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,KAAK,UAAU,kBAAkB,CAAC,IAAU,EAAE,MAAmB;IAC/D,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,IAAI,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACjF,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACzF,OAAO,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,sBAAsB,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAChF,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,sBAAsB,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,YAAY,CAAC,CAAS;IAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,yBAAyB,EAAE,mBAAmB,CAAC,CAAC,OAAO,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AACxH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,IAAU,EACV,QAAyB,EACzB,KAAoB;IAEpB,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IACvE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,KAAK,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,2BAA2B,CAAC;IACvF,MAAM,GAAG,GAAG,GAAG,IAAI,mBAAmB,CAAC;IAEvC,MAAM,MAAM,GACV,oJAAoJ,CAAC;IACvJ,MAAM,IAAI,GAAG,cAAc,MAAM,CAAC,EAAE,cAAc,MAAM,CAAC,OAAO,IAAI,QAAQ,mBAAmB,MAAM,EAAE,CAAC;IAExG,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE;aACpD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,KAAK,CAAC,OAAO;gBACpB,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;oBACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;iBAChC;gBACD,UAAU,EAAE,EAAE;gBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;aACzC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC3C,OAAO;gBACL,OAAO,EAAE,eAAe;gBACxB,UAAU,EAAE,kBAAkB;gBAC9B,OAAO,EAAE,8BAA8B,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;gBACpF,qBAAqB,EAAE,CAAC;aACzB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4D,CAAC;QAC3F,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC;QACxD,IAAI,MAA4B,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,eAAe;gBACxB,UAAU,EAAE,mBAAmB;gBAC/B,OAAO,EAAE,mCAAmC;gBAC5C,qBAAqB,EAAE,CAAC;aACzB,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/C,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,QAAQ;gBACjB,UAAU,EAAE,YAAY;gBACxB,OAAO,EAAE,qBAAqB;gBAC9B,qBAAqB,EAAE,CAAC;aACzB,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,QAAQ;gBACjB,UAAU,EAAE,YAAY;gBACxB,OAAO,EAAE,qBAAqB;gBAC9B,qBAAqB,EAAE,CAAC;aACzB,CAAC;QACJ,CAAC;QACD,OAAO;YACL,OAAO,EAAE,eAAe;YACxB,UAAU,EAAE,iBAAiB;YAC7B,OAAO,EAAE,0BAA0B;YACnC,qBAAqB,EAAE,CAAC;SACzB,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,eAAe;YACxB,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE,gBAAgB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;SACtE,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * @restormel/testing-runner — local suite execution (MVP).
+ */
+export { runBrowserGoal } from "./browser-goal.js";
+export type { RunBrowserGoalOptions, RunBrowserGoalResult } from "./browser-goal.js";
+export { evaluateBrowserSuccessCriteria } from "./evaluate-criteria.js";
+export type { CriteriaEvaluation } from "./evaluate-criteria.js";
+export { runGoalAttempts } from "./retries.js";
+export type { AttemptOutcome } from "./retries.js";
+export { runLocalSuite, runSuiteFromConfig } from "./run-suite.js";
+export type { RunLocalSuiteOptions, RunSuiteExecutionOptions, RunSuiteResult } from "./types.js";
+export { withTimeout, TimeoutError } from "./timeout.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,8BAA8B,EAAE,MAAM,wBAAwB,CAAC;AACxE,YAAY,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACnE,YAAY,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjG,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,9 @@
+/**
+ * @restormel/testing-runner — local suite execution (MVP).
+ */
+export { runBrowserGoal } from "./browser-goal.js";
+export { evaluateBrowserSuccessCriteria } from "./evaluate-criteria.js";
+export { runGoalAttempts } from "./retries.js";
+export { runLocalSuite, runSuiteFromConfig } from "./run-suite.js";
+export { withTimeout, TimeoutError } from "./timeout.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,8BAA8B,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}

package/dist/judge-ref.d.ts

@@ -0,0 +1,7 @@
+import type { SuccessCriteria } from "@restormel/testing-core";
+/**
+ * Logical ref for Keys resolution when a goal uses `judge_rubric`.
+ * Order: explicit `model_ref` on the rubric, then common slots on the merged environment keys map.
+ */
+export declare function judgeLogicalRefForCriteria(criteria: SuccessCriteria, resolvedKeys: Record<string, string>): string | undefined;
+//# sourceMappingURL=judge-ref.d.ts.map

package/dist/judge-ref.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-ref.d.ts","sourceRoot":"","sources":["../src/judge-ref.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,eAAe,EACzB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACnC,MAAM,GAAG,SAAS,CAWpB"}

package/dist/judge-ref.js

@@ -0,0 +1,17 @@
+/**
+ * Logical ref for Keys resolution when a goal uses `judge_rubric`.
+ * Order: explicit `model_ref` on the rubric, then common slots on the merged environment keys map.
+ */
+export function judgeLogicalRefForCriteria(criteria, resolvedKeys) {
+    const jr = criteria.judgeRubric;
+    if (!jr)
+        return undefined;
+    if (jr.modelRef)
+        return jr.modelRef;
+    return (resolvedKeys.llm_primary ??
+        resolvedKeys.llmPrimary ??
+        resolvedKeys.judge ??
+        resolvedKeys.llm_judge ??
+        resolvedKeys.llmJudge);
+}
+//# sourceMappingURL=judge-ref.js.map

package/dist/judge-ref.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-ref.js","sourceRoot":"","sources":["../src/judge-ref.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CACxC,QAAyB,EACzB,YAAoC;IAEpC,MAAM,EAAE,GAAG,QAAQ,CAAC,WAAW,CAAC;IAChC,IAAI,CAAC,EAAE;QAAE,OAAO,SAAS,CAAC;IAC1B,IAAI,EAAE,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC,QAAQ,CAAC;IACpC,OAAO,CACL,YAAY,CAAC,WAAW;QACxB,YAAY,CAAC,UAAU;QACvB,YAAY,CAAC,KAAK;QAClB,YAAY,CAAC,SAAS;QACtB,YAAY,CAAC,QAAQ,CACtB,CAAC;AACJ,CAAC"}

package/dist/retries.d.ts

@@ -0,0 +1,38 @@
+import type { Verdict } from "@restormel/testing-core";
+export type AttemptOutcome = {
+    kind: "stop";
+    verdict: "passed";
+    reasonCode: string;
+    summary: string;
+} | {
+    kind: "stop";
+    verdict: "indeterminate";
+    reasonCode: string;
+    summary: string;
+} | {
+    kind: "stop";
+    verdict: "failed";
+    reasonCode: string;
+    summary: string;
+    retryable: false;
+} | {
+    kind: "retry";
+    verdict: "failed";
+    reasonCode: string;
+    summary: string;
+    retryable: true;
+};
+/**
+ * `maxRetries` = extra attempts after the first (e.g. 2 → 3 total tries).
+ */
+export declare function runGoalAttempts(options: {
+    maxRetries: number;
+    backoffMs?: number;
+    runAttempt: (attemptIndex: number) => Promise<AttemptOutcome>;
+}): Promise<{
+    verdict: Verdict;
+    reasonCode: string;
+    summary: string;
+    retriesUsed: number;
+}>;
+//# sourceMappingURL=retries.d.ts.map

package/dist/retries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retries.d.ts","sourceRoot":"","sources":["../src/retries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,eAAe,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,KAAK,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,CAAC;AAE/F;;GAEG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;CAC/D,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC,CAoC1F"}

package/dist/retries.js

@@ -0,0 +1,40 @@
+/**
+ * `maxRetries` = extra attempts after the first (e.g. 2 → 3 total tries).
+ */
+export async function runGoalAttempts(options) {
+    const maxAttempts = Math.max(1, options.maxRetries + 1);
+    let lastFail;
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        if (attempt > 0 && options.backoffMs !== undefined && options.backoffMs > 0) {
+            await sleep(options.backoffMs);
+        }
+        const out = await options.runAttempt(attempt);
+        if (out.kind === "stop") {
+            return {
+                verdict: out.verdict,
+                reasonCode: out.reasonCode,
+                summary: out.summary,
+                retriesUsed: attempt,
+            };
+        }
+        lastFail = { reasonCode: out.reasonCode, summary: out.summary };
+        if (!out.retryable || attempt === maxAttempts - 1) {
+            return {
+                verdict: "failed",
+                reasonCode: out.reasonCode,
+                summary: out.summary,
+                retriesUsed: attempt,
+            };
+        }
+    }
+    return {
+        verdict: "failed",
+        reasonCode: lastFail?.reasonCode ?? "UNKNOWN",
+        summary: lastFail?.summary ?? "Goal failed",
+        retriesUsed: maxAttempts - 1,
+    };
+}
+function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+//# sourceMappingURL=retries.js.map

package/dist/retries.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retries.js","sourceRoot":"","sources":["../src/retries.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAIrC;IACC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACxD,IAAI,QAA6D,CAAC;IAElE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YAC5E,MAAM,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,WAAW,EAAE,OAAO;aACrB,CAAC;QACJ,CAAC;QAED,QAAQ,GAAG,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,OAAO,KAAK,WAAW,GAAG,CAAC,EAAE,CAAC;YAClD,OAAO;gBACL,OAAO,EAAE,QAAQ;gBACjB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,WAAW,EAAE,OAAO;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,QAAQ;QACjB,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,SAAS;QAC7C,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,aAAa;QAC3C,WAAW,EAAE,WAAW,GAAG,CAAC;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/run-suite.d.ts

@@ -0,0 +1,10 @@
+import type { RunLocalSuiteOptions, RunSuiteExecutionOptions, RunSuiteResult } from "./types.js";
+/**
+ * Execute one suite from an already-loaded config (local CLI / programmatic use).
+ */
+export declare function runSuiteFromConfig(options: RunSuiteExecutionOptions): Promise<RunSuiteResult>;
+/**
+ * Load config from disk, then run one suite.
+ */
+export declare function runLocalSuite(options: RunLocalSuiteOptions): Promise<RunSuiteResult>;
+//# sourceMappingURL=run-suite.d.ts.map

package/dist/run-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-suite.d.ts","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAqBjG;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqInG;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CA0B1F"}

package/dist/run-suite.js

@@ -0,0 +1,179 @@
+import { randomUUID } from "node:crypto";
+import { formatConfigErrors, isSafeHttpUrl, loadConfigFromFile, resolvedKeysForEnvironment, resolveEnvironmentProfile, resolveSuite, } from "@restormel/testing-config";
+import { resolve } from "node:path";
+import { runBrowserGoal } from "./browser-goal.js";
+import { resolveStorageStatePath } from "./storage-state.js";
+function aggregateVerdict(goalVerdicts) {
+    if (goalVerdicts.some((v) => v === "failed"))
+        return "failed";
+    if (goalVerdicts.some((v) => v === "indeterminate"))
+        return "indeterminate";
+    return "passed";
+}
+function mergeKeysMeta(fragments) {
+    const byRef = new Map();
+    for (const f of fragments) {
+        const prev = byRef.get(f.logicalRef);
+        if (!prev) {
+            byRef.set(f.logicalRef, { ...f });
+        }
+        else {
+            prev.invocationCount = (prev.invocationCount ?? 0) + (f.invocationCount ?? 0);
+        }
+    }
+    return [...byRef.values()];
+}
+/**
+ * Execute one suite from an already-loaded config (local CLI / programmatic use).
+ */
+export async function runSuiteFromConfig(options) {
+    const traces = [];
+    const warnings = [];
+    const errors = [];
+    const startedAt = new Date().toISOString();
+    const suiteRes = resolveSuite(options.config, options.suiteId);
+    if (!suiteRes.ok) {
+        return { ok: false, errors: [formatConfigErrors(suiteRes.errors)], traces, warnings };
+    }
+    const suite = suiteRes.suite;
+    const envId = options.environmentId ?? suite.environment;
+    const envRes = resolveEnvironmentProfile(options.config, envId);
+    if (!envRes.ok) {
+        return { ok: false, errors: [formatConfigErrors(envRes.errors)], traces, warnings };
+    }
+    let profile = envRes.profile;
+    const override = options.targetUrlOverride?.trim();
+    if (override !== undefined && override.length > 0) {
+        const urlCheck = isSafeHttpUrl(override);
+        if (!urlCheck.ok) {
+            return {
+                ok: false,
+                errors: [`Invalid target_url override: ${urlCheck.reason}`],
+                traces,
+                warnings,
+            };
+        }
+        profile = { ...profile, baseUrl: override };
+    }
+    let goals = suite.goals;
+    if (options.goalIds !== undefined && options.goalIds.length > 0) {
+        const suiteGoalIds = new Set(suite.goals.map((g) => g.id));
+        const missing = options.goalIds.filter((id) => !suiteGoalIds.has(id));
+        if (missing.length > 0) {
+            errors.push(`Unknown goal id(s) for suite "${suite.id}": ${missing.join(", ")}`);
+            return { ok: false, errors, traces, warnings };
+        }
+        const pick = new Set(options.goalIds);
+        goals = suite.goals.filter((g) => pick.has(g.id));
+    }
+    const resolvedKeys = resolvedKeysForEnvironment(options.config, envId);
+    const storageStatePath = options.configFilePath !== undefined
+        ? resolveStorageStatePath(profile, options.configFilePath)
+        : undefined;
+    const retryPolicy = suite.retryPolicy ?? { maxRetries: 0 };
+    const timeoutMs = suite.defaultTimeoutMs ?? options.config.defaults?.defaultTimeoutMs ?? 30_000;
+    const artifactPolicy = suite.artifactPolicy ?? options.config.defaults?.artifactPolicy;
+    const shots = artifactPolicy?.screenshots ?? "on_failure";
+    const captureScreenshotOnFailure = shots === "on_failure" || shots === "always";
+    const runId = randomUUID();
+    const trigger = options.trigger ?? "local";
+    const goalRuns = [];
+    let stepIndex = 0;
+    const allKeysFragments = [];
+    for (const goal of goals) {
+        if (goal.type !== "browser") {
+            goalRuns.push({
+                goalId: goal.id,
+                verdict: "indeterminate",
+                reasonCode: "GOAL_TYPE_UNSUPPORTED",
+                summary: `Goal type "${goal.type}" is not supported by the local runner (browser only in this slice)`,
+                retriesUsed: 0,
+                evidenceRefs: [],
+            });
+            continue;
+        }
+        const bg = await runBrowserGoal({
+            runId,
+            goal,
+            baseUrl: profile.baseUrl,
+            timeoutMs,
+            retryPolicy,
+            artifactDir: options.artifactDir,
+            captureScreenshotOnFailure,
+            headless: options.headless,
+            storageStatePath,
+            createBrowserSession: options.createBrowserSession,
+            resolvedKeys,
+            keysAdapterOptions: options.keysAdapterOptions,
+            startingStepIndex: stepIndex,
+        });
+        stepIndex = bg.nextStepIndex;
+        traces.push(...bg.traces);
+        warnings.push(...bg.warnings);
+        goalRuns.push(bg.goalRecord);
+        allKeysFragments.push(...bg.keysModelMetaFragments);
+    }
+    const verdict = aggregateVerdict(goalRuns.map((g) => g.verdict));
+    const mergedMeta = mergeKeysMeta(allKeysFragments);
+    const judgeInvocations = mergedMeta.reduce((n, m) => n + (m.invocationCount ?? 0), 0);
+    let costEstimate;
+    if (judgeInvocations > 0) {
+        costEstimate = {
+            tokenEstimate: { input: 0, output: judgeInvocations * 120 },
+        };
+    }
+    const run = {
+        id: runId,
+        suiteId: suite.id,
+        environmentId: envId,
+        trigger,
+        commitSha: options.commitSha,
+        repository: options.repository,
+        startedAt,
+        endedAt: new Date().toISOString(),
+        verdict,
+        goalRuns,
+        keysModelMeta: mergedMeta.length > 0 ? mergedMeta : undefined,
+        judgeInvocationCount: judgeInvocations,
+        costEstimate,
+    };
+    const suiteMeta = {
+        id: suite.id,
+        description: suite.description,
+        tags: suite.tags,
+        environmentId: envId,
+        goalCount: goals.length,
+    };
+    return { ok: true, run, traces, warnings, errors, suiteMeta };
+}
+/**
+ * Load config from disk, then run one suite.
+ */
+export async function runLocalSuite(options) {
+    const loaded = await loadConfigFromFile(options.configPath);
+    if (!loaded.ok) {
+        return {
+            ok: false,
+            errors: [formatConfigErrors(loaded.errors)],
+            traces: [],
+            warnings: [],
+        };
+    }
+    const configFilePath = resolve(process.cwd(), options.configPath);
+    return runSuiteFromConfig({
+        config: loaded.config,
+        suiteId: options.suiteId,
+        environmentId: options.environmentId,
+        targetUrlOverride: options.targetUrlOverride,
+        goalIds: options.goalIds,
+        trigger: options.trigger,
+        commitSha: options.commitSha,
+        repository: options.repository,
+        artifactDir: options.artifactDir,
+        keysAdapterOptions: options.keysAdapterOptions,
+        headless: options.headless,
+        createBrowserSession: options.createBrowserSession,
+        configFilePath,
+    });
+}
+//# sourceMappingURL=run-suite.js.map

package/dist/run-suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-suite.js","sourceRoot":"","sources":["../src/run-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,kBAAkB,EAClB,0BAA0B,EAC1B,yBAAyB,EACzB,YAAY,GACb,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAG7D,SAAS,gBAAgB,CAAC,YAAuB;IAC/C,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC9D,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,eAAe,CAAC;QAAE,OAAO,eAAe,CAAC;IAC5E,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,SAA0B;IAC/C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,eAAe,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAiC;IACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACxF,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;IAE7B,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC,WAAW,CAAC;IACzD,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACtF,CAAC;IACD,IAAI,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAE7B,MAAM,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC;IACnD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,CAAC,gCAAgC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC3D,MAAM;gBACN,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IACxB,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,iCAAiC,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACjD,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,YAAY,GAAG,0BAA0B,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACvE,MAAM,gBAAgB,GACpB,OAAO,CAAC,cAAc,KAAK,SAAS;QAClC,CAAC,CAAC,uBAAuB,CAAC,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC;QAC1D,CAAC,CAAC,SAAS,CAAC;IAEhB,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IAC3D,MAAM,SAAS,GAAG,KAAK,CAAC,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,gBAAgB,IAAI,MAAM,CAAC;IAChG,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC;IACvF,MAAM,KAAK,GAAG,cAAc,EAAE,WAAW,IAAI,YAAY,CAAC;IAC1D,MAAM,0BAA0B,GAAG,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,QAAQ,CAAC;IAEhF,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC;IAC3C,MAAM,QAAQ,GAA0B,EAAE,CAAC;IAC3C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,gBAAgB,GAAoB,EAAE,CAAC;IAE7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,OAAO,EAAE,eAAe;gBACxB,UAAU,EAAE,uBAAuB;gBACnC,OAAO,EAAE,cAAc,IAAI,CAAC,IAAI,qEAAqE;gBACrG,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,EAAE;aACjB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC;YAC9B,KAAK;YACL,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,SAAS;YACT,WAAW;YACX,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,0BAA0B;YAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,gBAAgB;YAChB,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;YAClD,YAAY;YACZ,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;YAC9C,iBAAiB,EAAE,SAAS;SAC7B,CAAC,CAAC;QACH,SAAS,GAAG,EAAE,CAAC,aAAa,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QAC7B,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;IACnD,MAAM,gBAAgB,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtF,IAAI,YAAsC,CAAC;IAC3C,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,YAAY,GAAG;YACb,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,GAAG,GAAG,EAAE;SAC5D,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAc;QACrB,EAAE,EAAE,KAAK;QACT,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,aAAa,EAAE,KAAK;QACpB,OAAO;QACP,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,SAAS;QACT,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACjC,OAAO;QACP,QAAQ;QACR,aAAa,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC7D,oBAAoB,EAAE,gBAAgB;QACtC,YAAY;KACb,CAAC;IAEF,MAAM,SAAS,GAAG;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,aAAa,EAAE,KAAK;QACpB,SAAS,EAAE,KAAK,CAAC,MAAM;KACxB,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA6B;IAC/D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC5D,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,CAAC,kBAAkB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,EAAE;SACb,CAAC;IACJ,CAAC;IACD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAClE,OAAO,kBAAkB,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;QAClD,cAAc;KACf,CAAC,CAAC;AACL,CAAC"}

package/dist/storage-state.d.ts

@@ -0,0 +1,8 @@
+import type { EnvironmentProfile } from "@restormel/testing-core";
+/**
+ * Resolve Playwright storage state path for `auth_mode: storage_state`.
+ * - `auth_ref: env:VAR` — path read from `process.env.VAR` (trimmed).
+ * - Otherwise — path relative to the config file directory, or absolute if `auth_ref` is absolute.
+ */
+export declare function resolveStorageStatePath(profile: EnvironmentProfile, configFilePath: string): string | undefined;
+//# sourceMappingURL=storage-state.d.ts.map

package/dist/storage-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-state.d.ts","sourceRoot":"","sources":["../src/storage-state.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAElE;;;;GAIG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,kBAAkB,EAC3B,cAAc,EAAE,MAAM,GACrB,MAAM,GAAG,SAAS,CAapB"}

package/dist/storage-state.js

@@ -0,0 +1,22 @@
+import { dirname, isAbsolute, join } from "node:path";
+/**
+ * Resolve Playwright storage state path for `auth_mode: storage_state`.
+ * - `auth_ref: env:VAR` — path read from `process.env.VAR` (trimmed).
+ * - Otherwise — path relative to the config file directory, or absolute if `auth_ref` is absolute.
+ */
+export function resolveStorageStatePath(profile, configFilePath) {
+    if (profile.authMode !== "storage_state" || profile.authRef === undefined || profile.authRef === "") {
+        return undefined;
+    }
+    const ref = profile.authRef.trim();
+    if (ref.startsWith("env:")) {
+        const varName = ref.slice("env:".length).trim();
+        if (!/^[A-Z][A-Z0-9_]*$/.test(varName))
+            return undefined;
+        const p = process.env[varName]?.trim();
+        return p && p.length > 0 ? p : undefined;
+    }
+    const base = dirname(configFilePath);
+    return isAbsolute(ref) ? ref : join(base, ref);
+}
+//# sourceMappingURL=storage-state.js.map

package/dist/storage-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-state.js","sourceRoot":"","sources":["../src/storage-state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGtD;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAA2B,EAC3B,cAAsB;IAEtB,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,KAAK,EAAE,EAAE,CAAC;QACpG,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,SAAS,CAAC;QACzD,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3C,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACrC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACjD,CAAC"}

package/dist/timeout.d.ts

@@ -0,0 +1,6 @@
+export declare class TimeoutError extends Error {
+    readonly code: "TIMEOUT";
+    constructor(message: string);
+}
+export declare function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T>;
+//# sourceMappingURL=timeout.d.ts.map

package/dist/timeout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timeout.d.ts","sourceRoot":"","sources":["../src/timeout.ts"],"names":[],"mappings":"AAAA,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,IAAI,EAAG,SAAS,CAAU;gBACvB,OAAO,EAAE,MAAM;CAI5B;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAmBzF"}

package/dist/timeout.js

@@ -0,0 +1,25 @@
+export class TimeoutError extends Error {
+    code = "TIMEOUT";
+    constructor(message) {
+        super(message);
+        this.name = "TimeoutError";
+    }
+}
+export function withTimeout(promise, ms, label) {
+    if (!Number.isFinite(ms) || ms <= 0) {
+        return promise;
+    }
+    return new Promise((resolve, reject) => {
+        const t = setTimeout(() => {
+            reject(new TimeoutError(`${label} exceeded ${ms}ms`));
+        }, ms);
+        promise.then((v) => {
+            clearTimeout(t);
+            resolve(v);
+        }, (e) => {
+            clearTimeout(t);
+            reject(e);
+        });
+    });
+}
+//# sourceMappingURL=timeout.js.map

package/dist/timeout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"timeout.js","sourceRoot":"","sources":["../src/timeout.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAa,SAAQ,KAAK;IAC5B,IAAI,GAAG,SAAkB,CAAC;IACnC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,UAAU,WAAW,CAAI,OAAmB,EAAE,EAAU,EAAE,KAAa;IAC3E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE;YACxB,MAAM,CAAC,IAAI,YAAY,CAAC,GAAG,KAAK,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;QACxD,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,OAAO,CAAC,IAAI,CACV,CAAC,CAAC,EAAE,EAAE;YACJ,YAAY,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,CAAC,EACD,CAAC,CAAC,EAAE,EAAE;YACJ,YAAY,CAAC,CAAC,CAAC,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,57 @@
+import type { KeysModelAdapterOptions } from "@restormel/testing-keys-adapter";
+import type { PlaywrightTestingSessionOptions } from "@restormel/testing-browser-playwright";
+import type { RestormelTestingConfig } from "@restormel/testing-config";
+import type { RunRecord, RunTrigger, SuiteReportSlice, TraceEvent } from "@restormel/testing-core";
+import type { TestingBrowserSession } from "@restormel/testing-browser-playwright";
+export interface RunLocalSuiteOptions {
+    /** Path to restormel-testing.yaml / .json */
+    configPath: string;
+    suiteId: string;
+    /** Defaults to the suite’s configured environment. */
+    environmentId?: string;
+    /**
+     * When set, replaces the resolved environment `base_url` (e.g. CI preview deploy URL).
+     * Must be a safe http(s) URL with no embedded credentials.
+     */
+    targetUrlOverride?: string;
+    goalIds?: string[];
+    trigger?: RunTrigger;
+    commitSha?: string;
+    repository?: string;
+    /** Directory for screenshots and future artefacts (created as needed). */
+    artifactDir?: string;
+    keysAdapterOptions?: KeysModelAdapterOptions;
+    headless?: boolean;
+    /**
+     * Test / advanced: override browser session creation (e.g. mocks).
+     * Default: Playwright Chromium via `createPlaywrightTestingSession`.
+     */
+    createBrowserSession?: (opts?: PlaywrightTestingSessionOptions) => Promise<TestingBrowserSession>;
+}
+export interface RunSuiteExecutionOptions {
+    config: RestormelTestingConfig;
+    suiteId: string;
+    environmentId?: string;
+    targetUrlOverride?: string;
+    goalIds?: string[];
+    trigger?: RunTrigger;
+    commitSha?: string;
+    repository?: string;
+    artifactDir?: string;
+    keysAdapterOptions?: KeysModelAdapterOptions;
+    headless?: boolean;
+    createBrowserSession?: (opts?: PlaywrightTestingSessionOptions) => Promise<TestingBrowserSession>;
+    configFilePath?: string;
+}
+export interface RunSuiteResult {
+    /** False when config could not be loaded or suite/environment invalid. */
+    ok: boolean;
+    run?: RunRecord;
+    traces: TraceEvent[];
+    warnings: string[];
+    /** Human-readable failures (config path, validation, missing env). */
+    errors: string[];
+    /** Present when `ok` and the suite was resolved (for report generators). */
+    suiteMeta?: SuiteReportSlice;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC/E,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACxE,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACnG,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uCAAuC,CAAC;AAEnF,MAAM,WAAW,oBAAoB;IACnC,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,uBAAuB,CAAC;IAC7C,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,CACrB,IAAI,CAAC,EAAE,+BAA+B,KACnC,OAAO,CAAC,qBAAqB,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,sBAAsB,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,uBAAuB,CAAC;IAC7C,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,oBAAoB,CAAC,EAAE,CACrB,IAAI,CAAC,EAAE,+BAA+B,KACnC,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,0EAA0E;IAC1E,EAAE,EAAE,OAAO,CAAC;IACZ,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,sEAAsE;IACtE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,gBAAgB,CAAC;CAC9B"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@restormel/testing-runner",
+  "version": "0.1.0",
+  "description": "Suite execution: browser goals, Keys-backed judges, retries, artefacts.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Allotment-Technology-Ltd/restormel-testing.git",
+    "directory": "packages/runner"
+  },
+  "bugs": {
+    "url": "https://github.com/Allotment-Technology-Ltd/restormel-testing/issues"
+  },
+  "homepage": "https://github.com/Allotment-Technology-Ltd/restormel-testing/tree/main/packages/runner#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "playwright": "^1.49.0",
+    "@restormel/testing-browser-playwright": "0.1.0",
+    "@restormel/testing-config": "0.1.0",
+    "@restormel/testing-core": "0.1.0",
+    "@restormel/testing-keys-adapter": "0.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.0"
+  },
+  "scripts": {
+    "build": "tsc -b tsconfig.json",
+    "typecheck": "tsc -b tsconfig.json"
+  }
+}