@jevitate/cli 0.1.0 → 0.2.0

package/dist/multi-run.d.ts

@@ -0,0 +1,339 @@
+import { type GoalOutcome, type MissionOutcome } from "@jevitate/domain";
+import { type UsageAggregate, type UsageCounts } from "@jevitate/ai-core";
+import { type EngineInfo } from "./engine.js";
+/**
+ * Multi-run orchestration over the existing explore strategies (#141 repeat-and-vote, #143 persona
+ * matrix). One orchestrator runs the SAME mission N times — once per persona (storageState) and
+ * `--repeat` times within each — strictly SEQUENTIALLY (never concurrently: conversational and
+ * stateful missions share server-side state), each run in a fresh browser context (every run is a
+ * separate mission, which opens its own context). It then aggregates the runs with code only:
+ *
+ *  - repeat-and-vote (#141): a finding is kept when it recurs in ≥ k of N runs; the rest are
+ *    reported as `flaky` (seen, not counted). The overall outcome is the agreed one (the most
+ *    common per-run outcome, when it was reached by ≥ k runs), else `intermittent`.
+ *  - persona matrix (#143): per-persona outcome, findings, requests (method + templated path →
+ *    status) and visible controls, plus an ADVISORY diff: requests one persona made that another
+ *    did not, status differences for the same request (a 401/403 against a 2xx is a candidate RBAC
+ *    finding), controls visible to one persona only, and each persona's outcome.
+ *
+ * Storage-state CONTENTS are never read, logged or copied here — only their paths travel.
+ */
+export declare class MultiRunArgsError extends Error {
+    readonly code: "E_EXPLORE_ARGS";
+    constructor(message: string);
+}
+export interface Persona {
+    readonly name: string;
+    /** Absolute path of the persona's Playwright storageState file (its contents are never read). */
+    readonly storageState: string;
+}
+export interface MultiRunPlan {
+    /** Runs per persona (or per mission, without personas). */
+    readonly repeat: number;
+    /** A finding (and the outcome) counts when seen in at least this many of `repeat` runs. */
+    readonly minAgreement: number;
+    /** The persona matrix; `null` for a plain `--repeat`. */
+    readonly personas: readonly Persona[] | null;
+}
+/** Upper bound on `--repeat`: each run is a whole mission, run one after another. */
+export declare const MAX_REPEAT = 20;
+/** `--persona <name>=<storageState>`: the path is resolved against the working directory. */
+export declare function parsePersonaSpec(spec: string, cwd?: string): Persona;
+/**
+ * `--actor <name>=<storageState>` (#147): the same `<name>=<storageState>` shape and checks as a
+ * persona (the file must exist; its contents are never read here).
+ */
+export declare function parseActorSpec(spec: string, cwd?: string): Persona;
+/**
+ * `--personas <file>`: JSON, either `{"admin": "admin.json", "sales": "sales.json"}` or
+ * `{"personas": [{"name": "admin", "storageState": "admin.json"}]}` (a bare array also works).
+ * Relative paths resolve against the personas file's own directory.
+ */
+export declare function loadPersonasFile(path: string): Persona[];
+export interface MultiRunFlags {
+    readonly repeat?: string;
+    readonly minAgreement?: string;
+    readonly persona: readonly string[];
+    readonly personas?: string;
+    readonly storageState?: string;
+    readonly saveStorageState?: string;
+}
+/** Whether the flags ask for a multi-run at all. */
+export declare function wantsMultiRun(f: MultiRunFlags): boolean;
+/** Validates the multi-run flags (before any browser opens). Default k = a strict majority of N. */
+export declare function resolveMultiRunPlan(f: MultiRunFlags, cwd?: string): MultiRunPlan;
+/** One finding as a run reported it, reduced to what cross-run matching needs. */
+export interface RunFinding {
+    /** defect / invariant / hang / advisory / coverage-defect / ux (plus the defect's own kind). */
+    readonly kind: string;
+    /** The engine's own stable fingerprint, when the finding has one. */
+    readonly fingerprint?: string;
+    /** Templated route it was (first) seen on. */
+    readonly route: string;
+    readonly title: string;
+    /** For a UX finding: its rubric item and implicated controls (a UX finding has no fingerprint). */
+    readonly rubricItemId?: string;
+    readonly controls?: readonly string[];
+}
+/**
+ * THE cross-run finding identity (#141). Kept deliberately small and self-contained so it can be
+ * swapped for the shared finding-identity model of `jevitate report`/`diff`/`check` (#137–#139):
+ * the engine's own fingerprint (defect / hang / invariant / signal) plus the templated route; a UX
+ * finding (no fingerprint) is its rubric item + route + implicated controls.
+ */
+export declare function findingIdentity(f: RunFinding): string;
+/** Every finding a run's result carries, whatever its strategy. */
+export declare function extractRunFindings(data: unknown): RunFinding[];
+/**
+ * #226: a run's canonical verdict (the #217 contract): its own `missionOutcome`; else a goal run's
+ * ending folded (`GOAL_OUTCOME_FOLD`), or another strategy's `outcome` when it already is one; else
+ * `crashed` — a run that says nothing readable proves nothing.
+ */
+export declare function runMissionOutcomeOf(strategy: string, data: unknown): MissionOutcome;
+/** `METHOD /templated/path` → the distinct response statuses seen (assets excluded; pending dropped). */
+export declare function extractRequests(data: unknown): Record<string, number[]>;
+/** Every control identity the run's transcript saw (see `TranscriptEntry.controls`). */
+export declare function extractControls(transcript: unknown): string[];
+/** What one run contributed. */
+export interface RunSummary {
+    /** 1-based, within its persona. */
+    readonly index: number;
+    /** False when the run could not start or broke outside the mission (its envelope was an error). */
+    readonly ok: boolean;
+    /** The run's own ending: a goal run's `goalOutcome`, else its canonical outcome. */
+    readonly outcome: string;
+    /** #226: the run's canonical verdict (#217) — what the vote counts. */
+    readonly missionOutcome: MissionOutcome;
+    /** #226: a goal run's own ending (#217), beside `missionOutcome`. */
+    readonly goalOutcome?: GoalOutcome;
+    readonly exitCode: number;
+    readonly findings: readonly RunFinding[];
+    readonly requests: Readonly<Record<string, readonly number[]>>;
+    readonly controls: readonly string[];
+    /** The run's own persisted result (or report), when it wrote one. */
+    readonly resultPath?: string;
+    /** The run's envelope, as written next to its artifacts. */
+    readonly envelopePath?: string;
+    readonly error?: {
+        readonly code: string;
+        readonly message: string;
+    };
+    /** #220: why the run ended the way it did, when its result says (a crash's failure, a stop reason). */
+    readonly reason?: string;
+    /** The run's model usage (#163), as its envelope reported it. */
+    readonly usage?: UsageCounts;
+    /** #213: the run's `failure.kind` (or its crash's), when it had one. */
+    readonly failureKind?: string;
+    /** #226: a find-out run's answer (`{text, evidence}`), as its result carried it. */
+    readonly answer?: unknown;
+    /** #213: why the run's storage-state session (a persona's) was not honoured, when it was not. */
+    readonly sessionLost?: string;
+}
+export interface AggregatedFinding {
+    readonly id: string;
+    readonly kind: string;
+    readonly fingerprint?: string;
+    readonly route: string;
+    readonly title: string;
+    /** `seen/N`, e.g. `2/3`. */
+    readonly stability: string;
+    readonly seen: number;
+    readonly of: number;
+    /** Which runs (1-based) saw it. */
+    readonly runs: number[];
+    /** `agreed`: seen in ≥ k runs (counted). `flaky`: seen in fewer — reported, never counted. */
+    readonly status: "agreed" | "flaky";
+}
+export interface CellResult {
+    readonly persona: string | null;
+    readonly storageStatePath?: string;
+    /**
+     * The agreed outcome; else `intermittent` when the finished runs reached different VERDICTS — or
+     * `inconclusive` (#220) when they did not agree because a run broke (crashed/inconclusive: a lost
+     * browser proves nothing about the app) or runs are still missing. `reason` then says which.
+     */
+    readonly outcome: string;
+    /**
+     * #226: the canonical verdict (#217) the runs agreed on — voted over each run's `missionOutcome`, so
+     * goal runs that failed in different ways (exhausted, blocked) still agree they found a defect.
+     */
+    readonly missionOutcome: MissionOutcome;
+    /** #226: goal runs only — the goal ending ≥ k runs agreed on, else the canonical outcome (#217). */
+    readonly goalOutcome?: GoalOutcome;
+    /** #220: why the outcome is `inconclusive` (which runs broke, or how many are missing). */
+    readonly reason?: string;
+    /** The exit code of `missionOutcome`. */
+    readonly exitCode: number;
+    /** How many runs ended in each (own) outcome. */
+    readonly outcomes: Readonly<Record<string, number>>;
+    readonly runs: readonly RunSummary[];
+    /** Agreed findings (≥ k runs), most stable first. */
+    readonly findings: AggregatedFinding[];
+    /** Findings seen in fewer than k runs. */
+    readonly flaky: AggregatedFinding[];
+    /** Requests seen in ≥ k runs → every status seen for them. */
+    readonly requests: Record<string, number[]>;
+    /** Controls seen in ≥ k runs. */
+    readonly controls: string[];
+    /**
+     * #213: set when every run ended without observing the app (a starved host, an unreachable or
+     * unresponsive target) — why. Such a persona is left out of the persona diff: its missing requests
+     * and controls are the environment, never an access difference.
+     */
+    readonly notObserved?: string;
+    /**
+     * #213: the persona's session was not honoured in some run (its first page was a sign-in page) —
+     * what it tested was not this persona. Which runs, and why.
+     */
+    readonly sessionLost?: string;
+}
+/**
+ * Votes one persona's (or the mission's) runs: findings, outcome, requests and controls. `planned`
+ * is how many runs the plan has (default: the runs given) — fewer finished never reads `intermittent`.
+ */
+export declare function voteRuns(runs: readonly RunSummary[], k: number, persona?: Persona | null, planned?: number): CellResult;
+export interface PersonaPresenceDiff {
+    /** The request (`METHOD /templated/path`) or control (`role "name"`). */
+    readonly item: string;
+    /** Personas that made / saw it. */
+    readonly presentFor: string[];
+    /** Personas that did not. */
+    readonly absentFor: string[];
+}
+export interface PersonaStatusDiff {
+    readonly request: string;
+    /** Persona → the statuses its runs got for this same request. */
+    readonly statuses: Record<string, number[]>;
+}
+export interface RbacCandidate {
+    readonly request: string;
+    /** Personas refused (401/403) and the status they got. */
+    readonly denied: Record<string, number[]>;
+    /** Personas the same request succeeded (2xx) for. */
+    readonly allowed: string[];
+    readonly title: string;
+}
+export interface PersonaDiff {
+    /** Always true: the diff is code-computed evidence for a person to judge, never a verdict. */
+    readonly advisory: true;
+    readonly requestsOnlyIn: PersonaPresenceDiff[];
+    readonly statusDiffs: PersonaStatusDiff[];
+    readonly controlsOnlyIn: PersonaPresenceDiff[];
+    /** Persona → its agreed canonical outcome (`missionOutcome`, #226). */
+    readonly outcomes: Record<string, string>;
+    /** True when not every persona reached the same outcome. */
+    readonly outcomeDiffers: boolean;
+    /** A 401/403 for one persona where another got a 2xx on the same request. */
+    readonly rbacCandidates: RbacCandidate[];
+    /** #213: personas left out of the comparison because their runs never observed the app — and why. */
+    readonly notCompared: Array<{
+        readonly persona: string;
+        readonly reason: string;
+    }>;
+    /**
+     * #213: persona → why its session was lost (its first page was a sign-in page). Every other
+     * difference for that persona is then an access difference of whoever the run was, not the persona.
+     */
+    readonly sessionLost?: Record<string, string>;
+}
+/** The persona matrix's diff section (#143): pure, over the per-persona (voted) cells. */
+export declare function diffPersonas(cells: readonly CellResult[]): PersonaDiff;
+/** A run's JSON envelope, exactly as `jevitate explore --json` emits it. */
+export type RunEnvelope = {
+    readonly ok: true;
+    readonly data: unknown;
+} | {
+    readonly ok: false;
+    readonly error: {
+        readonly code: string;
+        readonly message: string;
+    };
+};
+export interface RunOnceArgs {
+    /** Persona storage state to start from (`undefined`: the mission's own `--storage-state`, if any). */
+    readonly storageState?: string;
+    /** Where this run writes its artifacts. */
+    readonly outDir: string;
+}
+export type RunOnce = (args: RunOnceArgs) => Promise<RunEnvelope>;
+export interface MultiRunResult {
+    readonly kind: "multi-run";
+    readonly strategy: string;
+    readonly repeat: number;
+    readonly minAgreement: number;
+    /**
+     * Without personas: the agreed outcome. With personas: every persona's, when they agree; else
+     * `mixed`. While runs are pending (`complete: false`, or a killed multi-run) it is `inconclusive` (#220).
+     */
+    readonly outcome: string;
+    /**
+     * #226: the canonical verdict (#217), like every mission result's: without personas, the agreed
+     * `missionOutcome`; with personas, the shared one when they agree, else the most severe persona's
+     * (`combineOutcomes`: a broken persona run dominates, a confirmed hang beats a defect). While runs
+     * are pending (`complete: false`, or a killed multi-run) it is `inconclusive`.
+     */
+    readonly missionOutcome: MissionOutcome;
+    /** #226: `--goal` multi-runs only — the goal ending the runs agreed on, else the canonical outcome (#217). */
+    readonly goalOutcome?: GoalOutcome;
+    /** #220: why the outcome is `inconclusive` (runs pending or interrupted, or a run broke). */
+    readonly reason?: string;
+    /** #220: the multi-run was stopped by this signal; the run in flight is recorded as interrupted. */
+    readonly interrupted?: {
+        readonly signal: string;
+    };
+    /** The exit code of `missionOutcome` (130/143 for a killed multi-run). */
+    readonly exitCode: number;
+    /** #226: the build that produced the aggregate (`{version, commit, builtAt}`), like every result. */
+    readonly engine: EngineInfo;
+    /** Agreed findings (with personas: each tagged by persona). */
+    readonly findings: Array<AggregatedFinding & {
+        readonly persona?: string;
+    }>;
+    /** Flaky findings (seen in fewer than k runs) — reported, not counted. */
+    readonly flaky: Array<AggregatedFinding & {
+        readonly persona?: string;
+    }>;
+    /** One cell per persona (a single, persona-less cell without personas). */
+    readonly cells: CellResult[];
+    /** The persona matrix's diff (present only with personas). */
+    readonly diff?: PersonaDiff;
+    /** The aggregate result file (`multi-run.result.json` in the multi-run directory). */
+    readonly resultPath: string;
+    /** False while runs are still pending (the file is rewritten after every run). */
+    readonly complete: boolean;
+    /**
+     * Model usage summed over every run so far (#163): equals the sum of the runs' own `usage`. A run
+     * whose envelope carried none (it crashed outside the mission) makes the total `partial`.
+     */
+    readonly usage: UsageAggregate;
+}
+/** A run a signal interrupted (#220): the killed mission's partial result, when one was armed. */
+export interface InterruptedRun {
+    readonly signal: string;
+    readonly exitCode: number;
+    /** The killed mission's partial result (absent when the signal landed before it was armed). */
+    readonly partial?: Record<string, unknown>;
+}
+export interface RunMultiRunOptions {
+    readonly plan: MultiRunPlan;
+    readonly strategy: string;
+    /** The multi-run directory: `<persona>/run-<i>/` per run plus `multi-run.result.json`. */
+    readonly outDir: string;
+    readonly runOnce: RunOnce;
+    /**
+     * #220: armed for the whole multi-run. `onKill` must be called SYNCHRONOUSLY on a kill signal: it
+     * records the run in flight as interrupted, rewrites the aggregate (incomplete, `inconclusive`)
+     * and returns it. Returns the disarm function.
+     */
+    readonly armKill?: (onKill: (interrupted: InterruptedRun) => MultiRunResult) => () => void;
+}
+/** Summarizes one run's envelope (the only IO: its transcript file when the result omits it). */
+export declare function summarizeRun(strategy: string, index: number, envelope: RunEnvelope, envelopePath?: string): RunSummary;
+/** Folds the voted cells into the aggregate (pure). */
+export declare function aggregateCells(strategy: string, plan: MultiRunPlan, cells: CellResult[], resultPath: string, complete: boolean, engine?: EngineInfo): MultiRunResult;
+/**
+ * Runs the plan SEQUENTIALLY — persona by persona, `repeat` runs each, one at a time (never
+ * concurrently) — writing each run's envelope next to its artifacts and rewriting the aggregate
+ * after every run (so a killed multi-run still leaves the runs it finished, marked incomplete).
+ */
+export declare function runMultiRun(opts: RunMultiRunOptions): Promise<MultiRunResult>;