@crvouga/atlas 0.0.0-stage → 0.2.0

package/dist/core/src/run/execute.d.ts

@@ -0,0 +1,854 @@
+import type { BusinessEventResult } from '../events/cloudevents.js';
+import type { ChartGraph } from '../graph.js';
+import type { ChartScope, PlannedJourney, PlannedPath, PlannedStep } from '../plan.js';
+import type { Composition } from '../spec/compose.js';
+import type { SpecBundle } from '../spec/types.js';
+import type { CheckResult, Clip, Driver, EventMatcher, EventSource, Image, Implementation, PrivacyRules, RecognizerResult, RunMode, Status, TimelineEntry } from './types.js';
+/** A seed as the plan sees it: where it starts, and whether it can run yet. */
+export type PlannedSeed = {
+    name: string;
+    active: string[];
+    blocked?: string;
+};
+/** One shard of a run split across machines: `index` is 1-based. */
+export type Shard = {
+    index: number;
+    count: number;
+};
+/** What a run reports about: the spec, the scope, and the implementation's metadata. */
+export type RunContext<C> = {
+    bundle: SpecBundle;
+    composition: Composition;
+    graph: ChartGraph;
+    scope: ChartScope;
+    implementation: Implementation<C>;
+    /** The journeys of the plan, as the paths they are cut into; journeys are reported from them. */
+    journeys?: PlannedJourney[];
+    seeds?: PlannedSeed[];
+    eventMatchers?: Record<string, EventMatcher>;
+    privacy?: PrivacyRules;
+    /** Recorded in the manifest as the run's environment. */
+    environment?: Record<string, unknown>;
+};
+export type RunOptions<C> = RunContext<C> & {
+    paths: PlannedPath[];
+    driver: Driver<C>;
+    mode: RunMode;
+    /** Runs land in `<outputRoot>/<run-id>/`. */
+    outputRoot: string;
+    retry?: boolean;
+    stepTimeoutMs?: number;
+    eventSources?: EventSource[];
+    /** Path attempts that run at once, capped by the driver's `concurrency`; 1 when omitted. */
+    workers?: number;
+    /** Reported in the manifest when the paths are one shard of a larger run. */
+    shard?: Shard;
+    /** Passed outcomes of an earlier run to keep instead of running those paths again. */
+    reuse?: {
+        runDir: string;
+        outcomes: Record<string, PathOutcome>;
+    };
+    log?: (line: string) => void;
+};
+type StepResult = {
+    transition: string;
+    event: string;
+    status: Status;
+    reason: string | null;
+};
+type Attempt = {
+    status: 'passed' | 'failed';
+    error: string | null;
+    stoppedAt: string | null;
+    trace: string | null;
+    traces?: Record<string, string>;
+};
+type Observed = {
+    step: PlannedStep;
+    target: string;
+    clip: (Clip & {
+        captions?: string;
+    }) | null;
+    timeline: TimelineEntry[];
+    events: BusinessEventResult;
+    recognizer: RecognizerResult;
+    shot: Image | null;
+    transient: boolean;
+    checks: CheckResult[];
+};
+/**
+ * Everything one path's run produced, with media paths relative to the run directory. A run's
+ * manifest is a pure function of its plan and these outcomes, folded in plan order, so outcomes
+ * from parallel workers, shards and earlier runs compose into the same report.
+ */
+export type PathOutcome = {
+    id: string;
+    key: string;
+    status: Status;
+    attempts: Attempt[];
+    steps: StepResult[];
+    stoppedAt: string | null;
+    observed: Observed[];
+    failure: {
+        state: string;
+        recognizer: RecognizerResult | null;
+        looksLike: string[];
+        shot: Image | null;
+    } | null;
+    leaks: string[];
+    durationMs: number;
+    /** How long the last attempt took to seed (or set up) and recognise its start. */
+    setupMs?: number;
+    /** The run the outcome was taken from, when it was reused rather than run again. */
+    reusedFrom?: string;
+};
+/** A rough cost of a path before it has ever run: setting it up, then each step. */
+export declare const estimateMs: (p: PlannedPath) => number;
+/**
+ * The paths one shard runs: longest first onto the least loaded shard, by the plan alone, so
+ * every machine computes the same split. Paths reported without running go to the first shard.
+ */
+export declare function shardPaths(paths: PlannedPath[], shard: Shard): PlannedPath[];
+/** The plan keys and outcomes a run wrote, for merging shards and reusing passed paths. */
+export declare function readOutcomes(runDir: string): {
+    runId: string;
+    outcomes: Record<string, PathOutcome>;
+} | null;
+/** How long each path took in the newest run under `outputRoot` that recorded it, by path key. */
+export declare function durationHistory(outputRoot: string): Record<string, number>;
+/** The newest run under `outputRoot` that wrote outcomes. */
+export declare function latestRun(outputRoot: string): string | null;
+/**
+ * Fold path outcomes into the run's report, in plan order: the same plan and outcomes always give
+ * the same manifest, however many workers ran them, in whatever order they finished.
+ */
+export declare function buildManifest<C>(context: RunContext<C>, paths: PlannedPath[], outcomes: Map<string, PathOutcome>, run: {
+    id: string;
+    startedAt: Date;
+    finished: boolean;
+    mode: RunMode;
+    driver: {
+        name: string;
+        clients?: Record<string, string>;
+    };
+    activePaths?: string[];
+    extra?: Record<string, unknown>;
+}): {
+    $schema: string;
+    schemaVersion: number;
+    run: {
+        id: string;
+        startedAt: string;
+        finishedAt: string | null;
+        progress: {
+            totalPaths: number;
+            completedPaths: number;
+            activePaths: string[];
+            updatedAt: string;
+        };
+        durationMs: number;
+        commit: string | null;
+        branch: string | null;
+        environment: {
+            clients?: Record<string, string> | undefined;
+            driver: string;
+        };
+        mode: RunMode;
+        specVersion: string;
+        specDirectory: string;
+        approvals: unknown[];
+        scope: {
+            state?: string | undefined;
+            chart: string;
+            start: string;
+        };
+    };
+    charts: {
+        id: string;
+        file: string;
+        description: string;
+        parentState: string | null;
+        handOffs: {
+            [k: string]: string;
+        };
+    }[];
+    map: {
+        nodes: {
+            id: string;
+            parent: string | null;
+            chart: string;
+            type: string;
+            initial: string | null;
+            inScope: boolean;
+        }[];
+        edges: {
+            id: string;
+            source: string;
+            target: string;
+            event: string;
+            kind: import('./types.js').EventKind;
+            inScope: boolean;
+        }[];
+    };
+    states: {
+        [k: string]: {
+            status: Status;
+            screenshot: Image | null;
+            screenshotsByPath: Record<string, Image>;
+            recognizer: RecognizerResult | null;
+            looksLike: string[];
+            businessEvents: BusinessEventResult | null;
+            checks: CheckResult[];
+            reachedBy: string[];
+            notes: string[];
+            confidence: "confirmed" | "assumed";
+            source: string[];
+            expectedEvents: string[];
+            seeds?: string[] | undefined;
+            client?: string | undefined;
+            name: string;
+            chart: string | null;
+            description: string;
+            snapshot: string;
+        };
+    };
+    transitions: {
+        [k: string]: {
+            status: Status;
+            clip: (Clip & {
+                captions?: string;
+            }) | null;
+            timeline: TimelineEntry[];
+            reason: string | null;
+            reachedBy: string[];
+            client?: string | undefined;
+            id: string;
+            source: string;
+            target: string | null;
+            event: string;
+            kind: import('./types.js').EventKind;
+            how: string | null;
+        };
+    };
+    paths: {
+        id: string;
+        key: string;
+        name: string;
+        kind: PlannedPath["kind"];
+        status: Status;
+        attempts: Attempt[];
+        steps: StepResult[];
+        stoppedAt: string | null;
+        seed?: string;
+        durationMs?: number;
+        setupMs?: number;
+        reusedFrom?: string;
+    }[];
+    journeys: {
+        name: string;
+        description: string;
+        status: Status | "not-run";
+        paths: string[];
+        stoppedAt: string | null;
+    }[];
+    seeds: {
+        paths: string[];
+        verifiedBy: string[];
+        blocked?: string | undefined;
+        for?: string[] | undefined;
+        shows?: string | undefined;
+        name: string;
+        at: string[];
+        how: string | null;
+    }[];
+    coverage: {
+        [x: string]: {
+            states: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+            transitions: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+        };
+        overall: {
+            states: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+            transitions: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+        };
+    };
+    privacy: {
+        findings: string[];
+        passed: boolean;
+    };
+    reports: {
+        junit: string;
+        ctrf: string;
+        mermaid: string;
+    };
+};
+export type Manifest = ReturnType<typeof buildManifest>;
+/** Write a run's manifest, reports and outcomes; returns the manifest. */
+export declare function writeRun<C>(runDir: string, context: RunContext<C>, paths: PlannedPath[], outcomes: Map<string, PathOutcome>, run: Parameters<typeof buildManifest>[3]): {
+    $schema: string;
+    schemaVersion: number;
+    run: {
+        id: string;
+        startedAt: string;
+        finishedAt: string | null;
+        progress: {
+            totalPaths: number;
+            completedPaths: number;
+            activePaths: string[];
+            updatedAt: string;
+        };
+        durationMs: number;
+        commit: string | null;
+        branch: string | null;
+        environment: {
+            clients?: Record<string, string> | undefined;
+            driver: string;
+        };
+        mode: RunMode;
+        specVersion: string;
+        specDirectory: string;
+        approvals: unknown[];
+        scope: {
+            state?: string | undefined;
+            chart: string;
+            start: string;
+        };
+    };
+    charts: {
+        id: string;
+        file: string;
+        description: string;
+        parentState: string | null;
+        handOffs: {
+            [k: string]: string;
+        };
+    }[];
+    map: {
+        nodes: {
+            id: string;
+            parent: string | null;
+            chart: string;
+            type: string;
+            initial: string | null;
+            inScope: boolean;
+        }[];
+        edges: {
+            id: string;
+            source: string;
+            target: string;
+            event: string;
+            kind: import('./types.js').EventKind;
+            inScope: boolean;
+        }[];
+    };
+    states: {
+        [k: string]: {
+            status: Status;
+            screenshot: Image | null;
+            screenshotsByPath: Record<string, Image>;
+            recognizer: RecognizerResult | null;
+            looksLike: string[];
+            businessEvents: BusinessEventResult | null;
+            checks: CheckResult[];
+            reachedBy: string[];
+            notes: string[];
+            confidence: "confirmed" | "assumed";
+            source: string[];
+            expectedEvents: string[];
+            seeds?: string[] | undefined;
+            client?: string | undefined;
+            name: string;
+            chart: string | null;
+            description: string;
+            snapshot: string;
+        };
+    };
+    transitions: {
+        [k: string]: {
+            status: Status;
+            clip: (Clip & {
+                captions?: string;
+            }) | null;
+            timeline: TimelineEntry[];
+            reason: string | null;
+            reachedBy: string[];
+            client?: string | undefined;
+            id: string;
+            source: string;
+            target: string | null;
+            event: string;
+            kind: import('./types.js').EventKind;
+            how: string | null;
+        };
+    };
+    paths: {
+        id: string;
+        key: string;
+        name: string;
+        kind: PlannedPath["kind"];
+        status: Status;
+        attempts: Attempt[];
+        steps: StepResult[];
+        stoppedAt: string | null;
+        seed?: string;
+        durationMs?: number;
+        setupMs?: number;
+        reusedFrom?: string;
+    }[];
+    journeys: {
+        name: string;
+        description: string;
+        status: Status | "not-run";
+        paths: string[];
+        stoppedAt: string | null;
+    }[];
+    seeds: {
+        paths: string[];
+        verifiedBy: string[];
+        blocked?: string | undefined;
+        for?: string[] | undefined;
+        shows?: string | undefined;
+        name: string;
+        at: string[];
+        how: string | null;
+    }[];
+    coverage: {
+        [x: string]: {
+            states: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+            transitions: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+        };
+        overall: {
+            states: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+            transitions: {
+                total: number;
+                passed: number;
+                failed: number;
+                flaky: number;
+                notReached: number;
+            };
+        };
+    };
+    privacy: {
+        findings: string[];
+        passed: boolean;
+    };
+    reports: {
+        junit: string;
+        ctrf: string;
+        mermaid: string;
+    };
+};
+/**
+ * Compose the outcomes of several runs of the same plan (the shards of one run, or a rerun of the
+ * paths that failed) into one run, folded in plan order. Media are copied in, so the merged run
+ * stands on its own. Later runs win when two ran the same path.
+ */
+export declare function mergeRuns<C>(context: RunContext<C>, paths: PlannedPath[], runDirs: string[], options: {
+    outputRoot: string;
+    mode?: RunMode;
+    driver?: {
+        name: string;
+        clients?: Record<string, string>;
+    };
+}): {
+    runDir: string;
+    manifest: {
+        $schema: string;
+        schemaVersion: number;
+        run: {
+            id: string;
+            startedAt: string;
+            finishedAt: string | null;
+            progress: {
+                totalPaths: number;
+                completedPaths: number;
+                activePaths: string[];
+                updatedAt: string;
+            };
+            durationMs: number;
+            commit: string | null;
+            branch: string | null;
+            environment: {
+                clients?: Record<string, string> | undefined;
+                driver: string;
+            };
+            mode: RunMode;
+            specVersion: string;
+            specDirectory: string;
+            approvals: unknown[];
+            scope: {
+                state?: string | undefined;
+                chart: string;
+                start: string;
+            };
+        };
+        charts: {
+            id: string;
+            file: string;
+            description: string;
+            parentState: string | null;
+            handOffs: {
+                [k: string]: string;
+            };
+        }[];
+        map: {
+            nodes: {
+                id: string;
+                parent: string | null;
+                chart: string;
+                type: string;
+                initial: string | null;
+                inScope: boolean;
+            }[];
+            edges: {
+                id: string;
+                source: string;
+                target: string;
+                event: string;
+                kind: import('./types.js').EventKind;
+                inScope: boolean;
+            }[];
+        };
+        states: {
+            [k: string]: {
+                status: Status;
+                screenshot: Image | null;
+                screenshotsByPath: Record<string, Image>;
+                recognizer: RecognizerResult | null;
+                looksLike: string[];
+                businessEvents: BusinessEventResult | null;
+                checks: CheckResult[];
+                reachedBy: string[];
+                notes: string[];
+                confidence: "confirmed" | "assumed";
+                source: string[];
+                expectedEvents: string[];
+                seeds?: string[] | undefined;
+                client?: string | undefined;
+                name: string;
+                chart: string | null;
+                description: string;
+                snapshot: string;
+            };
+        };
+        transitions: {
+            [k: string]: {
+                status: Status;
+                clip: (Clip & {
+                    captions?: string;
+                }) | null;
+                timeline: TimelineEntry[];
+                reason: string | null;
+                reachedBy: string[];
+                client?: string | undefined;
+                id: string;
+                source: string;
+                target: string | null;
+                event: string;
+                kind: import('./types.js').EventKind;
+                how: string | null;
+            };
+        };
+        paths: {
+            id: string;
+            key: string;
+            name: string;
+            kind: PlannedPath["kind"];
+            status: Status;
+            attempts: Attempt[];
+            steps: StepResult[];
+            stoppedAt: string | null;
+            seed?: string;
+            durationMs?: number;
+            setupMs?: number;
+            reusedFrom?: string;
+        }[];
+        journeys: {
+            name: string;
+            description: string;
+            status: Status | "not-run";
+            paths: string[];
+            stoppedAt: string | null;
+        }[];
+        seeds: {
+            paths: string[];
+            verifiedBy: string[];
+            blocked?: string | undefined;
+            for?: string[] | undefined;
+            shows?: string | undefined;
+            name: string;
+            at: string[];
+            how: string | null;
+        }[];
+        coverage: {
+            [x: string]: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+            overall: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+        };
+        privacy: {
+            findings: string[];
+            passed: boolean;
+        };
+        reports: {
+            junit: string;
+            ctrf: string;
+            mermaid: string;
+        };
+    };
+    ok: boolean;
+};
+/**
+ * Run planned paths against a system through a driver. Paths start from their seed (or the
+ * implementation's `setup`) and run on a pool of workers, longest first, each attempt in its own
+ * driver context; every state is recognised from the outside after every step; a failed path is
+ * retried once to tell flaky from broken. The manifest is rewritten as each path finishes, from
+ * all outcomes so far in plan order, so a crash keeps what ran and the order paths finished in
+ * never changes the report.
+ */
+export declare function runPaths<C>(options: RunOptions<C>): Promise<{
+    runDir: string;
+    manifest: {
+        $schema: string;
+        schemaVersion: number;
+        run: {
+            id: string;
+            startedAt: string;
+            finishedAt: string | null;
+            progress: {
+                totalPaths: number;
+                completedPaths: number;
+                activePaths: string[];
+                updatedAt: string;
+            };
+            durationMs: number;
+            commit: string | null;
+            branch: string | null;
+            environment: {
+                clients?: Record<string, string> | undefined;
+                driver: string;
+            };
+            mode: RunMode;
+            specVersion: string;
+            specDirectory: string;
+            approvals: unknown[];
+            scope: {
+                state?: string | undefined;
+                chart: string;
+                start: string;
+            };
+        };
+        charts: {
+            id: string;
+            file: string;
+            description: string;
+            parentState: string | null;
+            handOffs: {
+                [k: string]: string;
+            };
+        }[];
+        map: {
+            nodes: {
+                id: string;
+                parent: string | null;
+                chart: string;
+                type: string;
+                initial: string | null;
+                inScope: boolean;
+            }[];
+            edges: {
+                id: string;
+                source: string;
+                target: string;
+                event: string;
+                kind: import('./types.js').EventKind;
+                inScope: boolean;
+            }[];
+        };
+        states: {
+            [k: string]: {
+                status: Status;
+                screenshot: Image | null;
+                screenshotsByPath: Record<string, Image>;
+                recognizer: RecognizerResult | null;
+                looksLike: string[];
+                businessEvents: BusinessEventResult | null;
+                checks: CheckResult[];
+                reachedBy: string[];
+                notes: string[];
+                confidence: "confirmed" | "assumed";
+                source: string[];
+                expectedEvents: string[];
+                seeds?: string[] | undefined;
+                client?: string | undefined;
+                name: string;
+                chart: string | null;
+                description: string;
+                snapshot: string;
+            };
+        };
+        transitions: {
+            [k: string]: {
+                status: Status;
+                clip: (Clip & {
+                    captions?: string;
+                }) | null;
+                timeline: TimelineEntry[];
+                reason: string | null;
+                reachedBy: string[];
+                client?: string | undefined;
+                id: string;
+                source: string;
+                target: string | null;
+                event: string;
+                kind: import('./types.js').EventKind;
+                how: string | null;
+            };
+        };
+        paths: {
+            id: string;
+            key: string;
+            name: string;
+            kind: PlannedPath["kind"];
+            status: Status;
+            attempts: Attempt[];
+            steps: StepResult[];
+            stoppedAt: string | null;
+            seed?: string;
+            durationMs?: number;
+            setupMs?: number;
+            reusedFrom?: string;
+        }[];
+        journeys: {
+            name: string;
+            description: string;
+            status: Status | "not-run";
+            paths: string[];
+            stoppedAt: string | null;
+        }[];
+        seeds: {
+            paths: string[];
+            verifiedBy: string[];
+            blocked?: string | undefined;
+            for?: string[] | undefined;
+            shows?: string | undefined;
+            name: string;
+            at: string[];
+            how: string | null;
+        }[];
+        coverage: {
+            [x: string]: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+            overall: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+        };
+        privacy: {
+            findings: string[];
+            passed: boolean;
+        };
+        reports: {
+            junit: string;
+            ctrf: string;
+            mermaid: string;
+        };
+    };
+    ok: boolean;
+}>;
+export {};