@crvouga/atlas 0.0.0-stage → 0.2.0

package/dist/core/src/config.d.ts

@@ -0,0 +1,477 @@
+import type { StateValue } from './graph.js';
+import type { Shard } from './run/execute.js';
+import type { Driver, EventMatcher, EventSource, Implementation, PrivacyRules, RunMode } from './run/types.js';
+export type AtlasConfig<C = unknown> = {
+    /** The directory holding the charts (`*.scxml`, `*.machine.{json,yaml}`) and `journeys.yaml`. */
+    specs: string;
+    /** The chart to run; the root chart when omitted. */
+    chart?: string;
+    /** Where generated paths start (an XState state value); the initial state when omitted. */
+    start?: StateValue;
+    startLabel?: string;
+    /** Only these events enter the chart from outside it. */
+    entryEvents?: string[];
+    /** Run only this state and what is inside it, with the transitions into and out of it. */
+    state?: string;
+    /** Path attempts that run at once (capped by the driver's `concurrency`); one per CPU when omitted. */
+    workers?: number;
+    /** Runs once before any path: prepare what every seed relies on (reference data, config). */
+    beforeRun?: () => Promise<void> | void;
+    driver: Driver<C> | (() => Driver<C> | Promise<Driver<C>>);
+    implementation: Implementation<C>;
+    /** Where runs are written; `atlas-runs` beside the config when omitted. */
+    output?: string;
+    eventSources?: () => EventSource[] | Promise<EventSource[]>;
+    eventMatchers?: Record<string, EventMatcher>;
+    privacy?: PrivacyRules;
+    /** URLs the run will reach; checked against `privacy.allowHosts` (loopback by default). */
+    targets?: string[];
+    environment?: Record<string, unknown>;
+    stepTimeoutMs?: number;
+    /** Meta fields every state must carry (default: description). */
+    requiredMeta?: string[];
+};
+export declare function defineConfig<C>(config: AtlasConfig<C>): AtlasConfig<C>;
+/** Narrow what a config runs from the command line. */
+export type PrepareOptions = {
+    /** Scope the run to this state instead of the config's. */
+    state?: string;
+};
+/** Load, lint and plan a config without running anything. */
+export declare function prepare<C>(config: AtlasConfig<C>, baseDirectory?: string, options?: PrepareOptions): {
+    bundle: import('./index.js').SpecBundle;
+    lint: {
+        findings: import('./lint.js').LintFinding[];
+        composition: import('./index.js').Composition;
+        graph: import('./graph.js').ChartGraph;
+        journeys: {
+            journey: import('./index.js').Journey;
+            replay: {
+                ok: false;
+                steps: (import('./graph.js').Step & {
+                    from: import('./graph.js').Config;
+                    to: import('./graph.js').Config;
+                })[];
+                config: import('./graph.js').Config;
+                failedAt: string;
+            } | {
+                ok: true;
+                steps: (import('./graph.js').Step & {
+                    from: import('./graph.js').Config;
+                    to: import('./graph.js').Config;
+                })[];
+                config: import('./graph.js').Config;
+                failedAt?: undefined;
+            };
+        }[];
+        unimplemented: {
+            events: string[];
+            states: string[];
+        };
+    };
+    scope: import('./plan.js').ChartScope;
+    plan: {
+        paths: import('./plan.js').PlannedPath[];
+        unreachable: string[];
+        journeys: import('./plan.js').PlannedJourney[];
+        seeds: {
+            blocked?: string | undefined;
+            name: string;
+            active: string[];
+        }[];
+    };
+    specs: string;
+};
+export type RunCommandOptions = PrepareOptions & {
+    mode?: RunMode;
+    /** Only these paths, by id. */
+    paths?: string[];
+    /** Only paths that start from these seeds ("start" for paths that `setup` starts). */
+    seeds?: string[];
+    retry?: boolean;
+    output?: string;
+    workers?: number;
+    /** Run one shard of the plan; `atlas merge` composes the shards. */
+    shard?: Shard;
+    /** Keep the passed paths of this run (or the newest run, with "latest") and run the rest. */
+    reuse?: string;
+    log?: (line: string) => void;
+};
+/** Run a config end to end and write its manifest and reports. */
+export declare function runConfig<C>(config: AtlasConfig<C>, options?: RunCommandOptions, baseDirectory?: string): Promise<{
+    runDir: string;
+    manifest: {
+        $schema: string;
+        schemaVersion: number;
+        run: {
+            id: string;
+            startedAt: string;
+            finishedAt: string | null;
+            progress: {
+                totalPaths: number;
+                completedPaths: number;
+                activePaths: string[];
+                updatedAt: string;
+            };
+            durationMs: number;
+            commit: string | null;
+            branch: string | null;
+            environment: {
+                clients?: Record<string, string> | undefined;
+                driver: string;
+            };
+            mode: RunMode;
+            specVersion: string;
+            specDirectory: string;
+            approvals: unknown[];
+            scope: {
+                state?: string | undefined;
+                chart: string;
+                start: string;
+            };
+        };
+        charts: {
+            id: string;
+            file: string;
+            description: string;
+            parentState: string | null;
+            handOffs: {
+                [k: string]: string;
+            };
+        }[];
+        map: {
+            nodes: {
+                id: string;
+                parent: string | null;
+                chart: string;
+                type: string;
+                initial: string | null;
+                inScope: boolean;
+            }[];
+            edges: {
+                id: string;
+                source: string;
+                target: string;
+                event: string;
+                kind: import('./index.js').EventKind;
+                inScope: boolean;
+            }[];
+        };
+        states: {
+            [k: string]: {
+                status: import('./index.js').Status;
+                screenshot: import('./index.js').Image | null;
+                screenshotsByPath: Record<string, import('./index.js').Image>;
+                recognizer: import('./index.js').RecognizerResult | null;
+                looksLike: string[];
+                businessEvents: import('./index.js').BusinessEventResult | null;
+                checks: import('./index.js').CheckResult[];
+                reachedBy: string[];
+                notes: string[];
+                confidence: "confirmed" | "assumed";
+                source: string[];
+                expectedEvents: string[];
+                seeds?: string[] | undefined;
+                client?: string | undefined;
+                name: string;
+                chart: string | null;
+                description: string;
+                snapshot: string;
+            };
+        };
+        transitions: {
+            [k: string]: {
+                status: import('./index.js').Status;
+                clip: (import('./index.js').Clip & {
+                    captions?: string;
+                }) | null;
+                timeline: import('./index.js').TimelineEntry[];
+                reason: string | null;
+                reachedBy: string[];
+                client?: string | undefined;
+                id: string;
+                source: string;
+                target: string | null;
+                event: string;
+                kind: import('./index.js').EventKind;
+                how: string | null;
+            };
+        };
+        paths: {
+            id: string;
+            key: string;
+            name: string;
+            kind: import('./plan.js').PlannedPath["kind"];
+            status: import('./index.js').Status;
+            attempts: {
+                status: "passed" | "failed";
+                error: string | null;
+                stoppedAt: string | null;
+                trace: string | null;
+                traces?: Record<string, string>;
+            }[];
+            steps: {
+                transition: string;
+                event: string;
+                status: import('./index.js').Status;
+                reason: string | null;
+            }[];
+            stoppedAt: string | null;
+            seed?: string;
+            durationMs?: number;
+            setupMs?: number;
+            reusedFrom?: string;
+        }[];
+        journeys: {
+            name: string;
+            description: string;
+            status: import('./index.js').Status | "not-run";
+            paths: string[];
+            stoppedAt: string | null;
+        }[];
+        seeds: {
+            paths: string[];
+            verifiedBy: string[];
+            blocked?: string | undefined;
+            for?: string[] | undefined;
+            shows?: string | undefined;
+            name: string;
+            at: string[];
+            how: string | null;
+        }[];
+        coverage: {
+            [x: string]: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+            overall: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+        };
+        privacy: {
+            findings: string[];
+            passed: boolean;
+        };
+        reports: {
+            junit: string;
+            ctrf: string;
+            mermaid: string;
+        };
+    };
+    ok: boolean;
+}>;
+/** Compose runs of a config (its shards, or a rerun and the run before it) into one run. */
+export declare function mergeConfigRuns<C>(config: AtlasConfig<C>, runDirs: string[], options?: PrepareOptions & {
+    output?: string;
+}, baseDirectory?: string): {
+    runDir: string;
+    manifest: {
+        $schema: string;
+        schemaVersion: number;
+        run: {
+            id: string;
+            startedAt: string;
+            finishedAt: string | null;
+            progress: {
+                totalPaths: number;
+                completedPaths: number;
+                activePaths: string[];
+                updatedAt: string;
+            };
+            durationMs: number;
+            commit: string | null;
+            branch: string | null;
+            environment: {
+                clients?: Record<string, string> | undefined;
+                driver: string;
+            };
+            mode: RunMode;
+            specVersion: string;
+            specDirectory: string;
+            approvals: unknown[];
+            scope: {
+                state?: string | undefined;
+                chart: string;
+                start: string;
+            };
+        };
+        charts: {
+            id: string;
+            file: string;
+            description: string;
+            parentState: string | null;
+            handOffs: {
+                [k: string]: string;
+            };
+        }[];
+        map: {
+            nodes: {
+                id: string;
+                parent: string | null;
+                chart: string;
+                type: string;
+                initial: string | null;
+                inScope: boolean;
+            }[];
+            edges: {
+                id: string;
+                source: string;
+                target: string;
+                event: string;
+                kind: import('./index.js').EventKind;
+                inScope: boolean;
+            }[];
+        };
+        states: {
+            [k: string]: {
+                status: import('./index.js').Status;
+                screenshot: import('./index.js').Image | null;
+                screenshotsByPath: Record<string, import('./index.js').Image>;
+                recognizer: import('./index.js').RecognizerResult | null;
+                looksLike: string[];
+                businessEvents: import('./index.js').BusinessEventResult | null;
+                checks: import('./index.js').CheckResult[];
+                reachedBy: string[];
+                notes: string[];
+                confidence: "confirmed" | "assumed";
+                source: string[];
+                expectedEvents: string[];
+                seeds?: string[] | undefined;
+                client?: string | undefined;
+                name: string;
+                chart: string | null;
+                description: string;
+                snapshot: string;
+            };
+        };
+        transitions: {
+            [k: string]: {
+                status: import('./index.js').Status;
+                clip: (import('./index.js').Clip & {
+                    captions?: string;
+                }) | null;
+                timeline: import('./index.js').TimelineEntry[];
+                reason: string | null;
+                reachedBy: string[];
+                client?: string | undefined;
+                id: string;
+                source: string;
+                target: string | null;
+                event: string;
+                kind: import('./index.js').EventKind;
+                how: string | null;
+            };
+        };
+        paths: {
+            id: string;
+            key: string;
+            name: string;
+            kind: import('./plan.js').PlannedPath["kind"];
+            status: import('./index.js').Status;
+            attempts: {
+                status: "passed" | "failed";
+                error: string | null;
+                stoppedAt: string | null;
+                trace: string | null;
+                traces?: Record<string, string>;
+            }[];
+            steps: {
+                transition: string;
+                event: string;
+                status: import('./index.js').Status;
+                reason: string | null;
+            }[];
+            stoppedAt: string | null;
+            seed?: string;
+            durationMs?: number;
+            setupMs?: number;
+            reusedFrom?: string;
+        }[];
+        journeys: {
+            name: string;
+            description: string;
+            status: import('./index.js').Status | "not-run";
+            paths: string[];
+            stoppedAt: string | null;
+        }[];
+        seeds: {
+            paths: string[];
+            verifiedBy: string[];
+            blocked?: string | undefined;
+            for?: string[] | undefined;
+            shows?: string | undefined;
+            name: string;
+            at: string[];
+            how: string | null;
+        }[];
+        coverage: {
+            [x: string]: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+            overall: {
+                states: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+                transitions: {
+                    total: number;
+                    passed: number;
+                    failed: number;
+                    flaky: number;
+                    notReached: number;
+                };
+            };
+        };
+        privacy: {
+            findings: string[];
+            passed: boolean;
+        };
+        reports: {
+            junit: string;
+            ctrf: string;
+            mermaid: string;
+        };
+    };
+    ok: boolean;
+};

package/dist/core/src/config.js

@@ -0,0 +1,100 @@
+import { availableParallelism } from 'node:os';
+import path from 'node:path';
+import { lintBundle } from './lint.js';
+import { chartScope, planChart } from './plan.js';
+import { assertAllowedTarget, mergePrivacy } from './privacy.js';
+import { latestRun, mergeRuns, readOutcomes, runPaths, shardPaths } from './run/execute.js';
+import { loadSpecDirectory } from './spec/load.js';
+export function defineConfig(config) {
+    return config;
+}
+/** Load, lint and plan a config without running anything. */
+export function prepare(config, baseDirectory = process.cwd(), options = {}) {
+    const specs = path.resolve(baseDirectory, config.specs);
+    const bundle = loadSpecDirectory(specs);
+    const metadata = config.requiredMeta ? { requiredMeta: config.requiredMeta } : {};
+    const structure = lintBundle(bundle, metadata);
+    const state = options.state ?? config.state;
+    const scope = chartScope(bundle, structure.composition, structure.graph, {
+        ...(config.chart === undefined ? {} : { chart: config.chart }),
+        ...(config.start === undefined ? {} : { start: config.start }),
+        ...(config.startLabel === undefined ? {} : { startLabel: config.startLabel }),
+        ...(config.entryEvents === undefined ? {} : { entryEvents: config.entryEvents }),
+        ...(state === undefined ? {} : { state })
+    });
+    const impl = config.implementation;
+    const implemented = new Set(Object.entries(impl.events).filter(([, e]) => !e.blocked).map(([k]) => k));
+    const full = lintBundle(bundle, {
+        ...metadata,
+        scope: { states: scope.states, events: scope.events },
+        implementations: { events: implemented, states: new Set(Object.keys(impl.states)) }
+    });
+    const isBlocked = (event) => !impl.events[event] || Boolean(impl.events[event].blocked);
+    const seeds = Object.entries(impl.seeds ?? {}).map(([name, seed]) => ({
+        name,
+        at: typeof seed.at === 'string' ? full.graph.valueOf(seed.at) : seed.at,
+        ...(seed.blocked ? { blocked: seed.blocked } : {}),
+        ...(seed.for ? { for: seed.for } : {}),
+        ...(seed.shows ? { shows: seed.shows } : {})
+    }));
+    for (const seed of seeds) {
+        const shows = impl.seeds[seed.name].shows;
+        if (shows && !full.graph.resolve(seed.at).active.includes(shows))
+            throw new Error(`The seed "${seed.name}" shows "${shows}", which is not active where it starts`);
+        for (const event of seed.for ?? [])
+            if (!full.graph.stateNames().some((n) => full.graph.node(n)?.on?.[event]))
+                throw new Error(`The seed "${seed.name}" is for "${event}", which no state handles`);
+    }
+    const plan = planChart(bundle, full.graph, scope, { isBlocked, ...(impl.canStart ? { canStart: impl.canStart } : {}), seeds, hasSetup: Boolean(impl.setup) });
+    return { bundle, lint: full, scope, plan, specs };
+}
+const contextOf = (config, prepared) => ({
+    bundle: prepared.bundle,
+    composition: prepared.lint.composition,
+    graph: prepared.lint.graph,
+    scope: prepared.scope,
+    implementation: config.implementation,
+    journeys: prepared.plan.journeys,
+    seeds: prepared.plan.seeds,
+    ...(config.eventMatchers ? { eventMatchers: config.eventMatchers } : {}),
+    ...(config.privacy ? { privacy: config.privacy } : {}),
+    ...(config.environment ? { environment: config.environment } : {})
+});
+/** Run a config end to end and write its manifest and reports. */
+export async function runConfig(config, options = {}, baseDirectory = process.cwd()) {
+    const prepared = prepare(config, baseDirectory, options);
+    const structural = prepared.lint.findings.filter((f) => f.rule !== 'implemented');
+    if (structural.length) {
+        throw new Error(`The spec is not executable:\n${structural.map((f) => `  [${f.rule}] ${f.message}`).join('\n')}`);
+    }
+    assertAllowedTarget(config.targets ?? [], mergePrivacy(config.privacy));
+    await config.beforeRun?.();
+    const driver = typeof config.driver === 'function' ? await config.driver() : config.driver;
+    const outputRoot = path.resolve(baseDirectory, options.output ?? config.output ?? 'atlas-runs');
+    let paths = prepared.plan.paths.filter((p) => (!options.paths || options.paths.includes(p.id)) && (!options.seeds || options.seeds.includes(p.seed ?? 'start')));
+    if (options.shard)
+        paths = shardPaths(paths, options.shard);
+    const reuseDir = options.reuse === 'latest' ? latestRun(outputRoot) : options.reuse ? path.resolve(options.reuse) : null;
+    const reused = reuseDir ? readOutcomes(reuseDir) : null;
+    if (options.reuse && !reused)
+        throw new Error(`There is no earlier run to reuse${reuseDir ? ` at ${reuseDir}` : ''}`);
+    return runPaths({
+        ...contextOf(config, prepared),
+        paths,
+        driver,
+        mode: options.mode ?? 'fast',
+        outputRoot,
+        ...(options.retry === undefined ? {} : { retry: options.retry }),
+        ...(config.stepTimeoutMs === undefined ? {} : { stepTimeoutMs: config.stepTimeoutMs }),
+        eventSources: config.eventSources ? await config.eventSources() : [],
+        workers: options.workers ?? config.workers ?? availableParallelism(),
+        ...(options.shard ? { shard: options.shard } : {}),
+        ...(reused && reuseDir ? { reuse: { runDir: reuseDir, outcomes: reused.outcomes } } : {}),
+        ...(options.log ? { log: options.log } : {})
+    });
+}
+/** Compose runs of a config (its shards, or a rerun and the run before it) into one run. */
+export function mergeConfigRuns(config, runDirs, options = {}, baseDirectory = process.cwd()) {
+    const prepared = prepare(config, baseDirectory, options);
+    return mergeRuns(contextOf(config, prepared), prepared.plan.paths, runDirs.map((d) => path.resolve(d)), { outputRoot: path.resolve(baseDirectory, options.output ?? config.output ?? 'atlas-runs') });
+}

package/dist/core/src/events/cloudevents.d.ts

@@ -0,0 +1,56 @@
+import type { EventMatcher, EventSource } from '../run/types.js';
+/** A CloudEvents 1.0 event in the JSON format (https://cloudevents.io). */
+export type CloudEvent = {
+    specversion: '1.0';
+    id: string;
+    source: string;
+    type: string;
+    time?: string;
+    subject?: string;
+    datacontenttype?: string;
+    data?: unknown;
+    [extension: string]: unknown;
+};
+/** Parse an HTTP delivery in either CloudEvents content mode (structured, binary or batch). */
+export declare function cloudEventsFromHttp(headers: Record<string, string | string[] | undefined>, body: string): CloudEvent[];
+/**
+ * An HTTP endpoint the system under test sends CloudEvents to (any content mode). Point the
+ * product's event bus, webhook relay or OpenTelemetry collector's CloudEvents exporter at it.
+ */
+export declare function cloudEventsHttpSource(options?: {
+    port?: number;
+    host?: string;
+}): Promise<EventSource & {
+    url: string;
+}>;
+/** Lines appended to a log file during a step, for products whose only event stream is a log. */
+export declare function logFileSource(file: string, options?: {
+    settleMs?: number;
+}): EventSource;
+export type BusinessEventResult = {
+    expected: string[];
+    observed: string[];
+    missing: string[];
+    noSignal: string[];
+    events: CloudEvent[];
+};
+/**
+ * Keep what belongs to one attempt when attempts run side by side: events and log lines that
+ * mention one of its correlation ids. With no ids, everything is kept.
+ */
+export declare function correlated(collected: {
+    events: CloudEvent[];
+    text: string;
+}, ids: string[]): {
+    events: CloudEvent[];
+    text: string;
+};
+/**
+ * Compare the business events a state expects with what the sources saw. A business event is
+ * observed when its matcher matches; with no matcher, a CloudEvent whose `type` equals the name.
+ * Events with neither a matcher nor any CloudEvents source are "no signal", never "missing".
+ */
+export declare function compareBusinessEvents(expected: string[], collected: {
+    events: CloudEvent[];
+    text: string;
+}, matchers: Record<string, EventMatcher>, hasCloudEvents: boolean): BusinessEventResult;