@trylayout/qa 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Layout
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,161 @@
+# Layout QA
+
+Layout QA is a local browser QA protocol and runner for AI-built frontends. It runs deterministic Playwright-style flows against a local or preview URL, captures the screenshot sequence, checks browser health, and writes a static HTML report.
+
+The core loop is intentionally local:
+
+```bash
+npx @trylayout/qa init
+npx @trylayout/qa run --target-url http://localhost:5173 --scenario happy_path --open
+```
+
+## Why this exists
+
+Frontend agents can move faster when they have a visual feedback loop they can run themselves. Layout gives the agent a small protocol:
+
+- Wire app mocks behind a local env flag such as `VITE_LAYOUT_QA_MOCKS=1`.
+- Keep deterministic API/auth scenarios in the app repo.
+- Declare high-value browser flows in `.layout/qa-flows.json`.
+- Run the CLI locally and inspect the generated screenshots/report.
+
+Layout does not require uploading screenshots or source code. Hosted reports, PR comments, and AI review notes can be layered on later.
+
+## Install
+
+Use it directly with `npx`:
+
+```bash
+npx @trylayout/qa run --target-url http://localhost:5173
+```
+
+Or install it in a project:
+
+```bash
+npm install --save-dev @trylayout/qa
+npx trylayout run --target-url http://localhost:5173
+```
+
+The package uses Playwright. If your environment does not already have the browser binaries, run:
+
+```bash
+npx playwright install chromium
+```
+
+## Commands
+
+Initialize a starter flow file:
+
+```bash
+npx @trylayout/qa init
+```
+
+Run a flow:
+
+```bash
+npx @trylayout/qa run \
+  --target-url http://localhost:5173 \
+  --scenario happy_path \
+  --open
+```
+
+Useful options:
+
+```text
+--target-url <url>     URL of the running frontend to test.
+--scenario <name>      Mock scenario to activate. Defaults to happy_path.
+--flows <path>         Flow manifest path. Defaults to .layout/qa-flows.json.
+--out <path>           Artifact directory. Defaults to .layout/runs.
+--timeout <ms>         Browser run timeout. Defaults to 60000.
+--headed               Show the browser instead of running headless.
+--open                 Open the generated local HTML report after the run.
+--json                 Print machine-readable JSON.
+```
+
+Each run writes:
+
+```text
+.layout/runs/<timestamp-scenario>/
+  index.html
+  result.json
+  screenshots/
+    01-<step>.jpg
+    final.jpg
+```
+
+The process exits `0` on pass and `1` on failure, so the same command can run in CI.
+
+## Flow Manifest
+
+Default path: `.layout/qa-flows.json`.
+
+```json
+{
+  "$schema": "https://trylayout.com/schemas/qa-flows.v1.json",
+  "docsUrl": "https://trylayout.com/docs/qa",
+  "schemaVersion": 1,
+  "flows": [
+    {
+      "id": "workspace_smoke",
+      "name": "Workspace smoke",
+      "startUrl": "/",
+      "scenarios": ["happy_path"],
+      "steps": [
+        {
+          "id": "workspace_loaded",
+          "type": "assert_visible_text",
+          "text": "Dashboard",
+          "screenshot": true
+        }
+      ]
+    }
+  ]
+}
+```
+
+Supported step types:
+
+- `goto`: navigate to `url`.
+- `click`: click by `selector` or visible `text`.
+- `fill`: fill a `selector` with `value`.
+- `assert_visible_text`: require visible `text`.
+- `wait_for_text`: alias for a visible text wait.
+- `assert_url`: require current URL to equal `url` or contain `contains`.
+- `screenshot`: capture a screenshot checkpoint.
+
+The runner sets these before the app loads:
+
+```js
+localStorage.setItem("layout.qa.scenario", "<scenario>");
+sessionStorage.setItem("layout.qa.runner", "1");
+```
+
+Your app can use `layout.qa.scenario` to switch deterministic mock states. The `layout.qa.runner` flag is useful for hiding local-only QA switchers from screenshots.
+
+## Agent Setup Prompt
+
+Paste this into your coding agent inside the frontend repo:
+
+```text
+Set up Layout QA for this web app.
+
+Docs: https://trylayout.com/docs/qa
+Flow schema: https://trylayout.com/schemas/qa-flows.v1.json
+
+Add deterministic mock API/auth states behind a local-only env flag such as VITE_LAYOUT_QA_MOCKS=1. Use the scenario key localStorage["layout.qa.scenario"] with at least happy_path, empty, and error states. Keep mocks inside the app test/dev setup; do not add a standalone mock server.
+
+Add .layout/qa-flows.json with one smoke flow for the highest-value page and screenshot checkpoints after meaningful user-visible states. Prefer visible text and stable selectors. Hide any local QA scenario switcher when sessionStorage["layout.qa.runner"] === "1".
+
+Then run:
+npx @trylayout/qa run --target-url <local app url> --scenario happy_path --open
+```
+
+## npm Publishing
+
+This repo is configured for a public scoped npm package:
+
+```bash
+npm login
+npm publish --access public
+```
+
+If the `@trylayout` npm scope does not exist yet, create it in npm first, then rerun publish from this repo.

package/build/cli/layoutQa.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/build/cli/layoutQa.js

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const flows_1 = require("../flows");
+const runner_1 = require("../runner");
+const report_1 = require("../report");
+function printHelp() {
+    process.stdout.write(`Layout QA CLI
+
+Usage:
+  trylayout init [options]
+  trylayout run --target-url <url> [options]
+  npx @trylayout/qa run --target-url <url> [options]
+
+Commands:
+  init                  Write a starter .layout/qa-flows.json.
+  run                   Run browser QA and write a local HTML report.
+
+Options:
+  --target-url <url>     URL of the running frontend to test.
+  --scenario <name>      Mock scenario to activate. Defaults to happy_path.
+  --flows <path>         Flow manifest path. Defaults to .layout/qa-flows.json.
+  --out <path>           Artifact directory. Defaults to .layout/runs.
+  --timeout <ms>         Browser run timeout. Defaults to LAYOUT_QA_TEST_TIMEOUT_MS or 60000.
+  --headed               Show the browser instead of running headless.
+  --open                 Open the generated local HTML report after the run.
+  --json                 Print machine-readable JSON.
+  --force                Overwrite an existing flow file during init.
+  --help                 Show this help.
+`);
+}
+function readFlag(args, name) {
+    const index = args.indexOf(name);
+    if (index === -1)
+        return '';
+    return args[index + 1] || '';
+}
+function hasFlag(args, name) {
+    return args.includes(name);
+}
+function parseArgs(args) {
+    const command = args[0] && !args[0].startsWith('--') ? args[0] : 'help';
+    const timeoutValue = readFlag(args, '--timeout');
+    const parsedTimeoutMs = timeoutValue ? Number(timeoutValue) : undefined;
+    if (timeoutValue &&
+        (!Number.isFinite(parsedTimeoutMs) || Number(parsedTimeoutMs) <= 0)) {
+        throw new Error('--timeout must be a positive number of milliseconds.');
+    }
+    return {
+        command,
+        targetUrl: readFlag(args, '--target-url'),
+        scenario: readFlag(args, '--scenario') || 'happy_path',
+        flowsPath: readFlag(args, '--flows'),
+        outDir: readFlag(args, '--out'),
+        timeoutMs: parsedTimeoutMs,
+        headed: hasFlag(args, '--headed'),
+        json: hasFlag(args, '--json'),
+        open: hasFlag(args, '--open'),
+        force: hasFlag(args, '--force'),
+        help: hasFlag(args, '--help') || command === 'help',
+    };
+}
+async function exists(filePath) {
+    return promises_1.default
+        .access(filePath)
+        .then(() => true)
+        .catch(() => false);
+}
+async function initCommand(options) {
+    const manifestPath = options.flowsPath
+        ? path_1.default.resolve(process.cwd(), options.flowsPath)
+        : await (0, flows_1.resolveDefaultPath)(flows_1.FLOW_MANIFEST_PATH);
+    if ((await exists(manifestPath)) && !options.force) {
+        throw new Error(`${manifestPath} already exists. Use --force to overwrite it.`);
+    }
+    await promises_1.default.mkdir(path_1.default.dirname(manifestPath), { recursive: true });
+    await promises_1.default.writeFile(manifestPath, `${JSON.stringify((0, flows_1.starterFlowManifest)(), null, 2)}\n`);
+    process.stdout.write(`Created ${manifestPath}\n`);
+}
+function statusIcon(passed) {
+    return passed ? 'PASS' : 'FAIL';
+}
+function printHumanSummary(input) {
+    const passed = (0, runner_1.isQaRunPassed)(input.result);
+    process.stdout.write(`\nLayout QA ${passed ? 'passed' : 'failed'}\n` +
+        `Scenario: ${input.scenario}\n` +
+        `Target: ${input.targetUrl}\n` +
+        `Final URL: ${input.result.finalUrl || 'unavailable'}\n` +
+        `Flow: ${input.result.flow?.name || 'None'} (${input.result.flow?.source || 'none'})\n` +
+        `Manifest: ${input.manifestFound ? input.manifestPath : 'not found; default smoke'}\n\n`);
+    for (const check of input.result.checks) {
+        process.stdout.write(`${statusIcon(check.passed)} ${check.label}${check.detail ? ` - ${check.detail}` : ''}\n`);
+    }
+    if (input.result.flow?.steps.length) {
+        process.stdout.write('\nFlow steps:\n');
+        for (const step of input.result.flow.steps) {
+            process.stdout.write(`${statusIcon(step.status === 'passed')} ${step.label || step.id}${step.detail ? ` - ${step.detail}` : ''}\n`);
+        }
+    }
+    if (input.result.issues.length) {
+        process.stdout.write('\nIssues:\n');
+        for (const issue of input.result.issues) {
+            process.stdout.write(`- ${issue.type}: ${issue.message}${issue.source ? ` (${issue.source})` : ''}\n`);
+        }
+    }
+    if (input.result.nextAction) {
+        process.stdout.write(`\nNext action: ${input.result.nextAction.title}\n` +
+            `${input.result.nextAction.detail}\n`);
+    }
+    process.stdout.write(`\nArtifacts: ${input.artifacts.runDir}\n`);
+    process.stdout.write(`Report: ${input.artifacts.reportPath}\n`);
+}
+function resultForConsole(result) {
+    const clean = JSON.parse(JSON.stringify(result));
+    delete clean.screenshotDataUrl;
+    const flow = clean.flow;
+    for (const step of flow?.steps || []) {
+        delete step.screenshotDataUrl;
+    }
+    return clean;
+}
+async function runCommand(options) {
+    if (!options.targetUrl) {
+        throw new Error('--target-url is required.');
+    }
+    const { flow, manifestPath, manifestFound } = await (0, flows_1.loadFlow)({
+        flowsPath: options.flowsPath,
+        scenario: options.scenario,
+    });
+    let result;
+    try {
+        result = await (0, runner_1.runLayoutQaBrowser)({
+            targetUrl: options.targetUrl,
+            scenario: options.scenario,
+            flow,
+            timeoutMs: options.timeoutMs || (0, flows_1.getTestTimeoutMs)(),
+            headless: !options.headed,
+        });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        result = (0, runner_1.buildRunnerErrorResult)(message);
+    }
+    const artifacts = await (0, report_1.writeArtifacts)({
+        outDir: options.outDir,
+        scenario: options.scenario,
+        targetUrl: options.targetUrl,
+        manifestPath,
+        manifestFound,
+        result,
+    });
+    const passed = (0, runner_1.isQaRunPassed)(result);
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify({
+            status: passed ? 'passed' : 'failed',
+            scenario: options.scenario,
+            targetUrl: options.targetUrl,
+            manifestPath,
+            manifestFound,
+            artifacts,
+            result: resultForConsole(result),
+        }, null, 2)}\n`);
+    }
+    else {
+        printHumanSummary({
+            result,
+            scenario: options.scenario,
+            targetUrl: options.targetUrl,
+            manifestPath,
+            manifestFound,
+            artifacts,
+        });
+    }
+    if (options.open) {
+        await (0, report_1.openReport)(artifacts.reportPath);
+    }
+    process.exitCode = passed ? 0 : 1;
+}
+async function main() {
+    const options = parseArgs(process.argv.slice(2));
+    if (options.help) {
+        printHelp();
+        return;
+    }
+    if (options.command === 'init') {
+        await initCommand(options);
+        return;
+    }
+    if (options.command === 'run') {
+        await runCommand(options);
+        return;
+    }
+    throw new Error(`Unsupported command: ${options.command}`);
+}
+main().catch(error => {
+    process.stderr.write(`Layout QA failed to start: ${error instanceof Error ? error.message : String(error)}\n`);
+    process.exitCode = 1;
+});

package/build/flows.d.ts

@@ -0,0 +1,35 @@
+import { LoadedQaFlow, QaFlowDefinition } from './types';
+export declare const DEFAULT_TEST_TIMEOUT_MS: number;
+export declare const SCREENSHOT_LIMIT_BYTES: number;
+export declare const FLOW_MANIFEST_PATH = ".layout/qa-flows.json";
+export declare const QA_DOCS_URL = "https://trylayout.com/docs/qa";
+export declare function getTestTimeoutMs(): number;
+export declare function selectFlowFromManifest(raw: unknown, scenario: string): QaFlowDefinition | null;
+export declare function parseFlowManifestContent(content: string, scenario: string): LoadedQaFlow | null;
+export declare function defaultFlow(): LoadedQaFlow;
+export declare function resolveDefaultPath(defaultPath: string): Promise<string>;
+export declare function loadFlow(input: {
+    flowsPath: string;
+    scenario: string;
+}): Promise<{
+    flow: LoadedQaFlow;
+    manifestPath: string;
+    manifestFound: boolean;
+}>;
+export declare function starterFlowManifest(): {
+    $schema: string;
+    docsUrl: string;
+    schemaVersion: number;
+    flows: {
+        id: string;
+        name: string;
+        startUrl: string;
+        scenarios: string[];
+        steps: {
+            id: string;
+            type: string;
+            label: string;
+            screenshot: boolean;
+        }[];
+    }[];
+};

package/build/flows.js

@@ -0,0 +1,182 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QA_DOCS_URL = exports.FLOW_MANIFEST_PATH = exports.SCREENSHOT_LIMIT_BYTES = exports.DEFAULT_TEST_TIMEOUT_MS = void 0;
+exports.getTestTimeoutMs = getTestTimeoutMs;
+exports.selectFlowFromManifest = selectFlowFromManifest;
+exports.parseFlowManifestContent = parseFlowManifestContent;
+exports.defaultFlow = defaultFlow;
+exports.resolveDefaultPath = resolveDefaultPath;
+exports.loadFlow = loadFlow;
+exports.starterFlowManifest = starterFlowManifest;
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+exports.DEFAULT_TEST_TIMEOUT_MS = 60 * 1000;
+exports.SCREENSHOT_LIMIT_BYTES = 300 * 1024;
+exports.FLOW_MANIFEST_PATH = '.layout/qa-flows.json';
+exports.QA_DOCS_URL = 'https://trylayout.com/docs/qa';
+function getTestTimeoutMs() {
+    const value = Number(process.env.LAYOUT_QA_TEST_TIMEOUT_MS);
+    return Number.isFinite(value) && value > 0 ? value : exports.DEFAULT_TEST_TIMEOUT_MS;
+}
+function isRecord(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+function stringValue(value, fallback = '') {
+    return typeof value === 'string' ? value : fallback;
+}
+function booleanValue(value, fallback = false) {
+    return typeof value === 'boolean' ? value : fallback;
+}
+function numberValue(value) {
+    return typeof value === 'number' && Number.isFinite(value)
+        ? value
+        : undefined;
+}
+function stringArray(value) {
+    return Array.isArray(value)
+        ? value.filter(item => typeof item === 'string')
+        : [];
+}
+function normalizeFlowStep(value, index) {
+    if (!isRecord(value))
+        return null;
+    const type = stringValue(value.type).trim();
+    if (!type)
+        return null;
+    return {
+        id: stringValue(value.id).trim() ||
+            `${type.replace(/[^a-zA-Z0-9_-]+/g, '_')}_${index + 1}`,
+        type,
+        label: stringValue(value.label || value.name).trim(),
+        text: stringValue(value.text).trim(),
+        selector: stringValue(value.selector).trim(),
+        value: stringValue(value.value),
+        url: stringValue(value.url || value.path).trim(),
+        contains: stringValue(value.contains).trim(),
+        exact: booleanValue(value.exact),
+        screenshot: booleanValue(value.screenshot, type === 'screenshot'),
+        timeoutMs: numberValue(value.timeoutMs),
+    };
+}
+function normalizeFlow(value, index) {
+    if (!isRecord(value))
+        return null;
+    const rawSteps = Array.isArray(value.steps) ? value.steps : [];
+    const steps = rawSteps
+        .map((step, stepIndex) => normalizeFlowStep(step, stepIndex))
+        .filter((step) => Boolean(step));
+    if (steps.length === 0)
+        return null;
+    const id = stringValue(value.id).trim() || `flow_${index + 1}`;
+    return {
+        id,
+        name: stringValue(value.name).trim() || id,
+        startUrl: stringValue(value.startUrl).trim() || '/',
+        scenarios: stringArray(value.scenarios),
+        steps,
+    };
+}
+function selectFlowFromManifest(raw, scenario) {
+    if (!isRecord(raw) || !Array.isArray(raw.flows))
+        return null;
+    const flows = raw.flows
+        .map((flow, index) => normalizeFlow(flow, index))
+        .filter((flow) => Boolean(flow));
+    if (flows.length === 0)
+        return null;
+    return (flows.find(flow => flow.scenarios.length === 0 || flow.scenarios.includes(scenario)) || flows[0]);
+}
+function parseFlowManifestContent(content, scenario) {
+    const flow = selectFlowFromManifest(JSON.parse(content), scenario);
+    return flow ? { ...flow, source: 'manifest' } : null;
+}
+function defaultFlow() {
+    return {
+        id: 'target_smoke',
+        name: 'Target smoke',
+        startUrl: '/',
+        scenarios: [],
+        source: 'default',
+        steps: [
+            {
+                id: 'initial_screen',
+                type: 'screenshot',
+                label: 'Initial screen',
+                screenshot: true,
+            },
+        ],
+    };
+}
+async function exists(filePath) {
+    return promises_1.default
+        .access(filePath)
+        .then(() => true)
+        .catch(() => false);
+}
+async function resolveDefaultPath(defaultPath) {
+    const cwdPath = path_1.default.resolve(process.cwd(), defaultPath);
+    if (await exists(cwdPath))
+        return cwdPath;
+    const parentPath = path_1.default.resolve(process.cwd(), '..', defaultPath);
+    if (await exists(parentPath))
+        return parentPath;
+    if (defaultPath === '.layout' || defaultPath.startsWith('.layout/')) {
+        const cwdLayoutDir = path_1.default.resolve(process.cwd(), '.layout');
+        const parentLayoutDir = path_1.default.resolve(process.cwd(), '..', '.layout');
+        if (await exists(cwdLayoutDir))
+            return cwdPath;
+        if (await exists(parentLayoutDir))
+            return parentPath;
+    }
+    return cwdPath;
+}
+async function loadFlow(input) {
+    const manifestPath = input.flowsPath
+        ? path_1.default.resolve(process.cwd(), input.flowsPath)
+        : await resolveDefaultPath(exports.FLOW_MANIFEST_PATH);
+    const content = await promises_1.default.readFile(manifestPath, 'utf8').catch(error => {
+        if (error.code === 'ENOENT') {
+            return '';
+        }
+        throw error;
+    });
+    if (!content) {
+        return {
+            flow: defaultFlow(),
+            manifestPath,
+            manifestFound: false,
+        };
+    }
+    const flow = parseFlowManifestContent(content, input.scenario);
+    return {
+        flow: flow || defaultFlow(),
+        manifestPath,
+        manifestFound: Boolean(flow),
+    };
+}
+function starterFlowManifest() {
+    return {
+        $schema: 'https://trylayout.com/schemas/qa-flows.v1.json',
+        docsUrl: exports.QA_DOCS_URL,
+        schemaVersion: 1,
+        flows: [
+            {
+                id: 'smoke',
+                name: 'Smoke',
+                startUrl: '/',
+                scenarios: ['happy_path'],
+                steps: [
+                    {
+                        id: 'initial_screen',
+                        type: 'screenshot',
+                        label: 'Initial screen',
+                        screenshot: true,
+                    },
+                ],
+            },
+        ],
+    };
+}

package/build/index.d.ts

@@ -0,0 +1,4 @@
+export * from './flows';
+export * from './report';
+export * from './runner';
+export * from './types';

package/build/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./flows"), exports);
+__exportStar(require("./report"), exports);
+__exportStar(require("./runner"), exports);
+__exportStar(require("./types"), exports);

package/build/report.d.ts

@@ -0,0 +1,12 @@
+import { ArtifactSummary, QaTestRunResult } from './types';
+export declare function safeName(value: string): string;
+export declare function stepScreenshotFileName(index: number, stepId: string): string;
+export declare function writeArtifacts(input: {
+    outDir: string;
+    scenario: string;
+    targetUrl: string;
+    manifestPath: string;
+    manifestFound: boolean;
+    result: QaTestRunResult;
+}): Promise<ArtifactSummary>;
+export declare function openReport(reportPath: string): Promise<void>;