canary-lab 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Canary Lab Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,282 @@
+# Canary Lab
+
+Canary Lab is an npm package for end-to-end testing with Playwright, local service orchestration, and agent-assisted debugging. It scaffolds a runnable E2E project, starts dependent services, captures service logs, and provides a structured self-fixing workflow for Claude or Codex.
+
+If you are looking for a small E2E testing starter for local development, a Playwright-based test harness, or a self-fixing demo for AI coding agents, this package aims to be a useful starting point.
+
+## What It Tries To Help With
+
+- Scaffold an E2E test project quickly
+- Run Playwright tests against local services from one command
+- Capture per-service logs and a machine-readable failure summary
+- Give Claude or Codex a documented workflow for diagnosing and fixing failures
+- Keep generated projects visible while shipping package internals as compiled code
+
+## For Users
+
+Use Canary Lab when you want to bootstrap a local E2E testing project and experiment with an agent-assisted self-fixing loop.
+
+### Quick Start
+
+```bash
+npx canary-lab init my-lab
+cd my-lab
+npm install
+npm run install:browsers
+npx canary-lab run
+```
+
+### What Gets Scaffolded
+
+The generated project includes:
+
+- `features/example_todo_api` as a working Playwright E2E sample
+- `features/broken_todo_api` as an intentionally broken sample for self-fixing
+- `CLAUDE.md` and `.claude/skills/self-fixing-loop.md` for Claude
+- `AGENTS.md` and `.codex/self-fixing-loop.md` for Codex
+
+### Main Commands
+
+```bash
+npx canary-lab init <folder>
+npx canary-lab run
+npx canary-lab env
+npx canary-lab new-feature <name> "Description"
+```
+
+### Environment Switching
+
+`npx canary-lab env` helps manage temporary environment files for a feature.
+
+This is useful when you want to:
+
+- apply a known env setup before running E2E tests
+- switch between different local test configurations
+- avoid manually editing `.env` files each time
+- restore the previous env files after the run
+
+When you run `npx canary-lab env`, Canary Lab:
+
+- finds features that define env sets
+- lets you choose whether to apply or revert
+- backs up the current target env files
+- applies the selected env set
+- restores the previous files when you revert
+
+An env set is a named group of environment files for a feature, usually stored under `features/<feature>/envsets/`.
+
+Typical usage:
+
+```bash
+npx canary-lab env
+```
+
+Then choose the feature and env set you want, such as `local`.
+
+### How `envsets.config.json` Works
+
+Each feature can define reusable environment setups under:
+
+```bash
+features/<feature>/envsets/
+```
+
+The main config file is `envsets.config.json`. It tells Canary Lab:
+
+- where your local apps live
+- which files it is allowed to swap
+- which of those files belong to the feature
+
+Example:
+
+```json
+{
+  "appRoots": {
+    "CANARY_LAB": "/Users/me/Documents/canary-lab",
+    "APP_A": "/Users/me/Documents/app-a",
+    "APP_B": "/Users/me/Documents/app-b",
+    "APP_C": "/Users/me/Documents/app-c"
+  },
+  "slots": {
+    "feature.env": {
+      "description": "Feature .env file",
+      "target": "$CANARY_LAB/features/sample_feature/.env"
+    },
+    "app-a.env.local": {
+      "description": "App A local env file",
+      "target": "$APP_A/.env.local"
+    },
+    "app-b.env.local": {
+      "description": "App B local env file",
+      "target": "$APP_B/.env.local"
+    },
+    "app-c.config.json": {
+      "description": "App C local config file",
+      "target": "$APP_C/config/local.json"
+    }
+  },
+  "feature": {
+    "slots": [
+      "feature.env",
+      "app-a.env.local",
+      "app-b.env.local",
+      "app-c.config.json"
+    ],
+    "testCommand": "npm run test:e2e",
+    "testCwd": "$CANARY_LAB/features/sample_feature"
+  }
+}
+```
+
+A simple way to read this:
+
+- `appRoots` defines the base folders for your local repos or apps
+- `slots` defines the real files Canary Lab is allowed to replace temporarily
+- `feature.slots` lists which of those files this feature uses
+- folders like `envsets/local/` or `envsets/debug/` contain the actual replacement files
+
+When you apply an env set, Canary Lab:
+
+1. finds the target files listed in `feature.slots`
+2. backs up the current files
+3. copies the selected env-set files into place
+4. lets you revert them later
+
+This is useful when one test flow depends on env files across multiple local apps and you want to switch them together instead of editing them by hand.
+
+### Self-Fixing Workflow
+
+1. Run `npx canary-lab run`
+2. Choose the broken sample
+3. Leave the runner open in watch mode
+4. Open Claude or Codex in the generated project
+5. Type:
+
+```text
+self heal
+```
+
+`self heal` is a documented agent phrase, not a CLI command. It tells the agent to follow its canonical self-fixing workflow, inspect `logs/e2e-summary.json` and `logs/svc-*.log`, fix implementation only, and trigger `touch logs/.restart` or `touch logs/.rerun` as needed.
+
+### What Makes It Different
+
+Compared with a minimal E2E starter, Canary Lab also includes:
+
+- a runner that launches services and waits on health checks
+- structured failure context for agents
+- generated project guidance for both Claude and Codex
+- a built-in broken sample to exercise the workflow end to end
+
+### Observability Matters
+
+Canary Lab works best when the application under test emits useful logs.
+
+The self-fixing workflow depends on being able to correlate a failing test with the service output that happened during that test window. If a service produces little or no log output, the agent has much less context to work with, and the loop becomes correspondingly less useful.
+
+In practice, this means basic observability still matters:
+
+- services should write meaningful stdout or stderr logs
+- error paths should emit enough context to explain what failed
+- health checks and startup logs should make service state visible
+
+Canary Lab can organize and narrow the logs for a given test case, but it cannot recover context that the application never emitted.
+
+## For Contributors
+
+Use this repo when you are improving the package itself: the CLI, scaffold templates, runtime, smoke tests, or publish flow.
+
+### Local Development
+
+```bash
+npm install
+npm run build
+```
+
+### How It Works
+
+```mermaid
+flowchart TD
+    A["Canary Lab spawns app processes in terminal tabs"] --> B["Apps write stdout to logs/svc-*.log"]
+    B --> C["Canary Lab waits for health checks to pass"]
+    C --> D["Canary Lab runs Playwright tests"]
+    D --> E["Before each test, the runner appends &lt;test-tag&gt; to each service log"]
+    E --> F["After each test, the runner appends &lt;/test-tag&gt;"]
+    F --> G["The run writes logs/e2e-summary.json"]
+    G --> H["Agent is told: self heal"]
+    H --> I["Agent slices the matching log chunk, reads the failure, and fixes code"]
+    I --> J["Agent signals .restart or .rerun"]
+    J --> D
+```
+
+The basic flow is:
+
+- Canary Lab starts the required apps in terminal tabs
+- those apps write stdout to `logs/svc-*.log`
+- once health checks pass, Canary Lab runs Playwright
+- before a test runs, the runner appends a `<test-tag>` marker to each service log
+- after the test finishes, the runner appends the matching `</test-tag>`
+- the run writes `logs/e2e-summary.json`
+
+After that, an agent can be told:
+
+```text
+self heal
+```
+
+At that point the agent can:
+
+- `logs/e2e-summary.json`
+- `logs/svc-*.log`
+- the canonical self-fixing workflow doc
+
+and use the tags to read the matching chunk of each service log for the failed test, fix the implementation, and signal `touch logs/.restart` or `touch logs/.rerun`.
+
+### Repository Areas
+
+- `scripts/` contains the package CLI and scaffold commands
+- `shared/` contains the runtime behind `run` and `env`
+- `templates/project/` contains the generated project files
+- `feature-support/` contains the public imports used by generated projects
+
+### Contributor Workflow
+
+Typical loop:
+
+```bash
+npm run build
+npm run smoke:pack
+```
+
+If you change scaffold docs, generated files, or packaging behavior, validate by scaffolding a fresh temp project from the built package.
+
+### Packaging and Release Checks
+
+```bash
+npm run pack:check
+npm run smoke:pack
+npm run publish:package
+```
+
+- `pack:check` inspects the npm tarball contents
+- `smoke:pack` builds, packs, scaffolds a temp project, installs dependencies, and verifies the scaffold flow
+- `publish:package` runs build and tarball checks before `npm publish`
+
+### Local Tarball Test
+
+Before publishing, prefer testing the packed tarball instead of `npm link`:
+
+```bash
+npm run build
+npm pack
+mkdir /tmp/canary-lab-smoke
+cd /tmp/canary-lab-smoke
+npm init -y
+npm install /absolute/path/to/canary-lab-0.1.0.tgz
+npx canary-lab init test-folder
+```
+
+This tests the exact tarball npm would publish.
+
+## License
+
+[MIT](LICENSE)

package/dist/feature-support/load-env.d.ts

@@ -0,0 +1 @@
+export { loadFeatureEnv } from '../shared/configs/loadEnv';

package/dist/feature-support/load-env.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadFeatureEnv = void 0;
+var loadEnv_1 = require("../shared/configs/loadEnv");
+Object.defineProperty(exports, "loadFeatureEnv", { enumerable: true, get: function () { return loadEnv_1.loadFeatureEnv; } });

package/dist/feature-support/log-marker-fixture.d.ts

@@ -0,0 +1 @@
+export { test, expect, type APIRequestContext, type Page, } from '../shared/e2e-runner/log-marker-fixture';

package/dist/feature-support/log-marker-fixture.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.expect = exports.test = void 0;
+var log_marker_fixture_1 = require("../shared/e2e-runner/log-marker-fixture");
+Object.defineProperty(exports, "test", { enumerable: true, get: function () { return log_marker_fixture_1.test; } });
+Object.defineProperty(exports, "expect", { enumerable: true, get: function () { return log_marker_fixture_1.expect; } });

package/dist/feature-support/playwright-base.d.ts

@@ -0,0 +1 @@
+export { baseConfig } from '../shared/configs/playwright.base';

package/dist/feature-support/playwright-base.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.baseConfig = void 0;
+var playwright_base_1 = require("../shared/configs/playwright.base");
+Object.defineProperty(exports, "baseConfig", { enumerable: true, get: function () { return playwright_base_1.baseConfig; } });

package/dist/feature-support/types.d.ts

@@ -0,0 +1 @@
+export type { FeatureConfig, RepoPrerequisite, StartCommand, HealthCheck, NgrokTunnel, } from '../shared/launcher/types';

package/dist/feature-support/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/scripts/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/scripts/cli.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const runner_1 = require("../shared/e2e-runner/runner");
+const root_cli_1 = require("../shared/env-switcher/root-cli");
+const new_feature_1 = require("./new-feature");
+const init_project_1 = require("./init-project");
+function printUsage() {
+    console.log(`Canary Lab
+
+Usage:
+  canary-lab init <folder> [--package-spec <spec>]
+  canary-lab run
+  canary-lab env
+  canary-lab new-feature <name> [description]
+`);
+}
+async function main() {
+    const [command, ...args] = process.argv.slice(2);
+    switch (command) {
+        case 'init':
+            await (0, init_project_1.main)(args);
+            return;
+        case 'run':
+            await (0, runner_1.main)();
+            return;
+        case 'env':
+            await (0, root_cli_1.main)(args);
+            return;
+        case 'new-feature':
+            await (0, new_feature_1.main)(args);
+            return;
+        case '-h':
+        case '--help':
+        case undefined:
+            printUsage();
+            return;
+        default:
+            console.error(`Unknown command: ${command}`);
+            printUsage();
+            process.exit(1);
+    }
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/scripts/init-project.d.ts

@@ -0,0 +1 @@
+export declare function main(args?: string[]): Promise<void>;

package/dist/scripts/init-project.js

@@ -0,0 +1,110 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.main = main;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+function resolveFirstExisting(pathsToTry) {
+    const match = pathsToTry.find((candidate) => fs_1.default.existsSync(candidate));
+    if (!match) {
+        throw new Error(`Could not resolve any expected path: ${pathsToTry.join(', ')}`);
+    }
+    return match;
+}
+function getPackageJsonPath() {
+    return resolveFirstExisting([
+        path_1.default.resolve(__dirname, '../package.json'),
+        path_1.default.resolve(__dirname, '../../package.json'),
+    ]);
+}
+function getTemplateRoot() {
+    return resolveFirstExisting([
+        path_1.default.resolve(__dirname, '../templates/project'),
+        path_1.default.resolve(__dirname, '../../templates/project'),
+    ]);
+}
+function copyDir(sourceDir, targetDir) {
+    fs_1.default.mkdirSync(targetDir, { recursive: true });
+    for (const entry of fs_1.default.readdirSync(sourceDir, { withFileTypes: true })) {
+        const sourcePath = path_1.default.join(sourceDir, entry.name);
+        const targetPath = path_1.default.join(targetDir, entry.name);
+        if (entry.isDirectory()) {
+            copyDir(sourcePath, targetPath);
+            continue;
+        }
+        fs_1.default.copyFileSync(sourcePath, targetPath);
+    }
+}
+function readPackageVersion() {
+    const pkgPath = getPackageJsonPath();
+    return JSON.parse(fs_1.default.readFileSync(pkgPath, 'utf-8')).version;
+}
+function parseArgs(args) {
+    const folder = args[0];
+    if (!folder) {
+        console.error('Usage: canary-lab init <folder> [--package-spec <spec>]');
+        process.exit(1);
+    }
+    let packageSpec = `^${readPackageVersion()}`;
+    for (let i = 1; i < args.length; i += 1) {
+        if (args[i] === '--package-spec') {
+            packageSpec = args[i + 1];
+            i += 1;
+        }
+    }
+    if (!packageSpec) {
+        console.error('Missing value for --package-spec');
+        process.exit(1);
+    }
+    return { folder, packageSpec };
+}
+function buildPackageJson(projectName, packageSpec) {
+    return JSON.stringify({
+        name: projectName,
+        private: true,
+        version: '0.1.0',
+        description: 'Canary Lab project scaffold',
+        scripts: {
+            'canary-lab:run': 'canary-lab run',
+            'canary-lab:env': 'canary-lab env',
+            'canary-lab:new-feature': 'canary-lab new-feature',
+            'install:browsers': 'playwright install chromium',
+        },
+        devDependencies: {
+            '@playwright/test': '^1.54.2',
+            'canary-lab': packageSpec,
+        },
+    }, null, 2) + '\n';
+}
+async function main(args = process.argv.slice(2)) {
+    const { folder, packageSpec } = parseArgs(args);
+    const targetDir = path_1.default.resolve(process.cwd(), folder);
+    if (fs_1.default.existsSync(targetDir)) {
+        const entries = fs_1.default.readdirSync(targetDir);
+        if (entries.length > 0) {
+            console.error(`Target directory is not empty: ${targetDir}`);
+            process.exit(1);
+        }
+    }
+    else {
+        fs_1.default.mkdirSync(targetDir, { recursive: true });
+    }
+    copyDir(getTemplateRoot(), targetDir);
+    const projectName = path_1.default.basename(targetDir);
+    fs_1.default.writeFileSync(path_1.default.join(targetDir, 'package.json'), buildPackageJson(projectName, packageSpec));
+    console.log(`\n  Canary Lab project created at ${targetDir}\n`);
+    console.log('  Next steps:');
+    console.log(`    1. cd ${folder}`);
+    console.log('    2. npm install');
+    console.log('    3. npm run install:browsers');
+    console.log('    4. npx canary-lab run');
+    console.log('');
+}
+if (require.main === module) {
+    main().catch((err) => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/scripts/new-feature.d.ts

@@ -0,0 +1 @@
+export declare function main(args?: string[]): Promise<void>;

package/dist/scripts/new-feature.js

@@ -0,0 +1,135 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.main = main;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const project_root_1 = require("../shared/runtime/project-root");
+function buildFeatureConfig(name, description) {
+    return `const config = {
+  name: '${name}',
+  description: '${description}',
+  envs: ['local'],
+  repos: [
+    // {
+    //   name: 'your-repo',
+    //   localPath: '/absolute/path/to/your-repo',
+    //   cloneUrl: 'git@github.com:your-org/your-repo.git',
+    //   startCommands: [
+    //     {
+    //       name: 'your-repo dev server',
+    //       command: 'npm run dev',
+    //       healthCheck: {
+    //         url: 'http://localhost:3000/',
+    //         timeoutMs: 2000,
+    //       },
+    //     },
+    //   ],
+    // },
+  ],
+  featureDir: __dirname,
+}
+
+module.exports = { config }
+`;
+}
+function buildPlaywrightConfig() {
+    return `const { defineConfig } = require('@playwright/test')
+const { baseConfig } = require('canary-lab/feature-support/playwright-base')
+
+module.exports = defineConfig({
+  ...baseConfig,
+})
+`;
+}
+function buildFeatureConfigJs() {
+    return `const { loadFeatureEnv } = require('canary-lab/feature-support/load-env')
+
+loadFeatureEnv(__dirname + '/..')
+
+const GATEWAY_URL = process.env.GATEWAY_URL ?? 'http://localhost:3000'
+
+module.exports = {
+  GATEWAY_URL,
+}
+`;
+}
+function buildEnvsetsConfig(name) {
+    return JSON.stringify({
+        appRoots: {},
+        slots: {
+            [`${name}.env`]: {
+                description: `Canary Lab ${name} feature .env`,
+                target: `$CANARY_LAB_PROJECT_ROOT/features/${name}/.env`,
+            },
+        },
+        feature: {
+            slots: [`${name}.env`],
+            testCommand: 'npx playwright test',
+            testCwd: `$CANARY_LAB_PROJECT_ROOT/features/${name}`,
+        },
+    }, null, 2) + '\n';
+}
+function buildSpec(name) {
+    return `const { test, expect } = require('canary-lab/feature-support/log-marker-fixture')
+
+test.describe('${name}', () => {
+  test('example test', async () => {
+    expect(true).toBe(true)
+  })
+})
+`;
+}
+async function main(args = process.argv.slice(2)) {
+    const name = args[0];
+    const description = args.slice(1).join(' ') || 'TODO: add description';
+    if (!name) {
+        console.error('Usage: canary-lab new-feature <name> [description]');
+        process.exit(1);
+    }
+    if (!/^[a-z][a-z0-9_]*$/.test(name)) {
+        console.error(`Invalid feature name "${name}". Use snake_case (e.g. cns_webhooks).`);
+        process.exit(1);
+    }
+    const featureDir = path_1.default.join((0, project_root_1.getFeaturesDir)(), name);
+    if (fs_1.default.existsSync(featureDir)) {
+        console.error(`Feature "${name}" already exists at ${featureDir}`);
+        process.exit(1);
+    }
+    const files = [
+        ['feature.config.cjs', buildFeatureConfig(name, description)],
+        ['playwright.config.js', buildPlaywrightConfig()],
+        ['.env.example', 'GATEWAY_URL=http://localhost:3000\n'],
+        ['src/config.js', buildFeatureConfigJs()],
+        ['envsets/envsets.config.json', buildEnvsetsConfig(name)],
+        [`envsets/local/${name}.env`, 'GATEWAY_URL=http://localhost:3000\n'],
+        [`e2e/${name}.spec.js`, buildSpec(name)],
+    ];
+    for (const [relPath, content] of files) {
+        const fullPath = path_1.default.join(featureDir, relPath);
+        fs_1.default.mkdirSync(path_1.default.dirname(fullPath), { recursive: true });
+        fs_1.default.writeFileSync(fullPath, content);
+    }
+    fs_1.default.mkdirSync(path_1.default.join(featureDir, 'e2e/helpers'), { recursive: true });
+    console.log(`\n  Feature "${name}" created at features/${name}/\n`);
+    console.log('  Created files:');
+    for (const [relPath] of files) {
+        console.log(`    features/${name}/${relPath}`);
+    }
+    console.log(`    features/${name}/e2e/helpers/`);
+    console.log('');
+    console.log('  Next steps:');
+    console.log('    1. Edit feature.config.cjs — add your repos, start commands, and health checks');
+    console.log('    2. Edit src/config.js — add feature-specific env var exports');
+    console.log(`    3. Write your tests in e2e/${name}.spec.js`);
+    console.log('    4. Run: npx canary-lab run');
+    console.log('');
+}
+if (require.main === module) {
+    main().catch((err) => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/shared/configs/loadEnv.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Load .env file from a feature directory. Call this at the top of
+ * your feature's src/config.ts before exporting typed constants.
+ */
+export declare function loadFeatureEnv(featureDir: string): void;

package/dist/shared/configs/loadEnv.js

@@ -0,0 +1,19 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadFeatureEnv = loadFeatureEnv;
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const dotenv_1 = __importDefault(require("dotenv"));
+/**
+ * Load .env file from a feature directory. Call this at the top of
+ * your feature's src/config.ts before exporting typed constants.
+ */
+function loadFeatureEnv(featureDir) {
+    const envFile = path_1.default.join(featureDir, '.env');
+    if (fs_1.default.existsSync(envFile)) {
+        dotenv_1.default.config({ path: envFile });
+    }
+}

package/dist/shared/configs/playwright.base.d.ts

@@ -0,0 +1,12 @@
+export declare const baseConfig: {
+    testDir: string;
+    fullyParallel: boolean;
+    workers: number;
+    retries: number;
+    reporter: any[];
+    timeout: number;
+    use: {
+        trace: "retain-on-failure";
+        screenshot: "only-on-failure";
+    };
+};