@kirrosh/apitool 0.4.3 → 0.5.1

package/APITOOL.md

@@ -39,7 +39,7 @@ src/
 │   ├── reporter/     Console, JSON, JUnit XML
 │   └── agent/        AI Chat (AI SDK v6, tool calling)
 ├── db/               SQLite (runs, collections, environments)
-├── mcp/              MCP Server (15 tools)
+├── mcp/              MCP Server (17 tools)
 ├── web/              Hono + HTMX dashboard
 └── cli/              16 CLI commands
 ```
@@ -74,12 +74,56 @@ SQLite auto-created. Tables: `collections`, `runs`, `results`, `environments`, `
 Single-page dashboard: API selector → env selector → Run Tests → results + coverage + history. JUnit/JSON export. Hono + HTMX.
 
 ### MCP Server
-15 tools for AI agent integration. Primary test generation flow:
+17 tools for AI agent integration. Primary test generation flow:
 
 ```
 generate_tests_guide → [agent writes YAML] → save_test_suite → run_tests → diagnose_failure → ci_init
 ```
 
+### Safe Test Coverage Workflow
+
+**When the user asks to "safely cover", "test without breaking anything", or "start with read-only tests" — follow this 4-phase approach:**
+
+**Step 0 (required for npx MCP — single shared server):**
+```
+set_work_dir(workDir: "<absolute path to project root>")
+```
+Call this once at the start of the session so `apitool.db` and all relative paths resolve to your project directory.
+
+**Phase 0 — Register + static analysis (zero requests)**
+```
+setup_api(...)
+coverage_analysis(specPath, testsDir)   ← baseline, no HTTP
+```
+
+**Phase 1 — Smoke tests (GET-only, safe for production)**
+```
+generate_tests_guide(specPath, methodFilter: ["GET"])   ← GET endpoints only
+save_test_suite(...)                                    ← tags: [smoke]
+run_tests(testPath, safe: true)                         ← --safe enforces GET-only
+```
+Stop here if the user hasn't explicitly confirmed a staging/test environment.
+
+**Phase 2 — CRUD tests (only with explicit user confirmation + staging env)**
+```
+run_tests(testPath, tag: ["crud"], dryRun: true)        ← show requests first, no sending
+[show user what would be sent, ask confirmation]
+run_tests(testPath, tag: ["crud"], envName: "staging")  ← only after confirmation
+```
+
+**Phase 3 — Regression tracking**
+```
+query_db(action: "compare_runs", runId: prev, runIdB: curr)
+ci_init()
+```
+
+**Key safety rules:**
+- `safe: true` on `run_tests` → only GET requests execute, write ops are skipped
+- `dryRun: true` on `run_tests` → shows all requests without sending any
+- `methodFilter: ["GET"]` on `generate_tests_guide` → only generates GET test stubs
+- Always use `tags: [smoke]` for GET-only suites, `tags: [crud]` for write operations
+- Never run CRUD tests unless user confirmed environment is safe (staging/test)
+
 ### CI/CD
 `apitool ci init` scaffolds GitHub Actions or GitLab CI workflow. Supports schedule, repository_dispatch, manual triggers. See [docs/ci.md](docs/ci.md).
 
@@ -102,6 +146,8 @@ generate_tests_guide → [agent writes YAML] → save_test_suite → run_tests
 | `manage_environment` | CRUD for environments |
 | `manage_server` | Start/stop WebUI server |
 | `ci_init` | Generate CI/CD workflow (GitHub Actions / GitLab CI) |
+| `set_work_dir` | Set project root for the session (call FIRST with npx MCP) |
+| `describe_endpoint` | Full details for one endpoint: params, schemas, response headers, security |
 
 ## CLI Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kirrosh/apitool",
-  "version": "0.4.3",
+  "version": "0.5.1",
   "description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
   "license": "MIT",
   "module": "index.ts",

package/scripts/run-mocked-tests.ts

@@ -6,7 +6,6 @@
 const MOCKED_FILES = [
   "tests/agent/tools/diagnose-failure.test.ts",
   "tests/agent/tools/explore-api.test.ts",
-  "tests/agent/tools/generate-tests.test.ts",
   "tests/agent/tools/manage-environment.test.ts",
   "tests/agent/tools/query-results.test.ts",
   "tests/agent/tools/run-tests.test.ts",
@@ -15,7 +14,6 @@ const MOCKED_FILES = [
   "tests/mcp/coverage-analysis.test.ts",
   "tests/mcp/explore-api.test.ts",
   "tests/mcp/send-request.test.ts",
-  "tests/cli/request.test.ts",
   "tests/cli/coverage.test.ts",
 ];
 

package/src/cli/commands/ci-init.ts

@@ -33,43 +33,80 @@ jobs:
       - name: Install apitool
         run: curl -fsSL https://raw.githubusercontent.com/kirrosh/apitool/master/install.sh | sh
 
-      - name: Run tests
+      - name: Check coverage
+        run: apitool coverage --api myapi --fail-on-coverage 60
+        # Fails if coverage drops below 60% — adjust threshold as needed
+
+      - name: Run smoke tests (read-only, safe for production)
         run: |
           mkdir -p test-results
-          apitool run apis/ --report junit --no-db > test-results/junit.xml
-          # Add --env <name> to load .env.<name>.yaml from test directory
+          apitool run apis/ --tag smoke --safe --report junit --no-db > test-results/smoke.xml
+          # Use --env-var "API_KEY=\${{ secrets.API_KEY }}" to inject secrets without writing to disk
+        continue-on-error: true
+
+      - name: Run CRUD tests (staging only)
+        run: |
+          apitool run apis/ --tag crud --env staging --report junit --no-db > test-results/crud.xml
+          # Add --env-var "BASE_URL=\${{ secrets.STAGING_URL }}" for staging URL
         continue-on-error: true
 
       - name: Publish test results
         uses: EnricoMi/publish-unit-test-result-action@v2
         if: always()
         with:
-          files: test-results/junit.xml
+          files: test-results/*.xml
 
       - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: test-results
-          path: test-results/junit.xml
+          path: test-results/
 `;
 
 const GITLAB_CI_TEMPLATE = `# Trigger via API: curl -X POST --form ref=main --form token=TRIGGER_TOKEN $CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline
 
-api-tests:
+variables:
+  # Set API_KEY in GitLab CI/CD → Settings → Variables
+  API_KEY: ""
+
+api-coverage:
+  image: ubuntu:latest
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq curl
+    - curl -fsSL https://raw.githubusercontent.com/kirrosh/apitool/master/install.sh | sh
+  script:
+    - apitool coverage --api myapi --fail-on-coverage 60
+
+api-smoke:
+  image: ubuntu:latest
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq curl
+    - curl -fsSL https://raw.githubusercontent.com/kirrosh/apitool/master/install.sh | sh
+  script:
+    - mkdir -p test-results
+    # Use --env-var to inject secrets without writing to disk
+    - apitool run apis/ --tag smoke --safe --report junit --no-db --env-var "API_KEY=$API_KEY" > test-results/smoke.xml
+  allow_failure:
+    exit_codes: 1
+  artifacts:
+    when: always
+    reports:
+      junit: test-results/smoke.xml
+
+api-crud:
   image: ubuntu:latest
   before_script:
     - apt-get update -qq && apt-get install -y -qq curl
     - curl -fsSL https://raw.githubusercontent.com/kirrosh/apitool/master/install.sh | sh
   script:
     - mkdir -p test-results
-    - apitool run apis/ --report junit --no-db > test-results/junit.xml
-    # Add --env <name> to load .env.<name>.yaml from test directory
+    - apitool run apis/ --tag crud --env staging --report junit --no-db > test-results/crud.xml
   allow_failure:
     exit_codes: 1
   artifacts:
     when: always
     reports:
-      junit: test-results/junit.xml
+      junit: test-results/crud.xml
 `;
 
 function writeIfMissing(filePath: string, content: string, force: boolean): boolean {
@@ -86,10 +123,10 @@ function writeIfMissing(filePath: string, content: string, force: boolean): bool
   return true;
 }
 
-function detectPlatform(cwd: string): "github" | "gitlab" | null {
+function detectPlatform(cwd: string): "github" | "gitlab" | undefined {
   if (existsSync(resolve(cwd, ".github"))) return "github";
   if (existsSync(resolve(cwd, ".gitlab-ci.yml"))) return "gitlab";
-  return null;
+  return undefined;
 }
 
 export async function ciInitCommand(options: CiInitOptions): Promise<number> {

package/src/cli/commands/compare.ts

@@ -0,0 +1,129 @@
+import { getDb } from "../../db/schema.ts";
+import { getRunById, getResultsByRunId } from "../../db/queries.ts";
+import { printError } from "../output.ts";
+
+const RESET = "\x1b[0m";
+const GREEN = "\x1b[32m";
+const RED = "\x1b[31m";
+const YELLOW = "\x1b[33m";
+const BOLD = "\x1b[1m";
+
+function useColor(): boolean {
+  return process.stdout.isTTY ?? false;
+}
+
+export interface CompareOptions {
+  runA: number;
+  runB: number;
+  dbPath?: string;
+}
+
+export async function compareCommand(options: CompareOptions): Promise<number> {
+  const { runA, runB, dbPath } = options;
+
+  try {
+    getDb(dbPath);
+
+    const runARecord = getRunById(runA);
+    const runBRecord = getRunById(runB);
+
+    if (!runARecord) {
+      printError(`Run #${runA} not found`);
+      return 2;
+    }
+    if (!runBRecord) {
+      printError(`Run #${runB} not found`);
+      return 2;
+    }
+
+    const resultsA = getResultsByRunId(runA);
+    const resultsB = getResultsByRunId(runB);
+
+    // Build lookup maps: "suite_name::test_name" → status
+    const mapA = new Map<string, string>();
+    const mapB = new Map<string, string>();
+
+    for (const r of resultsA) {
+      mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
+    }
+    for (const r of resultsB) {
+      mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
+    }
+
+    const regressions: Array<{ suite: string; test: string; before: string; after: string }> = [];
+    const fixes: Array<{ suite: string; test: string; before: string; after: string }> = [];
+    const unchanged: number[] = [];
+    let newTests = 0;
+    let removedTests = 0;
+
+    // Check all keys from B (current run)
+    for (const [key, statusB] of mapB) {
+      const statusA = mapA.get(key);
+      if (statusA === undefined) {
+        newTests++;
+        continue;
+      }
+      const wasPass = statusA === "pass";
+      const isPass = statusB === "pass";
+      const wasFail = statusA === "fail" || statusA === "error";
+      const isFail = statusB === "fail" || statusB === "error";
+
+      const [suite, test] = key.split("::") as [string, string];
+
+      if (wasPass && isFail) {
+        regressions.push({ suite, test, before: statusA, after: statusB });
+      } else if (wasFail && isPass) {
+        fixes.push({ suite, test, before: statusA, after: statusB });
+      } else {
+        unchanged.push(1);
+      }
+    }
+
+    // Count removed tests
+    for (const key of mapA.keys()) {
+      if (!mapB.has(key)) removedTests++;
+    }
+
+    const color = useColor();
+
+    // Header
+    console.log(`\nComparing run #${runA} (${runARecord.started_at.slice(0, 19)}) → run #${runB} (${runBRecord.started_at.slice(0, 19)})\n`);
+
+    // Summary line
+    const parts = [
+      `${color ? BOLD : ""}${regressions.length} regressions${color ? RESET : ""}`,
+      `${fixes.length} fixes`,
+      `${unchanged.length} unchanged`,
+    ];
+    if (newTests > 0) parts.push(`${newTests} new`);
+    if (removedTests > 0) parts.push(`${removedTests} removed`);
+    console.log(parts.join("  |  ") + "\n");
+
+    // Regressions
+    if (regressions.length > 0) {
+      console.log(`${color ? RED + BOLD : ""}Regressions (pass → fail):${color ? RESET : ""}`);
+      for (const r of regressions) {
+        console.log(`  ${color ? RED : ""}✗${color ? RESET : ""} [${r.suite}] ${r.test}  (${r.before} → ${r.after})`);
+      }
+      console.log("");
+    }
+
+    // Fixes
+    if (fixes.length > 0) {
+      console.log(`${color ? GREEN : ""}Fixes (fail → pass):${color ? RESET : ""}`);
+      for (const f of fixes) {
+        console.log(`  ${color ? GREEN : ""}✓${color ? RESET : ""} [${f.suite}] ${f.test}  (${f.before} → ${f.after})`);
+      }
+      console.log("");
+    }
+
+    if (regressions.length === 0 && fixes.length === 0) {
+      console.log(`${color ? GREEN : ""}No regressions detected.${color ? RESET : ""}`);
+    }
+
+    return regressions.length > 0 ? 1 : 0;
+  } catch (err) {
+    printError(err instanceof Error ? err.message : String(err));
+    return 2;
+  }
+}

package/src/cli/commands/coverage.ts

@@ -4,6 +4,7 @@ import { printError, printSuccess } from "../output.ts";
 export interface CoverageOptions {
   spec: string;
   tests: string;
+  failOnCoverage?: number;
 }
 
 const RESET = "\x1b[0m";
@@ -57,6 +58,9 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
       }
     }
 
+    if (options.failOnCoverage !== undefined) {
+      return percentage < options.failOnCoverage ? 1 : 0;
+    }
     return uncovered.length > 0 ? 1 : 0;
   } catch (err) {
     printError(err instanceof Error ? err.message : String(err));

package/src/cli/commands/run.ts

@@ -1,4 +1,5 @@
 import { dirname } from "path";
+import { stat } from "node:fs/promises";
 import { parse } from "../../core/parser/yaml-parser.ts";
 import { loadEnvironment } from "../../core/parser/variables.ts";
 import { filterSuitesByTags } from "../../core/parser/filter.ts";
@@ -22,6 +23,8 @@ export interface RunOptions {
   authToken?: string;
   safe?: boolean;
   tag?: string[];
+  envVars?: string[];
+  dryRun?: boolean;
 }
 
 export async function runCommand(options: RunOptions): Promise<number> {
@@ -61,12 +64,16 @@ export async function runCommand(options: RunOptions): Promise<number> {
   }
 
   // 2. Load environment (resolve collection for scoped envs)
-  const searchDir = dirname(options.path);
+  // Use path itself as searchDir if it's a directory; dirname() on a dir path gives the parent
+  const pathStat = await stat(options.path).catch(() => null);
+  const searchDir = pathStat?.isDirectory() ? options.path : dirname(options.path);
   let collectionForEnv: { id: number } | null = null;
-  try {
-    getDb(options.dbPath);
-    collectionForEnv = findCollectionByTestPath(options.path);
-  } catch { /* DB not available — OK */ }
+  if (!options.noDb) {
+    try {
+      getDb(options.dbPath);
+      collectionForEnv = findCollectionByTestPath(options.path);
+    } catch { /* DB not available — OK */ }
+  }
 
   let env: Record<string, string> = {};
   try {
@@ -81,6 +88,16 @@ export async function runCommand(options: RunOptions): Promise<number> {
     env.auth_token = options.authToken;
   }
 
+  // Inject --env-var KEY=VALUE overrides (highest priority)
+  if (options.envVars && options.envVars.length > 0) {
+    for (const pair of options.envVars) {
+      const eqIdx = pair.indexOf("=");
+      if (eqIdx > 0) {
+        env[pair.slice(0, eqIdx)] = pair.slice(eqIdx + 1);
+      }
+    }
+  }
+
   // Warn if --env was explicitly set but file was not found (empty env)
   if (options.env && Object.keys(env).length === 0) {
     printWarning(`Environment file .env.${options.env}.yaml not found in ${searchDir}`);
@@ -95,18 +112,19 @@ export async function runCommand(options: RunOptions): Promise<number> {
 
   // 4. Run suites
   const results: TestRunResult[] = [];
+  const dryRun = options.dryRun === true;
   if (options.bail) {
     // Sequential with bail at suite level
     for (const suite of suites) {
-      const result = await runSuite(suite, env);
+      const result = await runSuite(suite, env, dryRun);
       results.push(result);
-      if (result.failed > 0 || result.steps.some((s) => s.status === "error")) {
+      if (!dryRun && (result.failed > 0 || result.steps.some((s) => s.status === "error"))) {
         break;
       }
     }
   } else {
     // Parallel
-    const all = await Promise.all(suites.map((suite) => runSuite(suite, env)));
+    const all = await Promise.all(suites.map((suite) => runSuite(suite, env, dryRun)));
     results.push(...all);
   }
 
@@ -131,7 +149,8 @@ export async function runCommand(options: RunOptions): Promise<number> {
     }
   }
 
-  // 7. Exit code
+  // 7. Exit code (always 0 in dry-run mode)
+  if (dryRun) return 0;
   const hasFailures = results.some((r) => r.failed > 0 || r.steps.some((s) => s.status === "error"));
   return hasFailures ? 1 : 0;
 }

package/src/cli/index.ts

@@ -15,13 +15,14 @@ import { coverageCommand } from "./commands/coverage.ts";
 import { doctorCommand } from "./commands/doctor.ts";
 import { addApiCommand } from "./commands/add-api.ts";
 import { ciInitCommand } from "./commands/ci-init.ts";
+import { compareCommand } from "./commands/compare.ts";
 import { printError } from "./output.ts";
 import { getRuntimeInfo } from "./runtime.ts";
 import { getDb } from "../db/schema.ts";
 import { findCollectionByNameOrId } from "../db/queries.ts";
 import type { ReporterName } from "../core/reporter/types.ts";
 
-export const VERSION = "0.3.0";
+export const VERSION = "0.5.0";
 
 export interface ParsedArgs {
   command: string | undefined;
@@ -88,6 +89,7 @@ Usage:
   apitool mcp              Start MCP server (stdio transport for AI agents)
                            --dir <path>  Set working directory (relative paths resolve here)
   apitool chat             Start interactive AI chat for API testing
+  apitool compare <runA> <runB>  Compare two test runs (regressions/fixes)
   apitool doctor           Run diagnostic checks
   apitool update           Update to latest version
 
@@ -117,12 +119,19 @@ Options for 'runs':
   runs <id>            Show run details with step results
   --limit <n>          Number of runs to show (default: 20)
 
+Options for 'compare':
+  compare <runA> <runB>   Compare two run IDs
+  Exit code 1 if regressions found, 0 otherwise
+
 Options for 'coverage':
   --api <name>         Use API collection (auto-resolves spec and tests dir)
   --spec <path>        Path to OpenAPI spec (required unless --api used)
   --tests <dir>        Path to test files directory (required unless --api used)
+  --fail-on-coverage N Exit 1 when coverage percentage is below N (0–100)
 
 Options for 'run':
+  --dry-run            Show requests without sending them (exit code always 0)
+  --env-var KEY=VALUE  Inject env variable (repeatable, overrides env file)
   --api <name>         Use API collection (resolves test path automatically)
   --env <name>         Use environment file (.env.<name>.yaml)
   --report <format>    Output format: console, json, junit (default: console)
@@ -248,15 +257,22 @@ async function main(): Promise<number> {
         }
       }
 
-      // Collect all --tag flags (parseArgs only stores last one, so re-parse)
+      // Collect all --tag and --env-var flags (parseArgs only stores last one, so re-parse)
       const tagValues: string[] = [];
+      const envVarValues: string[] = [];
       const rawRunArgs = process.argv.slice(2);
       for (let i = 0; i < rawRunArgs.length; i++) {
-        if (rawRunArgs[i] === "--tag" && rawRunArgs[i + 1]) {
+        const arg = rawRunArgs[i]!;
+        if (arg === "--tag" && rawRunArgs[i + 1]) {
           tagValues.push(rawRunArgs[i + 1]!);
           i++;
-        } else if (rawRunArgs[i]?.startsWith("--tag=")) {
-          tagValues.push(rawRunArgs[i]!.slice("--tag=".length));
+        } else if (arg.startsWith("--tag=")) {
+          tagValues.push(arg.slice("--tag=".length));
+        } else if (arg === "--env-var" && rawRunArgs[i + 1]) {
+          envVarValues.push(rawRunArgs[i + 1]!);
+          i++;
+        } else if (arg.startsWith("--env-var=")) {
+          envVarValues.push(arg.slice("--env-var=".length));
         }
       }
       // Support comma-separated: --tag smoke,crud → ["smoke", "crud"]
@@ -273,6 +289,8 @@ async function main(): Promise<number> {
         authToken: typeof flags["auth-token"] === "string" ? flags["auth-token"] : undefined,
         safe: flags["safe"] === true,
         tag: tags.length > 0 ? tags : undefined,
+        envVars: envVarValues.length > 0 ? envVarValues : undefined,
+        dryRun: flags["dry-run"] === true,
       });
     }
 
@@ -440,6 +458,26 @@ async function main(): Promise<number> {
       });
     }
 
+    case "compare": {
+      const rawA = positional[0];
+      const rawB = positional[1];
+      if (!rawA || !rawB) {
+        printError("Usage: apitool compare <runA> <runB>");
+        return 2;
+      }
+      const runA = parseInt(rawA, 10);
+      const runB = parseInt(rawB, 10);
+      if (isNaN(runA) || isNaN(runB)) {
+        printError("Run IDs must be integers");
+        return 2;
+      }
+      return compareCommand({
+        runA,
+        runB,
+        dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
+      });
+    }
+
     case "doctor": {
       return doctorCommand({
         dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
@@ -472,7 +510,16 @@ async function main(): Promise<number> {
         printError("Missing --tests <dir>. Usage: apitool coverage --spec <path> --tests <dir>");
         return 2;
       }
-      return coverageCommand({ spec, tests });
+      const failOnCoverageRaw = flags["fail-on-coverage"];
+      let failOnCoverage: number | undefined;
+      if (typeof failOnCoverageRaw === "string") {
+        failOnCoverage = parseInt(failOnCoverageRaw, 10);
+        if (isNaN(failOnCoverage) || failOnCoverage < 0 || failOnCoverage > 100) {
+          printError(`Invalid --fail-on-coverage value: ${failOnCoverageRaw} (must be 0–100)`);
+          return 2;
+        }
+      }
+      return coverageCommand({ spec, tests, failOnCoverage });
     }
 
     default: {

package/src/core/generator/openapi-reader.ts

@@ -10,7 +10,7 @@ export async function readOpenApiSpec(specPath: string): Promise<OpenAPIV3.Docum
     const resp = await fetch(specPath);
     if (!resp.ok) throw new Error(`Failed to fetch spec: ${resp.status} ${resp.statusText}`);
     const spec = await resp.json();
-    const api = await dereference(spec);
+    const api = await dereference(spec as string);
     return api as OpenAPIV3.Document;
   }
   const api = await dereference(specPath);

package/src/core/parser/types.ts

@@ -45,6 +45,8 @@ export interface TestSuite {
   headers?: Record<string, string>;
   config: SuiteConfig;
   tests: TestStep[];
+  /** Absolute path to the source file, set by yaml-parser */
+  filePath?: string;
 }
 
 export type Environment = Record<string, string>;

package/src/core/parser/yaml-parser.ts

@@ -19,7 +19,7 @@ export async function parseFile(filePath: string): Promise<TestSuite> {
 
   try {
     const suite = validateSuite(raw);
-    (suite as any)._source = filePath;
+    suite.filePath = filePath;
     return suite;
   } catch (err) {
     throw new Error(`Validation error in ${filePath}: ${(err as Error).message}`);

package/src/core/runner/execute-run.ts

@@ -15,6 +15,8 @@ export interface ExecuteRunOptions {
   dbPath?: string;
   safe?: boolean;
   tag?: string[];
+  envVars?: Record<string, string>;
+  dryRun?: boolean;
 }
 
 export interface ExecuteRunResult {
@@ -50,7 +52,8 @@ export async function executeRun(options: ExecuteRunOptions): Promise<ExecuteRun
   }
 
   const fileStat = await stat(testPath).catch(() => null);
-  const envDir = fileStat?.isDirectory() ? testPath : dirname(testPath);
+  const isDirectory = fileStat?.isDirectory() ?? false;
+  const envDir = isDirectory ? testPath : dirname(testPath);
 
   getDb(dbPath);
   const resolvedPath = resolve(testPath);
@@ -59,8 +62,28 @@ export async function executeRun(options: ExecuteRunOptions): Promise<ExecuteRun
 
   // If no envName given but a collection exists, fall back to "default" for DB lookup
   const effectiveEnvName = envName ?? (collection ? "default" : undefined);
-  const env = await loadEnvironment(effectiveEnvName, envDir, collection?.id);
-  const results = await Promise.all(suites.map((s) => runSuite(s, env)));
+
+  // Helper: load env with optional --env-var overrides merged on top
+  async function loadEnvWithOverrides(dir: string): Promise<Record<string, string>> {
+    const env = await loadEnvironment(effectiveEnvName, dir, collection?.id);
+    if (options.envVars && Object.keys(options.envVars).length > 0) {
+      Object.assign(env, options.envVars);
+    }
+    return env;
+  }
+
+  let results: Awaited<ReturnType<typeof runSuite>>[];
+  if (isDirectory) {
+    // Per-suite env: load env from each suite's own directory
+    results = await Promise.all(suites.map(async (s) => {
+      const suiteDir = s.filePath ? dirname(s.filePath) : envDir;
+      const env = await loadEnvWithOverrides(suiteDir);
+      return runSuite(s, env, options.dryRun);
+    }));
+  } else {
+    const env = await loadEnvWithOverrides(envDir);
+    results = await Promise.all(suites.map((s) => runSuite(s, env, options.dryRun)));
+  }
 
   const runId = createRun({
     started_at: results[0]?.started_at ?? new Date().toISOString(),