ai-test-guardrails 0.2.0

package/.cursor/mcp.json

@@ -0,0 +1,8 @@
+{
+  "mcpServers": {
+    "ai-test-guardrails": {
+      "command": "node",
+      "args": ["/Users/jason/Documents/GitHub/ai-test-guardrails/dist/server.js"]
+    }
+  }
+}

package/.prettierrc

@@ -0,0 +1,8 @@
+{
+  "semi": true,
+  "trailingComma": "all",
+  "singleQuote": false,
+  "printWidth": 100,
+  "tabWidth": 2,
+  "arrowParens": "always"
+}

package/README.md

@@ -0,0 +1,438 @@
+# ai-test-guardrails
+
+An MCP (Model Context Protocol) server that provides deterministic guardrails for AI-generated and existing test automation. It validates Playwright and Cypress test proposals using AST-based analysis — detecting flake-prone patterns, enforcing architecture rules, scoring risk, and scanning entire projects in a single call.
+
+## What It Does
+
+- **Validates** AI-generated test code for determinism, flake risk, and architecture compliance
+- **Classifies** violations as `critical`, `major`, or `minor` for prioritised remediation
+- **Scans** entire project directories, validating every test file in one pass with severity breakdown
+- **Detects** flake-prone constructs: hard sleeps, unbounded retries, unmocked network calls, dynamic selectors
+- **Enforces** architectural rules: page object patterns, selector hygiene, nesting depth limits
+- **Scores** flake risk on a 0–1 scale with detailed factor breakdown
+- **Rejects gracefully** unsupported frameworks (e.g. k6) with a clear, actionable message
+- **Returns** structured JSON results with transparent policy output via the MCP tool protocol
+
+## What It Does Not Do
+
+- Generate tests
+- Run Playwright or Cypress
+- Replace CI pipelines
+- Modify your repository
+
+It is a pure validation and scoring engine.
+
+## Installation
+
+```bash
+npm install
+npm run build
+```
+
+Requires Node.js >= 20.
+
+## Usage
+
+### As an MCP Server (stdio)
+
+```bash
+node dist/server.js
+```
+
+Configure in your MCP client. For **Cursor**, add a `.cursor/mcp.json` file to your project root:
+
+```json
+{
+  "mcpServers": {
+    "ai-test-guardrails": {
+      "command": "node",
+      "args": ["/path/to/ai-test-guardrails/dist/server.js"]
+    }
+  }
+}
+```
+
+For **Claude Desktop**, add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "ai-test-guardrails": {
+      "command": "node",
+      "args": ["/path/to/ai-test-guardrails/dist/server.js"]
+    }
+  }
+}
+```
+
+### Enforcement Modes
+
+All tools support a three-tier enforcement model designed for organisational adoption.
+
+| Mode | Action | Use case |
+|------|--------|----------|
+| `"advisory"` | Always `PASSED` or `ADVISED`. Never blocks. | Rollout phase — build trust without friction |
+| `"warn"` *(default)* | `WARNED` if under threshold, `REJECTED` if exceeded | Balanced adoption — enforce policy gradually |
+| `"block"` | `REJECTED` on any violation | Full enforcement once confidence is established |
+
+All modes produce identical scores and violation lists. Only `valid` and the `policy.action` field change.
+
+### Configurable Thresholds (warn mode)
+
+In `warn` mode, enforcement only triggers when detected issues exceed your configured thresholds:
+
+```json
+{
+  "mode": "warn",
+  "architectureThreshold": 3,
+  "flakeRiskThreshold": 0.7,
+  "determinismThreshold": 0
+}
+```
+
+| Threshold | Default | Meaning |
+|-----------|---------|---------|
+| `architectureThreshold` | `3` | Max architecture violations before REJECTED |
+| `flakeRiskThreshold` | `0.7` | Max flake risk score (0–1) before REJECTED |
+| `determinismThreshold` | `0` | Max determinism violations before REJECTED |
+
+### Policy Output
+
+Every result includes a `policy` block that makes enforcement transparent. As of v0.2, `detected` also includes a severity breakdown:
+
+```json
+{
+  "valid": false,
+  "policy": {
+    "mode": "warn",
+    "thresholds": { "architectureThreshold": 3, "flakeRiskThreshold": 0.7, "determinismThreshold": 0 },
+    "detected": {
+      "architectureViolations": 10,
+      "flakeRiskScore": 0,
+      "determinismViolations": 0,
+      "criticalCount": 0,
+      "majorCount": 8,
+      "minorCount": 2
+    },
+    "action": "REJECTED",
+    "reasons": ["10 architecture violations exceeded threshold of 3"]
+  }
+}
+```
+
+This makes it clear the test was rejected because of **policy**, not arbitrary tool behaviour.
+
+---
+
+## MCP Tools
+
+### `scan_project`
+
+Scans an entire project directory, validates every test file in one pass, and returns an aggregate summary with per-file results, severity breakdown, project-wide scores, and a ranked list of top offenders.
+
+**Input:**
+
+```json
+{
+  "projectPath": "/path/to/your/tests",
+  "framework": "playwright",
+  "mode": "warn",
+  "architectureThreshold": 3,
+  "flakeRiskThreshold": 0.7,
+  "determinismThreshold": 0
+}
+```
+
+**Output:**
+
+```json
+{
+  "scannedAt": "2026-02-26T22:12:36.742Z",
+  "projectPath": "/path/to/your/tests",
+  "framework": "playwright",
+  "mode": "warn",
+  "thresholds": { "architectureThreshold": 3, "flakeRiskThreshold": 0.7, "determinismThreshold": 0 },
+  "totals": {
+    "files": 19,
+    "passed": 6,
+    "warned": 8,
+    "rejected": 5,
+    "totalViolations": 59,
+    "criticalViolations": 12,
+    "majorViolations": 30,
+    "minorViolations": 17
+  },
+  "scores": {
+    "averageDeterminism": 0.99,
+    "averageFlakeRisk": 0.17,
+    "averageArchitecture": 0.93
+  },
+  "topOffenders": [
+    {
+      "file": "admin/email-sender-restrictor.spec.ts",
+      "policy": { "action": "REJECTED", "reasons": ["20 architecture violations exceeded threshold of 3"] },
+      "violations": ["..."]
+    }
+  ],
+  "files": ["...per-file results..."],
+  "unsupportedFiles": [
+    { "file": "perf/loadTest.js",  "detectedFramework": "k6" },
+    { "file": "perf/soakTest.js",  "detectedFramework": "k6" }
+  ]
+}
+```
+
+**File discovery:** The scanner picks up files in two passes:
+1. **Conventional names** — `*.spec.ts/js`, `*.test.ts/js`, `*.cy.ts/js` are always included.
+2. **Framework-detected** — any other `.ts`/`.js` file whose imports identify it as Playwright, Cypress, or an unsupported framework (e.g. k6) is also picked up. Files with no recognised test-framework imports (config files, helpers, etc.) are silently skipped.
+
+Directories `node_modules`, `.git`, `dist`, and `coverage` are always ignored.
+
+---
+
+### `validate_test`
+
+Validates a single test file for determinism, flake risk, and architecture compliance. Each violation carries a `severity` (`critical`, `major`, or `minor`), a `rule` identifier, and a `message`.
+
+**Input (warn mode with custom thresholds):**
+
+```json
+{
+  "testCode": "test('login', async ({ page }) => {\n  await page.waitForTimeout(1000);\n  await page.locator('.btn').click();\n});",
+  "framework": "playwright",
+  "mode": "warn",
+  "architectureThreshold": 3,
+  "flakeRiskThreshold": 0.7,
+  "determinismThreshold": 0
+}
+```
+
+**Output — REJECTED:**
+
+```json
+{
+  "valid": false,
+  "policy": {
+    "mode": "warn",
+    "thresholds": { "architectureThreshold": 3, "flakeRiskThreshold": 0.7, "determinismThreshold": 0 },
+    "detected": {
+      "architectureViolations": 1,
+      "flakeRiskScore": 0,
+      "determinismViolations": 1,
+      "criticalCount": 1,
+      "majorCount": 1,
+      "minorCount": 0
+    },
+    "action": "REJECTED",
+    "reasons": ["1 determinism violations exceeded threshold of 0"]
+  },
+  "determinismScore": 0.833,
+  "flakeRiskScore": 0,
+  "architectureScore": 0.75,
+  "violations": [
+    {
+      "severity": "critical",
+      "rule": "no-wait-for-timeout",
+      "message": "[line 2] waitForTimeout introduces non-deterministic timing. Use waitForSelector or expect assertions instead."
+    },
+    {
+      "severity": "major",
+      "rule": "no-raw-selector",
+      "message": "[line 3] Direct CSS selector \".btn\" in test code. Extract selectors to page objects and use data-testid or role-based selectors."
+    }
+  ]
+}
+```
+
+**Input (advisory mode):**
+
+```json
+{
+  "testCode": "test('login', async ({ page }) => {\n  await page.waitForTimeout(1000);\n});",
+  "framework": "playwright",
+  "mode": "advisory"
+}
+```
+
+**Output — ADVISED (violations surfaced, CI not blocked):**
+
+```json
+{
+  "valid": true,
+  "policy": { "mode": "advisory", "action": "ADVISED", "reasons": [] },
+  "violations": [
+    { "severity": "critical", "rule": "no-wait-for-timeout", "message": "[line 2] waitForTimeout introduces non-deterministic timing..." }
+  ]
+}
+```
+
+---
+
+### `score_flake_risk`
+
+Analyses the flake risk of a single test file and returns a numeric risk score (0–1) with contributing factors.
+
+**Input:**
+
+```json
+{
+  "testCode": "test('checkout', async ({ page }) => {\n  await page.goto('/cart');\n  await page.goto('/checkout');\n  const res = await fetch('/api/order');\n});",
+  "framework": "playwright"
+}
+```
+
+**Output:**
+
+```json
+{
+  "score": 0.45,
+  "factors": [
+    { "name": "async-heavy",          "weight": 0.15, "detected": false, "description": "High number of async operations increases timing sensitivity" },
+    { "name": "network-dependency",   "weight": 0.25, "detected": true,  "description": "Network calls without mocking create external dependencies" },
+    { "name": "multiple-navigations", "weight": 0.20, "detected": true,  "description": "Multiple navigation steps increase page load timing variability" },
+    { "name": "shared-state",         "weight": 0.20, "detected": false, "description": "Module-level mutable state can leak between tests" },
+    { "name": "timing-assertions",    "weight": 0.20, "detected": false, "description": "Timing-dependent assertions are sensitive to execution speed" }
+  ]
+}
+```
+
+---
+
+### `enforce_architecture`
+
+Checks a single test file for architectural compliance: page object usage, selector patterns, nesting depth, and duplicate titles. Violations are severity-classified. Supports all three enforcement modes.
+
+**Input:**
+
+```json
+{
+  "testCode": "let count = 0;\ndescribe('a', () => {\n  describe('b', () => {\n    describe('c', () => {\n      it('test', () => { count++; });\n      it('test', () => {});\n    });\n  });\n});",
+  "framework": "playwright"
+}
+```
+
+**Output:**
+
+```json
+{
+  "valid": false,
+  "policy": {
+    "mode": "warn",
+    "detected": { "architectureViolations": 3, "criticalCount": 1, "majorCount": 0, "minorCount": 2 },
+    "action": "REJECTED",
+    "reasons": ["3 architecture violations exceeded threshold of 3"]
+  },
+  "score": 0,
+  "violations": [
+    { "severity": "critical", "rule": "no-global-state",      "message": "[line 1] Module-level mutable variable \"count\" can leak state between tests..." },
+    { "severity": "minor",    "rule": "no-deep-nesting",      "message": "Test nesting depth is 3 (max allowed: 2). Flatten describe blocks..." },
+    { "severity": "minor",    "rule": "no-duplicate-title",   "message": "Duplicate test title \"test\". Each test should have a unique title..." }
+  ]
+}
+```
+
+---
+
+### Unsupported Framework Detection
+
+If test code imports from an unsupported framework (e.g. k6), tools return a graceful rejection instead of attempting validation:
+
+```json
+{
+  "supported": false,
+  "detectedFramework": "k6",
+  "message": "Framework \"k6\" is not supported by ai-test-guardrails v0.2. Supported frameworks: playwright, cypress.",
+  "supportedFrameworks": ["playwright", "cypress"]
+}
+```
+
+---
+
+## Validation Rules (v0.2)
+
+### Determinism Rules
+
+| Rule | Severity | Detects |
+|------|----------|---------|
+| `no-wait-for-timeout` | **critical** | `page.waitForTimeout()`, `cy.wait(number)` |
+| `no-hard-sleep` | **critical** | `setTimeout`, `sleep()` |
+| `no-unbounded-retry` | **critical** | `while(true)`, `for(;;)` |
+| `no-random-without-seed` | **major** | `Math.random()` |
+| `no-unmocked-network` | **major** | `fetch()`, `axios.*()` without `route()`/`intercept()` |
+| `no-dynamic-selector` | **major** | Template literals in `locator()`, `cy.get()`, etc. |
+
+### Architecture Rules
+
+| Rule | Severity | Enforces |
+|------|----------|----------|
+| `no-global-state` | **critical** | No module-level `let`/`var` declarations |
+| `no-raw-selector` | **major** | No direct CSS/ID selectors — use `data-testid` or role-based |
+| `no-deep-nesting` | **minor** | Max 2 levels of `describe`/`context` nesting |
+| `no-duplicate-title` | **minor** | No duplicate `it()`/`test()` titles |
+
+### Flake Risk Factors
+
+| Factor | Weight | Triggers |
+|--------|--------|----------|
+| Async-heavy | 0.15 | > 5 `await` expressions |
+| Network dependency | 0.25 | `fetch()`, `request()` calls |
+| Multiple navigations | 0.20 | > 1 `goto()`/`visit()` call |
+| Shared state | 0.20 | Module-level `let`/`var` |
+| Timing assertions | 0.20 | `waitForTimeout`, assertions with `timeout` option |
+
+---
+
+## Development
+
+```bash
+npm run dev          # Run server in dev mode (tsx)
+npm run build        # Compile TypeScript
+npm test             # Run tests
+npm run test:watch   # Watch mode
+npm run lint         # Lint with ESLint
+npm run format       # Format with Prettier
+npm run typecheck    # Type-check without emitting
+```
+
+## Project Structure
+
+```
+src/
+├── server.ts              # MCP server entry point with tool registration
+├── validators/
+│   ├── determinism.ts     # Determinism rule checks
+│   ├── architecture.ts    # Architecture compliance checks
+│   └── flakeRisk.ts       # Flake risk scoring engine
+├── types/
+│   └── guardrail.types.ts # Shared TypeScript interfaces
+├── utils/
+│   ├── astParser.ts       # TypeScript compiler API utilities
+│   ├── enforcement.ts     # Three-mode policy engine
+│   ├── frameworkDetector.ts # Framework detection from imports (k6, Playwright, Cypress)
+│   └── projectScanner.ts  # Two-pass file discovery, classification, and aggregator
+└── config/
+    └── defaultRules.ts    # Default rule configuration
+tests/
+├── astParser.test.ts
+├── determinism.test.ts
+├── architecture.test.ts
+├── flakeRisk.test.ts
+├── mode.test.ts
+├── projectScanner.test.ts
+└── severity.test.ts
+```
+
+## Roadmap
+
+- [x] **v0.1** — Core validation engine, three enforcement modes, project scanning
+- [x] **v0.2** — Violation severity tiers (critical / major / minor), k6 detection, severity breakdown in scan summary, two-pass `.js`/`.ts` file discovery
+- [ ] **v0.3** — Auto-fix suggestions in violation messages
+- [ ] **v0.3** — Cypress-specific selector pattern detection
+- [ ] **v0.4** — Support for custom rule plugins
+- [ ] **v0.4** — CI artefact export (scan results to JSON file)
+- [ ] **v0.5** — Historical flake score tracking
+- [ ] **v1.0** — Stable API with semver guarantees
+
+## Licence
+
+MIT

package/dist/config/defaultRules.d.ts

@@ -0,0 +1,28 @@
+export interface DeterminismRuleConfig {
+    detectWaitForTimeout: boolean;
+    detectHardSleeps: boolean;
+    detectRandomWithoutSeed: boolean;
+    detectUnboundedRetries: boolean;
+    detectUnmockedNetworkCalls: boolean;
+    detectDynamicSelectors: boolean;
+}
+export interface ArchitectureRuleConfig {
+    enforcePageObjects: boolean;
+    forbidGlobalState: boolean;
+    maxDescribeDepth: number;
+    forbidDuplicateTestTitles: boolean;
+}
+export interface FlakeRiskWeightConfig {
+    asyncHeavyWeight: number;
+    networkDependencyWeight: number;
+    multipleNavigationsWeight: number;
+    sharedStateWeight: number;
+    timingAssertionsWeight: number;
+}
+export interface RuleConfig {
+    determinism: DeterminismRuleConfig;
+    architecture: ArchitectureRuleConfig;
+    flakeRisk: FlakeRiskWeightConfig;
+}
+export declare const DEFAULT_RULES: RuleConfig;
+//# sourceMappingURL=defaultRules.d.ts.map

package/dist/config/defaultRules.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"defaultRules.d.ts","sourceRoot":"","sources":["../../src/config/defaultRules.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,qBAAqB;IACpC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,uBAAuB,EAAE,OAAO,CAAC;IACjC,sBAAsB,EAAE,OAAO,CAAC;IAChC,0BAA0B,EAAE,OAAO,CAAC;IACpC,sBAAsB,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,kBAAkB,EAAE,OAAO,CAAC;IAC5B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,qBAAqB,CAAC;IACnC,YAAY,EAAE,sBAAsB,CAAC;IACrC,SAAS,EAAE,qBAAqB,CAAC;CAClC;AAED,eAAO,MAAM,aAAa,EAAE,UAsB3B,CAAC"}

package/dist/config/defaultRules.js

@@ -0,0 +1,24 @@
+export const DEFAULT_RULES = {
+    determinism: {
+        detectWaitForTimeout: true,
+        detectHardSleeps: true,
+        detectRandomWithoutSeed: true,
+        detectUnboundedRetries: true,
+        detectUnmockedNetworkCalls: true,
+        detectDynamicSelectors: true,
+    },
+    architecture: {
+        enforcePageObjects: true,
+        forbidGlobalState: true,
+        maxDescribeDepth: 2,
+        forbidDuplicateTestTitles: true,
+    },
+    flakeRisk: {
+        asyncHeavyWeight: 0.15,
+        networkDependencyWeight: 0.25,
+        multipleNavigationsWeight: 0.2,
+        sharedStateWeight: 0.2,
+        timingAssertionsWeight: 0.2,
+    },
+};
+//# sourceMappingURL=defaultRules.js.map

package/dist/config/defaultRules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defaultRules.js","sourceRoot":"","sources":["../../src/config/defaultRules.ts"],"names":[],"mappings":"AA8BA,MAAM,CAAC,MAAM,aAAa,GAAe;IACvC,WAAW,EAAE;QACX,oBAAoB,EAAE,IAAI;QAC1B,gBAAgB,EAAE,IAAI;QACtB,uBAAuB,EAAE,IAAI;QAC7B,sBAAsB,EAAE,IAAI;QAC5B,0BAA0B,EAAE,IAAI;QAChC,sBAAsB,EAAE,IAAI;KAC7B;IACD,YAAY,EAAE;QACZ,kBAAkB,EAAE,IAAI;QACxB,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,CAAC;QACnB,yBAAyB,EAAE,IAAI;KAChC;IACD,SAAS,EAAE;QACT,gBAAgB,EAAE,IAAI;QACtB,uBAAuB,EAAE,IAAI;QAC7B,yBAAyB,EAAE,GAAG;QAC9B,iBAAiB,EAAE,GAAG;QACtB,sBAAsB,EAAE,GAAG;KAC5B;CACF,CAAC"}

package/dist/server.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":""}