canary-test-cli 5.15.0 → 6.0.0

package/agent/frameworks/registry.json

@@ -0,0 +1,655 @@
+{
+  "frameworks": [
+    {
+      "name": "playwright",
+      "display_name": "Playwright",
+      "category": "e2e_ui",
+      "categories": ["e2e_ui", "api"],
+      "languages": ["typescript", "javascript"],
+      "file_extensions": ["spec.ts", "spec.js", "test.ts", "test.js"],
+      "execution_command": "npx --yes playwright test {file}",
+      "ci_flags": ["--reporter=list"],
+      "ecosystems": ["react", "vue", "angular", "node"],
+      "status": "preferred",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "end-to-end testing",
+        "cross-browser testing",
+        "modern frontend applications"
+      ],
+      "strengths": [
+        "Reliable auto-waiting",
+        "Strong TypeScript support",
+        "Modern testing architecture",
+        "Fast execution"
+      ],
+      "avoid_when": ["legacy selenium-only environments"]
+    },
+    {
+      "name": "vitest",
+      "display_name": "Vitest",
+      "category": "frontend_unit",
+      "languages": ["typescript", "javascript"],
+      "file_extensions": ["test.ts", "test.js", "spec.ts", "spec.js"],
+      "execution_command": "npx --yes vitest run {file}",
+      "ci_flags": ["--reporter=verbose"],
+      "ecosystems": ["vite", "react", "vue"],
+      "status": "preferred",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "frontend unit testing",
+        "component testing",
+        "modern vite ecosystems"
+      ],
+      "strengths": [
+        "Fast execution",
+        "Modern developer experience",
+        "Vite-native integration"
+      ],
+      "avoid_when": ["legacy jest-only environments"]
+    },
+    {
+      "name": "pytest",
+      "display_name": "pytest",
+      "category": "api",
+      "languages": ["python"],
+      "file_extensions": ["py"],
+      "file_patterns": ["test_*.py", "*_test.py"],
+      "execution_command": "pytest {file}",
+      "ci_flags": ["--tb=short", "-p", "no:cacheprovider"],
+      "ecosystems": ["python"],
+      "status": "preferred",
+      "maturity": "very_high",
+      "community_size": "massive",
+      "recommended_for": [
+        "python backend testing",
+        "api testing",
+        "integration testing"
+      ],
+      "strengths": [
+        "Extremely mature ecosystem",
+        "Powerful fixtures",
+        "Excellent plugin ecosystem"
+      ],
+      "avoid_when": []
+    },
+    {
+      "name": "schemathesis",
+      "display_name": "Schemathesis",
+      "category": "api",
+      "categories": ["api", "contract"],
+      "languages": ["python"],
+      "file_extensions": ["py"],
+      "file_patterns": ["test_*.py", "*_test.py"],
+      "execution_command": "schemathesis run {file}",
+      "ci_flags": ["--checks", "all", "--max-response-time", "2000"],
+      "ecosystems": ["python", "openapi", "rest"],
+      "status": "supported",
+      "license": "MIT",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "property-based API testing",
+        "OpenAPI fuzz testing",
+        "finding schema/implementation drift"
+      ],
+      "strengths": [
+        "Automatically generates test cases from OpenAPI schemas",
+        "Finds coercion bugs and undocumented responses hand-written tests miss",
+        "Zero test code required — point at an OpenAPI URL",
+        "MIT licensed, no API key, no external service"
+      ],
+      "avoid_when": [
+        "No OpenAPI/Swagger spec available",
+        "API has destructive side-effects not safe to trigger randomly (use --dry-run or filter operations)"
+      ]
+    },
+    {
+      "name": "k6",
+      "display_name": "k6",
+      "category": "performance",
+      "languages": ["javascript"],
+      "file_extensions": ["js"],
+      "file_patterns": ["load.js", "*.load.js"],
+      "execution_command": "k6 run {file}",
+      "ci_flags": [],
+      "ecosystems": ["api", "backend", "microservices"],
+      "status": "preferred",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "performance testing",
+        "load testing",
+        "api stress testing"
+      ],
+      "strengths": [
+        "Developer-friendly scripting",
+        "Strong CI/CD integration",
+        "Modern architecture"
+      ],
+      "avoid_when": []
+    },
+    {
+      "name": "axe-core",
+      "display_name": "axe-core",
+      "category": "accessibility",
+      "languages": ["typescript", "javascript"],
+      "file_extensions": ["spec.ts", "spec.js"],
+      "execution_command": "npx --yes playwright test {file}",
+      "ecosystems": ["playwright", "react", "vue", "angular"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "accessibility audits (WCAG)",
+        "a11y regression checks in existing E2E suites"
+      ],
+      "strengths": [
+        "Pairs with an existing Playwright suite via @axe-core/playwright",
+        "Maps findings to WCAG success criteria",
+        "No separate runner to maintain"
+      ],
+      "avoid_when": ["projects with no browser-driven test layer"]
+    },
+    {
+      "name": "zap",
+      "display_name": "OWASP ZAP",
+      "category": "security",
+      "languages": ["any"],
+      "execution_command": "zap-baseline.py -t {target}",
+      "ecosystems": ["web", "api"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "dynamic application security testing (DAST)",
+        "baseline vulnerability scans in CI"
+      ],
+      "strengths": [
+        "Mature OWASP-backed scanner",
+        "Scriptable baseline scan for CI gating",
+        "Active + passive scan modes"
+      ],
+      "avoid_when": [
+        "static-only analysis needs (use static_analysis category instead)"
+      ]
+    },
+    {
+      "name": "backstopjs",
+      "display_name": "BackstopJS",
+      "category": "visual",
+      "languages": ["javascript", "typescript"],
+      "file_extensions": ["js", "json"],
+      "execution_command": "npx --yes backstop test",
+      "ecosystems": ["web"],
+      "status": "supported",
+      "maturity": "medium",
+      "community_size": "medium",
+      "recommended_for": [
+        "visual regression testing",
+        "pixel-diff snapshots across viewports"
+      ],
+      "strengths": [
+        "Dedicated visual-diff workflow",
+        "Configurable viewports and scenarios"
+      ],
+      "avoid_when": ["teams already using Playwright snapshots inline with E2E"]
+    },
+    {
+      "name": "pact",
+      "display_name": "Pact",
+      "category": "contract",
+      "languages": ["typescript", "javascript", "python", "java"],
+      "execution_command": "npx --yes pact verify",
+      "ecosystems": ["microservices", "api"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "consumer-driven contract testing between services",
+        "preventing integration drift across teams"
+      ],
+      "strengths": [
+        "Consumer-driven contracts",
+        "Broker for sharing pacts across teams",
+        "Polyglot language support"
+      ],
+      "avoid_when": ["single-team monolith with no cross-service contracts"]
+    },
+    {
+      "name": "chaos-toolkit",
+      "display_name": "Chaos Toolkit",
+      "category": "chaos",
+      "languages": ["python"],
+      "file_extensions": ["json", "yaml"],
+      "execution_command": "chaos run {file}",
+      "ecosystems": ["kubernetes", "cloud", "microservices"],
+      "status": "supported",
+      "maturity": "medium",
+      "community_size": "medium",
+      "recommended_for": [
+        "chaos engineering experiments",
+        "resilience and fault-injection testing"
+      ],
+      "strengths": [
+        "Declarative experiment definitions",
+        "Extensible driver ecosystem (k8s, cloud providers)"
+      ],
+      "avoid_when": ["systems without redundancy to absorb induced failures"]
+    },
+    {
+      "name": "sdv",
+      "display_name": "SDV (Synthetic Data Vault)",
+      "category": "synthetic_data",
+      "languages": ["python"],
+      "execution_command": "python -m pytest {file}",
+      "ecosystems": ["python", "data"],
+      "status": "preferred",
+      "license": "BSL-1.1",
+      "license_note": "Business Source License — source-available, not OSI open-source. Internal test-data generation is generally within permitted use, but BSL carries a production-use restriction. Downstream adopters must review BSL against their own business/procurement rules before relying on it; fall back to Faker if BSL is unacceptable.",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "statistically faithful synthetic datasets",
+        "schema-aware multi-table data preserving distributions and correlations"
+      ],
+      "strengths": [
+        "Learns real-data distributions and cross-column relationships",
+        "Relational / multi-table synthesis",
+        "Higher fidelity than field-level fakers"
+      ],
+      "avoid_when": [
+        "BSL is unacceptable under the adopter's license policy (use Faker)"
+      ]
+    },
+    {
+      "name": "faker",
+      "display_name": "Faker",
+      "category": "synthetic_data",
+      "languages": ["python", "javascript", "typescript"],
+      "execution_command": "python -m pytest {file}",
+      "ecosystems": ["any"],
+      "status": "supported",
+      "maturity": "very_high",
+      "community_size": "massive",
+      "recommended_for": [
+        "generating realistic synthetic test data",
+        "seeding fixtures without production data"
+      ],
+      "strengths": [
+        "Permissive MIT license",
+        "Huge provider catalog",
+        "Python and JS implementations"
+      ],
+      "avoid_when": [
+        "statistically faithful, distribution-matching datasets are required (prefer SDV, subject to its BSL review)"
+      ]
+    },
+    {
+      "name": "opentelemetry",
+      "display_name": "OpenTelemetry",
+      "category": "observability",
+      "languages": ["any"],
+      "ecosystems": ["distributed-systems", "microservices"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "instrumentation and telemetry assertions",
+        "validating traces/metrics/logs emitted under test"
+      ],
+      "strengths": [
+        "Vendor-neutral standard",
+        "Traces, metrics, and logs in one SDK"
+      ],
+      "avoid_when": [
+        "simple apps with no distributed call graph (Stage 2 routes this, issue #129)"
+      ]
+    },
+    {
+      "name": "maestro",
+      "display_name": "Maestro",
+      "category": "mobile",
+      "languages": ["yaml"],
+      "file_extensions": ["yaml", "yml"],
+      "execution_command": "maestro test {file}",
+      "ecosystems": ["android", "ios", "react-native", "flutter"],
+      "status": "supported",
+      "maturity": "medium",
+      "community_size": "medium",
+      "recommended_for": [
+        "mobile UI flow testing (Android/iOS)",
+        "cross-platform mobile smoke tests"
+      ],
+      "strengths": [
+        "Simple YAML flow syntax",
+        "Built-in flakiness tolerance",
+        "Single tool across Android and iOS"
+      ],
+      "avoid_when": [
+        "deep native-instrumentation needs (consider Appium/Espresso)"
+      ]
+    },
+    {
+      "name": "locust",
+      "display_name": "Locust",
+      "category": "load",
+      "languages": ["python"],
+      "file_extensions": ["py"],
+      "file_patterns": ["locustfile.py", "*_locust.py"],
+      "execution_command": "locust -f {file} --headless",
+      "ecosystems": ["api", "web", "backend"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "Python-defined load scenarios",
+        "soak and spike testing with concurrent users"
+      ],
+      "strengths": [
+        "Load scenarios as plain Python",
+        "Distributed mode for high concurrency",
+        "Live web UI for run monitoring"
+      ],
+      "avoid_when": [
+        "teams standardized on k6 for performance (overlapping concern)"
+      ]
+    },
+    {
+      "name": "stryker",
+      "display_name": "Stryker Mutator",
+      "category": "mutation",
+      "languages": ["typescript", "javascript"],
+      "file_extensions": ["js", "ts"],
+      "execution_command": "npx --yes stryker run",
+      "ecosystems": ["node", "react", "vue"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "mutation testing to measure test-suite effectiveness",
+        "finding assertions that never actually fail"
+      ],
+      "strengths": [
+        "Reveals weak/ineffective tests",
+        "Integrates with Jest/Vitest/Mocha"
+      ],
+      "avoid_when": [
+        "suites too slow to tolerate the mutation runtime multiplier"
+      ]
+    },
+    {
+      "name": "semgrep",
+      "display_name": "Semgrep",
+      "category": "static_analysis",
+      "languages": ["any"],
+      "file_extensions": ["yaml", "yml"],
+      "execution_command": "semgrep --config auto {target}",
+      "ecosystems": ["any"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "static analysis and custom lint rules",
+        "catching anti-patterns and code smells pre-merge"
+      ],
+      "strengths": [
+        "Pattern rules across many languages",
+        "Fast, CI-friendly, custom-rule authoring"
+      ],
+      "avoid_when": ["runtime/dynamic behavior is what needs verifying"]
+    },
+    {
+      "name": "testcontainers",
+      "display_name": "Testcontainers",
+      "category": "integration",
+      "languages": ["python", "typescript", "javascript", "java", "go"],
+      "execution_command": "pytest {file}",
+      "ecosystems": ["docker", "microservices", "backend"],
+      "status": "supported",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "integration tests against real dependencies (DBs, brokers) in Docker",
+        "replacing brittle mocks with disposable real services"
+      ],
+      "strengths": [
+        "Real dependencies in ephemeral containers",
+        "Polyglot bindings",
+        "Deterministic teardown"
+      ],
+      "avoid_when": ["CI without a Docker daemon available"]
+    },
+    {
+      "name": "tosca",
+      "display_name": "Tricentis Tosca",
+      "category": "e2e_ui",
+      "languages": ["any"],
+      "ecosystems": ["enterprise", "sap", "web"],
+      "status": "commercial",
+      "license": "commercial",
+      "license_gate": "CANARY_LICENSE_TRICENTIS",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "model-based / codeless E2E in orgs that already license Tricentis"
+      ],
+      "strengths": [
+        "Codeless model-based authoring",
+        "Broad enterprise app coverage"
+      ],
+      "avoid_when": [
+        "no active Tricentis license — prefer the OSS default (Playwright)"
+      ]
+    },
+    {
+      "name": "neoload",
+      "display_name": "Tricentis NeoLoad",
+      "category": "performance",
+      "languages": ["any"],
+      "ecosystems": ["enterprise", "api", "web"],
+      "status": "commercial",
+      "license": "commercial",
+      "license_gate": "CANARY_LICENSE_TRICENTIS",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "enterprise load testing in orgs that already license Tricentis"
+      ],
+      "strengths": [
+        "Enterprise protocol coverage",
+        "Integrated with Tricentis suite"
+      ],
+      "avoid_when": [
+        "no active Tricentis license — prefer the OSS default (k6)"
+      ]
+    },
+    {
+      "name": "lambdatest",
+      "display_name": "LambdaTest",
+      "category": "e2e_ui",
+      "languages": ["any"],
+      "ecosystems": ["cloud", "cross-browser"],
+      "status": "commercial",
+      "license": "commercial-org",
+      "license_gate": "CANARY_SCOPE",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "cloud cross-browser grids for orgs holding a LambdaTest license"
+      ],
+      "strengths": [
+        "Large device/browser cloud",
+        "Parallel cross-browser runs"
+      ],
+      "avoid_when": [
+        "no org license scope set — prefer the OSS default (Playwright)"
+      ]
+    },
+    {
+      "name": "mutmut",
+      "display_name": "mutmut",
+      "category": "mutation",
+      "languages": ["python"],
+      "file_extensions": ["py"],
+      "file_patterns": ["test_*.py", "*_test.py"],
+      "execution_command": "mutmut run",
+      "ci_flags": [],
+      "ecosystems": ["python", "pytest"],
+      "status": "supported",
+      "license": "BSD-3-Clause",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "mutation testing for Python suites",
+        "measuring whether pytest assertions actually catch regressions"
+      ],
+      "strengths": [
+        "The de-facto Python mutation runner (closes the Python-mutation routing gap Stryker cannot serve)",
+        "Reads pytest configuration — no separate test harness",
+        "Reveals assertions that never fail"
+      ],
+      "avoid_when": [
+        "suites too slow to tolerate the mutation runtime multiplier",
+        "TypeScript/JavaScript projects (use Stryker)"
+      ]
+    },
+    {
+      "name": "wdio",
+      "display_name": "Appium + WebdriverIO",
+      "category": "mobile",
+      "languages": ["typescript", "javascript"],
+      "file_extensions": ["spec.ts", "spec.js"],
+      "execution_command": "npx --yes wdio run wdio.conf.ts --spec {file}",
+      "ci_flags": [],
+      "ecosystems": ["android", "ios", "react-native", "appium"],
+      "status": "supported",
+      "license": "MIT",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "deep native-instrumentation mobile testing (Android/iOS) via Appium",
+        "cross-platform mobile suites needing real element interaction"
+      ],
+      "strengths": [
+        "Full Appium capability access (native selectors, gestures, contexts)",
+        "Mature WebdriverIO runner with rich reporter/service ecosystem",
+        "TypeScript-first authoring; `canary init wdio` scaffolds wdio.conf plus a capabilities stub"
+      ],
+      "avoid_when": [
+        "simple UI-flow smoke tests where Maestro's YAML flows are enough"
+      ]
+    },
+    {
+      "name": "hurl",
+      "display_name": "Hurl",
+      "category": "api",
+      "categories": ["api"],
+      "languages": ["hurl"],
+      "file_extensions": ["hurl"],
+      "file_patterns": ["*.hurl"],
+      "execution_command": "hurl --test {file}",
+      "ci_flags": ["--report-junit", "hurl-report.xml"],
+      "ecosystems": ["api", "rest", "http"],
+      "status": "supported",
+      "license": "Apache-2.0",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "single-file plain-text HTTP API tests",
+        "API checks for less-technical personas — no programming language required"
+      ],
+      "strengths": [
+        "One `.hurl` file per test fits the one-file execution model exactly",
+        "Human-readable HTTP DSL with assertions and captures",
+        "Fast native runner, no runtime dependency to install per project"
+      ],
+      "avoid_when": [
+        "complex programmatic setup/teardown is needed (use pytest or Playwright request)"
+      ]
+    },
+    {
+      "name": "fast-check",
+      "display_name": "fast-check",
+      "category": "property",
+      "languages": ["typescript", "javascript"],
+      "file_extensions": ["test.ts", "test.js", "spec.ts", "spec.js"],
+      "execution_command": "npx --yes vitest run {file}",
+      "ci_flags": ["--reporter=verbose"],
+      "ecosystems": ["node", "vitest", "jest"],
+      "status": "preferred",
+      "license": "MIT",
+      "maturity": "high",
+      "community_size": "large",
+      "recommended_for": [
+        "property-based / invariant testing in TypeScript and JavaScript",
+        "finding edge cases hand-written examples miss via generated inputs"
+      ],
+      "strengths": [
+        "Piggybacks the existing host runner (Vitest/Jest) — no new runner",
+        "Shrinking reports the minimal failing case",
+        "Rich arbitraries for structured input generation"
+      ],
+      "avoid_when": [
+        "behavior has no expressible invariant (example-based tests fit better)"
+      ]
+    },
+    {
+      "name": "hypothesis",
+      "display_name": "Hypothesis",
+      "category": "property",
+      "languages": ["python"],
+      "file_extensions": ["py"],
+      "file_patterns": ["test_*.py", "*_test.py"],
+      "execution_command": "pytest {file}",
+      "ci_flags": ["--tb=short", "-p", "no:cacheprovider"],
+      "ecosystems": ["python", "pytest"],
+      "status": "supported",
+      "license": "MPL-2.0",
+      "maturity": "very_high",
+      "community_size": "large",
+      "recommended_for": [
+        "property-based / invariant testing in Python",
+        "generative edge-case discovery over pytest suites"
+      ],
+      "strengths": [
+        "Runs inside pytest as the host runner — no new harness",
+        "Automatic shrinking to a minimal failing example",
+        "Strategy library for composing input generators"
+      ],
+      "avoid_when": [
+        "behavior has no expressible invariant (example-based tests fit better)"
+      ]
+    },
+    {
+      "name": "promptfoo",
+      "display_name": "promptfoo",
+      "category": "llm_eval",
+      "languages": ["yaml"],
+      "file_extensions": ["yaml", "yml"],
+      "file_patterns": ["promptfooconfig.yaml", "*.promptfoo.yaml"],
+      "execution_command": "npx --yes promptfoo eval -c {file}",
+      "ci_flags": ["--no-progress-bar"],
+      "ecosystems": ["llm", "prompts"],
+      "status": "supported",
+      "license": "MIT",
+      "maturity": "high",
+      "community_size": "medium",
+      "recommended_for": [
+        "LLM-behavior regression testing across prompts and model versions",
+        "assertion-based eval of prompt outputs (quality, format, safety)"
+      ],
+      "strengths": [
+        "Declarative YAML eval config — versioned alongside prompts",
+        "Model-agnostic assertions with side-by-side comparison",
+        "CI-friendly pass/fail gating on eval thresholds"
+      ],
+      "avoid_when": [
+        "no LLM under test (this is not a general functional-testing tool)"
+      ],
+      "provider_access_note": "Route provider access through the agent-plugin path — never embed raw API keys in the eval config or CI."
+    }
+  ]
+}