@dzhechkov/keysarium 1.7.14 → 1.7.18

package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-regression-testing/schemas/output.json

@@ -0,0 +1,311 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://agentic-qe.dev/schemas/regression-testing-output.json",
+  "title": "AQE Regression Testing Skill Output Schema",
+  "description": "Schema for regression testing output including test selection analysis, impact assessment, and suite optimization.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "regression-testing",
+      "description": "Skill identifier"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+-]\\d{2}:\\d{2})?$"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "testSelection", "impactAnalysis", "metrics"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 10,
+          "maxLength": 2000
+        },
+        "score": {
+          "$ref": "#/$defs/regressionScore"
+        },
+        "testSelection": {
+          "$ref": "#/$defs/testSelection",
+          "description": "Test selection strategy results"
+        },
+        "impactAnalysis": {
+          "$ref": "#/$defs/impactAnalysis",
+          "description": "Change impact analysis"
+        },
+        "executionResults": {
+          "$ref": "#/$defs/executionResults",
+          "description": "Regression test execution results"
+        },
+        "suiteHealth": {
+          "$ref": "#/$defs/suiteHealth",
+          "description": "Test suite health metrics"
+        },
+        "findings": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/regressionFinding"
+          },
+          "maxItems": 500,
+          "description": "Regressions detected"
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/recommendation"
+          },
+          "maxItems": 100
+        },
+        "metrics": {
+          "$ref": "#/$defs/regressionMetrics"
+        },
+        "artifacts": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/artifact"
+          },
+          "maxItems": 50
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "toolsUsed": { "type": "array", "items": { "type": "string" } },
+        "agentId": { "type": "string", "pattern": "^qe-[a-z][a-z0-9-]*$" },
+        "prNumber": { "type": "integer", "description": "PR number if applicable" },
+        "baseBranch": { "type": "string" },
+        "headBranch": { "type": "string" }
+      }
+    },
+    "validation": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "learning": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    }
+  },
+  "$defs": {
+    "regressionScore": {
+      "type": "object",
+      "required": ["value", "max"],
+      "properties": {
+        "value": { "type": "number", "minimum": 0, "maximum": 100 },
+        "max": { "type": "number", "const": 100 },
+        "grade": { "type": "string", "pattern": "^[A-F][+-]?$" },
+        "riskCoverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Risk coverage achieved (0.0-1.0)"
+        }
+      }
+    },
+    "testSelection": {
+      "type": "object",
+      "required": ["strategy"],
+      "properties": {
+        "strategy": {
+          "type": "string",
+          "enum": ["change-based", "risk-based", "historical", "time-budget", "full", "smoke"],
+          "description": "Selection strategy used"
+        },
+        "mustRun": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Tests that must run (directly impacted)"
+        },
+        "shouldRun": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Tests that should run (indirectly impacted)"
+        },
+        "canSkip": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Tests that can be safely skipped"
+        },
+        "totalTests": { "type": "integer", "minimum": 0 },
+        "selectedTests": { "type": "integer", "minimum": 0 },
+        "selectionRatio": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Percentage of tests selected"
+        },
+        "estimatedTime": {
+          "type": "string",
+          "description": "Estimated execution time"
+        },
+        "riskCoverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Risk coverage of selected tests"
+        }
+      }
+    },
+    "impactAnalysis": {
+      "type": "object",
+      "properties": {
+        "changedFiles": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "impactedModules": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "impactedTests": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "changeRisk": {
+          "type": "string",
+          "enum": ["low", "medium", "high", "critical"],
+          "description": "Overall change risk level"
+        },
+        "coverageMap": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "description": "Map of files to covering tests"
+        }
+      }
+    },
+    "executionResults": {
+      "type": "object",
+      "properties": {
+        "total": { "type": "integer", "minimum": 0 },
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "skipped": { "type": "integer", "minimum": 0 },
+        "flaky": { "type": "integer", "minimum": 0 },
+        "passRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "regressions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "test": { "type": "string" },
+              "previousStatus": { "type": "string" },
+              "currentStatus": { "type": "string" },
+              "failureMessage": { "type": "string" }
+            }
+          },
+          "description": "Tests that regressed"
+        },
+        "executionTimeMs": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "suiteHealth": {
+      "type": "object",
+      "properties": {
+        "flakiness": {
+          "type": "object",
+          "properties": {
+            "flakyTests": { "type": "array", "items": { "type": "string" } },
+            "flakinessRate": { "type": "number", "minimum": 0, "maximum": 100 }
+          }
+        },
+        "executionTrend": {
+          "type": "string",
+          "enum": ["improving", "stable", "degrading"],
+          "description": "Execution time trend"
+        },
+        "averageExecutionTime": { "type": "integer", "description": "Average test execution time in ms" },
+        "slowTests": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "test": { "type": "string" },
+              "duration": { "type": "integer" }
+            }
+          }
+        },
+        "recommendations": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "regressionFinding": {
+      "type": "object",
+      "required": ["id", "title", "severity"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REG-\\d{3,6}$" },
+        "title": { "type": "string", "minLength": 5, "maxLength": 200 },
+        "description": { "type": "string", "maxLength": 2000 },
+        "severity": { "type": "string", "enum": ["critical", "high", "medium", "low", "info"] },
+        "test": { "type": "string", "description": "Test that detected the regression" },
+        "affectedComponent": { "type": "string" },
+        "introducedIn": { "type": "string", "description": "Commit/PR that introduced the regression" },
+        "evidence": { "type": "string" },
+        "remediation": { "type": "string" }
+      }
+    },
+    "regressionMetrics": {
+      "type": "object",
+      "properties": {
+        "totalTests": { "type": "integer", "minimum": 0 },
+        "testsSelected": { "type": "integer", "minimum": 0 },
+        "testsExecuted": { "type": "integer", "minimum": 0 },
+        "passRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "regressionsFound": { "type": "integer", "minimum": 0 },
+        "selectionEfficiency": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "Percentage of tests run vs total"
+        },
+        "riskCoverageAchieved": { "type": "number", "minimum": 0, "maximum": 1 },
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "timeSaved": { "type": "string", "description": "Time saved vs full regression" }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REC-\\d{3,6}$" },
+        "title": { "type": "string" },
+        "description": { "type": "string" },
+        "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "effort": { "type": "string", "enum": ["trivial", "low", "medium", "high", "major"] }
+      }
+    },
+    "artifact": {
+      "type": "object",
+      "required": ["type", "path"],
+      "properties": {
+        "type": { "type": "string", "enum": ["report", "coverage", "log", "data"] },
+        "path": { "type": "string" },
+        "format": { "type": "string", "enum": ["json", "html", "md", "xml", "txt"] }
+      }
+    }
+  }
+}

package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-regression-testing/scripts/validate-config.json

@@ -0,0 +1,50 @@
+{
+  "skillName": "regression-testing",
+  "skillVersion": "1.0.0",
+  "requiredTools": [
+    "jq"
+  ],
+  "optionalTools": [
+    "ajv",
+    "jsonschema",
+    "python3"
+  ],
+  "schemaPath": "schemas/output.json",
+  "requiredFields": [
+    "skillName",
+    "status",
+    "output",
+    "output.testSelection",
+    "output.impactAnalysis",
+    "output.metrics"
+  ],
+  "requiredNonEmptyFields": [
+    "output.summary",
+    "output.testSelection"
+  ],
+  "mustContainTerms": [
+    "regression",
+    "test"
+  ],
+  "mustNotContainTerms": [
+    "TODO",
+    "placeholder",
+    "FIXME"
+  ],
+  "enumValidations": {
+    ".status": [
+      "success",
+      "partial",
+      "failed",
+      "skipped"
+    ],
+    ".output.testSelection.strategy": [
+      "change-based",
+      "risk-based",
+      "historical",
+      "time-budget",
+      "full",
+      "smoke"
+    ]
+  }
+}

package/templates/.claude/skills/feature-adr/references/agentic-qe/{risk-based-testing.md → qe-ref-risk-based-testing/REFERENCE.md}

@@ -16,28 +16,26 @@ validation:
   schema_path: schemas/output.json
   validator_path: scripts/validate-config.json
   eval_path: evals/risk-based-testing.yaml
-
 ---
 
+<!-- Vendored from https://github.com/proffesor-for-testing/agentic-qe at 0a6de058575cbe292afe220df105cc74120f7674 (MIT, Copyright (c) 2025 Agentic QE Contributors), 2026-10-23. Bundled references do not install aqe, MCP tools, pattern memory, or fleet runtime. See ../LICENSE.agentic-qe and ../README.md. -->
+
+## Keysarium execution adapter
+
+This pinned upstream body is methodology inside the bundled feature-ADR pipeline. In Reference Mode, the feature-ADR module that loaded this file owns orchestration and checkpoints. Use the reasoning and output shape below with the host's available tools. Do not treat upstream `aqe` CLI/MCP calls, named agent roles, browser tools, or example absolute paths as installed capabilities. Name a missing capability or evidence in the checkpoint instead of reporting that an upstream step ran. The schemas and eval files are bundled for explicit validation; bundling them does not run those checks automatically.
+
+Relative paths such as `schemas/output.json`, `evals/...` and `steps/...` resolve from this skill directory under `references/agentic-qe/`. When Direct Mode has a separately installed `aqe` runtime, follow the feature-ADR module's direct-mode branch and verify that each required runtime capability is actually available.
+
+<!-- BEGIN AGENTIC-QE UPSTREAM BODY -->
+
 # Risk-Based Testing
 
 <default_to_action>
 When planning tests or allocating testing resources:
-1. IDENTIFY risks: What can go wrong? What's the impact? What's the likelihood?
-2. CALCULATE risk: Risk = Probability × Impact (use 1-5 scale for each)
-3. PRIORITIZE: Critical (20+) → High (12-19) → Medium (6-11) → Low (1-5)
-4. ALLOCATE effort: 60% critical, 25% high, 10% medium, 5% low
-5. REASSESS continuously: New info, changes, production incidents
-
-**Quick Risk Assessment:**
-- Probability factors: Complexity, change frequency, developer experience, technical debt
-- Impact factors: User count, revenue, safety, reputation, regulatory
-- Dynamic adjustment: Production bugs increase risk; stable code decreases
-
-**Critical Success Factors:**
-- Test where bugs hurt most, not everywhere equally
-- Risk is dynamic - reassess with new information
-- Production data informs risk (shift-right feeds shift-left)
+1. IDENTIFY risks per component (use 1-5 scale for probability and impact)
+2. PRIORITIZE: Critical (20+) → High (12-19) → Medium (6-11) → Low (1-5)
+3. ALLOCATE effort: 60% critical, 25% high, 10% medium, 5% low
+4. REASSESS continuously: Production incidents raise risk; stable code lowers it
 </default_to_action>
 
 ## Quick Reference Card
@@ -48,11 +46,7 @@ When planning tests or allocating testing resources:
 - Allocating limited testing time
 - Justifying test coverage decisions
 
-### Risk Calculation
-```
-Risk Score = Probability (1-5) × Impact (1-5)
-```
-
+### Effort Allocation by Risk Score
 | Score | Priority | Effort | Action |
 |-------|----------|--------|--------|
 | 20-25 | Critical | 60% | Comprehensive testing, multiple techniques |
@@ -60,37 +54,9 @@ Risk Score = Probability (1-5) × Impact (1-5)
 | 6-11 | Medium | 10% | Standard testing, basic automation |
 | 1-5 | Low | 5% | Smoke test, exploratory only |
 
-### Probability Factors
-| Factor | Low (1) | Medium (3) | High (5) |
-|--------|---------|------------|----------|
-| Complexity | Simple CRUD | Business logic | Algorithms, integrations |
-| Change Rate | Stable 6+ months | Monthly changes | Weekly/daily changes |
-| Developer Experience | Senior, domain expert | Mid-level | Junior, new to codebase |
-| Technical Debt | Clean code | Some debt | Legacy, no tests |
-
-### Impact Factors
-| Factor | Low (1) | Medium (3) | High (5) |
-|--------|---------|------------|----------|
-| Users Affected | Admin only | Department | All users |
-| Revenue | None | Indirect | Direct (checkout) |
-| Safety | Convenience | Data loss | Physical harm |
-| Reputation | Internal | Industry | Public scandal |
-
 ---
 
-## Risk Assessment Workflow
-
-### Step 1: List Features/Components
-```
-Feature | Probability | Impact | Risk | Priority
---------|-------------|--------|------|----------
-Checkout | 4 | 5 | 20 | Critical
-User Auth | 3 | 5 | 15 | High
-Admin Panel | 2 | 2 | 4 | Low
-Search | 3 | 3 | 9 | Medium
-```
-
-### Step 2: Apply Test Depth
+## Apply Test Depth by Risk
 ```typescript
 await Task("Risk-Based Test Generation", {
   critical: {
@@ -207,6 +173,11 @@ const riskFleet = await FleetManager.coordinate({
 
 ## Remember
 
-**Risk = Probability × Impact.** Test where bugs hurt most. Critical gets 60%, low gets 5%. Risk is dynamic - reassess with new info. Production incidents raise risk scores.
-
 **With Agents:** Agents calculate risk using ML on historical data, select risk-appropriate tests, and adjust scores from production feedback. Use agents to maintain dynamic risk profiles at scale.
+<!-- END AGENTIC-QE UPSTREAM BODY -->
+
+## Keysarium local addition: probability and impact worksheet
+
+For each candidate failure, write down the conditions that make it likely and the consequence if it occurs. Complexity, recent change, fragile integration, and missing observability can raise probability; data loss, user reach, and recovery difficulty can raise impact. Record the reason for each rating so another reviewer can challenge it.
+
+Use the ranking to choose test depth. A high-impact boundary may need a real integration check and a failure-recovery check; a low-impact presentation detail may need a focused component check. Reassess after incidents or architecture changes. Numerical effort shares are examples only, not promises of defect detection.

package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-risk-based-testing/evals/risk-based-testing.yaml

@@ -0,0 +1,141 @@
+skill: risk-based-testing
+version: 1.0.0
+description: >
+  Evaluation suite for risk-based testing strategy.
+  Tests risk scoring, test prioritization, and coverage optimization.
+
+models_to_test:
+  - claude-sonnet-4-6    # Primary (high accuracy expected)
+  - claude-haiku-4-5     # Fast model (minimum quality floor)
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  target_agents:
+    - qe-learning-coordinator
+
+learning:
+  store_success_patterns: true
+  pattern_ttl_days: 90
+
+result_format:
+  json_output: true
+  include_timing: true
+  include_token_usage: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+
+  - id: tc001_risk_scoring
+    description: "Score and prioritize risks"
+    category: risk_assessment
+    priority: critical
+
+    input:
+      components:
+        - { name: "payment_processor", criticality: "critical", change_frequency: "low" }
+        - { name: "notification_service", criticality: "low", change_frequency: "high" }
+
+    expected_output:
+      must_contain:
+        - "risk"
+        - "score"
+        - "priority"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_test_prioritization
+    description: "Prioritize tests based on risk"
+    category: prioritization
+    priority: high
+
+    input:
+      available_tests: 500
+      time_budget_hours: 2
+      risk_scores: { "critical": 10, "high": 5, "medium": 2 }
+
+    expected_output:
+      must_contain:
+        - "prioritize"
+        - "test"
+        - "critical"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_coverage_optimization
+    description: "Optimize test coverage for risk mitigation"
+    category: coverage
+    priority: high
+
+    input:
+      high_risk_areas: ["authentication", "payment", "data_export"]
+      current_coverage_percent: 65
+
+    expected_output:
+      must_contain:
+        - "coverage"
+        - "optimize"
+
+    validation:
+      schema_check: true
+
+  - id: tc004_regression_risk
+    description: "Identify regression risk from changes"
+    category: regression
+    priority: medium
+
+    input:
+      changed_files:
+        - "auth/login.js"
+        - "auth/session.js"
+      dependent_modules: ["dashboard", "profile", "settings"]
+
+    expected_output:
+      must_contain:
+        - "regression"
+        - "risk"
+
+    validation:
+      schema_check: true
+
+  - id: tc005_risk_mitigation_strategy
+    description: "Generate risk mitigation testing strategy"
+    category: strategy
+    priority: medium
+
+    input:
+      identified_risks:
+        - "SQL injection in search"
+        - "XSS in user input"
+        - "CSRF on form submission"
+
+    expected_output:
+      must_contain:
+        - "strategy"
+        - "mitigation"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+success_criteria:
+  pass_rate: 0.9
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.75
+  max_execution_time_ms: 300000
+
+metadata:
+  author: "qe-risk-assessor"
+  created: "2026-02-02"
+  coverage_target: >
+    Risk-based testing with 5 test cases covering risk scoring,
+    test prioritization, coverage optimization, regression risk, and mitigation strategies.