@bugzy-ai/bugzy 1.15.1 → 1.16.0

package/dist/index.js

@@ -220,6 +220,7 @@ var TASK_SLUGS = {
   PROCESS_EVENT: "process-event",
   RUN_TESTS: "run-tests",
   VERIFY_CHANGES: "verify-changes",
+  TRIAGE_RESULTS: "triage-results",
   /** @deprecated Use ONBOARD_TESTING instead */
   FULL_TEST_COVERAGE: "onboard-testing"
 };
@@ -1419,6 +1420,7 @@ Before running tests, confirm the selection with the user if ambiguous:
     },
     // Step 7-10: Test Execution (library steps)
     "run-tests",
+    "normalize-test-results",
     "parse-test-results",
     "triage-failures",
     "fix-test-issues",
@@ -1427,14 +1429,7 @@ Before running tests, confirm the selection with the user if ambiguous:
       stepId: "log-product-bugs",
       conditionalOnSubagent: "issue-tracker"
     },
-    // Step 12: Knowledge Base Update (library)
-    "update-knowledge-base",
-    // Step 13: Team Communication (conditional - library step)
-    {
-      stepId: "notify-team",
-      conditionalOnSubagent: "team-communicator"
-    },
-    // Step 14: Handle Special Cases (inline - task-specific)
+    // Step 12: Handle Special Cases (inline - reference material, positioned before final action steps)
     {
       inline: true,
       title: "Handle Special Cases",
@@ -1482,6 +1477,13 @@ If selected test cases have formatting issues:
 **Related Documentation**:
 - \`./tests/docs/test-execution-strategy.md\` - When and why to run specific tests
 - \`./tests/docs/testing-best-practices.md\` - How to write tests (patterns and anti-patterns)`
+    },
+    // Step 13: Knowledge Base Update (library)
+    "update-knowledge-base",
+    // Step 14: Team Communication (conditional - library step, LAST actionable step)
+    {
+      stepId: "notify-team",
+      conditionalOnSubagent: "team-communicator"
     }
   ],
   requiredSubagents: ["browser-automation", "test-debugger-fixer"],
@@ -2201,6 +2203,108 @@ var exploreApplicationTask = {
   dependentTasks: []
 };
 
+// src/tasks/library/triage-results.ts
+var triageResultsTask = {
+  slug: TASK_SLUGS.TRIAGE_RESULTS,
+  name: "Triage Results",
+  description: "Analyze externally-submitted test results and triage failures as product bugs or test issues",
+  frontmatter: {
+    description: "Analyze externally-submitted test results and triage failures as product bugs or test issues",
+    "argument-hint": "[event payload with test results]"
+  },
+  steps: [
+    // Step 1: Overview (inline)
+    {
+      inline: true,
+      title: "Triage Results Overview",
+      content: `# Triage External Test Results
+
+Analyze test results submitted from an external CI pipeline. The results were sent via webhook and are available in the event payload \u2014 either as inline data or a URL to download.
+
+**Goal**: Normalize the results into the standard manifest format, classify each failure as a PRODUCT BUG or TEST ISSUE, and generate a triage report.
+
+This task is triggered automatically when test results are submitted to the Bugzy webhook from a CI system (GitHub Actions, GitLab CI, etc.).`
+    },
+    // Step 2: Security Notice (library)
+    "security-notice",
+    // Step 3: Arguments (inline)
+    {
+      inline: true,
+      title: "Arguments",
+      content: `Arguments: $ARGUMENTS`
+    },
+    // Step 4: Load Project Context (library)
+    "load-project-context",
+    // Step 5: Knowledge Base Read (library)
+    "read-knowledge-base",
+    // Step 6: Normalize Test Results (library — handles URL/inline results + manifest creation)
+    "normalize-test-results",
+    // Step 7: Triage Failures (existing library step)
+    "triage-failures",
+    // Step 8: Fix Test Issues (library — uses test-debugger-fixer)
+    "fix-test-issues",
+    // Step 9: Log Product Bugs (conditional — requires issue-tracker)
+    {
+      stepId: "log-product-bugs",
+      conditionalOnSubagent: "issue-tracker"
+    },
+    // Step 10: Update Knowledge Base (library)
+    "update-knowledge-base",
+    // Step 11: Notify Team (conditional — requires team-communicator)
+    {
+      stepId: "notify-team",
+      conditionalOnSubagent: "team-communicator"
+    },
+    // Step 12: Generate Triage Report (inline)
+    {
+      inline: true,
+      title: "Generate Triage Report",
+      content: `## Generate Triage Report
+
+Create a structured triage report as the task output. This report is stored in \`task_executions.result\` and displayed in the Bugzy dashboard.
+
+**Report Structure:**
+\`\`\`json
+{
+  "summary": {
+    "total": <number>,
+    "passed": <number>,
+    "failed": <number>,
+    "skipped": <number>,
+    "duration_ms": <number or null>
+  },
+  "ci_metadata": {
+    "pipeline_url": "<from event payload>",
+    "commit_sha": "<from event payload>",
+    "branch": "<from event payload>"
+  },
+  "triage": {
+    "product_bugs": [
+      {
+        "test_name": "<name>",
+        "error": "<brief error>",
+        "reason": "<why this is a product bug>"
+      }
+    ],
+    "test_issues": [
+      {
+        "test_name": "<name>",
+        "error": "<brief error>",
+        "reason": "<why this is a test issue>"
+      }
+    ]
+  }
+}
+\`\`\`
+
+Output this JSON as the final result of the task.`
+    }
+  ],
+  requiredSubagents: ["browser-automation", "test-debugger-fixer"],
+  optionalSubagents: ["issue-tracker", "team-communicator"],
+  dependentTasks: []
+};
+
 // src/tasks/index.ts
 var TASK_TEMPLATES = {
   [TASK_SLUGS.GENERATE_TEST_CASES]: generateTestCasesTask,
@@ -2210,7 +2314,8 @@ var TASK_TEMPLATES = {
   [TASK_SLUGS.RUN_TESTS]: runTestsTask,
   [TASK_SLUGS.VERIFY_CHANGES]: verifyChangesTask,
   [TASK_SLUGS.ONBOARD_TESTING]: onboardTestingTask,
-  [TASK_SLUGS.EXPLORE_APPLICATION]: exploreApplicationTask
+  [TASK_SLUGS.EXPLORE_APPLICATION]: exploreApplicationTask,
+  [TASK_SLUGS.TRIAGE_RESULTS]: triageResultsTask
 };
 function getTaskTemplate(slug) {
   return TASK_TEMPLATES[slug];
@@ -6803,6 +6908,10 @@ The agent will:
 4. Apply appropriate fix pattern from \`./tests/CLAUDE.md\`
 5. Rerun the test
 6. The custom reporter will automatically create the next exec-N/ folder
+6b. If no custom reporter (BYOT mode \u2014 check for \`reporters/bugzy-reporter.ts\`):
+   Run the parse script to update the manifest with re-run results:
+   \`npx tsx reporters/parse-results.ts --input <re-run-output> --timestamp <current> --test-id <testCaseId>\`
+   This creates exec-N+1/ and updates the manifest.
 7. Repeat up to 3 times if needed (exec-1, exec-2, exec-3)
 8. Report success or escalate as likely product bug
 
@@ -6994,6 +7103,87 @@ ls -t test-runs/ | head -1
   tags: ["execution", "exploration"]
 };
 
+// src/tasks/steps/execution/normalize-test-results.ts
+var normalizeTestResultsStep = {
+  id: "normalize-test-results",
+  title: "Normalize Test Results",
+  category: "execution",
+  content: `## Normalize Test Results
+
+Convert test results into the standard Bugzy \`test-runs/\` manifest format. This step handles both external CI results (via webhook) and local BYOT test output. In managed mode (bugzy-reporter already created the manifest), this step is skipped.
+
+### 1. Check for Existing Manifest
+
+Look for a \`test-runs/*/manifest.json\` from the most recent run. If a manifest already exists from the bugzy-reporter (managed mode), **skip this step entirely** \u2014 the results are already normalized.
+
+### 2. Determine Input Source
+
+Check how test results are available:
+
+**From event payload** (external CI \u2014 \`$ARGUMENTS\` contains event data):
+- \`data.results_url\` \u2014 URL to download results from (the parse script handles the download)
+- \`data.results\` \u2014 inline results (write to a temp file first: \`/tmp/bugzy-results-<random>.json\`)
+
+**From local test run** (agent executed BYOT tests):
+- Read \`./tests/CLAUDE.md\` for the native test output location
+- Find the most recent test output file
+
+### 3. Locate and Run Parse Script
+
+Look for the parse script at \`reporters/parse-results.ts\`.
+
+**If the parse script exists:**
+\`\`\`bash
+npx tsx reporters/parse-results.ts --input <source>
+\`\`\`
+Where \`<source>\` is the file path, temp file path, or URL determined in step 2.
+
+**If the parse script is missing** (fallback for robustness):
+Create the manifest inline using the same approach \u2014 parse the results format by inspecting the data structure:
+- JSON with \`suites\` or \`specs\` arrays: Likely Playwright JSON report
+- XML with \`<testsuites>\` or \`<testsuite>\` root: JUnit XML format
+- JSON with \`results\` array and \`stats\` object: Likely Cypress/Mocha JSON
+- Other: Inspect structure and adapt
+
+Then create:
+1. \`test-runs/{timestamp}/manifest.json\` with the standard Bugzy schema
+2. \`test-runs/{timestamp}/{testCaseId}/exec-1/result.json\` for each failed test
+
+Save the inline parse logic to \`reporters/parse-results.ts\` for future reuse.
+
+### 4. Verify Manifest
+
+Confirm \`manifest.json\` was created:
+- Read the manifest and validate the structure
+- Check that \`stats\` counts match the \`testCases\` array
+
+### 5. Generate Summary
+
+Read the manifest and produce a summary:
+
+\`\`\`markdown
+## Test Results Summary
+
+- Total Tests: [count]
+- Passed: [count] ([percentage]%)
+- Failed: [count] ([percentage]%)
+- Skipped: [count] ([percentage]%)
+- Duration: [time if available]
+\`\`\`
+
+### 6. Include CI Metadata (if from event payload)
+
+If the results came from an external CI event (\`$ARGUMENTS\` contains \`data.metadata\`), include:
+- **Pipeline URL**: \`data.metadata.pipeline_url\`
+- **Commit**: \`data.metadata.commit_sha\`
+- **Branch**: \`data.metadata.branch\`
+
+### 7. All Tests Passed?
+
+If there are **no failures**, note that all tests passed. Downstream triage and fix steps can be skipped.`,
+  tags: ["execution", "results", "normalization", "byot"]
+};
+
 // src/tasks/steps/generation/generate-test-plan.ts
 var generateTestPlanStep = {
   id: "generate-test-plan",
@@ -7178,6 +7368,116 @@ TEST_API_KEY=secret_key_here
   tags: ["generation", "environment"]
 };
 
+// src/tasks/steps/generation/create-results-parser.ts
+var createResultsParserStep = {
+  id: "create-results-parser",
+  title: "Create Results Parser Script",
+  category: "generation",
+  content: `## Create Results Parser Script
+
+Create a reusable script that normalizes test results from the project's test framework into Bugzy's standard \`test-runs/\` manifest format. This script is used at runtime by both external CI events and agent-executed BYOT test runs.
+
+### Inspect the Test Project
+
+1. Read \`./tests/CLAUDE.md\` to understand:
+   - Which test framework is used (Playwright, Cypress, Jest, Mocha, etc.)
+   - How tests are run and where output goes
+   - The native report format (JSON, JUnit XML, etc.)
+2. Check the test runner config file (e.g., \`playwright.config.ts\`, \`cypress.config.ts\`, \`jest.config.ts\`) for report settings
+3. If a sample test output exists, read it to understand the exact structure
+
+### Create the Parse Script
+
+Create \`reporters/parse-results.ts\` \u2014 a Node.js/TypeScript CLI script.
+
+**Interface:**
+\`\`\`
+npx tsx reporters/parse-results.ts --input <file-or-url> [--timestamp <existing>] [--test-id <id>]
+\`\`\`
+
+**Arguments:**
+- \`--input\` (required): file path or URL to the test results
+  - If URL (starts with \`http://\` or \`https://\`): download with 30s timeout
+  - If file path: read directly from disk
+- \`--timestamp\` (optional): existing run timestamp for incremental updates
+- \`--test-id\` (optional): specific test case ID for incremental updates (used with \`--timestamp\`)
+
+**Normal mode** (no \`--timestamp\`):
+1. Parse the project-specific test output format
+2. Generate a timestamp: \`YYYYMMDD-HHmmss\`
+3. Create \`test-runs/{timestamp}/manifest.json\` with the standard Bugzy schema:
+\`\`\`json
+{
+  "bugzyExecutionId": "<from BUGZY_EXECUTION_ID env var or 'local'>",
+  "timestamp": "<YYYYMMDD-HHmmss>",
+  "startTime": "<ISO8601>",
+  "endTime": "<ISO8601>",
+  "status": "completed",
+  "stats": {
+    "totalTests": 0,
+    "passed": 0,
+    "failed": 0,
+    "totalExecutions": 0
+  },
+  "testCases": [
+    {
+      "id": "<slugified test name, e.g. TC-001-login>",
+      "name": "<original test name>",
+      "totalExecutions": 1,
+      "finalStatus": "passed|failed",
+      "executions": [
+        {
+          "executionNumber": 1,
+          "status": "passed|failed",
+          "error": "<error message if failed, null if passed>",
+          "duration": null,
+          "hasTrace": false,
+          "hasScreenshots": false
+        }
+      ]
+    }
+  ]
+}
+\`\`\`
+4. For each failed test, create:
+   - Directory: \`test-runs/{timestamp}/{testCaseId}/exec-1/\`
+   - File: \`test-runs/{timestamp}/{testCaseId}/exec-1/result.json\` containing:
+\`\`\`json
+{
+  "status": "failed",
+  "error": "<full error message>",
+  "stackTrace": "<stack trace if available>",
+  "duration": null,
+  "testFile": "<file path if available>"
+}
+\`\`\`
+5. Print the manifest path to stdout
+6. Exit code 0 on success, non-zero on failure
+
+**Incremental mode** (\`--timestamp\` + \`--test-id\` provided):
+1. Read existing \`test-runs/{timestamp}/manifest.json\`
+2. Parse the new test results for the specified test case
+3. Find the next execution number (e.g., if exec-2 exists, create exec-3)
+4. Create \`test-runs/{timestamp}/{testCaseId}/exec-N/result.json\`
+5. Update the manifest: add execution entry, update \`totalExecutions\`, update \`finalStatus\` and stats
+6. Print the manifest path to stdout
+
+### Test the Script
+
+1. Run the project's tests to generate a sample output (or use an existing one)
+2. Run the parse script: \`npx tsx reporters/parse-results.ts --input <sample-output>\`
+3. Verify \`test-runs/\` was created with correct manifest.json structure
+4. Check that failed test directories have result.json files
+
+### Document in CLAUDE.md
+
+Add to \`./tests/CLAUDE.md\`:
+- Location: \`reporters/parse-results.ts\`
+- Usage: \`npx tsx reporters/parse-results.ts --input <file-or-url> [--timestamp <ts>] [--test-id <id>]\`
+- Where the project's native test output is located (for local runs)`,
+  tags: ["generation", "byot", "results", "parser"]
+};
+
 // src/tasks/steps/communication/notify-team.ts
 var notifyTeamStep = {
   id: "notify-team",
@@ -7426,11 +7726,13 @@ var STEP_LIBRARY = {
   "create-exploration-test-case": createExplorationTestCaseStep,
   "run-exploration": runExplorationStep,
   "process-exploration-results": processExplorationResultsStep,
+  "normalize-test-results": normalizeTestResultsStep,
   // Generation
   "generate-test-plan": generateTestPlanStep,
   "generate-test-cases": generateTestCasesStep,
   "automate-test-cases": automateTestCasesStep,
   "extract-env-variables": extractEnvVariablesStep,
+  "create-results-parser": createResultsParserStep,
   // Communication
   "notify-team": notifyTeamStep,
   // Maintenance