@probelabs/visor 0.1.106 → 0.1.111

package/defaults/visor.tests.yaml

@@ -0,0 +1,685 @@
+version: "1.0"
+extends: "visor.yaml"
+
+# Integration test suite for Visor default configuration
+# - Driven by events + fixtures; no manual step lists
+# - Strict by default: every executed step must have an expect
+# - AI mocks accept structured JSON when a schema is defined; plain uses text
+# - GitHub calls are recorded by default by the test runner (no network)
+
+tests:
+  defaults:
+    strict: true
+    ai_provider: mock
+    fail_on_unexpected_calls: true
+    # Enable GitHub + Slack frontends for PR stages in this suite.
+    # Slack is disabled automatically for non‑PR events in tests.
+    frontends:
+      - name: github
+      - name: slack
+        config:
+          defaultChannel: C_TEST
+    # Run GitHub-tagged checks in tests (e.g., label application)
+    tags: "github"
+    # Example: enable negative GitHub recorder for all tests
+    # github_recorder: { error_code: 429 }
+  # Built-in fixtures are provided by the test runner (gh.* namespace).
+  # Custom fixtures may still be added here if needed.
+  fixtures: []
+
+  cases:
+    - name: label-flow
+      description: |
+        Validates the happy path for PR open:
+        - overview runs and emits tags.label and tags.review-effort (mocked)
+        - apply-overview-labels adds two labels (feature and review/effort:2)
+        - overview prompt includes PR title and unified diff header
+      event: pr_opened
+      fixture: gh.pr_open.minimal
+      mocks:
+        overview:
+          text: |
+            High‑level summary of the changes and impact.
+          tags:
+            label: feature
+            review-effort: 2
+      expect:
+        calls:
+          - step: overview
+            exactly: 1
+          - step: apply-overview-labels
+            exactly: 1
+          - provider: github
+            op: labels.add
+            at_least: 1
+            args:
+              contains:
+                - feature
+                - "review/effort:2"
+        outputs:
+          - step: overview
+            path: "tags.label"
+            equals: feature
+          - step: overview
+            path: "tags['review-effort']"
+            equals: 2
+        prompts:
+          - step: overview
+            contains:
+              - "feat: add user search"
+
+    - name: references-example-link-issue
+      description: Ensure issue-assistant prompt renders example with HEAD fallback and includes References section.
+      event: issue_opened
+      fixture: gh.issue_open.minimal
+      ai_include_code_context: true
+      strict: false
+      expect:
+        prompts:
+          - step: issue-assistant
+            contains:
+              - "References:"
+              - "https://github.com/owner/repo/blob/HEAD/path/to/file.ext#LSTART-LEND"
+
+    - name: references-example-link-comment
+      description: Ensure comment-assistant prompt renders example with HEAD fallback and includes References section.
+      event: issue_comment
+      fixture: gh.issue_comment.visor_help
+      ai_include_code_context: true
+      strict: false
+      expect:
+        prompts:
+          - step: comment-assistant
+            contains:
+              - "References:"
+              - "https://github.com/owner/repo/blob/HEAD/path/to/file.ext#LSTART-LEND"
+
+    - name: issue-triage
+      skip: true
+      description: |
+        Ensures the issue assistant triages a newly opened issue and applies labels.
+        Asserts the structured output (intent=issue_triage) and the GitHub label op.
+      event: issue_opened
+      fixture: gh.issue_open.minimal
+      mocks:
+        issue-assistant:
+          text: |
+            Thanks for the detailed report! We will investigate.
+          intent: issue_triage
+          labels: [bug, priority/medium]
+      expect:
+        calls:
+          - step: issue-assistant
+            exactly: 1
+          - step: apply-issue-labels
+            exactly: 1
+          - step: init-fact-validation
+            exactly: 1
+          - step: extract-facts
+            exactly: 1
+          - step: validate-fact
+            at_least: 1
+          - provider: github
+            op: labels.add
+            at_least: 1
+            args:
+              contains:
+                - bug
+        outputs:
+          - step: issue-assistant
+            path: intent
+            equals: issue_triage
+        prompts:
+          - step: issue-assistant
+            contains:
+              - "Bug: crashes on search edge case"
+
+    - name: issue-triage-labels
+      description: |
+        Minimal issue triage path to validate label application from issue-assistant output.
+        Provider derives labels from dependency outputs and normalizes them.
+      event: issue_opened
+      fixture: gh.issue_open.minimal
+      mocks:
+        issue-assistant:
+          text: |
+            Thanks for the report — we will triage shortly.
+          intent: issue_triage
+          labels: [bug, priority/medium]
+      expect:
+        calls:
+          - step: issue-assistant
+            exactly: 1
+          - step: apply-issue-labels
+            exactly: 1
+          - provider: github
+            op: labels.add
+            at_least: 1
+            args:
+              contains:
+                - bug
+
+    - name: pr-review-e2e-flow
+      description: |
+        End-to-end PR lifecycle covering multiple external events:
+        1) PR opened → overview + labels
+        2) Standard comment → no bot reply
+        3) /visor help → single assistant reply (no retrigger)
+        4) /visor Regenerate reviews → retrigger overview
+        5) Fact validation enabled on comment → extract/validate/aggregate
+        6) Fact validation disabled on comment → only assistant, no validation steps
+        7) PR synchronized (new commit) → overview runs again
+      strict: true
+      flow:
+        - name: pr-open
+          description: |
+            PR open event. Mocks overview/security/quality/performance as empty issue lists.
+            Expects all review steps to run and labels to be added.
+          event: pr_opened
+          fixture: gh.pr_open.minimal
+          mocks:
+            overview:
+              text: "Overview body"
+              tags: { label: feature, review-effort: 2 }
+            security: { issues: [] }
+            architecture: { issues: [] }
+            quality: { issues: [] }
+            performance: { issues: [] }
+          expect:
+            calls:
+              - step: overview
+                at_least: 1
+              - step: security
+                exactly: 1
+              - step: architecture
+                exactly: 1
+              - step: performance
+                exactly: 1
+              - step: quality
+                exactly: 1
+              - step: apply-overview-labels
+                exactly: 1
+              - provider: github
+                op: issues.createComment
+                at_least: 1
+              - provider: github
+                op: labels.add
+                at_least: 1
+                args:
+                  contains: [feature]
+            prompts:
+              - step: overview
+                contains:
+                  - "feat: add user search"
+
+        - name: standard-comment
+          description: |
+            A regular human comment on a PR should not produce a bot reply.
+          event: issue_comment
+          fixture: gh.issue_comment.standard
+          mocks:
+            comment-assistant:
+              text: ""   # empty text to avoid posting a reply
+              intent: comment_reply
+          expect:
+            no_calls:
+              - provider: github
+                op: issues.createComment
+              - provider: slack
+                op: chat.postMessage
+              - provider: slack
+                op: chat.update
+            calls:
+              - step: comment-assistant
+                exactly: 1
+
+        - name: visor-plain
+          description: |
+            A "/visor help" comment should be recognized and answered once by the assistant.
+          event: issue_comment
+          fixture: gh.issue_comment.visor_help
+          mocks:
+            comment-assistant:
+              text: "Sure, here’s how I can help."
+              intent: comment_reply
+          expect:
+            calls:
+              - step: comment-assistant
+                exactly: 1
+              - provider: github
+                op: issues.createComment
+                exactly: 1
+            outputs:
+              - step: comment-assistant
+                path: intent
+                equals: comment_reply
+            prompts:
+              - step: comment-assistant
+                matches: "(?i)\\/visor\\s+help"
+
+        - name: visor-retrigger
+          description: |
+            A "/visor Regenerate reviews" comment should set intent=comment_retrigger
+            and schedule a new overview.
+          event: issue_comment
+          fixture: gh.issue_comment.visor_regenerate
+          mocks:
+            comment-assistant:
+              text: "Regenerating."
+              intent: comment_retrigger
+            overview:
+              text: "Overview (regenerated)"
+              tags: { label: feature, review-effort: 2 }
+          expect:
+            calls:
+              - step: comment-assistant
+                exactly: 1
+              - step: overview
+                exactly: 1
+              - step: security
+                exactly: 1
+              - step: architecture
+                exactly: 1
+              - step: performance
+                exactly: 1
+              - step: quality
+                exactly: 1
+              - provider: github
+                op: issues.createComment
+                at_least: 1
+            outputs:
+              - step: comment-assistant
+                path: intent
+                equals: comment_retrigger
+            prompts:
+              - step: comment-assistant
+                contains: ["Regenerate reviews"]
+
+        - name: facts-enabled (comment)
+          description: |
+            With fact validation enabled, the assistant reply is followed by
+            extract-facts and validate-fact (per fact). No aggregator is required for routing.
+          event: issue_comment
+          fixture: gh.issue_comment.visor_help
+          env:
+            ENABLE_FACT_VALIDATION: "true"
+          mocks:
+            comment-assistant:
+              text: "We rely on defaults/visor.yaml line 11 for max_parallelism=4."
+              intent: comment_reply
+            extract-facts:
+              - { id: f1, category: Configuration, claim: "max_parallelism defaults to 4", verifiable: true }
+            validate-fact[]:
+              - fact_id: f1
+                claim: "max_parallelism defaults to 4"
+                is_valid: true
+                confidence: high
+                evidence: "defaults/visor.yaml:11"
+          expect:
+            calls:
+              - step: comment-assistant
+                exactly: 1
+              - step: extract-facts
+                exactly: 1
+              - step: validate-fact
+                at_least: 1
+          # Prompt assertions are validated separately in stage-level prompt tests
+
+        - name: facts-invalid (comment)
+          description: |
+            Invalid fact path: after assistant reply, extract-facts finds one claim and
+            validate-fact returns is_valid=false; the runner detects problems from
+            validate-fact history and reruns the assistant once with correction context.
+            Due to goto forward-running dependents, extract-facts and validate-fact also run again.
+          event: issue_comment
+          fixture: gh.issue_comment.visor_help
+          env:
+            ENABLE_FACT_VALIDATION: "true"
+          routing:
+            max_loops: 1
+          mocks:
+            comment-assistant:
+              text: "We rely on defaults/visor.yaml line 11 for max_parallelism=4."
+              intent: comment_reply
+            extract-facts:
+              - { id: f1, category: Configuration, claim: "max_parallelism defaults to 4", verifiable: true }
+            validate-fact[]:
+              - fact_id: f1
+                claim: "max_parallelism defaults to 4"
+                is_valid: false
+                confidence: high
+                evidence: "defaults/visor.yaml:11 does not set 4"
+                correction: "max_parallelism defaults to 3"
+          expect:
+            calls:
+              - step: comment-assistant
+                exactly: 2
+              - step: extract-facts
+                exactly: 2
+              - step: validate-fact
+                exactly: 2
+              - step: aggregate
+                exactly: 1
+            outputs:
+              - step: validate-fact
+                where: { path: fact_id, equals: f1 }
+                path: correction
+                equals: "max_parallelism defaults to 3"
+
+        - name: facts-two-items (comment)
+          description: |
+            Two facts extracted; only the invalid fact should appear in the correction pass.
+            Due to goto forward-running dependents, extract-facts and validate-fact run again on retry.
+          event: issue_comment
+          fixture: gh.issue_comment.visor_help
+          env:
+            ENABLE_FACT_VALIDATION: "true"
+          routing:
+            max_loops: 1
+          mocks:
+            comment-assistant:
+              text: "We rely on defaults/visor.yaml for concurrency defaults."
+              intent: comment_reply
+            extract-facts:
+              - { id: f1, category: Configuration, claim: "max_parallelism defaults to 4", verifiable: true }
+              - { id: f2, category: Feature,       claim: "Fast mode is enabled by default", verifiable: true }
+            validate-fact[]:
+              - { fact_id: f1, claim: "max_parallelism defaults to 4", is_valid: false, confidence: high, evidence: "defaults/visor.yaml:11", correction: "max_parallelism defaults to 3" }
+              - { fact_id: f2, claim: "Fast mode is enabled by default", is_valid: true, confidence: high, evidence: "src/config.ts:FAST_MODE=true" }
+          expect:
+            calls:
+              - step: comment-assistant
+                exactly: 2
+              - step: extract-facts
+                exactly: 2
+              - step: validate-fact
+                exactly: 4
+              - step: aggregate
+                exactly: 1
+            outputs:
+              - step: validate-fact
+                where: { path: fact_id, equals: f1 }
+                path: is_valid
+                equals: false
+              - step: validate-fact
+                where: { path: fact_id, equals: f2 }
+                path: is_valid
+                equals: true
+              - step: validate-fact
+                where: { path: fact_id, equals: f1 }
+                path: correction
+                equals: "max_parallelism defaults to 3"
+
+        - name: facts-disabled
+          description: |
+            With fact validation disabled, only the assistant runs; no validation steps execute.
+          event: issue_comment
+          fixture: gh.issue_comment.standard
+          env:
+            ENABLE_FACT_VALIDATION: "false"
+          expect:
+            calls:
+              - step: comment-assistant
+                exactly: 1
+            no_calls:
+              - step: extract-facts
+              - step: validate-fact
+
+        - name: facts-enabled (issue)
+          description: Fact validation runs from an issue creation path.
+          event: issue_opened
+          fixture: gh.issue_open.minimal
+          env:
+            ENABLE_FACT_VALIDATION: "true"
+          mocks:
+            issue-assistant:
+              text: "Opening issue with claim about defaults/visor.yaml"
+              intent: issue_triage
+              labels: ["bug", "triage"]
+            extract-facts:
+              - { id: f1, category: Configuration, claim: "max_parallelism defaults to 4", verifiable: true }
+            validate-fact[]:
+              - fact_id: f1
+                claim: "max_parallelism defaults to 4"
+                is_valid: true
+                confidence: high
+                evidence: "defaults/visor.yaml:11"
+          expect:
+            calls:
+              - step: issue-assistant
+                at_least: 1
+              - step: apply-issue-labels
+                exactly: 1
+              - step: extract-facts
+                exactly: 1
+              - step: validate-fact
+                at_least: 1
+
+        - name: facts-invalid (issue)
+          description: |
+            Invalid claim triggers correction by rerunning issue-assistant.
+            Due to goto forward-running dependents, extract-facts and validate-fact also run again.
+          event: issue_opened
+          fixture: gh.issue_open.minimal
+          env:
+            ENABLE_FACT_VALIDATION: "true"
+          routing:
+            max_loops: 1
+          mocks:
+            issue-assistant:
+              text: "Claim: max_parallelism defaults to 4"
+              intent: issue_triage
+              labels: ["bug", "triage"]
+            extract-facts:
+              - { id: f1, category: Configuration, claim: "max_parallelism defaults to 4", verifiable: true }
+            validate-fact[]:
+              - fact_id: f1
+                claim: "max_parallelism defaults to 4"
+                is_valid: false
+                confidence: high
+                evidence: "defaults/visor.yaml:11 does not set 4"
+                correction: "max_parallelism defaults to 3"
+          expect:
+            calls:
+              - step: issue-assistant
+                exactly: 2
+              - step: apply-issue-labels
+                exactly: 2
+              - step: extract-facts
+                exactly: 2
+              - step: validate-fact
+                exactly: 2
+              - step: aggregate
+                exactly: 1
+            outputs:
+              - step: validate-fact
+                where: { path: fact_id, equals: f1 }
+                path: correction
+                equals: "max_parallelism defaults to 3"
+
+        - name: pr-updated
+          description: |
+            When a new commit is pushed (synchronize), overview should run again
+            and post/refresh a comment.
+          event: pr_updated
+          fixture: gh.pr_sync.minimal
+          mocks:
+            overview:
+              text: "Overview for new commit"
+              tags: { label: feature, review-effort: 3 }
+          expect:
+            calls:
+              - step: overview
+                exactly: 1
+              - step: security
+                exactly: 1
+              - step: architecture
+                exactly: 1
+              - step: performance
+                exactly: 1
+              - step: quality
+                exactly: 1
+
+    - name: security-fail-if
+      description: |
+        Verifies that the global fail_if trips when security produces an error‑severity issue.
+      event: pr_opened
+      fixture: gh.pr_open.minimal
+      mocks:
+        overview:
+          text: "Overview text"
+          tags:
+            label: bug
+            review-effort: 3
+        security:
+          issues:
+            - id: S-001
+              file: src/search.ts
+              line: 10
+              message: "Command injection risk"
+              severity: error
+              category: security
+      expect:
+        calls:
+          - step: overview
+            exactly: 1
+          - step: security
+            exactly: 1
+        outputs:
+          - step: security
+            path: "issues[0].severity"
+            equals: error
+        fail:
+          message_contains: "fail_if"
+
+    - name: strict-mode-example
+      skip: true
+      description: |
+        Demonstrates strict mode: a step executed without a corresponding expect
+        (apply-overview-labels) triggers a strict_violation with a helpful message.
+      event: pr_opened
+      fixture: gh.pr_open.minimal
+      mocks:
+        overview:
+          text: "Short overview"
+          tags:
+            label: chore
+            review-effort: 1
+      expect:
+        calls:
+          - step: overview
+            exactly: 1
+        strict_violation:
+          for_step: apply-overview-labels
+          message_contains: "Add an expect for this step or set strict: false"
+
+    - name: visor-plain-prompt
+      description: |
+        Standalone prompt check for a "/visor help" comment.
+      event: issue_comment
+      fixture: gh.issue_comment.visor_help
+      mocks:
+        comment-assistant:
+          text: "Here is how I can help."
+          intent: comment_reply
+      expect:
+        calls:
+          - step: comment-assistant
+            exactly: 1
+        prompts:
+          - step: comment-assistant
+            matches: "(?i)\\/visor\\s+help"
+
+    - name: visor-retrigger-prompt
+      description: |
+        Standalone prompt check for "/visor Regenerate reviews".
+      event: issue_comment
+      fixture: gh.issue_comment.visor_regenerate
+      strict: false
+      mocks:
+        comment-assistant:
+          text: "Regenerating."
+          intent: comment_retrigger
+      expect:
+        calls:
+          - step: comment-assistant
+            exactly: 1
+        prompts:
+          - step: comment-assistant
+            contains: ["Regenerate reviews"]
+
+    - name: command-mock-shape
+      description: |
+        Illustrates command provider mocking and output assertions.
+        Skipped by default; enable when command steps exist.
+      skip: true  # illustrative only, enable when a command step exists
+      event: pr_opened
+      fixture: gh.pr_open.minimal
+      mocks:
+        unit-tests:
+          stdout: '{"passed": 128, "failed": 0, "duration_sec": 1.2}'
+          exit_code: 0
+      expect:
+        calls:
+          - step: unit-tests
+            exactly: 1
+        outputs:
+          - step: unit-tests
+            path: failed
+            equals: 0
+
+    - name: github-negative-mode
+      description: |
+        Demonstrates negative GitHub recorder mode: simulate a 429 error and assert failure path.
+      event: pr_opened
+      fixture: gh.pr_open.minimal
+      github_recorder: { error_code: 429 }
+      # Override defaults for this case only by specifying a local recorder via env-like knob
+      # The runner reads tests.defaults.github_recorder; we provide it at the suite level by default.
+      mocks:
+        overview:
+          text: "Overview body"
+          tags: { label: feature, review-effort: 2 }
+      expect:
+        calls:
+          - step: overview
+            exactly: 1
+          - step: apply-overview-labels
+            exactly: 1
+        fail:
+          message_contains: "github/op_failed"
+
+    - name: facts-invalid
+      skip: true
+      description: |
+        With fact validation enabled and an invalid fact, aggregate-validations should detect
+        not-all-valid and route back to the assistant for a correction pass in the same stage.
+      event: issue_comment
+      fixture: gh.issue_comment.visor_help
+      env:
+        ENABLE_FACT_VALIDATION: "true"
+      mocks:
+        comment-assistant:
+          text: "We rely on defaults/visor.yaml line 11 for max_parallelism=4."
+          intent: comment_reply
+        extract-facts:
+          - { id: f1, category: Configuration, claim: "max_parallelism defaults to 4", verifiable: true }
+        validate-fact[]:
+          - fact_id: f1
+            claim: "max_parallelism defaults to 4"
+            is_valid: false
+            confidence: high
+            evidence: "defaults/visor.yaml:11 does not set 4"
+            correction: "max_parallelism defaults to 3"
+      expect:
+        calls:
+          - step: comment-assistant
+            exactly: 1
+          - step: extract-facts
+            exactly: 1
+          - step: validate-fact
+            at_least: 1
+          - step: aggregate-validations
+            exactly: 1
+          - step: init-fact-validation
+            exactly: 1