ace-test-runner-e2e 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +7 -0
  2. data/.ace-defaults/e2e-runner/config.yml +70 -0
  3. data/.ace-defaults/nav/protocols/guide-sources/ace-test-runner-e2e.yml +11 -0
  4. data/.ace-defaults/nav/protocols/skill-sources/ace-test-runner-e2e.yml +19 -0
  5. data/.ace-defaults/nav/protocols/tmpl-sources/ace-test-runner-e2e.yml +12 -0
  6. data/.ace-defaults/nav/protocols/wfi-sources/ace-test-runner-e2e.yml +11 -0
  7. data/CHANGELOG.md +1166 -0
  8. data/LICENSE +21 -0
  9. data/README.md +42 -0
  10. data/Rakefile +15 -0
  11. data/exe/ace-test-e2e +15 -0
  12. data/exe/ace-test-e2e-sh +67 -0
  13. data/exe/ace-test-e2e-suite +13 -0
  14. data/handbook/guides/e2e-testing.g.md +124 -0
  15. data/handbook/guides/scenario-yml-reference.g.md +182 -0
  16. data/handbook/guides/tc-authoring.g.md +131 -0
  17. data/handbook/skills/as-e2e-create/SKILL.md +30 -0
  18. data/handbook/skills/as-e2e-fix/SKILL.md +35 -0
  19. data/handbook/skills/as-e2e-manage/SKILL.md +31 -0
  20. data/handbook/skills/as-e2e-plan-changes/SKILL.md +30 -0
  21. data/handbook/skills/as-e2e-review/SKILL.md +35 -0
  22. data/handbook/skills/as-e2e-rewrite/SKILL.md +31 -0
  23. data/handbook/skills/as-e2e-run/SKILL.md +48 -0
  24. data/handbook/skills/as-e2e-setup-sandbox/SKILL.md +34 -0
  25. data/handbook/templates/ace-taskflow-fixture.template.md +322 -0
  26. data/handbook/templates/agent-experience-report.template.md +89 -0
  27. data/handbook/templates/metadata.template.yml +49 -0
  28. data/handbook/templates/scenario.yml.template.yml +60 -0
  29. data/handbook/templates/tc-file.template.md +45 -0
  30. data/handbook/templates/test-report.template.md +94 -0
  31. data/handbook/workflow-instructions/e2e/analyze-failures.wf.md +126 -0
  32. data/handbook/workflow-instructions/e2e/create.wf.md +395 -0
  33. data/handbook/workflow-instructions/e2e/execute.wf.md +253 -0
  34. data/handbook/workflow-instructions/e2e/fix.wf.md +166 -0
  35. data/handbook/workflow-instructions/e2e/manage.wf.md +179 -0
  36. data/handbook/workflow-instructions/e2e/plan-changes.wf.md +255 -0
  37. data/handbook/workflow-instructions/e2e/review.wf.md +286 -0
  38. data/handbook/workflow-instructions/e2e/rewrite.wf.md +281 -0
  39. data/handbook/workflow-instructions/e2e/run.wf.md +355 -0
  40. data/handbook/workflow-instructions/e2e/setup-sandbox.wf.md +461 -0
  41. data/lib/ace/test/end_to_end_runner/atoms/display_helpers.rb +234 -0
  42. data/lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb +199 -0
  43. data/lib/ace/test/end_to_end_runner/atoms/result_parser.rb +166 -0
  44. data/lib/ace/test/end_to_end_runner/atoms/skill_prompt_builder.rb +166 -0
  45. data/lib/ace/test/end_to_end_runner/atoms/skill_result_parser.rb +244 -0
  46. data/lib/ace/test/end_to_end_runner/atoms/suite_report_prompt_builder.rb +103 -0
  47. data/lib/ace/test/end_to_end_runner/atoms/tc_fidelity_validator.rb +39 -0
  48. data/lib/ace/test/end_to_end_runner/atoms/test_case_parser.rb +108 -0
  49. data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb +130 -0
  50. data/lib/ace/test/end_to_end_runner/cli/commands/run_test.rb +156 -0
  51. data/lib/ace/test/end_to_end_runner/models/test_case.rb +47 -0
  52. data/lib/ace/test/end_to_end_runner/models/test_result.rb +115 -0
  53. data/lib/ace/test/end_to_end_runner/models/test_scenario.rb +90 -0
  54. data/lib/ace/test/end_to_end_runner/molecules/affected_detector.rb +92 -0
  55. data/lib/ace/test/end_to_end_runner/molecules/config_loader.rb +75 -0
  56. data/lib/ace/test/end_to_end_runner/molecules/failure_finder.rb +203 -0
  57. data/lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb +35 -0
  58. data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb +121 -0
  59. data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb +182 -0
  60. data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb +321 -0
  61. data/lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb +131 -0
  62. data/lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb +172 -0
  63. data/lib/ace/test/end_to_end_runner/molecules/report_writer.rb +259 -0
  64. data/lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb +254 -0
  65. data/lib/ace/test/end_to_end_runner/molecules/setup_executor.rb +181 -0
  66. data/lib/ace/test/end_to_end_runner/molecules/simple_display_manager.rb +72 -0
  67. data/lib/ace/test/end_to_end_runner/molecules/suite_progress_display_manager.rb +223 -0
  68. data/lib/ace/test/end_to_end_runner/molecules/suite_report_writer.rb +277 -0
  69. data/lib/ace/test/end_to_end_runner/molecules/suite_simple_display_manager.rb +116 -0
  70. data/lib/ace/test/end_to_end_runner/molecules/test_discoverer.rb +136 -0
  71. data/lib/ace/test/end_to_end_runner/molecules/test_executor.rb +332 -0
  72. data/lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb +830 -0
  73. data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb +442 -0
  74. data/lib/ace/test/end_to_end_runner/version.rb +9 -0
  75. data/lib/ace/test/end_to_end_runner.rb +71 -0
  76. metadata +220 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 4da112162d04922844f2e503ae070c0e9aca7e2d0cdcad550b1d493884954060
4
+ data.tar.gz: f5208aaf067e103f395394979c9033efca722ce9528d0b6183d07ed09eef24bf
5
+ SHA512:
6
+ metadata.gz: 13b0ca8d93245494ad10999a5643e34b1c3fbef66626689178ee5f66a9f5c0b86d49f746f23737abb862454e61b64ba1bce40947718b3f3099a51effc53deedf
7
+ data.tar.gz: 3020c9bb4d524a5585b0af08bbcba71a64ab7cd267559a783a26a24b4182f6162e529d36c7c2e44a7b1fb464789fe2fe70beea5f9da577093b7b303e18a1d3ba
data/.ace-defaults/e2e-runner/config.yml ADDED
@@ -0,0 +1,70 @@
1
+ # Default configuration for e2e test runner
2
+ # This file provides defaults for the ace-test-runner-e2e gem
3
+
4
+ paths:
5
+ # Where test scenarios are stored in packages
6
+ scenarios: "test/e2e"
7
+ # Directory for test execution artifacts (gitignored)
8
+ cache_dir: ".ace-local/test-e2e"
9
+
10
+ patterns:
11
+ # Glob pattern for finding test scenarios (TS-format directories)
12
+ discovery: "test/e2e/TS-*/scenario.yml"
13
+
14
+ # Test scenario frontmatter requirements
15
+ required_fields:
16
+ - test-id
17
+ - title
18
+ - area
19
+
20
+ # Optional fields with defaults
21
+ optional_fields:
22
+ priority: medium
23
+ duration: "~5min"
24
+ automation-candidate: false
25
+
26
+ # Cleanup behavior
27
+ cleanup:
28
+ # Whether to delete test artifacts after test completion
29
+ # Set to false to keep artifacts for debugging
30
+ enabled: false
31
+
32
+ # Reporting defaults (suite final report LLM synthesis)
33
+ reporting:
34
+ # LLM model alias for suite report generation
35
+ model: "glite"
36
+ # Timeout in seconds for report generation
37
+ timeout: 60
38
+
39
+ # Execution defaults
40
+ execution:
41
+ # Default LLM provider:model for test execution
42
+ provider: "claude:sonnet@yolo"
43
+ # Timeout per test in seconds
44
+ timeout: 600
45
+ # Number of tests to run in parallel (1 = sequential)
46
+ parallel: 3
47
+
48
+ # Provider configuration
49
+ providers:
50
+ # CLI providers use deterministic pipeline execution (runner + verifier)
51
+ cli:
52
+ - claude
53
+ - gemini
54
+ - codex
55
+ - codexoss
56
+ - opencode
57
+ - pi
58
+
59
+ # Test ID format validation
60
+ test_id:
61
+ # Pattern: TS-{AREA}-{NNN}
62
+ pattern: "^TS-[A-Z]+-\\d{3}$"
63
+ # Known area codes
64
+ areas:
65
+ - LINT
66
+ - REVIEW
67
+ - BUILD
68
+ - GIT
69
+ - SEARCH
70
+ - BUNDLE
data/.ace-defaults/nav/protocols/guide-sources/ace-test-runner-e2e.yml ADDED
@@ -0,0 +1,11 @@
1
+ ---
2
+ # Guide Sources Protocol Configuration for ace-test-runner-e2e gem
3
+ name: ace-test-runner-e2e
4
+ type: gem
5
+ description: E2E testing guides from ace-test-runner-e2e gem
6
+ priority: 10
7
+
8
+ config:
9
+ relative_path: handbook/guides
10
+ pattern: "*.g.md"
11
+ enabled: true
data/.ace-defaults/nav/protocols/skill-sources/ace-test-runner-e2e.yml ADDED
@@ -0,0 +1,19 @@
1
+ ---
2
+ # Skill Sources Protocol Configuration for ace-test-runner-e2e gem
3
+ # This enables canonical skill discovery from the installed ace-test-runner-e2e gem
4
+
5
+ name: ace-test-runner-e2e
6
+ type: gem
7
+ description: Canonical skills from ace-test-runner-e2e gem
8
+ priority: 10
9
+
10
+ # Configuration for skill discovery within the gem
11
+ config:
12
+ # Relative path within the gem
13
+ relative_path: handbook/skills
14
+
15
+ # Pattern for finding canonical skill files
16
+ pattern: "*/SKILL.md"
17
+
18
+ # Enable discovery
19
+ enabled: true
data/.ace-defaults/nav/protocols/tmpl-sources/ace-test-runner-e2e.yml ADDED
@@ -0,0 +1,12 @@
1
+ ---
2
+ # Template Sources Protocol Configuration for ace-test-runner-e2e gem
3
+ name: ace-test-runner-e2e
4
+ type: gem
5
+ description: E2E test templates from ace-test-runner-e2e gem
6
+ priority: 10
7
+
8
+ config:
9
+ relative_path: handbook/templates
10
+ # Pattern matches both .template.md and .template.yml files
11
+ pattern: "*.template.*"
12
+ enabled: true
data/.ace-defaults/nav/protocols/wfi-sources/ace-test-runner-e2e.yml ADDED
@@ -0,0 +1,11 @@
1
+ ---
2
+ # WFI Sources Protocol Configuration for ace-test-runner-e2e gem
3
+ name: ace-test-runner-e2e
4
+ type: gem
5
+ description: E2E test workflow instructions from ace-test-runner-e2e gem
6
+ priority: 10
7
+
8
+ config:
9
+ relative_path: handbook/workflow-instructions
10
+ pattern: "*.wf.md"
11
+ enabled: true