ace-test-runner-e2e 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +7 -0
  2. data/.ace-defaults/e2e-runner/config.yml +70 -0
  3. data/.ace-defaults/nav/protocols/guide-sources/ace-test-runner-e2e.yml +11 -0
  4. data/.ace-defaults/nav/protocols/skill-sources/ace-test-runner-e2e.yml +19 -0
  5. data/.ace-defaults/nav/protocols/tmpl-sources/ace-test-runner-e2e.yml +12 -0
  6. data/.ace-defaults/nav/protocols/wfi-sources/ace-test-runner-e2e.yml +11 -0
  7. data/CHANGELOG.md +1166 -0
  8. data/LICENSE +21 -0
  9. data/README.md +42 -0
  10. data/Rakefile +15 -0
  11. data/exe/ace-test-e2e +15 -0
  12. data/exe/ace-test-e2e-sh +67 -0
  13. data/exe/ace-test-e2e-suite +13 -0
  14. data/handbook/guides/e2e-testing.g.md +124 -0
  15. data/handbook/guides/scenario-yml-reference.g.md +182 -0
  16. data/handbook/guides/tc-authoring.g.md +131 -0
  17. data/handbook/skills/as-e2e-create/SKILL.md +30 -0
  18. data/handbook/skills/as-e2e-fix/SKILL.md +35 -0
  19. data/handbook/skills/as-e2e-manage/SKILL.md +31 -0
  20. data/handbook/skills/as-e2e-plan-changes/SKILL.md +30 -0
  21. data/handbook/skills/as-e2e-review/SKILL.md +35 -0
  22. data/handbook/skills/as-e2e-rewrite/SKILL.md +31 -0
  23. data/handbook/skills/as-e2e-run/SKILL.md +48 -0
  24. data/handbook/skills/as-e2e-setup-sandbox/SKILL.md +34 -0
  25. data/handbook/templates/ace-taskflow-fixture.template.md +322 -0
  26. data/handbook/templates/agent-experience-report.template.md +89 -0
  27. data/handbook/templates/metadata.template.yml +49 -0
  28. data/handbook/templates/scenario.yml.template.yml +60 -0
  29. data/handbook/templates/tc-file.template.md +45 -0
  30. data/handbook/templates/test-report.template.md +94 -0
  31. data/handbook/workflow-instructions/e2e/analyze-failures.wf.md +126 -0
  32. data/handbook/workflow-instructions/e2e/create.wf.md +395 -0
  33. data/handbook/workflow-instructions/e2e/execute.wf.md +253 -0
  34. data/handbook/workflow-instructions/e2e/fix.wf.md +166 -0
  35. data/handbook/workflow-instructions/e2e/manage.wf.md +179 -0
  36. data/handbook/workflow-instructions/e2e/plan-changes.wf.md +255 -0
  37. data/handbook/workflow-instructions/e2e/review.wf.md +286 -0
  38. data/handbook/workflow-instructions/e2e/rewrite.wf.md +281 -0
  39. data/handbook/workflow-instructions/e2e/run.wf.md +355 -0
  40. data/handbook/workflow-instructions/e2e/setup-sandbox.wf.md +461 -0
  41. data/lib/ace/test/end_to_end_runner/atoms/display_helpers.rb +234 -0
  42. data/lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb +199 -0
  43. data/lib/ace/test/end_to_end_runner/atoms/result_parser.rb +166 -0
  44. data/lib/ace/test/end_to_end_runner/atoms/skill_prompt_builder.rb +166 -0
  45. data/lib/ace/test/end_to_end_runner/atoms/skill_result_parser.rb +244 -0
  46. data/lib/ace/test/end_to_end_runner/atoms/suite_report_prompt_builder.rb +103 -0
  47. data/lib/ace/test/end_to_end_runner/atoms/tc_fidelity_validator.rb +39 -0
  48. data/lib/ace/test/end_to_end_runner/atoms/test_case_parser.rb +108 -0
  49. data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb +130 -0
  50. data/lib/ace/test/end_to_end_runner/cli/commands/run_test.rb +156 -0
  51. data/lib/ace/test/end_to_end_runner/models/test_case.rb +47 -0
  52. data/lib/ace/test/end_to_end_runner/models/test_result.rb +115 -0
  53. data/lib/ace/test/end_to_end_runner/models/test_scenario.rb +90 -0
  54. data/lib/ace/test/end_to_end_runner/molecules/affected_detector.rb +92 -0
  55. data/lib/ace/test/end_to_end_runner/molecules/config_loader.rb +75 -0
  56. data/lib/ace/test/end_to_end_runner/molecules/failure_finder.rb +203 -0
  57. data/lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb +35 -0
  58. data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb +121 -0
  59. data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb +182 -0
  60. data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb +321 -0
  61. data/lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb +131 -0
  62. data/lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb +172 -0
  63. data/lib/ace/test/end_to_end_runner/molecules/report_writer.rb +259 -0
  64. data/lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb +254 -0
  65. data/lib/ace/test/end_to_end_runner/molecules/setup_executor.rb +181 -0
  66. data/lib/ace/test/end_to_end_runner/molecules/simple_display_manager.rb +72 -0
  67. data/lib/ace/test/end_to_end_runner/molecules/suite_progress_display_manager.rb +223 -0
  68. data/lib/ace/test/end_to_end_runner/molecules/suite_report_writer.rb +277 -0
  69. data/lib/ace/test/end_to_end_runner/molecules/suite_simple_display_manager.rb +116 -0
  70. data/lib/ace/test/end_to_end_runner/molecules/test_discoverer.rb +136 -0
  71. data/lib/ace/test/end_to_end_runner/molecules/test_executor.rb +332 -0
  72. data/lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb +830 -0
  73. data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb +442 -0
  74. data/lib/ace/test/end_to_end_runner/version.rb +9 -0
  75. data/lib/ace/test/end_to_end_runner.rb +71 -0
  76. metadata +220 -0
metadata ADDED
@@ -0,0 +1,220 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ace-test-runner-e2e
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.29.0
5
+ platform: ruby
6
+ authors:
7
+ - Michal Czyz
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: ace-support-cli
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '0.3'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '0.3'
26
+ - !ruby/object:Gem::Dependency
27
+ name: ace-support-core
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '0.25'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0.25'
40
+ - !ruby/object:Gem::Dependency
41
+ name: ace-support-config
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '0.8'
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '0.8'
54
+ - !ruby/object:Gem::Dependency
55
+ name: ace-llm
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '0.26'
61
+ type: :runtime
62
+ prerelease: false
63
+ version_requirements: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '0.26'
68
+ - !ruby/object:Gem::Dependency
69
+ name: ace-b36ts
70
+ requirement: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '0.7'
75
+ type: :runtime
76
+ prerelease: false
77
+ version_requirements: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '0.7'
82
+ - !ruby/object:Gem::Dependency
83
+ name: minitest
84
+ requirement: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '5.0'
89
+ type: :development
90
+ prerelease: false
91
+ version_requirements: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - "~>"
94
+ - !ruby/object:Gem::Version
95
+ version: '5.0'
96
+ - !ruby/object:Gem::Dependency
97
+ name: rake
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: '13.0'
103
+ type: :development
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '13.0'
110
+ description: Runs package E2E scenarios through agents with reproducible setup, sandbox
111
+ isolation, and structured reports for reliable validation workflows.
112
+ email:
113
+ - mc@cs3b.com
114
+ executables:
115
+ - ace-test-e2e
116
+ - ace-test-e2e-sh
117
+ - ace-test-e2e-suite
118
+ extensions: []
119
+ extra_rdoc_files: []
120
+ files:
121
+ - ".ace-defaults/e2e-runner/config.yml"
122
+ - ".ace-defaults/nav/protocols/guide-sources/ace-test-runner-e2e.yml"
123
+ - ".ace-defaults/nav/protocols/skill-sources/ace-test-runner-e2e.yml"
124
+ - ".ace-defaults/nav/protocols/tmpl-sources/ace-test-runner-e2e.yml"
125
+ - ".ace-defaults/nav/protocols/wfi-sources/ace-test-runner-e2e.yml"
126
+ - CHANGELOG.md
127
+ - LICENSE
128
+ - README.md
129
+ - Rakefile
130
+ - exe/ace-test-e2e
131
+ - exe/ace-test-e2e-sh
132
+ - exe/ace-test-e2e-suite
133
+ - handbook/guides/e2e-testing.g.md
134
+ - handbook/guides/scenario-yml-reference.g.md
135
+ - handbook/guides/tc-authoring.g.md
136
+ - handbook/skills/as-e2e-create/SKILL.md
137
+ - handbook/skills/as-e2e-fix/SKILL.md
138
+ - handbook/skills/as-e2e-manage/SKILL.md
139
+ - handbook/skills/as-e2e-plan-changes/SKILL.md
140
+ - handbook/skills/as-e2e-review/SKILL.md
141
+ - handbook/skills/as-e2e-rewrite/SKILL.md
142
+ - handbook/skills/as-e2e-run/SKILL.md
143
+ - handbook/skills/as-e2e-setup-sandbox/SKILL.md
144
+ - handbook/templates/ace-taskflow-fixture.template.md
145
+ - handbook/templates/agent-experience-report.template.md
146
+ - handbook/templates/metadata.template.yml
147
+ - handbook/templates/scenario.yml.template.yml
148
+ - handbook/templates/tc-file.template.md
149
+ - handbook/templates/test-report.template.md
150
+ - handbook/workflow-instructions/e2e/analyze-failures.wf.md
151
+ - handbook/workflow-instructions/e2e/create.wf.md
152
+ - handbook/workflow-instructions/e2e/execute.wf.md
153
+ - handbook/workflow-instructions/e2e/fix.wf.md
154
+ - handbook/workflow-instructions/e2e/manage.wf.md
155
+ - handbook/workflow-instructions/e2e/plan-changes.wf.md
156
+ - handbook/workflow-instructions/e2e/review.wf.md
157
+ - handbook/workflow-instructions/e2e/rewrite.wf.md
158
+ - handbook/workflow-instructions/e2e/run.wf.md
159
+ - handbook/workflow-instructions/e2e/setup-sandbox.wf.md
160
+ - lib/ace/test/end_to_end_runner.rb
161
+ - lib/ace/test/end_to_end_runner/atoms/display_helpers.rb
162
+ - lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb
163
+ - lib/ace/test/end_to_end_runner/atoms/result_parser.rb
164
+ - lib/ace/test/end_to_end_runner/atoms/skill_prompt_builder.rb
165
+ - lib/ace/test/end_to_end_runner/atoms/skill_result_parser.rb
166
+ - lib/ace/test/end_to_end_runner/atoms/suite_report_prompt_builder.rb
167
+ - lib/ace/test/end_to_end_runner/atoms/tc_fidelity_validator.rb
168
+ - lib/ace/test/end_to_end_runner/atoms/test_case_parser.rb
169
+ - lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb
170
+ - lib/ace/test/end_to_end_runner/cli/commands/run_test.rb
171
+ - lib/ace/test/end_to_end_runner/models/test_case.rb
172
+ - lib/ace/test/end_to_end_runner/models/test_result.rb
173
+ - lib/ace/test/end_to_end_runner/models/test_scenario.rb
174
+ - lib/ace/test/end_to_end_runner/molecules/affected_detector.rb
175
+ - lib/ace/test/end_to_end_runner/molecules/config_loader.rb
176
+ - lib/ace/test/end_to_end_runner/molecules/failure_finder.rb
177
+ - lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb
178
+ - lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb
179
+ - lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb
180
+ - lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb
181
+ - lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb
182
+ - lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb
183
+ - lib/ace/test/end_to_end_runner/molecules/report_writer.rb
184
+ - lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb
185
+ - lib/ace/test/end_to_end_runner/molecules/setup_executor.rb
186
+ - lib/ace/test/end_to_end_runner/molecules/simple_display_manager.rb
187
+ - lib/ace/test/end_to_end_runner/molecules/suite_progress_display_manager.rb
188
+ - lib/ace/test/end_to_end_runner/molecules/suite_report_writer.rb
189
+ - lib/ace/test/end_to_end_runner/molecules/suite_simple_display_manager.rb
190
+ - lib/ace/test/end_to_end_runner/molecules/test_discoverer.rb
191
+ - lib/ace/test/end_to_end_runner/molecules/test_executor.rb
192
+ - lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb
193
+ - lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb
194
+ - lib/ace/test/end_to_end_runner/version.rb
195
+ homepage: https://github.com/cs3b/ace
196
+ licenses:
197
+ - MIT
198
+ metadata:
199
+ homepage_uri: https://github.com/cs3b/ace
200
+ source_code_uri: https://github.com/cs3b/ace/tree/main/ace-test-runner-e2e/
201
+ changelog_uri: https://github.com/cs3b/ace/blob/main/ace-test-runner-e2e/CHANGELOG.md
202
+ rdoc_options: []
203
+ require_paths:
204
+ - lib
205
+ required_ruby_version: !ruby/object:Gem::Requirement
206
+ requirements:
207
+ - - ">="
208
+ - !ruby/object:Gem::Version
209
+ version: 3.2.0
210
+ required_rubygems_version: !ruby/object:Gem::Requirement
211
+ requirements:
212
+ - - ">="
213
+ - !ruby/object:Gem::Version
214
+ version: '0'
215
+ requirements: []
216
+ rubygems_version: 3.6.9
217
+ specification_version: 4
218
+ summary: Agent-executed end-to-end tests with reproducible sandboxes and structured
219
+ reporting
220
+ test_files: []