roast-ai 0.4.8 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.rubocop.yml +1 -0
  4. data/Gemfile.lock +3 -3
  5. data/README.md +9 -5
  6. data/dsl/less_simple.rb +112 -0
  7. data/dsl/prototype.rb +17 -0
  8. data/dsl/simple.rb +5 -7
  9. data/dsl/step_communication.rb +18 -0
  10. data/examples/grading/README.md +46 -0
  11. data/examples/grading/analyze_coverage/prompt.md +52 -0
  12. data/examples/grading/calculate_final_grade.rb +64 -0
  13. data/examples/grading/format_result.rb +61 -0
  14. data/examples/grading/generate_grades/prompt.md +105 -0
  15. data/examples/grading/generate_recommendations/output.txt +17 -0
  16. data/examples/grading/generate_recommendations/prompt.md +60 -0
  17. data/examples/grading/read_dependencies/prompt.md +15 -0
  18. data/examples/grading/verify_mocks_and_stubs/prompt.md +12 -0
  19. data/examples/grading/verify_test_helpers/prompt.md +53 -0
  20. data/examples/grading/workflow.md +5 -0
  21. data/examples/grading/workflow.yml +28 -0
  22. data/lib/roast/dsl/cog/config.rb +31 -0
  23. data/lib/roast/dsl/cog/stack.rb +21 -0
  24. data/lib/roast/dsl/cog/store.rb +26 -0
  25. data/lib/roast/dsl/cog.rb +70 -0
  26. data/lib/roast/dsl/cog_execution_context.rb +29 -0
  27. data/lib/roast/dsl/cogs/cmd.rb +55 -0
  28. data/lib/roast/dsl/cogs/graph.rb +53 -0
  29. data/lib/roast/dsl/cogs.rb +65 -0
  30. data/lib/roast/dsl/config_context.rb +54 -0
  31. data/lib/roast/dsl/executor.rb +62 -7
  32. data/lib/roast/dsl/workflow_execution_context.rb +47 -0
  33. data/lib/roast/error.rb +7 -0
  34. data/lib/roast/errors.rb +3 -3
  35. data/lib/roast/graph/edge.rb +25 -0
  36. data/lib/roast/graph/node.rb +40 -0
  37. data/lib/roast/graph/quantum_edge.rb +27 -0
  38. data/lib/roast/graph/threaded_exec.rb +93 -0
  39. data/lib/roast/graph.rb +233 -0
  40. data/lib/roast/resources/api_resource.rb +2 -2
  41. data/lib/roast/resources/url_resource.rb +2 -2
  42. data/lib/roast/tools/apply_diff.rb +1 -1
  43. data/lib/roast/tools/ask_user.rb +1 -1
  44. data/lib/roast/tools/bash.rb +1 -1
  45. data/lib/roast/tools/cmd.rb +2 -2
  46. data/lib/roast/tools/coding_agent.rb +2 -2
  47. data/lib/roast/tools/grep.rb +1 -1
  48. data/lib/roast/tools/read_file.rb +1 -1
  49. data/lib/roast/tools/search_file.rb +1 -1
  50. data/lib/roast/tools/swarm.rb +1 -1
  51. data/lib/roast/tools/update_files.rb +2 -2
  52. data/lib/roast/tools/write_file.rb +1 -1
  53. data/lib/roast/tools.rb +1 -1
  54. data/lib/roast/value_objects/api_token.rb +1 -1
  55. data/lib/roast/value_objects/uri_base.rb +1 -1
  56. data/lib/roast/value_objects/workflow_path.rb +1 -1
  57. data/lib/roast/version.rb +1 -1
  58. data/lib/roast/workflow/base_workflow.rb +38 -2
  59. data/lib/roast/workflow/command_executor.rb +1 -1
  60. data/lib/roast/workflow/configuration_loader.rb +1 -1
  61. data/lib/roast/workflow/error_handler.rb +1 -1
  62. data/lib/roast/workflow/step_executor_registry.rb +1 -1
  63. data/lib/roast/workflow/step_loader.rb +1 -1
  64. data/lib/roast/workflow/workflow_executor.rb +1 -1
  65. data/lib/roast.rb +1 -1
  66. data/sorbet/config +2 -0
  67. data/sorbet/rbi/annotations/.gitattributes +1 -0
  68. data/sorbet/rbi/annotations/activesupport.rbi +495 -0
  69. data/sorbet/rbi/annotations/faraday.rbi +17 -0
  70. data/sorbet/rbi/annotations/minitest.rbi +119 -0
  71. data/sorbet/rbi/annotations/mocha.rbi +34 -0
  72. data/sorbet/rbi/annotations/rainbow.rbi +269 -0
  73. data/sorbet/rbi/annotations/webmock.rbi +9 -0
  74. data/sorbet/rbi/gems/rbs-inline@0.12.0.rbi +2170 -0
  75. data/sorbet/rbi/gems/{rexml@3.4.1.rbi → rexml@3.4.2.rbi} +284 -239
  76. data/sorbet/rbi/shims/lib/roast/dsl/config_context.rbi +11 -0
  77. data/sorbet/rbi/shims/lib/roast/dsl/workflow_execution_context.rbi +11 -0
  78. data/sorbet/rbi/todo.rbi +7 -0
  79. metadata +46 -5
  80. data/package-lock.json +0 -6
  81. /data/sorbet/rbi/gems/{rack@2.2.17.rbi → rack@2.2.18.rbi} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '0019e9658942394301e91581ecc45b259c76f1999f17cc8f1ddf4f97692ea639'
4
- data.tar.gz: feb3d19b4316a2ea2751347a614f816c22cbd53182821eed661488c760a9897a
3
+ metadata.gz: fc01ff6a90dbe17c735b17935bb75bcfda0ee47c0b9dbee6234292cc4f5f3799
4
+ data.tar.gz: caa21e581b5476a65c2dc8b0b7a27f1b20baf2f493260d954785724f73a3ff40
5
5
  SHA512:
6
- metadata.gz: b8c5442247e1e0aae1cb7aa4509a3724fb1867bf9b05b15cfe1da4a8287db1db8229026ad3be66cc450b57650e8dd00d4eca632021680f3320d2bbf8dd19fc89
7
- data.tar.gz: bbf18e07720a9cfe48abc2780f0f2cdba7d091bb3b2103318556910693dc54b09980ac0f2133876a811700345cdb6fa9e32d87cd4622f9039fe1bbbb7ffe3151
6
+ metadata.gz: 6ea65bb2381627119b5740f9353b064d046e26b2769e0be36baf3f13b9696e30e6bd6332fbbf8c4305fea33d1b2bb12d81a4a56871f3793c3718e2ac8407c6ae
7
+ data.tar.gz: 23ff60c01817d64bae76bb16c0395d8cc7aa95cd621550cebc5fc6763fcf5ba8a023542bf07b49067fa9210d8f228f538b0e82159d897397d08532deece73bc4
data/.gitignore CHANGED
@@ -8,6 +8,7 @@
8
8
  /pkg/
9
9
  /spec/reports/
10
10
  /tmp/
11
+ /sig/generated/
11
12
  .CLAUDE.local.md
12
13
  .rspec_status
13
14
  **/.claude/settings.local.json
data/.rubocop.yml CHANGED
@@ -15,6 +15,7 @@ AllCops:
15
15
  - 'bin/*'
16
16
  - 'spec/fixtures/**/*'
17
17
  - 'test/fixtures/**/*'
18
+ - 'examples/**/*'
18
19
  SuggestExtensions: false
19
20
  TargetRubyVersion: 3.4
20
21
  UseCache: true
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- roast-ai (0.4.8)
4
+ roast-ai (0.4.9)
5
5
  activesupport (>= 7.0)
6
6
  cli-kit (~> 5.0)
7
7
  cli-ui (= 2.3.0)
@@ -167,7 +167,7 @@ GEM
167
167
  method_source (~> 1.0)
168
168
  public_suffix (6.0.2)
169
169
  racc (1.8.1)
170
- rack (2.2.17)
170
+ rack (2.2.19)
171
171
  rainbow (3.1.1)
172
172
  raix (1.0.2)
173
173
  activesupport (>= 6.0)
@@ -185,7 +185,7 @@ GEM
185
185
  rbs (3.9.4)
186
186
  logger
187
187
  regexp_parser (2.10.0)
188
- rexml (3.4.1)
188
+ rexml (3.4.2)
189
189
  rubocop (1.77.0)
190
190
  json (~> 2.3)
191
191
  language_server-protocol (~> 3.17.0.2)
data/README.md CHANGED
@@ -374,20 +374,24 @@ Roast supports several types of steps:
374
374
  ```yaml
375
375
  steps:
376
376
  - analyze_code
377
- - get_user_feedback:
377
+ - input:
378
+ name: get_user_feedback
378
379
  prompt: "Should we proceed with the refactoring? (yes/no)"
379
380
  type: confirm
380
- - review_changes:
381
+ - input:
382
+ name: review_changes
381
383
  prompt: "Enter your review comments"
382
384
  type: text
383
- - select_strategy:
385
+ - input:
386
+ name: select_strategy
384
387
  prompt: "Choose optimization strategy"
385
- type: select
388
+ type: choice
386
389
  options:
387
390
  - "Performance optimization"
388
391
  - "Memory optimization"
389
392
  - "Code clarity"
390
- - api_configuration:
393
+ - input:
394
+ name: api_configuration
391
395
  prompt: "Enter API key"
392
396
  type: password
393
397
  ```
@@ -0,0 +1,112 @@
1
+ # typed: false
2
+ # frozen_string_literal: true
3
+
4
+ #### cmd
5
+
6
+ # Passing just the command to execute will run it and return the output
7
+ cmd <<~SHELLSTEP
8
+ echo "raw run without storing, should see me once""
9
+ SHELLSTEP
10
+
11
+ # Passing a name finds or creates an object and returns that
12
+ cmd_cog = cmd(:hello)
13
+ puts "This is our new cmd_cog named ':hello': #{cmd_cog}"
14
+
15
+ # We can set a command to run for later
16
+ cmd(:set_and_run).set("echo 'set_and_run, should see me once'")
17
+ cmd(:set_and_run).run
18
+
19
+ # Similarly, we can run immediately and then re-run later
20
+ cmd(:run_and_rerun).run("echo 'run_and_rerun, should see me twice'")
21
+ cmd(:run_and_rerun).run
22
+
23
+ #### graph
24
+
25
+ # We can open and re-open a graph, and then execute it
26
+ graph(:updatable) do |graph|
27
+ graph.node(:open_cmd) do |state|
28
+ state[:open] = cmd("echo 'From a node added in first open, should see me once'")
29
+ end
30
+ end
31
+
32
+ graph(:updatable) do |graph|
33
+ graph.node(:reopen_cmd) do |state|
34
+ state[:reopen] = cmd("echo 'From a node added in reopen, should see me once'")
35
+ end
36
+ end
37
+
38
+ graph(:yea).execute
39
+
40
+ # We can also just populate and execute a graph in one go by calling graph.execute in the block.
41
+ graph(:define_and_exec) do |graph|
42
+ graph.node(:hi) do |state|
43
+ state[:hi_msg] = cmd("echo 'hi msg'")
44
+ end
45
+
46
+ graph.execute
47
+ end
48
+
49
+ # We can have subgraphs, because why not
50
+ graph(:outer) do |graph|
51
+ graph.subgraph(:inner) do |subgraph|
52
+ subgraph.node(:inner_node) do |inner_state|
53
+ inner_state[:foo] = cmd("echo 'inner_state foo'")
54
+ end
55
+ end
56
+
57
+ graph.node(:outer) do |outer_state|
58
+ outer_state[:bar] = cmd("echo 'outer_state bar'")
59
+ end
60
+
61
+ graph.execute
62
+ end
63
+
64
+ # We can specify our own edges
65
+ graph(:edges) do |graph|
66
+ graph.node(:thing1) do |state|
67
+ state[:thing1] = cmd("echo 'thing1'")
68
+ end
69
+
70
+ graph.node(:thing2) do |state|
71
+ state[:thing2] = cmd("echo 'thing2'")
72
+ end
73
+
74
+ graph.edge(from: :START, to: :thing1)
75
+ graph.edge(from: :thing1, to: :thing2)
76
+ graph.edge(from: :thing2, to: :DONE)
77
+
78
+ graph.execute
79
+ end
80
+
81
+ # We can have parallel execution
82
+ graph(:parallel) do |graph|
83
+ graph.node(:thing1) do |state|
84
+ state[:thing1] = cmd("sleep 0.5 && echo 'parallel thing1'")
85
+ end
86
+
87
+ graph.node(:thing2) do |state|
88
+ state[:thing2] = cmd("sleep 0.5 && echo 'parallel thing2'")
89
+ end
90
+
91
+ graph.edge(from: :START, to: [:thing1, :thing2])
92
+ graph.edge(from: [:thing1, :thing2], to: :DONE)
93
+
94
+ graph.execute
95
+ end
96
+
97
+ # We can have edges that are defined with a block
98
+ graph(:quantum) do |graph|
99
+ graph.node(:thing1) do |state|
100
+ state[:thing1] = cmd("echo 'quantum thing1'")
101
+ end
102
+
103
+ graph.edge(from: :START) do |_state|
104
+ :thing1
105
+ end
106
+
107
+ graph.edge(from: :thing1) do |_state|
108
+ :DONE
109
+ end
110
+
111
+ graph.execute
112
+ end
data/dsl/prototype.rb ADDED
@@ -0,0 +1,17 @@
1
+ # typed: true
2
+ # frozen_string_literal: true
3
+
4
+ #: self as Roast::DSL::Executor
5
+
6
+ config do
7
+ cmd(:echo) { print_all! }
8
+ end
9
+
10
+ execute do
11
+ # Anonymous cog. Added to the stack directly and given an autogenerated key in cog storage
12
+ # Use for actions you do once and forget about, don't need configuration
13
+ cmd { "touch tmp/#{SecureRandom.uuid} " }
14
+
15
+ # Named cog. Configuration for this specific instance will be looked up from config block
16
+ cmd(:echo) { "echo 'Hello World!'" }
17
+ end
data/dsl/simple.rb CHANGED
@@ -1,10 +1,8 @@
1
- # typed: true
1
+ # typed: false
2
2
  # frozen_string_literal: true
3
3
 
4
- #: self as Roast::DSL::Executor
5
-
6
- # This is a dead simple workflow that calls two shell scripts
7
- shell <<~SHELLSTEP
4
+ # This is a dead simple workflow that calls two commands
5
+ cmd <<~CMDSTEP
8
6
  echo "I have no idea what's going on"
9
- SHELLSTEP
10
- shell "pwd"
7
+ CMDSTEP
8
+ cmd "pwd"
@@ -0,0 +1,18 @@
1
+ # typed: false
2
+ # frozen_string_literal: true
3
+
4
+ # How do we pass information between steps?
5
+ # Demonstrate by passing result of a command output to another step
6
+
7
+ config do
8
+ cmd(:echo) { display! }
9
+ end
10
+
11
+ execute do
12
+ cmd(:ls) { "ls -al" }
13
+ cmd(:echo) do
14
+ # TODO: this is a bespoke output object for cmd, is there a generic one we can offer
15
+ first_line = cmd(:ls).command_output.split("\n").second
16
+ "echo '#{first_line}'"
17
+ end
18
+ end
@@ -0,0 +1,46 @@
1
+ # Test Grading Workflow
2
+
3
+ This workflow acts as a senior software engineer and testing expert to evaluate the quality of test files based on best practices and guidelines.
4
+
5
+ ## Usage
6
+
7
+ ```bash
8
+ # Run the grading workflow on a test file
9
+ roast execute examples/grading/workflow.yml path/to/your_test.rb
10
+ ```
11
+
12
+ ## How it Works
13
+
14
+ 1. **read_dependencies**: Analyzes the test file and its dependencies
15
+ 2. **run_coverage**: Executes the test with coverage tracking
16
+ 3. **generate_grades**: Evaluates test quality across multiple dimensions
17
+ 4. **verify_test_helpers**: Checks for proper test helper usage
18
+ 5. **verify_mocks_and_stubs**: Ensures appropriate use of test doubles
19
+ 6. **analyze_coverage**: Reviews code coverage metrics
20
+ 7. **generate_recommendations**: Provides improvement suggestions
21
+ 8. **calculate_final_grade**: Computes an overall grade (A-F scale)
22
+ 9. **format_result**: Formats the final output
23
+
24
+ ## Customization
25
+
26
+ Feel free to adapt this workflow to your testing environment:
27
+
28
+ - **Different test frameworks**: Modify `run_coverage.rb` to work with RSpec, Jest, pytest, etc.
29
+ - **Coverage tools**: Replace the coverage command with your preferred tool (SimpleCov, Istanbul, Coverage.py)
30
+ - **Grading criteria**: Adjust the prompts in each step to match your team's standards
31
+
32
+ ## Example Output
33
+
34
+ ```
35
+ ========== TEST GRADE REPORT ==========
36
+ Test file: test/example_test.rb
37
+
38
+ FINAL GRADE:
39
+ Score: 85/100
40
+ Letter Grade: B
41
+
42
+ RECOMMENDATIONS:
43
+ - Add edge case testing for error conditions
44
+ - Improve test descriptions for clarity
45
+ - Consider extracting common setup to helper methods
46
+ ```
@@ -0,0 +1,52 @@
1
+ <coverage_results>
2
+ <%= workflow.output["run_coverage"] %>
3
+ </coverage_results>
4
+
5
+ Analyze the results and score them on a scale of 1-10 using the following rubrics:
6
+
7
+ <line_coverage>
8
+ 0-1: Critical failure (0-20% coverage) - Core functionality remains completely untested
9
+ 2-3: Poor coverage (21-40%) - Major gaps; many key functions lack any testing
10
+ 4-5: Inadequate coverage (41-60%) - Several important code paths are not executed
11
+ 6-7: Moderate coverage (61-80%) - Notable gaps remain; some important functionality lacks coverage
12
+ 8-9: Good coverage (81-95%) - Only minor or edge case code paths remain untested
13
+ 10: Excellent coverage (96-100%)
14
+ </line_coverage>
15
+
16
+ <branch_coverage>
17
+ 0-1: Critical failure (0-20% branch coverage) - Almost no conditional branches are tested
18
+ 2-3: Poor coverage (21-40%) - Most conditional logic remains untested
19
+ 4-5: Inadequate coverage (41-60%) - Many conditions are only tested for one outcome
20
+ 6-7: Moderate coverage (61-80%) - Some conditions lack testing for all outcomes
21
+ 8-9: Good coverage (81-95%) - Most conditions are tested for most outcomes
22
+ 10: Excellent coverage (96-100%)
23
+ </branch_coverage>
24
+
25
+ <method_coverage>
26
+ 0-1: Critical failure (0-20% method coverage) - Most or core functionality methods are untested
27
+ 2-3: Poor coverage (21-40%) - Several public API methods remain untested
28
+ 4-5: Inadequate coverage (41-60%) - Some important public methods lack tests
29
+ 6-7: Moderate coverage (61-80%) - Notable gaps remain; some public methods may lack comprehensive testing
30
+ 8-9: Good coverage (81-95%) - Nearly all public methods are tested; private methods are mostly covered via public method tests
31
+ 10: Excellent coverage (96-100%)
32
+ </method_coverage>
33
+
34
+ RESPONSE FORMAT
35
+ You must respond in JSON format within <json> XML tags. Example:
36
+
37
+ <json>
38
+ {
39
+ "method_coverage": {
40
+ "score": "10",
41
+ "justification": "The source file has 100% method coverage, indicating all methods are being tested."
42
+ },
43
+ "line_coverage": {
44
+ "score": 10,
45
+ "justification": "The source file has 100% line coverage, indicating all executable lines are tested."
46
+ },
47
+ "branch_coverage": {
48
+ "score": 8,
49
+ "justification": "The source file has 80% branch coverage, indicating some branches need testing."
50
+ }
51
+ }
52
+ </json>
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CalculateFinalGrade < Roast::Workflow::BaseStep
4
+ WEIGHTS = {
5
+ test_helpers: 0.2,
6
+ mocks_and_stubs: 0.2,
7
+ readability: 0.2,
8
+ maintainability: 0.2,
9
+ effectiveness: 0.2,
10
+ }.freeze
11
+
12
+ def call
13
+ llm_analysis = workflow.output["generate_grades"]
14
+
15
+ weighted_sum = WEIGHTS.sum do |criterion, weight|
16
+ score = llm_analysis[criterion.to_s]["score"].to_f / 10.0
17
+ score * weight
18
+ end
19
+
20
+ {
21
+ final_score: {
22
+ weighted_score: weighted_sum,
23
+ letter_grade: calculate_letter_grade(weighted_sum),
24
+ },
25
+ rubric_scores: calculate_rubric_scores(llm_analysis),
26
+ }
27
+ end
28
+
29
+ private
30
+
31
+ def calculate_letter_grade(score)
32
+ case score
33
+ when 0.9..1.0
34
+ "A"
35
+ when 0.8...0.9
36
+ "B"
37
+ when 0.7...0.8
38
+ "C"
39
+ when 0.6...0.7
40
+ "D"
41
+ else
42
+ "F"
43
+ end
44
+ end
45
+
46
+ def calculate_rubric_scores(llm_analysis)
47
+ scores = {}
48
+
49
+ WEIGHTS.each_key do |criterion|
50
+ next 1 unless llm_analysis[criterion.to_s]
51
+ raw_score = llm_analysis[criterion.to_s]["score"].to_f
52
+ normalized_score = raw_score / 10.0
53
+
54
+ scores[criterion] = {
55
+ raw_value: raw_score,
56
+ score: normalized_score,
57
+ description: llm_analysis[criterion.to_s]["justification"],
58
+ weighted_score: normalized_score * WEIGHTS[criterion],
59
+ }
60
+ end
61
+
62
+ scores
63
+ end
64
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ class FormatResult < Roast::Workflow::BaseStep
4
+ RUBRIC = {
5
+ test_helpers: { description: "Test Helpers Usage", weight: 0.2 },
6
+ mocks_and_stubs: { description: "Mocks and Stubs Usage", weight: 0.2 },
7
+ readability: { description: "Test Readability", weight: 0.2 },
8
+ maintainability: { description: "Test Maintainability", weight: 0.2 },
9
+ effectiveness: { description: "Test Effectiveness", weight: 0.2 },
10
+ }.freeze
11
+
12
+ def call
13
+ append_to_final_output(<<~OUTPUT)
14
+ ========== TEST GRADE REPORT ==========
15
+ Test file: #{workflow.file}
16
+ OUTPUT
17
+
18
+ format_results
19
+ append_to_final_output("\n\n")
20
+ end
21
+
22
+ private
23
+
24
+ def format_results
25
+ # With HashWithIndifferentAccess, we can simply access with either syntax
26
+ grade_data = workflow.output["calculate_final_grade"]
27
+
28
+ unless grade_data
29
+ return append_to_final_output("Error: Grading data not available. This may be because you're replaying the workflow from this step, but the previous step data is missing or not found in the selected session.")
30
+ end
31
+
32
+ format_grade(grade_data)
33
+
34
+ # Make sure rubric_scores exists before trying to iterate over it
35
+ unless grade_data[:rubric_scores]
36
+ return append_to_final_output("Error: Rubric scores data not available in the workflow output.")
37
+ end
38
+
39
+ append_to_final_output("RUBRIC SCORES:")
40
+ grade_data[:rubric_scores].each do |category, data|
41
+ # Safely access RUBRIC with a fallback for potentially missing categories
42
+ rubric_item = RUBRIC[category.to_sym] || { description: "Unknown Category", weight: 0 }
43
+
44
+ append_to_final_output(" #{rubric_item[:description]} (#{(rubric_item[:weight] * 100).round}% of grade):")
45
+ append_to_final_output(" Value: #{data[:raw_value] || "N/A"}")
46
+ append_to_final_output(" Score: #{data[:score] ? (data[:score] * 10).round : "N/A"}/10 - \"#{data[:description] || "No description available"}\"")
47
+ end
48
+ end
49
+
50
+ def format_grade(grade_data)
51
+ return append_to_final_output("\nError: Final grade data not available.") unless grade_data && grade_data[:final_score]
52
+
53
+ letter_grade = grade_data[:final_score][:letter_grade]
54
+ celebration_emoji = letter_grade == "A" ? "🎉" : ""
55
+ append_to_final_output(<<~OUTPUT)
56
+ \nFINAL GRADE:
57
+ Score: #{(grade_data[:final_score][:weighted_score] * 100).round}/100
58
+ Letter Grade: #{letter_grade} #{celebration_emoji}
59
+ OUTPUT
60
+ end
61
+ end
@@ -0,0 +1,105 @@
1
+ These are the key testing guidelines to consider in your evaluation:
2
+
3
+ - Tests should serve as specifications that define expected behaviors
4
+ - Tests should have descriptive names that clearly communicate intent
5
+ - Tests should focus on behavior rather than implementation details
6
+ - Excessive mocking/stubbing should be avoided in favor of testing real behavior
7
+ - Tests should be well-structured with minimal setup complexity
8
+ - Tests should be maintainable and not break when implementation details change
9
+ - Tests should cover edge cases and error conditions
10
+ - Tests should follow proper naming conventions and directory structure
11
+ - Tests should not modify the behaviour of the code being tested (e.g. making a private method public in tests)
12
+
13
+ Now consider the full transcript and evaluate the test being graded based on the following rubrics on a scale of 1-10:
14
+
15
+ <test_helpers>
16
+ 0-1: Extremely poor helper usage - Helpers used incorrectly or inappropriately, making tests harder to understand
17
+ 2-3: Poor helper usage - Helpers are poorly designed, tightly coupled to implementation, or used incorrectly
18
+ 4-5: Basic helper usage - Helpers work but may be poorly organized or not reusable
19
+ 6-7: Good helper usage - Helpers are well-designed and used appropriately
20
+ 8-9: Very good helper usage - Helpers are well-factored, reusable, and make tests clearer
21
+ 10: Excellent helper usage - Helpers are perfectly designed, highly reusable, and significantly improve test clarity and maintainability. Also give this score to tests that DO NOT use test helpers at all.
22
+ </test_helpers>
23
+
24
+ <mocks_and_stubs>
25
+ 0-1: Extremely poor mocking - Mocks/stubs used incorrectly or excessively, completely hiding real behavior
26
+ 2-3: Poor mocking - Heavy reliance on mocks that couple tests to implementation; mocks don't match real behavior
27
+ 4-5: Basic mocking - Mocks used appropriately but may be overused or not match implementation exactly
28
+ 6-7: Good mocking - Mocks used judiciously where needed; generally match implementation
29
+ 8-9: Very good mocking - Minimal mocking focused on external dependencies; accurately reflects real behavior
30
+ 10: Excellent mocking - Mocks used only where absolutely necessary (external APIs, etc); perfectly match real implementations; maintain loose coupling
31
+ </mocks_and_stubs>
32
+
33
+ <readability>
34
+ 0-1: Extremely poor readability - Test purpose is impossible to understand; no structure or organization
35
+ 2-3: Poor readability - Test names are vague or misleading; structure is confusing with no clear assertions
36
+ 4-5: Basic readability - Structure is understandable but not optimized for clarity
37
+ 6-7: Good readability - Structure is logical with clear assertions
38
+ 8-9: Very readable - Well-organized with explicit, meaningful test names and assertions
39
+ 10: Exceptionally readable - Test names serve as perfect specifications; elegant structure with context-providing descriptions; self-documenting with clear setup, execution, and verification phases
40
+ </readability>
41
+
42
+ <maintenability>
43
+ 0-1: Extremely brittle - Tests are completely coupled to implementation details
44
+ 2-3: Highly unmaintainable - Will require significant rework when code changes because of heavy coupling to implementation details
45
+ 4-5: Somewhat maintainable - Some coupling to implementation details
46
+ 6-7: Reasonably maintainable - Tests mostly focus on behavior over implementation; limited coupling to implementation details
47
+ 8-9: Highly maintainable - Tests focus on behavior rather than implementation; changes to implementation should rarely break tests
48
+ 10: Exceptionally maintainable - Tests purely focus on behavior and public interfaces; implementation can be completely refactored without breaking tests; well-factored test helpers and fixtures
49
+ </maintenability>
50
+
51
+ <effectiveness>
52
+ 0-1: Ineffective - Don't validate actual behavior and could pass even if code is broken
53
+ 2-3: Minimally effective - Only the most basic functionality validated. Many incorrect behaviors would not be caught
54
+ 4-5: Partially effective - Only catch obvious issues but miss subtle bugs; limited validation of actual outcomes
55
+ 6-7: Reasonably effective - Should catch most common bugs
56
+ 8-9: Highly effective - Should catch nearly all bugs
57
+ 10: Exceptionally effective - Should catch even subtle edge case bugs; validate both positive and negative cases
58
+ </effectiveness>
59
+
60
+ While grading, consider the following goals as being applicable across all rubrics:
61
+
62
+ SUBJECTIVE:
63
+ - Well-written: Organized, easy to understand, and follow best practices
64
+ - Real behavior: Validate what the code does rather than implementation details
65
+ - Isolated: Should not depend on external systems, services, or APIs. Note: The use of fixtures such as `shops(:snowdevil)` is expected and should not be penalized. The only exception is when the SUT is being loaded as a fixture unnecessarily when it could be instantiated directly.
66
+
67
+ OBJECTIVE
68
+ - Idempotent: Should be able to run repeatedly without affecting outcome or side effects.
69
+ - Deterministic: Should produce the same results across all runs and environments.
70
+ - No sleep: Does not include sleep calls or rely on timing for synchronization.
71
+ - Concurrent: Properly handles concurrent execution paths without errors.
72
+ - Timeless: Does not depend on the current date or time. Will not fail due to changes such as daylight savings or leap years. Specifically with regards to handling time, look for anti-patterns like `Time.current + 7.days.to_i`, which fails on DST changes. The correct approach is `7.days.from_now`.
73
+
74
+ VIOLATING ANY OBJECTIVE GOAL SHOULD RESULT IN AN OVERALL SCORE LESS THAN 5!
75
+
76
+ Provide a brief justification for each score, using a maximum of 1-3 sentences. (Note that specific recommendations for improvement are not needed at this step.)
77
+
78
+ You are acting as a stern and relentless striver for excellence in programming, so you must be highly critical. The point of this grading exercise is to facilitate substantial improvement, not just stroking the programmer's ego. Do not hesitate to give a failing overall score (0) for serious violations!
79
+
80
+ RESPONSE FORMAT: You must respond in JSON format within <json> XML tags.
81
+
82
+ <json>
83
+ {
84
+ "test_helpers": {
85
+ "score": 4,
86
+ "justification": "Helpers are used incorrectly in several places, reducing test maintainability and clarity. The assert_valid_record helper is being misused with hashes instead of model instances."
87
+ },
88
+ "mocks_and_stubs": {
89
+ "score": 4,
90
+ "justification": "Several mocks don't match the actual implementation, making tests brittle and potentially hiding production bugs. For example, mocking success: true when the service returns status: 'success'."
91
+ },
92
+ "readability": {
93
+ "score": 8,
94
+ "justification": "Test names clearly describe behavior being tested."
95
+ },
96
+ "maintainability": {
97
+ "score": 6,
98
+ "justification": "Tests mostly focus on behavior but have some coupling to implementation."
99
+ },
100
+ "effectiveness": {
101
+ "score": 7,
102
+ "justification": "Tests validate most expected behaviors and would catch common bugs."
103
+ }
104
+ }
105
+ </json>
@@ -0,0 +1,17 @@
1
+ ========== TEST RECOMMENDATIONS ==========
2
+ <%- if response.recommendations.empty? -%>
3
+ No recommendations found.
4
+ <%- else -%>
5
+ <%- response.recommendations.each_with_index do |rec, index| -%>
6
+ Recommendation #<%= index + 1 %>:
7
+ Description: <%= rec.description %>
8
+ Impact: <%= rec.impact %>
9
+ Priority: <%= rec.priority %>
10
+
11
+ Code Suggestion:
12
+
13
+ <%= rec.code_suggestion %>
14
+
15
+ <%- end -%>
16
+ <%- end -%>
17
+ ===========================================
@@ -0,0 +1,60 @@
1
+ Finally, based on the conversation transcript above, go ahead and provide specific, actionable recommendations that would most effectively improve the overall test score.
2
+
3
+ Focus on recommendations that would:
4
+
5
+ 1. Increase coverage
6
+ 2. Add more assertions where needed
7
+ 3. Make the tests more maintainable or readable
8
+ 4. Ensure tests serve as specifications by having clear, descriptive names
9
+ 5. Reduce excessive mocking/stubbing that couples tests to implementation details
10
+ 6. Improve test structure to reduce setup complexity
11
+ 7. Ensure tests focus on behavior rather than implementation details
12
+ 8. Ensure gaps in private methods are tested through public methods
13
+ 9. Fix any issues with test helpers that are used incorrectly or unnecessarily
14
+ 10. Improve efficiency by combining or deleting tests where appropriate (note that having more than one assertion per test is acceptable)
15
+ 11. Fix any violations of the objective criteria (idempotency, determinism, etc.)
16
+ 12. Be specific about edge cases that should be covered by tests. Write down in the recommendations which edge cases you are referring to.
17
+ 13. Do not recommend the use of RSpec features like `let` for Minispec tests.
18
+
19
+ IF YOU IDENTIFY EDGE CASES, YOU MUST BE SPECIFIC ABOUT THEM IN THE RECOMMENDATIONS.
20
+
21
+ RESPONSE FORMAT: You must respond in JSON format inside <json> XML tags without additional commentary.
22
+
23
+ Example:
24
+
25
+ <json>
26
+ {
27
+ "recommendations": [
28
+ {
29
+ "description": "Add tests for uncovered method X",
30
+ "impact": "Would increase method coverage by Y%",
31
+ "priority": "High",
32
+ "code_suggestion": "def test_method_x_with_valid_input\n result = subject.method_x('valid_input')\n assert_equal expected_result, result\nend"
33
+ },
34
+ {
35
+ "description": "Fix time handling to avoid DST issues",
36
+ "impact": "Would make tests deterministic across DST changes",
37
+ "priority": "High",
38
+ "code_suggestion": "# Replace\nexpiry_time = Time.current + 7.days.to_i\n\n# With\nexpiry_time = 7.days.from_now"
39
+ },
40
+ {
41
+ "description": "Add edge case tests for the show action for when the parameter X is blank",
42
+ "impact": "Would improve test completeness and effectiveness",
43
+ "priority": "Medium",
44
+ "code_suggestion": "..."
45
+ },
46
+ {
47
+ "description": "Improve test descriptions to better serve as specifications",
48
+ "impact": "Would make tests more valuable as documentation",
49
+ "priority": "Medium",
50
+ "code_suggestion": "# Replace\ndef test_process\n\n# With\ndef test_process_returns_success_with_valid_input"
51
+ },
52
+ {
53
+ "description": "Replace implementation-focused mocks with behavior assertions",
54
+ "impact": "Would make tests less brittle and more maintainable",
55
+ "priority": "High",
56
+ "code_suggestion": "# Replace\nUserNotifier.expects(:notify).with(user, 'welcome')\n\n# With\nassert_sends_notification(user, 'welcome') do\n subject.process\nend"
57
+ }
58
+ ]
59
+ }
60
+ </json>