roast-ai 0.4.7 → 0.4.8

data/examples/grading/calculate_final_grade.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-class CalculateFinalGrade < Roast::Workflow::BaseStep
-  attr_accessor :llm_analysis
-
-  WEIGHTS = {
-    line_coverage: 0.1,
-    method_coverage: 0.1,
-    branch_coverage: 0.3,
-    test_helpers: 0.1,
-    mocks_and_stubs: 0.1,
-    readability: 0.1,
-    maintainability: 0.1,
-    effectiveness: 0.1,
-  }.freeze
-
-  def call
-    @llm_analysis = workflow.output["generate_grades"].merge(workflow.output["analyze_coverage"])
-    weighted_sum = WEIGHTS.sum do |criterion, weight|
-      score = llm_analysis[criterion.to_s]["score"].to_f / 10.0
-      score * weight
-    end
-
-    {
-      final_score: {
-        weighted_score: weighted_sum,
-        letter_grade: calculate_letter_grade(weighted_sum),
-      },
-      rubric_scores: calculate_rubric_scores,
-    }
-  end
-
-  private
-
-  def calculate_letter_grade(score)
-    case score
-    when 0.9..1.0
-      "A"
-    when 0.8...0.9
-      "B"
-    when 0.7...0.8
-      "C"
-    when 0.6...0.7
-      "D"
-    else
-      "F"
-    end
-  end
-
-  def calculate_rubric_scores
-    scores = {}
-
-    WEIGHTS.each_key do |criterion|
-      raw_score = llm_analysis[criterion.to_s]["score"].to_f
-      normalized_score = raw_score / 10.0
-
-      scores[criterion] = {
-        raw_value: raw_score,
-        score: normalized_score,
-        description: llm_analysis[criterion.to_s]["justification"],
-        weighted_score: normalized_score * WEIGHTS[criterion],
-      }
-    end
-
-    scores
-  end
-end

data/examples/grading/format_result.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-class FormatResult < Roast::Workflow::BaseStep
-  RUBRIC = {
-    line_coverage: { description: "Line Coverage", weight: 0.1 },
-    method_coverage: { description: "Method Coverage", weight: 0.1 },
-    branch_coverage: { description: "Branch Coverage", weight: 0.3 },
-    test_helpers: { description: "Test Helpers Usage", weight: 0.1 },
-    mocks_and_stubs: { description: "Mocks and Stubs Usage", weight: 0.1 },
-    readability: { description: "Test Readability", weight: 0.1 },
-    maintainability: { description: "Test Maintainability", weight: 0.1 },
-    effectiveness: { description: "Test Effectiveness", weight: 0.1 },
-  }.freeze
-
-  def call
-    append_to_final_output(<<~OUTPUT)
-      ========== TEST GRADE REPORT ==========
-      Test file: #{workflow.file}
-    OUTPUT
-
-    format_results
-    append_to_final_output("\n\n")
-  end
-
-  private
-
-  def format_results
-    # With HashWithIndifferentAccess, we can simply access with either syntax
-    grade_data = workflow.output["calculate_final_grade"]
-
-    unless grade_data
-      return append_to_final_output("Error: Grading data not available. This may be because you're replaying the workflow from this step, but the previous step data is missing or not found in the selected session.")
-    end
-
-    format_grade(grade_data)
-
-    # Make sure rubric_scores exists before trying to iterate over it
-    unless grade_data[:rubric_scores]
-      return append_to_final_output("Error: Rubric scores data not available in the workflow output.")
-    end
-
-    append_to_final_output("RUBRIC SCORES:")
-    grade_data[:rubric_scores].each do |category, data|
-      # Safely access RUBRIC with a fallback for potentially missing categories
-      rubric_item = RUBRIC[category.to_sym] || { description: "Unknown Category", weight: 0 }
-
-      append_to_final_output("  #{rubric_item[:description]} (#{(rubric_item[:weight] * 100).round}% of grade):")
-      append_to_final_output("    Value: #{data[:raw_value] || "N/A"}")
-      append_to_final_output("    Score: #{data[:score] ? (data[:score] * 10).round : "N/A"}/10 - \"#{data[:description] || "No description available"}\"")
-    end
-  end
-
-  def format_grade(grade_data)
-    return append_to_final_output("\nError: Final grade data not available.") unless grade_data && grade_data[:final_score]
-
-    letter_grade = grade_data[:final_score][:letter_grade]
-    celebration_emoji = letter_grade == "A" ? "🎉" : ""
-    append_to_final_output(<<~OUTPUT)
-      \nFINAL GRADE:
-        Score: #{(grade_data[:final_score][:weighted_score] * 100).round}/100
-        Letter Grade: #{letter_grade} #{celebration_emoji}
-    OUTPUT
-  end
-end

data/examples/grading/generate_grades/prompt.md

@@ -1,105 +0,0 @@
-These are the key testing guidelines to consider in your evaluation:
-
-- Tests should serve as specifications that define expected behaviors
-- Tests should have descriptive names that clearly communicate intent
-- Tests should focus on behavior rather than implementation details
-- Excessive mocking/stubbing should be avoided in favor of testing real behavior
-- Tests should be well-structured with minimal setup complexity
-- Tests should be maintainable and not break when implementation details change
-- Tests should cover edge cases and error conditions
-- Tests should follow proper naming conventions and directory structure
-- Tests should not modify the behaviour of the code being tested (e.g. making a private method public in tests)
-
-Now consider the full transcript and evaluate the test being graded based on the following rubrics on a scale of 1-10:
-
-<test_helpers>
-0-1: Extremely poor helper usage - Helpers used incorrectly or inappropriately, making tests harder to understand
-2-3: Poor helper usage - Helpers are poorly designed, tightly coupled to implementation, or used incorrectly
-4-5: Basic helper usage - Helpers work but may be poorly organized or not reusable
-6-7: Good helper usage - Helpers are well-designed and used appropriately
-8-9: Very good helper usage - Helpers are well-factored, reusable, and make tests clearer
-10: Excellent helper usage - Helpers are perfectly designed, highly reusable, and significantly improve test clarity and maintainability. Also give this score to tests that DO NOT use test helpers at all.
-</test_helpers>
-
-<mocks_and_stubs>
-0-1: Extremely poor mocking - Mocks/stubs used incorrectly or excessively, completely hiding real behavior
-2-3: Poor mocking - Heavy reliance on mocks that couple tests to implementation; mocks don't match real behavior
-4-5: Basic mocking - Mocks used appropriately but may be overused or not match implementation exactly
-6-7: Good mocking - Mocks used judiciously where needed; generally match implementation
-8-9: Very good mocking - Minimal mocking focused on external dependencies; accurately reflects real behavior
-10: Excellent mocking - Mocks used only where absolutely necessary (external APIs, etc); perfectly match real implementations; maintain loose coupling
-</mocks_and_stubs>
-
-<readability>
-0-1: Extremely poor readability - Test purpose is impossible to understand; no structure or organization
-2-3: Poor readability - Test names are vague or misleading; structure is confusing with no clear assertions
-4-5: Basic readability - Structure is understandable but not optimized for clarity
-6-7: Good readability - Structure is logical with clear assertions
-8-9: Very readable - Well-organized with explicit, meaningful test names and assertions
-10: Exceptionally readable - Test names serve as perfect specifications; elegant structure with context-providing descriptions; self-documenting with clear setup, execution, and verification phases
-</readability>
-
-<maintenability>
-0-1: Extremely brittle - Tests are completely coupled to implementation details
-2-3: Highly unmaintainable - Will require significant rework when code changes because of heavy coupling to implementation details
-4-5: Somewhat maintainable - Some coupling to implementation details
-6-7: Reasonably maintainable - Tests mostly focus on behavior over implementation; limited coupling to implementation details
-8-9: Highly maintainable - Tests focus on behavior rather than implementation; changes to implementation should rarely break tests
-10: Exceptionally maintainable - Tests purely focus on behavior and public interfaces; implementation can be completely refactored without breaking tests; well-factored test helpers and fixtures
-</maintenability>
-
-<effectiveness>
-0-1: Ineffective - Don't validate actual behavior and could pass even if code is broken
-2-3: Minimally effective - Only the most basic functionality validated. Many incorrect behaviors would not be caught
-4-5: Partially effective - Only catch obvious issues but miss subtle bugs; limited validation of actual outcomes
-6-7: Reasonably effective - Should catch most common bugs
-8-9: Highly effective - Should catch nearly all bugs
-10: Exceptionally effective - Should catch even subtle edge case bugs; validate both positive and negative cases
-</effectiveness>
-
-While grading, consider the following goals as being applicable across all rubrics:
-
-SUBJECTIVE:
-- Well-written: Organized, easy to understand, and follow best practices
-- Real behavior: Validate what the code does rather than implementation details
-- Isolated: Should not depend on external systems, services, or APIs. Note: The use of fixtures such as `shops(:snowdevil)` is expected and should not be penalized. The only exception is when the SUT is being loaded as a fixture unnecessarily when it could be instantiated directly.
-
-OBJECTIVE
-- Idempotent: Should be able to run repeatedly without affecting outcome or side effects.
-- Deterministic: Should produce the same results across all runs and environments.
-- No sleep: Does not include sleep calls or rely on timing for synchronization.
-- Concurrent: Properly handles concurrent execution paths without errors.
-- Timeless: Does not depend on the current date or time. Will not fail due to changes such as daylight savings or leap years. Specifically with regards to handling time, look for anti-patterns like `Time.current + 7.days.to_i`, which fails on DST changes. The correct approach is `7.days.from_now`.
-
-VIOLATING ANY OBJECTIVE GOAL SHOULD RESULT IN AN OVERALL SCORE LESS THAN 5!
-
-Provide a brief justification for each score, using a maximum of 1-3 sentences. (Note that specific recommendations for improvement are not needed at this step.)
-
-You are acting as a stern and relentless striver for excellence in programming, so you must be highly critical. The point of this grading exercise is to facilitate substantial improvement, not just stroking the programmer's ego. Do not hesitate to give a failing overall score (0) for serious violations!
-
-RESPONSE FORMAT: You must respond in JSON format within <json> XML tags.
-
-<json>
-{
-  "test_helpers": {
-    "score": 4,
-    "justification": "Helpers are used incorrectly in several places, reducing test maintainability and clarity. The assert_valid_record helper is being misused with hashes instead of model instances."
-  },
-  "mocks_and_stubs": {
-    "score": 4,
-    "justification": "Several mocks don't match the actual implementation, making tests brittle and potentially hiding production bugs. For example, mocking success: true when the service returns status: 'success'."
-  },
-  "readability": {
-    "score": 8,
-    "justification": "Test names clearly describe behavior being tested."
-  },
-  "maintainability": {
-    "score": 6,
-    "justification": "Tests mostly focus on behavior but have some coupling to implementation."
-  },
-  "effectiveness": {
-    "score": 7,
-    "justification": "Tests validate most expected behaviors and would catch common bugs."
-  }
-}
-</json>

data/examples/grading/generate_recommendations/output.txt

@@ -1,17 +0,0 @@
-========== TEST RECOMMENDATIONS ==========
-<%- if response.recommendations.empty? -%>
-No recommendations found.
-<%- else -%>
-<%- response.recommendations.each_with_index do |rec, index| -%>
-Recommendation #<%= index + 1 %>:
-Description: <%= rec.description %>
-Impact: <%= rec.impact %>
-Priority: <%= rec.priority %>
-
-Code Suggestion:
-
-<%= rec.code_suggestion %>
-
-<%- end -%>
-<%- end -%>
-===========================================

data/examples/grading/generate_recommendations/prompt.md

@@ -1,60 +0,0 @@
-Finally, based on the conversation transcript above, go ahead and provide specific, actionable recommendations that would most effectively improve the overall test score.
-
-Focus on recommendations that would:
-
-1. Increase coverage
-2. Add more assertions where needed
-3. Make the tests more maintainable or readable
-4. Ensure tests serve as specifications by having clear, descriptive names
-5. Reduce excessive mocking/stubbing that couples tests to implementation details
-6. Improve test structure to reduce setup complexity
-7. Ensure tests focus on behavior rather than implementation details
-8. Ensure gaps in private methods are tested through public methods
-9. Fix any issues with test helpers that are used incorrectly or unnecessarily
-10. Improve efficiency by combining or deleting tests where appropriate (note that having more than one assertion per test is acceptable)
-11. Fix any violations of the objective criteria (idempotency, determinism, etc.)
-12. Be specific about edge cases that should be covered by tests. Write down in the recommendations which edge cases you are referring to.
-13. Do not recommend the use of RSpec features like `let` for Minispec tests.
-
-IF YOU IDENTIFY EDGE CASES, YOU MUST BE SPECIFIC ABOUT THEM IN THE RECOMMENDATIONS.
-
-RESPONSE FORMAT: You must respond in JSON format inside <json> XML tags without additional commentary.
-
-Example:
-
-<json>
-{
-  "recommendations": [
-    {
-      "description": "Add tests for uncovered method X",
-      "impact": "Would increase method coverage by Y%",
-      "priority": "High",
-      "code_suggestion": "def test_method_x_with_valid_input\n  result = subject.method_x('valid_input')\n  assert_equal expected_result, result\nend"
-    },
-    {
-      "description": "Fix time handling to avoid DST issues",
-      "impact": "Would make tests deterministic across DST changes",
-      "priority": "High",
-      "code_suggestion": "# Replace\nexpiry_time = Time.current + 7.days.to_i\n\n# With\nexpiry_time = 7.days.from_now"
-    },
-    {
-      "description": "Add edge case tests for the show action for when the parameter X is blank",
-      "impact": "Would improve test completeness and effectiveness",
-      "priority": "Medium",
-      "code_suggestion": "..."
-    },  
-    {
-      "description": "Improve test descriptions to better serve as specifications",
-      "impact": "Would make tests more valuable as documentation",
-      "priority": "Medium",
-      "code_suggestion": "# Replace\ndef test_process\n\n# With\ndef test_process_returns_success_with_valid_input"
-    },
-    {
-      "description": "Replace implementation-focused mocks with behavior assertions",
-      "impact": "Would make tests less brittle and more maintainable",
-      "priority": "High",
-      "code_suggestion": "# Replace\nUserNotifier.expects(:notify).with(user, 'welcome')\n\n# With\nassert_sends_notification(user, 'welcome') do\n  subject.process\nend"
-    }
-  ]
-}
-</json>

data/examples/grading/js_test_runner

@@ -1,31 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-if ARGV.length != 2
-  puts "Usage: #{File.basename($PROGRAM_NAME)} SUBJECT_FILE TEST_FILE"
-  exit 1
-end
-
-subject_file, test_file = ARGV
-
-def detect_package_manager
-  return "pnpm" if File.exist?(File.join(Dir.pwd, "pnpm-lock.yaml"))
-  return "yarn" if File.exist?(File.join(Dir.pwd, "yarn.lock"))
-
-  "npm"
-end
-
-jest_options = [
-  "--verbose",
-  "--no-colors",
-  "--ci",
-  "--coverageReporters=text-summary",
-  "--collectCoverageFrom=#{subject_file}",
-]
-
-# Assumes the test command is `test:coverage`
-# Both admin-web and checkout-web use this command
-command = "#{detect_package_manager} run test:coverage -- #{test_file} #{jest_options.join(" ")}"
-
-$stderr.puts "Running: #{command}"
-puts Roast::Helpers::CmdRunner.system(command)

data/examples/grading/rb_test_runner

@@ -1,19 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "rubygems"
-require "bundler/setup"
-
-require "roast/helpers/minitest_coverage_runner"
-
-# Suppress fancy minitest reporting
-ENV["RM_INFO"] = "true"
-
-if ARGV.length != 2
-  puts "Usage: #{File.basename($PROGRAM_NAME)} SUBJECT_FILE TEST_FILE"
-  exit 1
-end
-
-test_file, subject_file = ARGV
-
-Roast::Helpers::MinitestCoverageRunner.new(test_file, subject_file).run

data/examples/grading/read_dependencies/prompt.md

@@ -1,16 +0,0 @@
-Use the provided functions to find and read important dependencies of the provided test file named <%= workflow.file %>.
-
-The first dependency you should always look for is the source file for the prime subject of the test (whatever class this test file is claiming to test). Use `read_file` to read the subject's source code into your conversation transcript, but only if it's not already there from a previous chat.
-
-If you can identify other important application-level dependencies then read them too.
-How many extra dependencies to research is left to your discretion, but ALWAYS make sure you have the subject under test (SUT) in your context before responding.
-
-Once you are finished using tool functions, respond with the relative path to the source file of the SUT inside <sut> tags. IMPORTANT: Include the full relative path from the project root, including any directory prefixes like lib/, app/, etc.
-
-Example:
-
-If you are told to find the dependencies of `test/services/country_db_interface_test.rb`,
-then you would use the functions as explained above and ultimately respond with `<sut>app/services/country_db_interface.rb</sut>`
-
-If the file is found at `lib/roast/workflow/workflow_initializer.rb`, respond with `<sut>lib/roast/workflow/workflow_initializer.rb</sut>` (include the lib/ prefix)
-

data/examples/grading/run_coverage.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-class RunCoverage < Roast::Workflow::BaseStep
-  def call
-    # Run the test with coverage analysis
-    run_test_with_coverage
-  end
-
-  private
-
-  def run_test_with_coverage
-    subject_file = workflow.output["read_dependencies"]
-    subject_file = subject_file.match(%r{<sut>(.*?)</sut>})&.[](1) || subject_file
-    test_file = workflow.file
-    extension = File.extname(test_file).gsub(".", "")
-
-    # Handle JS/TS test files
-    extension = "js" if ["js", "jsx", "ts", "tsx"].include?(extension)
-
-    # Get the absolute path to the test_runner executable
-    test_runner_path = File.expand_path("#{extension}_test_runner", __dir__)
-
-    # Make sure the test_runner executable exists
-    unless File.exist?(test_runner_path)
-      Roast::Helpers::Logger.error("Test runner executable not found: #{test_runner_path}")
-      exit(1)
-    end
-
-    # Resolve paths to prevent issues when pwd differs from project root
-    resolved_subject_file = Roast::Helpers::PathResolver.resolve(subject_file)
-    unless File.exist?(resolved_subject_file)
-      Roast::Helpers::Logger.error("Subject file not found: #{resolved_subject_file}")
-      exit(1)
-    end
-
-    resolved_test_file = Roast::Helpers::PathResolver.resolve(test_file)
-    unless File.exist?(resolved_test_file)
-      Roast::Helpers::Logger.error("Test file not found: #{resolved_test_file}")
-      exit(1)
-    end
-
-    # Run the test_runner using shadowenv for environment consistency
-    command = "shadowenv exec --dir . -- #{test_runner_path} #{resolved_test_file} #{resolved_subject_file}"
-    output, status = Roast::Helpers::CmdRunner.capture2(command)
-
-    unless status.success?
-      Roast::Helpers::Logger.error("Test runner exited with non-zero status: #{status.exitstatus}")
-      Roast::Helpers::Logger.error(output)
-      exit(status.exitstatus)
-    end
-
-    output
-  end
-end

data/examples/grading/verify_mocks_and_stubs/prompt.md

@@ -1,12 +0,0 @@
-Find places in the provided test code where stubbing and mocking are used. Search for the corresponding implementation source code of those dependencies elsewhere in the codebase to validate that the stub or mock matches the implementation that it is doubling. Use the tool functions provided to find and read the dependencies.
-
-Once you've found the dependencies, verify that any mocks and stubs accurately reflect the real implementation. If there are discrepancies, list them out alphabetically with:
-
-1. The name of the mocked/stubbed method
-2. What the mock/stub expects in terms of arguments and/or return values
-3. What the actual implementation actually takes as arguments and returns
-4. Suggestions for fixing the discrepancy
-
-Note: If there are no discrepancies, do not summarize those that accurately reflect their real implementations in the codebase, just respond "All mocks and stubs verified."
-
-IMPORTANT: There's absolutely no need for you to waste time grepping for methods/functions that you know belong to testing libraries such as Mocha's `expects` and `stubs`. Only search for the implementation of things that are stubbed and/or mocked in the test to verify whether the test code matches the implementation code.

data/examples/grading/verify_test_helpers/prompt.md

@@ -1,53 +0,0 @@
-Now identify custom test helpers used in this test for the following purpose:
-
-1. Analyzing if they are used correctly
-2. Understanding test code that has had significant chunks of implementation abstracted away into helpers
-3. Fully understanding custom assertions that are not included by default in Ruby on Rails or part of your base knowledge
-
-Your grep tool function is vital for this work. It provides 4 lines of context before and after the matching line.
-
-For example, if you call `grep(string: "def assert_sql")`, the output will include:
-
-```
-.test/support/helpers/sql_assertions.rb-101-    end
-.test/support/helpers/sql_assertions.rb-102-    result
-.test/support/helpers/sql_assertions.rb-103-  end
-.test/support/helpers/sql_assertions.rb-104-
-.test/support/helpers/sql_assertions.rb:105:  def assert_sql(*patterns_to_match, **kwargs, &block)
-.test/support/helpers/sql_assertions.rb-106-    mysql_only_test!
-.test/support/helpers/sql_assertions.rb-107-
-.test/support/helpers/sql_assertions.rb-108-    result = T.let(nil, T.nilable(T::Boolean))
-.test/support/helpers/sql_assertions.rb-109-    counter = ActiveRecord::SQLCounter.new(**kwargs)
-```
-
-Unfortunately, many test helper methods are undocumented. In those cases (like the example above) the pre-context will be junk. However, there are a number of helper methods that do have very specific and narrow use cases, and those do tend to be well-documented. In those cases, you should use `read_file` to be able to read the full documentation.
-
-For example, here is the result of calling `grep(string: "def assert_sql_events")`
-
-```
-.test/support/helpers/externals_helper.rb-93-  # @example Logs events in the list that did not occur
-.test/support/helpers/externals_helper.rb-94-  #   expected_queries = { "Shop Load" => 1, "User Load" => 1 }
-.test/support/helpers/externals_helper.rb-95-  #   # Fails and reports that User Load occured 0 times instead of expected 1
-.test/support/helpers/externals_helper.rb-96-  #   assert_sql_events(expected_queries) { Shop.current_or_find(shop.id) }
-.test/support/helpers/externals_helper.rb:97:  def assert_sql_events(expected_events, &block)
-.test/support/helpers/externals_helper.rb-98-    mysql_only_test!
-.test/support/helpers/externals_helper.rb-99-
-.test/support/helpers/externals_helper.rb-100-    mysql_events = ExternalsCollector.new(&block).events
-.test/support/helpers/externals_helper.rb-101-      .select { |e| e.first == :mysql }
-```
-
-Notice that the documentation for the `assert_sql_events` method is cutoff. Use your `read_file` tool function to get the whole test helper source code and gain better understanding of how it is intended to be used, with the side benefit of also being able to see how it is implemented.
-
-Note: You will undoubtedly already be familiar with some of Minitest and RSpec's built-in helpers. There is no need to search for those, since they are packaged as gems you won't find them anyway.
-
-DO NOT FORGET TO PREPEND `def` TO YOUR QUERY TO FIND A METHOD DEFINITION INSTEAD OF USAGES, otherwise you may bring back a very large and useless result set!!!
-
-Once you are done understanding the custom test helpers used in the test file, analyze and report on whether it seems like any of the helpers are:
-
-1. Used incorrectly
-2. Used unnecessarily
-3. Any other problem related to the use of helper methods
-
-Where possible, use your best judgment to make recommendations for how to fix problems that you find, but ONLY related to test helpers.
-
-Note: You are only being used to help find problems so it is not necessary to report on correct usage of helpers or to make positive comments.

data/examples/grading/workflow.md

@@ -1,8 +0,0 @@
-As a senior software engineer and testing expert, evaluate the quality of this test file based on guidelines that will be subsequently provided.
-
-Next I will now provide the source code of the test that we will be analyzing, and then step you through a series of analysis activities, before finally asking you to provided a final report.
-
-<test>
-# <%= file %>
-<%= File.read(file) %>
-</test>

data/examples/grading/workflow.rb.md

@@ -1,6 +0,0 @@
-As a senior Ruby engineer and testing expert, evaluate the quality of this Ruby test file. Next I will now provide the source code of the test that we will be analyzing, and then step you through a series of analysis activities, before finally asking you to provided a final report.
-
-<test>
-# <%= file %>
-<%= File.read(file) %>
-</test>

data/examples/grading/workflow.ts+tsx.md

@@ -1,6 +0,0 @@
-As a senior front-end engineer and testing expert, evaluate the quality of this test file. Next I will now provide the source code of the test that we will be analyzing, and then step you through a series of analysis activities, before finally asking you to provided a final report.
-
-<test>
-# <%= file %>
-<%= File.read(file) %>
-</test>

data/examples/grading/workflow.yml

@@ -1,41 +0,0 @@
-name: Test Grading
-api_token: $(echo $OPENAI_API_KEY)
-# model: anthropic:claude-opus-4
-model: gpt-4.1-mini
-
-tools:
-  - Roast::Tools::Grep
-  - Roast::Tools::ReadFile
-  - Roast::Tools::SearchFile
-
-# Uncomment this to run the workflow on modified tests automatically
-# each: '% cd $(git rev-parse --show-toplevel) && git status --porcelain | grep "_test\.rb" | cut -c4- | xargs realpath'
-
-steps:
-  - read_dependencies
-  - run_coverage
-  -
-    - analyze_coverage
-    - verify_test_helpers
-    - verify_mocks_and_stubs
-  - generate_grades
-  - calculate_final_grade
-  - format_result
-  - generate_recommendations
-
-# set non-default attributes for steps below
-analyze_coverage:
-#  model: gpt-4.1-mini
-  json: true
-  
-generate_grades:
-#  model: o3
-  json: true
-
-generate_recommendations:
-#  model: o3
-  json: true
-  params:
-    max_completion_tokens: 5_000
-
-

data/examples/instrumentation.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-# Demonstration of how to use the Roast instrumentation hooks
-# This file would typically be placed in PROJECT_ROOT/.roast/initializers/
-# for automatic loading during workflow execution
-
-# Example: Log all workflow and step events
-ActiveSupport::Notifications.subscribe(/roast\.workflow\./) do |name, start, finish, _id, payload|
-  duration = finish - start
-
-  case name
-  when "roast.workflow.start"
-    puts "\n🚀 Workflow starting: #{payload[:name]}"
-    puts "   Path: #{payload[:workflow_path]}"
-    puts "   Options: #{payload[:options]}"
-  when "roast.workflow.complete"
-    status = payload[:success] ? "✅ Successfully" : "❌ With errors"
-    puts "\n#{status} completed workflow in #{duration.round(2)} seconds"
-  end
-end
-
-# Example: Track step execution times
-ActiveSupport::Notifications.subscribe(/roast\.step\./) do |name, start, finish, _id, payload|
-  duration = finish - start
-
-  case name
-  when "roast.step.start"
-    puts "\n  ▶️ Step starting: #{payload[:step_name]}"
-  when "roast.step.complete"
-    puts "  ✅ Step completed: #{payload[:step_name]} (#{duration.round(3)}s)"
-  when "roast.step.error"
-    puts "  ❌ Step failed: #{payload[:step_name]}"
-    puts "     Error: #{payload[:error]} - #{payload[:message]}"
-  end
-end
-
-# Example: Monitor AI interactions
-ActiveSupport::Notifications.subscribe(/roast\.chat_completion\./) do |name, start, finish, _id, payload|
-  case name
-  when "roast.chat_completion.start"
-    puts "\n  🤖 AI request starting (model: #{payload[:model]})"
-    puts "     Parameters: #{payload[:parameters].inspect}" if payload[:parameters].any?
-  when "roast.chat_completion.complete"
-    duration = finish - start
-    puts "  🤖 AI request completed in #{duration.round(2)}s (execution time: #{payload[:execution_time].round(2)}s)"
-    puts "     Response size: #{payload[:response_size]} characters"
-  when "roast.chat_completion.error"
-    puts "  🤖 AI request failed: #{payload[:error]} - #{payload[:message]}"
-    puts "     Execution time: #{payload[:execution_time].round(2)}s"
-  end
-end
-
-# Example: Track tool executions
-ActiveSupport::Notifications.subscribe(/roast\.tool\./) do |name, _start, _finish, _id, payload|
-  case name
-  when "roast.tool.execute"
-    puts "\n  🔧 Executing tool: #{payload[:function_name]}"
-  when "roast.tool.complete"
-    puts "  🔧 Tool completed: #{payload[:function_name]} (#{payload[:execution_time].round(3)}s)"
-    puts "     Cache enabled: #{payload[:cache_enabled]}"
-  when "roast.tool.error"
-    puts "  🔧 Tool failed: #{payload[:function_name]}"
-    puts "     Error: #{payload[:error]} - #{payload[:message]}"
-    puts "     Execution time: #{payload[:execution_time].round(3)}s"
-  end
-end
-
-# In a Shopify-specific initializer (e.g., .roast/initializers/monorail.rb),
-# you could replace these logging examples with actual Monorail tracking:
-#
-# ActiveSupport::Notifications.subscribe("roast.workflow.start") do |name, start, finish, id, payload|
-#   Roast::Support::Monorail.track_command("run", {
-#     "workflow_path" => payload[:workflow_path],
-#     "name" => payload[:name]
-#   })
-# end