roast-ai 0.4.7 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1d2d1b26f4618a6036bc890e0426b5fe6c10dfd530e1c391d68b46680d2c2bc
-  data.tar.gz: e6b5670d414a2349c639b45a9b9d7b45b74a6dafc48cd706fedf8b55af88b6d2
+  metadata.gz: fc01ff6a90dbe17c735b17935bb75bcfda0ee47c0b9dbee6234292cc4f5f3799
+  data.tar.gz: caa21e581b5476a65c2dc8b0b7a27f1b20baf2f493260d954785724f73a3ff40
 SHA512:
-  metadata.gz: 447c20f3a2b170ce7ced6c14c5a1f62117299a984f513a6fb83f9c53934bbb6239c3dd11a791fe0437475af127a315120140b6e1918c124c25d774e3cc1cf2d3
-  data.tar.gz: cabe9380d1d1859c49d71d79547a1cc13f0d15f550aa373d82523384a0a8a8dda1f21bf1a6a2bfb2196bf3e4d6f1161ff047cbd498ca790c90e86006629e972b
+  metadata.gz: 6ea65bb2381627119b5740f9353b064d046e26b2769e0be36baf3f13b9696e30e6bd6332fbbf8c4305fea33d1b2bb12d81a4a56871f3793c3718e2ac8407c6ae
+  data.tar.gz: 23ff60c01817d64bae76bb16c0395d8cc7aa95cd621550cebc5fc6763fcf5ba8a023542bf07b49067fa9210d8f228f538b0e82159d897397d08532deece73bc4

data/.gitignore

@@ -8,6 +8,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/sig/generated/
 .CLAUDE.local.md
 .rspec_status
 **/.claude/settings.local.json

data/.rubocop.yml

@@ -15,6 +15,7 @@ AllCops:
     - 'bin/*'
     - 'spec/fixtures/**/*'
     - 'test/fixtures/**/*'
+    - 'examples/**/*'
   SuggestExtensions: false
   TargetRubyVersion: 3.4
   UseCache: true

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    roast-ai (0.4.7)
+    roast-ai (0.4.9)
       activesupport (>= 7.0)
       cli-kit (~> 5.0)
       cli-ui (= 2.3.0)
@@ -167,7 +167,7 @@ GEM
       method_source (~> 1.0)
     public_suffix (6.0.2)
     racc (1.8.1)
-    rack (2.2.17)
+    rack (2.2.19)
     rainbow (3.1.1)
     raix (1.0.2)
       activesupport (>= 6.0)
@@ -185,7 +185,7 @@ GEM
     rbs (3.9.4)
       logger
     regexp_parser (2.10.0)
-    rexml (3.4.1)
+    rexml (3.4.2)
     rubocop (1.77.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)

data/README.md

@@ -374,20 +374,24 @@ Roast supports several types of steps:
     ```yaml
     steps:
       - analyze_code
-      - get_user_feedback:
+      - input:
+          name: get_user_feedback
           prompt: "Should we proceed with the refactoring? (yes/no)"
           type: confirm
-      - review_changes:
+      - input:
+          name: review_changes
           prompt: "Enter your review comments"
           type: text
-      - select_strategy:
+      - input:
+          name: select_strategy
           prompt: "Choose optimization strategy"
-          type: select
+          type: choice
           options:
             - "Performance optimization"
             - "Memory optimization"
             - "Code clarity"
-      - api_configuration:
+      - input:
+          name: api_configuration
           prompt: "Enter API key"
           type: password
     ```

data/Rakefile

@@ -31,3 +31,5 @@ RuboCop::RakeTask.new(:rubocop) do |task|
 end
 
 task default: [:test, :rubocop]
+
+task lint: [:rubocop]

data/dsl/less_simple.rb

@@ -0,0 +1,112 @@
+# typed: false
+# frozen_string_literal: true
+
+#### cmd
+
+# Passing just the command to execute will run it and return the output
+cmd <<~SHELLSTEP
+  echo "raw run without storing, should see me once""
+SHELLSTEP
+
+# Passing a name finds or creates an object and returns that
+cmd_cog = cmd(:hello)
+puts "This is our new cmd_cog named ':hello': #{cmd_cog}"
+
+# We can set a command to run for later
+cmd(:set_and_run).set("echo 'set_and_run, should see me once'")
+cmd(:set_and_run).run
+
+# Similarly, we can run immediately and then re-run later
+cmd(:run_and_rerun).run("echo 'run_and_rerun, should see me twice'")
+cmd(:run_and_rerun).run
+
+#### graph
+
+# We can open and re-open a graph, and then execute it
+graph(:updatable) do |graph|
+  graph.node(:open_cmd) do |state|
+    state[:open] = cmd("echo 'From a node added in first open, should see me once'")
+  end
+end
+
+graph(:updatable) do |graph|
+  graph.node(:reopen_cmd) do |state|
+    state[:reopen] = cmd("echo 'From a node added in reopen, should see me once'")
+  end
+end
+
+graph(:yea).execute
+
+# We can also just populate and execute a graph in one go by calling graph.execute in the block.
+graph(:define_and_exec) do |graph|
+  graph.node(:hi) do |state|
+    state[:hi_msg] = cmd("echo 'hi msg'")
+  end
+
+  graph.execute
+end
+
+# We can have subgraphs, because why not
+graph(:outer) do |graph|
+  graph.subgraph(:inner) do |subgraph|
+    subgraph.node(:inner_node) do |inner_state|
+      inner_state[:foo] = cmd("echo 'inner_state foo'")
+    end
+  end
+
+  graph.node(:outer) do |outer_state|
+    outer_state[:bar] = cmd("echo 'outer_state bar'")
+  end
+
+  graph.execute
+end
+
+# We can specify our own edges
+graph(:edges) do |graph|
+  graph.node(:thing1) do |state|
+    state[:thing1] = cmd("echo 'thing1'")
+  end
+
+  graph.node(:thing2) do |state|
+    state[:thing2] = cmd("echo 'thing2'")
+  end
+
+  graph.edge(from: :START, to: :thing1)
+  graph.edge(from: :thing1, to: :thing2)
+  graph.edge(from: :thing2, to: :DONE)
+
+  graph.execute
+end
+
+# We can have parallel execution
+graph(:parallel) do |graph|
+  graph.node(:thing1) do |state|
+    state[:thing1] = cmd("sleep 0.5 && echo 'parallel thing1'")
+  end
+
+  graph.node(:thing2) do |state|
+    state[:thing2] = cmd("sleep 0.5 && echo 'parallel thing2'")
+  end
+
+  graph.edge(from: :START, to: [:thing1, :thing2])
+  graph.edge(from: [:thing1, :thing2], to: :DONE)
+
+  graph.execute
+end
+
+# We can have edges that are defined with a block
+graph(:quantum) do |graph|
+  graph.node(:thing1) do |state|
+    state[:thing1] = cmd("echo 'quantum thing1'")
+  end
+
+  graph.edge(from: :START) do |_state|
+    :thing1
+  end
+
+  graph.edge(from: :thing1) do |_state|
+    :DONE
+  end
+
+  graph.execute
+end

data/dsl/prototype.rb

@@ -0,0 +1,17 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Executor
+
+config do
+  cmd(:echo) { print_all! }
+end
+
+execute do
+  # Anonymous cog. Added to the stack directly and given an autogenerated key in cog storage
+  # Use for actions you do once and forget about, don't need configuration
+  cmd { "touch tmp/#{SecureRandom.uuid} " }
+
+  # Named cog. Configuration for this specific instance will be looked up from config block
+  cmd(:echo) { "echo 'Hello World!'" }
+end

data/dsl/simple.rb

@@ -1,10 +1,8 @@
-# typed: true
+# typed: false
 # frozen_string_literal: true
 
-#: self as Roast::DSL::Executor
-
-# This is a dead simple workflow that calls two shell scripts
-shell <<~SHELLSTEP
+# This is a dead simple workflow that calls two commands
+cmd <<~CMDSTEP
   echo "I have no idea what's going on"
-SHELLSTEP
-shell "pwd"
+CMDSTEP
+cmd "pwd"

data/dsl/step_communication.rb

@@ -0,0 +1,18 @@
+# typed: false
+# frozen_string_literal: true
+
+# How do we pass information between steps?
+# Demonstrate by passing result of a command output to another step
+
+config do
+  cmd(:echo) { display! }
+end
+
+execute do
+  cmd(:ls) { "ls -al" }
+  cmd(:echo) do
+    # TODO: this is a bespoke output object for cmd, is there a generic one we can offer
+    first_line = cmd(:ls).command_output.split("\n").second
+    "echo '#{first_line}'"
+  end
+end

data/examples/README.md

@@ -0,0 +1,9 @@
+These examples demonstrate various usages and features of Roast. They are automatically tested and verified by functional tests (see `test/functional/roast_examples_test.rb`).
+
+### available_tools_demo
+
+Demonstrates use of Roast tool implementations in LLM steps. Steps are implemented with `prompt.md` in directories that match the step name
+
+### basic_prompt_workflow
+
+Demonstrates using LLM steps to glean insights from a provided data file (`test/fixtures/sample_data/skateboard_orders.csv`).

data/examples/available_tools_demo/workflow.yml

@@ -1,4 +1,4 @@
-model: openai:gpt-4o-mini
+model: gpt-4o-mini
 
 tools:
   - Roast::Tools::Grep

data/examples/basic_prompt_workflow/workflow.md

@@ -0,0 +1 @@
+You are a skateboard shop owner looking for insights on their business.

data/examples/basic_prompt_workflow/workflow.yml

@@ -0,0 +1,14 @@
+name: Analyze My Business
+description: Examines a CSV list of orders for insights
+
+# Default model for all steps
+model: google:gemini-2.5-flash
+tools:
+  - Roast::Tools::ReadFile
+  - Roast::Tools::Grep
+
+steps:
+  - Read the provided CSV files.
+  - "Where are most of my customers from?"
+  - "What is my most popular product?"
+  - Summarize the insights from the report.

data/examples/grading/README.md

@@ -2,30 +2,6 @@
 
 This workflow acts as a senior software engineer and testing expert to evaluate the quality of test files based on best practices and guidelines.
 
-## Prerequisites
-
-This example uses `shadowenv` for environment management, which is specific to Shopify's development environment. If you're not using shadowenv, you'll need to adapt the commands to your own setup.
-
-### If you're using shadowenv:
-```bash
-brew install shadowenv
-```
-
-### If you're NOT using shadowenv:
-You'll need to modify the `run_coverage.rb` file to remove the shadowenv commands. Look for lines like:
-```ruby
-command = "shadowenv exec -- bundle exec ruby ..."
-```
-
-And change them to match your environment:
-```ruby
-# For standard Ruby/Bundler setup:
-command = "bundle exec ruby ..."
-
-# Or if you're using rbenv/rvm:
-command = "ruby ..."
-```
-
 ## Usage
 
 ```bash
@@ -52,7 +28,6 @@ Feel free to adapt this workflow to your testing environment:
 - **Different test frameworks**: Modify `run_coverage.rb` to work with RSpec, Jest, pytest, etc.
 - **Coverage tools**: Replace the coverage command with your preferred tool (SimpleCov, Istanbul, Coverage.py)
 - **Grading criteria**: Adjust the prompts in each step to match your team's standards
-- **Environment setup**: Remove or replace shadowenv with your environment management tool
 
 ## Example Output
 
@@ -68,4 +43,4 @@ RECOMMENDATIONS:
 - Add edge case testing for error conditions
 - Improve test descriptions for clarity
 - Consider extracting common setup to helper methods
-```
+```

data/examples/grading/analyze_coverage/prompt.md

@@ -49,4 +49,4 @@ You must respond in JSON format within <json> XML tags. Example:
     "justification": "The source file has 80% branch coverage, indicating some branches need testing."
   }
 }
-</json>
+</json>

data/examples/grading/calculate_final_grade.rb

@@ -1,21 +1,17 @@
 # frozen_string_literal: true
 
 class CalculateFinalGrade < Roast::Workflow::BaseStep
-  attr_accessor :llm_analysis
-
   WEIGHTS = {
-    line_coverage: 0.1,
-    method_coverage: 0.1,
-    branch_coverage: 0.3,
-    test_helpers: 0.1,
-    mocks_and_stubs: 0.1,
-    readability: 0.1,
-    maintainability: 0.1,
-    effectiveness: 0.1,
+    test_helpers: 0.2,
+    mocks_and_stubs: 0.2,
+    readability: 0.2,
+    maintainability: 0.2,
+    effectiveness: 0.2,
   }.freeze
 
   def call
-    @llm_analysis = workflow.output["generate_grades"].merge(workflow.output["analyze_coverage"])
+    llm_analysis = workflow.output["generate_grades"]
+
     weighted_sum = WEIGHTS.sum do |criterion, weight|
       score = llm_analysis[criterion.to_s]["score"].to_f / 10.0
       score * weight
@@ -26,7 +22,7 @@ class CalculateFinalGrade < Roast::Workflow::BaseStep
         weighted_score: weighted_sum,
         letter_grade: calculate_letter_grade(weighted_sum),
       },
-      rubric_scores: calculate_rubric_scores,
+      rubric_scores: calculate_rubric_scores(llm_analysis),
     }
   end
 
@@ -47,10 +43,11 @@ class CalculateFinalGrade < Roast::Workflow::BaseStep
     end
   end
 
-  def calculate_rubric_scores
+  def calculate_rubric_scores(llm_analysis)
     scores = {}
 
     WEIGHTS.each_key do |criterion|
+      next 1 unless llm_analysis[criterion.to_s]
       raw_score = llm_analysis[criterion.to_s]["score"].to_f
       normalized_score = raw_score / 10.0
 

data/examples/grading/format_result.rb

@@ -2,14 +2,11 @@
 
 class FormatResult < Roast::Workflow::BaseStep
   RUBRIC = {
-    line_coverage: { description: "Line Coverage", weight: 0.1 },
-    method_coverage: { description: "Method Coverage", weight: 0.1 },
-    branch_coverage: { description: "Branch Coverage", weight: 0.3 },
-    test_helpers: { description: "Test Helpers Usage", weight: 0.1 },
-    mocks_and_stubs: { description: "Mocks and Stubs Usage", weight: 0.1 },
-    readability: { description: "Test Readability", weight: 0.1 },
-    maintainability: { description: "Test Maintainability", weight: 0.1 },
-    effectiveness: { description: "Test Effectiveness", weight: 0.1 },
+    test_helpers: { description: "Test Helpers Usage", weight: 0.2 },
+    mocks_and_stubs: { description: "Mocks and Stubs Usage", weight: 0.2 },
+    readability: { description: "Test Readability", weight: 0.2 },
+    maintainability: { description: "Test Maintainability", weight: 0.2 },
+    effectiveness: { description: "Test Effectiveness", weight: 0.2 },
   }.freeze
 
   def call

data/examples/grading/generate_grades/prompt.md

@@ -102,4 +102,4 @@ RESPONSE FORMAT: You must respond in JSON format within <json> XML tags.
     "justification": "Tests validate most expected behaviors and would catch common bugs."
   }
 }
-</json>
+</json>

data/examples/grading/generate_recommendations/prompt.md

@@ -57,4 +57,4 @@ Example:
     }
   ]
 }
-</json>
+</json>

data/examples/grading/read_dependencies/prompt.md

@@ -13,4 +13,3 @@ If you are told to find the dependencies of `test/services/country_db_interface_
 then you would use the functions as explained above and ultimately respond with `<sut>app/services/country_db_interface.rb</sut>`
 
 If the file is found at `lib/roast/workflow/workflow_initializer.rb`, respond with `<sut>lib/roast/workflow/workflow_initializer.rb</sut>` (include the lib/ prefix)
-

data/examples/grading/verify_test_helpers/prompt.md

@@ -50,4 +50,4 @@ Once you are done understanding the custom test helpers used in the test file, a
 
 Where possible, use your best judgment to make recommendations for how to fix problems that you find, but ONLY related to test helpers.
 
-Note: You are only being used to help find problems so it is not necessary to report on correct usage of helpers or to make positive comments.
+Note: You are only being used to help find problems so it is not necessary to report on correct usage of helpers or to make positive comments.

data/examples/grading/workflow.md

@@ -2,7 +2,4 @@ As a senior software engineer and testing expert, evaluate the quality of this t
 
 Next I will now provide the source code of the test that we will be analyzing, and then step you through a series of analysis activities, before finally asking you to provided a final report.
 
-<test>
-# <%= file %>
-<%= File.read(file) %>
-</test>
+# <%= file %> <%= File.read(file) %>

data/examples/grading/workflow.yml

@@ -1,21 +1,13 @@
 name: Test Grading
-api_token: $(echo $OPENAI_API_KEY)
-# model: anthropic:claude-opus-4
-model: gpt-4.1-mini
 
 tools:
   - Roast::Tools::Grep
   - Roast::Tools::ReadFile
   - Roast::Tools::SearchFile
 
-# Uncomment this to run the workflow on modified tests automatically
-# each: '% cd $(git rev-parse --show-toplevel) && git status --porcelain | grep "_test\.rb" | cut -c4- | xargs realpath'
-
 steps:
   - read_dependencies
-  - run_coverage
   -
-    - analyze_coverage
     - verify_test_helpers
     - verify_mocks_and_stubs
   - generate_grades
@@ -23,19 +15,14 @@ steps:
   - format_result
   - generate_recommendations
 
-# set non-default attributes for steps below
 analyze_coverage:
-#  model: gpt-4.1-mini
   json: true
-  
+
 generate_grades:
-#  model: o3
+  model: o3
   json: true
 
 generate_recommendations:
-#  model: o3
+  model: o3
   json: true
-  params:
-    max_completion_tokens: 5_000
-
 

data/lib/roast/dsl/cog/config.rb

@@ -0,0 +1,31 @@
+# typed: false
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    class Cog
+      class Config
+        attr_reader :values
+
+        def initialize(initial = {})
+          @values = initial
+        end
+
+        def merge(config_object)
+          self.class.new(values.merge(config_object.values))
+        end
+
+        # It is recommended to implement a custom config object for a nicer interface,
+        # but for simple cases where it would just be a key value store we provide one by default.
+
+        def []=(key, value)
+          @values[key] = value
+        end
+
+        def [](key)
+          @values[key]
+        end
+      end
+    end
+  end
+end

data/lib/roast/dsl/cog/stack.rb

@@ -0,0 +1,21 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    class Cog
+      class Stack
+        delegate :map, :push, :size, :empty?, to: :@queue
+
+        def initialize
+          @queue = []
+        end
+
+        #: () -> Roast::DSL::Cog?
+        def pop
+          @queue.shift
+        end
+      end
+    end
+  end
+end

data/lib/roast/dsl/cog/store.rb

@@ -0,0 +1,26 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    class Cog
+      class Store
+        class CogAlreadyDefinedError < Roast::Error; end
+
+        delegate :[], to: :store
+
+        #: (Symbol, Roast::DSL::Cog) -> Roast::DSL::Cog
+        def insert(id, inst)
+          raise CogAlreadyDefinedError if store.key?(id)
+
+          store[id] = inst
+        end
+
+        #: () -> Hash[Symbol, Roast::DSL::Cog]
+        def store
+          @store ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/roast/dsl/cog.rb

@@ -0,0 +1,70 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    class Cog
+      class CogAlreadyRanError < StandardError; end
+
+      class << self
+        def on_create
+          eigen = self
+          proc do |instance_name = Random.uuid, &action|
+            #: self as Roast::DSL::WorkflowExecutionContext
+            add_cog_instance(instance_name, eigen.new(instance_name, action))
+          end
+        end
+
+        def on_config
+          eigen = self
+          proc do |cog_name = nil, &configuration|
+            #: self as Roast::DSL::ConfigContext
+            config_object = if cog_name.nil?
+              fetch_execution_scope(eigen)
+            else
+              fetch_or_create_cog_config(eigen, cog_name)
+            end
+
+            config_object.instance_exec(&configuration) if configuration
+          end
+        end
+
+        def config_class
+          @config_class ||= find_child_config_or_default
+        end
+
+        private
+
+        def find_child_config_or_default
+          config_constant = "#{name}::Config"
+          const_defined?(config_constant) ? const_get(config_constant) : Cog::Config # rubocop:disable Sorbet/ConstantsFromStrings
+        end
+      end
+
+      attr_reader :name, :output
+
+      def initialize(name, cog_input_proc)
+        @name = name
+        @cog_input_proc = cog_input_proc
+        @finished = false
+      end
+
+      def run!(config, cog_execution_context)
+        raise CogAlreadyRanError if ran?
+
+        @config = config
+        @output = execute(cog_execution_context.instance_exec(&@cog_input_proc))
+        @finished = true
+      end
+
+      def ran?
+        @finished
+      end
+
+      # Inheriting cog must implement this
+      def execute(input)
+        raise NotImplementedError
+      end
+    end
+  end
+end