roast-ai 0.3.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f94565fdc94547f9ab2fa57b401930d189c2e63f5f31897165a07cf6828b10d
-  data.tar.gz: '068450417abc06ed7fb98af7c36ec972d6671b23e0e85cfdadde8a934c327fdc'
+  metadata.gz: b31e95a8d65d0b1fa798c0baed8bcbf9249cc67fca752154865d914d01ffbd8d
+  data.tar.gz: 21205a0a50e2301dda0936e67b02d05af94085b0e383f859aec282827649ebb3
 SHA512:
-  metadata.gz: 07e5a581c469a37d3f0371627c92ef0b465cfeb5e977a90221a3cdb7783193de7a1c6600df3b1235705ef65e0e76f566bf88730621172314d3bc0ba70adc5dc4
-  data.tar.gz: 12e7ae70eee847f11eac8c4e593e11b5a94c0574634254a559d41d006e8fac01dc7b7c26fa2ba1a80fc13de0172ac810ff390e8bda2555e96489c0385f815537
+  metadata.gz: 80b5ceada69faff53f5893e9378e628474b103c3d6b60a8701be80b1bfe11cb6e4bde252670d890a2ae7e460d90d956daed7813d251cf6d84399ab70d321b0d8
+  data.tar.gz: 620e7d46fc6f7e9b10a71807f213ba101cb2bbd9a3bad073a153057bd67bc842cf74e4ffaa179596a2ed3655dc7a864c366034a37f4a720d2fd6faafa86f5541

data/.github/workflows/ci.yaml

@@ -23,8 +23,8 @@ jobs:
       BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
     steps:
       - uses: actions/checkout@v4
-      - name: Install ripgrep
-        run: sudo apt-get update && sudo apt-get install -y ripgrep
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y ripgrep graphviz
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}

data/.gitignore

@@ -40,3 +40,4 @@ bin/ruby-rewrite
 bin/thor
 
 gemfiles/*.lock
+bin/claude-swarm

data/CHANGELOG.md

@@ -5,6 +5,91 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.1] - 2025-06-18
+
+### Added
+- **SQLite session storage** as the default storage backend (#252)
+  - Provides better performance and advanced querying capabilities
+  - Sessions are stored in `~/.roast/sessions.db` by default (configurable via `ROAST_SESSIONS_DB`)
+  - New `roast sessions` command to list and filter stored sessions
+  - New `roast session <id>` command to view detailed session information
+  - Session cleanup with `roast sessions --cleanup --older-than <duration>`
+  - Filter sessions by status, workflow name, or age
+  - Maintains full backward compatibility with filesystem storage
+- **`--file-storage` CLI option** to use legacy filesystem storage instead of SQLite
+  - Use `-f` or `--file-storage` flag to opt into filesystem storage
+  - Environment variable `ROAST_STATE_STORAGE=file` still supported for compatibility
+- **Foundation for wait_for_event feature** (#251)
+  - New `roast resume` command infrastructure for resuming paused workflows
+  - Event storage and tracking in SQLite sessions table
+- **Configurable agent step options** for CodingAgent (#266)
+  - New `continue` option for agent steps to maintain session context across multiple agent invocations
+  - New `include_context_summary` option to provide AI-generated workflow context summaries to agents
+  - Context summaries are intelligently tailored to the agent's specific task using LLM analysis
+  - Helps reduce token usage by including only relevant context information
+- **Token consumption reporting** for step execution (#264)
+  - Displays token usage (prompt and completion) after each step execution
+  - Helps users monitor and optimize their LLM token consumption
+  - Automatically enabled for all workflow runs
+- **Timeout functionality for bash and cmd steps** (#261)
+  - New `timeout` option for bash and cmd steps to prevent hanging commands
+  - Configurable timeout duration in seconds
+  - Commands are automatically terminated if they exceed the specified timeout
+  - Prevents workflows from getting stuck on unresponsive commands
+- **Claude Swarm tool integration** (#254)
+  - New `Roast::Tools::Swarm` for integrating with Claude Swarm framework
+  - Enables using Swarm's multi-agent orchestration capabilities within Roast workflows
+  - Provides seamless handoffs between specialized AI agents
+- **Workflow visualization with diagram command** (#256)
+  - New `roast diagram` command to generate visual representations of workflows
+  - Creates GraphViz-based diagrams showing workflow structure and flow
+  - Supports both DOT format output and PNG/SVG image generation
+  - Helps understand complex workflow logic at a glance
+- **Comprehensive workflow validation** (#244)
+  - New `roast validate` command to check workflow syntax and structure
+  - Validates YAML syntax, step references, and configuration options
+  - Provides detailed error messages for invalid workflows
+  - Helps catch errors before running workflows
+- **apply_diff tool** (#246)
+  - New built-in tool for applying unified diff patches to files
+  - Supports standard diff format for making precise file modifications
+  - Enables AI models to suggest changes in diff format
+  - More reliable than search-and-replace for complex edits
+- **Model fallback mechanism** (#257)
+  - Workflows without explicit model configuration now use a sensible default
+  - Prevents errors when model is not specified at workflow or step level
+  - Improves user experience for simple workflows
+- **Context management foundation for auto-compaction** (#264)
+  - Infrastructure for future automatic context size management
+  - Enables intelligent token usage optimization in long-running workflows
+
+### Changed
+- Session storage now defaults to SQLite instead of filesystem
+  - Existing filesystem sessions remain accessible when using `--file-storage` flag
+  - No migration required - both storage backends can coexist
+
+[0.4.1]: https://github.com/Shopify/roast/compare/v0.4.0...v0.4.1
+
+## [0.4.0] - 2025-06-12
+
+### Added
+- **Input step type** for collecting user input during workflow execution (#154)
+  - Interactive prompts pause workflow execution to collect user input
+  - Supports multiple input types: `text` (default), `confirm`, `select`, and `password`
+  - `confirm` type provides yes/no prompts with boolean results
+  - `select` type allows choosing from a list of options
+  - `password` type masks input for sensitive data using io/console
+  - Input values are stored in workflow output and accessible via dot notation (e.g., `{{output.step_name}}`)
+  - Integrates with CLI::UI for consistent formatting and user experience
+- **Agent step type** for direct pass-through to coding agents (#151)
+  - Steps prefixed with `^` send prompts directly to the CodingAgent tool
+  - Supports both file-based and inline agent prompts
+  - Bypasses LLM interpretation for precise agent instructions
+
+### Fixed
+- DotAccessHash array wrapping and template response handling
+- CLI::UI formatting and color handling for better terminal output
+
 ## [0.3.1] - 2025-06-05
 
 ### Added

data/CLAUDE.md

@@ -4,17 +4,15 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 
 ## About the codebase
 - This is a Ruby gem called Roast. Its purpose is to run AI workflows defined in a YAML file.
+- Note that this project now uses Zeitwerk, which means you don't have to manually require project files anymore
 
 ## Commands
-
-- Default (tests + lint): `bundle exec rake`
-- Test all: `bundle exec test`
+- Default THE SUITE RUNS FAST SO USE THIS  IN MOST CASES (tests + lint w/autocorrect): `bundle exec rake`
 - Run single test: `bundle exec ruby -Itest test/path/to/test_file.rb`
 - Lint: `bundle exec rubocop`
 - Lint (with autocorrect, preferred): `bundle exec rubocop -A`
 - Whenever you want to run the whole test suite just run `bundle exec rake` to also run linting, and note the linting errors too (most will auto correct but not all)
 - **Run roast locally**: Use `bin/roast` (not `bundle exec roast` which may use the installed gem)
-- Alternative: `bundle exec exe/roast`
 
 ## Tech stack
 - `thor` and `cli-ui` for the CLI tool
@@ -22,7 +20,6 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - Prefer using the more literate `test "this is a test description" do` type of testing that we get from extending ActiveSupport::TestCase over the primitive XUnit-style def test_description headings for tests
 
 ## Code Style Guidelines
-
 - Naming: snake_case for variables/methods, CamelCase for classes/modules, ALL_CAPS for constants
 - Module structure: Use nested modules under the `Roast` namespace
 - Command pattern: Commands implement a `call` method and class-level `help` method
@@ -40,7 +37,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - Always leave a blank line after module includes and before the rest of the class
 
 ## Architecture Guidelines
-
+- **SOLID principles are important** - don't violate them
 - **Maintain proper separation of concerns**: Don't mix unrelated concepts in the same class or module
   - Example: Conditional execution (if/unless) should NOT be mixed with iteration execution (each/repeat)
   - Each concept should have its own executor class and be handled separately
@@ -51,14 +48,67 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
   - ConditionalExecutor handles conditionals (if, unless)
   - Don't combine different responsibilities in one class
 - **Do not implement prompts "inline" using a prompt: attribute nested under step names, that violates the primary design architecture of Roast**
+- When faced with the choice between working around an architectural issue or code smell versus actually diving into fixing the design issue or code smell, choose the latter more principled approach
+- When fixing code smells, you don't have to worry about internal backwards compatibility 
+
+## Workflow Configuration Syntax
+- The `steps` key in the workflow configuration is an array of step names
+- Only step names, inline prompts, and control flow keywords are allowed in the steps array
+- Additional per-step configuration is provided in a top-level hash with the step name as the key, not within steps!!! (Very important)
+- The reason that steps are not configured "inline" within the steps array is so that the shape of the workflow is as obvious as possible
+- Step labels are inferred for most steps and optional for inline prompts, but required for steps that need custom configuration
+- The result of running a step is stored in the `workflow.output` hash with the step label as the key
+
+## How Roast Tools Work (CRITICAL - READ THIS!)
+**Tools in Roast are NOT explicitly invoked in workflow steps!** This is a fundamental concept that differs from many other workflow systems.
+
+### Key Concepts:
+1. **Tools are capabilities available to the LLM** - They are functions the LLM can choose to call while executing a step
+2. **Steps contain prompts** - Steps describe what needs to be done, not how to do it
+3. **The LLM decides when to use tools** - While executing a step's prompt, the LLM analyzes the task and calls tools as needed
+4. **Tools are registered, not declared in steps** - Use the `tools:` section to make tools available, but never use a `tool:` key in step configuration
+
+### Correct inline prompt syntax:
+```yaml
+steps:
+  - analyze_code: |
+      Analyze the codebase and identify performance bottlenecks.
+      Use any available tools to read files and search for patterns.
+```
+
+### INCORRECT syntax (DO NOT USE):
+```yaml
+# WRONG - no 'prompt:' key
+steps:
+  - analyze_code:
+      prompt: "Analyze the codebase"
+      
+# WRONG - no 'tool:' key
+steps:
+  - run_analysis:
+      tool: coding_agent
+      prompt: "Analyze code"
+```
 
-## Guidance and Expectations
+### How tools are actually used:
+When the LLM executes the `analyze_code` step above, it might:
+1. Decide it needs to read files and call `read_file(path)`
+2. Decide it needs to search and call `grep(pattern, path)`
+3. Decide it needs Claude Swarm and call `swarm(prompt, config_path)`
 
+The LLM makes these decisions based on the prompt and available tools, similar to how Claude (you) decides when to use Bash, Read, or other tools when responding to user requests.
+
+## Step Configuration
+- The `path` key in a step configuration is the path to a Ruby file that defines a custom step.
+- The `model` key in a step configuration is the model to use for the step.
+- The `print_response` key in a step configuration is a boolean that determines whether the step's response should be printed to the console.
+
+## Coding Guidance and Expectations
 - Do not decide unilaterally to leave code for the sake of "backwards compatibility"... always run those decisions by me first.
 - Don't ever commit and push changes unless directly told to do so
+- You can't test input steps yourself since they block, so ask me to do it manually
 
 ## Git Workflow Practices
-
 1. **Amending Commits**:
    - Use `git commit --amend --no-edit` to add staged changes to the last commit without changing the commit message
    - This is useful for incorporating small fixes or changes that belong with the previous commit
@@ -130,6 +180,48 @@ gh pr view {pr_number}
 gh pr diff {pr_number}
 ```
 
+### Issue Labeling and Project Management
+
+When creating GitHub issues, always check available labels, projects, and milestones first:
+
+```bash
+# List all available labels
+gh api repos/Shopify/roast/labels | jq '.[].name'
+
+# List all milestones
+gh api repos/Shopify/roast/milestones | jq '.[] | {title: .title, number: .number, state: .state}'
+
+# List projects linked to the roast repository
+gh api graphql -f query='
+{
+  repository(owner: "Shopify", name: "roast") {
+    projectsV2(first: 10) {
+      nodes {
+        title
+        number
+        url
+      }
+    }
+  }
+}' --jq '.data.repository.projectsV2.nodes[] | {title: .title, number: .number, url: .url}'
+```
+
+**Issue Creation Workflow**:
+1. First check what labels exist and apply appropriate ones when creating the issue
+2. After creating the issue, ask the user if they want it added to an existing milestone
+3. Ask the user if they want it added to a particular project board
+
+```bash
+# Create issue with labels
+gh api repos/Shopify/roast/issues -X POST -F title="Issue Title" -F body="Issue description" -F 'labels=["bug", "enhancement"]'
+
+# Add issue to a milestone (after creation)
+gh api repos/Shopify/roast/issues/{issue_number} -X PATCH -F milestone={milestone_number}
+
+# Add issue to a GitHub Project
+gh project item-add {project_number} --url https://github.com/Shopify/roast/issues/{issue_number}
+```
+
 #### Formatting Tips for GitHub API
 1. Use literal newlines in the body text instead of `\n` escape sequences
 2. When formatting is stripped (like backticks), use alternatives:
@@ -167,4 +259,9 @@ gh pr diff {pr_number}
     - Avoid premature optimization outside of hot paths
     - Consider the tradeoff between readability and performance
     - Suggest optimizations that improve both clarity and performance
-```
+
+## CLI::UI Formatting Tips
+- To apply color to terminal output using CLI::UI, use the following syntax:
+  ```ruby
+  puts ::CLI::UI.fmt("{{red:This field is required. Please provide a value.}}")
+  ```

data/Gemfile

@@ -8,7 +8,7 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 gemspec
 
 gem "cgi"
-gem "cli-ui"
+gem "cli-ui", "2.3.0"
 gem "dotenv"
 gem "guard"
 gem "guard-minitest"
@@ -18,3 +18,6 @@ gem "rubocop-shopify", require: false
 gem "vcr", require: false
 gem "webmock", require: false
 gem "minitest-rg"
+gem "sqlite3", "~> 1.7"
+
+gem "claude_swarm"

data/Gemfile.lock

@@ -1,15 +1,17 @@
 PATH
   remote: .
   specs:
-    roast-ai (0.3.1)
+    roast-ai (0.4.1)
       activesupport (>= 7.0)
-      cli-ui
+      cli-ui (= 2.3.0)
       diff-lcs (~> 1.5)
       faraday-retry
       json-schema
       open_router (~> 0.3)
       raix (~> 1.0)
+      ruby-graphviz (~> 1.2)
       thor (~> 1.3)
+      zeitwerk (~> 2.6)
 
 GEM
   remote: https://rubygems.org/
@@ -33,17 +35,49 @@ GEM
     base64 (0.3.0)
     benchmark (0.4.1)
     bigdecimal (3.2.2)
-    cgi (0.4.2)
-    cli-ui (2.3.1)
+    cgi (0.5.0)
+    claude_swarm (0.1.15)
+      fast-mcp-annotations
+      thor (~> 1.3)
+    cli-ui (2.3.0)
     coderay (1.1.3)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
     crack (1.0.0)
       bigdecimal
       rexml
-    diff-lcs (1.6.1)
+    diff-lcs (1.6.2)
     dotenv (3.1.8)
     drb (2.2.3)
+    dry-configurable (1.3.0)
+      dry-core (~> 1.1)
+      zeitwerk (~> 2.6)
+    dry-core (1.1.0)
+      concurrent-ruby (~> 1.0)
+      logger
+      zeitwerk (~> 2.6)
+    dry-inflector (1.2.0)
+    dry-initializer (3.2.0)
+    dry-logic (1.6.0)
+      bigdecimal
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.1)
+      zeitwerk (~> 2.6)
+    dry-schema (1.14.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 1.0, >= 1.0.1)
+      dry-core (~> 1.1)
+      dry-initializer (~> 3.2)
+      dry-logic (~> 1.5)
+      dry-types (~> 1.8)
+      zeitwerk (~> 2.6)
+    dry-types (1.8.3)
+      bigdecimal (~> 3.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0)
+      dry-inflector (~> 1.0)
+      dry-logic (~> 1.4)
+      zeitwerk (~> 2.6)
     event_stream_parser (1.0.0)
     faraday (2.13.1)
       faraday-net_http (>= 2.0, < 3.5)
@@ -55,6 +89,13 @@ GEM
       net-http (>= 0.5.0)
     faraday-retry (2.3.1)
       faraday (~> 2.0)
+    fast-mcp-annotations (1.5.2)
+      addressable (~> 2.8)
+      base64
+      dry-schema (~> 1.14)
+      json (~> 2.0)
+      mime-types (~> 3.4)
+      rack (~> 3.1)
     ffi (1.17.2-arm64-darwin)
     ffi (1.17.2-x86_64-linux-gnu)
     formatador (1.1.0)
@@ -73,14 +114,14 @@ GEM
     guard-minitest (2.4.6)
       guard-compat (~> 1.2)
       minitest (>= 3.0)
-    hashdiff (1.1.2)
+    hashdiff (1.2.0)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     json (2.12.2)
     json-schema (5.1.1)
       addressable (~> 2.8)
       bigdecimal (~> 3.1)
-    language_server-protocol (3.17.0.4)
+    language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
@@ -88,6 +129,11 @@ GEM
     logger (1.7.0)
     lumberjack (1.2.10)
     method_source (1.1.0)
+    mime-types (3.7.0)
+      logger
+      mime-types-data (~> 3.2025, >= 3.2025.0507)
+    mime-types-data (3.2025.0603)
+    mini_portile2 (2.8.9)
     minitest (5.25.5)
     minitest-rg (5.3.0)
       minitest (~> 5.0)
@@ -114,22 +160,23 @@ GEM
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (6.0.1)
+    public_suffix (6.0.2)
     racc (1.8.1)
+    rack (3.1.16)
     rainbow (3.1.1)
-    raix (1.0.0)
+    raix (1.0.1)
       activesupport (>= 6.0)
       faraday-retry (~> 2.0)
       open_router (~> 0.2)
       ostruct
       ruby-openai (~> 7)
-    rake (13.2.1)
+    rake (13.3.0)
     rb-fsevent (0.11.2)
     rb-inotify (0.11.1)
       ffi (~> 1.0)
     regexp_parser (2.10.0)
     rexml (3.4.1)
-    rubocop (1.75.3)
+    rubocop (1.76.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -137,14 +184,16 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.44.0, < 2.0)
+      rubocop-ast (>= 1.45.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.44.1)
+    rubocop-ast (1.45.0)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
-    rubocop-shopify (2.17.0)
+    rubocop-shopify (2.17.1)
       rubocop (~> 1.62)
+    ruby-graphviz (1.2.5)
+      rexml
     ruby-openai (7.4.0)
       event_stream_parser (>= 0.3.0, < 2.0.0)
       faraday (>= 1)
@@ -153,6 +202,8 @@ GEM
     ruby2_keywords (0.0.5)
     securerandom (0.4.1)
     shellany (0.0.1)
+    sqlite3 (1.7.3)
+      mini_portile2 (~> 2.8.0)
     thor (1.3.2)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
@@ -166,14 +217,16 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    zeitwerk (2.7.3)
 
 PLATFORMS
-  arm64-darwin-23
+  arm64-darwin
   x86_64-linux
 
 DEPENDENCIES
   cgi
-  cli-ui
+  claude_swarm
+  cli-ui (= 2.3.0)
   dotenv
   guard
   guard-minitest
@@ -182,6 +235,7 @@ DEPENDENCIES
   rake
   roast-ai!
   rubocop-shopify
+  sqlite3 (~> 1.7)
   vcr
   webmock