roast-ai 0.4.9 → 0.5.0

data/Rakefile

@@ -4,6 +4,8 @@ require "bundler/gem_tasks"
 require "rubocop/rake_task"
 require "rake/testtask"
 
+### Test Tasks
+
 Rake::TestTask.new(:minitest_all) do |t|
   t.libs << "test"
   t.libs << "lib"
@@ -19,10 +21,32 @@ end
 Rake::TestTask.new(:minitest_old) do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList["test/functional/**/*_test.rb"]
+  t.test_files = FileList["test/**/*_test.rb"].exclude(
+    "test/functional/**/*_test.rb",
+    "test/dsl/**/*_test.rb",
+    "test/roast/dsl/**/*_test.rb",
+  )
+end
+
+Rake::TestTask.new(:minitest_dsl_fast) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/dsl/**/*_test.rb", "test/roast/dsl/**/*_test.rb"]
+end
+
+Rake::TestTask.new(:minitest_dsl_slow) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/dsl/**/*_test.rb", "test/roast/dsl/**/*_test.rb"]
 end
+task :set_slow_env do
+  ENV["ROAST_RUN_SLOW_TESTS"] = "true"
+end
+task minitest_dsl_slow: :set_slow_env
+
+task test: [:minitest_dsl_fast, :minitest_dsl_slow, :minitest_functional, :minitest_old]
 
-task test: [:minitest_all]
+### Rubocop Tasks
 
 RuboCop::RakeTask.new(:rubocop_ci)
 
@@ -30,6 +54,17 @@ RuboCop::RakeTask.new(:rubocop) do |task|
   task.options = ["--autocorrect"]
 end
 
-task default: [:test, :rubocop]
+### Sorbet Tasks
+
+desc "Run Sorbet type checker"
+task :sorbet do
+  sh "bin/srb tc" do |ok, _|
+    abort "Sorbet type checking failed" unless ok
+  end
+end
+
+### Task Groups
+
+task default: [:sorbet, :rubocop, :minitest_dsl_fast]
 
-task lint: [:rubocop]
+task check: [:sorbet, :rubocop]

data/dev.yml

@@ -0,0 +1,29 @@
+name: roast
+
+nix: true
+
+type: ruby
+
+up:
+  - ruby
+  - bundler
+
+check:
+  style: bundle exec rake rubocop
+  tc: bundle exec rake sorbet
+  test: bundle exec rake minitest_dsl_fast
+
+commands:
+  __default__: execute
+  execute:
+    desc: Execute a roast workflow
+    run: bundle exec roast execute --executor=dsl
+  style:
+    desc: Run Rubocop style checks (and autocorrect)
+    run: bundle exec rake rubocop:autocorrect
+  tc:
+    desc: Run Sorbet type checks
+    run: bundle exec rake sorbet
+  test:
+    desc: Run all tests
+    run: bundle exec rake minitest_dsl_fast

data/dsl/agent_sessions.rb

@@ -0,0 +1,20 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  agent do
+    provider :claude
+    model "haiku"
+    dump_raw_agent_messages_to "tmp/claude-messages.log"
+  end
+end
+
+execute do
+  agent(:one) { "The magic word is 'pomegranate'" }
+  agent(:two) do |my|
+    my.session = agent!(:one).session
+    "What is the magic word?"
+  end
+end

data/dsl/async_cogs.rb

@@ -0,0 +1,49 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd { display! }
+
+  # Any type of cog can be configured to run asynchronously in the background.
+  # The cog that follows it in the workflow will be able to begin running immediately.
+  cmd(/slow/) { async! }
+end
+
+execute do
+  # 'first' is synchronous; nothing else runs until it completes.
+  cmd(:first) { "echo first" }
+
+  # These cogs are async and slow, so we kick them off in the background (via the 'async!' config option)
+  # Other cogs can continue while these cog runs in the background
+  cmd(:slow_background_task_1) do
+    sleep 0.1
+    "echo slow background task 1"
+  end
+  cmd(:slow_background_task_2) do
+    sleep 0.2
+    "echo slow background task 2"
+  end
+
+  cmd(:second) { "echo second" }
+  cmd(:third) { "echo third" }
+
+  # 'fourth' accesses the output of 'slow_background_task_1'.
+  # It will block until 'slow_background_task_1' completes.
+  # 'fourth' is synchronous, so cogs that follow it will also be forced to wait (whether they are async or not)
+  cmd(:fourth) do
+    "echo \"fourth <-- '#{cmd!(:slow_background_task_1).text}'\""
+  end
+
+  cmd(:fifth) { "echo fifth" }
+
+  # The overall completion order of these cogs is:
+  # first
+  # second : run right after first; does not have to wait for the slow background tasks to complete
+  # third
+  # slow_background_task_1
+  # fourth : forced to wait for slow_background_task_1 to complete to access its input
+  # fifth
+  # slow_background_task_2 : the workflow will not complete until all async cogs have completed
+end

data/dsl/async_cogs_complex.rb

@@ -0,0 +1,67 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd { display! }
+
+  # Any type of cog can be configured to run asynchronously.
+  # The cog that follows it in the workflow will be able to begin running immediately, before the async cog has finished.
+  cmd(:second) { async! }
+
+  # If there are a mix of sync and async cogs in a workflow, any synchronous cog may begin running before any
+  # async cogs above it have completed, but no cogs below it -- sync or async -- will start until it has completed.
+  cmd(:third) { no_async! } # NOTE: `no_async!` is redundant here (it's the default); but included here for extra clarity.
+  cmd(:fourth) { async! }
+  cmd(:fifth) { async! }
+  cmd(:sixth) { async! }
+end
+
+execute do
+  # 'first' is synchronous; nothing else runs until it completes.
+  cmd(:first) { "echo first" }
+
+  # 'second' is async and slow; it will not complete until after 'third' is done,
+  # but 'third' will be allowed to start right away.
+  cmd(:second) do
+    sleep 0.2
+    "echo second"
+  end
+
+  # 'third is synchronous and medium-speed; it will not wait for the async cogs above it,
+  # so it will complete before the slow, async 'second' cog.
+  cmd(:third) do
+    sleep 0.1
+    "echo third"
+  end
+
+  # 'fourth' is async, but it will not be allowed to start until the synchronous 'third' cog about it has completed.
+  # It will still complete before the slow-running 'second' cog, though.
+  cmd(:fourth) { "echo fourth" }
+
+  # 'fifth' is async, and fast, and will start before the slow-running 'second' cog.
+  # However, its input depends on the output of 'second', so it will automatically wait for 'second' to complete.
+  cmd(:fifth) do
+    # All the output accessors -- cmd?(:name), cmd(:name), cmd!(:name) -- will block
+    # until the named async cog is complete IF AND ONLY IF it has already started.
+    # If the named cog is defined later in the workflow and thus not already started,
+    # these methods will return immediately.
+    "echo \"fifth (second said: '#{cmd!(:second).text}')\""
+  end
+
+  # 'sixth' is async as well, and it will complete before the slow-running 'second' cog AND before the 'fifth' cog
+  # that is blocking on the output of 'second'
+  cmd(:sixth) do
+    sleep 0.01 # just to make sure this runs slightly slower than 'fourth' to maintain deterministic ordering
+    "echo sixth"
+  end
+
+  # The overall completion order of these cogs is:
+  # first : synchronous; nothing else started until it was done
+  # third : synchronous; did not block on the async 'second' cog
+  # fourth : async; could not start until after 'third'
+  # sixth : async; started at the same time as 'fourth', ran very slightly slower
+  # second : async, slow; started right after 'first', but didn't complete until now
+  # fifth : async; faster than 'second' but depended on its output, so it blocked until 'second' was complete
+end

data/dsl/call.rb

@@ -0,0 +1,44 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd do
+    display!
+  end
+end
+
+execute(:capitalize_a_random_word) do
+  cmd(:word) { "shuf /usr/share/dict/words -n 1" }
+  cmd(:capitalize) do |my, value_from_call|
+    # Optional second block argument lets you use a value passed to the sub-executor by `call`
+    # from withing any cog in the sub-executor
+    word = value_from_call || cmd!(:word).text
+    my.command = "/bin/sh"
+    my.args << "-c"
+    my.args << "/bin/echo \"#{word}\" | tr '[:lower:]' '[:upper:]'"
+  end
+end
+
+execute do
+  cmd(:before) { "echo '--> before'" }
+
+  # First positional argument is always the (optional) cog name
+  # Keyword argument "run" is the (required) scope name of the executor to run
+  call(:first_call, run: :capitalize_a_random_word)
+  call(:other_named_call, run: :capitalize_a_random_word)
+  call(run: :capitalize_a_random_word) # anonymous call cog
+
+  cmd { "echo '---'" }
+
+  # Can invoke a sub-executor with an executor-scoped value, that will be passed to each cog's input proc in that scope
+  call(run: :capitalize_a_random_word) do |my|
+    my.value = "scope value: roast"
+  end
+
+  # Shorthand input coercion for passing a scope value
+  call(run: :capitalize_a_random_word) { "scope value: other" }
+
+  cmd(:after) { "echo '--> after'" }
+end

data/dsl/collect_from.rb

@@ -0,0 +1,72 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd { display! }
+  cmd(/to_/) { no_display! }
+end
+
+execute(:capitalize_a_word) do
+  cmd(:to_original) { |_, word| "echo \"#{word}\"" }
+  cmd(:to_upper) do |my, word|
+    my.command = "sh"
+    my.args << "-c"
+    my.args << "echo \"#{word}\" | tr '[:lower:]' '[:upper:]'"
+  end
+  cmd(:to_lower) do |my, word|
+    my.command = "sh"
+    my.args << "-c"
+    my.args << "echo \"#{word}\" | tr '[:upper:]' '[:lower:]'"
+  end
+end
+
+execute do
+  # Call a subroutine with `call` or `map`
+  call(:hello, run: :capitalize_a_word) { "Hello" }
+  call(:world, run: :capitalize_a_word) { "World" }
+  map(:other_words, run: :capitalize_a_word) { ["Goodnight", "Moon"] }
+
+  cmd do
+    # Normally, you can only reference the output of cogs that run in the same executor scope.
+    begin
+      # There is no :to_upper cog that this could sensibly be referring to
+      cmd(:to_upper)
+    rescue Roast::DSL::CogInputManager::CogDoesNotExistError
+      puts "Could not access :to_upper directly"
+    end
+
+    # Using `from`, you can access cogs from the executor scope that was run by a specific named `call`.
+    # The block you pass to `from` runs in the input context of the specified scope, rather than the current scope.
+    original = from(call!(:hello)) { cmd!(:to_original).text }
+    # The type of the return value of `from` is the same type as the return value of the block.
+    upper = from(call!(:hello)) { cmd!(:to_upper) }.text
+    # You can also use this shorthand `from` syntax to get the output of the last cog in the call's executor scope.
+    # In this syntax, the return value of `from` is untyped
+    lower = from(call!(:hello)).text
+    "echo \"#{original} --> #{upper} --> #{lower}\""
+  end
+
+  cmd do
+    # You can also grab the `call`'s output once and pass it to multiple `from` invocations.
+    my_scope = call!(:world)
+    original = from(my_scope) { cmd!(:to_original).text }
+    upper = from(my_scope) { cmd!(:to_upper).text }
+    lower = from(my_scope) { cmd!(:to_lower).text }
+    "echo \"#{original} --> #{upper} --> #{lower}\""
+  end
+
+  cmd do
+    # Using `collect`, you can access cogs from the executor scopes that were run by a specific named `map`.
+    # The block you pass to `collect` runs in the input context of each specified scope.
+    # `collect` returns an array containing the output of each invocation of that block.
+    originals = collect(map!(:other_words)) { cmd!(:to_original).text }
+    # The type of the return value of `call` is an Array of the same type as the return value of the block.
+    uppers = collect(map!(:other_words)) { cmd!(:to_upper) }.map(&:text)
+    # You can also use this shorthand `collect` syntax to get the output of the last cog in each of the map's executor scopes.
+    # In this syntax, the return value of `collect` is Array[untyped]
+    lowers = collect(map!(:other_words)).map(&:text)
+    "echo \"#{originals.join(",")} --> #{uppers.join(",")} --> #{lowers.join(",")}\""
+  end
+end

data/dsl/demo/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+gem "roast-ai", path: "../.."
+gem "plugin-gem-example", path: "../plugin-gem-example"

data/dsl/demo/Gemfile.lock

@@ -0,0 +1,120 @@
+PATH
+  remote: ../..
+  specs:
+    roast-ai (0.4.9)
+      activesupport (>= 7.0)
+      cli-kit (~> 5.0)
+      cli-ui (= 2.3.0)
+      diff-lcs (~> 1.5)
+      json-schema
+      open_router (~> 0.3)
+      raix (~> 1.0.2)
+      ruby-graphviz (~> 1.2)
+      ruby_llm
+      sqlite3 (~> 2.6)
+      thor (~> 1.3)
+      zeitwerk (~> 2.6)
+
+PATH
+  remote: ../plugin-gem-example
+  specs:
+    plugin-gem-example (0.1.0)
+      roast-ai
+
+GEM
+  specs:
+    activesupport (8.0.3)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    base64 (0.3.0)
+    benchmark (0.4.1)
+    bigdecimal (3.3.1)
+    cli-kit (5.0.1)
+      cli-ui (~> 2.0)
+    cli-ui (2.3.0)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.4)
+    diff-lcs (1.6.2)
+    dotenv (3.1.8)
+    drb (2.2.3)
+    event_stream_parser (1.0.0)
+    faraday (2.14.0)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-multipart (1.1.1)
+      multipart-post (~> 2.0)
+    faraday-net_http (3.4.1)
+      net-http (>= 0.5.0)
+    faraday-retry (2.3.2)
+      faraday (~> 2.0)
+    i18n (1.14.7)
+      concurrent-ruby (~> 1.0)
+    json (2.15.1)
+    json-schema (6.0.0)
+      addressable (~> 2.8)
+      bigdecimal (~> 3.1)
+    logger (1.7.0)
+    marcel (1.1.0)
+    minitest (5.26.0)
+    multipart-post (2.4.1)
+    net-http (0.6.0)
+      uri
+    open_router (0.3.3)
+      activesupport (>= 6.0)
+      dotenv (>= 2)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
+    ostruct (0.6.3)
+    public_suffix (6.0.2)
+    raix (1.0.3)
+      activesupport (>= 6.0)
+      faraday-retry (~> 2.0)
+      open_router (~> 0.2)
+      ostruct
+      ruby-openai (~> 8.1)
+    rexml (3.4.4)
+    ruby-graphviz (1.2.5)
+      rexml
+    ruby-openai (8.3.0)
+      event_stream_parser (>= 0.3.0, < 2.0.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
+    ruby_llm (1.8.2)
+      base64
+      event_stream_parser (~> 1)
+      faraday (>= 1.10.0)
+      faraday-multipart (>= 1)
+      faraday-net_http (>= 1)
+      faraday-retry (>= 1)
+      marcel (~> 1.0)
+      zeitwerk (~> 2)
+    securerandom (0.4.1)
+    sqlite3 (2.7.4-arm64-darwin)
+    thor (1.4.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    uri (1.0.4)
+    zeitwerk (2.7.3)
+
+PLATFORMS
+  arm64-darwin
+
+DEPENDENCIES
+  plugin-gem-example!
+  roast-ai!
+
+BUNDLED WITH
+   2.6.8

data/dsl/demo/cogs/local.rb

@@ -0,0 +1,15 @@
+# typed: true
+# frozen_string_literal: true
+
+class Local < Roast::DSL::Cog
+  class Input < Roast::DSL::Cog::Input
+    def validate!
+      true
+    end
+  end
+
+  #: (Input) -> void
+  def execute(input)
+    puts "I'm a workflow-specific cog!"
+  end
+end

data/dsl/demo/simple_external_cog.rb

@@ -0,0 +1,17 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+use "simple", from: "plugin_gem_example"
+use "MyCogNamespace::Other", from: "plugin_gem_example"
+use "local"
+
+# Use multiple cogs
+# use ["simple", "other"], from: "plugin_gem_example"
+
+execute do
+  simple
+  other
+  local
+end

data/dsl/json_output.rb

@@ -0,0 +1,28 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd { display! }
+end
+
+execute do
+  cmd(:json) do |my|
+    my.command = "echo"
+    my.args << <<~JSON
+      {
+        "hello": "world",
+        "letters": [
+          "aaa",
+          "bbb"
+        ]
+      }
+    JSON
+  end
+
+  ruby do
+    puts "RAW OUTPUT: #{cmd!(:json).text}"
+    puts "SOME VALUE FROM PARSED OUTPUT: #{cmd!(:json).json![:letters].first}"
+  end
+end

data/dsl/map.rb

@@ -0,0 +1,55 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd do
+    display!
+  end
+end
+
+execute(:capitalize_a_word) do
+  cmd(:capitalize) do |my, word|
+    # Use the `fail!` method in a cog's input proc to terminate the cog with a failure state
+    # e.g., if a condition prevents its successful execution.
+    # If `fail!` is called, the input proc will immediately terminate.
+    # The entire workflow may or may not terminate as a result, based on its configuration and the cog's configuration.
+    fail! unless word.present?
+    my.command = "sh"
+    my.args << "-c"
+    my.args << "echo \"#{word}\" | tr '[:lower:]' '[:upper:]'"
+  end
+end
+
+execute do
+  words = ["hello", "world", "goodnight", "moon"]
+
+  # WITHOUT MAP COG
+  # You can use plain Ruby to generate a collection of cogs programmatically
+  # This is equivalent to simply defining each of the cogs explicitly, one by one
+  # In this case, four copies of the `call` cog invoking :capitalize_a_word
+  words.each { |word| call(run: :capitalize_a_word) { word } }
+
+  cmd(:foo) { "echo" }
+
+  # USING MAP COG
+  # You can use the `map` cog to apply a scoped executor to each item in a collection of values
+  map(:some_name, run: :capitalize_a_word) do |my|
+    my.items = words
+  end
+
+  cmd { "echo" }
+
+  # USING MAP COG (SHORTHAND)
+  # - name of execute scope is required
+  # - name of the map cog itself can be omitted (anonymous cog)
+  # - items over which to map coerced from return value of input proc
+  map(run: :capitalize_a_word) { words.reverse }
+
+  # ACCESSING THE OUTPUT OF A SPECIFIC MAP ITERATION
+  ruby do
+    puts ""
+    puts "#{words[2]} -> #{from(map!(:some_name).iteration(2)) { cmd!(:capitalize).text }}"
+  end
+end

data/dsl/map_reduce.rb

@@ -0,0 +1,37 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd { display! }
+  cmd(/to_/) { no_display! }
+end
+
+execute(:capitalize_a_word) do
+  cmd(:to_original) { |_, word| "echo \"#{word}\"" }
+  cmd(:to_upper) do |my, word|
+    my.command = "sh"
+    my.args << "-c"
+    my.args << "echo \"#{word}\" | tr '[:lower:]' '[:upper:]'"
+  end
+  cmd(:to_lower) do |my, word|
+    my.command = "sh"
+    my.args << "-c"
+    my.args << "echo \"#{word}\" | tr '[:upper:]' '[:lower:]'"
+  end
+end
+
+execute do
+  # Call a subroutine with `map`
+  map(:words, run: :capitalize_a_word) { ["Hello", "World"] }
+
+  cmd do
+    # Use `reduce` to apply a block to the input context of each executor scope run by `map` in turn.
+    # The return value of each invocation of the block will be passed as the 'accumulator' to the next invocation.
+    # You can provide an optional initial value for the accumulator as the second argument to `reduce`.
+    # If an initial value is present, the return type is non-nilable. If absent, the return type is nilable.
+    words = reduce(map!(:words), "lower case words:") { |acc| acc + " " + cmd!(:to_lower).text }
+    "echo \"#{words}\""
+  end
+end

data/dsl/map_with_index.rb

@@ -0,0 +1,49 @@
+# typed: true
+# frozen_string_literal: true
+
+#: self as Roast::DSL::Workflow
+
+config do
+  cmd do
+    display!
+  end
+end
+
+execute(:capitalize_a_word) do
+  cmd(:capitalize) do |my, word, index|
+    fail! unless word.present?
+    my.command = "sh"
+    my.args << "-c"
+    # When this execution scope is called by the `map` cog, the optional 'index' argument to this block
+    # will contain the index of its element in the enumerable on which `map` is invoked
+    my.args << "echo \"[#{index}] #{word}\" | tr '[:lower:]' '[:upper:]'"
+  end
+end
+
+execute do
+  words = ["hello", "world", "goodnight", "moon"]
+
+  # When calling an execute scope with `map`, the index of the item in the enumerable
+  # will be provided to the cogs in that scope
+  map(:some_name, run: :capitalize_a_word) { words }
+
+  cmd { "echo" }
+
+  # When calling an execute scope with `call`, the cogs in that scope will receive 0 as the index value by default
+  call(run: :capitalize_a_word) { "default" }
+
+  call(run: :capitalize_a_word) do |my|
+    my.value = "specific"
+    # It is possible to specify a custom index value when invoking `call`
+    my.index = 23
+  end
+
+  cmd { "echo" }
+
+  # `map` will also take a custom 'initial_index' value, at which to start the value of 'index' passed
+  # to the executors it invokes. This parallels the syntax of Ruby's `items.map.with_index(initial_value) { ... }`
+  map(run: :capitalize_a_word) do |my|
+    my.items = words[1, 2]
+    my.initial_index = 8
+  end
+end