rcrewai 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d8a012bca72730f032aa41c0b8b9baeda9bc2dffaad12b3d742e1904fa2b879
-  data.tar.gz: 3bffa51d2181e09b651eeeb4b109005016048a7595f1d1a4a13a96175b6473dd
+  metadata.gz: 1e33f9998ef9f32aa53d4c6df3e8e7b04314f7b5cbb9fe06e5c7d1936b466222
+  data.tar.gz: 294a6a4a5ed7d5f1b5fbb58c3edd0bb69a0cd5d9e61da07b9f8dabf6c4457414
 SHA512:
-  metadata.gz: f7a78f7ea2652e8511324c737041bdd290e3df2ea5af75ee736292aa57ff47f26669dcb3fd09788b18f8f731341d46f9cc384468add953f05a8b2840b3fde79c
-  data.tar.gz: 9526bde229c78e8d4344a1c1cdb3343db511cc8145c8b6bd97dc2e047af23d063e3afe17cef6af51fefbfa70df3c91e6696c43b1d3cbfbbcb3a5f24e22dd978a
+  metadata.gz: f64195cd3065e2c26dd65ddea08246bec905754192e8ba0fad1a23f7bcbfc998c2f8a4d3cad91f7491f5d8e0f2fc5c4cd53be60de47e737f9d41b2c898089ba7
+  data.tar.gz: 5107c4953f3ff80f86125fdbb5325bf582b1c624b07e97b739890fd21d0c83c892c7adff4fd982556141dd5a60b186814bcc3b1644056a89d589788810fee860

data/CHANGELOG.md

@@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.10.0] - 2026-10-19
+
+### Added
+- Anthropic live streaming for real-time callers (voice agents):
+  - `RCrewAI::CancelToken`: pass `cancel:` to `chat`; `cancel!` aborts the
+    stream at the next chunk, closes the connection and raises
+    `RCrewAI::StreamCancelled`. Under an Async reactor, stopping the calling
+    task does the same.
+  - Streaming requests reuse one keep-alive connection per client
+    (`RCrewAI::PersistentHTTP`), so back-to-back turns skip TCP/TLS setup.
+  - `chat(model:)` overrides the configured model per call.
+  - Usage now carries `cached_tokens` (Anthropic `cache_read_input_tokens`)
+    in the result hash and on the `Events::Usage` stream event.
+- The Anthropic client honours `config.base_url`.
+
+### Notes
+- Verified fiber-safe on Ruby >= 3.2 / async 2.x: Anthropic streaming inside an
+  Async reactor never blocks other fibers (spec covers it, skipped < 3.2).
+
 ## [0.9.0] - 2026-09-10
 
 Concurrency. Three points where independent, IO-bound work ran in sequence now

data/ROADMAP.md

@@ -139,6 +139,12 @@ since this fragility lives exactly in 3.0/3.1 scheduler support.
 Threads cost none of that: `concurrent-ruby` is already a dependency, the Ruby
 floor is unchanged, and failures are loud.
 
+**Update (2026-10):** the fiber failure is specific to Ruby 3.0/3.1. On Ruby
+3.3 with `async` 2.46, Net::HTTP streaming inside a fiber yields to the
+reactor correctly (`spec/anthropic_live_streaming_spec.rb` asserts a 10 ms
+ticker never stalls, and that stopping the task closes the stream). Threads
+stay the internal concurrency model; fiber callers on 3.2+ are supported.
+
 Upstream's async/await shape is a Python idiom; porting its *form* rather than
 its *effect* would buy nothing here.
 

data/lib/rcrewai/cancel_token.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module RCrewAI
+  # Raised inside a streaming call when its CancelToken is triggered.
+  class StreamCancelled < Error; end
+
+  # Cooperative cancellation for a streaming LLM call. Pass it as
+  # `cancel:` to a client's #chat; calling #cancel! (from the stream sink,
+  # another thread, or another fiber) aborts the stream at the next chunk,
+  # closes the connection and raises StreamCancelled in the caller.
+  #
+  # Under an Async reactor, stopping the calling task has the same effect.
+  class CancelToken
+    def initialize
+      @cancelled = false
+    end
+
+    def cancel!
+      @cancelled = true
+    end
+
+    def cancelled?
+      @cancelled
+    end
+  end
+end

data/lib/rcrewai/events.rb

@@ -17,7 +17,8 @@ module RCrewAI
     ToolCallResult  = Struct.new(*BASE_ATTRS, :tool, :call_id, :result, :duration_ms, keyword_init: true)
     ToolCallError   = Struct.new(*BASE_ATTRS, :tool, :call_id, :error,                keyword_init: true)
     Thinking        = Struct.new(*BASE_ATTRS, :text,                                  keyword_init: true)
-    Usage           = Struct.new(*BASE_ATTRS, :prompt_tokens, :completion_tokens, :total_tokens, :cost_usd, keyword_init: true)
+    Usage           = Struct.new(*BASE_ATTRS, :prompt_tokens, :completion_tokens, :total_tokens, :cost_usd,
+                                 :cached_tokens, keyword_init: true)
     IterationStart  = Struct.new(*BASE_ATTRS, :iteration_index,                       keyword_init: true)
     IterationEnd    = Struct.new(*BASE_ATTRS, :finish_reason,                         keyword_init: true)
     Error           = Struct.new(*BASE_ATTRS, :error,                                 keyword_init: true)

data/lib/rcrewai/llm_clients/anthropic.rb

@@ -7,6 +7,8 @@ require_relative '../events'
 require_relative '../sse_parser'
 require_relative '../provider_schema'
 require_relative '../pricing'
+require_relative '../cancel_token'
+require_relative '../persistent_http'
 
 module RCrewAI
   module LLMClients
@@ -23,7 +25,7 @@ module RCrewAI
 
       def initialize(config = RCrewAI.configuration, **hooks)
         super(config, **hooks)
-        @base_url = BASE_URL
+        @base_url = config.base_url || BASE_URL
       end
 
       def chat(messages:, tools: nil, tool_choice: :auto, stream: nil, **options)
@@ -31,7 +33,7 @@ module RCrewAI
         non_system = messages.reject { |m| m.is_a?(Hash) && m[:role] == 'system' }
 
         payload = {
-          model: config.model,
+          model: options[:model] || config.model,
           messages: format_messages(non_system),
           max_tokens: options[:max_tokens] || config.max_tokens || 1000,
           temperature: options[:temperature] || config.temperature
@@ -57,7 +59,7 @@ module RCrewAI
 
         if stream
           payload[:stream] = true
-          stream_chat(payload, stream)
+          stream_chat(payload, stream, cancel: options[:cancel])
         else
           plain_chat(payload)
         end
@@ -91,7 +93,7 @@ module RCrewAI
         apply_after_response(normalize_non_streaming(body), started_at)
       end
 
-      def stream_chat(payload, sink) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      def stream_chat(payload, sink, cancel: nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength
         url = "#{@base_url}/messages"
         payload = apply_before_request(payload)
         started_at = Time.now
@@ -102,6 +104,7 @@ module RCrewAI
         blocks = {}
         finish_reason = nil
         prompt_tokens = nil
+        cached_tokens = nil
         completion_tokens = nil
 
         parser = SSEParser.new do |sse|
@@ -109,6 +112,7 @@ module RCrewAI
           case data['type']
           when 'message_start'
             prompt_tokens = data.dig('message', 'usage', 'input_tokens')
+            cached_tokens = data.dig('message', 'usage', 'cache_read_input_tokens')
           when 'content_block_start'
             cb = data['content_block'] || {}
             blocks[data['index']] = { id: cb['id'], name: cb['name'], arguments: +'' } if cb['type'] == 'tool_use'
@@ -131,7 +135,7 @@ module RCrewAI
           end
         end
 
-        streaming_post(url, payload) { |chunk| parser.feed(chunk) }
+        streaming_post(payload, cancel: cancel) { |chunk| parser.feed(chunk) }
 
         tool_calls = blocks.values.map do |b|
           {
@@ -144,15 +148,16 @@ module RCrewAI
         usage = {
           prompt_tokens: prompt_tokens,
           completion_tokens: completion_tokens,
-          total_tokens: (prompt_tokens || 0) + (completion_tokens || 0)
+          total_tokens: (prompt_tokens || 0) + (completion_tokens || 0),
+          cached_tokens: cached_tokens
         }
 
         if prompt_tokens || completion_tokens
           sink.call(Events::Usage.new(
                       type: :usage, timestamp: Time.now, agent: nil, iteration: nil,
                       prompt_tokens: prompt_tokens, completion_tokens: completion_tokens,
-                      total_tokens: usage[:total_tokens],
-                      cost_usd: Pricing.cost_for(config.model,
+                      total_tokens: usage[:total_tokens], cached_tokens: cached_tokens,
+                      cost_usd: Pricing.cost_for(payload[:model],
                                                  prompt_tokens: prompt_tokens || 0,
                                                  completion_tokens: completion_tokens || 0)
                     ))
@@ -163,23 +168,25 @@ module RCrewAI
           tool_calls: tool_calls,
           usage: usage,
           finish_reason: finish_reason || :stop,
-          model: config.model,
+          model: payload[:model],
           provider: :anthropic
         }
         apply_after_response(result, started_at)
       end
 
-      def streaming_post(url, payload, &on_chunk)
-        conn = Faraday.new do |f|
-          f.request :json
-          f.options.timeout = config.timeout
-          f.adapter Faraday.default_adapter
-        end
-        conn.post(url) do |req|
-          req.headers = build_headers.merge(auth_header)
-          req.body = payload.to_json
-          req.options.on_data = proc { |chunk, _| on_chunk.call(chunk) }
-        end
+      # Streams over one kept-alive connection per client (see PersistentHTTP).
+      def streaming_post(payload, cancel: nil, &on_chunk)
+        uri = URI("#{@base_url}/messages")
+        response = streaming_connection.post_stream(
+          uri.request_uri, headers: build_headers.merge(auth_header), body: payload.to_json, cancel: cancel, &on_chunk
+        )
+        return response if response.is_a?(Net::HTTPSuccess)
+
+        handle_response(Struct.new(:status, :body).new(response.code.to_i, response.body))
+      end
+
+      def streaming_connection
+        @streaming_connection ||= PersistentHTTP.new(@base_url, timeout: config.timeout)
       end
 
       def normalize_non_streaming(body)

data/lib/rcrewai/persistent_http.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module RCrewAI
+  # One keep-alive HTTP connection per client for streaming POSTs, so
+  # back-to-back calls skip TCP/TLS setup. Requests on a connection are
+  # serialised. A stream that does not finish cleanly (cancel, task stop,
+  # error) discards the connection; the next call reconnects. Under Ruby's
+  # fiber scheduler Net::HTTP yields on IO, so this is reactor-friendly.
+  class PersistentHTTP
+    RETRYABLE = [EOFError, IOError, Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNREFUSED].freeze
+
+    def initialize(base_url, timeout:)
+      @uri = URI(base_url)
+      @timeout = timeout
+      @lock = Mutex.new
+      @http = nil
+    end
+
+    # Yields body chunks as they arrive. Returns the Net::HTTPResponse.
+    # Non-2xx responses are returned with their body read (not yielded).
+    def post_stream(path, headers:, body:, cancel: nil, &on_chunk)
+      @lock.synchronize do
+        attempts = 0
+        begin
+          perform(path, headers, body, cancel, &on_chunk)
+        rescue *RETRYABLE
+          # A kept-alive connection the server already closed fails before any
+          # bytes arrive; retry once on a fresh connection.
+          raise if @received || (attempts += 1) > 1
+
+          retry
+        end
+      end
+    end
+
+    def close
+      @http&.finish if @http&.started?
+    rescue IOError
+      nil
+    ensure
+      @http = nil
+    end
+
+    private
+
+    def perform(path, headers, body, cancel, &on_chunk)
+      @received = false
+      completed = false
+      request = Net::HTTP::Post.new(path, headers)
+      request.body = body
+      response = nil
+      connection.request(request) do |res|
+        response = res
+        unless res.is_a?(Net::HTTPSuccess)
+          res.read_body
+          next
+        end
+
+        res.read_body do |chunk|
+          @received = true
+          raise StreamCancelled, 'stream cancelled' if cancel&.cancelled?
+
+          on_chunk.call(chunk)
+          raise StreamCancelled, 'stream cancelled' if cancel&.cancelled?
+        end
+      end
+      completed = true
+      response
+    ensure
+      close unless completed
+    end
+
+    def connection
+      return @http if @http&.started?
+
+      @http = Net::HTTP.new(@uri.host, @uri.port)
+      @http.use_ssl = @uri.scheme == 'https'
+      @http.open_timeout = 10
+      @http.read_timeout = @timeout
+      @http.keep_alive_timeout = 30
+      @http.start
+    end
+  end
+end

data/lib/rcrewai/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RCrewAI
-  VERSION = '0.9.0'
+  VERSION = '0.10.0'
 end

data/lib/rcrewai.rb

@@ -21,6 +21,8 @@ require_relative 'rcrewai/configuration'
 require_relative 'rcrewai/events'
 require_relative 'rcrewai/checkpoint'
 require_relative 'rcrewai/sse_parser'
+require_relative 'rcrewai/cancel_token'
+require_relative 'rcrewai/persistent_http'
 require_relative 'rcrewai/pricing'
 require_relative 'rcrewai/llm_client'
 require_relative 'rcrewai/similarity'

data/rcrewai.gemspec

@@ -60,6 +60,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'thor', '~> 1.3'
 
   # Development dependencies
+  spec.add_development_dependency 'async', '~> 2.0'
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.12'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rcrewai
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - gkosmo
@@ -135,6 +135,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -347,6 +361,7 @@ files:
 - lib/rcrewai/agent.rb
 - lib/rcrewai/agent_augmentations.rb
 - lib/rcrewai/async_executor.rb
+- lib/rcrewai/cancel_token.rb
 - lib/rcrewai/checkpoint.rb
 - lib/rcrewai/checkpoint/cli.rb
 - lib/rcrewai/cli.rb
@@ -392,6 +407,7 @@ files:
 - lib/rcrewai/memory/tool_memory.rb
 - lib/rcrewai/multimodal.rb
 - lib/rcrewai/output_schema.rb
+- lib/rcrewai/persistent_http.rb
 - lib/rcrewai/planning.rb
 - lib/rcrewai/pricing.rb
 - lib/rcrewai/process.rb