rcrewai 0.3.0 → 0.4.0

data/lib/rcrewai/flow.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require_relative 'flow/state'
+require_relative 'flow/state_store'
+
+module RCrewAI
+  # Event-driven workflow engine — CrewAI's second pillar, in Ruby.
+  #
+  # Subclass Flow and declare methods with the class-level DSL:
+  #
+  #   class GuideFlow < RCrewAI::Flow
+  #     start :pick_topic
+  #     def pick_topic = state.topic = 'ruby'
+  #
+  #     listen :pick_topic
+  #     def research(prev) = "researched #{prev}"
+  #
+  #     router :research
+  #     def route(prev) = prev.include?('ruby') ? :publish : :revise
+  #
+  #     listen :publish
+  #     def publish = state.done = true
+  #   end
+  #
+  #   GuideFlow.new.kickoff
+  #
+  # Triggers combine with and_/or_. State is a schemaless object with a UUID and
+  # can be persisted/restored via a state store.
+  class Flow
+    # --- Trigger descriptors -------------------------------------------------
+    Trigger = Struct.new(:mode, :names) do
+      def satisfied_by?(completed)
+        case mode
+        when :single, :or then names.any? { |n| completed.include?(n) }
+        when :and then names.all? { |n| completed.include?(n) }
+        end
+      end
+    end
+
+    # --- Class-level DSL -----------------------------------------------------
+    class << self
+      def start_methods
+        @start_methods ||= []
+      end
+
+      # name => Trigger. Populated when a listen/router declaration is bound to
+      # the next defined method.
+      def listeners
+        @listeners ||= {}
+      end
+
+      def routers
+        @routers ||= {}
+      end
+
+      def start(method_name)
+        start_methods << method_name.to_sym
+      end
+
+      def listen(trigger)
+        @pending = [:listen, normalize_trigger(trigger)]
+      end
+
+      def router(trigger)
+        @pending = [:router, normalize_trigger(trigger)]
+      end
+
+      def or_(*names)
+        Trigger.new(:or, names.map(&:to_sym))
+      end
+
+      def and_(*names)
+        Trigger.new(:and, names.map(&:to_sym))
+      end
+
+      # Binds a pending listen/router declaration to the method just defined.
+      def method_added(method_name)
+        super
+        return unless @pending
+
+        kind, trigger = @pending
+        @pending = nil
+        case kind
+        when :listen then listeners[method_name.to_sym] = trigger
+        when :router then routers[method_name.to_sym] = trigger
+        end
+      end
+
+      # Merge inherited declarations so subclasses of a Flow subclass compose.
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@start_methods, start_methods.dup)
+        subclass.instance_variable_set(:@listeners, listeners.dup)
+        subclass.instance_variable_set(:@routers, routers.dup)
+      end
+
+      private
+
+      def normalize_trigger(trigger)
+        return trigger if trigger.is_a?(Trigger)
+
+        Trigger.new(:single, [trigger.to_sym])
+      end
+    end
+
+    # --- Instance ------------------------------------------------------------
+    attr_reader :state
+
+    def initialize(state_store: nil, feedback_handler: nil)
+      @state = State.new
+      @state_store = state_store
+      @feedback_handler = feedback_handler
+    end
+
+    # A pause point for human feedback. Calls the configured feedback_handler
+    # with the prompt and returns its response; without a handler, prompts on
+    # the console. Mirrors CrewAI's @human_feedback.
+    def human_feedback(prompt)
+      return @feedback_handler.call(prompt) if @feedback_handler
+
+      require_relative 'human_input'
+      response = HumanInput.new.request_input(prompt)
+      response.is_a?(Hash) ? response[:input] : response
+    end
+
+    # Runs the flow to completion. Optional inputs seed the state.
+    def kickoff(inputs: {})
+      inputs.each { |k, v| @state[k] = v }
+
+      @completed = []       # method names that have finished
+      @outputs = {}         # method name => return value
+      @router_labels = []   # labels emitted by routers, act as pseudo-triggers
+
+      self.class.start_methods.each { |m| run_method(m) }
+      drain_listeners
+
+      persist
+      @state
+    end
+
+    # Restores state previously persisted under +id+.
+    def restore(id)
+      raise FlowError, 'no state store configured' unless @state_store
+
+      hash = @state_store.load(id)
+      raise FlowError, "no persisted state for id #{id}" unless hash
+
+      @state = State.new(symbolize(hash))
+      @state
+    end
+
+    private
+
+    def run_method(method_name)
+      trigger = self.class.listeners[method_name] || self.class.routers[method_name]
+      arg = trigger ? @outputs[last_trigger_name(trigger)] : nil
+
+      result = arity_for(method_name).zero? ? send(method_name) : send(method_name, arg)
+
+      @completed << method_name
+      @outputs[method_name] = result
+
+      # A router's return value becomes a label that listeners can trigger on.
+      @router_labels << result.to_sym if self.class.routers.key?(method_name) && result
+    end
+
+    # Repeatedly fire any listeners/routers whose triggers are now satisfied,
+    # until no new method runs (fixed point).
+    def drain_listeners
+      reactive = self.class.listeners.merge(self.class.routers)
+      loop do
+        ran = false
+        reactive.each do |method_name, trigger|
+          next if fired_enough?(method_name, trigger)
+
+          if trigger.satisfied_by?(satisfied_set)
+            run_listener(method_name, trigger)
+            ran = true
+          end
+        end
+        break unless ran
+      end
+    end
+
+    # For :single/:or triggers we fire once per completed trigger name; for :and
+    # we fire once. Track how many times each listener has fired.
+    def run_listener(method_name, trigger)
+      @fired ||= Hash.new { |h, k| h[k] = [] }
+
+      case trigger.mode
+      when :and
+        @fired[method_name] << :once
+        invoke_listener(method_name, @outputs[trigger.names.last])
+      else
+        pending = trigger.names.select { |n| satisfied_set.include?(n) } - @fired[method_name]
+        pending.each do |name|
+          @fired[method_name] << name
+          invoke_listener(method_name, @outputs[name])
+        end
+      end
+    end
+
+    def invoke_listener(method_name, arg)
+      result = arity_for(method_name).zero? ? send(method_name) : send(method_name, arg)
+      @completed << method_name
+      @outputs[method_name] = result
+      @router_labels << result.to_sym if self.class.routers.key?(method_name) && result
+    end
+
+    def fired_enough?(method_name, trigger)
+      @fired ||= Hash.new { |h, k| h[k] = [] }
+      case trigger.mode
+      when :and then @fired[method_name].any?
+      else (trigger.names & satisfied_set).all? { |n| @fired[method_name].include?(n) }
+      end
+    end
+
+    # Names available to satisfy triggers: completed methods + router labels.
+    def satisfied_set
+      @completed + @router_labels
+    end
+
+    def last_trigger_name(trigger)
+      (trigger.names & @completed).last || trigger.names.last
+    end
+
+    def arity_for(method_name)
+      method(method_name).arity
+    end
+
+    def persist
+      return unless @state_store
+
+      @state_store.save(@state.id, @state.to_h)
+    end
+
+    def symbolize(hash)
+      hash.transform_keys(&:to_sym)
+    end
+  end
+
+  class FlowError < Error; end
+end

data/lib/rcrewai/knowledge/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative 'chunker'
+require_relative 'store'
+require_relative 'sources'
+require_relative 'embedder'
+
+module RCrewAI
+  module Knowledge
+    # A knowledge base: loads sources, chunks their text, embeds the chunks, and
+    # answers similarity queries. Attach one to an Agent (role-specific) or a
+    # Crew (shared) via the +knowledge_sources:+ option.
+    class Base
+      attr_reader :sources
+
+      def initialize(sources: [], embedder: nil, chunk_size: 1000, overlap: 100)
+        @sources = Array(sources)
+        @embedder = embedder || Embedder.new
+        @chunker = Chunker.new(chunk_size: chunk_size, overlap: overlap)
+        @store = Store.new
+        @built = false
+      end
+
+      # Loads, chunks, and embeds all sources. Idempotent.
+      def build!
+        return self if @built
+
+        chunks = @sources.flat_map { |source| @chunker.chunk(source.read) }
+        unless chunks.empty?
+          vectors = @embedder.embed(chunks)
+          chunks.zip(vectors).each { |text, vector| @store.add(text, vector) }
+        end
+
+        @built = true
+        self
+      end
+
+      # Returns up to k chunks most relevant to the query string.
+      def search(query, k: 3)
+        build! unless @built
+        return [] if @store.empty?
+
+        query_vector = @embedder.embed([query]).first
+        @store.search(query_vector, k: k)
+      end
+
+      def empty?
+        @sources.empty?
+      end
+    end
+  end
+end

data/lib/rcrewai/knowledge/chunker.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module RCrewAI
+  module Knowledge
+    # Splits text into fixed-size, overlapping character windows. Overlap keeps
+    # context from spilling across chunk boundaries during retrieval.
+    class Chunker
+      def initialize(chunk_size: 1000, overlap: 100)
+        raise ArgumentError, 'overlap must be smaller than chunk_size' if overlap >= chunk_size
+
+        @chunk_size = chunk_size
+        @overlap = overlap
+      end
+
+      def chunk(text)
+        text = text.to_s
+        return [] if text.empty?
+        return [text] if text.length <= @chunk_size
+
+        chunks = []
+        start = 0
+        step = @chunk_size - @overlap
+        while start < text.length
+          chunks << text[start, @chunk_size]
+          start += step
+        end
+        chunks
+      end
+    end
+  end
+end

data/lib/rcrewai/knowledge/embedder.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'faraday'
+
+module RCrewAI
+  module Knowledge
+    # Turns text into embedding vectors. Defaults to OpenAI's embeddings API;
+    # #embed takes an array of strings and returns an array of vectors. Any
+    # object responding to #embed can be substituted (see specs).
+    class Embedder
+      DEFAULT_MODEL = 'text-embedding-3-small'
+      OPENAI_URL = 'https://api.openai.com/v1/embeddings'
+
+      def initialize(model: DEFAULT_MODEL, api_key: nil, config: RCrewAI.configuration)
+        @model = model
+        @api_key = api_key || config.openai_api_key || config.api_key
+      end
+
+      def embed(texts)
+        texts = Array(texts)
+        return [] if texts.empty?
+
+        response = connection.post(OPENAI_URL) do |req|
+          req.headers['Authorization'] = "Bearer #{@api_key}"
+          req.headers['Content-Type'] = 'application/json'
+          req.body = JSON.generate(model: @model, input: texts)
+        end
+
+        raise EmbeddingError, "embedding request failed: #{response.status}" unless response.success?
+
+        body = response.body
+        body = JSON.parse(body) if body.is_a?(String)
+        body['data'].map { |d| d['embedding'] }
+      end
+
+      private
+
+      def connection
+        @connection ||= Faraday.new do |f|
+          f.adapter Faraday.default_adapter
+        end
+      end
+    end
+
+    class EmbeddingError < RCrewAI::Error; end
+  end
+end

data/lib/rcrewai/knowledge/sources.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module RCrewAI
+  module Knowledge
+    # A knowledge source yields plain text via #read. Concrete sources load from
+    # strings, files, PDFs, CSVs, or URLs.
+    class Source
+      def read
+        raise NotImplementedError, 'Subclasses must implement #read'
+      end
+    end
+
+    class StringSource < Source
+      def initialize(text)
+        super()
+        @text = text.to_s
+      end
+
+      def read
+        @text
+      end
+    end
+
+    class FileSource < Source
+      def initialize(path)
+        super()
+        @path = path
+      end
+
+      def read
+        File.read(@path)
+      end
+    end
+
+    class PdfSource < Source
+      def initialize(path)
+        super()
+        @path = path
+      end
+
+      def read
+        require 'pdf-reader'
+        reader = PDF::Reader.new(@path)
+        reader.pages.map(&:text).join("\n")
+      end
+    end
+
+    class CsvSource < Source
+      def initialize(path)
+        super()
+        @path = path
+      end
+
+      def read
+        require 'csv'
+        CSV.read(@path).map { |row| row.join(', ') }.join("\n")
+      end
+    end
+
+    class UrlSource < Source
+      def initialize(url, fetcher: nil)
+        super()
+        @url = url
+        @fetcher = fetcher
+      end
+
+      def read
+        html = @fetcher ? @fetcher.call(@url) : fetch(@url)
+        require 'nokogiri'
+        doc = Nokogiri::HTML(html)
+        doc.search('script, style').remove
+        doc.text.gsub(/\s+/, ' ').strip
+      end
+
+      private
+
+      def fetch(url)
+        require 'faraday'
+        Faraday.get(url).body
+      end
+    end
+  end
+end

data/lib/rcrewai/knowledge/store.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RCrewAI
+  module Knowledge
+    # In-memory vector store with cosine-similarity search. The default backing
+    # store for Knowledge — no external service required. The interface
+    # (#add, #search) is intentionally small so a Chroma/Qdrant-backed store can
+    # be swapped in later.
+    class Store
+      Entry = Struct.new(:text, :vector)
+
+      def initialize
+        @entries = []
+      end
+
+      def add(text, vector)
+        @entries << Entry.new(text, vector)
+      end
+
+      # Returns the texts of the top-k entries most similar to +query_vector+.
+      def search(query_vector, k: 3)
+        return [] if @entries.empty?
+
+        @entries
+          .map { |e| [e.text, cosine_similarity(query_vector, e.vector)] }
+          .sort_by { |(_text, score)| -score }
+          .first(k)
+          .map(&:first)
+      end
+
+      def size
+        @entries.length
+      end
+
+      def empty?
+        @entries.empty?
+      end
+
+      private
+
+      def cosine_similarity(a, b)
+        dot = 0.0
+        norm_a = 0.0
+        norm_b = 0.0
+        a.each_index do |i|
+          ai = a[i].to_f
+          bi = (b[i] || 0).to_f
+          dot += ai * bi
+          norm_a += ai * ai
+          norm_b += bi * bi
+        end
+        return 0.0 if norm_a.zero? || norm_b.zero?
+
+        dot / (Math.sqrt(norm_a) * Math.sqrt(norm_b))
+      end
+    end
+  end
+end

data/lib/rcrewai/knowledge.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative 'knowledge/chunker'
+require_relative 'knowledge/store'
+require_relative 'knowledge/sources'
+require_relative 'knowledge/embedder'
+require_relative 'knowledge/base'
+
+module RCrewAI
+  # Retrieval-augmented knowledge for agents and crews. See Knowledge::Base.
+  module Knowledge
+  end
+end

data/lib/rcrewai/llm_client.rb

@@ -28,6 +28,29 @@ module RCrewAI
       end
     end
 
+    # Resolves a per-agent / per-pass LLM spec into a client.
+    #   nil            -> global provider
+    #   Symbol/String  -> that provider, global model
+    #   Hash           -> { provider:, model:, api_key:, temperature: } overrides
+    #   client object  -> returned as-is (anything responding to #chat)
+    def self.resolve(spec, config = RCrewAI.configuration)
+      case spec
+      when nil
+        for_provider(nil, config)
+      when Symbol, String
+        overridden = config.with_overrides(provider: spec)
+        for_provider(overridden.llm_provider, overridden)
+      when Hash
+        overridden = config.with_overrides(**spec)
+        for_provider(overridden.llm_provider, overridden)
+      else
+        return spec if spec.respond_to?(:chat)
+
+        raise ConfigurationError,
+              "Invalid llm: expected a provider symbol, an options hash, or a client responding to #chat, got #{spec.class}"
+      end
+    end
+
     def self.chat(messages:, **options)
       client = for_provider
       client.chat(messages: messages, **options)

data/lib/rcrewai/output_schema.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RCrewAI
+  # Validates and coerces a task's raw string output against a JSON-Schema
+  # subset (object / type / required / property types). Used by Task for the
+  # `output_schema:` option. Kept intentionally small: it covers the shapes an
+  # LLM is realistically asked to emit, not the whole JSON Schema spec.
+  module OutputSchema
+    module_function
+
+    # Returns the validated/coerced object.
+    # Raises OutputSchemaError if the string can't be parsed or doesn't conform.
+    def coerce(raw, schema)
+      data = parse(raw)
+      validate!(data, schema)
+      data
+    end
+
+    # Extracts a JSON document from a string that may contain surrounding prose,
+    # then parses it. Prefers a fenced ```json block, then the first balanced
+    # object/array, then the whole string.
+    def parse(raw)
+      candidate = extract_json(raw.to_s)
+      JSON.parse(candidate)
+    rescue JSON::ParserError => e
+      raise OutputSchemaError, "output is not valid JSON: #{e.message}"
+    end
+
+    def extract_json(text)
+      if (fenced = text[/```(?:json)?\s*(\{.*?\}|\[.*?\])\s*```/m, 1])
+        return fenced
+      end
+
+      first = text.index(/[{\[]/)
+      last  = text.rindex(/[}\]]/)
+      return text if first.nil? || last.nil? || last < first
+
+      text[first..last]
+    end
+
+    def validate!(data, schema)
+      type = (schema[:type] || schema['type'])&.to_s
+      case type
+      when 'object' then validate_object!(data, schema)
+      when 'array'  then raise_unless(data.is_a?(Array), 'expected an array')
+      when 'string' then raise_unless(data.is_a?(String), 'expected a string')
+      when 'integer' then raise_unless(data.is_a?(Integer), 'expected an integer')
+      when 'number' then raise_unless(data.is_a?(Numeric), 'expected a number')
+      when 'boolean' then raise_unless([true, false].include?(data), 'expected a boolean')
+      end
+      data
+    end
+
+    def validate_object!(data, schema)
+      raise_unless(data.is_a?(Hash), 'expected a JSON object')
+
+      required = schema[:required] || schema['required'] || []
+      required.each do |key|
+        raise_unless(data.key?(key.to_s), "missing required property '#{key}'")
+      end
+
+      props = schema[:properties] || schema['properties'] || {}
+      props.each do |name, subschema|
+        value = data[name.to_s]
+        next if value.nil?
+
+        validate!(value, subschema)
+      end
+    end
+
+    def raise_unless(condition, message)
+      raise OutputSchemaError, message unless condition
+    end
+  end
+
+  class OutputSchemaError < Error; end
+end

data/lib/rcrewai/planning.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'output_schema'
+
+module RCrewAI
+  # Runs a single planning pass over a crew's tasks before execution. Asks an
+  # LLM to draft a short, concrete plan for each task and folds that plan into
+  # the task's description, so the executing agent starts with a game plan.
+  #
+  # Mirrors CrewAI's `planning=True`. Best-effort: if the planner errors or
+  # returns unparseable output, execution proceeds with the original tasks.
+  class Planning
+    def initialize(crew, llm: nil, logger: nil)
+      @crew = crew
+      @llm = llm || LLMClient.for_provider
+      @logger = logger
+    end
+
+    def plan!
+      return if @crew.tasks.empty?
+
+      plans = request_plans
+      return if plans.nil? || plans.empty?
+
+      @crew.tasks.each do |task|
+        step = plans[task.name] || plans[task.name.to_s]
+        task.enrich_description("Plan: #{step}") if step
+      end
+    rescue StandardError => e
+      @logger&.warn("Planning pass failed, continuing without a plan: #{e.message}")
+      nil
+    end
+
+    private
+
+    def request_plans
+      response = @llm.chat(messages: [
+                             { role: 'system', content: system_prompt },
+                             { role: 'user', content: user_prompt }
+                           ])
+      content = response.is_a?(Hash) ? response[:content].to_s : response.to_s
+      parse_plans(content)
+    end
+
+    def parse_plans(content)
+      OutputSchema.parse(content)
+    rescue OutputSchemaError
+      nil
+    end
+
+    def system_prompt
+      'You are a planning assistant. Given a list of tasks, produce a short, ' \
+        'concrete plan for each. Respond ONLY with a JSON object mapping each ' \
+        'task name to a one-sentence plan string.'
+    end
+
+    def user_prompt
+      lines = @crew.tasks.map do |t|
+        "- #{t.name}: #{t.description} (expected: #{t.expected_output || 'n/a'})"
+      end
+      "Tasks:\n#{lines.join("\n")}\n\nReturn JSON: { \"<task name>\": \"<plan>\", ... }"
+    end
+  end
+end