phronomy 0.7.1 → 0.9.0

data/lib/phronomy/knowledge_source/rag_knowledge.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module KnowledgeSource
-    # A KnowledgeSource that retrieves semantically relevant chunks from a VectorStore.
-    #
-    # On each #fetch call, the query is embedded and the k nearest documents are
-    # returned as knowledge chunks.
-    #
-    # @example
-    #   store = Phronomy::VectorStore::InMemory.new
-    #   embeddings = Phronomy::Embeddings::RubyLLMEmbeddings.new(model: "text-embedding-3-small")
-    #   ks = Phronomy::KnowledgeSource::RAGKnowledge.new(
-    #     store: store,
-    #     embeddings: embeddings,
-    #     k: 5
-    #   )
-    class RAGKnowledge < Base
-      # @param store      [Phronomy::VectorStore::Base]    vector store holding documents
-      # @param embeddings [Phronomy::Embeddings::Base]     embeddings adapter
-      # @param k          [Integer]                        number of chunks to retrieve
-      # @param type       [Symbol]                         semantic tag (default :rag)
-      # @param source     [String, nil]                    default source label; falls back to
-      #   each document's :source metadata when nil
-      # @api public
-      def initialize(store:, embeddings:, k: 5, type: :rag, source: nil)
-        @store = store
-        @embeddings = embeddings
-        @k = k
-        @type = type
-        @source = source
-      end
-
-      # Embed the query and retrieve the k nearest chunks from the vector store.
-      #
-      # Returns an empty array when query is nil or blank.
-      #
-      # @param query              [String, nil]
-      # @param cancellation_token [Phronomy::CancellationToken, nil] optional; raises CancellationError when cancelled
-      # @return [Array<Hash>]
-      # @api public
-      def fetch(query: nil, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        return [] if query.nil? || query.strip.empty?
-
-        vector = @embeddings.embed(query, cancellation_token)
-        results = @store.search(query_embedding: vector, k: @k, cancellation_token: cancellation_token)
-        results.map do |doc|
-          chunk = {content: doc[:metadata][:content], type: @type}
-          src = @source || doc[:metadata][:source]
-          chunk[:source] = src if src
-          chunk
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/knowledge_source/static_knowledge.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module KnowledgeSource
-    # A KnowledgeSource backed by fixed text provided at construction time.
-    #
-    # Useful for injecting static documents, policy files, or configuration
-    # knowledge that does not change per request.
-    #
-    # @example
-    #   ks = Phronomy::KnowledgeSource::StaticKnowledge.new(
-    #     "Our refund policy: ...",
-    #     type: :policy
-    #   )
-    #   agent.invoke("What is the refund policy?", config: { knowledge_sources: [ks] })
-    class StaticKnowledge < Base
-      # @param text   [String] the static knowledge text to inject
-      # @param type   [Symbol] semantic tag for the chunk (default :static)
-      # @param source [String, nil] label identifying where this knowledge came from
-      #   (e.g. a filename). Included in the context XML tag and exposed to the LLM
-      #   so that agents can produce grounded citations.
-      # @api public
-      def initialize(text, type: :static, source: nil)
-        @text = text.to_s
-        @type = type
-        @source = source
-      end
-
-      # Returns the fixed text as a single chunk, regardless of query.
-      #
-      # @param query              [String, nil]                    ignored for static knowledge
-      # @param cancellation_token [Phronomy::CancellationToken, nil] optional; raises CancellationError when cancelled
-      # @return [Array<Hash>]
-      # @api public
-      def fetch(query: nil, cancellation_token: nil)
-        cancellation_token&.raise_if_cancelled!
-        return [] if @text.empty?
-
-        chunk = {content: @text, type: @type}
-        chunk[:source] = @source if @source
-        [chunk]
-      end
-
-      # Static knowledge content never changes between invocations.
-      # @return [true]
-      # @api public
-      def static?
-        true
-      end
-    end
-  end
-end

data/lib/phronomy/loader/base.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Loader
-    # Abstract base class for document loaders.
-    #
-    # A loader converts an external source (file path, URL, etc.) into an
-    # Array of document hashes understood by the rest of the pipeline:
-    #
-    #   [{ text: String, metadata: Hash }, ...]
-    #
-    # Subclasses must implement {#load}.
-    class Base
-      # Load documents from +source+ and return an array of document hashes.
-      #
-      # @param source [String] file path, URL, or other source identifier
-      # @return [Array<Hash>] array of <tt>{ text: String, metadata: Hash }</tt>
-      # @raise [NotImplementedError] when not overridden by a subclass
-      # @api public
-      def load(source)
-        raise NotImplementedError, "#{self.class}#load is not implemented"
-      end
-    end
-  end
-end

data/lib/phronomy/loader/csv_loader.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require "csv"
-
-module Phronomy
-  module Loader
-    # Loads a CSV file, converting each row into a separate document.
-    #
-    # By default the first row is treated as a header and column names are
-    # available in the document metadata.  The full row is serialised to
-    # a human-readable "key: value" string for embedding.
-    #
-    # @example
-    #   loader = Phronomy::Loader::CsvLoader.new
-    #   docs   = loader.load("products.csv")
-    #   # => [
-    #   #   { text: "name: Widget\nprice: 9.99", metadata: { source: "...", row: 1, name: "Widget", price: "9.99" } },
-    #   #   ...
-    #   # ]
-    class CsvLoader < Base
-      # @param headers     [Boolean] treat the first row as headers (default: true)
-      # @param text_column [String, nil] if set, use only this column as the document text
-      # @api public
-      def initialize(headers: true, text_column: nil)
-        @headers = headers
-        @text_column = text_column
-      end
-
-      # @param source [String] path to a CSV file
-      # @return [Array<Hash>]
-      # @raise [Errno::ENOENT] if the file does not exist
-      # @api public
-      def load(source)
-        rows = CSV.read(source, headers: @headers, encoding: "UTF-8")
-
-        if @headers
-          rows.each_with_index.map do |row, idx|
-            row_hash = row.to_h
-            text = if @text_column
-              row_hash[@text_column].to_s
-            else
-              row_hash.map { |k, v| "#{k}: #{v}" }.join("\n")
-            end
-            metadata = row_hash.transform_keys(&:to_sym).merge(source: source, row: idx + 1)
-            {text: text, metadata: metadata}
-          end
-        else
-          rows.each_with_index.map do |row, idx|
-            text = row.join(", ")
-            {text: text, metadata: {source: source, row: idx + 1}}
-          end
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/loader/markdown_loader.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Loader
-    # Loads a Markdown file, optionally splitting on top-level headings.
-    #
-    # When +split_on_headings:+ is true (the default), each H1/H2 section
-    # becomes a separate document so that embeddings capture section semantics
-    # rather than the full file at once.
-    #
-    # @example Single document (heading split disabled)
-    #   loader = Phronomy::Loader::MarkdownLoader.new(split_on_headings: false)
-    #   docs   = loader.load("README.md")
-    #   # => [{ text: "# Title\n...", metadata: { source: "README.md" } }]
-    #
-    # @example Split per heading (default)
-    #   loader = Phronomy::Loader::MarkdownLoader.new
-    #   docs   = loader.load("guide.md")
-    #   # => [
-    #   #   { text: "# Section 1\n...", metadata: { source: "guide.md", section: "Section 1" } },
-    #   #   { text: "## Sub-section\n...", metadata: { source: "guide.md", section: "Sub-section" } },
-    #   # ]
-    class MarkdownLoader < Base
-      HEADING_RE = /^(\#{1,6})\s+(.+)$/
-
-      # @param split_on_headings [Boolean] split on H1–H6 boundaries (default: true)
-      # @api public
-      def initialize(split_on_headings: true)
-        @split_on_headings = split_on_headings
-      end
-
-      # @param source [String] path to a Markdown file
-      # @return [Array<Hash>]
-      # @raise [Errno::ENOENT] if the file does not exist
-      # @api public
-      def load(source)
-        content = File.read(source, encoding: "UTF-8")
-        return [{text: content, metadata: {source: source}}] unless @split_on_headings
-
-        split_by_headings(content, source)
-      end
-
-      private
-
-      def split_by_headings(content, source)
-        sections = []
-        current_lines = []
-        current_heading = nil
-
-        content.each_line do |line|
-          if (m = HEADING_RE.match(line.chomp))
-            flush_section(sections, current_lines, current_heading, source) if current_lines.any?
-            current_heading = m[2].strip
-            current_lines = [line]
-          else
-            current_lines << line
-          end
-        end
-
-        flush_section(sections, current_lines, current_heading, source) if current_lines.any?
-
-        # Fall back to single document if no headings were found
-        sections.empty? ? [{text: content, metadata: {source: source}}] : sections
-      end
-
-      def flush_section(sections, lines, heading, source)
-        text = lines.join
-        return if text.strip.empty?
-
-        metadata = {source: source}
-        metadata[:section] = heading if heading
-        sections << {text: text, metadata: metadata}
-      end
-    end
-  end
-end

data/lib/phronomy/loader/plain_text_loader.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Loader
-    # Loads a plain-text file as a single document.
-    #
-    # @example
-    #   loader = Phronomy::Loader::PlainTextLoader.new
-    #   docs   = loader.load("/path/to/file.txt")
-    #   # => [{ text: "...", metadata: { source: "/path/to/file.txt" } }]
-    class PlainTextLoader < Base
-      # @param source [String] absolute or relative path to a text file
-      # @return [Array<Hash>] single-element array with the file contents
-      # @raise [Errno::ENOENT] if the file does not exist
-      # @api public
-      def load(source)
-        text = File.read(source, encoding: "UTF-8")
-        [{text: text, metadata: {source: source}}]
-      end
-    end
-  end
-end

data/lib/phronomy/loader.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  # Document loader implementations for ingesting files into a RAG pipeline.
-  #
-  # Sub-classes are auto-loaded by Zeitwerk:
-  #   Phronomy::Loader::Base
-  #   Phronomy::Loader::PlainTextLoader
-  #   Phronomy::Loader::MarkdownLoader
-  #   Phronomy::Loader::CsvLoader
-  module Loader
-  end
-end

data/lib/phronomy/prompt_template.rb

@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  # A prompt template that substitutes {{variable}} placeholders in a string.
-  #
-  # @example Simple human template
-  #   t = Phronomy::PromptTemplate.new(template: "Translate to {{lang}}: {{text}}")
-  #   t.format(lang: "French", text: "Hello")
-  #   # => "Translate to French: Hello"
-  #
-  # @example With a system template
-  #   t = Phronomy::PromptTemplate.new(
-  #     template: "{{question}}",
-  #     system_template: "You are a {{role}} assistant."
-  #   )
-  #   t.format_system(role: "helpful")
-  #   # => "You are a helpful assistant."
-  #
-  # As a Runnable, #invoke accepts a Hash of variables and returns a Hash
-  # with :prompt (and optionally :system) keys.
-  class PromptTemplate
-    include Phronomy::Runnable
-
-    PLACEHOLDER = /\{\{(\w+)\}\}/
-
-    attr_reader :template, :system_template
-
-    # @param template        [String] human message template with {{var}} placeholders
-    # @param system_template [String, nil] optional system message template
-    # @api public
-    def initialize(template:, system_template: nil)
-      @template = template
-      @system_template = system_template
-    end
-
-    # Substitute all {{var}} placeholders in the human template.
-    #
-    # @param variables [Hash{Symbol => String}]
-    # @return [String]
-    # @api public
-    def format(**variables)
-      substitute(@template, variables)
-    end
-
-    # Substitute all {{var}} placeholders in the system template.
-    # Returns nil when no system template was set.
-    #
-    # @param variables [Hash{Symbol => String}]
-    # @return [String, nil]
-    # @api public
-    def format_system(**variables)
-      @system_template && substitute(@system_template, variables)
-    end
-
-    # Runnable interface: accepts a Hash of variable values.
-    # Returns { prompt: String, system: String|nil }.
-    #
-    # @param input  [Hash{Symbol => String}]
-    # @return [Hash]
-    # @api public
-    def invoke(input, config: {})
-      vars = normalize_input(input)
-      result = {prompt: format(**vars)}
-      sys = format_system(**vars)
-      result[:system] = sys if sys
-      result
-    end
-
-    # Returns the list of placeholder names found in both templates.
-    #
-    # @return [Array<Symbol>]
-    # @api public
-    def variables
-      names = @template.scan(PLACEHOLDER).flatten
-      names += @system_template.scan(PLACEHOLDER).flatten if @system_template
-      names.map(&:to_sym).uniq
-    end
-
-    private
-
-    def substitute(text, variables)
-      text.gsub(PLACEHOLDER) do |match|
-        key = Regexp.last_match(1).to_sym
-        variables.fetch(key) { raise KeyError, "Missing variable: {{#{key}}}" }
-      end
-    end
-
-    def normalize_input(input)
-      case input
-      when Hash then input
-      when String then {input: input}
-      else raise ArgumentError, "PromptTemplate#invoke expects a Hash of variables, got #{input.class}"
-      end
-    end
-  end
-end

data/lib/phronomy/splitter/base.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Splitter
-    # Abstract base class for text splitters.
-    #
-    # A splitter takes a single document hash (or plain text) and returns an
-    # array of smaller chunk documents:
-    #
-    #   [{ text: String, metadata: Hash }, ...]
-    #
-    # Subclasses must implement {#split}.
-    class Base
-      # Split +document+ into an array of chunk documents.
-      #
-      # @param document [Hash, String]
-      #   Either a document hash (<tt>{ text: String, metadata: Hash }</tt>)
-      #   returned by a Loader, or a plain String.
-      # @return [Array<Hash>] array of <tt>{ text: String, metadata: Hash }</tt>
-      # @raise [NotImplementedError] when not overridden by a subclass
-      # @api public
-      def split(document)
-        raise NotImplementedError, "#{self.class}#split is not implemented"
-      end
-
-      # Convenience method: split an array of documents.
-      #
-      # @param documents [Array<Hash, String>]
-      # @return [Array<Hash>]
-      # @api public
-      def split_all(documents)
-        documents.flat_map { |doc| split(doc) }
-      end
-
-      private
-
-      # Normalise a document-or-string argument into {text:, metadata:}.
-      def normalise(document)
-        case document
-        when Hash then {text: document[:text].to_s, metadata: document.fetch(:metadata, {})}
-        when String then {text: document, metadata: {}}
-        else raise ArgumentError, "document must be a Hash or String, got #{document.class}"
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/splitter/fixed_size_splitter.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Splitter
-    # Splits text into fixed-size character chunks with optional overlap.
-    #
-    # @example
-    #   splitter = Phronomy::Splitter::FixedSizeSplitter.new(chunk_size: 200, chunk_overlap: 20)
-    #   chunks   = splitter.split({ text: long_text, metadata: { source: "doc.txt" } })
-    #   # => [
-    #   #   { text: "...(200 chars)...", metadata: { source: "doc.txt", chunk: 0 } },
-    #   #   { text: "...(200 chars, 20-char overlap)...", metadata: { source: "doc.txt", chunk: 1 } },
-    #   # ]
-    class FixedSizeSplitter < Base
-      # @param chunk_size    [Integer] maximum characters per chunk (default: 1000)
-      # @param chunk_overlap [Integer] characters to repeat at the start of each
-      #   subsequent chunk (default: 200); must be less than chunk_size
-      # @api public
-      def initialize(chunk_size: 1000, chunk_overlap: 200)
-        raise ArgumentError, "chunk_overlap must be less than chunk_size" if chunk_overlap >= chunk_size
-
-        @chunk_size = chunk_size
-        @chunk_overlap = chunk_overlap
-      end
-
-      # @param document [Hash, String]
-      # @return [Array<Hash>]
-      # @api public
-      def split(document)
-        doc = normalise(document)
-        text = doc[:text]
-        base_metadata = doc[:metadata]
-
-        chunks = []
-        start = 0
-        index = 0
-
-        while start < text.length
-          chunk_text = text[start, @chunk_size]
-          chunks << {text: chunk_text, metadata: base_metadata.merge(chunk: index)}
-          break if start + @chunk_size >= text.length
-
-          start += @chunk_size - @chunk_overlap
-          index += 1
-        end
-
-        chunks
-      end
-    end
-  end
-end

data/lib/phronomy/splitter/recursive_splitter.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Splitter
-    # Splits text recursively using a prioritised list of separator strings.
-    #
-    # The splitter tries each separator in order.  When a separator produces
-    # chunks that are still larger than +chunk_size+, it recurses with the
-    # next separator in the list.  This mirrors LangChain's
-    # RecursiveCharacterTextSplitter behaviour.
-    #
-    # Default separators (in priority order):
-    #   1. "\n\n"  — paragraph breaks
-    #   2. "\n"    — line breaks
-    #   3. ". "    — sentence boundaries
-    #   4. " "     — word boundaries
-    #   5. ""      — character-level fallback
-    #
-    # @example
-    #   splitter = Phronomy::Splitter::RecursiveSplitter.new(chunk_size: 300, chunk_overlap: 30)
-    #   chunks   = splitter.split({ text: long_markdown, metadata: { source: "guide.md" } })
-    class RecursiveSplitter < Base
-      DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""].freeze
-
-      # @param chunk_size    [Integer] maximum characters per chunk (default: 1000)
-      # @param chunk_overlap [Integer] overlap characters (default: 200)
-      # @param separators    [Array<String>] separator list in priority order
-      # @api public
-      def initialize(chunk_size: 1000, chunk_overlap: 200, separators: DEFAULT_SEPARATORS)
-        raise ArgumentError, "chunk_overlap must be less than chunk_size" if chunk_overlap >= chunk_size
-
-        @chunk_size = chunk_size
-        @chunk_overlap = chunk_overlap
-        @separators = separators
-      end
-
-      # @param document [Hash, String]
-      # @return [Array<Hash>]
-      # @api public
-      def split(document)
-        doc = normalise(document)
-        texts = recursive_split(doc[:text], @separators)
-        merge_with_overlap(texts).each_with_index.map do |text, idx|
-          {text: text, metadata: doc[:metadata].merge(chunk: idx)}
-        end
-      end
-
-      private
-
-      # Split +text+ using the first separator that yields non-trivial pieces,
-      # then recurse on any piece that is still too large.
-      def recursive_split(text, separators)
-        return [text] if text.length <= @chunk_size || separators.empty?
-
-        sep, *rest_seps = separators
-
-        # Character-level fallback: just slice
-        if sep == ""
-          return FixedSizeSplitter
-              .new(chunk_size: @chunk_size, chunk_overlap: @chunk_overlap)
-              .split(text)
-              .map { |c| c[:text] }
-        end
-
-        parts = text.split(sep)
-
-        # If this separator doesn't split, try the next
-        return recursive_split(text, rest_seps) if parts.length <= 1
-
-        # Re-attach the separator to each part except the last so context is preserved
-        parts_with_sep = parts.each_with_index.map do |part, i|
-          (i < parts.length - 1) ? part + sep : part
-        end
-
-        parts_with_sep.flat_map do |part|
-          if part.length > @chunk_size
-            recursive_split(part, rest_seps)
-          else
-            [part]
-          end
-        end.reject { |t| t.strip.empty? }
-      end
-
-      # Merge small adjacent pieces and apply overlap between chunks.
-      def merge_with_overlap(texts)
-        merged = []
-        current = +""
-
-        texts.each do |text|
-          if current.length + text.length <= @chunk_size
-            current << text
-          else
-            merged << current.strip unless current.strip.empty?
-            # Start next chunk with overlap from the end of current
-            overlap_text = (current.length > @chunk_overlap) ? current[-@chunk_overlap..] : current
-            current = overlap_text + text
-          end
-        end
-
-        merged << current.strip unless current.strip.empty?
-        merged
-      end
-    end
-  end
-end

data/lib/phronomy/splitter.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  # Text splitter implementations for chunking documents before embedding.
-  #
-  # Sub-classes are auto-loaded by Zeitwerk:
-  #   Phronomy::Splitter::Base
-  #   Phronomy::Splitter::FixedSizeSplitter
-  #   Phronomy::Splitter::RecursiveSplitter
-  module Splitter
-  end
-end