phronomy 0.7.0 → 0.8.0

data/lib/phronomy/agent/context/knowledge/loader/markdown_loader.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Loader
+          # Loads a Markdown file, optionally splitting on top-level headings.
+          #
+          # When +split_on_headings:+ is true (the default), each H1/H2 section
+          # becomes a separate document so that embeddings capture section semantics
+          # rather than the full file at once.
+          #
+          # @example Single document (heading split disabled)
+          #   loader = Phronomy::Agent::Context::Knowledge::Loader::MarkdownLoader.new(split_on_headings: false)
+          #   docs   = loader.load("README.md")
+          #   # => [{ text: "# Title\n...", metadata: { source: "README.md" } }]
+          #
+          # @example Split per heading (default)
+          #   loader = Phronomy::Agent::Context::Knowledge::Loader::MarkdownLoader.new
+          #   docs   = loader.load("guide.md")
+          #   # => [
+          #   #   { text: "# Section 1\n...", metadata: { source: "guide.md", section: "Section 1" } },
+          #   #   { text: "## Sub-section\n...", metadata: { source: "guide.md", section: "Sub-section" } },
+          #   # ]
+          class MarkdownLoader < Base
+            HEADING_RE = /^(\#{1,6})\s+(.+)$/
+
+            # @param split_on_headings [Boolean] split on H1–H6 boundaries (default: true)
+            # @api public
+            def initialize(split_on_headings: true)
+              @split_on_headings = split_on_headings
+            end
+
+            # @param source [String] path to a Markdown file
+            # @return [Array<Hash>]
+            # @raise [Errno::ENOENT] if the file does not exist
+            # @api public
+            def load(source)
+              content = File.read(source, encoding: "UTF-8")
+              return [{text: content, metadata: {source: source}}] unless @split_on_headings
+
+              split_by_headings(content, source)
+            end
+
+            private
+
+            def split_by_headings(content, source)
+              sections = []
+              current_lines = []
+              current_heading = nil
+
+              content.each_line do |line|
+                if (m = HEADING_RE.match(line.chomp))
+                  flush_section(sections, current_lines, current_heading, source) if current_lines.any?
+                  current_heading = m[2].strip
+                  current_lines = [line]
+                else
+                  current_lines << line
+                end
+              end
+
+              flush_section(sections, current_lines, current_heading, source) if current_lines.any?
+
+              # Fall back to single document if no headings were found
+              sections.empty? ? [{text: content, metadata: {source: source}}] : sections
+            end
+
+            def flush_section(sections, lines, heading, source)
+              text = lines.join
+              return if text.strip.empty?
+
+              metadata = {source: source}
+              metadata[:section] = heading if heading
+              sections << {text: text, metadata: metadata}
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/loader/plain_text_loader.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Loader
+          # Loads a plain-text file as a single document.
+          #
+          # @example
+          #   loader = Phronomy::Agent::Context::Knowledge::Loader::PlainTextLoader.new
+          #   docs   = loader.load("/path/to/file.txt")
+          #   # => [{ text: "...", metadata: { source: "/path/to/file.txt" } }]
+          class PlainTextLoader < Base
+            # @param source [String] absolute or relative path to a text file
+            # @return [Array<Hash>] single-element array with the file contents
+            # @raise [Errno::ENOENT] if the file does not exist
+            # @api public
+            def load(source)
+              text = File.read(source, encoding: "UTF-8")
+              [{text: text, metadata: {source: source}}]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/source/base.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Source
+          # Abstract base class for all KnowledgeSource implementations.
+          #
+          # Subclasses must implement #fetch(query:) and return an Array of chunk Hashes.
+          # Each chunk Hash must contain:
+          #   :content [String]  the text to inject into the context
+          #   :type    [Symbol]  semantic tag (e.g. :static, :rag, :entity)
+          class Base
+            # Retrieve knowledge chunks relevant to the given query.
+            #
+            # @param query              [String, nil]                    the current user input used to select relevant chunks
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil] optional token; raises CancellationError when cancelled
+            # @return [Array<Hash>] array of { content: String, type: Symbol }
+            # @api public
+            def fetch(query: nil, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              raise NotImplementedError, "#{self.class}#fetch is not implemented"
+            end
+
+            # Submits a {#fetch} call to {BlockingAdapterPool} and returns a
+            # {BlockingAdapterPool::PendingOperation}.
+            # Callers can fan out multiple fetches in parallel and await them all.
+            #
+            # @param query              [String, nil]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
+            # @param timeout            [Numeric, nil] seconds before the operation is abandoned
+            # @return [BlockingAdapterPool::PendingOperation]
+            # @api public
+            def fetch_async(query: nil, cancellation_token: nil, timeout: nil)
+              Phronomy::Runtime.instance.blocking_io.submit(
+                timeout: timeout,
+                cancellation_token: cancellation_token
+              ) do
+                fetch(query: query, cancellation_token: cancellation_token)
+              end
+            end
+
+            # Returns true when this source's content is considered static (i.e. does
+            # not change between agent invocations). Static sources are eligible for
+            # fingerprint-based caching in ContextVersionCache.
+            #
+            # Override in subclasses that return fixed content.
+            #
+            # @return [Boolean]
+            # @api public
+            def static?
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/source/entity_knowledge.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Source
+          # A KnowledgeSource that extracts named-entity facts from conversation history.
+          #
+          # This is the knowledge-injection counterpart of the old EntityMemory.
+          # It scans saved user messages with a regex heuristic (no LLM call) and
+          # returns the discovered facts as a single knowledge chunk tagged :entity.
+          #
+          # EntityKnowledge is stateful: it accumulates extracted facts via #update(messages:)
+          # which should be called each time new messages are saved.
+          #
+          # Supported extraction patterns (case-insensitive):
+          #   "my name is Alice"          → { name: "Alice" }
+          #   "I am Alice"                → { identity: "Alice" }
+          #   "I'm a software engineer"   → { occupation: "software engineer" }
+          #   "I work at / for Acme"      → { workplace: "Acme" }
+          #   "I live in Tokyo"           → { location: "Tokyo" }
+          #   "I'm from Tokyo"            → { location: "Tokyo" }
+          #   "I like / love Ruby"        → { preference: "Ruby" }
+          #
+          # @example
+          #   ks = Phronomy::Agent::Context::Knowledge::Source::EntityKnowledge.new
+          #   ks.update(messages: chat_messages)
+          #   agent.invoke("What is my name?", config: { knowledge_sources: [ks] })
+          class EntityKnowledge < Base
+            PATTERNS = [
+              [:name, /\bmy name is\s+([A-Za-z][A-Za-z0-9 \-']*)/i],
+              [:identity, /\bI\s+am\s+([A-Z][A-Za-z0-9 \-']+)/],
+              [:occupation, /\bI(?:'m| am) a(?:n)?\s+([A-Za-z][A-Za-z0-9 \-']*)/i],
+              [:workplace, /\bI (?:work|worked) (?:at|for|in)\s+([A-Za-z0-9][A-Za-z0-9 \-'.&,]*)/i],
+              [:location, /\bI live in\s+([A-Za-z][A-Za-z0-9 \-']*)/i],
+              [:location, /\bI(?:'m| am) from\s+([A-Za-z][A-Za-z0-9 \-']*)/i],
+              [:preference, /\bI (?:like|love|enjoy)\s+([A-Za-z][A-Za-z0-9 \-']*)/i]
+            ].freeze
+
+            def initialize
+              @entities = {}
+            end
+
+            # Scan messages and accumulate entity facts.
+            # Call this after saving a new set of messages (e.g. from a ConversationManager save hook).
+            #
+            # @param messages [Array] message objects responding to #role and #content
+            # @api public
+            def update(messages:)
+              messages.each do |msg|
+                next unless msg.role.to_sym == :user
+
+                extract(msg.content.to_s).each { |key, value| @entities[key] = value }
+              end
+            end
+
+            # Returns a single chunk containing all known entity facts in XML context format.
+            # Returns an empty array when no entities have been discovered.
+            #
+            # @param query              [String, nil]                    unused — entity knowledge is always fully injected
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil] optional; raises CancellationError when cancelled
+            # @return [Array<Hash>]
+            # @api public
+            def fetch(query: nil, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              return [] if @entities.empty?
+
+              lines = @entities.map { |key, value| "- #{key}: #{value}" }.join("\n")
+              content = <<~CONTENT.chomp
+                Known facts about the user:
+                #{lines}
+              CONTENT
+              [{content: content, type: :entity}]
+            end
+
+            # Returns the current entity store (primarily for testing).
+            #
+            # @return [Hash]
+            # @api public
+            def entities
+              @entities.dup
+            end
+
+            private
+
+            def extract(text)
+              found = {}
+              PATTERNS.each do |key, pattern|
+                if (match = text.match(pattern))
+                  value = match[1].strip.sub(/[.!?]\s+.*$/, "").gsub(/[.,;!?]+$/, "")
+                  found[key] = value unless value.empty?
+                end
+              end
+              found
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/source/rag_knowledge.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Source
+          # A KnowledgeSource that retrieves semantically relevant chunks from a VectorStore.
+          #
+          # On each #fetch call, the query is embedded and the k nearest documents are
+          # returned as knowledge chunks.
+          #
+          # @example
+          #   store = Phronomy::Agent::Context::Knowledge::VectorStore::InMemory.new
+          #   embeddings = Phronomy::Agent::Context::Knowledge::Embeddings::RubyLLMEmbeddings.new(model: "text-embedding-3-small")
+          #   ks = Phronomy::Agent::Context::Knowledge::Source::RAGKnowledge.new(
+          #     store: store,
+          #     embeddings: embeddings,
+          #     k: 5
+          #   )
+          class RAGKnowledge < Base
+            # @param store      [Phronomy::Agent::Context::Knowledge::VectorStore::Base]    vector store holding documents
+            # @param embeddings [Phronomy::Agent::Context::Knowledge::Embeddings::Base]     embeddings adapter
+            # @param k          [Integer]                        number of chunks to retrieve
+            # @param type       [Symbol]                         semantic tag (default :rag)
+            # @param source     [String, nil]                    default source label; falls back to
+            #   each document's :source metadata when nil
+            # @api public
+            def initialize(store:, embeddings:, k: 5, type: :rag, source: nil)
+              @store = store
+              @embeddings = embeddings
+              @k = k
+              @type = type
+              @source = source
+            end
+
+            # Embed the query and retrieve the k nearest chunks from the vector store.
+            #
+            # Returns an empty array when query is nil or blank.
+            #
+            # @param query              [String, nil]
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil] optional; raises CancellationError when cancelled
+            # @return [Array<Hash>]
+            # @api public
+            def fetch(query: nil, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              return [] if query.nil? || query.strip.empty?
+
+              vector = @embeddings.embed(query, cancellation_token)
+              results = @store.search(query_embedding: vector, k: @k, cancellation_token: cancellation_token)
+              results.map do |doc|
+                chunk = {content: doc[:metadata][:content], type: @type}
+                src = @source || doc[:metadata][:source]
+                chunk[:source] = src if src
+                chunk
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/source/static_knowledge.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Source
+          # A KnowledgeSource backed by fixed text provided at construction time.
+          #
+          # Useful for injecting static documents, policy files, or configuration
+          # knowledge that does not change per request.
+          #
+          # @example
+          #   ks = Phronomy::Agent::Context::Knowledge::Source::StaticKnowledge.new(
+          #     "Our refund policy: ...",
+          #     type: :policy
+          #   )
+          #   agent.invoke("What is the refund policy?", config: { knowledge_sources: [ks] })
+          class StaticKnowledge < Base
+            # @param text   [String] the static knowledge text to inject
+            # @param type   [Symbol] semantic tag for the chunk (default :static)
+            # @param source [String, nil] label identifying where this knowledge came from
+            #   (e.g. a filename). Included in the context XML tag and exposed to the LLM
+            #   so that agents can produce grounded citations.
+            # @api public
+            def initialize(text, type: :static, source: nil)
+              @text = text.to_s
+              @type = type
+              @source = source
+            end
+
+            # Returns the fixed text as a single chunk, regardless of query.
+            #
+            # @param query              [String, nil]                    ignored for static knowledge
+            # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil] optional; raises CancellationError when cancelled
+            # @return [Array<Hash>]
+            # @api public
+            def fetch(query: nil, cancellation_token: nil)
+              cancellation_token&.raise_if_cancelled!
+              return [] if @text.empty?
+
+              chunk = {content: @text, type: @type}
+              chunk[:source] = @source if @source
+              [chunk]
+            end
+
+            # Static knowledge content never changes between invocations.
+            # @return [true]
+            # @api public
+            def static?
+              true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/splitter/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Splitter
+          # Abstract base class for text splitters.
+          #
+          # A splitter takes a single document hash (or plain text) and returns an
+          # array of smaller chunk documents:
+          #
+          #   [{ text: String, metadata: Hash }, ...]
+          #
+          # Subclasses must implement {#split}.
+          class Base
+            # Split +document+ into an array of chunk documents.
+            #
+            # @param document [Hash, String]
+            #   Either a document hash (<tt>{ text: String, metadata: Hash }</tt>)
+            #   returned by a Loader, or a plain String.
+            # @return [Array<Hash>] array of <tt>{ text: String, metadata: Hash }</tt>
+            # @raise [NotImplementedError] when not overridden by a subclass
+            # @api public
+            def split(document)
+              raise NotImplementedError, "#{self.class}#split is not implemented"
+            end
+
+            # Convenience method: split an array of documents.
+            #
+            # @param documents [Array<Hash, String>]
+            # @return [Array<Hash>]
+            # @api public
+            def split_all(documents)
+              documents.flat_map { |doc| split(doc) }
+            end
+
+            private
+
+            # Normalise a document-or-string argument into {text:, metadata:}.
+            def normalise(document)
+              case document
+              when Hash then {text: document[:text].to_s, metadata: document.fetch(:metadata, {})}
+              when String then {text: document, metadata: {}}
+              else raise ArgumentError, "document must be a Hash or String, got #{document.class}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/splitter/fixed_size_splitter.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Splitter
+          # Splits text into fixed-size character chunks with optional overlap.
+          #
+          # @example
+          #   splitter = Phronomy::Agent::Context::Knowledge::Splitter::FixedSizeSplitter.new(chunk_size: 200, chunk_overlap: 20)
+          #   chunks   = splitter.split({ text: long_text, metadata: { source: "doc.txt" } })
+          #   # => [
+          #   #   { text: "...(200 chars)...", metadata: { source: "doc.txt", chunk: 0 } },
+          #   #   { text: "...(200 chars, 20-char overlap)...", metadata: { source: "doc.txt", chunk: 1 } },
+          #   # ]
+          class FixedSizeSplitter < Base
+            # @param chunk_size    [Integer] maximum characters per chunk (default: 1000)
+            # @param chunk_overlap [Integer] characters to repeat at the start of each
+            #   subsequent chunk (default: 200); must be less than chunk_size
+            # @api public
+            def initialize(chunk_size: 1000, chunk_overlap: 200)
+              raise ArgumentError, "chunk_overlap must be less than chunk_size" if chunk_overlap >= chunk_size
+
+              @chunk_size = chunk_size
+              @chunk_overlap = chunk_overlap
+            end
+
+            # @param document [Hash, String]
+            # @return [Array<Hash>]
+            # @api public
+            def split(document)
+              doc = normalise(document)
+              text = doc[:text]
+              base_metadata = doc[:metadata]
+
+              chunks = []
+              start = 0
+              index = 0
+
+              while start < text.length
+                chunk_text = text[start, @chunk_size]
+                chunks << {text: chunk_text, metadata: base_metadata.merge(chunk: index)}
+                break if start + @chunk_size >= text.length
+
+                start += @chunk_size - @chunk_overlap
+                index += 1
+              end
+
+              chunks
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/agent/context/knowledge/splitter/recursive_splitter.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    module Context
+      module Knowledge
+        module Splitter
+          # Splits text recursively using a prioritised list of separator strings.
+          #
+          # The splitter tries each separator in order.  When a separator produces
+          # chunks that are still larger than +chunk_size+, it recurses with the
+          # next separator in the list.  This mirrors LangChain's
+          # RecursiveCharacterTextSplitter behaviour.
+          #
+          # Default separators (in priority order):
+          #   1. "\n\n"  — paragraph breaks
+          #   2. "\n"    — line breaks
+          #   3. ". "    — sentence boundaries
+          #   4. " "     — word boundaries
+          #   5. ""      — character-level fallback
+          #
+          # @example
+          #   splitter = Phronomy::Agent::Context::Knowledge::Splitter::RecursiveSplitter.new(chunk_size: 300, chunk_overlap: 30)
+          #   chunks   = splitter.split({ text: long_markdown, metadata: { source: "guide.md" } })
+          class RecursiveSplitter < Base
+            DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""].freeze
+
+            # @param chunk_size    [Integer] maximum characters per chunk (default: 1000)
+            # @param chunk_overlap [Integer] overlap characters (default: 200)
+            # @param separators    [Array<String>] separator list in priority order
+            # @api public
+            def initialize(chunk_size: 1000, chunk_overlap: 200, separators: DEFAULT_SEPARATORS)
+              raise ArgumentError, "chunk_overlap must be less than chunk_size" if chunk_overlap >= chunk_size
+
+              @chunk_size = chunk_size
+              @chunk_overlap = chunk_overlap
+              @separators = separators
+            end
+
+            # @param document [Hash, String]
+            # @return [Array<Hash>]
+            # @api public
+            def split(document)
+              doc = normalise(document)
+              texts = recursive_split(doc[:text], @separators)
+              merge_with_overlap(texts).each_with_index.map do |text, idx|
+                {text: text, metadata: doc[:metadata].merge(chunk: idx)}
+              end
+            end
+
+            private
+
+            # Split +text+ using the first separator that yields non-trivial pieces,
+            # then recurse on any piece that is still too large.
+            def recursive_split(text, separators)
+              return [text] if text.length <= @chunk_size || separators.empty?
+
+              sep, *rest_seps = separators
+
+              # Character-level fallback: just slice
+              if sep == ""
+                return FixedSizeSplitter
+                    .new(chunk_size: @chunk_size, chunk_overlap: @chunk_overlap)
+                    .split(text)
+                    .map { |c| c[:text] }
+              end
+
+              parts = text.split(sep)
+
+              # If this separator doesn't split, try the next
+              return recursive_split(text, rest_seps) if parts.length <= 1
+
+              # Re-attach the separator to each part except the last so context is preserved
+              parts_with_sep = parts.each_with_index.map do |part, i|
+                (i < parts.length - 1) ? part + sep : part
+              end
+
+              parts_with_sep.flat_map do |part|
+                if part.length > @chunk_size
+                  recursive_split(part, rest_seps)
+                else
+                  [part]
+                end
+              end.reject { |t| t.strip.empty? }
+            end
+
+            # Merge small adjacent pieces and apply overlap between chunks.
+            def merge_with_overlap(texts)
+              merged = []
+              current = +""
+
+              texts.each do |text|
+                if current.length + text.length <= @chunk_size
+                  current << text
+                else
+                  merged << current.strip unless current.strip.empty?
+                  # Start next chunk with overlap from the end of current
+                  overlap_text = (current.length > @chunk_overlap) ? current[-@chunk_overlap..] : current
+                  current = overlap_text + text
+                end
+              end
+
+              merged << current.strip unless current.strip.empty?
+              merged
+            end
+          end
+        end
+      end
+    end
+  end
+end