robot_lab-document_store 0.1.0 → 0.1.1

data/lib/robot_lab/document_store.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-require "fastembed"
-require_relative "document_store/version"
+require_relative 'document_store/version'
 
 module RobotLab
   # Embedding-based document store for semantic search over arbitrary text.
@@ -13,6 +12,11 @@ module RobotLab
   # The embedding model is initialised lazily on first use — the ONNX model
   # file is downloaded on that first call (cached locally afterwards).
   #
+  # When fastembed is not installed, DocumentStore falls back to a lightweight
+  # TF-IDF word-frequency embedder. The fallback is lower quality (no semantic
+  # understanding, only lexical overlap) but works offline with no downloads,
+  # making it suitable for development and testing.
+  #
   # @example Standalone
   #   store = RobotLab::DocumentStore.new
   #   store.store(:q4_report, "Q4 revenue came in at $4.2M, up 18% YoY…")
@@ -26,15 +30,26 @@ module RobotLab
   #   memory.search_documents("how to configure redis", limit: 3)
   #
   class DocumentStore
+    # @api private
+    FASTEMBED_AVAILABLE = begin
+      require 'fastembed'
+      true
+      # :nocov:
+    rescue LoadError
+      false
+      # :nocov:
+    end
+
     # Default embedding model used when none is specified.
-    DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
+    DEFAULT_MODEL = 'BAAI/bge-small-en-v1.5'
 
-    # @param model_name [String] fastembed model name
+    # @param model_name [String] fastembed model name (ignored when fastembed unavailable)
     def initialize(model_name: DEFAULT_MODEL)
-      @model_name = model_name
-      @documents  = {}  # key (Symbol) => { text: String, vector: Array<Float> }
-      @mutex      = Mutex.new
-      @model      = nil  # lazy: initialised on first embed call
+      @model_name      = model_name
+      @documents       = {} # key (Symbol) => { text: String, vector: Array<Float> }
+      @mutex           = Mutex.new
+      @fastembed_model = nil # lazy: initialised on first embed call
+      @using_fastembed = FASTEMBED_AVAILABLE
     end
 
     # Embed +text+ and store it under +key+.
@@ -106,39 +121,117 @@ module RobotLab
       self
     end
 
+    STOP_WORDS = %w[
+      a an the is are was were be been being am do does did
+      to of in and or but for with on at by from as into
+      it its this that these those i you he she we they
+      not no nor so yet
+    ].to_set.freeze
+
     private
 
-    def model
-      @model ||= Fastembed::TextEmbedding.new(model_name: @model_name, show_progress: false)
+    # ── Fastembed path ──────────────────────────────────────────────────────
+
+    def fastembed_model
+      @fastembed_model ||= Fastembed::TextEmbedding.new(model_name: @model_name, show_progress: false)
     end
 
     def passage_vector(text)
-      model.passage_embed([text]).to_a.first
+      if @using_fastembed
+        fastembed_model.passage_embed([text]).to_a.first
+      else
+        fallback_vector(text)
+      end
     end
 
     def query_vector(text)
-      model.query_embed([text]).to_a.first
+      if @using_fastembed
+        fastembed_model.query_embed([text]).to_a.first
+      else
+        fallback_vector(text)
+      end
     end
 
     def cosine_similarity(vec_a, vec_b)
       return 0.0 unless vec_a && vec_b
+
+      if @using_fastembed
+        return 0.0 if vec_a.empty? || vec_b.empty?
+        return 0.0 if vec_a.length != vec_b.length
+
+        dot    = 0.0
+        norm_a = 0.0
+        norm_b = 0.0
+
+        vec_a.each_with_index do |a, i|
+          b       = vec_b[i]
+          dot    += a * b
+          norm_a += a * a
+          norm_b += b * b
+        end
+
+        return 0.0 if norm_a.zero? || norm_b.zero?
+
+        dot / (Math.sqrt(norm_a) * Math.sqrt(norm_b))
+      else
+        sparse_cosine(vec_a, vec_b)
+      end
+    end
+
+    # ── Fallback TF-IDF word-frequency path ─────────────────────────────────
+
+    # Returns a sparse Hash{String => Float} L2-normalised term-frequency vector.
+    def fallback_vector(text)
+      counts = Hash.new(0)
+      text.downcase.scan(/[a-z]+/).each do |w|
+        next if STOP_WORDS.include?(w)
+
+        counts[stem(w)] += 1
+      end
+
+      return {} if counts.empty?
+
+      norm = Math.sqrt(counts.values.sum { |c| c * c }.to_f)
+      counts.transform_values { |c| c / norm }
+    end
+
+    def sparse_cosine(vec_a, vec_b)
       return 0.0 if vec_a.empty? || vec_b.empty?
-      return 0.0 if vec_a.length != vec_b.length
 
       dot    = 0.0
       norm_a = 0.0
       norm_b = 0.0
 
-      vec_a.each_with_index do |a, i|
-        b       = vec_b[i]
-        dot    += a * b
-        norm_a += a * a
-        norm_b += b * b
+      vec_a.each do |k, v|
+        dot += v * vec_b[k].to_f
+        norm_a += v * v
       end
+      vec_b.each_value { |v| norm_b += v * v }
 
+      # :nocov:
       return 0.0 if norm_a.zero? || norm_b.zero?
+      # :nocov:
 
       dot / (Math.sqrt(norm_a) * Math.sqrt(norm_b))
     end
+
+    # Very basic Porter-style stemmer for English: strip common suffixes.
+    def stem(word)
+      word = word.dup
+      word.sub!(/ies$/, 'y')    ||
+        word.sub!(/ness$/, '')  ||
+        word.sub!(/ment$/, '')  ||
+        word.sub!(/tion$/, '')  ||
+        word.sub!(/ing$/, '')   ||
+        word.sub!(/ed$/, '')    ||
+        word.sub!(/er$/, '')    ||
+        word.sub!(/ly$/, '')    ||
+        word.sub!(/s$/, '')
+      word
+    end
   end
 end
+
+if defined?(RobotLab) && RobotLab.respond_to?(:register_extension)
+  RobotLab.register_extension(:document_store, RobotLab::DocumentStore)
+end

data/mkdocs.yml

@@ -7,6 +7,7 @@ copyright: Copyright © 2025 Dewayne VanHoozer
 repo_name: MadBomber/robot_lab-document_store
 repo_url: https://github.com/MadBomber/robot_lab-document_store
 edit_uri: edit/main/docs/
+docs_dir: docs
 
 theme:
   name: material
@@ -111,3 +112,7 @@ extra:
 
 nav:
   - Home: index.md
+  - Getting Started: getting_started.md
+  - API Reference: api_reference.md
+  - How It Works: how_it_works.md
+  - RAG Patterns: rag_patterns.md

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: robot_lab-document_store
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -36,17 +36,24 @@ extra_rdoc_files: []
 files:
 - ".envrc"
 - ".github/workflows/deploy-github-pages.yml"
+- ".rubocop.yml"
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
 - Rakefile
+- docs/api_reference.md
+- docs/assets/architecture.svg
+- docs/getting_started.md
+- docs/how_it_works.md
 - docs/index.md
-- examples/26_document_store.rb
-- examples/26_document_store/api_versioning_adr.md
-- examples/26_document_store/incident_postmortem.md
-- examples/26_document_store/postgres_runbook.md
-- examples/26_document_store/redis_caching_guide.md
-- examples/26_document_store/sidekiq_guide.md
+- docs/pluggable_backends_design.md
+- docs/rag_patterns.md
+- examples/01_basic_usage.rb
+- examples/01_basic_usage/api_versioning_adr.md
+- examples/01_basic_usage/incident_postmortem.md
+- examples/01_basic_usage/postgres_runbook.md
+- examples/01_basic_usage/redis_caching_guide.md
+- examples/01_basic_usage/sidekiq_guide.md
 - lib/robot_lab/document_store.rb
 - lib/robot_lab/document_store/version.rb
 - mkdocs.yml