documentrix 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c45e75b570207ac77d9a04e95939050c33d8f5e0645c7ebac96c862de4e252f3
-  data.tar.gz: 18b0f526ec16115483de74c027a7d8eaa8c8ba2461ac41d4695f53f1ceff32c7
+  metadata.gz: c3ab97426ab9fbd4ec832a422a18d574f4c03b11f22391cbc6eded16b9ba0609
+  data.tar.gz: 991215fb26b4165a4b3562075d9742e3756218e2a24f3f32a550e232f59fb92f
 SHA512:
-  metadata.gz: dddf96ef71ab25c35c6872905cea070c8a27036caf841d97626fadc1db172e496a3b8f13748000cbb9a096b66ae59f459bae145fc2a4a06abffad7de267362a9
-  data.tar.gz: cab52c41f1749fe0ff56538cd01e4525787921e700dca91e44f10d939ec942afc59478c4f8256033fdbf5ef4a38ac649d80eadfbc711042bccb85e38b056a569
+  metadata.gz: d67e453e27428bcd8364349e988556af844caab5faa8a8edb3dcfba9650554b8dbfc66ad322a6f9f59d5ac64350d7006bf899a16b6c83f92909793301b6954d5
+  data.tar.gz: 49706d4984b27ee7b3d0f8ef8a9efec4a8e74d8bef78d78f15ed5c76f5bacc0faf20c36cef739aa78c1f68e5733696d16999bbd14228ae7be8e6cf9c9baf0a75

data/CHANGES.md

@@ -1,5 +1,85 @@
 # Changes
 
+## 2026-05-18 v0.3.1
+
+- Fixed scoping bugs in `clear_by_source` and `source_exist?` by implementing a
+  `key LIKE ?` constraint.
+- Added regression tests for prefix isolation in
+  `spec/documentrix/documents/cache/sqlite_cache_spec.rb`.
+- Introduced the `start_with_prefix` method in
+  `lib/documentrix/documents/cache/sqlite_cache.rb` to unify prefix patterns.
+- Updated `tags`, `size`, `clear_all_with_prefix`, and `each` to utilize the
+  new `start_with_prefix` method.
+
+## 2026-05-17 v0.3.0
+
+### New Features
+
+- **Source Tracking & Versioning**:
+    - Introduced `Documentrix::Utils::Digests` for SHA256 hashing of strings
+      and files, including an `mtime`-based cache.
+    - Implemented source-based document management in `Documentrix::Documents`
+      via `normalize_source`, `source_exist?`, `source_modified?`,
+      `source_update`, and `source_remove`.
+    - Updated `Documentrix::Documents#add` and
+      `Documentrix::Documents#source_update` to support `digest` for version
+      tracking.
+- **Text Splitting**:
+    - Added `Documentrix::Documents::Splitters::Common` to implement
+      `force_split` behavior.
+    - Integrated `force` splitting into `Character`, `RecursiveCharacter`, and
+      `Semantic` splitters.
+- **Cache Enhancements**:
+    - Added `each_source` to `Documentrix::Documents::Cache::Common` and an
+      optimized `SELECT DISTINCT source` implementation in
+      `Documentrix::Documents::Cache::SQLiteCache`.
+    - Added a SQLite trigger `delete_embedding_after_record` to automatically
+      clean the `embeddings` table.
+
+### Improvements & Refactorings
+
+- **Search & Retrieval**:
+    - Added `min_similarity` parameter to `Documentrix::Documents#find`,
+      `Documentrix::Documents::Cache::Common#find_records`, and
+      `Documentrix::Documents::Cache::SQLiteCache#find_records`.
+    - Optimized `Documentrix::Documents::Cache::SQLiteCache#find_records` by
+      moving similarity calculations into the SQL query using `1 -
+      vec_distance_cosine`.
+    - Simplified `Documentrix::Documents#find_where` by streamlining
+      `take_while` logic and utilizing `opts[:max_records]`.
+- **Cache Implementations**:
+    - Made `object_class` a required keyword argument in
+      `Documentrix::Documents::RedisCache#initialize`.
+    - Refactored `Documentrix::Documents::Cache::Common#clear_by_source` and
+      `Documentrix::Documents::Cache::Common#source_exist?` to use ternary
+      operators.
+    - Improved `Documentrix::Documents::Cache::SQLiteCache#each_source` and
+      `Documentrix::Documents::Cache::SQLiteCache#find_records` for better
+      robustness and formatting.
+- **Documentation & Tooling**:
+    - Expanded YARD documentation for
+      `Documentrix::Documents::Splitters::Character`, `RecursiveCharacter`,
+      `Semantic`, and `Documentrix::Utils::ColorizeTexts`.
+    - Centralized RSpec configuration via a `.rspec` file.
+
+### Bug Fixes
+
+- Fixed an issue in `Documentrix::Documents#find` where `max_records` was
+  hardcoded to `nil` when calling the cache.
+- Adjusted default handling of `min_similarity` in
+  `Documentrix::Documents#find` to use `min_similarity ||= -1` within the
+  method body.
+
+### Testing
+
+- Significantly expanded test suites for `SQLiteCache`, `MemoryCache`, and
+  `RedisCache`, specifically covering `each_source`, `tags`, `clear_for_tags`,
+  and digest-based checks.
+- Added new test cases in `spec/documents_spec.rb` for source management and
+  `Documentrix::Documents#source_update`.
+- Added `spec/utils/digests_spec.rb` and updated splitter specs to verify
+  `force` splitting behavior.
+
 ## 2026-05-12 v0.2.0
 
 ### Added

data/documentrix.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-# stub: documentrix 0.2.0 ruby lib
+# stub: documentrix 0.3.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "documentrix".freeze
-  s.version = "0.2.0".freeze
+  s.version = "0.3.1".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -11,15 +11,15 @@ Gem::Specification.new do |s|
   s.date = "1980-01-02"
   s.description = "The Ruby library, Documentrix, is designed to provide a way to build and\nquery vector databases for applications in natural language processing\n(NLP) and large language models (LLMs). It allows users to store and\nretrieve dense vector embeddings for text strings.\n".freeze
   s.email = "flori@ping.de".freeze
-  s.extra_rdoc_files = ["README.md".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze]
-  s.files = [".envrc".freeze, ".utilsrc".freeze, ".yardopts".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "docker-compose.yml".freeze, "documentrix.gemspec".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze, "redis/redis.conf".freeze, "spec/assets/embeddings.json".freeze, "spec/documentrix/documents/cache/interface_spec.rb".freeze, "spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
+  s.extra_rdoc_files = ["README.md".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/common.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/digests.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze]
+  s.files = [".envrc".freeze, ".utilsrc".freeze, ".yardopts".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "docker-compose.yml".freeze, "documentrix.gemspec".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/common.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/digests.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze, "redis/redis.conf".freeze, "spec/assets/embeddings.json".freeze, "spec/documentrix/documents/cache/interface_spec.rb".freeze, "spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/digests_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
   s.homepage = "https://github.com/flori/documentrix".freeze
   s.licenses = ["MIT".freeze]
   s.rdoc_options = ["--title".freeze, "Documentrix - Ruby library for embedding vector database".freeze, "--main".freeze, "README.md".freeze]
   s.required_ruby_version = Gem::Requirement.new(">= 3.1".freeze)
   s.rubygems_version = "4.0.10".freeze
   s.summary = "Ruby library for embedding vector database".freeze
-  s.test_files = ["spec/documentrix/documents/cache/interface_spec.rb".freeze, "spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
+  s.test_files = ["spec/documentrix/documents/cache/interface_spec.rb".freeze, "spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/digests_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
 
   s.specification_version = 4
 

data/lib/documentrix/documents/cache/common.rb

@@ -12,6 +12,7 @@
 # memory, Redis, and SQLite.
 module Documentrix::Documents::Cache::Common
   include Documentrix::Utils::Math
+  include Documentrix::Utils::Digests
   include Enumerable
 
   # The initialize method sets up the Documentrix::Documents::Cache instance's
@@ -62,27 +63,29 @@ module Documentrix::Documents::Cache::Common
   # @param needle [ Array ] an array containing the embedding vector
   # @param tags [ String, Array ] a string or array of strings representing the tags to search for
   # @param max_records [ Integer ] the maximum number of records to return
+  # @param min_similarity [ Float ] the minimum similarity score required for a record to be returned (defaults to -1)
   #
-  # @yield [ record ]
-  #
-  # @return [ Array<Documentrix::Documents::Records> ] an array containing the matching records
-  def find_records(needle, tags: nil, max_records: nil)
+  # @return [ Array<Documentrix::Documents::Record> ] an array containing the matching records
+  def find_records(needle, tags: nil, max_records: nil, min_similarity: -1)
     tags    = Documentrix::Utils::Tags.new(Array(tags)).to_a
     records = self
     if tags.present?
       records = records.select { |_key, record| (tags & record.tags).size >= 1 }
     end
+
     needle_norm = norm(needle)
-    records     = records.sort_by { |key, record|
+    records     = records.map do |key, record|
       record.key        = key
       record.similarity = cosine_similarity(
-        a: needle,
-        b: record.embedding,
+        a:      needle,
+        b:      record.embedding,
         a_norm: needle_norm,
         b_norm: record.norm,
       )
-    }
-    records.transpose.last&.reverse.to_a
+      record
+    end.sort_by(&:similarity).reverse.select { _1.similarity >= min_similarity }
+
+    max_records ? records.take(max_records) : records
   end
 
   # Returns a set of unique tags found in the cache records.
@@ -116,19 +119,68 @@ module Documentrix::Documents::Cache::Common
     self
   end
 
+  # Yields each unique, full source present in the cache records.
+  #
+  # @yield [source] the full source string
+  # @return [Enumerator] an enumerator if no block is given, nil otherwise.
+  def each_source(&block)
+    block or return enum_for(__method__)
+    seen = {}
+    each do |_key, record|
+      source = record.source.full? or next
+      seen.key?(source) and next
+      seen[source] = true
+      block.(source)
+    end
+    nil
+  end
+
   # The clear_by_source method removes all records from the cache that
   # have a source matching the given source.
   #
   # @param source [String] the source to filter records by
+  # @param digest [String, nil] the SHA256 hexadecimal digest of the source.
+  # @param operator [Symbol, String] the operator to compare the digest with ('=' or '!=')
   #
   # @return [self] self
-  def clear_by_source(source)
+  def clear_by_source(source, digest: nil, operator: ?=)
+    operator = operator == '=' ? '==' : '!='
+
     each do |key, record|
-      delete(unpre(key)) if record.source == source
+      next unless record.source == source
+      if digest
+        should_delete = record.digest.send(operator, digest)
+        delete(unpre(key)) if should_delete
+      else
+        delete(unpre(key))
+      end
     end
     self
   end
 
+  # Checks if any records associated with the given source exist in the cache.
+  #
+  # @param source [String] the source to check for existence
+  # @param digest [String, nil] the SHA256 hexadecimal digest to compare against
+  # @param operator [Symbol, String] the operator to compare the digest with ('=' or '!=')
+  #
+  # @return [Boolean] true if a matching record is found, false otherwise.
+  def source_exist?(source, digest: nil, operator: ?=)
+    operator = operator == '=' ? '==' : '!='
+
+    each do |_, record|
+      next unless record.source == source
+      if digest
+        if record.digest.send(operator, digest)
+          return true
+        end
+      else
+        return true
+      end
+    end
+    false
+  end
+
   # The clear method removes cached records based on the provided tags or
   # clears all records with the current prefix.
   #

data/lib/documentrix/documents/cache/records.rb

@@ -27,7 +27,7 @@ module Documentrix::Documents::Cache::Records
     # The to_s method returns a string representation of the object.
     #
     # @return [String] A string containing the text and tags of the record,
-    # along with its similarity score.
+    #   along with its similarity score.
     def to_s
       my_tags = tags_set
       my_tags.empty? or my_tags = " #{my_tags}"

data/lib/documentrix/documents/cache/redis_cache.rb

@@ -23,7 +23,7 @@ class Documentrix::Documents::RedisCache
   # @param [String] prefix the string to be used as the prefix for this cache
   # @param [String] url the URL of the Redis server (default: ENV['REDIS_URL'])
   # @param [Class] object_class the class of objects stored in Redis (default: nil)
-  def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil)
+  def initialize(prefix:, url: ENV['REDIS_URL'], object_class:)
     super(prefix:)
     url or raise ArgumentError, 'require redis url'
     @url, @object_class = url, object_class
@@ -46,7 +46,7 @@ class Documentrix::Documents::RedisCache
   def [](key)
     value = redis.get(pre(key))
     unless value.nil?
-      object_class ? JSON.parse(value, object_class:) : JSON.parse(value)
+      JSON.parse(value, object_class:)
     end
   end
 
@@ -153,7 +153,7 @@ class Documentrix::Documents::RedisCache
 
     redis.scan_each(match: prefix + ?*) do |key|
       value = redis.get(key) or next
-      value = object_class ? JSON.parse(value, object_class:) : JSON.parse(value)
+      value = JSON.parse(value, object_class:)
       block.(key, value)
     end
   end

data/lib/documentrix/documents/cache/sqlite_cache.rb

@@ -46,17 +46,17 @@ class Documentrix::Documents::Cache::SQLiteCache
     result = execute(
       %{
         SELECT records.key, records.text, records.norm, records.source,
-          records.tags, embeddings.embedding
+          records.digest, records.tags, embeddings.embedding
         FROM records
         INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
         WHERE records.key = ?
       },
       pre(key)
     )&.first or return
-    key, text, norm, source, tags, embedding = *result
+    key, text, norm, source, digest, tags, embedding = *result
     embedding = embedding.unpack("f*")
     tags      = Documentrix::Utils::Tags.new(JSON(tags.to_s).to_a, source:)
-    convert_value_to_record(key:, text:, norm:, source:, tags:, embedding:)
+    convert_value_to_record(key:, text:, norm:, source:, digest:, tags:, embedding:)
   end
 
   # The []= method sets the value for a given key by inserting it into the
@@ -66,15 +66,16 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @param [Hash, Documentrix::Documents::Record] value the hash or record
   #        containing the text, embedding, and other metadata
   def []=(key, value)
-    value = convert_value_to_record(value)
+    value     = convert_value_to_record(value)
+    digest    = compute_file_digest(value.source)
     embedding = value.embedding.pack("f*")
     execute(%{BEGIN})
     execute(%{INSERT INTO embeddings(embedding) VALUES(?)}, [ embedding ])
     embedding_id, = execute(%{ SELECT last_insert_rowid() }).flatten
     execute(%{
-      INSERT INTO records(key,text,embedding_id,norm,source,tags)
-      VALUES(?,?,?,?,?,?)
-    }, [ pre(key), value.text, embedding_id, value.norm, value.source, JSON(value.tags) ])
+      INSERT INTO records(key,text,embedding_id,norm,source,digest,tags)
+      VALUES(?,?,?,?,?,?,?)
+    }, [ pre(key), value.text, embedding_id, value.norm, value.source, digest, JSON(value.tags) ])
     execute(%{COMMIT})
   end
 
@@ -113,7 +114,7 @@ class Documentrix::Documents::Cache::SQLiteCache
     result = Documentrix::Utils::Tags.new
     execute(%{
         SELECT DISTINCT(tags) FROM records WHERE key LIKE ?
-      }, [ "#@prefix%" ]
+      }, [ start_with_prefix ]
     ).flatten.each do
       JSON(_1).each { |t| result.add(t) }
     end
@@ -125,7 +126,10 @@ class Documentrix::Documents::Cache::SQLiteCache
   #
   # @return [ Integer ] the count of records
   def size
-    execute(%{SELECT COUNT(*) FROM records WHERE key LIKE ?}, [ "#@prefix%" ]).flatten.first
+    execute(
+      %{SELECT COUNT(*) FROM records WHERE key LIKE ?},
+      [ start_with_prefix ]
+    ).flatten.first
   end
 
   # The clear_for_tags method clears the cache for specific tags by deleting
@@ -153,19 +157,93 @@ class Documentrix::Documents::Cache::SQLiteCache
   #
   # @return [ Documentrix::Documents::RedisBackedMemoryCache ] self
   def clear_all_with_prefix
-    execute(%{DELETE FROM records WHERE key LIKE ?}, [ "#@prefix%" ])
+    execute(%{DELETE FROM records WHERE key LIKE ?}, [ start_with_prefix ])
     self
   end
 
-  # The clear_by_source method removes all records from the cache that
-  # have a source matching the given source.
+  # Removes all records associated with the specified source from the cache.
+  #
+  # If a digest is provided, the method will only remove records that do NOT
+  # match this digest. This allows for updating a source by wiping old versions
+  # while preserving records that are already up-to-date.
+  #
+  # @param source [String] the source identifier used to filter records
+  # @param digest [String, nil] the SHA256 hexadecimal digest of the source.
+  #   Records matching this digest will be preserved.
+  #
+  # @return [self] the cache instance for method chaining
+  def clear_by_source(source, digest: nil, operator: ?=)
+    operator = '!=' if operator != ?=
+    if digest
+      execute(
+        %{
+          DELETE FROM records
+          WHERE key LIKE ? AND source = ? AND digest #{operator} ?
+        },
+        [ start_with_prefix, source, digest ]
+      )
+    else
+      execute(
+        %{
+          DELETE FROM records
+          WHERE key LIKE ? AND source = ?
+        },
+        [ start_with_prefix, source ]
+      )
+    end
+    self
+  end
+
+  # The source_exist? method checks if any records associated with the given
+  # source exist in the cache. If a digest is provided, it verifies if the
+  # source exists and matches the specified digest using the provided operator.
+  #
+  # @param source [#to_s] the source to check for existence
+  # @param digest [String, nil] the SHA256 hexadecimal digest to compare
+  #   against the stored source digest (optional)
+  # @param operator [String] the operator to use for comparison ('=' or '!=').
+  #   Defaults to '='.
+  #
+  # @return [Boolean] true if the source exists (and matches the digest
+  #   condition if provided), false otherwise.
+  def source_exist?(source, digest: nil, operator: ?=)
+    operator = '!=' if operator != ?=
+    if digest
+      !!execute(
+        %{
+          SELECT 1 FROM records WHERE key LIKE ? AND source = ? AND digest #{operator} ?
+        },
+        [ start_with_prefix, source, digest ]
+      ).first
+    else
+      !!execute(
+        %{
+          SELECT 1 FROM records WHERE key LIKE ? AND source = ?
+        },
+        [ start_with_prefix, source ]
+      ).first
+    end
+  end
+
+  # Yields each unique, full source present in the cache records.
   #
-  # @param source [String] the source to filter records by
+  # This is a high-performance override for SQLite that avoids loading
+  # embeddings and parsing JSON for every record.
   #
-  # @return [Documentrix::Documents::Cache::SQLiteCache] self
-  def clear_by_source(source)
-    execute(%{DELETE FROM records WHERE source = ?}, [ source ])
-    self
+  # @yield [source] the full source string
+  # @return [Enumerator] an enumerator if no block is given, nil otherwise.
+  def each_source(&block)
+    block or return enum_for(__method__)
+
+    execute(%{
+      SELECT DISTINCT source FROM records
+      WHERE key LIKE ? AND source IS NOT NULL
+    }, [ start_with_prefix ]).each do |source,|
+      source = source.full? or next
+
+      block.(source)
+    end
+    nil
   end
 
   # Move a key prefix in the cache.
@@ -203,19 +281,19 @@ class Documentrix::Documents::Cache::SQLiteCache
   #   cache.each do |key, value|
   #     puts "#{key}: #{value}"
   #   end
-  def each(prefix: "#@prefix%", &block)
+  def each(prefix: start_with_prefix, &block)
     block or return enum_for(__method__, prefix:)
 
     execute(%{
       SELECT records.key, records.text, records.norm, records.source,
-        records.tags, embeddings.embedding
+        records.digest, records.tags, embeddings.embedding
       FROM records
       INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
       WHERE records.key LIKE ?
-    }, [ prefix ]).each do |key, text, norm, source, tags, embedding|
+    }, [ prefix ]).each do |key, text, norm, source, digest, tags, embedding|
       embedding = embedding.unpack("f*")
       tags      = Documentrix::Utils::Tags.new(JSON(tags.to_s).to_a, source:)
-      value     = convert_value_to_record(key:, text:, norm:, source:, tags:, embedding:)
+      value     = convert_value_to_record(key:, text:, norm:, source:, digest:, tags:, embedding:)
       block.(key, value)
     end
     self
@@ -261,7 +339,7 @@ class Documentrix::Documents::Cache::SQLiteCache
       SELECT key, tags, embedding_id
       FROM records
       WHERE key LIKE ?#{tags_where}
-    }, [ "#@prefix%" ])
+    }, [ start_with_prefix ])
     if tags_filter
       records = records.select { |key, tags, embedding_id|
         (tags_filter & JSON(tags.to_s).to_a).size >= 1
@@ -275,39 +353,52 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @param needle [ Array ] the embedding vector
   # @param tags [ Array ] the list of tags to filter by (optional)
   # @param max_records [ Integer ] the maximum number of records to return (optional)
+  # @param min_similarity [ Float ] the minimum similarity score to include (defaults to -1)
   #
   # @yield [ key, value ]
   #
   # @raise [ ArgumentError ] if needle size does not match embedding length
   #
   # @example
-  #   documents.find_records([ 0.1 ] * 1_024, tags: %w[ test ])
+  #   documents.find_records([ 0.1 ] * 1_024, tags: %w[ test ], min_similarity: 0.7)
   #
   # @return [ Array<Documentrix::Documents::Record> ] the list of matching records
-  def find_records(needle, tags: nil, max_records: nil)
+  def find_records(needle, tags: nil, max_records: nil, min_similarity: -1)
     needle.size != @embedding_length and
       raise ArgumentError, "needle embedding length != %s" % @embedding_length
     needle_binary = needle.pack("f*")
     max_records   = [ max_records, size, 4_096 ].compact.min
     records = find_records_for_tags(tags)
     rowids_where = '(%s)' % records.transpose.last&.join(?,)
-    execute(%{
-      SELECT records.key, records.text, records.norm, records.source,
-        records.tags, embeddings.embedding
-      FROM records
-      INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
-      WHERE embeddings.rowid IN #{rowids_where}
-        AND embeddings.embedding MATCH ? AND embeddings.k = ?
-    }, [ needle_binary, max_records ]).map do |key, text, norm, source, tags, embedding|
+    execute(
+      %{
+        SELECT records.key, records.text, records.norm, records.source,
+          records.digest, records.tags, embeddings.embedding,
+          1 - vec_distance_cosine(?, vec_f32(embeddings.embedding)) AS similarity
+        FROM records
+        INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
+        WHERE embeddings.rowid IN #{rowids_where}
+          AND embeddings.embedding MATCH ? AND similarity >= ?
+          AND embeddings.k = ?
+        ORDER BY similarity DESC
+      }, [ needle_binary, needle_binary, min_similarity, max_records ]
+    ).map do |key, text, norm, source, digest, tags, embedding, similarity|
       key       = unpre(key)
       embedding = embedding.unpack("f*")
       tags      = Documentrix::Utils::Tags.new(JSON(tags.to_s).to_a, source:)
-      convert_value_to_record(key:, text:, norm:, source:, tags:, embedding:)
+      convert_value_to_record(key:, text:, norm:, source:, digest:, tags:, embedding:, similarity:)
     end
   end
 
   private
 
+  # Returns the SQL LIKE pattern for records starting with the current prefix.
+  #
+  # @return [ String ] the prefix pattern used in SQL WHERE clauses
+  def start_with_prefix
+    "#@prefix%"
+  end
+
   # The execute method executes an SQL query on the database by calling the
   # \@database.execute method.
   #
@@ -362,10 +453,18 @@ class Documentrix::Documents::Cache::SQLiteCache
         embedding_id integer,
         norm         float NOT NULL DEFAULT 0.0,
         source       text,
+        digest       text,
         tags         json NOT NULL DEFAULT [],
         FOREIGN KEY(embedding_id) REFERENCES embeddings(id) ON DELETE CASCADE
       )
     }
+    execute %{
+      CREATE TRIGGER IF NOT EXISTS delete_embedding_after_record AFTER DELETE ON records
+      FOR EACH ROW
+      BEGIN
+        DELETE FROM embeddings WHERE rowid = OLD.embedding_id;
+      END
+    }
     nil
   end