documentrix 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 681b65f2c422db0a749e2ec33fba7eadabfaa1d2edc3c47d88ee6dfa0ac9cdd0
-  data.tar.gz: ba48587f3434dbd91b074781712ea4117f6f52874cc03890f2a1e79fb36b1891
+  metadata.gz: e8872a69e718d4c77ca45805be09efb3078f99b809d6fd32078592046b13198e
+  data.tar.gz: f20b52e12f99c836a7790ce326159f883ffce9b4bf75d68318592c71ed206d23
 SHA512:
-  metadata.gz: 4a4028ee84733d6e5df6300d60c2ede2ae8e003ac893f3583e1cb3288ca6e85451085d0fd8c3e26654eea490b51849d47c0ea027f5663a12a38002c96ef2117c
-  data.tar.gz: ca0193dea4d6c751a69e001e18676a3ec9afe4cc02d30175fb74a4ae21a7fc39e2978ee8af7d2a50e65a2275ff86cc51bf5e72a22aa0252f3ac13634d247590b
+  metadata.gz: 2f6ccb24968240b798bfa580ea24004c61baaf1a6455917d46b485591a6a97b67539f8a078116c7c8f659bf504f956fc0f24991730b1fba701627ec7aad7987d
+  data.tar.gz: 54eac6998b2e1f5bdee7f27aa5ed9e580c3da0ed6c87d1dfeb53dcfbb8750a3b40db7a8b964f6fbe9517f611dd2b1b18092acf07d6ac4e5b8a1f71b3a770bc2e

data/.envrc

@@ -0,0 +1 @@
+export REDIS_URL=redis://localhost:9736

data/CHANGES.md

@@ -1,5 +1,32 @@
 # Changes
 
+## 2025-11-12 v0.0.3
+
+- Replaced `numo-narray` dependency with `numo-narray-alt` in `Rakefile` and
+  `documentrix.gemspec` to prevent compilation issues
+- Updated `all_images` development dependency version from `~> 0.6` to `~> 0.9`
+  in `Rakefile`
+- Updated `s.rubygems_version` from **3.6.7** to **3.7.2**
+- Updated `gem_hadar` development dependency from `~> 1.20` to `~> 2.8`
+- Added `openssl-dev` to the package list in `.all_images.yml` for Docker builds
+- Removed duplicate `tags` method
+- Replaced `RSpec.describe` with `describe` for simplified syntax
+- Updated Redis service image in `docker-compose.yml` from
+  `valkey/valkey:7.2.8-alpine` to `valkey/valkey:**8.1.1-alpine**`
+
+## 2025-05-26 v0.0.2
+
+* Documentrix::Utils::Tags enhancements for improved tagging functionality:
+  * Added `valid_tag` parameter to `initialize` method with default value.
+  * Introduced `DEFAULT_VALID_TAG` regular expression constant.
+  * Updated `initialize` and `add` methods to use new `valid_tag` parameter.
+  * Added `attr_reader` for `valid_tag`.
+* Added `describe` block for `Tag` in `tags_spec.rb`.
+  * Added three `it` blocks to test instantiation, default tag value trimming, and custom regex usage.
+* Update `.envrc` file to include Redis connection settings:
+  * Added `REDIS_URL` environment variable with value `redis://localhost:9736`
+  * Created new `.envrc` file in the root directory.
+
 ## 2025-01-29 v0.0.1
 
 * Added docker-compose redis

data/Rakefile

@@ -26,17 +26,17 @@ GemHadar do
 
   required_ruby_version  '~> 3.1'
 
-  dependency 'infobar',       '~> 0.8'
-  dependency 'json',          '~> 2.0'
-  dependency 'tins',          '~> 1.34'
-  dependency 'sqlite-vec',    '~> 0.0'
-  dependency 'sqlite3',       '~> 2.0', '>= 2.0.1'
-  dependency 'kramdown-ansi', '~> 0.0', '>= 0.0.1'
-  dependency 'numo-narray',   '~> 0.9'
-  dependency 'redis',         '~> 5.0'
-  dependency 'more_math',     '~> 1.1'
-
-  development_dependency 'all_images', '~> 0.6'
+  dependency 'infobar',         '~> 0.9'
+  dependency 'json',            '~> 2.0'
+  dependency 'tins',            '~> 1.34'
+  dependency 'sqlite-vec',      '~> 0.0'
+  dependency 'sqlite3',         '~> 2.0', '>= 2.0.1'
+  dependency 'kramdown-ansi',   '~> 0.0', '>= 0.0.1'
+  dependency 'numo-narray-alt', '~> 0.9'
+  dependency 'redis',           '~> 5.0'
+  dependency 'more_math',       '~> 1.1'
+
+  development_dependency 'all_images', '~> 0.9'
   development_dependency 'rspec',      '~> 3.2'
   development_dependency 'kramdown',   '~> 2.0'
   development_dependency 'debug'

data/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   redis:
-    image: valkey/valkey:7.2.8-alpine
+    image: valkey/valkey:8.1.1-alpine
     restart: unless-stopped
     ports: [ "127.0.0.1:9736:6379" ]
     volumes:

data/documentrix.gemspec

@@ -1,41 +1,41 @@
 # -*- encoding: utf-8 -*-
-# stub: documentrix 0.0.1 ruby lib
+# stub: documentrix 0.0.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "documentrix".freeze
-  s.version = "0.0.1".freeze
+  s.version = "0.0.3".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Florian Frank".freeze]
-  s.date = "2025-01-29"
+  s.date = "1980-01-02"
   s.description = "The Ruby library, Documentrix, is designed to provide a way to build and\nquery vector databases for applications in natural language processing\n(NLP) and large language models (LLMs). It allows users to store and\nretrieve dense vector embeddings for text strings.\n".freeze
   s.email = "flori@ping.de".freeze
   s.extra_rdoc_files = ["README.md".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_backed_memory_cache.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze]
-  s.files = [".yardopts".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "docker-compose.yml".freeze, "documentrix.gemspec".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_backed_memory_cache.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze, "redis/redis.conf".freeze, "spec/assets/embeddings.json".freeze, "spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
+  s.files = [".envrc".freeze, ".yardopts".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "docker-compose.yml".freeze, "documentrix.gemspec".freeze, "lib/documentrix.rb".freeze, "lib/documentrix/documents.rb".freeze, "lib/documentrix/documents/cache/common.rb".freeze, "lib/documentrix/documents/cache/memory_cache.rb".freeze, "lib/documentrix/documents/cache/records.rb".freeze, "lib/documentrix/documents/cache/redis_backed_memory_cache.rb".freeze, "lib/documentrix/documents/cache/redis_cache.rb".freeze, "lib/documentrix/documents/cache/sqlite_cache.rb".freeze, "lib/documentrix/documents/splitters/character.rb".freeze, "lib/documentrix/documents/splitters/semantic.rb".freeze, "lib/documentrix/utils.rb".freeze, "lib/documentrix/utils/colorize_texts.rb".freeze, "lib/documentrix/utils/math.rb".freeze, "lib/documentrix/utils/tags.rb".freeze, "lib/documentrix/version.rb".freeze, "redis/redis.conf".freeze, "spec/assets/embeddings.json".freeze, "spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
   s.homepage = "https://github.com/flori/documentrix".freeze
   s.licenses = ["MIT".freeze]
   s.rdoc_options = ["--title".freeze, "Documentrix - Ruby library for embedding vector database".freeze, "--main".freeze, "README.md".freeze]
   s.required_ruby_version = Gem::Requirement.new("~> 3.1".freeze)
-  s.rubygems_version = "3.6.2".freeze
+  s.rubygems_version = "3.7.2".freeze
   s.summary = "Ruby library for embedding vector database".freeze
   s.test_files = ["spec/documentrix/documents/cache/memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb".freeze, "spec/documentrix/documents/cache/redis_cache_spec.rb".freeze, "spec/documentrix/documents/cache/sqlite_cache_spec.rb".freeze, "spec/documentrix/documents/splitters/character_spec.rb".freeze, "spec/documentrix/documents/splitters/semantic_spec.rb".freeze, "spec/documents_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/utils/colorize_texts_spec.rb".freeze, "spec/utils/tags_spec.rb".freeze]
 
   s.specification_version = 4
 
-  s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.19".freeze])
-  s.add_development_dependency(%q<all_images>.freeze, ["~> 0.6".freeze])
+  s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 2.8".freeze])
+  s.add_development_dependency(%q<all_images>.freeze, ["~> 0.9".freeze])
   s.add_development_dependency(%q<rspec>.freeze, ["~> 3.2".freeze])
   s.add_development_dependency(%q<kramdown>.freeze, ["~> 2.0".freeze])
   s.add_development_dependency(%q<debug>.freeze, [">= 0".freeze])
   s.add_development_dependency(%q<simplecov>.freeze, [">= 0".freeze])
-  s.add_runtime_dependency(%q<infobar>.freeze, ["~> 0.8".freeze])
+  s.add_runtime_dependency(%q<infobar>.freeze, ["~> 0.9".freeze])
   s.add_runtime_dependency(%q<json>.freeze, ["~> 2.0".freeze])
   s.add_runtime_dependency(%q<tins>.freeze, ["~> 1.34".freeze])
   s.add_runtime_dependency(%q<sqlite-vec>.freeze, ["~> 0.0".freeze])
   s.add_runtime_dependency(%q<sqlite3>.freeze, ["~> 2.0".freeze, ">= 2.0.1".freeze])
   s.add_runtime_dependency(%q<kramdown-ansi>.freeze, ["~> 0.0".freeze, ">= 0.0.1".freeze])
-  s.add_runtime_dependency(%q<numo-narray>.freeze, ["~> 0.9".freeze])
+  s.add_runtime_dependency(%q<numo-narray-alt>.freeze, ["~> 0.9".freeze])
   s.add_runtime_dependency(%q<redis>.freeze, ["~> 5.0".freeze])
   s.add_runtime_dependency(%q<more_math>.freeze, ["~> 1.1".freeze])
 end

data/lib/documentrix/documents/cache/records.rb

@@ -130,16 +130,5 @@ module Documentrix::Documents::Cache::Records
         end
       end
     end
-    def tags
-      if defined? super
-        super
-      else
-        each_with_object(Documentrix::Utils::Tags.new) do |(_, record), t|
-          record.tags.each do |tag|
-            t.add(tag, source: record.source)
-          end
-        end
-      end
-    end
   end
 end

data/lib/documentrix/utils/tags.rb

@@ -1,6 +1,10 @@
 require 'term/ansicolor'
 
 class Documentrix::Utils::Tags
+  # Matches tags with optional leading # characters and at least one non-space
+  # character by default:
+  DEFAULT_VALID_TAG = /\A#*(\S+)/
+
   class Tag < String
     include Term::ANSIColor
 
@@ -9,13 +13,15 @@ class Documentrix::Utils::Tags
     #
     # @param tag [String] the string representation of the tag
     # @param source [String, nil] the source URL for the tag (default: nil)
-    def initialize(tag, source: nil)
-      super(tag.to_s.gsub(/\A#+/, ''))
+    def initialize(tag, valid_tag: DEFAULT_VALID_TAG, source: nil)
+      super(tag.to_s[valid_tag, 1])
       self.source = source
     end
 
     attr_accessor :source # the source URL for the tag
 
+    attr_reader :valid_tag # the regular expression capturing a valid tag's content
+
     # The to_s method formats the tag string for output, including source URL
     # if requested.
     #
@@ -46,15 +52,18 @@ class Documentrix::Utils::Tags
   #   Documentrix::Utils::Tags.new(%w[ foo bar ])
   #
   # @return [Documentrix::Utils::Tags] an instance of Documentrix::Utils::Tags
-  def initialize(tags = [], source: nil)
-    tags = Array(tags)
-    @set = []
+  def initialize(tags = [], valid_tag: DEFAULT_VALID_TAG, source: nil)
+    tags       = Array(tags)
+    @valid_tag = valid_tag
+    @set       = []
     tags.each { |tag| add(tag, source:) }
   end
 
+  attr_reader :valid_tag # the regular expression capturing a valid tag's content
+
   def add(tag, source: nil)
     unless tag.is_a?(Tag)
-      tag = Tag.new(tag, source:)
+      tag = Tag.new(tag, valid_tag:, source:)
     end
     index = @set.bsearch_index { _1 >= tag }
     if index == nil

data/lib/documentrix/version.rb

@@ -1,6 +1,6 @@
 module Documentrix
   # Documentrix version
-  VERSION         = '0.0.1'
+  VERSION         = '0.0.3'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/documentrix/documents/cache/memory_cache_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents::MemoryCache do
+describe Documentrix::Documents::MemoryCache do
   let :prefix do
     'test-'
   end

data/spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents::RedisBackedMemoryCache do
+describe Documentrix::Documents::RedisBackedMemoryCache do
   let :prefix do
     'test-'
   end

data/spec/documentrix/documents/cache/redis_cache_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents::RedisCache do
+describe Documentrix::Documents::RedisCache do
   let :prefix do
     'test-'
   end

data/spec/documentrix/documents/cache/sqlite_cache_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents::SQLiteCache do
+describe Documentrix::Documents::SQLiteCache do
   let :prefix do
     'test-'
   end

data/spec/documentrix/documents/splitters/character_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents::Splitters::Character do
+describe Documentrix::Documents::Splitters::Character do
   let :splitter do
     described_class.new chunk_size: 23, combining_string: ''
   end
@@ -58,7 +58,7 @@ RSpec.describe Documentrix::Documents::Splitters::Character do
   end
 end
 
-RSpec.describe Documentrix::Documents::Splitters::RecursiveCharacter do
+describe Documentrix::Documents::Splitters::RecursiveCharacter do
   let :splitter do
     described_class.new chunk_size: 23, combining_string: ''
   end

data/spec/documentrix/documents/splitters/semantic_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents::Splitters::Semantic do
+describe Documentrix::Documents::Splitters::Semantic do
   let :ollama do
     double('Ollama::Client')
   end

data/spec/documents_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Documents do
+describe Documentrix::Documents do
   let :ollama do
     double('Ollama::Client')
   end

data/spec/utils/colorize_texts_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Utils::ColorizeTexts do
+describe Documentrix::Utils::ColorizeTexts do
   it 'colorizes texts' do
     ct = described_class.new(%w[ foo bar ])
     colored   = ct.to_s

data/spec/utils/tags_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Documentrix::Utils::Tags do
+describe Documentrix::Utils::Tags do
   it 'can be instantiated' do
     expect(described_class.new).to be_a described_class
   end
@@ -50,4 +50,19 @@ RSpec.describe Documentrix::Utils::Tags do
       "\e]8;;file:///path/to/bar.html\e\\#bar\e]8;;\e\\ \e]8;;https://foo.example.com\e\\#foo\e]8;;\e\\"
     )
   end
+
+  describe described_class::Tag do
+    it 'can be instantiated' do
+      expect(described_class.new('#foo')).to be_a described_class
+    end
+
+    it 'can cannot contain a space by default' do
+      expect(described_class.new('#foo bar')).to eq 'foo'
+    end
+
+    it 'can cannot use a custom regular expression for capturing the tag value' do
+      valid_tag = /\A#*([\w\]\[]+)/
+      expect(described_class.new('#foo[bar]!!!!', valid_tag:)).to eq 'foo[bar]'
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: documentrix
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Florian Frank
 bindir: bin
 cert_chain: []
-date: 2025-01-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gem_hadar
@@ -15,28 +15,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '2.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: all_images
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -99,14 +99,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -190,7 +190,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.1
 - !ruby/object:Gem::Dependency
-  name: numo-narray
+  name: numo-narray-alt
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -257,6 +257,7 @@ extra_rdoc_files:
 - lib/documentrix/utils/tags.rb
 - lib/documentrix/version.rb
 files:
+- ".envrc"
 - ".yardopts"
 - CHANGES.md
 - Gemfile
@@ -314,7 +315,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Ruby library for embedding vector database
 test_files: