documentrix 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.utilsrc +107 -0
- data/CHANGES.md +28 -0
- data/Rakefile +5 -1
- data/docker-compose.yml +1 -1
- data/documentrix.gemspec +8 -8
- data/lib/documentrix/documents/cache/common.rb +102 -6
- data/lib/documentrix/documents/cache/memory_cache.rb +36 -8
- data/lib/documentrix/documents/cache/records.rb +14 -90
- data/lib/documentrix/documents/cache/redis_cache.rb +57 -25
- data/lib/documentrix/documents/cache/sqlite_cache.rb +41 -5
- data/lib/documentrix/documents.rb +62 -6
- data/lib/documentrix/utils/colorize_texts.rb +11 -0
- data/lib/documentrix/utils/math.rb +5 -0
- data/lib/documentrix/utils/tags.rb +34 -0
- data/lib/documentrix/utils.rb +10 -0
- data/lib/documentrix/version.rb +1 -1
- data/lib/documentrix.rb +6 -0
- data/spec/documentrix/documents/cache/interface_spec.rb +188 -0
- data/spec/documentrix/documents/cache/memory_cache_spec.rb +29 -0
- data/spec/documentrix/documents/cache/redis_cache_spec.rb +13 -11
- data/spec/documentrix/documents/cache/sqlite_cache_spec.rb +34 -0
- data/spec/documents_spec.rb +22 -0
- data/spec/spec_helper.rb +2 -6
- metadata +11 -12
- data/lib/documentrix/documents/cache/redis_backed_memory_cache.rb +0 -64
- data/spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb +0 -121
|
@@ -3,6 +3,14 @@ require 'sqlite3'
|
|
|
3
3
|
require 'sqlite_vec'
|
|
4
4
|
require 'digest/md5'
|
|
5
5
|
|
|
6
|
+
# SQLiteCache is a cache implementation that uses SQLite database for storing
|
|
7
|
+
# document embeddings and related metadata.
|
|
8
|
+
#
|
|
9
|
+
# This class provides a persistent cache storage solution for document
|
|
10
|
+
# embeddings, leveraging SQLite's capabilities to store both the embedding
|
|
11
|
+
# vectors and associated text data, tags, and source information.
|
|
12
|
+
# It supports efficient vector similarity searches using the sqlite_vec
|
|
13
|
+
# extension for fast nearest neighbor queries.
|
|
6
14
|
class Documentrix::Documents::Cache::SQLiteCache
|
|
7
15
|
include Documentrix::Documents::Cache::Common
|
|
8
16
|
|
|
@@ -135,20 +143,45 @@ class Documentrix::Documents::Cache::SQLiteCache
|
|
|
135
143
|
keys = '(%s)' % records.transpose.first.map { "'%s'" % quote(_1) }.join(?,)
|
|
136
144
|
execute(%{DELETE FROM records WHERE key IN #{keys}})
|
|
137
145
|
else
|
|
138
|
-
|
|
146
|
+
clear_all_with_prefix
|
|
139
147
|
end
|
|
140
148
|
self
|
|
141
149
|
end
|
|
142
150
|
|
|
143
|
-
# The
|
|
144
|
-
# executing a SQL query.
|
|
151
|
+
# The clear_all_with_prefix method deletes all records for prefix `prefix`
|
|
152
|
+
# from the cache by executing a SQL query.
|
|
145
153
|
#
|
|
146
154
|
# @return [ Documentrix::Documents::RedisBackedMemoryCache ] self
|
|
147
|
-
def
|
|
155
|
+
def clear_all_with_prefix
|
|
148
156
|
execute(%{DELETE FROM records WHERE key LIKE ?}, [ "#@prefix%" ])
|
|
149
157
|
self
|
|
150
158
|
end
|
|
151
159
|
|
|
160
|
+
# Move a key prefix in the cache.
|
|
161
|
+
#
|
|
162
|
+
# This operation updates every record whose key starts with +old_prefix+,
|
|
163
|
+
# rewriting the prefix to +new_prefix+. It uses SQLite’s built‑in `replace()`
|
|
164
|
+
# string function, which means the change is atomic and performed entirely
|
|
165
|
+
# inside the database engine—no Ruby‑side iteration or temporary data
|
|
166
|
+
# structures are needed.
|
|
167
|
+
#
|
|
168
|
+
# @param old_prefix [String] the current prefix to replace (must match the
|
|
169
|
+
# beginning of the keys you want to move).
|
|
170
|
+
# @param new_prefix [String] the new prefix that should take its place.
|
|
171
|
+
#
|
|
172
|
+
# @return [Documentrix::Documents::Cache::SQLiteCache] the cache instance,
|
|
173
|
+
# enabling method chaining.
|
|
174
|
+
def move_prefix(old_prefix, new_prefix)
|
|
175
|
+
execute(
|
|
176
|
+
%{
|
|
177
|
+
UPDATE records
|
|
178
|
+
SET key = replace(key, '#{quote(old_prefix)}', '#{quote(new_prefix)}')
|
|
179
|
+
WHERE key LIKE ?
|
|
180
|
+
},
|
|
181
|
+
old_prefix + '%'
|
|
182
|
+
)
|
|
183
|
+
end
|
|
184
|
+
|
|
152
185
|
# The each method iterates over records matching the given prefix and yields
|
|
153
186
|
# them to the block.
|
|
154
187
|
#
|
|
@@ -160,6 +193,8 @@ class Documentrix::Documents::Cache::SQLiteCache
|
|
|
160
193
|
# puts "#{key}: #{value}"
|
|
161
194
|
# end
|
|
162
195
|
def each(prefix: "#@prefix%", &block)
|
|
196
|
+
block or return enum_for(__method__, prefix:)
|
|
197
|
+
|
|
163
198
|
execute(%{
|
|
164
199
|
SELECT records.key, records.text, records.norm, records.source,
|
|
165
200
|
records.tags, embeddings.embedding
|
|
@@ -174,7 +209,6 @@ class Documentrix::Documents::Cache::SQLiteCache
|
|
|
174
209
|
end
|
|
175
210
|
self
|
|
176
211
|
end
|
|
177
|
-
include Enumerable
|
|
178
212
|
|
|
179
213
|
# The full_each method iterates over all keys and values in the cache,
|
|
180
214
|
# regardless of their prefix.
|
|
@@ -183,6 +217,8 @@ class Documentrix::Documents::Cache::SQLiteCache
|
|
|
183
217
|
#
|
|
184
218
|
# @return [ Documentrix::Documents::Cache::SQLiteCache ] self
|
|
185
219
|
def full_each(&block)
|
|
220
|
+
block or return enum_for(__method__)
|
|
221
|
+
|
|
186
222
|
each(prefix: ?%, &block)
|
|
187
223
|
end
|
|
188
224
|
|
|
@@ -4,22 +4,63 @@ require 'kramdown/ansi'
|
|
|
4
4
|
|
|
5
5
|
class Documentrix::Documents
|
|
6
6
|
end
|
|
7
|
+
# Module for managing document caches in Documentrix
|
|
8
|
+
#
|
|
9
|
+
# This module provides the foundational cache interface and implementations for
|
|
10
|
+
# storing and retrieving document embeddings and related metadata. It supports
|
|
11
|
+
# multiple cache backends including memory, Redis, and SQLite, enabling
|
|
12
|
+
# flexible storage options for vector databases.
|
|
13
|
+
#
|
|
14
|
+
# The cache system handles operations such as setting, retrieving, and deleting
|
|
15
|
+
# cached entries, as well as querying and filtering cached data based on tags
|
|
16
|
+
# and similarity searches. It includes common functionality for managing cache
|
|
17
|
+
# prefixes, enumerating collections, extracting tags, and clearing cache entries.
|
|
7
18
|
module Documentrix::Documents::Cache
|
|
8
19
|
end
|
|
9
20
|
require 'documentrix/documents/cache/records'
|
|
10
21
|
require 'documentrix/documents/cache/memory_cache'
|
|
11
22
|
require 'documentrix/documents/cache/redis_cache'
|
|
12
|
-
require 'documentrix/documents/cache/redis_backed_memory_cache'
|
|
13
23
|
require 'documentrix/documents/cache/sqlite_cache'
|
|
24
|
+
|
|
25
|
+
# Module for text splitting operations in Documentrix
|
|
26
|
+
#
|
|
27
|
+
# This module provides functionality for splitting text into smaller chunks
|
|
28
|
+
# using various strategies. It includes both simple character-based splitting
|
|
29
|
+
# and more sophisticated semantic splitting that considers the meaning
|
|
30
|
+
# and structure of the text when determining split points.
|
|
31
|
+
#
|
|
32
|
+
# The splitters are designed to work with the Documentrix::Documents class
|
|
33
|
+
# to prepare text data for embedding and storage in vector databases.
|
|
14
34
|
module Documentrix::Documents::Splitters
|
|
15
35
|
end
|
|
16
36
|
require 'documentrix/documents/splitters/character'
|
|
17
37
|
require 'documentrix/documents/splitters/semantic'
|
|
18
38
|
|
|
39
|
+
# Documentrix::Documents is a class that provides functionality for building
|
|
40
|
+
# and querying vector databases for natural language processing and large
|
|
41
|
+
# language model applications.
|
|
42
|
+
#
|
|
43
|
+
# It allows users to store and retrieve dense vector embeddings for text
|
|
44
|
+
# strings, supporting various cache backends including memory, Redis, and
|
|
45
|
+
# SQLite for efficient data management.
|
|
46
|
+
#
|
|
47
|
+
# The class handles the complete workflow of adding documents, computing their
|
|
48
|
+
# embeddings using a specified model, storing them in a cache, and performing
|
|
49
|
+
# similarity searches to find relevant documents based on query text.
|
|
50
|
+
#
|
|
51
|
+
# @example
|
|
52
|
+
# documents = Documentrix::Documents.new(
|
|
53
|
+
# ollama: ollama_client,
|
|
54
|
+
# model: 'mxbai-embed-large',
|
|
55
|
+
# collection: 'my-collection'
|
|
56
|
+
# )
|
|
57
|
+
# documents.add(['text1', 'text2'])
|
|
58
|
+
# results = documents.find('search query')
|
|
19
59
|
class Documentrix::Documents
|
|
20
60
|
include Kramdown::ANSI::Width
|
|
21
61
|
include Documentrix::Documents::Cache
|
|
22
62
|
|
|
63
|
+
# Shortcut for Documentrix::Documents::Cache::Records::Record
|
|
23
64
|
Record = Class.new Documentrix::Documents::Cache::Records::Record
|
|
24
65
|
|
|
25
66
|
# The initialize method sets up the Documentrix::Documents instance by
|
|
@@ -229,6 +270,21 @@ class Documentrix::Documents
|
|
|
229
270
|
([ default_collection ] + @cache.collections('%s-' % class_prefix)).uniq
|
|
230
271
|
end
|
|
231
272
|
|
|
273
|
+
# Rename the current collection, moving all keys from the old prefix to a new
|
|
274
|
+
# one. After the rename the instance’s `collection` attribute points to
|
|
275
|
+
# `new_collection`, and the cache keys are updated accordingly.
|
|
276
|
+
#
|
|
277
|
+
# @param new_collection [Symbol] The name of the collection to rename to.
|
|
278
|
+
# @return [Documentrix::Documents] Returns `self` to allow method chaining.
|
|
279
|
+
def rename_collection(new_collection)
|
|
280
|
+
new_collection = new_collection.to_sym
|
|
281
|
+
collections.member?(new_collection) and
|
|
282
|
+
raise ArgumentError, "new collection #{new_collection} already exists!"
|
|
283
|
+
new_prefix = '%s-%s-' % [ class_prefix, new_collection ]
|
|
284
|
+
@cache.move_prefix(prefix, new_prefix)
|
|
285
|
+
self.collection = new_collection
|
|
286
|
+
end
|
|
287
|
+
|
|
232
288
|
# The tags method returns an array of unique tags from the cache.
|
|
233
289
|
#
|
|
234
290
|
# @return [Documentrix::Utils::Tags] A set of unique tags
|
|
@@ -269,11 +325,6 @@ class Documentrix::Documents
|
|
|
269
325
|
end
|
|
270
326
|
ensure
|
|
271
327
|
cache ||= MemoryCache.new(prefix:,)
|
|
272
|
-
cache.respond_to?(:find_records) or cache.extend(Records::FindRecords)
|
|
273
|
-
cache.extend(Records::Tags)
|
|
274
|
-
if cache.respond_to?(:redis)
|
|
275
|
-
cache.extend(Records::RedisFullEach)
|
|
276
|
-
end
|
|
277
328
|
return cache
|
|
278
329
|
end
|
|
279
330
|
|
|
@@ -305,6 +356,11 @@ class Documentrix::Documents
|
|
|
305
356
|
@ollama.embed(model:, input:, options:).embeddings
|
|
306
357
|
end
|
|
307
358
|
|
|
359
|
+
# Returns the base prefix string used for cache keys.
|
|
360
|
+
# This string is prefixed to the collection name to form the full key
|
|
361
|
+
# namespace.
|
|
362
|
+
#
|
|
363
|
+
# @return [String] the prefix identifier for Documentrix documents.
|
|
308
364
|
def class_prefix
|
|
309
365
|
'Documents'
|
|
310
366
|
end
|
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
require 'term/ansicolor'
|
|
2
2
|
require 'kramdown/ansi'
|
|
3
3
|
|
|
4
|
+
# A utility class for colorizing and formatting text output with ANSI color
|
|
5
|
+
# codes and size information.
|
|
6
|
+
#
|
|
7
|
+
# The ColorizeTexts class takes an array of text strings and formats them with
|
|
8
|
+
# different ANSI colors for visual distinction. It also appends the size of each
|
|
9
|
+
# text block to the output, making it useful for debugging or displaying
|
|
10
|
+
# information about text chunks in a visually appealing way.
|
|
11
|
+
#
|
|
12
|
+
# @example
|
|
13
|
+
# colorizer = Documentrix::Utils::ColorizeTexts.new('foo', 'bar')
|
|
14
|
+
# puts colorizer.to_s
|
|
4
15
|
class Documentrix::Utils::ColorizeTexts
|
|
5
16
|
include Math
|
|
6
17
|
include Term::ANSIColor
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
# A module that provides mathematical utilities for vector operations.
|
|
2
|
+
#
|
|
3
|
+
# This module includes methods for calculating vector norms, computing cosine
|
|
4
|
+
# similarity between vectors, and converting arrays into Numo::NArray objects
|
|
5
|
+
# for numerical computations.
|
|
1
6
|
module Documentrix::Utils::Math
|
|
2
7
|
# Returns the cosine similarity between two vectors `a` and `b`, 1.0 is
|
|
3
8
|
# exactly the same, 0.0 means decorrelated.
|
|
@@ -1,10 +1,31 @@
|
|
|
1
1
|
require 'term/ansicolor'
|
|
2
2
|
|
|
3
|
+
# A collection of tags with optional source tracking and formatting
|
|
4
|
+
# capabilities.
|
|
5
|
+
#
|
|
6
|
+
# The Tags class manages a collection of unique tags, ensuring no duplicates while maintaining sorted order.
|
|
7
|
+
# Each tag can optionally be associated with a source URL for tracking origins.
|
|
8
|
+
# The class provides methods for adding, iterating, and formatting tags for display.
|
|
9
|
+
#
|
|
10
|
+
# @example
|
|
11
|
+
# tags = Documentrix::Utils::Tags.new(%w[ foo bar ])
|
|
12
|
+
# tags.add('baz')
|
|
13
|
+
# tags.to_s # => "bar baz foo"
|
|
3
14
|
class Documentrix::Utils::Tags
|
|
4
15
|
# Matches tags with optional leading # characters and at least one non-space
|
|
5
16
|
# character by default:
|
|
6
17
|
DEFAULT_VALID_TAG = /\A#*(\S+)/
|
|
7
18
|
|
|
19
|
+
# A tag string that includes optional source tracking functionality.
|
|
20
|
+
#
|
|
21
|
+
# The Tag class extends String and adds the ability to track the source URL
|
|
22
|
+
# where the tag originated. It provides methods for formatting the tag string
|
|
23
|
+
# for output, including the option to include a hyperlink to the source.
|
|
24
|
+
#
|
|
25
|
+
# @example
|
|
26
|
+
# tag = Documentrix::Utils::Tags::Tag.new('example')
|
|
27
|
+
# tag.source = 'https://example.com'
|
|
28
|
+
# tag.to_s # => "\e]8;;https://example.com\e\\\\#example\e]8;;\e\\\\"
|
|
8
29
|
class Tag < String
|
|
9
30
|
include Term::ANSIColor
|
|
10
31
|
|
|
@@ -61,6 +82,19 @@ class Documentrix::Utils::Tags
|
|
|
61
82
|
|
|
62
83
|
attr_reader :valid_tag # the regular expression capturing a valid tag's content
|
|
63
84
|
|
|
85
|
+
# The add method adds a tag to the collection, ensuring uniqueness and
|
|
86
|
+
# maintaining sorted order.
|
|
87
|
+
#
|
|
88
|
+
# If the tag is not already a Tag instance, it creates one using the provided
|
|
89
|
+
# source. The method uses binary search to find the correct insertion point
|
|
90
|
+
# to maintain the sorted order.
|
|
91
|
+
# Duplicate tags are not added, and the method returns self to allow for
|
|
92
|
+
# method chaining.
|
|
93
|
+
#
|
|
94
|
+
# @param tag [ String, Documentrix::Utils::Tags::Tag ] the tag to be added
|
|
95
|
+
# @param source [ String, nil ] the source URL associated with the tag (optional)
|
|
96
|
+
#
|
|
97
|
+
# @return [ Documentrix::Utils::Tags ] self
|
|
64
98
|
def add(tag, source: nil)
|
|
65
99
|
unless tag.is_a?(Tag)
|
|
66
100
|
tag = Tag.new(tag, valid_tag:, source:)
|
data/lib/documentrix/utils.rb
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
# A module that provides utility classes and methods for Documentrix
|
|
2
|
+
#
|
|
3
|
+
# The Utils module contains helper classes and modules that support various
|
|
4
|
+
# functionalities within Documentrix, including mathematical operations,
|
|
5
|
+
# text processing, and tag management.
|
|
6
|
+
#
|
|
7
|
+
# @example
|
|
8
|
+
# Documentrix::Utils::Math
|
|
9
|
+
# Documentrix::Utils::Tags
|
|
10
|
+
# Documentrix::Utils::ColorizeTexts
|
|
1
11
|
module Documentrix::Utils
|
|
2
12
|
end
|
|
3
13
|
require 'documentrix/utils/colorize_texts'
|
data/lib/documentrix/version.rb
CHANGED
data/lib/documentrix.rb
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
|
+
# Documentrix is a Ruby library designed to facilitate the creation and
|
|
2
|
+
# querying of vector databases for natural language processing and large
|
|
3
|
+
# language model applications. It provides functionality for storing and
|
|
4
|
+
# retrieving dense vector embeddings of text strings, supporting various cache
|
|
5
|
+
# backends including memory, Redis, and SQLite for efficient data management.
|
|
1
6
|
module Documentrix
|
|
2
7
|
module Utils
|
|
3
8
|
end
|
|
4
9
|
end
|
|
5
10
|
|
|
11
|
+
require 'set'
|
|
6
12
|
require 'json'
|
|
7
13
|
require 'infobar'
|
|
8
14
|
require 'tins'
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
describe 'Documentrix::Documents::Cache Interface' do
|
|
4
|
+
describe 'MemoryCache Interface' do
|
|
5
|
+
let(:cache) { Documentrix::Documents::MemoryCache.new(prefix: 'test-') }
|
|
6
|
+
|
|
7
|
+
it 'has proper method resolution' do
|
|
8
|
+
# Basic cache operations
|
|
9
|
+
expect(cache).to respond_to(:[])
|
|
10
|
+
expect(cache.method(:[]).owner).to eq Documentrix::Documents::MemoryCache
|
|
11
|
+
|
|
12
|
+
expect(cache).to respond_to(:[]=)
|
|
13
|
+
expect(cache.method(:[]=).owner).to eq Documentrix::Documents::MemoryCache
|
|
14
|
+
|
|
15
|
+
expect(cache).to respond_to(:key?)
|
|
16
|
+
expect(cache.method(:key?).owner).to eq Documentrix::Documents::MemoryCache
|
|
17
|
+
|
|
18
|
+
expect(cache).to respond_to(:delete)
|
|
19
|
+
expect(cache.method(:delete).owner).to eq Documentrix::Documents::MemoryCache
|
|
20
|
+
|
|
21
|
+
expect(cache).to respond_to(:size)
|
|
22
|
+
expect(cache.method(:size).owner).to eq Documentrix::Documents::MemoryCache
|
|
23
|
+
|
|
24
|
+
expect(cache).to respond_to(:clear_all_with_prefix)
|
|
25
|
+
expect(cache.method(:clear_all_with_prefix).owner).to eq Documentrix::Documents::MemoryCache
|
|
26
|
+
|
|
27
|
+
expect(cache).to respond_to(:each)
|
|
28
|
+
expect(cache.method(:each).owner).to eq Documentrix::Documents::MemoryCache
|
|
29
|
+
|
|
30
|
+
expect(cache).to respond_to(:full_each)
|
|
31
|
+
expect(cache.method(:full_each).owner).to eq Documentrix::Documents::MemoryCache
|
|
32
|
+
|
|
33
|
+
expect(cache).to respond_to(:move_prefix)
|
|
34
|
+
expect(cache.method(:move_prefix).owner).to eq Documentrix::Documents::MemoryCache
|
|
35
|
+
|
|
36
|
+
# Common methods from Cache::Common
|
|
37
|
+
expect(cache).to respond_to(:collections)
|
|
38
|
+
expect(cache.method(:collections).owner).to eq Documentrix::Documents::Cache::Common
|
|
39
|
+
|
|
40
|
+
expect(cache).to respond_to(:pre)
|
|
41
|
+
expect(cache.method(:pre).owner).to eq Documentrix::Documents::Cache::Common
|
|
42
|
+
|
|
43
|
+
expect(cache).to respond_to(:unpre)
|
|
44
|
+
expect(cache.method(:unpre).owner).to eq Documentrix::Documents::Cache::Common
|
|
45
|
+
|
|
46
|
+
expect(cache).to respond_to(:find_records)
|
|
47
|
+
expect(cache.method(:find_records).owner).to eq Documentrix::Documents::Cache::Common
|
|
48
|
+
|
|
49
|
+
expect(cache).to respond_to(:tags)
|
|
50
|
+
expect(cache.method(:tags).owner).to eq Documentrix::Documents::Cache::Common
|
|
51
|
+
|
|
52
|
+
expect(cache).to respond_to(:clear_for_tags)
|
|
53
|
+
expect(cache.method(:clear_for_tags).owner).to eq Documentrix::Documents::Cache::Common
|
|
54
|
+
|
|
55
|
+
expect(cache).to respond_to(:clear)
|
|
56
|
+
expect(cache.method(:clear).owner).to eq Documentrix::Documents::Cache::Common
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
describe 'RedisCache Interface' do
|
|
61
|
+
let(:cache) { Documentrix::Documents::RedisCache.new(prefix: 'test-', url: 'redis://localhost:6379') }
|
|
62
|
+
|
|
63
|
+
it 'has proper method resolution' do
|
|
64
|
+
# Basic cache operations
|
|
65
|
+
expect(cache).to respond_to(:[])
|
|
66
|
+
expect(cache.method(:[]).owner).to eq Documentrix::Documents::RedisCache
|
|
67
|
+
|
|
68
|
+
expect(cache).to respond_to(:[]=)
|
|
69
|
+
expect(cache.method(:[]=).owner).to eq Documentrix::Documents::RedisCache
|
|
70
|
+
|
|
71
|
+
expect(cache).to respond_to(:key?)
|
|
72
|
+
expect(cache.method(:key?).owner).to eq Documentrix::Documents::RedisCache
|
|
73
|
+
|
|
74
|
+
expect(cache).to respond_to(:delete)
|
|
75
|
+
expect(cache.method(:delete).owner).to eq Documentrix::Documents::RedisCache
|
|
76
|
+
|
|
77
|
+
expect(cache).to respond_to(:size)
|
|
78
|
+
expect(cache.method(:size).owner).to eq Documentrix::Documents::RedisCache
|
|
79
|
+
|
|
80
|
+
expect(cache).to respond_to(:clear_all_with_prefix)
|
|
81
|
+
expect(cache.method(:clear_all_with_prefix).owner).to eq Documentrix::Documents::RedisCache
|
|
82
|
+
|
|
83
|
+
expect(cache).to respond_to(:each)
|
|
84
|
+
expect(cache.method(:each).owner).to eq Documentrix::Documents::RedisCache
|
|
85
|
+
|
|
86
|
+
expect(cache).to respond_to(:full_each)
|
|
87
|
+
expect(cache.method(:full_each).owner).to eq Documentrix::Documents::RedisCache
|
|
88
|
+
|
|
89
|
+
expect(cache).to respond_to(:move_prefix)
|
|
90
|
+
expect(cache.method(:move_prefix).owner).to eq Documentrix::Documents::RedisCache
|
|
91
|
+
|
|
92
|
+
# Common methods from Cache::Common
|
|
93
|
+
expect(cache).to respond_to(:collections)
|
|
94
|
+
expect(cache.method(:collections).owner).to eq Documentrix::Documents::Cache::Common
|
|
95
|
+
|
|
96
|
+
expect(cache).to respond_to(:pre)
|
|
97
|
+
expect(cache.method(:pre).owner).to eq Documentrix::Documents::Cache::Common
|
|
98
|
+
|
|
99
|
+
expect(cache).to respond_to(:unpre)
|
|
100
|
+
expect(cache.method(:unpre).owner).to eq Documentrix::Documents::Cache::Common
|
|
101
|
+
|
|
102
|
+
expect(cache).to respond_to(:find_records)
|
|
103
|
+
expect(cache.method(:find_records).owner).to eq Documentrix::Documents::Cache::Common
|
|
104
|
+
|
|
105
|
+
expect(cache).to respond_to(:tags)
|
|
106
|
+
expect(cache.method(:tags).owner).to eq Documentrix::Documents::Cache::Common
|
|
107
|
+
|
|
108
|
+
expect(cache).to respond_to(:clear_for_tags)
|
|
109
|
+
expect(cache.method(:clear_for_tags).owner).to eq Documentrix::Documents::Cache::Common
|
|
110
|
+
|
|
111
|
+
expect(cache).to respond_to(:clear)
|
|
112
|
+
expect(cache.method(:clear).owner).to eq Documentrix::Documents::Cache::Common
|
|
113
|
+
|
|
114
|
+
# Redis-specific methods
|
|
115
|
+
expect(cache).to respond_to(:redis)
|
|
116
|
+
expect(cache.method(:redis).owner).to eq Documentrix::Documents::RedisCache
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
describe 'SQLiteCache Interface' do
|
|
121
|
+
let(:cache) { Documentrix::Documents::Cache::SQLiteCache.new(prefix: 'test-') }
|
|
122
|
+
|
|
123
|
+
it 'has proper method resolution' do
|
|
124
|
+
# Basic cache operations
|
|
125
|
+
expect(cache).to respond_to(:[])
|
|
126
|
+
expect(cache.method(:[]).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
127
|
+
|
|
128
|
+
expect(cache).to respond_to(:[]=)
|
|
129
|
+
expect(cache.method(:[]=).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
130
|
+
|
|
131
|
+
expect(cache).to respond_to(:key?)
|
|
132
|
+
expect(cache.method(:key?).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
133
|
+
|
|
134
|
+
expect(cache).to respond_to(:delete)
|
|
135
|
+
expect(cache.method(:delete).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
136
|
+
|
|
137
|
+
expect(cache).to respond_to(:size)
|
|
138
|
+
expect(cache.method(:size).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
139
|
+
|
|
140
|
+
expect(cache).to respond_to(:clear_all_with_prefix)
|
|
141
|
+
expect(cache.method(:clear_all_with_prefix).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
142
|
+
|
|
143
|
+
expect(cache).to respond_to(:each)
|
|
144
|
+
expect(cache.method(:each).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
145
|
+
|
|
146
|
+
expect(cache).to respond_to(:full_each)
|
|
147
|
+
expect(cache.method(:full_each).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
148
|
+
|
|
149
|
+
expect(cache).to respond_to(:move_prefix)
|
|
150
|
+
expect(cache.method(:move_prefix).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
151
|
+
|
|
152
|
+
# Common methods from Cache::Common
|
|
153
|
+
expect(cache).to respond_to(:collections)
|
|
154
|
+
expect(cache.method(:collections).owner).to eq Documentrix::Documents::Cache::Common
|
|
155
|
+
|
|
156
|
+
expect(cache).to respond_to(:pre)
|
|
157
|
+
expect(cache.method(:pre).owner).to eq Documentrix::Documents::Cache::Common
|
|
158
|
+
|
|
159
|
+
expect(cache).to respond_to(:unpre)
|
|
160
|
+
expect(cache.method(:unpre).owner).to eq Documentrix::Documents::Cache::Common
|
|
161
|
+
|
|
162
|
+
expect(cache).to respond_to(:find_records)
|
|
163
|
+
expect(cache.method(:find_records).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
164
|
+
|
|
165
|
+
expect(cache).to respond_to(:tags)
|
|
166
|
+
expect(cache.method(:tags).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
167
|
+
|
|
168
|
+
expect(cache).to respond_to(:clear_for_tags)
|
|
169
|
+
expect(cache.method(:clear_for_tags).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
170
|
+
|
|
171
|
+
expect(cache).to respond_to(:clear)
|
|
172
|
+
expect(cache.method(:clear).owner).to eq Documentrix::Documents::Cache::Common
|
|
173
|
+
|
|
174
|
+
# SQLite-specific methods
|
|
175
|
+
expect(cache).to respond_to(:filename)
|
|
176
|
+
expect(cache.method(:filename).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
177
|
+
|
|
178
|
+
expect(cache).to respond_to(:embedding_length)
|
|
179
|
+
expect(cache.method(:embedding_length).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
180
|
+
|
|
181
|
+
expect(cache).to respond_to(:convert_to_vector)
|
|
182
|
+
expect(cache.method(:convert_to_vector).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
183
|
+
|
|
184
|
+
expect(cache).to respond_to(:find_records_for_tags)
|
|
185
|
+
expect(cache.method(:find_records_for_tags).owner).to eq Documentrix::Documents::Cache::SQLiteCache
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
@@ -51,6 +51,21 @@ describe Documentrix::Documents::MemoryCache do
|
|
|
51
51
|
expect(s).to eq 2
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
+
it 'can move prefixes' do
|
|
55
|
+
key, value = 'foo', { test1: true }
|
|
56
|
+
cache[key] = value
|
|
57
|
+
cache.prefix = 'test2-'
|
|
58
|
+
key, value = 'bar', { test2: true }
|
|
59
|
+
cache[key] = value
|
|
60
|
+
expect(cache.full_each.to_a).to eq(
|
|
61
|
+
[["test-foo", {test1: true}], ["test2-bar", {test2: true}]]
|
|
62
|
+
)
|
|
63
|
+
cache.move_prefix('test-', 'test3-')
|
|
64
|
+
expect(cache.full_each.to_a).to eq(
|
|
65
|
+
[["test2-bar", {test2: true}], ["test3-foo", {test1: true}]]
|
|
66
|
+
)
|
|
67
|
+
end
|
|
68
|
+
|
|
54
69
|
it 'can delete' do
|
|
55
70
|
key, value = 'foo', { test: true }
|
|
56
71
|
expect(cache.delete(key)).to be_falsy
|
|
@@ -70,6 +85,20 @@ describe Documentrix::Documents::MemoryCache do
|
|
|
70
85
|
expect(k).to eq prefix + key
|
|
71
86
|
expect(v).to eq value
|
|
72
87
|
end
|
|
88
|
+
expect(cache.each.to_a).to eq [ ["test-foo", {test: true}] ]
|
|
89
|
+
expect(cache.to_a).to eq [ ["test-foo", {test: true}] ]
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
it "can iterate over the full cache's keys, values" do
|
|
93
|
+
key, value = 'foo', { test: true }
|
|
94
|
+
cache[key] = value
|
|
95
|
+
cache.prefix = 'test2-'
|
|
96
|
+
key, value = 'bar', { test2: true }
|
|
97
|
+
cache[key] = value
|
|
98
|
+
expect(cache.full_each.to_a).to eq [
|
|
99
|
+
["test-foo", {test: true}],
|
|
100
|
+
["test2-bar", {test2: true}],
|
|
101
|
+
]
|
|
73
102
|
end
|
|
74
103
|
|
|
75
104
|
it 'returns size' do
|
|
@@ -50,19 +50,10 @@ describe Documentrix::Documents::RedisCache do
|
|
|
50
50
|
|
|
51
51
|
it 'can set a value for a key' do
|
|
52
52
|
key, value = 'foo', { test: true }
|
|
53
|
-
expect(redis).to receive(:set).with(prefix + key, JSON(value)
|
|
53
|
+
expect(redis).to receive(:set).with(prefix + key, JSON(value))
|
|
54
54
|
cache[key] = value
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
it 'can set a value for a key with ttl' do
|
|
58
|
-
cache = described_class.new prefix:, url: 'something', ex: 3_600
|
|
59
|
-
key, value = 'foo', { test: true }
|
|
60
|
-
expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: 3_600)
|
|
61
|
-
cache[key] = value
|
|
62
|
-
expect(redis).to receive(:ttl).with(prefix + key).and_return 3_600
|
|
63
|
-
expect(cache.ttl(key)).to eq 3_600
|
|
64
|
-
end
|
|
65
|
-
|
|
66
57
|
it 'can determine if key exists' do
|
|
67
58
|
key = 'foo'
|
|
68
59
|
expect(redis).to receive(:exists?).with(prefix + key).and_return(false, true)
|
|
@@ -70,6 +61,17 @@ describe Documentrix::Documents::RedisCache do
|
|
|
70
61
|
expect(cache.key?('foo')).to eq true
|
|
71
62
|
end
|
|
72
63
|
|
|
64
|
+
it 'can move prefixes' do
|
|
65
|
+
expect(redis).to receive(:get).with(prefix + 'foo').and_return(JSON(foo: true))
|
|
66
|
+
expect(redis).to receive(:get).with('test2-bar').and_return(JSON(foo: true))
|
|
67
|
+
expect(redis).to receive(:set).with('test3-foo', '{"foo":true}')
|
|
68
|
+
expect(redis).to receive(:del).with('test-foo')
|
|
69
|
+
expect(redis).to receive(:scan_each).with(match: ?*).
|
|
70
|
+
and_yield("#{prefix}foo").
|
|
71
|
+
and_yield("test2-bar")
|
|
72
|
+
cache.move_prefix('test-', 'test3-')
|
|
73
|
+
end
|
|
74
|
+
|
|
73
75
|
it 'can delete' do
|
|
74
76
|
key = 'foo'
|
|
75
77
|
expect(redis).to receive(:del).with(prefix + key).and_return 1
|
|
@@ -80,7 +82,7 @@ describe Documentrix::Documents::RedisCache do
|
|
|
80
82
|
|
|
81
83
|
it 'can iterate over keys, values' do
|
|
82
84
|
key, value = 'foo', { 'test' => true }
|
|
83
|
-
expect(redis).to receive(:set).with(prefix + key, JSON(value)
|
|
85
|
+
expect(redis).to receive(:set).with(prefix + key, JSON(value))
|
|
84
86
|
cache[key] = value
|
|
85
87
|
expect(redis).to receive(:scan_each).with(match: "#{prefix}*").
|
|
86
88
|
and_yield("#{prefix}foo")
|
|
@@ -75,6 +75,27 @@ describe Documentrix::Documents::SQLiteCache do
|
|
|
75
75
|
expect(s).to eq 2
|
|
76
76
|
end
|
|
77
77
|
|
|
78
|
+
it 'can move prefixes' do
|
|
79
|
+
key, value = 'foo', test_value
|
|
80
|
+
cache[key] = value
|
|
81
|
+
cache.prefix = 'test2-'
|
|
82
|
+
key, value = 'bar', test_value
|
|
83
|
+
cache[key] = value
|
|
84
|
+
expect(cache.full_each.to_a).to eq(
|
|
85
|
+
[
|
|
86
|
+
["test-foo", Documentrix::Documents::Record[test_value]],
|
|
87
|
+
["test2-bar", Documentrix::Documents::Record[test_value]],
|
|
88
|
+
]
|
|
89
|
+
)
|
|
90
|
+
cache.move_prefix('test-', 'test3-')
|
|
91
|
+
expect(cache.full_each.to_a).to eq(
|
|
92
|
+
[
|
|
93
|
+
["test3-foo", Documentrix::Documents::Record[test_value]],
|
|
94
|
+
["test2-bar", Documentrix::Documents::Record[test_value]],
|
|
95
|
+
]
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
|
|
78
99
|
it 'can delete' do
|
|
79
100
|
key, value = 'foo', test_value
|
|
80
101
|
expect(cache.delete(key)).to be_falsy
|
|
@@ -136,6 +157,19 @@ describe Documentrix::Documents::SQLiteCache do
|
|
|
136
157
|
|
|
137
158
|
it 'can iterate over keys under a prefix' do
|
|
138
159
|
cache['foo'] = test_value
|
|
160
|
+
expect(cache.each.to_a).to eq [ [ 'test-foo', Documentrix::Documents::Record[test_value] ] ]
|
|
139
161
|
expect(cache.to_a).to eq [ [ 'test-foo', Documentrix::Documents::Record[test_value] ] ]
|
|
140
162
|
end
|
|
163
|
+
|
|
164
|
+
it "can iterate over the full cache's keys, values" do
|
|
165
|
+
key, value = 'foo', test_value
|
|
166
|
+
cache[key] = value
|
|
167
|
+
cache.prefix = 'test2-'
|
|
168
|
+
key, value = 'bar', test_value
|
|
169
|
+
cache[key] = value
|
|
170
|
+
expect(cache.full_each.to_a).to eq [
|
|
171
|
+
["test-foo", Documentrix::Documents::Record[test_value] ],
|
|
172
|
+
["test2-bar", Documentrix::Documents::Record[test_value] ],
|
|
173
|
+
]
|
|
174
|
+
end
|
|
141
175
|
end
|
data/spec/documents_spec.rb
CHANGED
|
@@ -162,6 +162,28 @@ describe Documentrix::Documents do
|
|
|
162
162
|
expect(documents.collections).to eq [ :default ]
|
|
163
163
|
end
|
|
164
164
|
|
|
165
|
+
it 'can rename collections' do
|
|
166
|
+
documents.collection = :foo
|
|
167
|
+
documents << 'foo'
|
|
168
|
+
expect(documents.collections).to eq %i[ default foo ]
|
|
169
|
+
documents.rename_collection(:bar)
|
|
170
|
+
expect(documents.collection).to eq :bar
|
|
171
|
+
expect(documents.collections).to eq %i[ default bar ]
|
|
172
|
+
expect(documents.exist?('foo')).to be true
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
it 'cannot rename collections into already existing ones' do
|
|
176
|
+
documents.collection = :foo
|
|
177
|
+
documents << 'foo'
|
|
178
|
+
expect(documents.collections).to eq %i[ default foo ]
|
|
179
|
+
documents.collection = :bar
|
|
180
|
+
documents << 'foo'
|
|
181
|
+
expect(documents.collections).to eq %i[ default foo bar ]
|
|
182
|
+
expect {
|
|
183
|
+
documents.rename_collection(:foo)
|
|
184
|
+
}.to raise_error(ArgumentError, 'new collection foo already exists!')
|
|
185
|
+
end
|
|
186
|
+
|
|
165
187
|
it 'can change collection' do
|
|
166
188
|
expect(documents.instance_eval { @cache }).to receive(:prefix=).
|
|
167
189
|
with(/#@collection/).and_call_original
|