active_record-vector 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +15 -0
- data/LICENSE +21 -0
- data/README.md +142 -0
- data/lib/active_record_vector/chunker.rb +82 -0
- data/lib/active_record_vector/distance.rb +56 -0
- data/lib/active_record_vector/migration_helper.rb +40 -0
- data/lib/active_record_vector/model.rb +145 -0
- data/lib/active_record_vector/providers/base.rb +24 -0
- data/lib/active_record_vector/providers/cohere.rb +42 -0
- data/lib/active_record_vector/providers/custom.rb +21 -0
- data/lib/active_record_vector/providers/ollama.rb +41 -0
- data/lib/active_record_vector/providers/openai.rb +44 -0
- data/lib/active_record_vector/version.rb +5 -0
- data/lib/active_record_vector.rb +43 -0
- metadata +138 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 24566066c866ee7bcbbc82f66d629de57cc0a1dc22ff2f10a807aa422129513a
|
|
4
|
+
data.tar.gz: 8577baf91f536e46467ae6d1552c1500515a772fd8d437e70408caa4e73e2407
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: c20374c7843a1c6a6b5443a808adfcb6e51184731af5d88588b091d4f75a7b4175fce58ca48881c7e5dc63cc614d12048dd7ffc463d92cbe8d8024d70d0df6c5
|
|
7
|
+
data.tar.gz: 2fc8a4903f0e88df02ec050158a2a10e358b6e5d8b48c6802829b2ec559a7db427ce7e6bc488a291054c6829d3a42d33b3148ea4ec8e5e34552c2576a3d3a614
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## [0.1.0] - 2026-08-03
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- Initial release of `active_record-vector`
|
|
9
|
+
- `has_vector` macro for ActiveRecord models
|
|
10
|
+
- Multi-provider AI embedding support: OpenAI, Ollama (100% free local AI), Cohere, Custom lambdas
|
|
11
|
+
- Vector distance metrics: Cosine similarity, Euclidean distance (L2), Inner product
|
|
12
|
+
- `semantic_search` and `nearest_to` ActiveRecord scopes
|
|
13
|
+
- `ActiveRecordVector::Chunker` for splitting long document text in RAG pipelines
|
|
14
|
+
- Migration DSL extensions (`add_vector_column`, `add_vector_index`)
|
|
15
|
+
- Support for PostgreSQL `pgvector` index operations (`HNSW`, `IVFFlat`)
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aditya
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<h1 align="center">🧠 active_record-vector</h1>
|
|
3
|
+
<p align="center">
|
|
4
|
+
<strong>Native AI vector embeddings, semantic search & RAG for Rails ActiveRecord</strong>
|
|
5
|
+
</p>
|
|
6
|
+
<p align="center">
|
|
7
|
+
<a href="https://rubygems.org/gems/active_record-vector"><img src="https://img.shields.io/gem/v/active_record-vector?color=%23e9573f" alt="Gem Version"></a>
|
|
8
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License"></a>
|
|
9
|
+
<a href="https://rubygems.org/gems/active_record-vector"><img src="https://img.shields.io/gem/dt/active_record-vector?color=green" alt="Downloads"></a>
|
|
10
|
+
</p>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
**`active_record-vector`** gives any Rails ActiveRecord model native AI vector embedding generation, semantic similarity search, and RAG (Retrieval-Augmented Generation) document chunking capabilities in 2 lines of code.
|
|
16
|
+
|
|
17
|
+
Works out-of-the-box with **OpenAI**, **Ollama** (100% free local AI), **Cohere**, **pgvector** (PostgreSQL), and SQLite/MySQL.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## ✨ Features
|
|
22
|
+
|
|
23
|
+
- 🤖 **`has_vector` Macro**: Automatically generates & updates AI vector embeddings on model save callbacks.
|
|
24
|
+
- 🔍 **Native ActiveRecord Scopes**: Chain `.semantic_search("query")` and `.nearest_to(vector)` directly with standard Rails queries (`where`, `limit`, `order`).
|
|
25
|
+
- 🆓 **Free Local AI via Ollama**: Generate embeddings 100% offline, locally, and free using `nomic-embed-text` or `all-minilm`.
|
|
26
|
+
- ⚡ **Multi-Provider Support**: OpenAI (`text-embedding-3-small`), Ollama, Cohere, or custom Procs/Lambdas.
|
|
27
|
+
- 📑 **RAG Text Chunker**: Built-in document text splitting utility (`ActiveRecordVector::Chunker`) with token-aware overlap.
|
|
28
|
+
- 🗄️ **PostgreSQL `pgvector` Integration**: Migration DSL extensions (`add_vector_column`, `add_vector_index`) supporting `HNSW` and `IVFFlat` indexes with fallback Ruby distance algorithms for SQLite/MySQL.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## 📦 Installation
|
|
33
|
+
|
|
34
|
+
Add to your Rails application's `Gemfile`:
|
|
35
|
+
|
|
36
|
+
```ruby
|
|
37
|
+
gem "active_record-vector"
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
And execute:
|
|
41
|
+
```bash
|
|
42
|
+
bundle install
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## 🚀 Quick Start
|
|
48
|
+
|
|
49
|
+
### 1. Define Model Vector Embeddings
|
|
50
|
+
|
|
51
|
+
Add `has_vector` to your ActiveRecord model:
|
|
52
|
+
|
|
53
|
+
```ruby
|
|
54
|
+
class Article < ApplicationRecord
|
|
55
|
+
has_vector :embedding,
|
|
56
|
+
provider: :openai, # :openai, :ollama, :cohere, or custom proc
|
|
57
|
+
model: "text-embedding-3-small",
|
|
58
|
+
from: [:title, :body], # concatenated automatically
|
|
59
|
+
auto_generate: true # before_save callback
|
|
60
|
+
end
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### 2. Semantic Similarity Search
|
|
64
|
+
|
|
65
|
+
Perform vector similarity searches using standard Rails scopes:
|
|
66
|
+
|
|
67
|
+
```ruby
|
|
68
|
+
# Semantic search by query text
|
|
69
|
+
Article.semantic_search("Ruby on Rails 8 performance", limit: 5)
|
|
70
|
+
|
|
71
|
+
# Chain with standard ActiveRecord queries
|
|
72
|
+
Article.where(published: true)
|
|
73
|
+
.semantic_search("AI integration", limit: 10)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## 🦙 Free Local AI Embeddings with Ollama
|
|
79
|
+
|
|
80
|
+
Generate embeddings 100% offline, privately, and for free using [Ollama](https://ollama.com/):
|
|
81
|
+
|
|
82
|
+
```ruby
|
|
83
|
+
class Article < ApplicationRecord
|
|
84
|
+
has_vector :embedding,
|
|
85
|
+
provider: :ollama,
|
|
86
|
+
model: "nomic-embed-text", # or "all-minilm"
|
|
87
|
+
host: "http://localhost:11434",
|
|
88
|
+
from: :body
|
|
89
|
+
end
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## 📑 RAG Document Text Chunker
|
|
95
|
+
|
|
96
|
+
Split long documents into overlapping chunks for embedding generation in RAG pipelines:
|
|
97
|
+
|
|
98
|
+
```ruby
|
|
99
|
+
# Split long document text
|
|
100
|
+
chunks = ActiveRecordVector::Chunker.split(long_text, chunk_size: 1000, chunk_overlap: 200)
|
|
101
|
+
|
|
102
|
+
chunks.each do |chunk_text|
|
|
103
|
+
article.chunks.create!(content: chunk_text) # auto-generates vector embedding
|
|
104
|
+
end
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## 🛠️ Rails Migration Helpers
|
|
110
|
+
|
|
111
|
+
```ruby
|
|
112
|
+
class AddEmbeddingToArticles < ActiveRecord::Migration[7.2]
|
|
113
|
+
def change
|
|
114
|
+
# Adds pgvector column (or text column fallback on SQLite)
|
|
115
|
+
add_vector_column :articles, :embedding, dimensions: 1536
|
|
116
|
+
|
|
117
|
+
# Adds HNSW vector index for high-speed similarity queries
|
|
118
|
+
add_vector_index :articles, :embedding, type: :hnsw, distance: :cosine
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## 🛠️ Local Development & Testing
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
git clone https://github.com/aditya-8108/active_record-vector.git
|
|
129
|
+
cd active_record-vector
|
|
130
|
+
|
|
131
|
+
bundle config set --local path 'vendor/bundle'
|
|
132
|
+
bundle install
|
|
133
|
+
|
|
134
|
+
# Run test suite
|
|
135
|
+
bundle exec rspec
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## 📄 License
|
|
141
|
+
|
|
142
|
+
Distributed under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordVector
|
|
4
|
+
# Document text chunking utility for RAG (Retrieval-Augmented Generation) pipelines.
|
|
5
|
+
# Splits long texts into overlapping chunks for embedding generation.
|
|
6
|
+
class Chunker
|
|
7
|
+
attr_reader :chunk_size, :chunk_overlap, :separators
|
|
8
|
+
|
|
9
|
+
DEFAULT_SEPARATORS = ["\n\n", "\n", ". ", " ", ""].freeze
|
|
10
|
+
|
|
11
|
+
def initialize(chunk_size: 1000, chunk_overlap: 200, separators: DEFAULT_SEPARATORS)
|
|
12
|
+
@chunk_size = chunk_size
|
|
13
|
+
@chunk_overlap = chunk_overlap
|
|
14
|
+
@separators = separators
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Convenience class method
|
|
18
|
+
def self.split(text, chunk_size: 1000, chunk_overlap: 200)
|
|
19
|
+
new(chunk_size: chunk_size, chunk_overlap: chunk_overlap).split(text)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Split text into array of chunk strings
|
|
23
|
+
def split(text)
|
|
24
|
+
return [] if text.nil? || text.strip.empty?
|
|
25
|
+
return [text.strip] if text.length <= @chunk_size
|
|
26
|
+
|
|
27
|
+
splits = split_text(text, @separators)
|
|
28
|
+
merge_splits(splits)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def split_text(text, separators)
|
|
34
|
+
separator = separators.find { |s| s.empty? || text.include?(s) } || ""
|
|
35
|
+
return text.chars if separator.empty?
|
|
36
|
+
|
|
37
|
+
parts = text.split(separator)
|
|
38
|
+
result = []
|
|
39
|
+
parts.each_with_index do |part, idx|
|
|
40
|
+
piece = idx.zero? ? part : "#{separator}#{part}"
|
|
41
|
+
if piece.length > @chunk_size && separators.length > 1
|
|
42
|
+
next_separators = separators[(separators.index(separator) + 1)..]
|
|
43
|
+
result.concat(split_text(piece, next_separators))
|
|
44
|
+
else
|
|
45
|
+
result << piece
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
result
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def merge_splits(splits)
|
|
52
|
+
chunks = []
|
|
53
|
+
current_chunk = []
|
|
54
|
+
current_length = 0
|
|
55
|
+
|
|
56
|
+
splits.each do |split|
|
|
57
|
+
split_len = split.length
|
|
58
|
+
|
|
59
|
+
if current_length + split_len > @chunk_size && current_chunk.any?
|
|
60
|
+
chunk_str = current_chunk.join.strip
|
|
61
|
+
chunks << chunk_str unless chunk_str.empty?
|
|
62
|
+
|
|
63
|
+
# Keep overlap
|
|
64
|
+
while current_length > @chunk_overlap && current_chunk.any?
|
|
65
|
+
removed = current_chunk.shift
|
|
66
|
+
current_length -= removed.length
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
current_chunk << split
|
|
71
|
+
current_length += split_len
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
if current_chunk.any?
|
|
75
|
+
final_str = current_chunk.join.strip
|
|
76
|
+
chunks << final_str unless final_str.empty?
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
chunks
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordVector
|
|
4
|
+
# Vector distance and similarity calculations.
|
|
5
|
+
module Distance
|
|
6
|
+
class << self
|
|
7
|
+
# Cosine similarity between two float arrays (returns value between -1.0 and 1.0)
|
|
8
|
+
def cosine_similarity(vec_a, vec_b)
|
|
9
|
+
return 0.0 if vec_a.nil? || vec_b.nil? || vec_a.empty? || vec_b.empty?
|
|
10
|
+
return 0.0 unless vec_a.length == vec_b.length
|
|
11
|
+
|
|
12
|
+
dot = 0.0
|
|
13
|
+
norm_a = 0.0
|
|
14
|
+
norm_b = 0.0
|
|
15
|
+
|
|
16
|
+
vec_a.each_with_index do |val, idx|
|
|
17
|
+
b_val = vec_b[idx]
|
|
18
|
+
dot += val * b_val
|
|
19
|
+
norm_a += val * val
|
|
20
|
+
norm_b += b_val * b_val
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
denom = Math.sqrt(norm_a) * Math.sqrt(norm_b)
|
|
24
|
+
return 0.0 if denom.zero?
|
|
25
|
+
|
|
26
|
+
dot / denom
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Euclidean distance (L2 norm) between two float arrays
|
|
30
|
+
def euclidean_distance(vec_a, vec_b)
|
|
31
|
+
return Float::INFINITY if vec_a.nil? || vec_b.nil? || vec_a.empty? || vec_b.empty?
|
|
32
|
+
return Float::INFINITY unless vec_a.length == vec_b.length
|
|
33
|
+
|
|
34
|
+
sum = 0.0
|
|
35
|
+
vec_a.each_with_index do |val, idx|
|
|
36
|
+
diff = val - vec_b[idx]
|
|
37
|
+
sum += diff * diff
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
Math.sqrt(sum)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Dot product (inner product) between two float arrays
|
|
44
|
+
def inner_product(vec_a, vec_b)
|
|
45
|
+
return 0.0 if vec_a.nil? || vec_b.nil? || vec_a.empty? || vec_b.empty?
|
|
46
|
+
return 0.0 unless vec_a.length == vec_b.length
|
|
47
|
+
|
|
48
|
+
sum = 0.0
|
|
49
|
+
vec_a.each_with_index do |val, idx|
|
|
50
|
+
sum += val * vec_b[idx]
|
|
51
|
+
end
|
|
52
|
+
sum
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordVector
|
|
4
|
+
# Helper module providing migration DSL extensions for adding vector columns and indexes.
|
|
5
|
+
module MigrationHelper
|
|
6
|
+
# Add vector column to a table (supports PostgreSQL pgvector and JSON fallback)
|
|
7
|
+
def add_vector_column(table_name, column_name, dimensions: 1536, **options)
|
|
8
|
+
if postgresql_adapter?
|
|
9
|
+
execute "CREATE EXTENSION IF NOT EXISTS vector;"
|
|
10
|
+
execute "ALTER TABLE #{table_name} ADD COLUMN #{column_name} vector(#{dimensions});"
|
|
11
|
+
else
|
|
12
|
+
add_column table_name, column_name, :text, **options
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Add HNSW or IVFFlat vector index to a table
|
|
17
|
+
def add_vector_index(table_name, column_name, type: :hnsw, distance: :cosine)
|
|
18
|
+
return unless postgresql_adapter?
|
|
19
|
+
|
|
20
|
+
ops_class = case distance.to_sym
|
|
21
|
+
when :cosine then "vector_cosine_ops"
|
|
22
|
+
when :inner_product, :dot then "vector_ip_ops"
|
|
23
|
+
else "vector_l2_ops"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
index_type = type.to_s.upcase
|
|
27
|
+
index_name = "index_#{table_name}_on_#{column_name}_#{type}"
|
|
28
|
+
|
|
29
|
+
execute "CREATE INDEX IF NOT EXISTS #{index_name} ON #{table_name} USING #{index_type} (#{column_name} #{ops_class});"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def postgresql_adapter?
|
|
35
|
+
respond_to?(:adapter_name) && adapter_name.to_s.match?(/postgres/i)
|
|
36
|
+
rescue StandardError
|
|
37
|
+
false
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordVector
|
|
4
|
+
# Module included in ActiveRecord models to provide vector capabilities.
|
|
5
|
+
module Model
|
|
6
|
+
extend ActiveSupport::Concern if defined?(ActiveSupport::Concern)
|
|
7
|
+
|
|
8
|
+
def self.included(base)
|
|
9
|
+
base.extend(ClassMethods)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
module ClassMethods
|
|
13
|
+
# Macro to enable vector capabilities on a model attribute
|
|
14
|
+
#
|
|
15
|
+
# Example:
|
|
16
|
+
# has_vector :embedding,
|
|
17
|
+
# provider: :openai,
|
|
18
|
+
# model: "text-embedding-3-small",
|
|
19
|
+
# from: [:title, :body],
|
|
20
|
+
# auto_generate: true
|
|
21
|
+
def has_vector(attribute = :embedding, options = {})
|
|
22
|
+
@vector_configs ||= {}
|
|
23
|
+
@vector_configs[attribute.to_sym] = {
|
|
24
|
+
provider: options[:provider] || :openai,
|
|
25
|
+
model: options[:model],
|
|
26
|
+
from: Array(options[:from] || :content),
|
|
27
|
+
auto_generate: options.fetch(:auto_generate, true),
|
|
28
|
+
provider_options: options
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
# Setup auto-generation callback if supported
|
|
32
|
+
if options.fetch(:auto_generate, true) && respond_to?(:before_save)
|
|
33
|
+
before_save do
|
|
34
|
+
generate_vector_embeddings(attribute.to_sym)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Define semantic search scopes if supported
|
|
39
|
+
return unless respond_to?(:scope)
|
|
40
|
+
|
|
41
|
+
scope :semantic_search, lambda { |query, target_attribute: attribute, limit: 10, distance: :cosine|
|
|
42
|
+
nearest_to(query, attribute: target_attribute, distance: distance).limit(limit)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
scope :nearest_to, lambda { |query_or_vector, target_attribute: attribute, distance: :cosine|
|
|
46
|
+
cfg = @vector_configs[target_attribute.to_sym] || {}
|
|
47
|
+
query_vector = if query_or_vector.is_a?(Array)
|
|
48
|
+
query_or_vector
|
|
49
|
+
else
|
|
50
|
+
ActiveRecordVector.provider_for(cfg[:provider], cfg[:provider_options]).embed(query_or_vector.to_s)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
if pgvector_supported?
|
|
54
|
+
pgvector_nearest(query_vector, target_attribute, distance)
|
|
55
|
+
else
|
|
56
|
+
ruby_vector_nearest(query_vector, target_attribute, distance)
|
|
57
|
+
end
|
|
58
|
+
}
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def vector_configs
|
|
62
|
+
@vector_configs ||= {}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
def pgvector_supported?
|
|
68
|
+
respond_to?(:connection) && connection.class.name.include?("PostgreSQL")
|
|
69
|
+
rescue StandardError
|
|
70
|
+
false
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def pgvector_nearest(vector, attribute, distance)
|
|
74
|
+
vector_str = "[#{vector.join(',')}]"
|
|
75
|
+
col_name = connection.quote_column_name(attribute)
|
|
76
|
+
|
|
77
|
+
op = case distance.to_sym
|
|
78
|
+
when :cosine then "<=>"
|
|
79
|
+
when :inner_product, :dot then "<#>"
|
|
80
|
+
else "<->" # Euclidean L2
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
order("#{col_name} #{op} '#{vector_str}'")
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def ruby_vector_nearest(vector, attribute, distance)
|
|
87
|
+
# Fetch records and sort in memory using ActiveRecordVector::Distance
|
|
88
|
+
records = all.to_a
|
|
89
|
+
sorted = records.sort_by do |record|
|
|
90
|
+
rec_vec = record.public_send(attribute)
|
|
91
|
+
next Float::INFINITY if rec_vec.nil?
|
|
92
|
+
|
|
93
|
+
# Parse vector if stored as JSON string
|
|
94
|
+
rec_vec = JSON.parse(rec_vec) if rec_vec.is_a?(String)
|
|
95
|
+
|
|
96
|
+
case distance.to_sym
|
|
97
|
+
when :cosine
|
|
98
|
+
-Distance.cosine_similarity(vector, rec_vec) # descending similarity
|
|
99
|
+
when :inner_product, :dot
|
|
100
|
+
-Distance.inner_product(vector, rec_vec)
|
|
101
|
+
else
|
|
102
|
+
Distance.euclidean_distance(vector, rec_vec) # ascending distance
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Return a scope-like relation using primary keys
|
|
107
|
+
ids = sorted.map(&:id)
|
|
108
|
+
return where(id: nil) if ids.empty?
|
|
109
|
+
|
|
110
|
+
begin
|
|
111
|
+
where(id: ids).in_order_of(:id, ids)
|
|
112
|
+
rescue StandardError
|
|
113
|
+
where(id: ids)
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Instance Methods
|
|
119
|
+
def generate_vector_embeddings(attribute = :embedding)
|
|
120
|
+
cfg = self.class.vector_configs[attribute.to_sym]
|
|
121
|
+
return unless cfg
|
|
122
|
+
|
|
123
|
+
source_text = vector_source_text(attribute)
|
|
124
|
+
return if source_text.strip.empty?
|
|
125
|
+
|
|
126
|
+
provider = ActiveRecordVector.provider_for(cfg[:provider], cfg[:provider_options])
|
|
127
|
+
vector = provider.embed(source_text)
|
|
128
|
+
|
|
129
|
+
public_send("#{attribute}=", vector)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def vector_source_text(attribute = :embedding)
|
|
133
|
+
cfg = self.class.vector_configs[attribute.to_sym]
|
|
134
|
+
return "" unless cfg
|
|
135
|
+
|
|
136
|
+
from_attrs = cfg[:from]
|
|
137
|
+
parts = from_attrs.filter_map do |attr|
|
|
138
|
+
val = respond_to?(attr) ? public_send(attr) : nil
|
|
139
|
+
val.to_s.strip unless val.nil? || val.to_s.strip.empty?
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
parts.join("\n\n")
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordVector
|
|
4
|
+
module Providers
|
|
5
|
+
# Abstract base class for embedding providers.
|
|
6
|
+
class Base
|
|
7
|
+
attr_reader :options
|
|
8
|
+
|
|
9
|
+
def initialize(options = {})
|
|
10
|
+
@options = options
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Generate embedding vector array for input text string
|
|
14
|
+
def embed(_text)
|
|
15
|
+
raise NotImplementedError, "#{self.class.name}#embed must be implemented"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Generate batch embedding vector arrays for array of text strings
|
|
19
|
+
def embed_batch(texts)
|
|
20
|
+
texts.map { |text| embed(text) }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module ActiveRecordVector
|
|
8
|
+
module Providers
|
|
9
|
+
# Cohere Embedding Provider
|
|
10
|
+
class Cohere < Base
|
|
11
|
+
DEFAULT_MODEL = "embed-english-v3.0"
|
|
12
|
+
API_ENDPOINT = "https://api.cohere.com/v1/embed"
|
|
13
|
+
|
|
14
|
+
def embed(text)
|
|
15
|
+
api_key = options[:api_key] || ENV.fetch("COHERE_API_KEY", nil)
|
|
16
|
+
raise Error, "Cohere API key missing. Set ENV['COHERE_API_KEY'] or pass api_key: '...'" unless api_key
|
|
17
|
+
|
|
18
|
+
uri = URI.parse(options[:endpoint] || API_ENDPOINT)
|
|
19
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
20
|
+
http.use_ssl = (uri.scheme == "https")
|
|
21
|
+
|
|
22
|
+
payload = {
|
|
23
|
+
texts: [text],
|
|
24
|
+
model: options[:model] || DEFAULT_MODEL,
|
|
25
|
+
input_type: options[:input_type] || "search_document"
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
request = Net::HTTP::Post.new(uri.request_uri, {
|
|
29
|
+
"Content-Type" => "application/json",
|
|
30
|
+
"Authorization" => "Bearer #{api_key}"
|
|
31
|
+
})
|
|
32
|
+
request.body = payload.to_json
|
|
33
|
+
|
|
34
|
+
response = http.request(request)
|
|
35
|
+
raise Error, "Cohere API Error (#{response.code}): #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
|
36
|
+
|
|
37
|
+
data = JSON.parse(response.body)
|
|
38
|
+
data.dig("embeddings", 0) || []
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordVector
|
|
4
|
+
module Providers
|
|
5
|
+
# Custom provider allowing any Proc, Lambda, or custom object responding to #call or #embed
|
|
6
|
+
class Custom < Base
|
|
7
|
+
def embed(text)
|
|
8
|
+
callable = options[:with] || options[:proc]
|
|
9
|
+
raise Error, "Custom provider requires options[:with] to be a Proc or object responding to #call" unless callable
|
|
10
|
+
|
|
11
|
+
if callable.respond_to?(:call)
|
|
12
|
+
callable.call(text)
|
|
13
|
+
elsif callable.respond_to?(:embed)
|
|
14
|
+
callable.embed(text)
|
|
15
|
+
else
|
|
16
|
+
raise Error, "Custom provider options[:with] must respond to #call or #embed"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module ActiveRecordVector
|
|
8
|
+
module Providers
|
|
9
|
+
# Ollama Embedding Provider for 100% free, local, private offline AI embeddings
|
|
10
|
+
class Ollama < Base
|
|
11
|
+
DEFAULT_MODEL = "nomic-embed-text"
|
|
12
|
+
DEFAULT_HOST = "http://localhost:11434"
|
|
13
|
+
|
|
14
|
+
def embed(text)
|
|
15
|
+
base_host = options[:host] || ENV["OLLAMA_HOST"] || DEFAULT_HOST
|
|
16
|
+
uri = URI.parse("#{base_host.chomp('/')}/api/embeddings")
|
|
17
|
+
|
|
18
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
19
|
+
http.use_ssl = (uri.scheme == "https")
|
|
20
|
+
http.open_timeout = options[:timeout] || 15
|
|
21
|
+
http.read_timeout = options[:timeout] || 15
|
|
22
|
+
|
|
23
|
+
payload = {
|
|
24
|
+
model: options[:model] || DEFAULT_MODEL,
|
|
25
|
+
prompt: text
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
request = Net::HTTP::Post.new(uri.request_uri, {
|
|
29
|
+
"Content-Type" => "application/json"
|
|
30
|
+
})
|
|
31
|
+
request.body = payload.to_json
|
|
32
|
+
|
|
33
|
+
response = http.request(request)
|
|
34
|
+
raise Error, "Ollama API Error (#{response.code}): #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
|
35
|
+
|
|
36
|
+
data = JSON.parse(response.body)
|
|
37
|
+
data["embedding"] || []
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module ActiveRecordVector
|
|
8
|
+
module Providers
|
|
9
|
+
# OpenAI Embedding Provider (supports text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002)
|
|
10
|
+
class Openai < Base
|
|
11
|
+
DEFAULT_MODEL = "text-embedding-3-small"
|
|
12
|
+
API_ENDPOINT = "https://api.openai.com/v1/embeddings"
|
|
13
|
+
|
|
14
|
+
def embed(text)
|
|
15
|
+
api_key = options[:api_key] || ENV.fetch("OPENAI_API_KEY", nil)
|
|
16
|
+
raise Error, "OpenAI API key missing. Set ENV['OPENAI_API_KEY'] or pass api_key: '...' to has_vector" unless api_key
|
|
17
|
+
|
|
18
|
+
uri = URI.parse(options[:endpoint] || API_ENDPOINT)
|
|
19
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
20
|
+
http.use_ssl = (uri.scheme == "https")
|
|
21
|
+
http.open_timeout = options[:timeout] || 10
|
|
22
|
+
http.read_timeout = options[:timeout] || 10
|
|
23
|
+
|
|
24
|
+
payload = {
|
|
25
|
+
input: text,
|
|
26
|
+
model: options[:model] || DEFAULT_MODEL
|
|
27
|
+
}
|
|
28
|
+
payload[:dimensions] = options[:dimensions] if options[:dimensions]
|
|
29
|
+
|
|
30
|
+
request = Net::HTTP::Post.new(uri.request_uri, {
|
|
31
|
+
"Content-Type" => "application/json",
|
|
32
|
+
"Authorization" => "Bearer #{api_key}"
|
|
33
|
+
})
|
|
34
|
+
request.body = payload.to_json
|
|
35
|
+
|
|
36
|
+
response = http.request(request)
|
|
37
|
+
raise Error, "OpenAI API Error (#{response.code}): #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
|
38
|
+
|
|
39
|
+
data = JSON.parse(response.body)
|
|
40
|
+
data.dig("data", 0, "embedding") || []
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "active_record_vector/version"
|
|
4
|
+
require_relative "active_record_vector/distance"
|
|
5
|
+
require_relative "active_record_vector/chunker"
|
|
6
|
+
require_relative "active_record_vector/providers/base"
|
|
7
|
+
require_relative "active_record_vector/providers/openai"
|
|
8
|
+
require_relative "active_record_vector/providers/ollama"
|
|
9
|
+
require_relative "active_record_vector/providers/cohere"
|
|
10
|
+
require_relative "active_record_vector/providers/custom"
|
|
11
|
+
require_relative "active_record_vector/model"
|
|
12
|
+
require_relative "active_record_vector/migration_helper"
|
|
13
|
+
|
|
14
|
+
module ActiveRecordVector
|
|
15
|
+
class Error < StandardError; end
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
# Provider registry lookup
|
|
19
|
+
def provider_for(provider_name, options = {})
|
|
20
|
+
case provider_name.to_sym
|
|
21
|
+
when :openai
|
|
22
|
+
Providers::Openai.new(options)
|
|
23
|
+
when :ollama
|
|
24
|
+
Providers::Ollama.new(options)
|
|
25
|
+
when :cohere
|
|
26
|
+
Providers::Cohere.new(options)
|
|
27
|
+
when :custom
|
|
28
|
+
Providers::Custom.new(options)
|
|
29
|
+
else
|
|
30
|
+
if provider_name.is_a?(Class) && provider_name < Providers::Base
|
|
31
|
+
provider_name.new(options)
|
|
32
|
+
elsif provider_name.respond_to?(:call) || provider_name.respond_to?(:embed)
|
|
33
|
+
Providers::Custom.new(options.merge(with: provider_name))
|
|
34
|
+
else
|
|
35
|
+
raise Error, "Unknown embedding provider: #{provider_name.inspect}"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Auto-hook into ActiveRecord if defined
|
|
43
|
+
ActiveRecord::Base.include(ActiveRecordVector::Model) if defined?(ActiveRecord::Base)
|
metadata
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: active_record-vector
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Aditya
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-08-03 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: activesupport
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '6.0'
|
|
20
|
+
- - "<"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '9.0'
|
|
23
|
+
type: :runtime
|
|
24
|
+
prerelease: false
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
26
|
+
requirements:
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: '6.0'
|
|
30
|
+
- - "<"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '9.0'
|
|
33
|
+
- !ruby/object:Gem::Dependency
|
|
34
|
+
name: bundler
|
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '2.0'
|
|
40
|
+
type: :development
|
|
41
|
+
prerelease: false
|
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - "~>"
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '2.0'
|
|
47
|
+
- !ruby/object:Gem::Dependency
|
|
48
|
+
name: rake
|
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - "~>"
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '13.0'
|
|
54
|
+
type: :development
|
|
55
|
+
prerelease: false
|
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - "~>"
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '13.0'
|
|
61
|
+
- !ruby/object:Gem::Dependency
|
|
62
|
+
name: rspec
|
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - "~>"
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '3.0'
|
|
68
|
+
type: :development
|
|
69
|
+
prerelease: false
|
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - "~>"
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '3.0'
|
|
75
|
+
- !ruby/object:Gem::Dependency
|
|
76
|
+
name: rubocop
|
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '1.0'
|
|
82
|
+
type: :development
|
|
83
|
+
prerelease: false
|
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '1.0'
|
|
89
|
+
description: An elegant, lightweight Ruby gem bringing native AI vector embeddings,
|
|
90
|
+
semantic similarity search, and RAG document chunking to Rails ActiveRecord models.
|
|
91
|
+
Supports OpenAI, Ollama (free local AI), Cohere, and pgvector.
|
|
92
|
+
email:
|
|
93
|
+
executables: []
|
|
94
|
+
extensions: []
|
|
95
|
+
extra_rdoc_files: []
|
|
96
|
+
files:
|
|
97
|
+
- CHANGELOG.md
|
|
98
|
+
- LICENSE
|
|
99
|
+
- README.md
|
|
100
|
+
- lib/active_record_vector.rb
|
|
101
|
+
- lib/active_record_vector/chunker.rb
|
|
102
|
+
- lib/active_record_vector/distance.rb
|
|
103
|
+
- lib/active_record_vector/migration_helper.rb
|
|
104
|
+
- lib/active_record_vector/model.rb
|
|
105
|
+
- lib/active_record_vector/providers/base.rb
|
|
106
|
+
- lib/active_record_vector/providers/cohere.rb
|
|
107
|
+
- lib/active_record_vector/providers/custom.rb
|
|
108
|
+
- lib/active_record_vector/providers/ollama.rb
|
|
109
|
+
- lib/active_record_vector/providers/openai.rb
|
|
110
|
+
- lib/active_record_vector/version.rb
|
|
111
|
+
homepage: https://github.com/aditya-8108/active_record-vector
|
|
112
|
+
licenses:
|
|
113
|
+
- MIT
|
|
114
|
+
metadata:
|
|
115
|
+
homepage_uri: https://github.com/aditya-8108/active_record-vector
|
|
116
|
+
source_code_uri: https://github.com/aditya-8108/active_record-vector
|
|
117
|
+
changelog_uri: https://github.com/aditya-8108/active_record-vector/blob/main/CHANGELOG.md
|
|
118
|
+
rubygems_mfa_required: 'true'
|
|
119
|
+
post_install_message:
|
|
120
|
+
rdoc_options: []
|
|
121
|
+
require_paths:
|
|
122
|
+
- lib
|
|
123
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
124
|
+
requirements:
|
|
125
|
+
- - ">="
|
|
126
|
+
- !ruby/object:Gem::Version
|
|
127
|
+
version: 3.0.0
|
|
128
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
129
|
+
requirements:
|
|
130
|
+
- - ">="
|
|
131
|
+
- !ruby/object:Gem::Version
|
|
132
|
+
version: '0'
|
|
133
|
+
requirements: []
|
|
134
|
+
rubygems_version: 3.3.5
|
|
135
|
+
signing_key:
|
|
136
|
+
specification_version: 4
|
|
137
|
+
summary: Native vector embeddings, semantic search & RAG for Rails ActiveRecord
|
|
138
|
+
test_files: []
|