embed_sync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +18 -0
- data/LICENSE.txt +21 -0
- data/README.md +187 -0
- data/Rakefile +12 -0
- data/lib/embed_sync/backfill.rb +33 -0
- data/lib/embed_sync/configuration.rb +62 -0
- data/lib/embed_sync/definition.rb +75 -0
- data/lib/embed_sync/embed_job.rb +35 -0
- data/lib/embed_sync/errors.rb +18 -0
- data/lib/embed_sync/log_subscriber.rb +22 -0
- data/lib/embed_sync/model.rb +93 -0
- data/lib/embed_sync/providers/base.rb +13 -0
- data/lib/embed_sync/providers/fake.rb +33 -0
- data/lib/embed_sync/providers/ruby_llm.rb +56 -0
- data/lib/embed_sync/providers.rb +34 -0
- data/lib/embed_sync/query.rb +27 -0
- data/lib/embed_sync/railtie.rb +19 -0
- data/lib/embed_sync/sync.rb +90 -0
- data/lib/embed_sync/tasks/embed_sync.rake +45 -0
- data/lib/embed_sync/version.rb +5 -0
- data/lib/embed_sync.rb +98 -0
- data/lib/generators/embed_sync/embed_sync_generator.rb +89 -0
- data/lib/generators/embed_sync/install_generator.rb +18 -0
- data/lib/generators/embed_sync/templates/initializer.rb.tt +23 -0
- data/lib/generators/embed_sync/templates/migration.rb.tt +12 -0
- data/sig/embed_sync.rbs +4 -0
- metadata +155 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EmbedSync
|
|
4
|
+
module Providers
|
|
5
|
+
# Generates embeddings with the ruby_llm gem (OpenAI, Gemini, Voyage, Bedrock, Ollama, ...).
|
|
6
|
+
# API keys are configured through RubyLLM.configure, not here.
|
|
7
|
+
#
|
|
8
|
+
# EmbedSync.config.provider = :ruby_llm
|
|
9
|
+
# EmbedSync.config.provider_options = { dimensions: 512, provider: :openai }
|
|
10
|
+
class RubyLLM < Base
|
|
11
|
+
EMBED_OPTIONS = %i[dimensions provider assume_model_exists task_type provider_options].freeze
|
|
12
|
+
INVALID_REQUEST_ERRORS = %w[BadRequestError ContextLengthExceededError UnauthorizedError ForbiddenError
|
|
13
|
+
PaymentRequiredError ContentFilterError ModelNotFoundError].freeze
|
|
14
|
+
|
|
15
|
+
def initialize(**options)
|
|
16
|
+
super()
|
|
17
|
+
unknown = options.keys - EMBED_OPTIONS
|
|
18
|
+
raise ConfigurationError, "Unknown ruby_llm provider options: #{unknown.join(", ")}" if unknown.any?
|
|
19
|
+
|
|
20
|
+
@options = options.compact
|
|
21
|
+
load_ruby_llm
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def embed(texts, model:)
|
|
25
|
+
vectors = ::RubyLLM.embed(texts, model: model, **@options).vectors
|
|
26
|
+
# RubyLLM returns a flat vector when it embedded a single text.
|
|
27
|
+
vectors.first.is_a?(Numeric) ? [vectors] : vectors
|
|
28
|
+
rescue *invalid_request_errors => e
|
|
29
|
+
raise InvalidRequestError, "#{e.class}: #{e.message}"
|
|
30
|
+
rescue *retryable_errors => e
|
|
31
|
+
raise ProviderError, "#{e.class}: #{e.message}"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
# The request itself was rejected; sending it again gives the same answer.
|
|
37
|
+
def invalid_request_errors
|
|
38
|
+
INVALID_REQUEST_ERRORS.filter_map do |name|
|
|
39
|
+
::RubyLLM.const_get(name, false) if ::RubyLLM.const_defined?(name, false)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Everything else from the API (rate limits, 5xx, overloaded) plus network failures, which ruby_llm
|
|
44
|
+
# lets through from Faraday.
|
|
45
|
+
def retryable_errors
|
|
46
|
+
[::RubyLLM::Error, (::Faraday::Error if defined?(::Faraday::Error))].compact
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def load_ruby_llm
|
|
50
|
+
require "ruby_llm" unless defined?(::RubyLLM.embed)
|
|
51
|
+
rescue LoadError
|
|
52
|
+
raise ConfigurationError, 'The :ruby_llm provider needs the ruby_llm gem. Add gem "ruby_llm" to your Gemfile.'
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EmbedSync
|
|
4
|
+
# Registry of embedding providers, looked up by name.
|
|
5
|
+
module Providers
|
|
6
|
+
@registry = {}
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
# The class is given as a string so a provider's dependencies only load when it is used.
|
|
10
|
+
def register(name, class_name)
|
|
11
|
+
@registry[name.to_sym] = class_name
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def registered
|
|
15
|
+
@registry.keys
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def build(name, **)
|
|
19
|
+
class_name = @registry.fetch(name.to_sym) do
|
|
20
|
+
raise ConfigurationError,
|
|
21
|
+
"Unknown provider #{name.inspect}. Registered providers: #{registered.join(", ")}"
|
|
22
|
+
end
|
|
23
|
+
require_relative "providers/ruby_llm" if class_name == "EmbedSync::Providers::RubyLLM"
|
|
24
|
+
Object.const_get(class_name).new(**)
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
require_relative "providers/base"
|
|
31
|
+
require_relative "providers/fake"
|
|
32
|
+
|
|
33
|
+
EmbedSync::Providers.register(:fake, "EmbedSync::Providers::Fake")
|
|
34
|
+
EmbedSync::Providers.register(:ruby_llm, "EmbedSync::Providers::RubyLLM")
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EmbedSync
|
|
4
|
+
# Embeds search queries, optionally caching them (config.query_cache = Rails.cache).
|
|
5
|
+
module Query
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def embed(text, definition)
|
|
9
|
+
raise ArgumentError, "Search query can't be blank" if text.blank?
|
|
10
|
+
|
|
11
|
+
cache = EmbedSync.config.query_cache
|
|
12
|
+
return embed_now(text, definition) unless cache
|
|
13
|
+
|
|
14
|
+
cache.fetch(cache_key(text, definition), expires_in: EmbedSync.config.query_cache_expires_in) do
|
|
15
|
+
embed_now(text, definition)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def embed_now(text, definition)
|
|
20
|
+
Sync.embed_texts([text.to_s], definition).first
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def cache_key(text, definition)
|
|
24
|
+
"embed_sync/query/#{definition.model_name}/#{definition.dimensions}/#{Digest::SHA256.hexdigest(text.to_s)}"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EmbedSync
|
|
4
|
+
# Hooks EmbedSync into Rails: rake tasks, logging and job settings.
|
|
5
|
+
class Railtie < Rails::Railtie
|
|
6
|
+
rake_tasks do
|
|
7
|
+
load File.expand_path("tasks/embed_sync.rake", __dir__)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
initializer "embed_sync.log_subscriber" do
|
|
11
|
+
require_relative "log_subscriber"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
config.after_initialize do
|
|
15
|
+
limit = EmbedSync.config.concurrency_limit
|
|
16
|
+
EmbedSync::EmbedJob.apply_concurrency_limit(limit) if limit
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module EmbedSync
|
|
4
|
+
# Embeds a batch of records for one has_embedding column with a single provider call.
|
|
5
|
+
module Sync
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
# Returns the number of records written. Fresh records are skipped unless force: true.
|
|
9
|
+
# Records with blank source text get a nil vector without calling the provider.
|
|
10
|
+
# skip_invalid: records the provider rejects (e.g. text too long) are left stale instead of raising.
|
|
11
|
+
# recheck: before writing, reload the record under a row lock and skip it if its text changed while the
|
|
12
|
+
# provider was working (the save that changed it queues its own embedding).
|
|
13
|
+
def call(records, column, force: false, skip_invalid: false, recheck: false)
|
|
14
|
+
return 0 if records.empty?
|
|
15
|
+
|
|
16
|
+
definition = records.first.class.embed_sync_definition(column)
|
|
17
|
+
pending = pending_records(records, definition, force)
|
|
18
|
+
return 0 if pending.empty?
|
|
19
|
+
|
|
20
|
+
vectors = embed_pending(pending.map(&:last), definition, skip_invalid)
|
|
21
|
+
pending.zip(vectors).count do |(record, text), vector|
|
|
22
|
+
vector != :rejected && write(record, definition, text, vector, recheck)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def pending_records(records, definition, force)
|
|
27
|
+
records.filter_map do |record|
|
|
28
|
+
[record, definition.source_text(record)] if force || definition.stale?(record)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# One provider call for the whole batch. If the provider rejects it, embed the texts one at a time so
|
|
33
|
+
# a single bad record doesn't block the others; the rejected ones come back as :rejected.
|
|
34
|
+
def embed_pending(texts, definition, skip_invalid)
|
|
35
|
+
embed_with_blanks(texts, definition)
|
|
36
|
+
rescue InvalidRequestError
|
|
37
|
+
raise unless skip_invalid
|
|
38
|
+
return [:rejected] if texts.one?
|
|
39
|
+
|
|
40
|
+
texts.map { |text| embed_pending([text], definition, skip_invalid).first }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Blank texts get nil without being sent to the provider.
|
|
44
|
+
def embed_with_blanks(texts, definition)
|
|
45
|
+
vectors = embed_texts(texts.select(&:present?), definition).each
|
|
46
|
+
texts.map { |text| text.present? ? vectors.next : nil }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def write(record, definition, text, vector, recheck)
|
|
50
|
+
attributes = { definition.column => vector, definition.digest_column => definition.digest(text) }
|
|
51
|
+
return record.update_columns(attributes) unless recheck
|
|
52
|
+
|
|
53
|
+
record.with_lock do
|
|
54
|
+
next false unless definition.source_text(record) == text
|
|
55
|
+
|
|
56
|
+
record.update_columns(attributes)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def embed_texts(texts, definition)
|
|
61
|
+
return [] if texts.empty?
|
|
62
|
+
|
|
63
|
+
vectors = instrument(texts, definition) { EmbedSync.provider.embed(texts, model: definition.model_name) }
|
|
64
|
+
unless vectors.is_a?(Array) && vectors.size == texts.size
|
|
65
|
+
raise ProviderError, "Expected #{texts.size} vectors from the provider, got #{vectors.try(:size).inspect}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
check_dimensions!(vectors, definition)
|
|
69
|
+
|
|
70
|
+
vectors
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def check_dimensions!(vectors, definition)
|
|
74
|
+
return unless definition.dimensions
|
|
75
|
+
|
|
76
|
+
wrong = vectors.find { |vector| vector.size != definition.dimensions }
|
|
77
|
+
return unless wrong
|
|
78
|
+
|
|
79
|
+
raise DimensionMismatchError, "#{definition.column} expects #{definition.dimensions} dimensions, " \
|
|
80
|
+
"but #{definition.model_name} returned #{wrong.size}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Publishes "embed.embed_sync" with record_class, column, model and count (and exception, if raised).
|
|
84
|
+
def instrument(texts, definition, &)
|
|
85
|
+
payload = { record_class: definition.owner&.name, column: definition.column,
|
|
86
|
+
model: definition.model_name, count: texts.size }
|
|
87
|
+
ActiveSupport::Notifications.instrument("embed.embed_sync", payload, &)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :embed_sync do
|
|
4
|
+
find_models = lambda do |name|
|
|
5
|
+
if name.present?
|
|
6
|
+
[name.constantize]
|
|
7
|
+
else
|
|
8
|
+
Rails.application.eager_load! if defined?(Rails.application)
|
|
9
|
+
ActiveRecord::Base.descendants.select { |klass| klass.embed_sync_definitions.any? && !klass.abstract_class? }
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Without a model name, each embedding is visited once, on the class that declared it: STI subclasses
|
|
14
|
+
# inherit their parent's has_embedding and their rows are already covered by the parent's table scan.
|
|
15
|
+
each_embedding = lambda do |args, &block|
|
|
16
|
+
explicit = args[:model].present?
|
|
17
|
+
find_models.call(args[:model]).each do |model|
|
|
18
|
+
definitions = model.embed_sync_definitions.values
|
|
19
|
+
definitions = definitions.select { |d| d.column.to_s == args[:column].to_s } if args[:column].present?
|
|
20
|
+
definitions = definitions.select { |d| d.owner == model } unless explicit
|
|
21
|
+
if explicit && definitions.empty?
|
|
22
|
+
raise ArgumentError, "#{model.name} has no has_embedding for #{args[:column].inspect}"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
definitions.each { |definition| block.call(model, definition.column) }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
desc "Embed records that are missing or out of date. Usage: embed_sync:backfill[Product,embedding]"
|
|
30
|
+
task :backfill, %i[model column] => :environment do |_task, args|
|
|
31
|
+
each_embedding.call(args) do |model, column|
|
|
32
|
+
count = EmbedSync::Backfill.run(model, column: column) { |total| print "\r#{model.name}##{column}: #{total}" }
|
|
33
|
+
puts "\r#{model.name}##{column}: embedded #{count} records"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
desc "Report records whose embedding is missing or out of date. Usage: embed_sync:stale[Product,embedding]"
|
|
38
|
+
task :stale, %i[model column] => :environment do |_task, args|
|
|
39
|
+
each_embedding.call(args) do |model, column|
|
|
40
|
+
report = EmbedSync::Backfill.report(model, column: column)
|
|
41
|
+
puts "#{model.name}##{column}: #{report[:total]} total, #{report[:missing]} never embedded, " \
|
|
42
|
+
"#{report[:stale]} stale"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
data/lib/embed_sync.rb
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support"
|
|
4
|
+
require "active_support/lazy_load_hooks"
|
|
5
|
+
require "active_support/isolated_execution_state"
|
|
6
|
+
|
|
7
|
+
require_relative "embed_sync/version"
|
|
8
|
+
require_relative "embed_sync/errors"
|
|
9
|
+
require_relative "embed_sync/configuration"
|
|
10
|
+
require_relative "embed_sync/providers"
|
|
11
|
+
require_relative "embed_sync/definition"
|
|
12
|
+
require_relative "embed_sync/sync"
|
|
13
|
+
require_relative "embed_sync/backfill"
|
|
14
|
+
require_relative "embed_sync/query"
|
|
15
|
+
|
|
16
|
+
# Keeps vector embeddings in sync with ActiveRecord models.
|
|
17
|
+
module EmbedSync
|
|
18
|
+
autoload :Model, "embed_sync/model"
|
|
19
|
+
autoload :EmbedJob, "embed_sync/embed_job"
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
def config
|
|
23
|
+
@config ||= Configuration.new
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def configure
|
|
27
|
+
yield config
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def reset_config!
|
|
31
|
+
@config = nil
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def provider
|
|
35
|
+
config.resolved_provider
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Turns off automatic syncing inside the block (e.g. bulk imports), on this thread only.
|
|
39
|
+
def without_sync
|
|
40
|
+
previous = ActiveSupport::IsolatedExecutionState[:embed_sync_disabled]
|
|
41
|
+
ActiveSupport::IsolatedExecutionState[:embed_sync_disabled] = true
|
|
42
|
+
yield
|
|
43
|
+
ensure
|
|
44
|
+
ActiveSupport::IsolatedExecutionState[:embed_sync_disabled] = previous
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def sync_enabled?
|
|
48
|
+
!ActiveSupport::IsolatedExecutionState[:embed_sync_disabled]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# EmbedSync.backfill(Product) / EmbedSync.backfill(Product.includes(:category), column: :embedding)
|
|
52
|
+
def backfill(scope, **, &)
|
|
53
|
+
Backfill.run(scope, **, &)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def stale_report(scope, **)
|
|
57
|
+
Backfill.report(scope, **)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Called after commit: queues or runs the embedding when the record's text changed.
|
|
61
|
+
def sync_record(record, column)
|
|
62
|
+
return if !sync_enabled? || record.embed_sync_skipped?
|
|
63
|
+
|
|
64
|
+
definition = record.class.embed_sync_definition(column)
|
|
65
|
+
record = full_record(record, definition) or return
|
|
66
|
+
return unless definition.stale?(record)
|
|
67
|
+
|
|
68
|
+
case definition.sync_mode
|
|
69
|
+
when :async then EmbedJob.perform_later(record, column.to_s)
|
|
70
|
+
when :inline then sync_inline(record, column)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
# Records loaded with select(...) may lack the columns stale? reads; load the whole row instead.
|
|
77
|
+
def full_record(record, definition)
|
|
78
|
+
definition.stale?(record)
|
|
79
|
+
record
|
|
80
|
+
rescue ActiveModel::MissingAttributeError
|
|
81
|
+
record.class.unscoped.find_by(record.class.primary_key => record.id)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# The row is already committed, so a provider failure must not make save look like it failed.
|
|
85
|
+
# The error is logged by EmbedSync::LogSubscriber and the record stays stale for embed_sync:backfill.
|
|
86
|
+
def sync_inline(record, column)
|
|
87
|
+
Sync.call([record], column, recheck: true)
|
|
88
|
+
rescue ProviderError, InvalidRequestError
|
|
89
|
+
nil
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
ActiveSupport.on_load(:active_record) do
|
|
95
|
+
include EmbedSync::Model
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
require_relative "embed_sync/railtie" if defined?(Rails::Railtie)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators/named_base"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module EmbedSync
|
|
7
|
+
module Generators
|
|
8
|
+
# rails g embed_sync Product name description [--column=embedding] [--dimensions=1536]
|
|
9
|
+
class EmbedSyncGenerator < Rails::Generators::NamedBase
|
|
10
|
+
include Rails::Generators::Migration
|
|
11
|
+
|
|
12
|
+
OPCLASSES = { "cosine" => "vector_cosine_ops", "euclidean" => "vector_l2_ops",
|
|
13
|
+
"inner_product" => "vector_ip_ops" }.freeze
|
|
14
|
+
|
|
15
|
+
namespace "embed_sync"
|
|
16
|
+
source_root File.expand_path("templates", __dir__)
|
|
17
|
+
desc "Adds an embedding column, digest column and vector index to a model (PostgreSQL + pgvector)"
|
|
18
|
+
|
|
19
|
+
argument :source_attributes, type: :array, default: [], banner: "attribute attribute"
|
|
20
|
+
|
|
21
|
+
class_option :column, type: :string, default: "embedding", desc: "Vector column name"
|
|
22
|
+
class_option :dimensions, type: :numeric, default: 1536, desc: "Vector size of your embedding model"
|
|
23
|
+
class_option :index, type: :string, default: "hnsw", enum: %w[hnsw ivfflat none], desc: "Vector index type"
|
|
24
|
+
class_option :distance, type: :string, default: "cosine", enum: OPCLASSES.keys, desc: "Distance for the index"
|
|
25
|
+
|
|
26
|
+
def self.next_migration_number(dirname)
|
|
27
|
+
ActiveRecord::Generators::Base.next_migration_number(dirname)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def check_source_attributes
|
|
31
|
+
return if source_attributes.any?
|
|
32
|
+
|
|
33
|
+
raise Thor::Error, "Pass the attributes to embed, e.g. rails g embed_sync #{class_name} name description"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def create_migration_file
|
|
37
|
+
migration_template "migration.rb.tt", "db/migrate/add_#{column}_to_#{table_name}.rb"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def add_has_embedding_to_model
|
|
41
|
+
path = File.join("app/models", class_path, "#{file_name}.rb")
|
|
42
|
+
unless File.exist?(File.join(destination_root, path))
|
|
43
|
+
say_status :skip, "#{path} not found; add has_embedding to your model yourself", :yellow
|
|
44
|
+
return
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Matches both `class Admin::Product` and `module Admin; class Product`.
|
|
48
|
+
source = File.read(File.join(destination_root, path))
|
|
49
|
+
name = source.match?(/class\s+#{Regexp.escape(class_name)}\b/) ? class_name : class_name.demodulize
|
|
50
|
+
inject_into_class path, name, model_lines
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Without neighbor, Rails doesn't know the vector type: it drops limit:, the index fails and
|
|
54
|
+
# schema.rb can't dump the table.
|
|
55
|
+
def warn_without_neighbor
|
|
56
|
+
return if defined?(::Neighbor)
|
|
57
|
+
|
|
58
|
+
say_status :warning, 'Add gem "neighbor" to your Gemfile before running this migration ' \
|
|
59
|
+
"(it teaches Rails the pgvector column type).", :yellow
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def column
|
|
65
|
+
options[:column]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def index?
|
|
69
|
+
options[:index] != "none"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def opclass
|
|
73
|
+
OPCLASSES.fetch(options[:distance])
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def migration_version
|
|
77
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def model_lines
|
|
81
|
+
from = source_attributes.one? ? ":#{source_attributes.first}" : "%i[#{source_attributes.join(" ")}]"
|
|
82
|
+
lines = []
|
|
83
|
+
lines << " has_neighbors :#{column}\n" if defined?(::Neighbor)
|
|
84
|
+
lines << " has_embedding :#{column}, from: #{from}\n"
|
|
85
|
+
lines.join
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators/base"
|
|
4
|
+
|
|
5
|
+
module EmbedSync
|
|
6
|
+
module Generators
|
|
7
|
+
# rails g embed_sync:install
|
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
|
9
|
+
namespace "embed_sync:install"
|
|
10
|
+
source_root File.expand_path("templates", __dir__)
|
|
11
|
+
desc "Creates config/initializers/embed_sync.rb"
|
|
12
|
+
|
|
13
|
+
def create_initializer
|
|
14
|
+
template "initializer.rb.tt", "config/initializers/embed_sync.rb"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
EmbedSync.configure do |config|
|
|
4
|
+
# Service that generates embeddings: a registered name or any object that responds to
|
|
5
|
+
# #embed(texts, model:).
|
|
6
|
+
# :ruby_llm uses the ruby_llm gem; set API keys with RubyLLM.configure (e.g. config.openai_api_key)
|
|
7
|
+
# :fake deterministic vectors, never calls an API (good for tests)
|
|
8
|
+
config.provider = Rails.env.test? ? :fake : :ruby_llm
|
|
9
|
+
# Passed to RubyLLM.embed: dimensions:, provider:, assume_model_exists:, task_type:, provider_options:
|
|
10
|
+
# config.provider_options = { dimensions: 1536 }
|
|
11
|
+
|
|
12
|
+
# Embedding model used when has_embedding doesn't pass model:.
|
|
13
|
+
config.default_model = "text-embedding-3-small"
|
|
14
|
+
|
|
15
|
+
# :async (enqueue EmbedSync::EmbedJob after commit), :inline (embed during the commit) or false.
|
|
16
|
+
config.sync = Rails.env.test? ? false : :async
|
|
17
|
+
|
|
18
|
+
# Active Job queue for EmbedSync::EmbedJob.
|
|
19
|
+
config.queue = :default
|
|
20
|
+
|
|
21
|
+
# Records per provider call when backfilling.
|
|
22
|
+
config.batch_size = 100
|
|
23
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
class Add<%= column.camelize %>To<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
# Guarded so rolling back this migration doesn't try to drop the extension other vector columns use.
|
|
4
|
+
enable_extension "vector" unless extension_enabled?("vector")
|
|
5
|
+
|
|
6
|
+
add_column :<%= table_name %>, :<%= column %>, :vector, limit: <%= options[:dimensions] %>
|
|
7
|
+
add_column :<%= table_name %>, :<%= column %>_digest, :string
|
|
8
|
+
<% if index? -%>
|
|
9
|
+
add_index :<%= table_name %>, :<%= column %>, using: :<%= options[:index] %>, opclass: :<%= opclass %>
|
|
10
|
+
<% end -%>
|
|
11
|
+
end
|
|
12
|
+
end
|
data/sig/embed_sync.rbs
ADDED
metadata
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: embed_sync
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Umair0092
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-10-29 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: activejob
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '7.1'
|
|
20
|
+
- - "<"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '9'
|
|
23
|
+
type: :runtime
|
|
24
|
+
prerelease: false
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
26
|
+
requirements:
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: '7.1'
|
|
30
|
+
- - "<"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '9'
|
|
33
|
+
- !ruby/object:Gem::Dependency
|
|
34
|
+
name: activerecord
|
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '7.1'
|
|
40
|
+
- - "<"
|
|
41
|
+
- !ruby/object:Gem::Version
|
|
42
|
+
version: '9'
|
|
43
|
+
type: :runtime
|
|
44
|
+
prerelease: false
|
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
46
|
+
requirements:
|
|
47
|
+
- - ">="
|
|
48
|
+
- !ruby/object:Gem::Version
|
|
49
|
+
version: '7.1'
|
|
50
|
+
- - "<"
|
|
51
|
+
- !ruby/object:Gem::Version
|
|
52
|
+
version: '9'
|
|
53
|
+
- !ruby/object:Gem::Dependency
|
|
54
|
+
name: activesupport
|
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
|
56
|
+
requirements:
|
|
57
|
+
- - ">="
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: '7.1'
|
|
60
|
+
- - "<"
|
|
61
|
+
- !ruby/object:Gem::Version
|
|
62
|
+
version: '9'
|
|
63
|
+
type: :runtime
|
|
64
|
+
prerelease: false
|
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
66
|
+
requirements:
|
|
67
|
+
- - ">="
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: '7.1'
|
|
70
|
+
- - "<"
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
version: '9'
|
|
73
|
+
- !ruby/object:Gem::Dependency
|
|
74
|
+
name: railties
|
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
|
76
|
+
requirements:
|
|
77
|
+
- - ">="
|
|
78
|
+
- !ruby/object:Gem::Version
|
|
79
|
+
version: '7.1'
|
|
80
|
+
- - "<"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '9'
|
|
83
|
+
type: :runtime
|
|
84
|
+
prerelease: false
|
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - ">="
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '7.1'
|
|
90
|
+
- - "<"
|
|
91
|
+
- !ruby/object:Gem::Version
|
|
92
|
+
version: '9'
|
|
93
|
+
description: Declare which attributes an ActiveRecord model embeds, and EmbedSync
|
|
94
|
+
regenerates the vector in a background job whenever that text changes. Includes
|
|
95
|
+
change detection, backfill and stale-record tasks, and pluggable providers.
|
|
96
|
+
email:
|
|
97
|
+
- umairhassan0092@gmail.com
|
|
98
|
+
executables: []
|
|
99
|
+
extensions: []
|
|
100
|
+
extra_rdoc_files: []
|
|
101
|
+
files:
|
|
102
|
+
- CHANGELOG.md
|
|
103
|
+
- LICENSE.txt
|
|
104
|
+
- README.md
|
|
105
|
+
- Rakefile
|
|
106
|
+
- lib/embed_sync.rb
|
|
107
|
+
- lib/embed_sync/backfill.rb
|
|
108
|
+
- lib/embed_sync/configuration.rb
|
|
109
|
+
- lib/embed_sync/definition.rb
|
|
110
|
+
- lib/embed_sync/embed_job.rb
|
|
111
|
+
- lib/embed_sync/errors.rb
|
|
112
|
+
- lib/embed_sync/log_subscriber.rb
|
|
113
|
+
- lib/embed_sync/model.rb
|
|
114
|
+
- lib/embed_sync/providers.rb
|
|
115
|
+
- lib/embed_sync/providers/base.rb
|
|
116
|
+
- lib/embed_sync/providers/fake.rb
|
|
117
|
+
- lib/embed_sync/providers/ruby_llm.rb
|
|
118
|
+
- lib/embed_sync/query.rb
|
|
119
|
+
- lib/embed_sync/railtie.rb
|
|
120
|
+
- lib/embed_sync/sync.rb
|
|
121
|
+
- lib/embed_sync/tasks/embed_sync.rake
|
|
122
|
+
- lib/embed_sync/version.rb
|
|
123
|
+
- lib/generators/embed_sync/embed_sync_generator.rb
|
|
124
|
+
- lib/generators/embed_sync/install_generator.rb
|
|
125
|
+
- lib/generators/embed_sync/templates/initializer.rb.tt
|
|
126
|
+
- lib/generators/embed_sync/templates/migration.rb.tt
|
|
127
|
+
- sig/embed_sync.rbs
|
|
128
|
+
homepage: https://github.com/Umair0092/embed_sync
|
|
129
|
+
licenses:
|
|
130
|
+
- MIT
|
|
131
|
+
metadata:
|
|
132
|
+
rubygems_mfa_required: 'true'
|
|
133
|
+
homepage_uri: https://github.com/Umair0092/embed_sync
|
|
134
|
+
source_code_uri: https://github.com/Umair0092/embed_sync
|
|
135
|
+
changelog_uri: https://github.com/Umair0092/embed_sync/blob/main/CHANGELOG.md
|
|
136
|
+
post_install_message:
|
|
137
|
+
rdoc_options: []
|
|
138
|
+
require_paths:
|
|
139
|
+
- lib
|
|
140
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
141
|
+
requirements:
|
|
142
|
+
- - ">="
|
|
143
|
+
- !ruby/object:Gem::Version
|
|
144
|
+
version: 3.2.0
|
|
145
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
|
+
requirements:
|
|
147
|
+
- - ">="
|
|
148
|
+
- !ruby/object:Gem::Version
|
|
149
|
+
version: '0'
|
|
150
|
+
requirements: []
|
|
151
|
+
rubygems_version: 3.4.1
|
|
152
|
+
signing_key:
|
|
153
|
+
specification_version: 4
|
|
154
|
+
summary: Keep vector embeddings in sync with your ActiveRecord models.
|
|
155
|
+
test_files: []
|