neighbor 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5caf112992a87ac93ef7d05094b9d5beb71470fef6d5dc07da8f4ba74787c381
-  data.tar.gz: 85d47b002533704a0098e41fb2e752405ea8ae50ca3cdb7d5984e0cd93016ab0
+  metadata.gz: 1154f0138248270d73d1681962c7bfecc8f81e074104a24071d32d76d5405705
+  data.tar.gz: 23c823cc022efcfebf9533d0388dd3abaec0fca61cc7e0911a2972dfb3bf429b
 SHA512:
-  metadata.gz: af55fbcbe28352582379e84b0e831aae677411cf4a45432cd9a1caff7e33857d38362c6bc484a479e9c4e99f634cb094efa4ea10e02e789b4cbd552c72578b6c
-  data.tar.gz: b896e3edda9fc58a7e8aeb8aa71cf409ae5d0f8b30a785642e8380b071a0bff3e6504bc5123711405504ab7f54fb4ac856a2a26f5088bcd8bc4c963b2f4cab10
+  metadata.gz: 67cc38e53cb43b8775561dd234cf073d2d45115b1c517aa32f890ab9f3d059106915158ee8bc08d2af9091e2107273ba9a65da6c66a1646afc419b62a8a669e0
+  data.tar.gz: 63cf2a9765043884726a59a6c058730d004be2b15bed0565be70579ef1717b322bb424e1311ced9a5685c8f45b7521333b0c66cca42c83cd52cb94cd13a7a5fd

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.1.0 (2026-05-14)
+
+- Added experimental support for SQLite with no extension
+- Added experimental support for Vec1
+
 ## 1.0.0 (2026-04-04)
 
 - Dropped support for Ruby < 3.3 and Active Record < 7.2

data/README.md

@@ -4,10 +4,10 @@ Nearest neighbor search for Rails
 
 Supports:
 
-- Postgres (cube and pgvector)
+- Postgres (pgvector and cube)
 - MariaDB 11.8
 - MySQL 9 (searching requires HeatWave) - experimental
-- SQLite (sqlite-vec) - experimental
+- SQLite - experimental
 
 Also available for [Redis](https://github.com/ankane/neighbor-redis) and [S3 Vectors](https://github.com/ankane/neighbor-s3)
 
@@ -23,14 +23,7 @@ gem "neighbor"
 
 ### For Postgres
 
-Neighbor supports two extensions: [cube](https://www.postgresql.org/docs/current/cube.html) and [pgvector](https://github.com/pgvector/pgvector). cube ships with Postgres, while pgvector supports more dimensions and approximate nearest neighbor search.
-
-For cube, run:
-
-```sh
-rails generate neighbor:cube
-rails db:migrate
-```
+Neighbor supports two extensions for Postgres: [pgvector](https://github.com/pgvector/pgvector) and [cube](https://www.postgresql.org/docs/current/cube.html). cube ships with Postgres, while pgvector supports more dimensions and approximate nearest neighbor search.
 
 For pgvector, [install the extension](https://github.com/pgvector/pgvector#installation) and run:
 
@@ -39,18 +32,11 @@ rails generate neighbor:vector
 rails db:migrate
 ```
 
-### For SQLite
-
-Add this line to your application’s Gemfile:
-
-```ruby
-gem "sqlite-vec"
-```
-
-And run:
+For cube, run:
 
 ```sh
-rails generate neighbor:sqlite
+rails generate neighbor:cube
+rails db:migrate
 ```
 
 ## Getting Started
@@ -60,13 +46,13 @@ Create a migration
 ```ruby
 class AddEmbeddingToItems < ActiveRecord::Migration[8.1]
   def change
-    # cube
-    add_column :items, :embedding, :cube
-
     # pgvector, MariaDB, and MySQL
     add_column :items, :embedding, :vector, limit: 3 # dimensions
 
-    # sqlite-vec
+    # cube
+    add_column :items, :embedding, :cube
+
+    # SQLite
     add_column :items, :embedding, :binary
   end
 end
@@ -107,47 +93,14 @@ nearest_item.neighbor_distance
 
 See the additional docs for:
 
-- [cube](#cube)
 - [pgvector](#pgvector)
+- [cube](#cube)
 - [MariaDB](#mariadb)
 - [MySQL](#mysql)
-- [sqlite-vec](#sqlite-vec)
+- [SQLite](#sqlite)
 
 Or check out some [examples](#examples)
 
-## cube
-
-### Distance
-
-Supported values are:
-
-- `euclidean`
-- `cosine`
-- `taxicab`
-- `chebyshev`
-
-For cosine distance with cube, vectors must be normalized before being stored.
-
-```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, normalize: true
-end
-```
-
-For inner product with cube, see [this example](examples/disco/user_recs_cube.rb).
-
-### Dimensions
-
-The `cube` type can have up to 100 dimensions by default. See the [Postgres docs](https://www.postgresql.org/docs/current/cube.html) for how to increase this.
-
-For cube, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
-
-```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, dimensions: 3
-end
-```
-
 ## pgvector
 
 ### Distance
@@ -278,6 +231,39 @@ embedding = Neighbor::SparseVector.new({0 => 0.9, 1 => 1.3, 2 => 1.1}, 3)
 Item.nearest_neighbors(:embedding, embedding, distance: "euclidean").first(5)
 ```
 
+## cube
+
+### Distance
+
+Supported values are:
+
+- `euclidean`
+- `cosine`
+- `taxicab`
+- `chebyshev`
+
+For cosine distance with cube, vectors must be normalized before being stored.
+
+```ruby
+class Item < ApplicationRecord
+  has_neighbors :embedding, normalize: true
+end
+```
+
+For inner product with cube, see [this example](examples/disco/user_recs_cube.rb).
+
+### Dimensions
+
+The `cube` type can have up to 100 dimensions by default. See the [Postgres docs](https://www.postgresql.org/docs/current/cube.html) for how to increase this.
+
+For cube, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
+
+```ruby
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 3
+end
+```
+
 ## MariaDB
 
 ### Distance
@@ -353,20 +339,22 @@ Get the nearest neighbors by Hamming distance
 Item.nearest_neighbors(:embedding, "\x05", distance: "hamming").first(5)
 ```
 
-## sqlite-vec
+## SQLite
 
 ### Distance
 
 Supported values are:
 
 - `euclidean`
+- `inner_product`
 - `cosine`
 - `taxicab`
 - `hamming`
+- `jaccard`
 
 ### Dimensions
 
-For sqlite-vec, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
+For SQLite, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
 
 ```ruby
 class Item < ApplicationRecord
@@ -374,12 +362,97 @@ class Item < ApplicationRecord
 end
 ```
 
-### Virtual Tables
+### Int8 Vectors
+
+Use the `type` option for int8 vectors
+
+```ruby
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 3, type: :int8
+end
+```
+
+### Binary Vectors
+
+Use the `type` option for binary vectors
+
+```ruby
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 8, type: :bit
+end
+```
+
+Get the nearest neighbors by Hamming distance
+
+```ruby
+Item.nearest_neighbors(:embedding, "\x05", distance: "hamming").first(5)
+```
+
+### SQLite Extensions
+
+Improve performance with extensions:
+
+- [Vec1](#vec1)
+- [sqlite-vec](#sqlite-vec)
+
+### Vec1
+
+For [Vec1](https://sqlite.org/vec1/doc/trunk/doc/vec1.md), [build the extension](https://sqlite.org/vec1/doc/trunk/doc/vec1.md#2-building-the-extension) and create `config/initializers/neighbor.rb` with:
+
+```ruby
+Neighbor::SQLite.initialize!(extension: "/path/to/vec1.so")
+```
+
+This speeds up `euclidean` and `cosine` distance
+
+You can also use [virtual tables](https://sqlite.org/vec1/doc/trunk/doc/vec1intro.md#1-using-the-virtual-table)
+
+```ruby
+class CreateItems < ActiveRecord::Migration[8.1]
+  def change
+    # Rails 8+
+    create_virtual_table :items, :vec1, ["embedding", "id"]
+
+    # Rails < 8
+    execute "CREATE VIRTUAL TABLE items USING vec1(embedding, id)"
+  end
+end
+```
+
+You can optionally ignore any shadow tables that are created
+
+```ruby
+ActiveRecord::SchemaDumper.ignore_tables += [
+  "items_base", "items_config", "items_idx", "items_meta", "items_model"
+]
+```
+
+Get the `k` nearest neighbors
+
+```ruby
+Item.find_by_sql("SELECT * FROM items(vec1_from_json(?), ?)", [[1, 2, 3].to_json, {k: 5}.to_json])
+```
+
+### sqlite-vec
+
+For [sqlite-vec](https://github.com/asg017/sqlite-vec), add this line to your application’s Gemfile:
+
+```ruby
+gem "sqlite-vec"
+```
+
+And run:
+
+```sh
+rails generate neighbor:sqlite
+```
+
+This speeds up `euclidean`, `cosine`, `taxicab`, and `hamming` distance
 
 You can also use [virtual tables](https://alexgarcia.xyz/sqlite-vec/features/knn.html)
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[8.1]
+class CreateItems < ActiveRecord::Migration[8.1]
   def change
     # Rails 8+
     create_virtual_table :items, :vec0, [
@@ -420,32 +493,6 @@ Filter by primary key
 Item.where(id: [2, 3]).where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
 ```
 
-### Int8 Vectors
-
-Use the `type` option for int8 vectors
-
-```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, dimensions: 3, type: :int8
-end
-```
-
-### Binary Vectors
-
-Use the `type` option for binary vectors
-
-```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, dimensions: 8, type: :bit
-end
-```
-
-Get the nearest neighbors by Hamming distance
-
-```ruby
-Item.nearest_neighbors(:embedding, "\x05", distance: "hamming").first(5)
-```
-
 ## Examples
 
 - [Embeddings](#openai-embeddings) with OpenAI

data/lib/neighbor/model.rb

@@ -127,8 +127,10 @@ module Neighbor
           neighbor_distance =
             if distance == "cosine" && normalize_required
               "POWER(#{order}, 2) / 2.0"
-            elsif [:vector, :halfvec, :sparsevec].include?(column_type) && distance == "inner_product"
+            elsif distance == "inner_product"
               "(#{order}) * -1"
+            elsif adapter == :sqlite && order.start_with?("vec1_l2_distance")
+              "sqrt(#{order})"
             else
               order
             end

data/lib/neighbor/postgresql.rb

@@ -47,7 +47,7 @@ module Neighbor
 
     module ArrayMethods
       def type_cast_array(value, method, ...)
-        if (subtype.is_a?(Neighbor::Type::Vector) || subtype.is_a?(Neighbor::Type::Halfvec)) && method != :deserialize && value.is_a?(::Array) && value.all? { |v| v.is_a?(::Numeric) }
+        if (subtype.is_a?(Neighbor::Type::Vector) || subtype.is_a?(Neighbor::Type::Halfvec)) && method != :deserialize && value.is_a?(::Array) && value.all?(::Numeric)
           super(ArrayWrapper.new(value), method, ...)
         else
           super

data/lib/neighbor/sqlite.rb

@@ -1,27 +1,139 @@
 module Neighbor
   module SQLite
+    class << self
+      attr_reader :extensions
+    end
+
     # note: this is a public API (unlike PostgreSQL and MySQL)
-    def self.initialize!
-      return if defined?(@initialized)
+    def self.initialize!(extension: :sqlite_vec)
+      if extension == :sqlite_vec
+        require "sqlite_vec"
+      elsif !extension.is_a?(String)
+        raise ArgumentError, "Unsupported extension"
+      end
+
+      (@extensions ||= []) << extension
+    end
+
+    def self.initialize_adapter!
+      @extensions ||= []
 
       require_relative "type/sqlite_vector"
       require_relative "type/sqlite_int8_vector"
 
-      require "sqlite_vec"
       require "active_record/connection_adapters/sqlite3_adapter"
-
       ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(InstanceMethods)
+    end
+
+    def self.vec1?
+      extensions.any?(String)
+    end
+
+    def self.sqlite_vec?
+      extensions.include?(:sqlite_vec)
+    end
 
-      @initialized = true
+    def self.setup_functions(db)
+      db.create_function("neighbor_l2_distance", 2) do |func, a, b, c|
+        func.result =
+          if a.nil? || b.nil?
+            nil
+          else
+            raise SQLite3::SQLException, "different vector dimensions" if a.bytesize != b.bytesize
+            fmt = c == 1 ? "c*" : "f*"
+            a = a.unpack(fmt)
+            b = b.unpack(fmt)
+            Math.sqrt(a.zip(b).sum { |ai, bi| (ai - bi)**2 })
+          end
+      end
+
+      db.create_function("neighbor_max_inner_product", 2) do |func, a, b, c|
+        func.result =
+          if a.nil? || b.nil?
+            nil
+          else
+            raise SQLite3::SQLException, "different vector dimensions" if a.bytesize != b.bytesize
+            fmt = c == 1 ? "c*" : "f*"
+            a = a.unpack(fmt)
+            b = b.unpack(fmt)
+            -a.zip(b).sum { |ai, bi| ai * bi }
+          end
+      end
+
+      db.create_function("neighbor_cosine_distance", 2) do |func, a, b, c|
+        func.result =
+          if a.nil? || b.nil?
+            nil
+          else
+            raise SQLite3::SQLException, "different vector dimensions" if a.bytesize != b.bytesize
+            fmt = c == 1 ? "c*" : "f*"
+            a = a.unpack(fmt)
+            b = b.unpack(fmt)
+            similarity = a.zip(b).sum { |ai, bi| ai * bi }
+            norma = a.sum { |v| v * v }
+            normb = b.sum { |v| v * v }
+            1.0 - (similarity / Math.sqrt(norma * normb)).clamp(-1.0, 1.0)
+          end
+      end
+
+      db.create_function("neighbor_l1_distance", 2) do |func, a, b, c|
+        func.result =
+          if a.nil? || b.nil?
+            nil
+          else
+            raise SQLite3::SQLException, "different vector dimensions" if a.bytesize != b.bytesize
+            fmt = c == 1 ? "c*" : "f*"
+            a = a.unpack(fmt)
+            b = b.unpack(fmt)
+            a.zip(b).sum { |ai, bi| (ai - bi).abs }
+          end
+      end
+
+      db.create_function("neighbor_hamming_distance", 2) do |func, a, b|
+        func.result =
+          if a.nil? || b.nil?
+            nil
+          else
+            raise SQLite3::SQLException, "different vector dimensions" if a.bytesize != b.bytesize
+            # TODO improve
+            a.each_byte.zip(b.each_byte).sum { |ai, bi| (ai ^ bi).to_s(2).count("1") }
+          end
+      end
+
+      db.create_function("neighbor_jaccard_distance", 2) do |func, a, b|
+        func.result =
+          if a.nil? || b.nil?
+            nil
+          else
+            raise SQLite3::SQLException, "different vector dimensions" if a.bytesize != b.bytesize
+            # TODO improve
+            ab = a.each_byte.zip(b.each_byte).sum { |ai, bi| (ai & bi).to_s(2).count("1") }
+            aa = a.unpack1("B*").count("1")
+            bb = b.unpack1("B*").count("1")
+            ab == 0 ? 1.0 : 1.0 - (ab / (aa + bb - ab).to_f)
+          end
+      end
     end
 
     module InstanceMethods
       def configure_connection
         super
         db = @raw_connection
-        db.enable_load_extension(1)
-        SqliteVec.load(db)
-        db.enable_load_extension(0)
+        SQLite.setup_functions(db)
+        if SQLite.extensions.any?
+          db.enable_load_extension(1)
+          begin
+            SQLite.extensions.each do |extension|
+              if extension == :sqlite_vec
+                SqliteVec.load(db)
+              else
+                db.load_extension(extension)
+              end
+            end
+          ensure
+            db.enable_load_extension(0)
+          end
+        end
       end
     end
   end

data/lib/neighbor/utils.rb

@@ -76,13 +76,37 @@ module Neighbor
       when :sqlite
         case distance
         when "euclidean"
-          "vec_distance_L2"
+          if SQLite.vec1?
+            "vec1_l2_distance"
+          elsif SQLite.sqlite_vec?
+            "vec_distance_L2"
+          else
+            "neighbor_l2_distance"
+          end
         when "cosine"
-          "vec_distance_cosine"
+          if SQLite.vec1?
+            "vec1_cos_distance"
+          elsif SQLite.sqlite_vec?
+            "vec_distance_cosine"
+          else
+            "neighbor_cosine_distance"
+          end
         when "taxicab"
-          "vec_distance_L1"
+          if SQLite.sqlite_vec?
+            "vec_distance_L1"
+          else
+            "neighbor_l1_distance"
+          end
+        when "inner_product"
+          "neighbor_max_inner_product"
         when "hamming"
-          "vec_distance_hamming"
+          if SQLite.sqlite_vec?
+            "vec_distance_hamming"
+          else
+            "neighbor_hamming_distance"
+          end
+        when "jaccard"
+          "neighbor_jaccard_distance"
         end
       when :mariadb
         case column_type
@@ -158,10 +182,15 @@ module Neighbor
     def self.order(adapter, type, operator, quoted_attribute, query)
       case adapter
       when :sqlite
-        case type
-        when :int8
+        if operator.start_with?("neighbor")
+          if type == :bit
+            "#{operator}(#{quoted_attribute}, #{query})"
+          else
+            "#{operator}(#{quoted_attribute}, #{query}, #{type == :int8 ? 1 : 0})"
+          end
+        elsif type == :int8
           "#{operator}(vec_int8(#{quoted_attribute}), vec_int8(#{query}))"
-        when :bit
+        elsif type == :bit
           "#{operator}(vec_bit(#{quoted_attribute}), vec_bit(#{query}))"
         else
           "#{operator}(#{quoted_attribute}, #{query})"

data/lib/neighbor/version.rb

@@ -1,3 +1,3 @@
 module Neighbor
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/neighbor.rb

@@ -30,6 +30,7 @@ ActiveSupport.on_load(:active_record) do
   end
 
   Neighbor::MySQL.initialize!
+  Neighbor::SQLite.initialize_adapter!
 end
 
 require_relative "neighbor/railtie" if defined?(Rails::Railtie)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: neighbor
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -74,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Nearest neighbor search for Rails
 test_files: []