neighbor 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee3864aa1511aa273c2d619e408bff8bcb491adc6dd111c5c88f7b79ef0baafc
-  data.tar.gz: 7d79f79814a0041e77d18edf83cb87b5fba5f59d1a25a71370d8b28c6a5cd289
+  metadata.gz: 9f4fa263cadd7a89f4f99faef6238db1f355597ce1510a84c722563a669c62d8
+  data.tar.gz: b3876b14c21276b0e2bfefa87d98265a3c766ae2b1b32459e085bcedfde8afcb
 SHA512:
-  metadata.gz: c8d63b19a366d670212d6d5296f3b0f0de45e007635b7026bc9474bbd9248c9fc9a7ba4ee8b7c1b796ecdd7eef93460fe4e4e746eb49a8c19d471f4827530ecf
-  data.tar.gz: c371a55e3f1579b6a62fcc5cc55c5ac54cb318faa12278c736089b97de1990ecf56a17863bcc7043b4a2cde9be50cb02b8b7dcecaf5032d371e1675a062723f2
+  metadata.gz: f90e43582600d41411c3a9e129ca03b606d08ece9bfc39227aab97e2c2af77950eb885cfa4900bef94eb8524d121af1b25823e495b99756ea8f04398d2497c8f
+  data.tar.gz: 17cb10ce768c76f1a0e273c887486f1e0a9a2b73f3d5d8758ebaada7aa12366e4d9e061b6a125b81cb36167013bf2a6f07dba786a5c0de4af99543e70421d961

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.6.0 (2025-06-12)
+
+- Added support for MariaDB 11.8
+- Dropped experimental support for MariaDB 11.7
+- Dropped support for Ruby < 3.2 and Active Record < 7.1
+
+## 0.5.2 (2025-01-05)
+
+- Improved support for Postgres arrays
+
 ## 0.5.1 (2024-12-03)
 
 - Added experimental support for MariaDB 11.7

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2021-2024 Andrew Kane
+Copyright (c) 2021-2025 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -5,9 +5,9 @@ Nearest neighbor search for Rails
 Supports:
 
 - Postgres (cube and pgvector)
-- SQLite (sqlite-vec) - experimental
-- MariaDB 11.7 - experimental
+- MariaDB 11.8
 - MySQL 9 (searching requires HeatWave) - experimental
+- SQLite (sqlite-vec) - experimental
 
 [![Build Status](https://github.com/ankane/neighbor/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/neighbor/actions)
 
@@ -107,9 +107,9 @@ See the additional docs for:
 
 - [cube](#cube)
 - [pgvector](#pgvector)
-- [sqlite-vec](#sqlite-vec)
 - [MariaDB](#mariadb)
 - [MySQL](#mysql)
+- [sqlite-vec](#sqlite-vec)
 
 Or check out some [examples](#examples)
 
@@ -216,7 +216,7 @@ Index vectors at half precision for smaller indexes
 ```ruby
 class AddIndexToItemsEmbedding < ActiveRecord::Migration[8.0]
   def change
-    add_index :items, "(embedding::halfvec(3)) vector_l2_ops", using: :hnsw
+    add_index :items, "(embedding::halfvec(3)) halfvec_l2_ops", using: :hnsw
   end
 end
 ```
@@ -276,7 +276,7 @@ embedding = Neighbor::SparseVector.new({0 => 0.9, 1 => 1.3, 2 => 1.1}, 3)
 Item.nearest_neighbors(:embedding, embedding, distance: "euclidean").first(5)
 ```
 
-## sqlite-vec
+## MariaDB
 
 ### Distance
 
@@ -284,90 +284,64 @@ Supported values are:
 
 - `euclidean`
 - `cosine`
-- `taxicab`
 - `hamming`
 
-### Dimensions
+### Indexing
 
-For sqlite-vec, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
+Vector columns must use `null: false` to add a vector index
 
 ```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, dimensions: 3
+class CreateItems < ActiveRecord::Migration[8.0]
+  def change
+    create_table :items do |t|
+      t.vector :embedding, limit: 3, null: false
+      t.index :embedding, type: :vector
+    end
+  end
 end
 ```
 
-### Virtual Tables
+### Binary Vectors
 
-You can also use [virtual tables](https://alexgarcia.xyz/sqlite-vec/features/knn.html)
+Use the `bigint` type to store binary vectors
 
 ```ruby
 class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
-    # Rails 8+
-    create_virtual_table :items, :vec0, [
-      "embedding float[3] distance_metric=L2"
-    ]
-
-    # Rails < 8
-    execute <<~SQL
-      CREATE VIRTUAL TABLE items USING vec0(
-        embedding float[3] distance_metric=L2
-      )
-    SQL
+    add_column :items, :embedding, :bigint
   end
 end
 ```
 
-Use `distance_metric=cosine` for cosine distance
-
-You can optionally ignore any shadow tables that are created
-
-```ruby
-ActiveRecord::SchemaDumper.ignore_tables += [
-  "items_chunks", "items_rowids", "items_vector_chunks00"
-]
-```
-
-Create a model with `rowid` as the primary key
-
-```ruby
-class Item < ApplicationRecord
-  self.primary_key = "rowid"
-
-  has_neighbors :embedding, dimensions: 3
-end
-```
+Note: Binary vectors can have up to 64 dimensions
 
-Get the `k` nearest neighbors
+Get the nearest neighbors by Hamming distance
 
 ```ruby
-Item.where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
+Item.nearest_neighbors(:embedding, 5, distance: "hamming").first(5)
 ```
 
-Filter by primary key
+## MySQL
 
-```ruby
-Item.where(rowid: [2, 3]).where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
-```
+### Distance
 
-### Int8 Vectors
+Supported values are:
 
-Use the `type` option for int8 vectors
+- `euclidean`
+- `cosine`
+- `hamming`
 
-```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, dimensions: 3, type: :int8
-end
-```
+Note: The `DISTANCE()` function is [only available on HeatWave](https://dev.mysql.com/doc/refman/9.0/en/vector-functions.html)
 
 ### Binary Vectors
 
-Use the `type` option for binary vectors
+Use the `binary` type to store binary vectors
 
 ```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, dimensions: 8, type: :bit
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
+  def change
+    add_column :items, :embedding, :binary
+  end
 end
 ```
 
@@ -377,7 +351,7 @@ Get the nearest neighbors by Hamming distance
 Item.nearest_neighbors(:embedding, "\x05", distance: "hamming").first(5)
 ```
 
-## MariaDB
+## sqlite-vec
 
 ### Distance
 
@@ -385,64 +359,82 @@ Supported values are:
 
 - `euclidean`
 - `cosine`
+- `taxicab`
 - `hamming`
 
-### Indexing
+### Dimensions
 
-Vector columns must use `null: false` to add a vector index
+For sqlite-vec, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
 
 ```ruby
-class CreateItems < ActiveRecord::Migration[8.0]
-  def change
-    create_table :items do |t|
-      t.vector :embedding, limit: 3, null: false
-      t.index :embedding, type: :vector
-    end
-  end
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 3
 end
 ```
 
-### Binary Vectors
+### Virtual Tables
 
-Use the `bigint` type to store binary vectors
+You can also use [virtual tables](https://alexgarcia.xyz/sqlite-vec/features/knn.html)
 
 ```ruby
 class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
-    add_column :items, :embedding, :bigint
+    # Rails 8+
+    create_virtual_table :items, :vec0, [
+      "id integer PRIMARY KEY AUTOINCREMENT NOT NULL",
+      "embedding float[3] distance_metric=L2"
+    ]
+
+    # Rails < 8
+    execute <<~SQL
+      CREATE VIRTUAL TABLE items USING vec0(
+        id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+        embedding float[3] distance_metric=L2
+      )
+    SQL
   end
 end
 ```
 
-Note: Binary vectors can have up to 64 dimensions
+Use `distance_metric=cosine` for cosine distance
 
-Get the nearest neighbors by Hamming distance
+You can optionally ignore any shadow tables that are created
 
 ```ruby
-Item.nearest_neighbors(:embedding, 5, distance: "hamming").first(5)
+ActiveRecord::SchemaDumper.ignore_tables += [
+  "items_chunks", "items_rowids", "items_vector_chunks00"
+]
 ```
 
-## MySQL
+Get the `k` nearest neighbors
 
-### Distance
+```ruby
+Item.where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
+```
 
-Supported values are:
+Filter by primary key
 
-- `euclidean`
-- `cosine`
-- `hamming`
+```ruby
+Item.where(id: [2, 3]).where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
+```
 
-Note: The `DISTANCE()` function is [only available on HeatWave](https://dev.mysql.com/doc/refman/9.0/en/vector-functions.html)
+### Int8 Vectors
+
+Use the `type` option for int8 vectors
+
+```ruby
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 3, type: :int8
+end
+```
 
 ### Binary Vectors
 
-Use the `binary` type to store binary vectors
+Use the `type` option for binary vectors
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
-  def change
-    add_column :items, :embedding, :binary
-  end
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 8, type: :bit
 end
 ```
 
@@ -481,7 +473,7 @@ end
 Create a method to call the [embeddings API](https://platform.openai.com/docs/guides/embeddings)
 
 ```ruby
-def fetch_embeddings(input)
+def embed(input)
   url = "https://api.openai.com/v1/embeddings"
   headers = {
     "Authorization" => "Bearer #{ENV.fetch("OPENAI_API_KEY")}",
@@ -505,7 +497,7 @@ input = [
   "The cat is purring",
   "The bear is growling"
 ]
-embeddings = fetch_embeddings(input)
+embeddings = embed(input)
 ```
 
 Store the embeddings
@@ -532,7 +524,7 @@ See the [complete code](examples/openai/example.rb)
 Generate a model
 
 ```sh
-rails generate model Document content:text embedding:bit{1024}
+rails generate model Document content:text embedding:bit{1536}
 rails db:migrate
 ```
 
@@ -547,15 +539,15 @@ end
 Create a method to call the [embed API](https://docs.cohere.com/reference/embed)
 
 ```ruby
-def fetch_embeddings(input, input_type)
-  url = "https://api.cohere.com/v1/embed"
+def embed(input, input_type)
+  url = "https://api.cohere.com/v2/embed"
   headers = {
     "Authorization" => "Bearer #{ENV.fetch("CO_API_KEY")}",
     "Content-Type" => "application/json"
   }
   data = {
     texts: input,
-    model: "embed-english-v3.0",
+    model: "embed-v4.0",
     input_type: input_type,
     embedding_types: ["ubinary"]
   }
@@ -573,7 +565,7 @@ input = [
   "The cat is purring",
   "The bear is growling"
 ]
-embeddings = fetch_embeddings(input, "search_document")
+embeddings = embed(input, "search_document")
 ```
 
 Store the embeddings
@@ -590,7 +582,7 @@ Embed the search query
 
 ```ruby
 query = "forest"
-query_embedding = fetch_embeddings([query], "search_query")[0]
+query_embedding = embed([query], "search_query")[0]
 ```
 
 And search the documents
@@ -884,7 +876,7 @@ bundle exec rake test:postgresql
 bundle exec rake test:sqlite
 
 # MariaDB
-docker run -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=neighbor_test -p 3307:3306 mariadb:11.7-rc
+docker run -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=neighbor_test -p 3307:3306 mariadb:11.8
 bundle exec rake test:mariadb
 
 # MySQL

data/lib/neighbor/model.rb

@@ -40,16 +40,8 @@ module Neighbor
         end
 
         if normalize
-          if ActiveRecord::VERSION::STRING.to_f >= 7.1
-            attribute_names.each do |attribute_name|
-              normalizes attribute_name, with: ->(v) { Neighbor::Utils.normalize(v, column_info: columns_hash[attribute_name.to_s]) }
-            end
-          else
-            attribute_names.each do |attribute_name|
-              attribute attribute_name do |cast_type|
-                Neighbor::NormalizedAttribute.new(cast_type: cast_type, model: self, attribute_name: attribute_name)
-              end
-            end
+          attribute_names.each do |attribute_name|
+            normalizes attribute_name, with: ->(v) { Neighbor::Utils.normalize(v, column_info: columns_hash[attribute_name.to_s]) }
           end
         end
 

data/lib/neighbor/mysql.rb

@@ -15,9 +15,6 @@ module Neighbor
 
       # prevent unknown OID warning
       ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.singleton_class.prepend(RegisterTypes)
-      if ActiveRecord::VERSION::STRING.to_f < 7.1
-        ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.register_vector_type(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter::TYPE_MAP)
-      end
     end
 
     module RegisterTypes

data/lib/neighbor/postgresql.rb

@@ -19,6 +19,9 @@ module Neighbor
 
       # prevent unknown OID warning
       ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.singleton_class.prepend(RegisterTypes)
+
+      # support vector[]/halfvec[]
+      ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array.prepend(ArrayMethods)
     end
 
     module RegisterTypes
@@ -39,5 +42,17 @@ module Neighbor
         end
       end
     end
+
+    ArrayWrapper = Struct.new(:to_a)
+
+    module ArrayMethods
+      def type_cast_array(value, method, ...)
+        if (subtype.is_a?(Neighbor::Type::Vector) || subtype.is_a?(Neighbor::Type::Halfvec)) && method != :deserialize && value.is_a?(::Array) && value.all? { |v| v.is_a?(::Numeric) }
+          super(ArrayWrapper.new(value), method, ...)
+        else
+          super
+        end
+      end
+    end
   end
 end

data/lib/neighbor/sqlite.rb

@@ -18,7 +18,7 @@ module Neighbor
     module InstanceMethods
       def configure_connection
         super
-        db = ActiveRecord::VERSION::STRING.to_f >= 7.1 ? @raw_connection : @connection
+        db = @raw_connection
         db.enable_load_extension(1)
         SqliteVec.load(db)
         db.enable_load_extension(0)

data/lib/neighbor/version.rb

@@ -1,3 +1,3 @@
 module Neighbor
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: neighbor
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,15 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7'
-description:
+        version: '7.1'
 email: andrew@ankane.org
 executables: []
 extensions: []
@@ -62,7 +60,6 @@ homepage: https://github.com/ankane/neighbor
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -70,15 +67,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Nearest neighbor search for Rails
 test_files: []