neighbor 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d7036a69b1c57161eaeb11e38feee92c1d5082ddbe8907a83ac3126adf9ae56
-  data.tar.gz: c88e4400b75d2a87f766f7e0b7ff6c5311d9c7866d32e7c2b9aa7601f60c474f
+  metadata.gz: fb8418a71159a849442d146643f732c28d10a5237d94a5b5b7d7466a224a40b5
+  data.tar.gz: 8a7ededb3071fdef4a77bbbdf4596d74ed323327d4d29ea15538e344d25af2f7
 SHA512:
-  metadata.gz: d3c4c25404fb64f324fbba70edcf06d827d3708905ed4a84404a6c9ce39f27b6890d449b285ce302e24495a666c34f0bf3050270b54ef8d283f53ebeb19e4e91
-  data.tar.gz: 63927a8801a88edd48f74ce85d056d7112fa526b37d473b232256b5f2d47e5254b34b25e0312afc01bfedcd6a9d7826496f208e0ab0d3f57c3307fa298b8984e
+  metadata.gz: 05b3b8ccd07570f531aef0132a80563aef046b88508e732b2f16fe02f6c0303bd6f6fdb00804a646845086ba7fd133358503a7c7b69b098c02f548434c859575
+  data.tar.gz: 9bcb61d4e1453e30daeab1fff0364c023278405d170b435cdfb262aac0aa0d91622e19471fbf3f3076e4c944d09318dc81cf334fdac7767fa6f72d95c8560907

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.5.2 (2025-01-05)
+
+- Improved support for Postgres arrays
+
+## 0.5.1 (2024-12-03)
+
+- Added experimental support for MariaDB 11.7
+- Dropped experimental support for MariaDB 11.6 Vector
+
 ## 0.5.0 (2024-10-07)
 
 - Added experimental support for SQLite (sqlite-vec)

data/README.md

@@ -6,7 +6,7 @@ Supports:
 
 - Postgres (cube and pgvector)
 - SQLite (sqlite-vec) - experimental
-- MariaDB 11.6 Vector - experimental
+- MariaDB 11.7 - experimental
 - MySQL 9 (searching requires HeatWave) - experimental
 
 [![Build Status](https://github.com/ankane/neighbor/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/neighbor/actions)
@@ -56,15 +56,15 @@ rails generate neighbor:sqlite
 Create a migration
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
     # cube
     add_column :items, :embedding, :cube
 
-    # pgvector and MySQL
+    # pgvector, MariaDB, and MySQL
     add_column :items, :embedding, :vector, limit: 3 # dimensions
 
-    # sqlite-vec and MariaDB
+    # sqlite-vec
     add_column :items, :embedding, :binary
   end
 end
@@ -174,7 +174,7 @@ The `sparsevec` type can have up to 16,000 non-zero elements, and sparse vectors
 Add an approximate index to speed up queries. Create a migration with:
 
 ```ruby
-class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.2]
+class AddIndexToItemsEmbedding < ActiveRecord::Migration[8.0]
   def change
     add_index :items, :embedding, using: :hnsw, opclass: :vector_l2_ops
     # or
@@ -202,7 +202,7 @@ Item.connection.execute("SET ivfflat.probes = 3")
 Use the `halfvec` type to store half-precision vectors
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
     add_column :items, :embedding, :halfvec, limit: 3 # dimensions
   end
@@ -214,7 +214,7 @@ end
 Index vectors at half precision for smaller indexes
 
 ```ruby
-class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.2]
+class AddIndexToItemsEmbedding < ActiveRecord::Migration[8.0]
   def change
     add_index :items, "(embedding::halfvec(3)) vector_l2_ops", using: :hnsw
   end
@@ -232,7 +232,7 @@ Item.nearest_neighbors(:embedding, [0.9, 1.3, 1.1], distance: "euclidean", preci
 Use the `bit` type to store binary vectors
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
     add_column :items, :embedding, :bit, limit: 3 # dimensions
   end
@@ -250,7 +250,7 @@ Item.nearest_neighbors(:embedding, "101", distance: "hamming").first(5)
 Use expression indexing for binary quantization
 
 ```ruby
-class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.2]
+class AddIndexToItemsEmbedding < ActiveRecord::Migration[8.0]
   def change
     add_index :items, "(binary_quantize(embedding)::bit(3)) bit_hamming_ops", using: :hnsw
   end
@@ -262,7 +262,7 @@ end
 Use the `sparsevec` type to store sparse vectors
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
     add_column :items, :embedding, :sparsevec, limit: 3 # dimensions
   end
@@ -302,19 +302,21 @@ end
 You can also use [virtual tables](https://alexgarcia.xyz/sqlite-vec/features/knn.html)
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
+    # Rails 8+
+    create_virtual_table :items, :vec0, [
+      "id integer PRIMARY KEY AUTOINCREMENT NOT NULL",
+      "embedding float[3] distance_metric=L2"
+    ]
+
     # Rails < 8
     execute <<~SQL
       CREATE VIRTUAL TABLE items USING vec0(
+        id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
         embedding float[3] distance_metric=L2
       )
     SQL
-
-    # Rails 8+
-    create_virtual_table :items, :vec0, [
-      "embedding float[3] distance_metric=L2"
-    ]
   end
 end
 ```
@@ -329,16 +331,6 @@ ActiveRecord::SchemaDumper.ignore_tables += [
 ]
 ```
 
-Create a model with `rowid` as the primary key
-
-```ruby
-class Item < ApplicationRecord
-  self.primary_key = "rowid"
-
-  has_neighbors :embedding, dimensions: 3
-end
-```
-
 Get the `k` nearest neighbors
 
 ```ruby
@@ -348,7 +340,7 @@ Item.where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
 Filter by primary key
 
 ```ruby
-Item.where(rowid: [2, 3]).where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
+Item.where(id: [2, 3]).where("embedding MATCH ?", [1, 2, 3].to_s).where(k: 5).order(:distance)
 ```
 
 ### Int8 Vectors
@@ -387,23 +379,15 @@ Supported values are:
 - `cosine`
 - `hamming`
 
-For cosine distance with MariaDB, vectors must be normalized before being stored.
-
-```ruby
-class Item < ApplicationRecord
-  has_neighbors :embedding, normalize: true
-end
-```
-
 ### Indexing
 
 Vector columns must use `null: false` to add a vector index
 
 ```ruby
-class CreateItems < ActiveRecord::Migration[7.2]
+class CreateItems < ActiveRecord::Migration[8.0]
   def change
     create_table :items do |t|
-      t.binary :embedding, null: false
+      t.vector :embedding, limit: 3, null: false
       t.index :embedding, type: :vector
     end
   end
@@ -415,7 +399,7 @@ end
 Use the `bigint` type to store binary vectors
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
     add_column :items, :embedding, :bigint
   end
@@ -447,7 +431,7 @@ Note: The `DISTANCE()` function is [only available on HeatWave](https://dev.mysq
 Use the `binary` type to store binary vectors
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+class AddEmbeddingToItems < ActiveRecord::Migration[8.0]
   def change
     add_column :items, :embedding, :binary
   end
@@ -892,7 +876,7 @@ bundle exec rake test:postgresql
 bundle exec rake test:sqlite
 
 # MariaDB
-docker run -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=neighbor_test -p 3307:3306 quay.io/mariadb-foundation/mariadb-devel:11.6-vector-preview
+docker run -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=neighbor_test -p 3307:3306 mariadb:11.7-rc
 bundle exec rake test:mariadb
 
 # MySQL

data/lib/neighbor/postgresql.rb

@@ -19,6 +19,9 @@ module Neighbor
 
       # prevent unknown OID warning
       ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.singleton_class.prepend(RegisterTypes)
+
+      # support vector[]/halfvec[]
+      ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array.prepend(ArrayMethods)
     end
 
     module RegisterTypes
@@ -39,5 +42,17 @@ module Neighbor
         end
       end
     end
+
+    ArrayWrapper = Struct.new(:to_a)
+
+    module ArrayMethods
+      def type_cast_array(value, method, ...)
+        if (subtype.is_a?(Neighbor::Type::Vector) || subtype.is_a?(Neighbor::Type::Halfvec)) && method != :deserialize && value.is_a?(::Array) && value.all? { |v| v.is_a?(::Numeric) }
+          super(ArrayWrapper.new(value), method, ...)
+        else
+          super
+        end
+      end
+    end
   end
 end

data/lib/neighbor/utils.rb

@@ -33,7 +33,7 @@ module Neighbor
     def self.normalize(value, column_info:)
       return nil if value.nil?
 
-      raise Error, "Normalize not supported for type" unless [:cube, :vector, :halfvec, :binary].include?(column_info&.type)
+      raise Error, "Normalize not supported for type" unless [:cube, :vector, :halfvec].include?(column_info&.type)
 
       norm = Math.sqrt(value.sum { |v| v * v })
 
@@ -86,10 +86,12 @@ module Neighbor
         end
       when :mariadb
         case column_type
-        when :binary
+        when :vector
           case distance
-          when "euclidean", "cosine"
-            "VEC_DISTANCE"
+          when "euclidean"
+            "VEC_DISTANCE_EUCLIDEAN"
+          when "cosine"
+            "VEC_DISTANCE_COSINE"
           end
         when :integer
           case distance
@@ -168,7 +170,7 @@ module Neighbor
         if operator == "BIT_COUNT"
           "BIT_COUNT(#{quoted_attribute} ^ #{query})"
         else
-          "VEC_DISTANCE(#{quoted_attribute}, #{query})"
+          "#{operator}(#{quoted_attribute}, #{query})"
         end
       when :mysql
         if operator == "BIT_COUNT"
@@ -191,8 +193,6 @@ module Neighbor
       case adapter
       when :postgresql
         column_type == :cube
-      when :mariadb
-        true
       else
         false
       end

data/lib/neighbor/version.rb

@@ -1,3 +1,3 @@
 module Neighbor
-  VERSION = "0.5.0"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: neighbor
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-08 00:00:00.000000000 Z
+date: 2025-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -24,7 +23,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7'
-description:
 email: andrew@ankane.org
 executables: []
 extensions: []
@@ -62,7 +60,6 @@ homepage: https://github.com/ankane/neighbor
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -77,8 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Nearest neighbor search for Rails
 test_files: []