neighbor 0.4.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09edc5a7eebbf6b14f06cb51340c5def49117a318340b4d2265321a8ce6a0bec'
-  data.tar.gz: fc8c8319cf715612f195836c84861eb327765355a0430f2d58fb5ab57857844e
+  metadata.gz: dfc4af6302c7098ea40f96e9d8a19706aff46a2506cad541ff18ee07fcd11019
+  data.tar.gz: a79b59895ca3b99a7c048eddd20cb3602b2660425ab44463e7021ab763a26f62
 SHA512:
-  metadata.gz: caa86d17e8a3f710988486264434767c33f8b197f9a8721d6dc762235a0bc959d5c186670f7518b9d628a771454861df1beb603a175ec804aa67cf6eb9e14361
-  data.tar.gz: 3ac9d60c57cc3e82b617820f205282b42684517070de22af5d94878959ef00e3758fb88f821ba3d3f2369602919a41d1706314f77436c8b3e5ef95acc38e3c17
+  metadata.gz: 11081e687de4c79428351095477137f9140bc6c0363d09c54ece8fd5f7bbe2df802d740332f4474357f9e9e57157bd1f1f4dd3671c106d24dc7e01e2f0d84e2a
+  data.tar.gz: f18d787b22df7bbc00c69b1f9f6262e19c0214f6ef28814a5c05d9e8c3dae357f32596994bef60112d3f53dac0c1b51f6c99af6345c7e01b2f26cef1a7b42226

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.4.2 (2024-08-27)
+
+- Fixed error with `nil` values
+
+## 0.4.1 (2024-08-26)
+
+- Added `precision` option
+- Added support for `bit` dimensions to model generator
+- Fixed error with Numo arrays
+
 ## 0.4.0 (2024-06-25)
 
 - Added support for `halfvec` and `sparsevec` types

data/README.md

@@ -14,7 +14,7 @@ gem "neighbor"
 
 ## Choose An Extension
 
-Neighbor supports two extensions: [cube](https://www.postgresql.org/docs/current/cube.html) and [vector](https://github.com/pgvector/pgvector). cube ships with Postgres, while vector supports more dimensions and approximate nearest neighbor search.
+Neighbor supports two extensions: [cube](https://www.postgresql.org/docs/current/cube.html) and [pgvector](https://github.com/pgvector/pgvector). cube ships with Postgres, while pgvector supports more dimensions and approximate nearest neighbor search.
 
 For cube, run:
 
@@ -23,7 +23,7 @@ rails generate neighbor:cube
 rails db:migrate
 ```
 
-For vector, [install pgvector](https://github.com/pgvector/pgvector#installation) and run:
+For pgvector, [install the extension](https://github.com/pgvector/pgvector#installation) and run:
 
 ```sh
 rails generate neighbor:vector
@@ -35,7 +35,7 @@ rails db:migrate
 Create a migration
 
 ```ruby
-class AddEmbeddingToItems < ActiveRecord::Migration[7.1]
+class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
   def change
     add_column :items, :embedding, :cube
     # or
@@ -70,15 +70,30 @@ Get the nearest neighbors to a vector
 Item.nearest_neighbors(:embedding, [0.9, 1.3, 1.1], distance: "euclidean").first(5)
 ```
 
-## Distance
+Records returned from `nearest_neighbors` will have a `neighbor_distance` attribute
+
+```ruby
+nearest_item = item.nearest_neighbors(:embedding, distance: "euclidean").first
+nearest_item.neighbor_distance
+```
+
+See the additional docs for:
+
+- [cube](#cube)
+- [pgvector](#pgvector)
+
+Or check out some [examples](#examples)
+
+## cube
+
+### Distance
 
 Supported values are:
 
 - `euclidean`
 - `cosine`
-- `taxicab` (cube only)
-- `chebyshev` (cube only)
-- `inner_product` (vector only)
+- `taxicab`
+- `chebyshev`
 
 For cosine distance with cube, vectors must be normalized before being stored.
 
@@ -88,18 +103,11 @@ class Item < ApplicationRecord
 end
 ```
 
-For inner product with cube, see [this example](examples/disco_user_recs_cube.rb).
+For inner product with cube, see [this example](examples/disco/user_recs_cube.rb).
 
-Records returned from `nearest_neighbors` will have a `neighbor_distance` attribute
+### Dimensions
 
-```ruby
-nearest_item = item.nearest_neighbors(:embedding, distance: "euclidean").first
-nearest_item.neighbor_distance
-```
-
-## Dimensions
-
-The cube data type can have up to 100 dimensions by default. See the [Postgres docs](https://www.postgresql.org/docs/current/cube.html) for how to increase this. The vector data type can have up to 16,000 dimensions, and vectors with up to 2,000 dimensions can be indexed.
+The `cube` type can have up to 100 dimensions by default. See the [Postgres docs](https://www.postgresql.org/docs/current/cube.html) for how to increase this.
 
 For cube, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
 
@@ -109,12 +117,29 @@ class Item < ApplicationRecord
 end
 ```
 
-## Indexing
+## pgvector
+
+### Distance
+
+Supported values are:
+
+- `euclidean`
+- `inner_product`
+- `cosine`
+- `taxicab`
+- `hamming`
+- `jaccard`
 
-For vector, add an approximate index to speed up queries. Create a migration with:
+### Dimensions
+
+The `vector` type can have up to 16,000 dimensions, and vectors with up to 2,000 dimensions can be indexed.
+
+### Indexing
+
+Add an approximate index to speed up queries. Create a migration with:
 
 ```ruby
-class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.1]
+class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.2]
   def change
     add_index :items, :embedding, using: :hnsw, opclass: :vector_l2_ops
     # or
@@ -137,9 +162,91 @@ Or the number of probes with IVFFlat
 Item.connection.execute("SET ivfflat.probes = 3")
 ```
 
+### Half-Precision Vectors
+
+Use the `halfvec` type to store half-precision vectors
+
+```ruby
+class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+  def change
+    add_column :items, :embedding, :halfvec, limit: 3 # dimensions
+  end
+end
+```
+
+### Half-Precision Indexing
+
+Index vectors at half precision for smaller indexes
+
+```ruby
+class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.2]
+  def change
+    add_index :items, "(embedding::halfvec(3)) vector_l2_ops", using: :hnsw
+  end
+end
+```
+
+Get the nearest neighbors
+
+```ruby
+Item.nearest_neighbors(:embedding, [0.9, 1.3, 1.1], distance: "euclidean", precision: "half").first(5)
+```
+
+### Binary Vectors
+
+Use the `bit` type to store binary vectors
+
+```ruby
+class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+  def change
+    add_column :items, :embedding, :bit, limit: 3 # dimensions
+  end
+end
+```
+
+Get the nearest neighbors by Hamming distance
+
+```ruby
+Item.nearest_neighbors(:embedding, "101", distance: "hamming").first(5)
+```
+
+### Binary Quantization
+
+Use expression indexing for binary quantization
+
+```ruby
+class AddIndexToItemsEmbedding < ActiveRecord::Migration[7.2]
+  def change
+    add_index :items, "(binary_quantize(embedding)::bit(3)) bit_hamming_ops", using: :hnsw
+  end
+end
+```
+
+### Sparse Vectors
+
+Use the `sparsevec` type to store sparse vectors
+
+```ruby
+class AddEmbeddingToItems < ActiveRecord::Migration[7.2]
+  def change
+    add_column :items, :embedding, :sparsevec, limit: 3 # dimensions
+  end
+end
+```
+
+Get the nearest neighbors
+
+```ruby
+embedding = Neighbor::SparseVector.new({0 => 0.9, 1 => 1.3, 2 => 1.1}, 3)
+Item.nearest_neighbors(:embedding, embedding, distance: "euclidean").first(5)
+```
+
 ## Examples
 
 - [OpenAI Embeddings](#openai-embeddings)
+- [Cohere Embeddings](#cohere-embeddings)
+- [Sentence Embeddings](#sentence-embeddings)
+- [Sparse Embeddings](#sparse-embeddings)
 - [Disco Recommendations](#disco-recommendations)
 
 ### OpenAI Embeddings
@@ -170,10 +277,10 @@ def fetch_embeddings(input)
   }
   data = {
     input: input,
-    model: "text-embedding-ada-002"
+    model: "text-embedding-3-small"
   }
 
-  response = Net::HTTP.post(URI(url), data.to_json, headers)
+  response = Net::HTTP.post(URI(url), data.to_json, headers).tap(&:value)
   JSON.parse(response.body)["data"].map { |v| v["embedding"] }
 end
 ```
@@ -199,14 +306,221 @@ end
 Document.insert_all!(documents)
 ```
 
-And get similar articles
+And get similar documents
+
+```ruby
+document = Document.first
+document.nearest_neighbors(:embedding, distance: "cosine").first(5).map(&:content)
+```
+
+See the [complete code](examples/openai/example.rb)
+
+### Cohere Embeddings
+
+Generate a model
+
+```sh
+rails generate model Document content:text embedding:bit{1024}
+rails db:migrate
+```
+
+And add `has_neighbors`
+
+```ruby
+class Document < ApplicationRecord
+  has_neighbors :embedding
+end
+```
+
+Create a method to call the [embed API](https://docs.cohere.com/reference/embed)
+
+```ruby
+def fetch_embeddings(input, input_type)
+  url = "https://api.cohere.com/v1/embed"
+  headers = {
+    "Authorization" => "Bearer #{ENV.fetch("CO_API_KEY")}",
+    "Content-Type" => "application/json"
+  }
+  data = {
+    texts: input,
+    model: "embed-english-v3.0",
+    input_type: input_type,
+    embedding_types: ["ubinary"]
+  }
+
+  response = Net::HTTP.post(URI(url), data.to_json, headers).tap(&:value)
+  JSON.parse(response.body)["embeddings"]["ubinary"].map { |e| e.map { |v| v.chr.unpack1("B*") }.join }
+end
+```
+
+Pass your input
+
+```ruby
+input = [
+  "The dog is barking",
+  "The cat is purring",
+  "The bear is growling"
+]
+embeddings = fetch_embeddings(input, "search_document")
+```
+
+Store the embeddings
+
+```ruby
+documents = []
+input.zip(embeddings) do |content, embedding|
+  documents << {content: content, embedding: embedding}
+end
+Document.insert_all!(documents)
+```
+
+Embed the search query
+
+```ruby
+query = "forest"
+query_embedding = fetch_embeddings([query], "search_query")[0]
+```
+
+And search the documents
+
+```ruby
+Document.nearest_neighbors(:embedding, query_embedding, distance: "hamming").first(5).map(&:content)
+```
+
+See the [complete code](examples/cohere/example.rb)
+
+### Sentence Embeddings
+
+You can generate embeddings locally with [Informers](https://github.com/ankane/informers).
+
+Generate a model
+
+```sh
+rails generate model Document content:text embedding:vector{384}
+rails db:migrate
+```
+
+And add `has_neighbors`
+
+```ruby
+class Document < ApplicationRecord
+  has_neighbors :embedding
+end
+```
+
+Load a [model](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)
+
+```ruby
+model = Informers::Model.new("sentence-transformers/all-MiniLM-L6-v2")
+```
+
+Pass your input
+
+```ruby
+input = [
+  "The dog is barking",
+  "The cat is purring",
+  "The bear is growling"
+]
+embeddings = model.embed(input)
+```
+
+Store the embeddings
+
+```ruby
+documents = []
+input.zip(embeddings) do |content, embedding|
+  documents << {content: content, embedding: embedding}
+end
+Document.insert_all!(documents)
+```
+
+And get similar documents
 
 ```ruby
 document = Document.first
 document.nearest_neighbors(:embedding, distance: "cosine").first(5).map(&:content)
 ```
 
-See the [complete code](examples/openai_embeddings.rb)
+See the [complete code](examples/informers/example.rb)
+
+### Sparse Embeddings
+
+You can generate sparse embeddings locally with [Transformers.rb](https://github.com/ankane/transformers-ruby).
+
+Generate a model
+
+```sh
+rails generate model Document content:text embedding:sparsevec{30522}
+rails db:migrate
+```
+
+And add `has_neighbors`
+
+```ruby
+class Document < ApplicationRecord
+  has_neighbors :embedding
+end
+```
+
+Load a [model](https://huggingface.co/opensearch-project/opensearch-neural-sparse-encoding-v1) to generate embeddings
+
+```ruby
+class EmbeddingModel
+  def initialize(model_id)
+    @model = Transformers::AutoModelForMaskedLM.from_pretrained(model_id)
+    @tokenizer = Transformers::AutoTokenizer.from_pretrained(model_id)
+    @special_token_ids = @tokenizer.special_tokens_map.map { |_, token| @tokenizer.vocab[token] }
+  end
+
+  def embed(input)
+    feature = @tokenizer.(input, padding: true, truncation: true, return_tensors: "pt", return_token_type_ids: false)
+    output = @model.(**feature)[0]
+    values = Torch.max(output * feature[:attention_mask].unsqueeze(-1), dim: 1)[0]
+    values = Torch.log(1 + Torch.relu(values))
+    values[0.., @special_token_ids] = 0
+    values.to_a
+  end
+end
+
+model = EmbeddingModel.new("opensearch-project/opensearch-neural-sparse-encoding-v1")
+```
+
+Pass your input
+
+```ruby
+input = [
+  "The dog is barking",
+  "The cat is purring",
+  "The bear is growling"
+]
+embeddings = model.embed(input)
+```
+
+Store the embeddings
+
+```ruby
+documents = []
+input.zip(embeddings) do |content, embedding|
+  documents << {content: content, embedding: Neighbor::SparseVector.new(embedding)}
+end
+Document.insert_all!(documents)
+```
+
+Embed the search query
+
+```ruby
+query = "forest"
+query_embedding = model.embed([query])[0]
+```
+
+And search the documents
+
+```ruby
+Document.nearest_neighbors(:embedding, Neighbor::SparseVector.new(query_embedding), distance: "inner_product").first(5).map(&:content)
+```
+
+See the [complete code](examples/sparse/example.rb)
 
 ### Disco Recommendations
 
@@ -252,19 +566,7 @@ movie = Movie.find_by(name: "Star Wars (1977)")
 movie.nearest_neighbors(:factors, distance: "cosine").first(5).map(&:name)
 ```
 
-See the complete code for [cube](examples/disco_item_recs_cube.rb) and [vector](examples/disco_item_recs_vector.rb)
-
-## Upgrading
-
-### 0.2.0
-
-The `distance` option has been moved from `has_neighbors` to `nearest_neighbors`, and there is no longer a default. If you use cosine distance, set:
-
-```ruby
-class Item < ApplicationRecord
-  has_neighbors normalize: true
-end
-```
+See the complete code for [cube](examples/disco/item_recs_cube.rb) and [pgvector](examples/disco/item_recs_vector.rb)
 
 ## History
 

data/lib/neighbor/model.rb

@@ -49,7 +49,7 @@ module Neighbor
         # TODO move to normalizes when Active Record < 7.1 no longer supported
         before_save do
           self.class.neighbor_attributes.each do |k, v|
-            next unless v[:normalize]
+            next unless v[:normalize] && attribute_changed?(k)
             value = read_attribute(k)
             next if value.nil?
             self[k] = Neighbor::Utils.normalize(value, column_info: self.class.columns_hash[k.to_s])
@@ -61,6 +61,7 @@ module Neighbor
         scope :nearest_neighbors, ->(attribute_name, vector, options = nil) {
           raise ArgumentError, "missing keyword: :distance" unless options.is_a?(Hash) && options.key?(:distance)
           distance = options.delete(:distance)
+          precision = options.delete(:precision)
           raise ArgumentError, "unknown keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
 
           attribute_name = attribute_name.to_sym
@@ -126,6 +127,18 @@ module Neighbor
           vector = Neighbor::Utils.normalize(vector, column_info: column_info) if normalize
 
           query = connection.quote(column_attribute.serialize(vector))
+
+          if !precision.nil?
+            case precision.to_s
+            when "half"
+              cast_dimensions = dimensions || column_info&.limit
+              raise ArgumentError, "Unknown dimensions" unless cast_dimensions
+              quoted_attribute += "::halfvec(#{connection.quote(cast_dimensions.to_i)})"
+            else
+              raise ArgumentError, "Invalid precision"
+            end
+          end
+
           order = "#{quoted_attribute} #{operator} #{query}"
           if operator == "#"
             order = "bit_count(#{order})"

data/lib/neighbor/railtie.rb

@@ -10,7 +10,7 @@ module Neighbor
 
   module GeneratedAttribute
     def parse_type_and_options(type, *, **)
-      if type =~ /\A(vector|halfvec|sparsevec)\{(\d+)\}\z/
+      if type =~ /\A(vector|halfvec|bit|sparsevec)\{(\d+)\}\z/
         return $1, limit: $2.to_i
       end
       super

data/lib/neighbor/type/cube.rb

@@ -6,7 +6,8 @@ module Neighbor
       end
 
       def serialize(value)
-        if value.is_a?(Array)
+        if Utils.array?(value)
+          value = value.to_a
           if value.first.is_a?(Array)
             value = value.map { |v| serialize_point(v) }.join(", ")
           else
@@ -19,8 +20,8 @@ module Neighbor
       private
 
       def cast_value(value)
-        if value.is_a?(Array)
-          value
+        if Utils.array?(value)
+          value.to_a
         elsif value.is_a?(Numeric)
           [value]
         elsif value.is_a?(String)

data/lib/neighbor/type/halfvec.rb

@@ -6,8 +6,8 @@ module Neighbor
       end
 
       def serialize(value)
-        if value.is_a?(Array)
-          value = "[#{value.map(&:to_f).join(",")}]"
+        if Utils.array?(value)
+          value = "[#{value.to_a.map(&:to_f).join(",")}]"
         end
         super(value)
       end
@@ -17,8 +17,8 @@ module Neighbor
       def cast_value(value)
         if value.is_a?(String)
           value[1..-1].split(",").map(&:to_f)
-        elsif value.is_a?(Array)
-          value
+        elsif Utils.array?(value)
+          value.to_a
         else
           raise "can't cast #{value.class.name} to halfvec"
         end

data/lib/neighbor/type/sparsevec.rb

@@ -19,8 +19,8 @@ module Neighbor
           value
         elsif value.is_a?(String)
           SparseVector.from_text(value)
-        elsif value.is_a?(Array)
-          value = SparseVector.new(value)
+        elsif Utils.array?(value)
+          value = SparseVector.new(value.to_a)
         else
           raise "can't cast #{value.class.name} to sparsevec"
         end

data/lib/neighbor/type/vector.rb

@@ -6,8 +6,8 @@ module Neighbor
       end
 
       def serialize(value)
-        if value.is_a?(Array)
-          value = "[#{value.map(&:to_f).join(",")}]"
+        if Utils.array?(value)
+          value = "[#{value.to_a.map(&:to_f).join(",")}]"
         end
         super(value)
       end
@@ -17,8 +17,8 @@ module Neighbor
       def cast_value(value)
         if value.is_a?(String)
           value[1..-1].split(",").map(&:to_f)
-        elsif value.is_a?(Array)
-          value
+        elsif Utils.array?(value)
+          value.to_a
         else
           raise "can't cast #{value.class.name} to vector"
         end

data/lib/neighbor/utils.rb

@@ -38,5 +38,9 @@ module Neighbor
       # could also throw error
       norm > 0 ? value.map { |v| v / norm } : value
     end
+
+    def self.array?(value)
+      !value.nil? && value.respond_to?(:to_a)
+    end
   end
 end

data/lib/neighbor/version.rb

@@ -1,3 +1,3 @@
 module Neighbor
-  VERSION = "0.4.0"
+  VERSION = "0.4.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neighbor
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-26 00:00:00.000000000 Z
+date: 2024-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord