neighbor 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da5f64785d929d673be65b5cf224894b6e5d6bbd8d66a27b84339ced5e8c1334
-  data.tar.gz: fbb217005dcd6a3c1d946eec178acf7f0f87ac3521e2deb4b7c8c2bc5433b633
+  metadata.gz: a0ab3641b145d2f7b83ab5b9639b63a97243aeaadd82e057ff71d30440eccf86
+  data.tar.gz: 0042cdebaff064ee3c27658f2a1edf348768bc7022e14fc12a15eabf1f4cb20f
 SHA512:
-  metadata.gz: d293a32bfd18e00851bcbbaecac507374e9105071fbf7ab8446eaea093b9741b573356a7b292bf19dd420c2d7208c62023be4f7106711e19f9e84970aedba2bd
-  data.tar.gz: 07ed7bccab0924fe183df1176937d6ce98d1e53313e2be53d96605d691ad697ff4e14abc722b7a3501b20a5ca2880774625a306c274f81d308895fa78a1bf068
+  metadata.gz: 0c500661e9557575032762af3f54b36259a4b87493cd0ed30c559e9fb1b21c18c70b4c410becd1095aed64bbd86d8aac527cbd498a14717708030905ecc84d6e
+  data.tar.gz: 75a62b2a13ecb8a34bb6595b71d0aa5686111a390a32ab610c10bcaf8d1e688541b742073658498e20a4724a9f9a1e7ab718f0a2298a7214e5c1cbffdb47c2e2

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.3.0 (2023-07-24)
+
+- Dropped support for Ruby < 3 and Active Record < 6.1
+
+## 0.2.3 (2023-04-02)
+
+- Added support for dimensions to model generator
+
 ## 0.2.2 (2022-07-13)
 
 - Added support for configurable attribute name

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2021-2022 Andrew Kane
+Copyright (c) 2021-2023 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -35,11 +35,11 @@ rails db:migrate
 Create a migration
 
 ```ruby
-class AddNeighborVectorToItems < ActiveRecord::Migration[6.1]
+class AddNeighborVectorToItems < ActiveRecord::Migration[7.0]
   def change
-    add_column :items, :neighbor_vector, :cube
+    add_column :items, :embedding, :cube
     # or
-    add_column :items, :neighbor_vector, :vector, limit: 3 # dimensions
+    add_column :items, :embedding, :vector, limit: 3 # dimensions
   end
 end
 ```
@@ -48,26 +48,26 @@ Add to your model
 
 ```ruby
 class Item < ApplicationRecord
-  has_neighbors
+  has_neighbors :embedding
 end
 ```
 
 Update the vectors
 
 ```ruby
-item.update(neighbor_vector: [1.0, 1.2, 0.5])
+item.update(embedding: [1.0, 1.2, 0.5])
 ```
 
 Get the nearest neighbors to a record
 
 ```ruby
-item.nearest_neighbors(distance: "euclidean").first(5)
+item.nearest_neighbors(:embedding, distance: "euclidean").first(5)
 ```
 
 Get the nearest neighbors to a vector
 
 ```ruby
-Item.nearest_neighbors([0.9, 1.3, 1.1], distance: "euclidean").first(5)
+Item.nearest_neighbors(:embedding, [0.9, 1.3, 1.1], distance: "euclidean").first(5)
 ```
 
 ## Distance
@@ -84,7 +84,7 @@ For cosine distance with cube, vectors must be normalized before being stored.
 
 ```ruby
 class Item < ApplicationRecord
-  has_neighbors normalize: true
+  has_neighbors :embedding, normalize: true
 end
 ```
 
@@ -93,19 +93,19 @@ For inner product with cube, see [this example](examples/disco_user_recs_cube.rb
 Records returned from `nearest_neighbors` will have a `neighbor_distance` attribute
 
 ```ruby
-nearest_item = item.nearest_neighbors(distance: "euclidean").first
+nearest_item = item.nearest_neighbors(:embedding, distance: "euclidean").first
 nearest_item.neighbor_distance
 ```
 
 ## Dimensions
 
-The cube data type is limited 100 dimensions by default. See the [Postgres docs](https://www.postgresql.org/docs/current/cube.html) for how to increase this. The vector data type is limited to 1024 dimensions.
+The cube data type can have up to 100 dimensions by default. See the [Postgres docs](https://www.postgresql.org/docs/current/cube.html) for how to increase this. The vector data type can have up to 16,000 dimensions, and vectors with up to 2,000 dimensions can be indexed.
 
 For cube, it’s a good idea to specify the number of dimensions to ensure all records have the same number.
 
 ```ruby
-class Movie < ApplicationRecord
-  has_neighbors dimensions: 3
+class Item < ApplicationRecord
+  has_neighbors :embedding, dimensions: 3
 end
 ```
 
@@ -114,9 +114,9 @@ end
 For vector, add an approximate index to speed up queries. Create a migration with:
 
 ```ruby
-class AddIndexToItemsNeighborVector < ActiveRecord::Migration[6.1]
+class AddIndexToItemsNeighborVector < ActiveRecord::Migration[7.0]
   def change
-    add_index :items, :neighbor_vector, using: :ivfflat, opclass: :vector_l2_ops
+    add_index :items, :embedding, using: :ivfflat, opclass: :vector_l2_ops
   end
 end
 ```
@@ -129,14 +129,85 @@ Set the number of probes
 Item.connection.execute("SET ivfflat.probes = 3")
 ```
 
-## Example
+## Examples
+
+- [OpenAI Embeddings](#openai-embeddings)
+- [Disco Recommendations](#disco-recommendations)
+
+### OpenAI Embeddings
+
+Generate a model
+
+```sh
+rails generate model Article content:text embedding:vector{1536}
+rails db:migrate
+```
+
+And add `has_neighbors`
+
+```ruby
+class Article < ApplicationRecord
+  has_neighbors :embedding
+end
+```
+
+Create a method to call the [embeddings API](https://platform.openai.com/docs/guides/embeddings)
+
+```ruby
+def fetch_embeddings(input)
+  url = "https://api.openai.com/v1/embeddings"
+  headers = {
+    "Authorization" => "Bearer #{ENV.fetch("OPENAI_API_KEY")}",
+    "Content-Type" => "application/json"
+  }
+  data = {
+    input: input,
+    model: "text-embedding-ada-002"
+  }
+
+  response = Net::HTTP.post(URI(url), data.to_json, headers)
+  JSON.parse(response.body)["data"].map { |v| v["embedding"] }
+end
+```
+
+Pass your input
+
+```ruby
+input = [
+  "The dog is barking",
+  "The cat is purring",
+  "The bear is growling"
+]
+embeddings = fetch_embeddings(input)
+```
+
+Store the embeddings
+
+```ruby
+articles = []
+input.zip(embeddings) do |content, embedding|
+  articles << {content: content, embedding: embedding}
+end
+Article.insert_all!(articles) # use create! for Active Record < 6
+```
+
+And get similar articles
+
+```ruby
+article = Article.first
+article.nearest_neighbors(:embedding, distance: "inner_product").first(5).map(&:content)
+```
+
+See the [complete code](examples/openai_embeddings.rb)
+
+### Disco Recommendations
 
 You can use Neighbor for online item-based recommendations with [Disco](https://github.com/ankane/disco). We’ll use MovieLens data for this example.
 
 Generate a model
 
 ```sh
-rails generate model Movie name:string neighbor_vector:cube
+rails generate model Movie name:string factors:cube
 rails db:migrate
 ```
 
@@ -144,7 +215,7 @@ And add `has_neighbors`
 
 ```ruby
 class Movie < ApplicationRecord
-  has_neighbors dimensions: 20, normalize: true
+  has_neighbors :factors, dimensions: 20, normalize: true
 end
 ```
 
@@ -156,12 +227,12 @@ recommender = Disco::Recommender.new(factors: 20)
 recommender.fit(data)
 ```
 
-Use item factors for the neighbor vector
+Store the item factors
 
 ```ruby
 movies = []
 recommender.item_ids.each do |item_id|
-  movies << {name: item_id, neighbor_vector: recommender.item_factors(item_id)}
+  movies << {name: item_id, factors: recommender.item_factors(item_id)}
 end
 Movie.insert_all!(movies) # use create! for Active Record < 6
 ```
@@ -170,7 +241,7 @@ And get similar movies
 
 ```ruby
 movie = Movie.find_by(name: "Star Wars (1977)")
-movie.nearest_neighbors(distance: "cosine").first(5).map(&:name)
+movie.nearest_neighbors(:factors, distance: "cosine").first(5).map(&:name)
 ```
 
 See the complete code for [cube](examples/disco_item_recs_cube.rb) and [vector](examples/disco_item_recs_vector.rb)

data/lib/neighbor/model.rb

@@ -113,8 +113,8 @@ module Neighbor
           raise ArgumentError, "Invalid attribute" unless self.class.neighbor_attributes[attribute_name]
 
           self.class
-            .where.not(self.class.primary_key => send(self.class.primary_key))
-            .nearest_neighbors(attribute_name, send(attribute_name), **options)
+            .where.not(self.class.primary_key => self[self.class.primary_key])
+            .nearest_neighbors(attribute_name, self[attribute_name], **options)
         end
       end
     end

data/lib/neighbor/railtie.rb

@@ -0,0 +1,19 @@
+module Neighbor
+  class Railtie < Rails::Railtie
+    generators do
+      # rails generate model Item embedding:vector{3}
+      if defined?(Rails::Generators::GeneratedAttribute)
+        Rails::Generators::GeneratedAttribute.singleton_class.prepend(Neighbor::GeneratedAttribute)
+      end
+    end
+  end
+
+  module GeneratedAttribute
+    def parse_type_and_options(type, *, **)
+      if type =~ /\A(vector)\{(\d+)\}\z/
+        return $1, limit: $2.to_i
+      end
+      super
+    end
+  end
+end

data/lib/neighbor/version.rb

@@ -1,3 +1,3 @@
 module Neighbor
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/neighbor.rb

@@ -2,7 +2,7 @@
 require "active_support"
 
 # modules
-require "neighbor/version"
+require_relative "neighbor/version"
 
 module Neighbor
   class Error < StandardError; end
@@ -20,8 +20,8 @@ module Neighbor
 end
 
 ActiveSupport.on_load(:active_record) do
-  require "neighbor/model"
-  require "neighbor/vector"
+  require_relative "neighbor/model"
+  require_relative "neighbor/vector"
 
   extend Neighbor::Model
 
@@ -32,16 +32,7 @@ ActiveSupport.on_load(:active_record) do
   ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::NATIVE_DATABASE_TYPES[:vector] = {name: "vector"}
 
   # ensure schema can be loaded
-  if ActiveRecord::VERSION::MAJOR >= 6
-    ActiveRecord::ConnectionAdapters::TableDefinition.send(:define_column_methods, :cube, :vector)
-  else
-    ActiveRecord::ConnectionAdapters::TableDefinition.define_method :cube do |*args, **options|
-      args.each { |name| column(name, :cube, options) }
-    end
-    ActiveRecord::ConnectionAdapters::TableDefinition.define_method :vector do |*args, **options|
-      args.each { |name| column(name, :vector, options) }
-    end
-  end
+  ActiveRecord::ConnectionAdapters::TableDefinition.send(:define_column_methods, :cube, :vector)
 
   # prevent unknown OID warning
   if ActiveRecord::VERSION::MAJOR >= 7
@@ -50,3 +41,5 @@ ActiveSupport.on_load(:active_record) do
     ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(Neighbor::RegisterTypes)
   end
 end
+
+require_relative "neighbor/railtie" if defined?(Rails::Railtie)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neighbor
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-13 00:00:00.000000000 Z
+date: 2023-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
 description:
 email: andrew@ankane.org
 executables: []
@@ -39,6 +39,7 @@ files:
 - lib/generators/neighbor/vector_generator.rb
 - lib/neighbor.rb
 - lib/neighbor/model.rb
+- lib/neighbor/railtie.rb
 - lib/neighbor/vector.rb
 - lib/neighbor/version.rb
 homepage: https://github.com/ankane/neighbor
@@ -53,14 +54,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Nearest neighbor search for Rails and Postgres