neighbor 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfc4af6302c7098ea40f96e9d8a19706aff46a2506cad541ff18ee07fcd11019
-  data.tar.gz: a79b59895ca3b99a7c048eddd20cb3602b2660425ab44463e7021ab763a26f62
+  metadata.gz: e8d611fd277cd48d309b2a087fdeb22f39f43d8ef81fcab57763bd5b4b2e48b3
+  data.tar.gz: fe0a5f7e4aa1ebd81f8c5849be67dc0f3c948d53d313b80259def04b9b7e9e84
 SHA512:
-  metadata.gz: 11081e687de4c79428351095477137f9140bc6c0363d09c54ece8fd5f7bbe2df802d740332f4474357f9e9e57157bd1f1f4dd3671c106d24dc7e01e2f0d84e2a
-  data.tar.gz: f18d787b22df7bbc00c69b1f9f6262e19c0214f6ef28814a5c05d9e8c3dae357f32596994bef60112d3f53dac0c1b51f6c99af6345c7e01b2f26cef1a7b42226
+  metadata.gz: 0d9d9d0be9f2929f1eab7e5df52a0606ca8c220ea74e6ef349c2ab77e5da44bd4b98ed4d2271345b2d679882f1790ac54b8373c13a69f75607935322bdb68754
+  data.tar.gz: 834c7d6e26be9b6d8fc262280048e5104cd022587fed30d030f6c0edd849966a49a08bc793f1b49764b1fc5afc014c1463dd8acacfb5739bc507f4a77d3281a1

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.3 (2024-09-02)
+
+- Added `rrf` method
+
 ## 0.4.2 (2024-08-27)
 
 - Fixed error with `nil` values

data/README.md

@@ -243,11 +243,12 @@ Item.nearest_neighbors(:embedding, embedding, distance: "euclidean").first(5)
 
 ## Examples
 
-- [OpenAI Embeddings](#openai-embeddings)
-- [Cohere Embeddings](#cohere-embeddings)
-- [Sentence Embeddings](#sentence-embeddings)
-- [Sparse Embeddings](#sparse-embeddings)
-- [Disco Recommendations](#disco-recommendations)
+- [Embeddings](#openai-embeddings) with OpenAI
+- [Binary embeddings](#cohere-embeddings) with Cohere
+- [Sentence embeddings](#sentence-embeddings) with Informers
+- [Hybrid search](#hybrid-search) with Informers
+- [Sparse search](#sparse-search) with Transformers.rb
+- [Recommendations](#disco-recommendations) with Disco
 
 ### OpenAI Embeddings
 
@@ -411,7 +412,7 @@ end
 Load a [model](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)
 
 ```ruby
-model = Informers::Model.new("sentence-transformers/all-MiniLM-L6-v2")
+model = Informers.pipeline("embedding", "sentence-transformers/all-MiniLM-L6-v2")
 ```
 
 Pass your input
@@ -422,7 +423,7 @@ input = [
   "The cat is purring",
   "The bear is growling"
 ]
-embeddings = model.embed(input)
+embeddings = model.(input)
 ```
 
 Store the embeddings
@@ -444,7 +445,86 @@ document.nearest_neighbors(:embedding, distance: "cosine").first(5).map(&:conten
 
 See the [complete code](examples/informers/example.rb)
 
-### Sparse Embeddings
+### Hybrid Search
+
+You can use Neighbor for hybrid search with [Informers](https://github.com/ankane/informers).
+
+Generate a model
+
+```sh
+rails generate model Document content:text embedding:vector{768}
+rails db:migrate
+```
+
+And add `has_neighbors` and a scope for keyword search
+
+```ruby
+class Document < ApplicationRecord
+  has_neighbors :embedding
+
+  scope :search, ->(query) {
+    where("to_tsvector(content) @@ plainto_tsquery(?)", query)
+      .order(Arel.sql("ts_rank_cd(to_tsvector(content), plainto_tsquery(?)) DESC", query))
+  }
+end
+```
+
+Create some documents
+
+```ruby
+texts = [
+  "The dog is barking",
+  "The cat is purring",
+  "The bear is growling"
+]
+documents = Document.create!(texts.map { |v| {content: v} })
+```
+
+Generate an embedding for each document
+
+```ruby
+embed = Informers.pipeline("embedding", "Snowflake/snowflake-arctic-embed-m-v1.5")
+embed_options = {model_output: "sentence_embedding", pooling: "none"} # specific to embedding model
+embeddings = embed.(documents.map(&:content), **embed_options)
+
+documents.zip(embeddings) do |document, embedding|
+  document.update!(embedding: embedding)
+end
+```
+
+Perform keyword search
+
+```ruby
+query = "growling bear"
+keyword_results = Document.search(query).limit(20).load_async
+```
+
+And semantic search in parallel (the query prefix is specific to the [embedding model](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5))
+
+```ruby
+query_prefix = "Represent this sentence for searching relevant passages: "
+query_embedding = embed.(query_prefix + query, **embed_options)
+semantic_results =
+  Document.nearest_neighbors(:embedding, query_embedding, distance: "cosine").limit(20).load_async
+```
+
+To combine the results, use Reciprocal Rank Fusion (RRF)
+
+```ruby
+Neighbor::Reranking.rrf(keyword_results, semantic_results)
+```
+
+Or a reranking model
+
+```ruby
+rerank = Informers.pipeline("reranking", "mixedbread-ai/mxbai-rerank-xsmall-v1")
+results = (keyword_results + semantic_results).uniq
+rerank.(query, results.map(&:content), top_k: 5).map { |v| results[v[:doc_id]] }
+```
+
+See the [complete code](examples/hybrid/example.rb)
+
+### Sparse Search
 
 You can generate sparse embeddings locally with [Transformers.rb](https://github.com/ankane/transformers-ruby).
 
@@ -556,7 +636,7 @@ movies = []
 recommender.item_ids.each do |item_id|
   movies << {name: item_id, factors: recommender.item_factors(item_id)}
 end
-Movie.insert_all!(movies)
+Movie.create!(movies)
 ```
 
 And get similar movies

data/lib/neighbor/reranking.rb

@@ -0,0 +1,27 @@
+module Neighbor
+  module Reranking
+    def self.rrf(first_ranking, *rankings, k: 60)
+      rankings.unshift(first_ranking)
+
+      ranks = []
+      results = []
+      rankings.each do |ranking|
+        ranks << ranking.map.with_index.to_h { |v, i| [v, i + 1] }
+        results.concat(ranking)
+      end
+
+      results =
+        results.uniq.map do |result|
+          score =
+            ranks.sum do |rank|
+              r = rank[result]
+              r ? 1.0 / (k + r) : 0.0
+            end
+
+          {result: result, score: score}
+        end
+
+      results.sort_by { |v| -v[:score] }
+    end
+  end
+end

data/lib/neighbor/version.rb

@@ -1,3 +1,3 @@
 module Neighbor
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/lib/neighbor.rb

@@ -2,6 +2,7 @@
 require "active_support"
 
 # modules
+require_relative "neighbor/reranking"
 require_relative "neighbor/sparse_vector"
 require_relative "neighbor/utils"
 require_relative "neighbor/version"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neighbor
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-27 00:00:00.000000000 Z
+date: 2024-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -40,6 +40,7 @@ files:
 - lib/neighbor.rb
 - lib/neighbor/model.rb
 - lib/neighbor/railtie.rb
+- lib/neighbor/reranking.rb
 - lib/neighbor/sparse_vector.rb
 - lib/neighbor/type/cube.rb
 - lib/neighbor/type/halfvec.rb