pgvector 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 215f975cafa2f782f3777ec65dd2fea587b420286f7f5ccf63b6b68376756dfd
-  data.tar.gz: e1979a1aa4fd4157cb04ae73c0c62b378046ca97154a25d4355e827be5abc53b
+  metadata.gz: 07a80636c13841d2fa97f8b740feb095c1c2f94fa3691b374d1472bba918bddf
+  data.tar.gz: 50c9e67781fbbe23fa3dbe2f87790025f18d3686e412b46f62d1cb0d2995ecb3
 SHA512:
-  metadata.gz: 1e54d7c41f8750b99262021e402f329252b428678ee99510dd52a1a113aeffdf35628a23b6ca9408ce55350a237cb2d224e66dfe7aafefd645612a29b74529ff
-  data.tar.gz: c8a014bb3708690cf3e463954aa7a846ce79075da039e23b3f94644e50e3c28d05376539699fd8ffe3c3b93d54eecb393fed472bddb60619ba897a5204bc511a
+  metadata.gz: f0aa5b733e1bc4022c2052be6b04aa0656bf306f588baf2907827674aeadc7dc52507895043049246634999dea37de629c19a164bd289fbd979eb628aaab0c33
+  data.tar.gz: f52e2e9c1c074c4f809fe78d6926bcb630694d79199e288ebb33d136a05191c4b2fff6038f8d7fc56d4291f949caa1b536d942aa7d0912be2daccf5a7ee23f83

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.3.0 (2024-06-25)
+
+- Added support for `halfvec` and `sparsevec` types
+- Added `taxicab`, `hamming`, and `jaccard` distances for Sequel
+- Dropped support for Ruby < 3.1
+
+## 0.2.2 (2023-10-03)
+
+- Added `nearest_neighbors` method to datasets with Sequel
+
 ## 0.2.1 (2023-06-04)
 
 - Added support for Sequel

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2022-2023 Andrew Kane
+Copyright (c) 2022-2024 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -6,7 +6,7 @@ Supports [pg](https://github.com/ged/ruby-pg) and [Sequel](https://github.com/je
 
 For Rails, check out [Neighbor](https://github.com/ankane/neighbor)
 
-[![Build Status](https://github.com/pgvector/pgvector-ruby/workflows/build/badge.svg?branch=master)](https://github.com/pgvector/pgvector-ruby/actions)
+[![Build Status](https://github.com/pgvector/pgvector-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/pgvector/pgvector-ruby/actions)
 
 ## Installation
 
@@ -26,10 +26,17 @@ Or check out some examples:
 - [Embeddings](examples/openai_embeddings.rb) with OpenAI
 - [User-based recommendations](examples/disco_user_recs.rb) with Disco
 - [Item-based recommendations](examples/disco_item_recs.rb) with Disco
+- [Bulk loading](examples/bulk_loading.rb) with `COPY`
 
 ## pg
 
-Register the vector type with your connection
+Enable the extension
+
+```ruby
+conn.exec("CREATE EXTENSION IF NOT EXISTS vector")
+```
+
+Optionally enable type casting for results
 
 ```ruby
 registry = PG::BasicTypeRegistry.new.define_default_types
@@ -37,6 +44,12 @@ Pgvector::PG.register_vector(registry)
 conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn, registry: registry)
 ```
 
+Create a table
+
+```ruby
+conn.exec("CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3))")
+```
+
 Insert a vector
 
 ```ruby
@@ -50,8 +63,24 @@ Get the nearest neighbors to a vector
 conn.exec_params("SELECT * FROM items ORDER BY embedding <-> $1 LIMIT 5", [embedding]).to_a
 ```
 
+Add an approximate index
+
+```ruby
+conn.exec("CREATE INDEX ON items USING hnsw (embedding vector_l2_ops)")
+# or
+conn.exec("CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)")
+```
+
+Use `vector_ip_ops` for inner product and `vector_cosine_ops` for cosine distance
+
 ## Sequel
 
+Enable the extension
+
+```ruby
+DB.run("CREATE EXTENSION IF NOT EXISTS vector")
+```
+
 Create a table
 
 ```ruby
@@ -81,7 +110,7 @@ Get the nearest neighbors to a record
 item.nearest_neighbors(:embedding, distance: "euclidean").limit(5)
 ```
 
-Also supports `inner_product` and `cosine` distance
+Also supports `inner_product`, `cosine`, `taxicab`, `hamming`, and `jaccard` distance
 
 Get the nearest neighbors to a vector
 
@@ -89,6 +118,14 @@ Get the nearest neighbors to a vector
 Item.nearest_neighbors(:embedding, [1, 1, 1], distance: "euclidean").limit(5)
 ```
 
+Add an approximate index
+
+```ruby
+DB.add_index :items, :embedding, type: "hnsw", opclass: "vector_l2_ops"
+```
+
+Use `vector_ip_ops` for inner product and `vector_cosine_ops` for cosine distance
+
 ## History
 
 View the [changelog](https://github.com/pgvector/pgvector-ruby/blob/master/CHANGELOG.md)

data/lib/pgvector/half_vector.rb

@@ -0,0 +1,19 @@
+module Pgvector
+  class HalfVector
+    def initialize(data)
+      @data = data.to_a.map(&:to_f)
+    end
+
+    def self.from_text(string)
+      new(string[1..-2].split(",").map(&:to_f))
+    end
+
+    def to_s
+      "[#{@data.to_a.map(&:to_f).join(",")}]"
+    end
+
+    def to_a
+      @data
+    end
+  end
+end

data/lib/pgvector/pg.rb

@@ -5,14 +5,24 @@ module Pgvector
     def self.register_vector(registry)
       registry.register_type(0, "vector", nil, TextDecoder::Vector)
       registry.register_type(1, "vector", nil, BinaryDecoder::Vector)
+
+      # no binary decoder for halfvec since unpack does not have directive for half-precision
+      registry.register_type(0, "halfvec", nil, TextDecoder::Halfvec)
+
+      registry.register_type(0, "sparsevec", nil, TextDecoder::Sparsevec)
+      registry.register_type(1, "sparsevec", nil, BinaryDecoder::Sparsevec)
     end
 
     module BinaryDecoder
       class Vector < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          dim, unused = string[0, 4].unpack("nn")
-          raise "expected unused to be 0" if unused != 0
-          string[4..-1].unpack("g#{dim}")
+          ::Pgvector::Vector.from_binary(string).to_a
+        end
+      end
+
+      class Sparsevec < ::PG::SimpleDecoder
+        def decode(string, tuple = nil, field = nil)
+          SparseVector.from_binary(string)
         end
       end
     end
@@ -20,7 +30,19 @@ module Pgvector
     module TextDecoder
       class Vector < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          Pgvector.decode(string)
+          ::Pgvector::Vector.from_text(string).to_a
+        end
+      end
+
+      class Halfvec < ::PG::SimpleDecoder
+        def decode(string, tuple = nil, field = nil)
+          HalfVector.from_text(string).to_a
+        end
+      end
+
+      class Sparsevec < ::PG::SimpleDecoder
+        def decode(string, tuple = nil, field = nil)
+          SparseVector.from_text(string)
         end
       end
     end

data/lib/pgvector/sparse_vector.rb

@@ -0,0 +1,87 @@
+module Pgvector
+  class SparseVector
+    attr_reader :dimensions, :indices, :values
+
+    NO_DEFAULT = Object.new
+
+    def initialize(value, dimensions = NO_DEFAULT)
+      if value.is_a?(Hash)
+        if dimensions == NO_DEFAULT
+          raise ArgumentError, "missing dimensions"
+        end
+        from_hash(value, dimensions)
+      else
+        unless dimensions == NO_DEFAULT
+          raise ArgumentError, "extra argument"
+        end
+        from_array(value)
+      end
+    end
+
+    def to_s
+      "{#{@indices.zip(@values).map { |i, v| "#{i.to_i + 1}:#{v.to_f}" }.join(",")}}/#{@dimensions.to_i}"
+    end
+
+    def to_a
+      arr = Array.new(dimensions, 0.0)
+      @indices.zip(@values) do |i, v|
+        arr[i] = v
+      end
+      arr
+    end
+
+    private
+
+    def from_hash(data, dimensions)
+      elements = data.select { |_, v| v != 0 }.sort
+      @dimensions = dimensions.to_i
+      @indices = elements.map { |v| v[0].to_i }
+      @values = elements.map { |v| v[1].to_f }
+    end
+
+    def from_array(arr)
+      arr = arr.to_a
+      @dimensions = arr.size
+      @indices = []
+      @values = []
+      arr.each_with_index do |v, i|
+        if v != 0
+          @indices << i
+          @values << v.to_f
+        end
+      end
+    end
+
+    class << self
+      def from_text(string)
+        elements, dimensions = string.split("/", 2)
+        indices = []
+        values = []
+        elements[1..-2].split(",").each do |e|
+          index, value = e.split(":", 2)
+          indices << index.to_i - 1
+          values << value.to_f
+        end
+        from_parts(dimensions.to_i, indices, values)
+      end
+
+      def from_binary(string)
+        dim, nnz, unused = string[0, 12].unpack("l>l>l>")
+        raise "expected unused to be 0" if unused != 0
+        indices = string[12, nnz * 4].unpack("l>#{nnz}")
+        values = string[(12 + nnz * 4)..-1].unpack("g#{nnz}")
+        from_parts(dim, indices, values)
+      end
+
+      private
+
+      def from_parts(dimensions, indices, values)
+        vec = allocate
+        vec.instance_variable_set(:@dimensions, dimensions)
+        vec.instance_variable_set(:@indices, indices)
+        vec.instance_variable_set(:@values, values)
+        vec
+      end
+    end
+  end
+end

data/lib/pgvector/vector.rb

@@ -0,0 +1,25 @@
+module Pgvector
+  class Vector
+    def initialize(data)
+      @data = data.to_a.map(&:to_f)
+    end
+
+    def self.from_text(string)
+      Vector.new(string[1..-2].split(",").map(&:to_f))
+    end
+
+    def self.from_binary(string)
+      dim, unused = string[0, 4].unpack("nn")
+      raise "expected unused to be 0" if unused != 0
+      Vector.new(string[4..-1].unpack("g#{dim}"))
+    end
+
+    def to_s
+      "[#{@data.to_a.map(&:to_f).join(",")}]"
+    end
+
+    def to_a
+      @data
+    end
+  end
+end

data/lib/pgvector/version.rb

@@ -1,3 +1,3 @@
 module Pgvector
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/pgvector.rb

@@ -1,14 +1,27 @@
 # modules
+require_relative "pgvector/half_vector"
+require_relative "pgvector/sparse_vector"
+require_relative "pgvector/vector"
 require_relative "pgvector/version"
 
 module Pgvector
   autoload :PG, "pgvector/pg"
 
   def self.encode(data)
-    "[#{data.to_a.map(&:to_f).join(",")}]"
+    if data.is_a?(SparseVector)
+      data.to_s
+    else
+      Vector.new(data).to_s
+    end
   end
 
   def self.decode(string)
-    string[1..-2].split(",").map(&:to_f)
+    if string[0] == "["
+      Vector.from_text(string).to_a
+    elsif string[0] == "{"
+      SparseVector.from_text(string)
+    else
+      string
+    end
   end
 end

data/lib/sequel/plugins/pgvector.rb

@@ -8,12 +8,10 @@ module Sequel
         end
       end
 
-      module ClassMethods
-        attr_accessor :vector_columns
-
+      module DatasetMethods
         def nearest_neighbors(column, value, distance:)
           value = ::Pgvector.encode(value) unless value.is_a?(String)
-          quoted_column = dataset.quote_identifier(column)
+          quoted_column = quote_identifier(column)
           distance = distance.to_s
 
           operator =
@@ -24,6 +22,12 @@ module Sequel
               "<=>"
             when "euclidean"
               "<->"
+            when "taxicab"
+              "<+>"
+            when "hamming"
+              "<~>"
+            when "jaccard"
+              "<%>"
             end
 
           raise ArgumentError, "Invalid distance: #{distance}" unless operator
@@ -41,6 +45,12 @@ module Sequel
             .exclude(column => nil)
             .order(Sequel.lit(order, value))
         end
+      end
+
+      module ClassMethods
+        attr_accessor :vector_columns
+
+        Sequel::Plugins.def_dataset_methods(self, :nearest_neighbors)
 
         Plugins.inherited_instance_variables(self, :@vector_columns => :dup)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgvector
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-05 00:00:00.000000000 Z
+date: 2024-06-26 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org
@@ -20,7 +20,10 @@ files:
 - LICENSE.txt
 - README.md
 - lib/pgvector.rb
+- lib/pgvector/half_vector.rb
 - lib/pgvector/pg.rb
+- lib/pgvector/sparse_vector.rb
+- lib/pgvector/vector.rb
 - lib/pgvector/version.rb
 - lib/sequel/plugins/pgvector.rb
 homepage: https://github.com/pgvector/pgvector-ruby
@@ -35,14 +38,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: pgvector support for Ruby