pgvector 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07a80636c13841d2fa97f8b740feb095c1c2f94fa3691b374d1472bba918bddf
-  data.tar.gz: 50c9e67781fbbe23fa3dbe2f87790025f18d3686e412b46f62d1cb0d2995ecb3
+  metadata.gz: fbdde0af357aaae0f727dff71e8678c705b1b347ecca7b49a8ca097189b42db0
+  data.tar.gz: d9d8bee13760ca165905f888e79507350565425ae819c452bf723ff775899235
 SHA512:
-  metadata.gz: f0aa5b733e1bc4022c2052be6b04aa0656bf306f588baf2907827674aeadc7dc52507895043049246634999dea37de629c19a164bd289fbd979eb628aaab0c33
-  data.tar.gz: f52e2e9c1c074c4f809fe78d6926bcb630694d79199e288ebb33d136a05191c4b2fff6038f8d7fc56d4291f949caa1b536d942aa7d0912be2daccf5a7ee23f83
+  metadata.gz: '04629b78419ba4fd5325792d2b47f60a6a42fe2e5f8c681bcbd53d2f769b0c8db74ba7e53b9ef84c579b1c98ded45b51e3fe58347394b5f7b707efc81a6500f3'
+  data.tar.gz: 995963de5eb7bd7587bff1f70a709cb9b4e7f0791ef6f23b6930104c5a0ba61514a684b2dbf784b2e5b62464e84f5c6edbf569681578d75c5db860500c46bbae

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.3.2 (2024-07-17)
+
+- Added `to_binary` method to `Vector`, `Bit`, and `SparseVector`
+
+## 0.3.1 (2024-07-10)
+
+- Added support for `bit` type to pg
+- Added extension for Sequel
+
 ## 0.3.0 (2024-06-25)
 
 - Added support for `halfvec` and `sparsevec` types

data/README.md

@@ -24,6 +24,7 @@ And follow the instructions for your database library:
 Or check out some examples:
 
 - [Embeddings](examples/openai_embeddings.rb) with OpenAI
+- [Binary embeddings](examples/cohere_embeddings.rb) with Cohere
 - [User-based recommendations](examples/disco_user_recs.rb) with Disco
 - [Item-based recommendations](examples/disco_item_recs.rb) with Disco
 - [Bulk loading](examples/bulk_loading.rb) with `COPY`

data/lib/pgvector/bit.rb

@@ -0,0 +1,32 @@
+module Pgvector
+  class Bit
+    def initialize(data)
+      if data.is_a?(Array)
+        @data = data.map { |v| v ? "1" : "0" }.join
+      else
+        @data = data.to_str
+      end
+    end
+
+    def self.from_text(string)
+      Bit.new(string)
+    end
+
+    def self.from_binary(string)
+      length, data = string.unpack("l>B*")
+      Bit.new(data[...length])
+    end
+
+    def to_s
+      @data
+    end
+
+    def to_a
+      @data.each_char.map { |v| v != "0" }
+    end
+
+    def to_binary
+      [@data.length, @data].pack("l>B*")
+    end
+  end
+end

data/lib/pgvector/pg.rb

@@ -3,14 +3,17 @@ require "pg"
 module Pgvector
   module PG
     def self.register_vector(registry)
-      registry.register_type(0, "vector", nil, TextDecoder::Vector)
-      registry.register_type(1, "vector", nil, BinaryDecoder::Vector)
+      registry.register_type(0, "vector", TextEncoder::Vector, TextDecoder::Vector)
+      registry.register_type(1, "vector", BinaryEncoder::Vector, BinaryDecoder::Vector)
 
       # no binary decoder for halfvec since unpack does not have directive for half-precision
-      registry.register_type(0, "halfvec", nil, TextDecoder::Halfvec)
+      registry.register_type(0, "halfvec", TextEncoder::Halfvec, TextDecoder::Halfvec)
 
-      registry.register_type(0, "sparsevec", nil, TextDecoder::Sparsevec)
-      registry.register_type(1, "sparsevec", nil, BinaryDecoder::Sparsevec)
+      registry.register_type(0, "bit", TextEncoder::Bit, TextDecoder::Bit)
+      registry.register_type(1, "bit", BinaryEncoder::Bit, BinaryDecoder::Bit)
+
+      registry.register_type(0, "sparsevec", TextEncoder::Sparsevec, TextDecoder::Sparsevec)
+      registry.register_type(1, "sparsevec", BinaryEncoder::Sparsevec, BinaryDecoder::Sparsevec)
     end
 
     module BinaryDecoder
@@ -20,9 +23,44 @@ module Pgvector
         end
       end
 
+      class Bit < ::PG::SimpleDecoder
+        def decode(string, tuple = nil, field = nil)
+          ::Pgvector::Bit.from_binary(string).to_s
+        end
+      end
+
       class Sparsevec < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          SparseVector.from_binary(string)
+          ::Pgvector::SparseVector.from_binary(string)
+        end
+      end
+    end
+
+    module BinaryEncoder
+      # experimental
+      def self.type_map
+        tm = ::PG::TypeMapByClass.new
+        tm[::Pgvector::Vector] = Vector.new
+        tm[::Pgvector::Bit] = Bit.new
+        tm[::Pgvector::SparseVector] = Sparsevec.new
+        tm
+      end
+
+      class Vector < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_binary
+        end
+      end
+
+      class Bit < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_binary
+        end
+      end
+
+      class Sparsevec < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_binary
         end
       end
     end
@@ -36,13 +74,55 @@ module Pgvector
 
       class Halfvec < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          HalfVector.from_text(string).to_a
+          ::Pgvector::HalfVector.from_text(string).to_a
+        end
+      end
+
+      class Bit < ::PG::SimpleDecoder
+        def decode(string, tuple = nil, field = nil)
+          ::Pgvector::Bit.from_text(string).to_s
         end
       end
 
       class Sparsevec < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          SparseVector.from_text(string)
+          ::Pgvector::SparseVector.from_text(string)
+        end
+      end
+    end
+
+    module TextEncoder
+      # experimental
+      def self.type_map
+        tm = ::PG::TypeMapByClass.new
+        tm[::Pgvector::Vector] = Vector.new
+        tm[::Pgvector::HalfVector] = Halfvec.new
+        tm[::Pgvector::Bit] = Bit.new
+        tm[::Pgvector::SparseVector] = Sparsevec.new
+        tm
+      end
+
+      class Vector < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
+        end
+      end
+
+      class Halfvec < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
+        end
+      end
+
+      class Bit < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
+        end
+      end
+
+      class Sparsevec < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
         end
       end
     end

data/lib/pgvector/sparse_vector.rb

@@ -30,6 +30,14 @@ module Pgvector
       arr
     end
 
+    def to_binary
+      nnz = @indices.size
+      buffer = [dimensions, nnz, 0].pack("l>l>l>")
+      @indices.pack("l>#{nnz}", buffer: buffer)
+      @values.pack("g#{nnz}", buffer: buffer)
+      buffer
+    end
+
     private
 
     def from_hash(data, dimensions)

data/lib/pgvector/vector.rb

@@ -1,7 +1,13 @@
 module Pgvector
   class Vector
     def initialize(data)
-      @data = data.to_a.map(&:to_f)
+      # keep as NArray when possible for performance
+      @data =
+        if numo?(data)
+          data.cast_to(Numo::SFloat)
+        else
+          data.to_a.map(&:to_f)
+        end
     end
 
     def self.from_text(string)
@@ -19,7 +25,23 @@ module Pgvector
     end
 
     def to_a
-      @data
+      @data.to_a
+    end
+
+    def to_binary
+      if numo?(@data)
+        [@data.shape[0], 0].pack("s>s>") + @data.to_network.to_binary
+      else
+        buffer = [@data.size, 0].pack("s>s>")
+        @data.pack("g*", buffer: buffer)
+        buffer
+      end
+    end
+
+    private
+
+    def numo?(data)
+      defined?(Numo::NArray) && data.is_a?(Numo::NArray)
     end
   end
 end

data/lib/pgvector/version.rb

@@ -1,3 +1,3 @@
 module Pgvector
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

data/lib/pgvector.rb

@@ -1,4 +1,5 @@
 # modules
+require_relative "pgvector/bit"
 require_relative "pgvector/half_vector"
 require_relative "pgvector/sparse_vector"
 require_relative "pgvector/vector"
@@ -8,7 +9,7 @@ module Pgvector
   autoload :PG, "pgvector/pg"
 
   def self.encode(data)
-    if data.is_a?(SparseVector)
+    if data.is_a?(Vector) || data.is_a?(HalfVector) || data.is_a?(SparseVector)
       data.to_s
     else
       Vector.new(data).to_s

data/lib/sequel/extensions/pgvector.rb

@@ -0,0 +1,5 @@
+require_relative "../plugins/pgvector"
+
+module Sequel
+  Dataset.register_extension(:pgvector, Plugins::Pgvector::DatasetMethods)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgvector
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-26 00:00:00.000000000 Z
+date: 2024-07-17 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org
@@ -20,11 +20,13 @@ files:
 - LICENSE.txt
 - README.md
 - lib/pgvector.rb
+- lib/pgvector/bit.rb
 - lib/pgvector/half_vector.rb
 - lib/pgvector/pg.rb
 - lib/pgvector/sparse_vector.rb
 - lib/pgvector/vector.rb
 - lib/pgvector/version.rb
+- lib/sequel/extensions/pgvector.rb
 - lib/sequel/plugins/pgvector.rb
 homepage: https://github.com/pgvector/pgvector-ruby
 licenses: