pgvector 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d3ef6a53417383ff7f8a2a514df78513dfe9029e524f0f624df8828fb99dba0
-  data.tar.gz: f326a21d3942079cdadc22d9a61367e7a6651ae37c5eb07a3955182f9f569ad0
+  metadata.gz: 7238af114feedc000b706855f9af446a583455a94771ece1e90c3e3082e63c44
+  data.tar.gz: f926141da347dc8bc1f3663e271cf3efa6b73b7c84c11e17db92f0d32af1a47c
 SHA512:
-  metadata.gz: fa4d6519685d179e3d5712cbcf1e6989ca4f98e5b0902f5061eca2be55451162f199fcc75f19841dbd8ada8c18de69ac6fa39cad545d4b5a6c542e9cdd0109ea
-  data.tar.gz: fd71245492b2ff8a06a0af60dbd12e57e44025f2662d4ea77eb7176aaaa4c4cb3b39c1e159edce92944da5b54fddbb19d1b504acd76a12c4efdb9a0fb6e797da
+  metadata.gz: 6351a9f10f989fb92e8b5f5edea1d66dcf5110b65b748fef654faa9bfe189af64a095dd263be03af8513f552063b828730bbc09b3545f3ccf25bc7b3aef9f47a
+  data.tar.gz: 86d146518d40c9af209e9a671db10c024df4c256271de7062c4de657404508d2c0aef599cb5a9a29c6b0190f40b2bedfb91dbb41c6a31bb40b0e0b7242d4957b

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.3.3 (2026-03-18)
+
+- Fixed error with `nil` values for Sequel
+
+## 0.3.2 (2024-07-17)
+
+- Added `to_binary` method to `Vector`, `Bit`, and `SparseVector`
+
 ## 0.3.1 (2024-07-10)
 
 - Added support for `bit` type to pg

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2022-2024 Andrew Kane
+Copyright (c) 2022-2025 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -23,10 +23,17 @@ And follow the instructions for your database library:
 
 Or check out some examples:
 
-- [Embeddings](examples/openai_embeddings.rb) with OpenAI
-- [User-based recommendations](examples/disco_user_recs.rb) with Disco
-- [Item-based recommendations](examples/disco_item_recs.rb) with Disco
-- [Bulk loading](examples/bulk_loading.rb) with `COPY`
+- [Embeddings](examples/openai/example.rb) with OpenAI
+- [Binary embeddings](examples/cohere/example.rb) with Cohere
+- [Sentence embeddings](examples/informers/example.rb) with Informers
+- [Hybrid search](examples/hybrid/example.rb) with Informers (Reciprocal Rank Fusion)
+- [Sparse search](examples/sparse/example.rb) with Transformers.rb
+- [Morgan fingerprints](examples/rdkit/example.rb) with RDKit.rb
+- [Topic modeling](examples/tomoto/example.rb) with tomoto.rb
+- [User-based recommendations](examples/disco/user_recs.rb) with Disco
+- [Item-based recommendations](examples/disco/item_recs.rb) with Disco
+- [Horizontal scaling](examples/citus/example.rb) with Citus
+- [Bulk loading](examples/loading/example.rb) with `COPY`
 
 ## pg
 
@@ -126,6 +133,48 @@ DB.add_index :items, :embedding, type: "hnsw", opclass: "vector_l2_ops"
 
 Use `vector_ip_ops` for inner product and `vector_cosine_ops` for cosine distance
 
+## Reference
+
+### Sparse Vectors
+
+Create a sparse vector from an array
+
+```ruby
+vec = Pgvector::SparseVector.new([1, 0, 2, 0, 3, 0])
+```
+
+Or a hash of non-zero elements
+
+```ruby
+vec = Pgvector::SparseVector.new({0 => 1, 2 => 2, 4 => 3}, 6)
+```
+
+Note: Indices start at 0
+
+Get the number of dimensions
+
+```ruby
+dim = vec.dimensions
+```
+
+Get the indices of non-zero elements
+
+```ruby
+indices = vec.indices
+```
+
+Get the values of non-zero elements
+
+```ruby
+values = vec.values
+```
+
+Get an array
+
+```ruby
+arr = vec.to_a
+```
+
 ## History
 
 View the [changelog](https://github.com/pgvector/pgvector-ruby/blob/master/CHANGELOG.md)
@@ -148,3 +197,12 @@ createdb pgvector_ruby_test
 bundle install
 bundle exec rake test
 ```
+
+To run an example:
+
+```sh
+cd examples/loading
+bundle install
+createdb pgvector_example
+bundle exec ruby example.rb
+```

data/lib/pgvector/bit.rb

@@ -13,8 +13,8 @@ module Pgvector
     end
 
     def self.from_binary(string)
-      length = string[..3].unpack1("l>")
-      Bit.new(string[4..].unpack("B*").join[...length])
+      length, data = string.unpack("l>B*")
+      Bit.new(data[...length])
     end
 
     def to_s
@@ -24,5 +24,9 @@ module Pgvector
     def to_a
       @data.each_char.map { |v| v != "0" }
     end
+
+    def to_binary
+      [@data.length, @data].pack("l>B*")
+    end
   end
 end

data/lib/pgvector/pg.rb

@@ -3,17 +3,17 @@ require "pg"
 module Pgvector
   module PG
     def self.register_vector(registry)
-      registry.register_type(0, "vector", nil, TextDecoder::Vector)
-      registry.register_type(1, "vector", nil, BinaryDecoder::Vector)
+      registry.register_type(0, "vector", TextEncoder::Vector, TextDecoder::Vector)
+      registry.register_type(1, "vector", BinaryEncoder::Vector, BinaryDecoder::Vector)
 
       # no binary decoder for halfvec since unpack does not have directive for half-precision
-      registry.register_type(0, "halfvec", nil, TextDecoder::Halfvec)
+      registry.register_type(0, "halfvec", TextEncoder::Halfvec, TextDecoder::Halfvec)
 
-      registry.register_type(0, "bit", nil, TextDecoder::Bit)
-      registry.register_type(1, "bit", nil, BinaryDecoder::Bit)
+      registry.register_type(0, "bit", TextEncoder::Bit, TextDecoder::Bit)
+      registry.register_type(1, "bit", BinaryEncoder::Bit, BinaryDecoder::Bit)
 
-      registry.register_type(0, "sparsevec", nil, TextDecoder::Sparsevec)
-      registry.register_type(1, "sparsevec", nil, BinaryDecoder::Sparsevec)
+      registry.register_type(0, "sparsevec", TextEncoder::Sparsevec, TextDecoder::Sparsevec)
+      registry.register_type(1, "sparsevec", BinaryEncoder::Sparsevec, BinaryDecoder::Sparsevec)
     end
 
     module BinaryDecoder
@@ -31,7 +31,36 @@ module Pgvector
 
       class Sparsevec < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          SparseVector.from_binary(string)
+          ::Pgvector::SparseVector.from_binary(string)
+        end
+      end
+    end
+
+    module BinaryEncoder
+      # experimental
+      def self.type_map
+        tm = ::PG::TypeMapByClass.new
+        tm[::Pgvector::Vector] = Vector.new
+        tm[::Pgvector::Bit] = Bit.new
+        tm[::Pgvector::SparseVector] = Sparsevec.new
+        tm
+      end
+
+      class Vector < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_binary
+        end
+      end
+
+      class Bit < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_binary
+        end
+      end
+
+      class Sparsevec < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_binary
         end
       end
     end
@@ -45,7 +74,7 @@ module Pgvector
 
       class Halfvec < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          HalfVector.from_text(string).to_a
+          ::Pgvector::HalfVector.from_text(string).to_a
         end
       end
 
@@ -57,7 +86,43 @@ module Pgvector
 
       class Sparsevec < ::PG::SimpleDecoder
         def decode(string, tuple = nil, field = nil)
-          SparseVector.from_text(string)
+          ::Pgvector::SparseVector.from_text(string)
+        end
+      end
+    end
+
+    module TextEncoder
+      # experimental
+      def self.type_map
+        tm = ::PG::TypeMapByClass.new
+        tm[::Pgvector::Vector] = Vector.new
+        tm[::Pgvector::HalfVector] = Halfvec.new
+        tm[::Pgvector::Bit] = Bit.new
+        tm[::Pgvector::SparseVector] = Sparsevec.new
+        tm
+      end
+
+      class Vector < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
+        end
+      end
+
+      class Halfvec < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
+        end
+      end
+
+      class Bit < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
+        end
+      end
+
+      class Sparsevec < ::PG::SimpleEncoder
+        def encode(value)
+          value.to_s
         end
       end
     end

data/lib/pgvector/sparse_vector.rb

@@ -30,6 +30,14 @@ module Pgvector
       arr
     end
 
+    def to_binary
+      nnz = @indices.size
+      buffer = [dimensions, nnz, 0].pack("l>l>l>")
+      @indices.pack("l>#{nnz}", buffer: buffer)
+      @values.pack("g#{nnz}", buffer: buffer)
+      buffer
+    end
+
     private
 
     def from_hash(data, dimensions)

data/lib/pgvector/vector.rb

@@ -1,7 +1,13 @@
 module Pgvector
   class Vector
     def initialize(data)
-      @data = data.to_a.map(&:to_f)
+      # keep as NArray when possible for performance
+      @data =
+        if numo?(data)
+          data.cast_to(Numo::SFloat)
+        else
+          data.to_a.map(&:to_f)
+        end
     end
 
     def self.from_text(string)
@@ -19,7 +25,23 @@ module Pgvector
     end
 
     def to_a
-      @data
+      @data.to_a
+    end
+
+    def to_binary
+      if numo?(@data)
+        [@data.shape[0], 0].pack("s>s>") + @data.to_network.to_binary.force_encoding(Encoding::BINARY)
+      else
+        buffer = [@data.size, 0].pack("s>s>")
+        @data.pack("g*", buffer: buffer)
+        buffer
+      end
+    end
+
+    private
+
+    def numo?(data)
+      defined?(Numo::NArray) && data.is_a?(Numo::NArray)
     end
   end
 end

data/lib/pgvector/version.rb

@@ -1,3 +1,3 @@
 module Pgvector
-  VERSION = "0.3.1"
+  VERSION = "0.3.3"
 end

data/lib/sequel/plugins/pgvector.rb

@@ -10,6 +10,7 @@ module Sequel
 
       module DatasetMethods
         def nearest_neighbors(column, value, distance:)
+          return extension(:null_dataset).nullify if value.nil?
           value = ::Pgvector.encode(value) unless value.is_a?(String)
           quoted_column = quote_identifier(column)
           distance = distance.to_s
@@ -67,7 +68,7 @@ module Sequel
         end
 
         def []=(k, v)
-          if self.class.vector_columns.key?(k.to_sym) && !v.is_a?(String)
+          if self.class.vector_columns.key?(k.to_sym) && !v.is_a?(String) && !v.nil?
             super(k, ::Pgvector.encode(v))
           else
             super
@@ -75,10 +76,12 @@ module Sequel
         end
 
         def [](k)
-          if self.class.vector_columns.key?(k.to_sym)
-            ::Pgvector.decode(super)
+          v = super
+          if self.class.vector_columns.key?(k.to_sym) && !v.nil?
+            # to_s needed for JRuby
+            ::Pgvector.decode(v.to_s)
           else
-            super
+            v
           end
         end
       end

metadata

@@ -1,16 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgvector
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-11 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
-description:
 email: andrew@ankane.org
 executables: []
 extensions: []
@@ -32,7 +30,6 @@ homepage: https://github.com/pgvector/pgvector-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -47,8 +44,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: pgvector support for Ruby
 test_files: []