neighbor-redis 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15f6a05f33831afcb282262440f8a4ccd73264c109ada3d5de1383af76cab304
-  data.tar.gz: 977b93e67d4b62db75837bf72dff4959a5b6aae8c940828273bee22563f172ad
+  metadata.gz: 94d78e38cc8f93df780741fe7c08964657e9f965ca746ffeb9b80fa5950663f6
+  data.tar.gz: 8c4ed1a76383f1a7f0d99712cce6b5d835472eabf48357ec1b7b9044f212e8ff
 SHA512:
-  metadata.gz: f6148b2717ddbb7cef8e9d762866b3c0d981be8e57e6a9e7c8a28f3a37b8402b5add0383e2676bc21b1c93dd687e10ae2fdaf9bfb31969adc80d7fb43def01a4
-  data.tar.gz: e7bc98f4436a8e48c84215e837461312bf0d610e9f5103a8d6f0ce39aa800970f5adb4c341ff691e21a059d30e4540eda4ef1df3d17f7d313923e6799860beb3
+  metadata.gz: 83d4415cd858429b265e764f09fb34df4492a0fa93d94d6a766688a8e2f3a98ae9850102ca684322838a898c7c2fb430aa7fbc0082564679b775d7dce79d7e18
+  data.tar.gz: 8dd3a10a9fbaf716df69cc166cd4703e2a0c9489a3cddc58816e4ef17a0f0a5be7fe16da680553b4884606eaddc9e62fdbdea572b6891e44ed54da7335187018

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.3.0 (2025-09-12)
+
+- Added support for vector sets
+- Added support for SVS Vamana indexes
+- Added support for metadata
+- Added `info` and `count` methods to indexes
+- Updated `add` and `remove` methods to return boolean
+- Updated `add_all` method to return array of booleans
+- Updated `create` and `promote` methods to return `nil`
+- Dropped support for Ruby < 3.2
+
+## 0.2.1 (2025-05-06)
+
+- Added support for Redis 8
+
 ## 0.2.0 (2024-10-23)
 
 - Dropped support for Ruby < 3.1

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2023 Andrew Kane
+Copyright (c) 2023-2025 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,14 +2,16 @@
 
 Nearest neighbor search for Ruby and Redis
 
+Supports Redis 8 [vector sets](https://redis.io/docs/latest/develop/data-types/vector-sets/) and RediSearch [vector indexes](https://redis.io/docs/latest/develop/ai/search-and-query/vectors/)
+
 [![Build Status](https://github.com/ankane/neighbor-redis/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/neighbor-redis/actions)
 
 ## Installation
 
-First, [install RediSearch](https://redis.io/docs/stack/search/quick_start/). With Docker, use:
+First, install Redis. With Docker, use:
 
 ```sh
-docker run -p 6379:6379 redis/redis-stack-server
+docker run -p 6379:6379 redis:8
 ```
 
 Add this line to your application’s Gemfile:
@@ -29,11 +31,10 @@ Neighbor::Redis.client = RedisClient.config.new_pool
 Create an index
 
 ```ruby
-index = Neighbor::Redis::HNSWIndex.new("items", dimensions: 3, distance: "l2")
-index.create
+index = Neighbor::Redis::VectorSet.new("items")
 ```
 
-Add items
+Add vectors
 
 ```ruby
 index.add(1, [1, 1, 1])
@@ -41,105 +42,196 @@ index.add(2, [2, 2, 2])
 index.add(3, [1, 1, 2])
 ```
 
-Note: IDs are stored and returned as strings (uses less total memory)
+Search for nearest neighbors to a vector
 
-Get the nearest neighbors to an item
+```ruby
+index.search([1, 1, 1], count: 5)
+```
+
+Search for nearest neighbors to a vector in the index
 
 ```ruby
-index.nearest(1, count: 5)
+index.search_id(1, count: 5)
 ```
 
-Get the nearest neighbors to a vector
+IDs are treated as strings by default, but can also be treated as integers
 
 ```ruby
-index.search([1, 1, 1], count: 5)
+Neighbor::Redis::VectorSet.new("items", id_type: "integer")
 ```
 
-## Distance
+## Operations
 
-Supported values are:
+Add or update a vector
 
-- `l2`
-- `inner_product`
-- `cosine`
+```ruby
+index.add(id, vector)
+```
 
-## Index Types
+Add or update multiple vectors
+
+```ruby
+index.add_all(ids, vectors)
+```
 
-Hierarchical Navigable Small World (HNSW)
+Get a vector
 
 ```ruby
-Neighbor::Redis::HNSWIndex.new(
-  name,
-  initial_cap: nil,
-  m: 16,
-  ef_construction: 200,
-  ef_runtime: 10,
-  epsilon: 0.01
-)
+index.find(id)
 ```
 
-Flat
+Remove a vector
 
 ```ruby
-Neighbor::Redis::FlatIndex.new(
-  name,
-  initial_cap: nil,
-  block_size: 1024
-)
+index.remove(id)
 ```
 
-## Additional Options
+Remove multiple vectors
 
-Store vectors as double precision (instead of single precision)
+```ruby
+index.remove_all(ids)
+```
+
+Count vectors
 
 ```ruby
-Neighbor::Redis::HNSWIndex.new(name, type: "float64")
+index.count
 ```
 
-Store vectors as JSON (instead of a hash/blob)
+## Metadata
+
+Add a vector with metadata
 
 ```ruby
-Neighbor::Redis::HNSWIndex.new(name, redis_type: "json")
+index.add(id, vector, metadata: {category: "A"})
 ```
 
-## Changing Options
+Add multiple vectors with metadata
+
+```ruby
+index.add_all(ids, vectors, metadata: [{category: "A"}, {category: "B"}, ...])
+```
 
-Create a new index to change any index options
+Get metadata for a vector
 
 ```ruby
-Neighbor::Redis::HNSWIndex.new("items-v2", **new_options)
+index.metadata(id)
 ```
 
-## Additional Operations
+Get metadata with search results
+
+```ruby
+index.search(vector, with_metadata: true)
+```
 
-Add multiple items
+Set metadata
 
 ```ruby
-index.add_all(ids, embeddings)
+index.set_metadata(id, {category: "B"})
 ```
 
-Get an item
+Remove metadata
 
 ```ruby
-index.find(id)
+index.remove_metadata(id)
+```
+
+## Index Types
+
+[Vector sets](#vector-sets)
+
+- use cosine distance
+- use single-precision floats
+- support exact and approximate search
+- support quantization and dimensionality reduction
+
+[Vector indexes](#vector-indexes)
+
+- support L2, inner product, and cosine distance
+- support single or double-precision floats
+- support either exact (flat) or approximate (HNSW and SVS Vamana) search
+- can support quantization and dimensionality reduction (SVS Vamana)
+- require calling `create` before searching
+
+## Vector Sets
+
+Create a vector set
+
+```ruby
+Neighbor::Redis::VectorSet.new(name)
 ```
 
-Remove an item
+Specify parameters
 
 ```ruby
-index.remove(id)
+Neighbor::Redis::VectorSet.new(name, m: 16, ef_construction: 200, ef_search: 10)
 ```
 
-Remove multiple items
+Use quantization (`int8` or `binary`)
 
 ```ruby
-index.remove_all(ids)
+Neighbor::Redis::VectorSet.new(name, quantization: "int8")
 ```
 
-Drop the index
+Use dimensionality reduction
 
 ```ruby
-index.drop
+Neighbor::Redis::VectorSet.new(name, reduce: 2)
+```
+
+Perform exact search
+
+```ruby
+index.search(vector, exact: true)
+```
+
+## Vector Indexes
+
+Create a vector index (`l2`, `inner_product`, or `cosine` distance)
+
+```ruby
+index = Neighbor::Redis::HnswIndex.new(name, dimensions: 3, distance: "cosine")
+index.create
+```
+
+Store vectors as double precision (instead of single precision)
+
+```ruby
+Neighbor::Redis::HnswIndex.new(name, type: "float64")
+```
+
+Store vectors as JSON (instead of a hash/blob)
+
+```ruby
+Neighbor::Redis::HnswIndex.new(name, redis_type: "json")
+```
+
+### Index Options
+
+HNSW
+
+```ruby
+Neighbor::Redis::HnswIndex.new(name, m: 16, ef_construction: 200, ef_search: 10)
+```
+
+SVS Vamana - *Redis 8.2+*
+
+```ruby
+Neighbor::Redis::SvsVamanaIndex.new(
+  name,
+  compression: nil,
+  construction_window_size: 200,
+  graph_max_degree: 32,
+  search_window_size: 10,
+  training_threshold: nil,
+  reduce: nil
+)
+```
+
+Flat
+
+```ruby
+Neighbor::Redis::FlatIndex.new(name)
 ```
 
 ## Example
@@ -149,8 +241,7 @@ You can use Neighbor Redis for online item-based recommendations with [Disco](ht
 Create an index
 
 ```ruby
-index = Neighbor::Redis::HNSWIndex.new("movies", dimensions: 20, distance: "cosine")
-index.create
+index = Neighbor::Redis::VectorSet.new("movies")
 ```
 
 Fit the recommender
@@ -170,14 +261,30 @@ index.add_all(recommender.item_ids, recommender.item_factors)
 And get similar movies
 
 ```ruby
-index.nearest("Star Wars (1977)").map { |v| v[:id] }
+index.search_id("Star Wars (1977)").map { |v| v[:id] }
 ```
 
-See the [complete code](examples/disco_item_recs.rb)
+See the complete code for [vector sets](examples/disco_item_recs_vs.rb) and [vector indexes](examples/disco_item_recs.rb)
 
 ## Reference
 
-- [Vector similarity](https://redis.io/docs/stack/search/reference/vectors/)
+Get index info
+
+```ruby
+index.info
+```
+
+Check if an index exists
+
+```ruby
+index.exists?
+```
+
+Drop an index
+
+```ruby
+index.drop
+```
 
 ## History
 

data/lib/neighbor/redis/hnsw_index.rb

@@ -1,13 +1,22 @@
 module Neighbor
   module Redis
-    class HNSWIndex < Index
-      def initialize(*args, initial_cap: nil, m: nil, ef_construction: nil, ef_runtime: nil, epsilon: nil, **options)
+    class HnswIndex < Index
+      def initialize(
+        *args,
+        initial_cap: nil,
+        m: nil,
+        ef_construction: nil,
+        ef_search: nil,
+        ef_runtime: nil,
+        epsilon: nil,
+        **options
+      )
         super(*args, **options)
         @algorithm = "HNSW"
         @initial_cap = initial_cap
         @m = m
         @ef_construction = ef_construction
-        @ef_runtime = ef_runtime
+        @ef_runtime = ef_search || ef_runtime
         @epsilon = epsilon
       end
 
@@ -23,5 +32,7 @@ module Neighbor
         params
       end
     end
+
+    HNSWIndex = HnswIndex
   end
 end

data/lib/neighbor/redis/index.rb

@@ -1,20 +1,27 @@
 module Neighbor
   module Redis
     class Index
-      def initialize(name, dimensions:, distance:, type: "float32", redis_type: "hash")
+      def initialize(name, dimensions:, distance:, type: "float32", redis_type: "hash", id_type: "string")
         @index_name = index_name(name)
         @global_prefix = "neighbor:items:"
         @prefix = "#{@global_prefix}#{name}:"
 
-        @dimensions = dimensions
+        @dimensions = dimensions.to_i
 
         unless distance.nil?
           @distance_metric =
             case distance.to_s
-            when "l2", "cosine"
-              distance.to_s.upcase
+            when "l2"
+              "L2"
             when "inner_product"
               "IP"
+            when "cosine"
+              if Redis.server_type == :dragonfly
+                # uses inner product instead of cosine distance?
+                raise ArgumentError, "unsupported distance"
+              else
+                "COSINE"
+              end
             else
               raise ArgumentError, "invalid distance"
             end
@@ -40,15 +47,25 @@ module Neighbor
           else
             raise ArgumentError, "invalid redis_type"
           end
+
+        @int_ids =
+          case id_type.to_s
+          when "string"
+            false
+          when "integer"
+            true
+          else
+            raise ArgumentError, "invalid id_type"
+          end
       end
 
-      def self.create(...)
-        index = new(...)
-        index.create
+      def self.create(*args, _schema: nil, **options)
+        index = new(*args, **options)
+        index.create(_schema:)
         index
       end
 
-      def create
+      def create(_schema: nil)
         params = {
           "TYPE" => @float64 ? "FLOAT64" : "FLOAT32",
           "DIM" => @dimensions,
@@ -59,81 +76,246 @@ module Neighbor
         command.push("ON", "JSON") if @json
         command.push("PREFIX", "1", @prefix, "SCHEMA")
         command.push("$.v", "AS") if @json
-        command.push("v", "VECTOR", @algorithm, params.size * 2, params)
-        ft_command { redis.call(*command) }
+        command.push("v", "VECTOR", @algorithm, params.size * 2)
+        params.each do |k, v|
+          command.push(k, v)
+        end
+
+        (_schema || {}).each do |k, v|
+          k = k.to_s
+          # TODO improve
+          if k == "v" || !k.match?(/\A\w+\z/)
+            raise ArgumentError, "invalid schema"
+          end
+          command.push("$.#{k}", "AS") if @json
+          command.push(k, v.to_s)
+          # TODO figure out how to handle separator for hashes
+          # command.push("SEPARATOR", "") if !@json
+        end
+
+        run_command(*command)
+        nil
+      rescue => e
+        raise Error, "RediSearch not installed" if e.message.include?("ERR unknown command 'FT.")
+        raise e
       end
 
       def exists?
-        redis.call("FT.INFO", @index_name)
+        run_command("FT.INFO", @index_name)
         true
       rescue ArgumentError
         # fix for invalid value for Float(): "-nan"
         true
       rescue => e
-        raise unless e.message.downcase.include?("unknown index name")
+        message = e.message.downcase
+        raise e unless message.include?("unknown index name") || message.include?("no such index") || message.include?("not found")
         false
       end
 
-      def add(id, embedding)
-        add_all([id], [embedding])
+      def info
+        info = run_command("FT.INFO", @index_name)
+        if info.is_a?(Hash)
+          info
+        else
+          # for RESP2
+          info = hash_result(info)
+          ["index_definition", "gc_stats" ,"cursor_stats", "dialect_stats", "Index Errors"].each do |k|
+            info[k] = hash_result(info[k]) if info[k]
+          end
+          ["attributes", "field statistics"].each do |k|
+            info[k]&.map! { |v| hash_result(v) }
+          end
+          info["field statistics"]&.each do |v|
+            v["Index Errors"] = hash_result(v["Index Errors"]) if v["Index Errors"]
+          end
+          info
+        end
+      end
+
+      def count
+        info.fetch("num_docs").to_i
       end
 
-      def add_all(ids, embeddings)
-        ids = ids.to_a
-        embeddings = embeddings.to_a
+      def add(id, vector, metadata: nil)
+        add_all([id], [vector], metadata: metadata ? [metadata] : nil)[0]
+      end
 
-        raise ArgumentError, "different sizes" if ids.size != embeddings.size
+      def add_all(ids, vectors, metadata: nil)
+        # perform checks first to reduce chance of non-atomic updates
+        ids = ids.to_a.map { |v| item_id(v) }
+        vectors = vectors.to_a
+        metadata = metadata.to_a if metadata
 
-        embeddings.each { |e| check_dimensions(e) }
+        raise ArgumentError, "different sizes" if ids.size != vectors.size
 
-        redis.pipelined do |pipeline|
-          ids.zip(embeddings).each do |id, embedding|
-            if @json
-              pipeline.call("JSON.SET", item_key(id), "$", JSON.generate({v: embedding}))
-            else
-              pipeline.call("HSET", item_key(id), {v: to_binary(embedding)})
-            end
+        vectors.each { |e| check_dimensions(e) }
+
+        if metadata
+          raise ArgumentError, "different sizes" if metadata.size != ids.size
+
+          metadata = metadata.map { |v| v&.transform_keys(&:to_s) }
+          if metadata.any? { |v| v&.key?("v") }
+            # TODO improve
+            raise ArgumentError, "invalid metadata"
           end
         end
+
+        result =
+          client.pipelined do |pipeline|
+            ids.zip(vectors).each_with_index do |(id, vector), i|
+              attributes = metadata && metadata[i] || {}
+              if @json
+                pipeline.call("JSON.SET", item_key(id), "$", JSON.generate(attributes.merge({"v" => vector})))
+              else
+                pipeline.call("HSET", item_key(id), attributes.merge({"v" => to_binary(vector)}))
+              end
+            end
+          end
+        result.map { |v| v.is_a?(String) ? v == "OK" : v > 0 }
       end
 
-      def remove(id)
-        remove_all([id])
+      def member?(id)
+        key = item_key(id)
+
+        run_command("EXISTS", key) == 1
       end
+      alias_method :include?, :member?
 
-      def remove_all(ids)
-        redis.call("DEL", ids.map { |id| item_key(id) })
+      def remove(id)
+        remove_all([id]) == 1
       end
 
-      def search(embedding, count: 5)
-        check_dimensions(embedding)
+      def remove_all(ids)
+        keys = ids.to_a.map { |id| item_key(id) }
 
-        search_by_blob(to_binary(embedding), count)
+        run_command("DEL", *keys).to_i
       end
 
       def find(id)
+        key = item_key(id)
+
         if @json
-          s = redis.call("JSON.GET", item_key(id), "$.v")
+          s = run_command("JSON.GET", key, "$.v")
           JSON.parse(s)[0] if s
         else
-          from_binary(redis.call("HGET", item_key(id), "v"))
+          s = run_command("HGET", key, "v")
+          from_binary(s) if s
+        end
+      end
+
+      def metadata(id)
+        key = item_key(id)
+
+        if @json
+          v = run_command("JSON.GET", key)
+          JSON.parse(v).except("v") if v
+        else
+          v = hash_result(run_command("HGETALL", key))
+          v.except("v") if v.any?
+        end
+      end
+
+      def set_metadata(id, metadata)
+        key = item_key(id)
+
+        # TODO DRY with add_all
+        metadata = metadata.transform_keys(&:to_s)
+        raise ArgumentError, "invalid metadata" if metadata.key?("v")
+
+        if @json
+          # TODO use WATCH
+          keys = run_command("JSON.OBJKEYS", key)
+          return false unless keys
+
+          keys.each do |k|
+            next if k == "v"
+
+            # safe to modify in-place
+            metadata[k] = nil unless metadata.key?(k)
+          end
+
+          run_command("JSON.MERGE", key, "$", JSON.generate(metadata)) == "OK"
+        else
+          # TODO use WATCH
+          fields = run_command("HKEYS", key)
+          return false if fields.empty?
+
+          fields.delete("v")
+          if fields.any?
+            # TODO use MULTI
+            run_command("HDEL", key, *fields)
+          end
+
+          if metadata.any?
+            args = []
+            metadata.each do |k, v|
+              args.push(k, v)
+            end
+            run_command("HSET", key, *args) > 0
+          else
+            true
+          end
+        end
+      end
+
+      def remove_metadata(id)
+        key = item_key(id)
+
+        if @json
+          # TODO use WATCH
+          keys = run_command("JSON.OBJKEYS", key)
+          return false unless keys
+
+          keys.delete("v")
+          if keys.any?
+            # merge with null deletes key
+            run_command("JSON.MERGE", key, "$", JSON.generate(keys.to_h { |k| [k, nil] })) == "OK"
+          else
+            true
+          end
+        else
+          # TODO use WATCH
+          fields = run_command("HKEYS", key)
+          return false if fields.empty?
+
+          fields.delete("v")
+          if fields.any?
+            run_command("HDEL", key, *fields) > 0
+          else
+            true
+          end
         end
       end
 
-      def nearest(id, count: 5)
-        embedding =
+      def search(vector, count: 5, with_metadata: false, _filter: nil)
+        check_dimensions(vector)
+
+        search_command(to_binary(vector), count, with_metadata:, _filter:)
+      end
+
+      def search_id(id, count: 5, with_metadata: false, _filter: nil)
+        id = item_id(id)
+        key = item_key(id)
+
+        vector =
           if @json
-            s = redis.call("JSON.GET", item_key(id), "$.v")
+            s = run_command("JSON.GET", key, "$.v")
             to_binary(JSON.parse(s)[0]) if s
           else
-            redis.call("HGET", item_key(id), "v")
+            run_command("HGET", key, "v")
           end
 
-        unless embedding
+        unless vector
           raise Error, "Could not find item #{id}"
         end
 
-        search_by_blob(embedding, count + 1).reject { |v| v[:id] == id.to_s }.first(count)
+        search_command(vector, count + 1, with_metadata:, _filter:).reject { |v| v[:id] == id }.first(count)
+      end
+      alias_method :nearest, :search_id
+
+      def promote(alias_name)
+        run_command("FT.ALIASUPDATE", index_name(alias_name), @index_name)
+        nil
       end
 
       def drop
@@ -141,63 +323,81 @@ module Neighbor
         drop_keys
       end
 
-      def promote(alias_name)
-        redis.call("FT.ALIASUPDATE", index_name(alias_name), @index_name)
-      end
-
       private
 
       def index_name(name)
         if name.include?(":")
-          raise ArgumentError, "Invalid name"
+          raise ArgumentError, "invalid name"
         end
 
         "neighbor-idx-#{name}"
       end
 
-      def check_dimensions(embedding)
-        if embedding.size != @dimensions
+      def check_dimensions(vector)
+        if vector.size != @dimensions
           raise ArgumentError, "expected #{@dimensions} dimensions"
         end
       end
 
       def item_key(id)
-        "#{@prefix}#{id}"
+        "#{@prefix}#{item_id(id)}"
+      end
+
+      def item_id(id)
+        @int_ids ? Integer(id) : id.to_s
       end
 
-      def search_by_blob(blob, count)
-        resp = redis.call("FT.SEARCH", @index_name, "*=>[KNN #{count.to_i} @v $BLOB]", "PARAMS", "2", "BLOB", blob, "SORTBY", "__v_score", "DIALECT", "2")
-        resp.is_a?(Hash) ? parse_results_hash(resp) : parse_results_array(resp)
+      def search_command(blob, count, with_metadata:, _filter:)
+        filter = _filter ? "(#{_filter})" : "*"
+        return_args = with_metadata ? [] : ["RETURN", 1, "__v_score"]
+        resp = run_command("FT.SEARCH", @index_name, "#{filter}=>[KNN #{count.to_i} @v $BLOB AS __v_score]", "PARAMS", "2", "BLOB", blob, *search_sort_args, *return_args, "DIALECT", "2")
+        if resp.is_a?(Hash)
+          parse_results_hash(resp, with_metadata:)
+        else
+          parse_results_array(resp, with_metadata:)
+        end
       end
 
-      def parse_results_hash(resp)
+      def search_sort_args
+        @search_sort_args ||= Redis.server_type == :valkey ? [] : ["SORTBY", "__v_score"]
+      end
+
+      def parse_results_hash(resp, with_metadata:)
         prefix_length = nil
         resp["results"].map do |result|
           key = result["id"]
           info = result["extra_attributes"]
           prefix_length ||= find_prefix_length(key)
-          search_result(key, info, prefix_length)
+          search_result(key, info, prefix_length, with_metadata:)
         end
       end
 
-      def parse_results_array(resp)
+      def parse_results_array(resp, with_metadata:)
         prefix_length = nil
         resp.shift.times.map do |i|
           key, info = resp.shift(2)
-          info = info.each_slice(2).to_h
+          info = info.each_slice(2).to_h unless info.is_a?(Hash)
           prefix_length ||= find_prefix_length(key)
-          search_result(key, info, prefix_length)
+          search_result(key, info, prefix_length, with_metadata:)
         end
       end
 
-      def search_result(key, info, prefix_length)
+      def search_result(key, info, prefix_length, with_metadata:)
         score = info["__v_score"].to_f
         distance = calculate_distance(score)
 
-        {
-          id: key[prefix_length..-1],
+        result = {
+          id: item_id(key[prefix_length..-1]),
           distance: distance
         }
+        if with_metadata
+          if @json
+            result[:metadata] = JSON.parse(info["$"]).except("v")
+          else
+            result[:metadata] = info.except("v", "__v_score")
+          end
+        end
+        result
       end
 
       def calculate_distance(score)
@@ -217,19 +417,19 @@ module Neighbor
       end
 
       def drop_index
-        redis.call("FT.DROPINDEX", @index_name)
+        run_command("FT.DROPINDEX", @index_name)
       end
 
       def drop_keys
         cursor = 0
         begin
-          cursor, keys = redis.call("SCAN", cursor, "MATCH", "#{@prefix}*", "COUNT", 100)
-          redis.call("DEL", keys) if keys.any?
+          cursor, keys = run_command("SCAN", cursor, "MATCH", "#{@prefix}*", "COUNT", 100)
+          run_command("DEL", *keys) if keys.any?
         end while cursor != "0"
       end
 
-      def to_binary(embedding)
-        embedding.to_a.pack(pack_format)
+      def to_binary(vector)
+        vector.to_a.pack(pack_format)
       end
 
       def from_binary(s)
@@ -240,15 +440,18 @@ module Neighbor
         @pack_format ||= @float64 ? "d#{@dimensions}" : "f#{@dimensions}"
       end
 
-      # just use for create for now
-      def ft_command
-        yield
-      rescue => e
-        raise Error, "RediSearch not installed" if e.message.include?("ERR unknown command 'FT.")
-        raise
+      def hash_result(result)
+        result.is_a?(Array) ? result.each_slice(2).to_h : result
+      end
+
+      def run_command(*args)
+        if args.any? { |v| !(v.is_a?(String) || v.is_a?(Numeric)) }
+          raise TypeError, "unexpected argument type"
+        end
+        client.call(*args)
       end
 
-      def redis
+      def client
         Redis.client
       end
     end

data/lib/neighbor/redis/svs_vamana_index.rb

@@ -0,0 +1,41 @@
+module Neighbor
+  module Redis
+    class SvsVamanaIndex < Index
+      def initialize(
+        *args,
+        compression: nil,
+        construction_window_size: nil,
+        graph_max_degree: nil,
+        search_window_size: nil,
+        epsilon: nil,
+        training_threshold: nil,
+        reduce: nil,
+        **options
+      )
+        super(*args, **options)
+        @algorithm = "SVS-VAMANA"
+        @compression = compression
+        @construction_window_size = construction_window_size
+        @graph_max_degree = graph_max_degree
+        @search_window_size = search_window_size
+        @epsilon = epsilon
+        @training_threshold = training_threshold
+        @reduce = reduce
+      end
+
+      private
+
+      def create_params
+        params = {}
+        params["COMPRESSION"] = @compression if @compression
+        params["CONSTRUCTION_WINDOW_SIZE"] = @construction_window_size if @construction_window_size
+        params["GRAPH_MAX_DEGREE"] = @graph_max_degree if @graph_max_degree
+        params["SEARCH_WINDOW_SIZE"] = @search_window_size if @search_window_size
+        params["EPSILON"] = @epsilon if @epsilon
+        params["TRAINING_THRESHOLD"] = @training_threshold if @training_threshold
+        params["REDUCE"] = @reduce if @reduce
+        params
+      end
+    end
+  end
+end

data/lib/neighbor/redis/vector_set.rb

@@ -0,0 +1,265 @@
+module Neighbor
+  module Redis
+    class VectorSet
+      NO_DEFAULT = Object.new
+
+      def initialize(
+        name,
+        m: nil,
+        ef_construction: nil,
+        ef_search: nil,
+        epsilon: nil,
+        quantization: nil,
+        reduce: nil,
+        id_type: "string"
+      )
+        name = name.to_str
+        if name.include?(":")
+          raise ArgumentError, "invalid name"
+        end
+
+        @name = name
+        @m = m&.to_i
+        @ef_construction = ef_construction&.to_i
+        @ef_search = ef_search&.to_i
+        @epsilon = epsilon&.to_f
+
+        @quant_type =
+          case quantization&.to_s
+          when nil
+            "NOQUANT"
+          when "binary"
+            "BIN"
+          when "int8"
+            "Q8"
+          else
+            raise ArgumentError, "invalid quantization"
+          end
+
+        case id_type.to_s
+        when "string", "integer"
+          @int_ids = id_type == "integer"
+        else
+          raise ArgumentError, "invalid id_type"
+        end
+
+        @reduce_args = []
+        @reduce_args.push("REDUCE", reduce.to_i) if reduce
+
+        @add_args = []
+        @add_args.push("M", @m) if @m
+        @add_args.push("EF", @ef_construction) if @ef_construction
+      end
+
+      def exists?
+        !run_command("VINFO", key).nil?
+      end
+
+      def info
+        hash_result(run_command("VINFO", key))
+      end
+
+      def dimensions
+        run_command("VDIM", key)
+      rescue => e
+        raise e unless e.message.include?("key does not exist")
+        nil
+      end
+
+      def count
+        run_command("VCARD", key)
+      end
+
+      def add(id, vector, metadata: nil)
+        add_all([id], [vector], metadata: metadata ? [metadata] : nil)[0]
+      end
+
+      def add_all(ids, vectors, metadata: nil)
+        # perform checks first to reduce chance of non-atomic updates
+        ids = ids.to_a.map { |v| item_id(v) }
+        vectors = vectors.to_a
+        metadata = metadata.to_a if metadata
+
+        raise ArgumentError, "different sizes" if ids.size != vectors.size
+
+        if vectors.size > 1
+          dimensions = vectors.first.size
+          unless vectors.all? { |v| v.size == dimensions }
+            raise ArgumentError, "different dimensions"
+          end
+        end
+
+        if metadata
+          raise ArgumentError, "different sizes" if metadata.size != ids.size
+        end
+
+        result =
+          client.pipelined do |pipeline|
+            ids.zip(vectors).each_with_index do |(id, vector), i|
+              attributes = metadata[i] if metadata
+              attribute_args = []
+              attribute_args.push("SETATTR", JSON.generate(attributes)) if attributes
+              pipeline.call("VADD", key, *@reduce_args, "FP32", to_binary(vector), id, @quant_type, *attribute_args, *@add_args)
+            end
+          end
+        result.map { |v| bool_result(v) }
+      end
+
+      def member?(id)
+        id = item_id(id)
+
+        bool_result(run_command("VISMEMBER", key, id))
+      end
+      alias_method :include?, :member?
+
+      def remove(id)
+        id = item_id(id)
+
+        bool_result(run_command("VREM", key, id))
+      end
+
+      def remove_all(ids)
+        # perform checks first to reduce chance of non-atomic updates
+        ids = ids.to_a.map { |v| item_id(v) }
+
+        result =
+          client.pipelined do |pipeline|
+            ids.each do |id|
+              pipeline.call("VREM", key, id)
+            end
+          end
+        result.map { |v| bool_result(v) }
+      end
+
+      def find(id)
+        id = item_id(id)
+
+        run_command("VEMB", key, id)&.map(&:to_f)
+      end
+
+      def metadata(id)
+        id = item_id(id)
+
+        a = run_command("VGETATTR", key, id)
+        a ? JSON.parse(a) : nil
+      end
+
+      def set_metadata(id, metadata)
+        id = item_id(id)
+
+        bool_result(run_command("VSETATTR", key, id, JSON.generate(metadata)))
+      end
+
+      def remove_metadata(id)
+        id = item_id(id)
+
+        bool_result(run_command("VSETATTR", key, id, ""))
+      end
+
+      def search(vector, count: 5, with_metadata: false, ef_search: nil, exact: false, _filter: nil, _ef_filter: nil)
+        count = count.to_i
+
+        search_command(["FP32", to_binary(vector)], count:, with_metadata:, ef_search:, exact:, _filter:, _ef_filter:).map do |k, v|
+          search_result(k, v, with_metadata:)
+        end
+      end
+
+      def search_id(id, count: 5, with_metadata: false, ef_search: nil, exact: false, _filter: nil, _ef_filter: nil)
+        id = item_id(id)
+        count = count.to_i
+
+        result =
+          search_command(["ELE", id], count: count + 1, with_metadata:, ef_search:, exact:, _filter:, _ef_filter:).filter_map do |k, v|
+            if k != id.to_s
+              search_result(k, v, with_metadata:)
+            end
+          end
+        result.first(count)
+      end
+      alias_method :nearest, :search_id
+
+      def links(id)
+        id = item_id(id)
+
+        run_command("VLINKS", key, id, "WITHSCORES")&.map do |links|
+          hash_result(links).map do |k, v|
+            search_result(k, v)
+          end
+        end
+      end
+
+      def sample(n = NO_DEFAULT)
+        count = n == NO_DEFAULT ? 1 : n.to_i
+
+        result = run_command("VRANDMEMBER", key, count).map { |v| item_id(v) }
+        n == NO_DEFAULT ? result.first : result
+      end
+
+      def drop
+        bool_result(run_command("DEL", key))
+      end
+
+      private
+
+      def key
+        "neighbor:vs:#{@name}"
+      end
+
+      def item_id(id)
+        @int_ids ? Integer(id) : id.to_s
+      end
+
+      def to_binary(vector)
+        vector.pack("e*")
+      end
+
+      def search_command(args, count:, with_metadata:, ef_search:, exact:, _filter:, _ef_filter:)
+        ef_search = @ef_search if ef_search.nil?
+
+        args << "WITHATTRIBS" if with_metadata
+        args.push("EF", ef_search) if ef_search
+        args.push("EPSILON", @epsilon) if @epsilon
+        args.push("FILTER", _filter) if _filter
+        args.push("FILTER_EF", _ef_filter) if _ef_filter
+        args << "TRUTH" if exact
+
+        result = run_command("VSIM", key, *args, "WITHSCORES", "COUNT", count)
+        if result.is_a?(Array)
+          if with_metadata
+            result.each_slice(3).to_h { |v| [v[0], v[1..]] }
+          else
+            hash_result(result)
+          end
+        else
+          result
+        end
+      end
+
+      def search_result(k, v, with_metadata: false)
+        v, a = v if with_metadata
+        value = {id: item_id(k), distance: 2 * (1 - v.to_f)}
+        value.merge!(metadata: a ? JSON.parse(a) : {}) if with_metadata
+        value
+      end
+
+      def hash_result(result)
+        result.is_a?(Array) ? result.each_slice(2).to_h : result
+      end
+
+      def bool_result(result)
+        result == true || result == 1
+      end
+
+      def run_command(*args)
+        if args.any? { |v| !(v.is_a?(String) || v.is_a?(Numeric)) }
+          raise TypeError, "unexpected argument type"
+        end
+        client.call(*args)
+      end
+
+      def client
+        Redis.client
+      end
+    end
+  end
+end

data/lib/neighbor/redis/version.rb

@@ -1,5 +1,5 @@
 module Neighbor
   module Redis
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/neighbor/redis.rb

@@ -1,10 +1,15 @@
 # dependencies
 require "redis-client"
 
+# stdlib
+require "json"
+
 # modules
 require_relative "redis/index"
 require_relative "redis/flat_index"
 require_relative "redis/hnsw_index"
+require_relative "redis/svs_vamana_index"
+require_relative "redis/vector_set"
 require_relative "redis/version"
 
 module Neighbor
@@ -13,6 +18,21 @@ module Neighbor
 
     class << self
       attr_accessor :client
+
+      def server_type
+        unless defined?(@server_type)
+          info = client.call("INFO")
+          @server_type =
+            if info.include?("valkey_version")
+              :valkey
+            elsif info.include?("dragonfly_version")
+              :dragonfly
+            else
+              :redis
+            end
+        end
+        @server_type
+      end
     end
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: neighbor-redis
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-23 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -24,7 +23,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email: andrew@ankane.org
 executables: []
 extensions: []
@@ -38,12 +36,13 @@ files:
 - lib/neighbor/redis/flat_index.rb
 - lib/neighbor/redis/hnsw_index.rb
 - lib/neighbor/redis/index.rb
+- lib/neighbor/redis/svs_vamana_index.rb
+- lib/neighbor/redis/vector_set.rb
 - lib/neighbor/redis/version.rb
 homepage: https://github.com/ankane/neighbor-redis
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -51,15 +50,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Nearest neighbor search for Ruby and Redis
 test_files: []