anngler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 02dc26823e95142aba1c8f1468ee440e5850d17c940df0488227ceec9e5d9568
+  data.tar.gz: 0214635227152047fc51f98a16d753e38133e3bd00a1ef50bb6a86a9fc3e4fc4
+SHA512:
+  metadata.gz: fdabb190537d475693d21619d2d5ee700b1cb13c27f3c526df0e880023f021d95cf4c5263248fd8d0b68346ca0c04f707faf2f836c3ad7336e3a91f747c7efee
+  data.tar.gz: 91cd7f7c122227c3764d3293385125eb81521181e513095eeed4f3cafa8fb97bea20dc60edc7d8c5d0441375b1efcf22bfacb69ac9495a25c50162f42747130a

data/lib/anngler.rb

@@ -0,0 +1,4 @@
+require_relative 'anngler/index.rb'
+require_relative 'anngler/helpers.rb'
+require_relative 'anngler/storage/memory_backend.rb'
+require_relative 'anngler/storage/redis_backend.rb'

data/lib/anngler/helpers.rb

@@ -0,0 +1,16 @@
+module Anngler
+    module Helpers
+        class << self
+
+            def magnitude(vec)
+                Math.sqrt(vec.square.to_a.reduce(:+))
+            end
+
+            def cosine_distance(a, b)
+                1 - a.dot(b) / (magnitude(a) * magnitude(b))
+            end
+
+        end 
+
+    end
+end

data/lib/anngler/index.rb

@@ -0,0 +1,128 @@
+require 'json'
+require 'base64'
+require 'zlib'
+
+module Anngler
+    class Index
+        #the number of features of the vectors we are storing
+        attr_reader :n_features
+
+        #the number of projections to generate (more = less vectors per bucket)
+        attr_reader :n_projections
+
+        #seed for our random number generator, we ensure this is deterministic buy resorting to the base16 of the bucket name if no seed is provided
+        attr_reader :seed
+
+        #the name of the bucket to allow multiple different hash tables in redis
+        attr_reader :bucket_name
+
+        #the random number generator for the projection matrices
+        attr_reader :rng
+
+        #an n_trees x n_features x n_projections matrix to store our projections
+        attr_reader :trees
+
+        #how many different projections to overlap (more allows for better accuracy but will slow performance)
+        attr_reader :n_trees
+
+        #which storage service to use (either redis or local memory)
+        attr_reader :storage
+
+        def initialize(
+                bucket_name,
+                n_projections,
+                n_features,
+                seed: nil,
+                n_trees: 1,
+                storage: Anngler::Storage::MemoryBackend.new
+            )
+            @n_projections = n_projections
+            @n_features = n_features
+            @seed = seed
+            @seed ||= bucket_name.to_i(36)
+            @bucket_name = bucket_name
+            @rng = Random.new(@seed)
+            @n_trees = n_trees
+            @storage = storage
+
+            gen_trees
+        end
+
+        def add(vec, label: "")
+            hashes = calc_hashes(vec)
+            #Serialize the vector and the label
+            serialized_data = pack_data(vec, label)
+
+            #add the vector into each tree
+            hashes.each_with_index do |hash, i|
+                bucket = "#{@bucket_name}:#{i}:#{hash2string(hash)}"
+                @storage.add_vector(bucket, serialized_data)
+            end
+        end
+
+        def remove(vec)
+            hashes = calc_hashes(vec)
+
+            #remove vector from each tree
+            hashes.each_with_index do |hash, i|
+                bucket = "#{@bucket_name}:#{i}:#{hash2string(hash)}"
+                @storage.remove_vector(bucket, encode_vec(vec))
+            end
+        end
+
+        def query(vec)
+            hashes = calc_hashes(vec)
+            raw_results = []
+
+            #search each tree and append the results into raw_results
+            hashes.each_with_index do |hash, i|
+                bucket = "#{@bucket_name}:#{i}:#{hash2string(hash)}"
+                raw_results += @storage.query_bucket(bucket)
+            end
+
+            #remove duplicates and decode the data
+            raw_results.uniq.map do |encoded_data|
+                unpack_data(encoded_data)
+            end.sort_by do |data|
+                #sort the results by cosine distance
+                Helpers.cosine_distance(vec, data["vec"])
+            end
+        end
+
+        private
+
+        def gen_trees
+            vals = Array.new(@n_trees * @n_features * @n_projections) { (@rng.rand * 2) -1 }
+            @trees = Numo::DFloat.asarray(vals).reshape(@n_trees, @n_features, @n_projections)
+        end
+
+        #return a hash key for each tree
+        def calc_hashes(vec)
+            [0..@n_trees - 1].map do |i|
+                vec.dot(@trees[i, true, true]).ge(0.0)
+            end
+        end
+
+        #turn the vector into a hexadecimal string
+        def hash2string(hash)
+            hash.to_a.join.to_i(2).to_s(16)
+        end
+
+        def encode_vec(vec)
+            Base64.encode64(Zlib::Deflate.deflate(vec.to_a.join(",")))
+        end
+
+        def decode_vec(vec)
+            Numo::DFloat.asarray(Zlib::Inflate.inflate(Base64.decode64(vec)).split(",").map(&:to_f))
+        end
+
+        def pack_data(vec, label)
+            "#{encode_vec(vec)}:#{label}"
+        end
+
+        def unpack_data(encoded_data)
+            encoded_vector, label = encoded_data.split(":")
+            { "label" => label, "vec" => decode_vec(encoded_vector) }
+        end
+    end
+end

data/lib/anngler/storage/memory_backend.rb

@@ -0,0 +1,23 @@
+module Anngler
+    module Storage
+        class MemoryBackend
+            def initialize
+                @storage = Hash.new( [] )
+            end
+
+            def add_vector(bucket, data)
+                @storage[bucket] = [data] + @storage[bucket]
+            end
+
+            def remove_vector(bucket, encoded_vec)
+                @storage[bucket] = @storage[bucket].reject do |encoded_str|
+                    encoded_str.split(":")[0] == encoded_vec
+                end
+            end
+
+            def query_bucket(bucket)
+                @storage[bucket]
+            end
+        end
+    end
+end

data/lib/anngler/storage/redis_backend.rb

@@ -0,0 +1,30 @@
+require 'redis'
+
+module Anngler
+    module Storage
+        class RedisBackend
+
+            def initialize(instance)
+                @instance = instance
+            end
+
+            def add_vector(bucket, data)
+                @instance.lpush(bucket, data)
+            end
+
+            def remove_vector(bucket, encoded_vec)
+                @instance.lrange(bucket, 0, -1).each do |val|
+                    if(val.split(":")[0] == encoded_vec)
+                        @instance.lrem(bucket, 0, val)
+                        return
+                    end
+                end
+            end
+
+            def query_bucket(bucket)
+                @instance.lrange(bucket, 0, -1)
+            end
+            
+        end
+    end
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: anngler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Aiden Leeming
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/anngler.rb
+- lib/anngler/helpers.rb
+- lib/anngler/index.rb
+- lib/anngler/storage/memory_backend.rb
+- lib/anngler/storage/redis_backend.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: A ruby locality sensitive hashing implementation using Redis for storage
+test_files: []