lsh 0.4.2-java → 0.5.0-java

data/lib/lsh/index.rb

@@ -39,16 +39,18 @@ module LSH
     end
 
     def add(vector, id = nil)
-      storage.add_vector_id(vector, id) if id
+      vector_hash = vector.hash
+      storage.add_vector(vector, vector_hash)
+      storage.add_vector_id(vector_hash, id) if id
       hashes(vector).each_with_index do |hash, i|
         hash_i = array_to_hash(hash)
         bucket = storage.find_bucket(i)
-        storage.add_vector_to_bucket(bucket, hash_i, vector)
+        storage.add_vector_hash_to_bucket(bucket, hash_i, vector_hash)
       end
     end
 
-    def vector_to_id(vector)
-      storage.vector_to_id(vector)
+    def vector_hash_to_id(vector_hash)
+      storage.vector_hash_to_id(vector_hash)
     end
 
     def id_to_vector(id)
@@ -63,14 +65,15 @@ module LSH
       # Take query hashes, move them around at radius r, and use them to do another query
       # TODO: only works for binary LSH atm
       if multiprobe_radius > 0
+        raise Exception.new("Non-zero multiprobe radius only implemented for binary LSH") unless hashes_are_binary?
         mp_arrays = multiprobe_hashes_arrays(hash_arrays, multiprobe_radius)
         mp_arrays.each do |probes_arrays|
           probes_hashes = probes_arrays.map { |a| array_to_hash(a) }
           results += storage.query_buckets(probes_hashes)
         end
+        results.uniq! { |result| result[:hash] }
       end
-      results = MathUtil.uniq(results)
-      order_vectors_by_similarity(vector, results)
+      order_results_by_similarity(vector, results)
     end
 
     def query_ids(id, multiprobe_radius = 0)
@@ -79,10 +82,8 @@ module LSH
     end
 
     def query_ids_by_vector(vector, multiprobe_radius = 0)
-      vectors = query(vector, multiprobe_radius)
-      results = []
-      vectors.each { |v| results << vector_to_id(v) }
-      results
+      results = query(vector, multiprobe_radius)
+      results.map { |result| vector_hash_to_id(result[:hash]) }
     end
 
     def multiprobe_hashes_arrays(hash_arrays, multiprobe_radius)
@@ -97,8 +98,11 @@ module LSH
       mp_arrays
     end
 
-    def order_vectors_by_similarity(vector, vectors)
-      vectors.map { |v| [ v, similarity(vector, v) ] } .sort_by { |v, sim| sim } .reverse .map { |vs| vs[0] }
+    def order_results_by_similarity(vector, results)
+      # Faster than vectors.sort - we precompute all similarities to vector
+      # and order using those
+      similarities = results.map { |result| [ result[:hash], result[:id], result[:data], similarity(vector, result[:data]) ] }
+      similarities.sort_by { |hash, id, vector, sim| sim } .reverse .map { |vs| { :hash => vs[0], :id => vs[1], :data => vs[2] } }
     end
 
     def hashes(vector)
@@ -108,7 +112,7 @@ module LSH
       end
       hashes
     end
-
+ 
     def hash(vector, projection, bias = true)
       hash = []
       dot_products = (projection * vector.transpose).column(0).to_a
@@ -128,6 +132,10 @@ module LSH
       hash
     end
 
+    def hashes_are_binary?
+      storage.parameters[:window] == Float::INFINITY
+    end
+
     def random_vector(dim)
       MathUtil.random_gaussian_matrix(1, dim)
     end
@@ -150,7 +158,7 @@ module LSH
       #value
     end
 
-    def generate_projections(dim, k, l)
+     def generate_projections(dim, k, l)
       projections = []
       l.times do |i|
         projections << generate_projection(dim, k)
@@ -160,7 +168,7 @@ module LSH
 
     def generate_projection(dim, k)
       MathUtil.random_gaussian_matrix(k, dim)
-    end
+     end
 
     def similarity(v1, v2)
       MathUtil.dot(v1, v2)

data/lib/lsh/math_util_gsl.rb

@@ -51,14 +51,6 @@ module LSH
       v.norm
     end
 
-    def self.uniq(vs)
-      # Can't use uniq as
-      # [ v, JSON.parse(v.to_json, :create_additions => true) ].uniq.size == 2 with GSL
-      results = []
-      vs.each { |v| results << v unless results.member? v }
-      results
-    end
-
   end
 
 end

data/lib/lsh/math_util_jblas.rb

@@ -44,10 +44,6 @@ module LSH
       v.norm2
     end
 
-    def self.uniq(vs)
-      vs.uniq
-    end
-
   end
 
 end

data/lib/lsh/storage/memory.rb

@@ -30,6 +30,9 @@ module LSH
 
       def reset!
         @buckets = nil
+        @vectors = nil
+        @vector_hash_to_id = nil
+        @id_to_vector = nil
       end
 
       def create_new_bucket
@@ -37,27 +40,32 @@ module LSH
         @buckets << {}
       end
 
-      def add_vector_to_bucket(bucket, hash, vector)
+      def add_vector(vector, vector_hash)
+        @vectors ||= {}
+        @vectors[vector_hash] = vector
+      end
+
+      def add_vector_hash_to_bucket(bucket, hash, vector_hash)
         if bucket.has_key? hash
-          bucket[hash] << vector
+          bucket[hash] << vector_hash
         else
-          bucket[hash] = [vector]
+          bucket[hash] = [vector_hash]
         end
       end
 
-      def add_vector_id(vector, id)
-        @vector_to_id ||= {}
-        @vector_to_id[vector.hash] = id
+      def add_vector_id(vector_hash, id)
+        @vector_hash_to_id ||= {}
+        @vector_hash_to_id[vector_hash] = id
         @id_to_vector ||= {}
-        @id_to_vector[id] = vector
+        @id_to_vector[id] = vector_hash
       end
 
-      def vector_to_id(vector)
-        @vector_to_id[vector.hash] if @vector_to_id
+      def vector_hash_to_id(vector_hash)
+        @vector_hash_to_id[vector_hash] if @vector_hash_to_id
       end
 
       def id_to_vector(id)
-        @id_to_vector[id] if @id_to_vector
+        @vectors[@id_to_vector[id]] if @id_to_vector
       end
 
       def find_bucket(i)
@@ -65,13 +73,22 @@ module LSH
       end
 
       def query_buckets(hashes)
-        results = []
+        results_hashes = {}
         hashes.each_with_index do |hash, i|
-          bucket = find_bucket(i)
-          in_bucket = bucket[hash]
-          results += in_bucket if in_bucket
+          vectors_hashes_in_bucket = @buckets[i][hash]
+          if vectors_hashes_in_bucket
+            vectors_hashes_in_bucket.each do |vector_hash|
+              results_hashes[vector_hash] = true
+            end
+          end
+        end
+        results_hashes.keys.map do |vector_hash|
+          { 
+            :data => @vectors[vector_hash], 
+            :hash => vector_hash, 
+            :id => vector_hash_to_id(vector_hash)
+          }
         end
-        results
       end
 
     end

data/lib/lsh/storage/redis_backend.rb

@@ -28,10 +28,8 @@ module LSH
       def initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' })
         @redis = Redis.new(params[:redis])
         @data_dir = params[:data_dir]
-        unless File.exists?(@data_dir)
-          Dir.mkdir(@data_dir)
-          Dir.mkdir(File.join(@data_dir, 'projections'))
-        end
+        Dir.mkdir(@data_dir) unless File.exists?(@data_dir)
+        Dir.mkdir(File.join(@data_dir, 'projections')) unless File.exists?(File.join(@data_dir, 'projections'))
       end
 
       def reset!
@@ -42,6 +40,10 @@ module LSH
       def clear_data!
         keys = @redis.keys("lsh:bucket:*")
         @redis.del(keys) unless keys.empty?
+        keys = @redis.keys("lsh:vector_to_id:*")
+        @redis.del(keys) unless keys.empty?
+        keys = @redis.keys("lsh:id_to_vector:*")
+        @redis.del(keys) unless keys.empty?
         delete_dat_files_in_dir(@data_dir)
       end
 
@@ -107,8 +109,8 @@ module LSH
         @redis.incr "lsh:buckets"
       end
 
-      def save_vector(vector)
-        path = File.join(@data_dir, vector.hash.to_s+'.dat')
+      def save_vector(vector, vector_hash)
+        path = File.join(@data_dir, vector_hash.to_s+'.dat')
         vector.save(path) unless File.exists?(path)
       end
 
@@ -118,19 +120,21 @@ module LSH
         vector
       end
 
-      def add_vector_to_bucket(bucket, hash, vector)
-        save_vector(vector) # Writing vector to disk if not already there
-        @redis.sadd "#{bucket}:#{hash}", vector.hash.to_s # Only storing vector's hash in Redis
+      def add_vector(vector, vector_hash)
+        save_vector(vector, vector_hash) # Writing vector to disk if not already there
       end
 
-      def add_vector_id(vector, id)
-        save_vector(vector) # Writing vector to disk if not already there
-        @redis.set "lsh:vector_to_id:#{vector.hash}", id
-        @redis.set "lsh:id_to_vector:#{id}", vector.hash.to_s
+      def add_vector_hash_to_bucket(bucket, hash, vector_hash)
+        @redis.sadd "#{bucket}:#{hash}", vector_hash.to_s # Only storing vector's hash in Redis
       end
 
-      def vector_to_id(vector)
-        @redis.get "lsh:vector_to_id:#{vector.hash}"
+      def add_vector_id(vector_hash, id)
+        @redis.set "lsh:vector_to_id:#{vector_hash}", id
+        @redis.set "lsh:id_to_vector:#{id}", vector_hash.to_s
+      end
+
+      def vector_hash_to_id(vector_hash)
+        @redis.get "lsh:vector_to_id:#{vector_hash}"
       end
 
       def id_to_vector(id)
@@ -143,21 +147,23 @@ module LSH
       end
 
       def query_buckets(hashes)
-        vector_hashes = []
+        results_hashes = {}
         hashes.each_with_index do |hash, i|
           bucket = find_bucket(i)
-          results = @redis.smembers("#{bucket}:#{hash}")
-          vector_hashes += results if results
+          vector_hashes_in_bucket = @redis.smembers("#{bucket}:#{hash}")
+          if vector_hashes_in_bucket
+            vector_hashes_in_bucket.each do |vector_hash|
+              results_hashes[vector_hash] = true
+            end
+          end
         end
-        # Making sure we don't load the same vectors twice if they match
-        # in different random projections
-        vector_hashes.uniq!
-        results = []
-        vector_hashes.each do |vector_hash|
-          vector = load_vector(vector_hash)
-          results << vector
+        results_hashes.keys.map do |vector_hash|
+          {
+            :data => load_vector(vector_hash),
+            :hash => vector_hash.to_i,
+            :id => vector_hash_to_id(vector_hash)
+          }
         end
-        results
       end
 
     end

data/lib/lsh/web.rb

@@ -24,13 +24,7 @@ module LSH
       if mime_type == 'application/json'
         t0 = Time.now
         vector = JSON.parse(params[:data], :create_additions => true)
-        result_vectors = index.query(vector, params[:radius] || 0)
-        results = []
-        if params[:include] == 'id'
-          result_vectors.each { |v| results << { :id => index.vector_to_id(v), :data => v } }
-        else
-          result_vectors.each { |v| results << { :data => v } }
-        end
+        results = index.query(vector, params[:radius] || 0)
         content_type :json
         { "time" => Time.now - t0, "results" => results }.to_json
       else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lsh
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
   prerelease:
 platform: java
 authors: