lsh 0.5.0 → 0.6.0

data/lib/lsh/index.rb

@@ -39,20 +39,16 @@ module LSH
     end
 
     def add(vector, id = nil)
-      vector_hash = vector.hash
-      storage.add_vector(vector, vector_hash)
-      storage.add_vector_id(vector_hash, id) if id
+      id ||= storage.generate_id
+      storage.add_vector(vector, id)
       hashes(vector).each_with_index do |hash, i|
         hash_i = array_to_hash(hash)
         bucket = storage.find_bucket(i)
-        storage.add_vector_hash_to_bucket(bucket, hash_i, vector_hash)
+        storage.add_vector_id_to_bucket(bucket, hash_i, id)
       end
+      id
     end
-
-    def vector_hash_to_id(vector_hash)
-      storage.vector_hash_to_id(vector_hash)
-    end
-
+    
     def id_to_vector(id)
       storage.id_to_vector(id)
     end
@@ -71,7 +67,7 @@ module LSH
           probes_hashes = probes_arrays.map { |a| array_to_hash(a) }
           results += storage.query_buckets(probes_hashes)
         end
-        results.uniq! { |result| result[:hash] }
+        results.uniq! { |result| result[:id] }
       end
       order_results_by_similarity(vector, results)
     end
@@ -83,7 +79,7 @@ module LSH
 
     def query_ids_by_vector(vector, multiprobe_radius = 0)
       results = query(vector, multiprobe_radius)
-      results.map { |result| vector_hash_to_id(result[:hash]) }
+      results.map { |result| result[:id] }
     end
 
     def multiprobe_hashes_arrays(hash_arrays, multiprobe_radius)
@@ -99,10 +95,8 @@ module LSH
     end
 
     def order_results_by_similarity(vector, results)
-      # Faster than vectors.sort - we precompute all similarities to vector
-      # and order using those
-      similarities = results.map { |result| [ result[:hash], result[:id], result[:data], similarity(vector, result[:data]) ] }
-      similarities.sort_by { |hash, id, vector, sim| sim } .reverse .map { |vs| { :hash => vs[0], :id => vs[1], :data => vs[2] } }
+      vector_t = vector.transpose
+      results.sort_by { |result| similarity(result[:data], vector_t) } .reverse
     end
 
     def hashes(vector)

data/lib/lsh/math_util_gsl.rb

@@ -44,7 +44,7 @@ module LSH
     end
 
     def self.dot(v1, v2)
-      (v1 * v2.transpose)[0,0]
+      (v1 * v2)[0,0]
     end
 
     def self.norm(v)

data/lib/lsh/math_util_jblas.rb

@@ -37,7 +37,7 @@ module LSH
     end
 
     def self.dot(v1, v2)
-      (v1 * v2.t)[0,0]
+      (v1 * v2)[0,0]
     end
 
     def self.norm(v)

data/lib/lsh/storage/memory.rb

@@ -24,48 +24,42 @@ module LSH
       attr_accessor :parameters
       attr_reader   :buckets
 
+      def initialize
+        reset!
+      end
+
       def has_index?
         projections and parameters and @buckets
       end
 
       def reset!
-        @buckets = nil
-        @vectors = nil
-        @vector_hash_to_id = nil
-        @id_to_vector = nil
+        @buckets = []
+        @vectors = {}
+        @next_id = 0
       end
 
       def create_new_bucket
-        @buckets ||= []
         @buckets << {}
       end
 
-      def add_vector(vector, vector_hash)
-        @vectors ||= {}
-        @vectors[vector_hash] = vector
+      def generate_id
+        @next_id += 1
+      end
+
+      def add_vector(vector, id)
+        @vectors[id] = vector
       end
 
-      def add_vector_hash_to_bucket(bucket, hash, vector_hash)
+      def add_vector_id_to_bucket(bucket, hash, vector_id)
         if bucket.has_key? hash
-          bucket[hash] << vector_hash
+          bucket[hash] << vector_id
         else
-          bucket[hash] = [vector_hash]
+          bucket[hash] = [vector_id]
         end
       end
 
-      def add_vector_id(vector_hash, id)
-        @vector_hash_to_id ||= {}
-        @vector_hash_to_id[vector_hash] = id
-        @id_to_vector ||= {}
-        @id_to_vector[id] = vector_hash
-      end
-
-      def vector_hash_to_id(vector_hash)
-        @vector_hash_to_id[vector_hash] if @vector_hash_to_id
-      end
-
       def id_to_vector(id)
-        @vectors[@id_to_vector[id]] if @id_to_vector
+        @vectors[id]
       end
 
       def find_bucket(i)
@@ -73,20 +67,19 @@ module LSH
       end
 
       def query_buckets(hashes)
-        results_hashes = {}
+        result_ids = {}
         hashes.each_with_index do |hash, i|
           vectors_hashes_in_bucket = @buckets[i][hash]
           if vectors_hashes_in_bucket
-            vectors_hashes_in_bucket.each do |vector_hash|
-              results_hashes[vector_hash] = true
+            vectors_hashes_in_bucket.each do |vector_id|
+              result_ids[vector_id] = true
             end
           end
         end
-        results_hashes.keys.map do |vector_hash|
+        result_ids.keys.map do |vector_id|
           { 
-            :data => @vectors[vector_hash], 
-            :hash => vector_hash, 
-            :id => vector_hash_to_id(vector_hash)
+            :data => @vectors[vector_id], 
+            :id => vector_id, 
           }
         end
       end

data/lib/lsh/storage/redis_backend.rb

@@ -24,12 +24,17 @@ module LSH
     class RedisBackend
 
       attr_reader :redis, :data_dir
+      attr_accessor :vector_cache, :cache_vectors
 
-      def initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 }, :data_dir => 'data' })
+      def initialize(params = {})
+        defaults = {:redis => {}, :data_dir => "data", :cache_vectors => TRUE}
+        params = defaults.merge params
         @redis = Redis.new(params[:redis])
         @data_dir = params[:data_dir]
         Dir.mkdir(@data_dir) unless File.exists?(@data_dir)
         Dir.mkdir(File.join(@data_dir, 'projections')) unless File.exists?(File.join(@data_dir, 'projections'))
+        @cache_vectors = params[:cache_vectors]
+        @vector_cache = {}
       end
 
       def reset!
@@ -40,11 +45,9 @@ module LSH
       def clear_data!
         keys = @redis.keys("lsh:bucket:*")
         @redis.del(keys) unless keys.empty?
-        keys = @redis.keys("lsh:vector_to_id:*")
-        @redis.del(keys) unless keys.empty?
-        keys = @redis.keys("lsh:id_to_vector:*")
-        @redis.del(keys) unless keys.empty?
         delete_dat_files_in_dir(@data_dir)
+        @redis.set("lsh:max_vector_id", 0)
+        @vector_cache = {}
       end
 
       def clear_projections!
@@ -109,37 +112,36 @@ module LSH
         @redis.incr "lsh:buckets"
       end
 
-      def save_vector(vector, vector_hash)
-        path = File.join(@data_dir, vector_hash.to_s+'.dat')
-        vector.save(path) unless File.exists?(path)
-      end
-
-      def load_vector(hash)
-        vector = MathUtil.zeros(1, parameters[:dim])
-        vector.load(File.join(@data_dir, hash+'.dat'))
-        vector
+      def generate_id
+        (@redis.incr "lsh:max_vector_id").to_s
       end
 
-      def add_vector(vector, vector_hash)
-        save_vector(vector, vector_hash) # Writing vector to disk if not already there
+      def save_vector(vector, vector_id)
+        path = File.join(@data_dir, vector_id+'.dat')
+        raise "File #{path} already exists" if File.exists?(path)
+        vector.save(path) 
+        @vector_cache[vector_id] = vector if @cache_vectors
       end
 
-      def add_vector_hash_to_bucket(bucket, hash, vector_hash)
-        @redis.sadd "#{bucket}:#{hash}", vector_hash.to_s # Only storing vector's hash in Redis
+      def load_vector(vector_id)
+        @vector_cache[vector_id] || (
+          vector = MathUtil.zeros(1, parameters[:dim])
+          vector.load(File.join(@data_dir, vector_id+'.dat'))
+          @vector_cache[vector_id] = vector if @cache_vectors
+          vector
+        )
       end
 
-      def add_vector_id(vector_hash, id)
-        @redis.set "lsh:vector_to_id:#{vector_hash}", id
-        @redis.set "lsh:id_to_vector:#{id}", vector_hash.to_s
+      def add_vector(vector, vector_id)
+        save_vector(vector, vector_id) # Writing vector to disk if not already there
       end
 
-      def vector_hash_to_id(vector_hash)
-        @redis.get "lsh:vector_to_id:#{vector_hash}"
+      def add_vector_id_to_bucket(bucket, hash, vector_id)
+        @redis.sadd "#{bucket}:#{hash}", vector_id
       end
 
-      def id_to_vector(id)
-        vector_hash = @redis.get "lsh:id_to_vector:#{id}"
-        load_vector(vector_hash)
+      def id_to_vector(vector_id)
+        load_vector(vector_id)
       end
 
       def find_bucket(i)
@@ -147,21 +149,16 @@ module LSH
       end
 
       def query_buckets(hashes)
-        results_hashes = {}
-        hashes.each_with_index do |hash, i|
+        keys = hashes.each_with_index.map do |hash, i|
           bucket = find_bucket(i)
-          vector_hashes_in_bucket = @redis.smembers("#{bucket}:#{hash}")
-          if vector_hashes_in_bucket
-            vector_hashes_in_bucket.each do |vector_hash|
-              results_hashes[vector_hash] = true
-            end
-          end
+          "#{bucket}:#{hash}"
         end
-        results_hashes.keys.map do |vector_hash|
+        result_ids = @redis.sunion(keys)
+
+        result_ids.map do |vector_id|
           {
-            :data => load_vector(vector_hash),
-            :hash => vector_hash.to_i,
-            :id => vector_hash_to_id(vector_hash)
+            :data => load_vector(vector_id),
+            :id   => vector_id
           }
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lsh
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -75,6 +75,70 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: An implementation of LSH in Ruby, using JBLAS for JRuby and GSL for MRI
 email: yves.raimond@bbc.co.uk
 executables: []