lsh 0.2.0 → 0.3.1

data/lib/lsh/index.rb

@@ -38,7 +38,8 @@ module LSH
       Index.new(storage.parameters, storage) if storage.has_index? 
     end
 
-    def add(vector)
+    def add(vector, id = nil)
+      storage.add_vector_id(vector, id) if id
       hashes(vector).each_with_index do |hash, i|
         hash_i = array_to_hash(hash)
         bucket = storage.find_bucket(i)
@@ -46,6 +47,14 @@ module LSH
       end
     end
 
+    def vector_to_id(vector)
+      storage.vector_to_id(vector)
+    end
+
+    def id_to_vector(id)
+      storage.id_to_vector(id)
+    end
+
     def query(vector, multiprobe_radius = 0)
       hash_arrays = hashes(vector)
       hashes = hash_arrays.map { |a| array_to_hash(a) }
@@ -64,6 +73,18 @@ module LSH
       order_vectors_by_similarity(vector, results)
     end
 
+    def query_ids(id, multiprobe_radius = 0)
+      vector = id_to_vector(id)
+      query_ids_by_vector(vector, multiprobe_radius)
+    end
+
+    def query_ids_by_vector(vector, multiprobe_radius = 0)
+      vectors = query(vector, multiprobe_radius)
+      results = []
+      vectors.each { |v| results << vector_to_id(v) }
+      results
+    end
+
     def multiprobe_hashes_arrays(hash_arrays, multiprobe_radius)
       mp_arrays = []
       (1..multiprobe_radius).to_a.each do |radius|

data/lib/lsh/storage/memory.rb

@@ -45,6 +45,21 @@ module LSH
         end
       end
 
+      def add_vector_id(vector, id)
+        @vector_to_id ||= {}
+        @vector_to_id[vector.hash] = id
+        @id_to_vector ||= {}
+        @id_to_vector[id] = vector
+      end
+
+      def vector_to_id(vector)
+        @vector_to_id[vector.hash] if @vector_to_id
+      end
+
+      def id_to_vector(id)
+        @id_to_vector[id] if @id_to_vector
+      end
+
       def find_bucket(i)
         @buckets[i]
       end

data/lib/lsh/storage/redis_backend.rb

@@ -112,11 +112,37 @@ module LSH
         @redis.incr "lsh:buckets"
       end
 
+      def save_vector(vector)
+        path = File.join(@data_dir, vector.hash.to_s+'.dat')
+        vector.save(path) unless File.exists?(path)
+      end
+
+      def load_vector(hash)
+        vector = MathUtil.zeros(parameters[:dim])
+        vector.load(File.join(@data_dir, hash+'.dat'))
+        vector
+      end
+
       def add_vector_to_bucket(bucket, hash, vector)
-        vector.save(File.join(@data_dir, vector.hash.to_s+'.dat')) # Writing vector to disk
+        save_vector(vector) # Writing vector to disk if not already there
         @redis.sadd "#{bucket}:#{hash}", vector.hash.to_s # Only storing vector's hash in Redis
       end
 
+      def add_vector_id(vector, id)
+        save_vector(vector) # Writing vector to disk if not already there
+        @redis.set "lsh:vector_to_id:#{vector.hash}", id
+        @redis.set "lsh:id_to_vector:#{id}", vector.hash.to_s
+      end
+
+      def vector_to_id(vector)
+        @redis.get "lsh:vector_to_id:#{vector.hash}"
+      end
+
+      def id_to_vector(id)
+        vector_hash = @redis.get "lsh:id_to_vector:#{id}"
+        load_vector(vector_hash)
+      end
+
       def find_bucket(i)
         "lsh:bucket:#{i}"
       end
@@ -133,8 +159,7 @@ module LSH
         vector_hashes.uniq!
         results = []
         vector_hashes.each do |vector_hash|
-          vector = MathUtil.zeros(parameters[:dim])
-          vector.load(File.join(@data_dir, vector_hash+'.dat'))
+          vector = load_vector(vector_hash)
           results << vector
         end
         results

data/lib/lsh/web.rb

@@ -24,7 +24,13 @@ module LSH
       if mime_type == 'application/json'
         t0 = Time.now
         vector = JSON.parse(params[:data])
-        results = index.query(vector)
+        result_vectors = index.query(vector, params[:radius] || 0)
+        results = []
+        if params[:include] == 'id'
+          result_vectors.each { |v| results << { :id => index.vector_to_id(v), :data => v } }
+        else
+          result_vectors.each { |v| results << { :data => v } }
+        end
         content_type :json
         { "time" => Time.now - t0, "results" => results }.to_json
       else
@@ -32,13 +38,36 @@ module LSH
       end
     end
 
+    post '/query-ids' do
+      if params[:data] # We're querying with a vector
+        mime_type = (params[:mime_type] || 'application/json')
+        if mime_type == 'application/json'
+          t0 = Time.now
+          vector = JSON.parse(params[:data])
+          results = index.query_ids_by_vector(vector, params[:radius] || 0)
+          content_type :json
+          { "time" => Time.now - t0, "results" => results }.to_json
+        else
+          raise "Unrecognised mime-type"
+        end
+      elsif params[:id] # We're querying with an id
+        raise "Unknown id" unless index.id_to_vector(params[:id])
+        t0 = Time.now
+        results = index.query_ids(params[:id], params[:radius] || 0)
+        content_type :json
+        { "time" => Time.now - t0, "results" => results }.to_json
+      else
+        raise "Missing query"
+      end
+    end
+
     post '/index' do
       raise "Missing data" unless params[:data]
       mime_type = (params[:mime_type] || 'application/json')
       if mime_type == 'application/json'
         t0 = Time.now
         vector = JSON.parse(params[:data])
-        index.add(vector)
+        index.add(vector, params[:id])
         content_type :json
         { "time" => Time.now - t0, "status" => "indexed" }.to_json
       else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lsh
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-03 00:00:00.000000000 Z
+date: 2013-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gsl