lsh 0.0.5 → 0.1.0

data/lib/lsh.rb

@@ -1,6 +1,6 @@
 # ruby-lsh
 #
-# Copyright (c) 2011 British Broadcasting Corporation
+# Copyright (c) 2012 British Broadcasting Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
 # limitations under the License.
 
 require_relative 'lsh/index.rb'
+require_relative 'lsh/storage/memory.rb'
+require_relative 'lsh/storage/redis_backend.rb'
 if RUBY_PLATFORM == 'java'
   require_relative 'lsh/math_util_jblas.rb'
 else

data/lib/lsh/index.rb

@@ -1,6 +1,6 @@
 # ruby-lsh
 #
-# Copyright (c) 2011 British Broadcasting Corporation
+# Copyright (c) 2012 British Broadcasting Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,26 +18,31 @@ module LSH
 
   class Index
 
-    attr_reader :projections, :buckets
+    attr_reader :projections, :buckets, :storage
+
+    def initialize(parameters = {}, storage = LSH::Storage::Memory.new)
+      @storage = storage
+      unless storage.has_index?
+        storage.parameters = parameters
+        # Initializing projections and buckets
+        storage.projections = generate_projections(
+          parameters[:dim], 
+          parameters[:number_of_random_vectors], 
+          parameters[:number_of_independent_projections]
+        )
+        parameters[:number_of_independent_projections].times { |i| storage.create_new_bucket }
+      end
+    end
 
-    def initialize(dim, k, w = Float::INFINITY, l = 150)
-      @window = w
-      @dim = dim
-      @number_of_random_vectors = k
-      @number_of_independent_projections = l
-      @projections = generate_projections(dim, k, l)
-      @buckets = []
-      l.times { |i| @buckets << {} }
+    def self.load(storage)
+      Index.new(storage.parameters, storage) if storage.has_index? 
     end
 
     def add(vector)
       hashes(vector).each_with_index do |hash, i|
         hash_i = array_to_hash(hash)
-        if @buckets[i].has_key? hash_i
-          @buckets[i][hash_i] << vector
-        else
-          @buckets[i][hash_i] = [vector]
-        end
+        bucket = storage.find_bucket(i)
+        storage.add_vector_to_bucket(bucket, hash_i, vector)
       end
     end
 
@@ -45,22 +50,25 @@ module LSH
       results = []
       hashes(vector).each_with_index do |hash, i|
         hash_i = array_to_hash(hash)
-        bucket = @buckets[i]
+        bucket = storage.find_bucket(i)
+        # Multiprobe LSH
         # Take query hash, move it around at radius r, hash it and use the result as a query
         # TODO: only works for binary LSH atm
-        results += bucket[hash_i] if bucket[hash_i]
+        bucket_results = storage.query_bucket(bucket, hash_i)
+        results += bucket_results if bucket_results
         if multiprobe_radius > 0
           (1..multiprobe_radius).to_a.each do |radius|
-            (0..(@number_of_random_vectors - 1)).to_a.combination(radius).each do |flips|
+            (0..(storage.parameters[:number_of_random_vectors] - 1)).to_a.combination(radius).each do |flips|
               probe = hash.clone
               flips.each { |d| probe[d] = (probe[d] == 1) ? 0 : 1  }
               probe_hash = array_to_hash(probe)
-              results += bucket[probe_hash] if bucket.has_key?(probe_hash)
+              probe_bucket_results = storage.query_bucket(bucket, probe_hash)
+              results += probe_bucket_results if probe_bucket_results
             end
           end
         end
       end
-      results.uniq!
+      results = MathUtil.uniq(results)
       order_vectors_by_similarity(vector, results)
     end
 
@@ -70,7 +78,7 @@ module LSH
 
     def hashes(vector)
       hashes = []
-      @projections.each do |projection|
+      storage.projections.each do |projection|
         hashes << hash(vector, projection)
       end
       hashes
@@ -80,7 +88,8 @@ module LSH
       hash = []
       projection.each do |random_vector|
         dot_product = similarity(vector, random_vector)
-        if @window == Float::INFINITY # Binary LSH
+        window = storage.parameters[:window]
+        if window == Float::INFINITY # Binary LSH
           if dot_product >= 0
             hash << 1
           else
@@ -88,7 +97,7 @@ module LSH
           end
         else
           b = bias ? MathUtil.random_uniform : 0.0
-          hash << (b + dot_product / @window).floor
+          hash << (b + dot_product / window).floor
         end
       end
       hash
@@ -137,7 +146,7 @@ module LSH
     end
 
     def inspect
-      "LSH index; dimension: #{@dim}; window size: #{@window}; #{@number_of_random_vectors} random vectors; #{@number_of_independent_projections} independent projections"
+      "#<LSH index; dimension: #{storage.parameters.dim}; window size: #{storage.parameters.window}; #{storage.parameters.number_of_random_vectors} random vectors; #{storage.parameters.number_of_independent_projections} independent projections>"
     end
 
   end

data/lib/lsh/math_util_gsl.rb

@@ -1,6 +1,6 @@
 # ruby-lsh
 #
-# Copyright (c) 2011 British Broadcasting Corporation
+# Copyright (c) 2012 British Broadcasting Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 require 'gsl'
+require 'json'
 
 module LSH
 
@@ -42,6 +43,33 @@ module LSH
       v.norm
     end
 
+    def self.uniq(vs)
+      # Can't use uniq as
+      # [ v, JSON.parse(v.to_json) ].uniq.size == 2 with GSL
+      results = []
+      vs.each { |v| results << v unless results.member? v }
+      results
+    end
+
+  end
+
+end
+
+module GSL
+
+  class Vector
+
+    def to_json(*a)
+      {
+        'json_class' => self.class.name,
+        'data' => to_a,
+      }.to_json(*a)
+    end
+
+    def self.json_create(o)
+      alloc(*o['data'])
+    end
+
   end
 
 end

data/lib/lsh/math_util_jblas.rb

@@ -1,6 +1,6 @@
 # ruby-lsh
 #
-# Copyright (c) 2011 British Broadcasting Corporation
+# Copyright (c) 2012 British Broadcasting Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,6 +40,31 @@ module LSH
       v.norm2
     end
 
+    def self.uniq(vs)
+      vs.uniq
+    end
+
+  end
+
+end
+
+
+module JBLAS
+
+  class DoubleMatrix
+
+    def to_json(*a)
+      {
+        'json_class' => 'JBLAS::DoubleMatrix',
+        'data' => to_a,
+      }.to_json(*a)
+    end
+
+    def self.json_create(o)
+      from_array(o['data']).t
+    end
+
   end
 
 end
+

data/lib/lsh/storage/memory.rb

@@ -0,0 +1,60 @@
+# ruby-lsh
+#
+# Copyright (c) 2012 British Broadcasting Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module LSH
+
+  module Storage
+
+    class Memory
+
+      attr_accessor :projections
+      attr_accessor :parameters
+      attr_reader   :buckets
+
+      def has_index?
+        projections and parameters and @buckets
+      end
+
+      def reset!
+        @buckets = nil
+      end
+
+      def create_new_bucket
+        @buckets ||= []
+        @buckets << {}
+      end
+
+      def add_vector_to_bucket(bucket, hash, vector)
+        if bucket.has_key? hash
+          bucket[hash] << vector
+        else
+          bucket[hash] = [vector]
+        end
+      end
+
+      def find_bucket(i)
+        @buckets[i]
+      end
+
+      def query_bucket(bucket, hash)
+        bucket[hash]
+      end
+
+    end
+
+  end
+
+end

data/lib/lsh/storage/redis_backend.rb

@@ -0,0 +1,86 @@
+# ruby-lsh
+#
+# Copyright (c) 2012 British Broadcasting Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'redis'
+require 'json'
+
+module LSH
+
+  module Storage
+
+    class RedisBackend
+
+      attr_reader :redis
+
+      def initialize(params = { :redis => { :host => '127.0.0.1', :port => 6379 } })
+        @redis = Redis.new(params[:redis])
+      end
+
+      def reset!
+        @redis.flushall
+      end
+
+      def has_index?
+        projections and parameters and @redis.get("buckets") > 0
+      end
+
+      def projections=(projections)
+        @redis.set "projections", projections.to_json
+      end
+
+      def projections
+        begin
+          @projections ||= JSON.parse(@redis.get "projections")
+        rescue TypeError
+          nil
+        end
+      end
+
+      def parameters=(parms)
+        parms[:window] = 'Infinity' if parms[:window] == Float::INFINITY
+        @redis.set "parameters", parms.to_json
+      end
+
+      def parameters
+        @parms ||= (
+          parms = JSON.parse(@redis.get "parameters")
+          parms.keys.each { |k| parms[k.to_sym] = parms[k]; parms.delete(k) }
+          parms[:window] = Float::INFINITY if parms[:window] == 'Infinity'
+          parms
+        )
+      end
+
+      def create_new_bucket
+        @redis.incr "buckets"
+      end
+
+      def add_vector_to_bucket(bucket, hash, vector)
+        @redis.sadd "#{bucket}:#{hash}", vector.to_json
+      end
+
+      def find_bucket(i)
+        "bucket:#{i}" if @redis.get("buckets").to_i > i
+      end
+
+      def query_bucket(bucket, hash)
+        @redis.smembers("#{bucket}:#{hash}").map { |vector_json| JSON.parse(vector_json) }
+      end
+
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lsh
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-17 00:00:00.000000000 Z
+date: 2012-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gsl
@@ -27,6 +27,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: An implementation of LSH in Ruby, using JBLAS for JRuby and GSL for MRI
 email: yves.raimond@bbc.co.uk
 executables: []
@@ -37,6 +69,8 @@ files:
 - lib/lsh/index.rb
 - lib/lsh/math_util_gsl.rb
 - lib/lsh/math_util_jblas.rb
+- lib/lsh/storage/memory.rb
+- lib/lsh/storage/redis_backend.rb
 homepage: https://github.com/bbcrd/ruby-lsh
 licenses: []
 post_install_message: