commendo 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ccb7bd9fd0ad412db66aeda26a0048db1d3dc040
-  data.tar.gz: 0afcaf6c33e8eed6b27401e3bce8bf67dc59440a
+  metadata.gz: 01e2c317e998efe31d72e1a399d1d8eb677854a7
+  data.tar.gz: fad0b2a2c09333ef31eff3f3aaaf91c7b3d1862f
 SHA512:
-  metadata.gz: 71d5f32a19e6d7de531dc3e99727caafc400eb0d7f0a3aa326f7c03a4b11ed5962654aea49499a75fc1d49344e156987bf20424d778562b901924fa8af939fde
-  data.tar.gz: 12f3145df67a43dd59eb56be2884118becb03c93535e7bb25dffaa4fc0d2a9ddeda6a563ea100c658d752f20567c48099ae8cbd2a8af2cb0dd1e6818780a4e2f
+  metadata.gz: 8d0563d3909815866885f6f654c172f39675c442fb24a7306162090f848e2e50f4161aabf0fcc4f4779e474ff3b5c9902ec8d5c7cdc66db69cd7a3886f17a9bc
+  data.tar.gz: f05702dd19c5be8b6480ef4a513294c493f7138a2bb2d3c5f329fc6a734a9060660ab168f4d987f830c188142fc6bd5334c0a903b0f45024d7810ba7cb02e284

data/CHANGELOG.md

@@ -1,2 +1,5 @@
+# 0.0.2 / 2014-03-31
+* [FEATURE] Weighted groups. Incremental additions. Deletions.
+
 # 0.0.1 / 2014-03-28
 * [FEATURE] Initial Release

data/Rakefile

@@ -46,8 +46,8 @@ task :load_traffic_from_tsv, :filename do |task, args|
   end
 
   puts 'Processing...'
-  cs.calculate_similarity(0.1999999999) { |key|
-    puts key
+  cs.calculate_similarity(0) { |key, index, total|
+    puts "#{key} - #{index}/#{total} = #{(index/total.to_f*100).round(2)}%"
   }
 end
 

data/lib/commendo.rb

@@ -1,6 +1,7 @@
 require 'redis'
 require 'commendo/version'
 require 'commendo/content_set'
+require 'commendo/weighted_group'
 
 module Commendo
   # Your code goes here...

data/lib/commendo/content_set.rb

@@ -9,43 +9,70 @@ module Commendo
     end
 
     def add_by_group(group, *resources)
-      resources.each do |res|
-        redis.sadd("#{set_key_base}:#{res}", group)
+      redis.sadd(group_key(group), resources)
+      resources.each do |resource|
+        redis.sadd(resource_key(resource), group)
       end
     end
 
     def add(resource, *groups)
-      redis.sadd("#{set_key_base}:#{resource}", groups)
+      redis.sadd(resource_key(resource), groups)
+      groups.each do |group|
+        redis.sadd(group_key(group), resource)
+      end
     end
 
-    def calculate_similarity(threshold = 0)
-      keys = redis.keys("#{set_key_base}:*")
-      keys.each do |outer_key|
-        outer_res = outer_key.gsub(/^#{set_key_base}:/, '')
-        outer_similarity_key = "#{similar_key_base}:#{outer_res}"
-        calculate_similarity_in_redis(outer_key, outer_similarity_key, threshold)
-        yield(outer_key) if block_given?
+    def add_and_calculate(resource, *groups)
+      add(resource, *groups)
+      groups = redis.smembers(resource_key(resource))
+      group_keys = groups.map { |group| group_key(group) }
+      resources = redis.sunion(*group_keys)
+      resources.combination(2) do |l, r|
+        intersect = redis.sinter(resource_key(l), resource_key(r))
+        if (intersect.length > 0)
+          union = redis.sunion(resource_key(l), resource_key(r))
+          jaccard = intersect.length / union.length.to_f
+          redis.zadd(similarity_key(l), jaccard, r)
+          redis.zadd(similarity_key(r), jaccard, l)
+        end
+      end
+    end
+
+    def delete(resource)
+      similar = similar_to(resource)
+      similar.each do |other_resource|
+        redis.zrem(similarity_key(other_resource[:resource]), "#{resource}")
       end
+      redis.del(similarity_key(resource))
+      redis.del(resource_key(resource))
+    end
 
+    def calculate_similarity(threshold = 0)
+      #TODO make this use scan for scaling
+      keys = redis.keys("#{resource_key_base}:*")
+      keys.each_with_index do |outer_key, i|
+        outer_res = outer_key.gsub(/^#{resource_key_base}:/, '')
+        calculate_similarity_in_redis(outer_key, similarity_key(outer_res), threshold)
+        yield(outer_key, i, keys.length) if block_given?
+      end
     end
 
     def calculate_similarity_in_redis(set_key, similiarity_key, threshold)
-      redis.eval(similarity_lua, [set_key, similiarity_key], [set_key_base, threshold])
+      #TODO maybe consider using ary.combination to get finer grained operation in lua
+      redis.eval(similarity_lua, [set_key, similiarity_key], [resource_key_base, threshold])
     end
 
     def similar_to(resource)
-      similar = []
-      similar_resources = redis.hgetall("#{similar_key_base}:#{resource}")
-      similar_resources.each do |resource, similarity|
-        similar << {resource: resource, similarity: similarity.to_f}
-      end
-      similar.sort! do |x, y|
-        if y[:similarity] != x[:similarity]
-          y[:similarity] <=> x[:similarity]
-        else
-          y[:resource] <=> x[:resource]
-        end
+      similar_resources = redis.zrevrange(similarity_key(resource), 0, -1, with_scores: true)
+
+      similar_resources.map do |resource|
+        {resource: resource[0], similarity: resource[1].to_f}
       end
+
+    end
+
+    def similarity_key(resource)
+      "#{similar_key_base}:#{resource}"
     end
 
     private
@@ -59,14 +86,22 @@ module Commendo
       file.read
     end
 
-    def set_key_base
-      "#{key_base}:sets"
-    end
-
     def similar_key_base
       "#{key_base}:similar"
     end
 
+    def resource_key_base
+      "#{key_base}:resources"
+    end
+
+    def resource_key(resource)
+      "#{resource_key_base}:#{resource}"
+    end
+
+    def group_key(group)
+      "#{key_base}:groups:#{group}"
+    end
+
   end
 
 end

data/lib/commendo/similarity.lua

@@ -20,7 +20,7 @@ for _,key in ipairs(key_matches) do
                 -- table.insert(similar, key)
                 -- table.insert(similar, similarity)
                 local resource = key:gsub('%' .. set_key_base .. ':', '')
-                redis.call('HSET', sim_key, resource, similarity)
+                redis.call('ZADD', sim_key, similarity, resource)
             end
         end
     end

data/lib/commendo/version.rb

@@ -1,3 +1,3 @@
 module Commendo
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/commendo/weighted_group.rb

@@ -0,0 +1,31 @@
+module Commendo
+
+  class WeightedGroup
+
+    attr_accessor :content_sets, :redis, :key_base
+
+    def initialize(redis, key_base, *content_sets)
+      @content_sets, @redis, @key_base = content_sets, redis, key_base
+    end
+
+    def similar_to(resource)
+      keys = content_sets.map do |cs|
+        cs[:cs].similarity_key(resource)
+      end
+      weights = content_sets.map do |cs|
+        cs[:weight]
+      end
+      tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
+      redis.zunionstore(tmp_key, keys, weights: weights)
+      similar_resources = redis.zrevrange(tmp_key, 0, -1, with_scores: true)
+      redis.del(tmp_key)
+
+      similar_resources.map do |resource|
+        {resource: resource[0], similarity: resource[1].to_f}
+      end
+
+    end
+
+  end
+
+end

data/test/content_set_test.rb

@@ -9,6 +9,13 @@ module Commendo
 
   class ContentSetTest < Minitest::Test
 
+    def test_gives_similarity_key_for_resource
+      redis = Redis.new(db: 15)
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      assert_equal 'CommendoTests:similar:resource-1', cs.similarity_key('resource-1')
+    end
+
     def test_stores_sets_by_resource
       redis = Redis.new(db: 15)
       redis.flushdb
@@ -18,15 +25,15 @@ module Commendo
       cs.add('resource-2', 'group-1')
       cs.add('resource-3', 'group-1', 'group-2')
       cs.add('resource-4', 'group-2')
-      assert redis.sismember("#{key_base}:sets:resource-1", 'group-1')
-      assert redis.sismember("#{key_base}:sets:resource-2", 'group-1')
-      assert redis.sismember("#{key_base}:sets:resource-3", 'group-1')
-      refute redis.sismember("#{key_base}:sets:resource-4", 'group-1')
-
-      assert redis.sismember("#{key_base}:sets:resource-1", 'group-2')
-      refute redis.sismember("#{key_base}:sets:resource-2", 'group-2')
-      assert redis.sismember("#{key_base}:sets:resource-3", 'group-2')
-      assert redis.sismember("#{key_base}:sets:resource-4", 'group-2')
+      assert redis.sismember("#{key_base}:resources:resource-1", 'group-1')
+      assert redis.sismember("#{key_base}:resources:resource-2", 'group-1')
+      assert redis.sismember("#{key_base}:resources:resource-3", 'group-1')
+      refute redis.sismember("#{key_base}:resources:resource-4", 'group-1')
+
+      assert redis.sismember("#{key_base}:resources:resource-1", 'group-2')
+      refute redis.sismember("#{key_base}:resources:resource-2", 'group-2')
+      assert redis.sismember("#{key_base}:resources:resource-3", 'group-2')
+      assert redis.sismember("#{key_base}:resources:resource-4", 'group-2')
     end
 
     def test_stores_sets_by_group
@@ -36,15 +43,15 @@ module Commendo
       cs = ContentSet.new(redis, key_base)
       cs.add_by_group('group-1', 'resource-1', 'resource-2', 'resource-3')
       cs.add_by_group('group-2', 'resource-1', 'resource-3', 'resource-4')
-      assert redis.sismember("#{key_base}:sets:resource-1", 'group-1')
-      assert redis.sismember("#{key_base}:sets:resource-2", 'group-1')
-      assert redis.sismember("#{key_base}:sets:resource-3", 'group-1')
-      refute redis.sismember("#{key_base}:sets:resource-4", 'group-1')
-
-      assert redis.sismember("#{key_base}:sets:resource-1", 'group-2')
-      refute redis.sismember("#{key_base}:sets:resource-2", 'group-2')
-      assert redis.sismember("#{key_base}:sets:resource-3", 'group-2')
-      assert redis.sismember("#{key_base}:sets:resource-4", 'group-2')
+      assert redis.sismember("#{key_base}:resources:resource-1", 'group-1')
+      assert redis.sismember("#{key_base}:resources:resource-2", 'group-1')
+      assert redis.sismember("#{key_base}:resources:resource-3", 'group-1')
+      refute redis.sismember("#{key_base}:resources:resource-4", 'group-1')
+
+      assert redis.sismember("#{key_base}:resources:resource-1", 'group-2')
+      refute redis.sismember("#{key_base}:resources:resource-2", 'group-2')
+      assert redis.sismember("#{key_base}:resources:resource-3", 'group-2')
+      assert redis.sismember("#{key_base}:resources:resource-4", 'group-2')
     end
 
     def test_calculates_similarity_scores
@@ -95,6 +102,60 @@ module Commendo
       skip
     end
 
+    def test_deletes_resource_from_everywhere
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      (3..23).each do |group|
+        (3..23).each do |res|
+          cs.add_by_group(group, res) if res % group == 0
+        end
+      end
+      cs.calculate_similarity
+      assert similar_to(cs, 18, 12)
+
+      cs.delete(12)
+      assert_equal [], cs.similar_to(12)
+      refute similar_to(cs, 18, 12)
+
+      cs.calculate_similarity
+      assert_equal [], cs.similar_to(12)
+      refute similar_to(cs, 18, 12)
+
+    end
+
+    def test_accepts_incremental_updates
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      (3..23).each do |group|
+        (3..23).each do |res|
+          cs.add(res, group) if res % group == 0
+        end
+      end
+      cs.calculate_similarity
+      assert similar_to(cs, 18, 12)
+      refute similar_to(cs, 10, 12)
+
+      cs.add_and_calculate(12, 'foo', true)
+      cs.add_and_calculate(10, 'foo', true)
+      assert similar_to(cs, 10, 12)
+    end
+
+    def test_accepts_tag_collection
+      skip
+    end
+
+    def test_filters_by_tag_collection
+      skip
+    end
+
+    def similar_to(cs, resource, similar)
+      cs.similar_to(resource).select { |sim| sim[:resource] == "#{similar}" }.length > 0
+    end
+
   end
 
 end

data/test/weighted_group_test.rb

@@ -0,0 +1,47 @@
+gem 'minitest'
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'minitest/mock'
+require 'mocha/setup'
+require 'commendo'
+
+module Commendo
+
+  class WeightedGroupTest < Minitest::Test
+
+    def test_calls_each_content_set
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      cs1 = ContentSet.new(redis, 'CommendoTests:ContentSet1')
+      cs2 = ContentSet.new(redis, 'CommendoTests:ContentSet2')
+      cs3 = ContentSet.new(redis, 'CommendoTests:ContentSet3')
+      (3..23).each do |group|
+        (3..23).each do |res|
+          cs1.add_by_group(group, res) if (res % group == 0) && (res % 2 == 0)
+          cs2.add_by_group(group, res) if (res % group == 0) && (res % 3 == 0)
+          cs3.add_by_group(group, res) if (res % group == 0) && (res % 6 == 0)
+        end
+      end
+      [cs1, cs2, cs3].each { |cs| cs.calculate_similarity }
+      weighted_group = WeightedGroup.new(redis, 'CommendoTests:WeightedGroup', { cs: cs1, weight: 1.0 },  { cs: cs2, weight: 10.0 },  { cs: cs3, weight: 100.0 } )
+      expected = [
+        {resource: '6', similarity: 55.5},
+        {resource: '12', similarity: 36.99999999999963},
+        {resource: '9', similarity: 5.0},
+        {resource: '3', similarity: 2.5},
+        {resource: '21', similarity: 1.6666666666666998},
+        {resource: '15', similarity: 1.6666666666666998}
+      ]
+
+      assert_equal expected, weighted_group.similar_to(18)
+    end
+
+    def test_precalculates
+      skip
+    end
+
+
+
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: commendo
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Rob Styles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2014-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -112,7 +112,9 @@ files:
 - lib/commendo/content_set.rb
 - lib/commendo/similarity.lua
 - lib/commendo/version.rb
+- lib/commendo/weighted_group.rb
 - test/content_set_test.rb
+- test/weighted_group_test.rb
 homepage: ''
 licenses:
 - MIT
@@ -139,4 +141,5 @@ specification_version: 4
 summary: A Jaccard-similarity recommender using Redis sets
 test_files:
 - test/content_set_test.rb
+- test/weighted_group_test.rb
 has_rdoc: