commendo 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6882c6ed2001ecae7be9eae066e3f534beeb1559
-  data.tar.gz: 0a0982ccc0a17285537b1adfc12bfe854da6c10f
+  metadata.gz: 4fb3a3f98e3846e5f75d884630c3735ed57f62b1
+  data.tar.gz: 3944a25fc941e55a343b5201bf40ef77fbef08fb
 SHA512:
-  metadata.gz: 834aeca013589496b5e6b95a0d5ec7e327fc6192ba57e7dab7c2d8507055013058a96b354b13bb2422c21caa7cce92c0fd5f0011c60d3cb5c202b723eb94dbf4
-  data.tar.gz: d773e4c3776f66ba003cf7b7c1327a376e439f8f72480b903d1512db906c27aa55c9f01a052c11a24fbde4f4e90d7bb50b13d6494fbd963ea407d0d4807eb5fd
+  metadata.gz: fe3b6ea17ed10adc4ae2ffa12e7179d305970b4c8bf0d100671df8b6f800b7ffa042291c57a8d1c4d62550490654c245134f3f18c2f1733cfa0ca829018defcb
+  data.tar.gz: 7d01cc0c741ca514992e57b44c3defdc1a8d3de5db26f486eae3ef9bd1de62f8823974c6350178ae725bc0b61698f7cc9fdd4f1705f085a1b4799db3654c3276

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.0.4 / 2014-03-31
+* [FEATURE] Similarity score pushed down into redis using two different approaches depending on size of set
+
 # 0.0.3 / 2014-03-31
 * [FEATURE] Filtering of recommendations based on tags
 

data/lib/commendo/content_set.rb

@@ -43,6 +43,7 @@ module Commendo
       similar.each do |other_resource|
         redis.zrem(similarity_key(other_resource[:resource]), "#{resource}")
       end
+      #TODO delete from groups?
       redis.del(similarity_key(resource))
       redis.del(resource_key(resource))
     end
@@ -50,18 +51,22 @@ module Commendo
     def calculate_similarity(threshold = 0)
       #TODO make this use scan for scaling
       keys = redis.keys("#{resource_key_base}:*")
-      keys.each_with_index do |outer_key, i|
-        outer_res = outer_key.gsub(/^#{resource_key_base}:/, '')
-        calculate_similarity_in_redis(outer_key, similarity_key(outer_res), threshold)
-        yield(outer_key, i, keys.length) if block_given?
+      keys.each_with_index do |key, i|
+        yield(key, i, keys.length) if block_given?
+        completed = redis.eval(similarity_lua, keys: [key], argv: [resource_key_base, similar_key_base, group_key_base, threshold])
+        if completed == 999
+          resource = key.gsub(/^#{resource_key_base}:/, '')
+          groups = redis.smembers(resource_key(resource))
+          group_keys = groups.map { |group| group_key(group) }
+          resources = redis.sunion(*group_keys)
+          resources.each do |to_compare|
+            next if resource == to_compare
+            redis.eval(pair_comparison_lua, keys: [key, resource_key(to_compare), similarity_key(resource), similarity_key(to_compare)], argv: [resource, to_compare, threshold])
+          end
+        end
       end
     end
 
-    def calculate_similarity_in_redis(set_key, similiarity_key, threshold)
-      #TODO maybe consider using ary.combination to get finer grained operation in lua
-      redis.eval(similarity_lua, [set_key, similiarity_key], [resource_key_base, threshold])
-    end
-
     def similar_to(resource)
       similar_resources = redis.zrevrange(similarity_key(resource), 0, -1, with_scores: true)
 
@@ -98,6 +103,15 @@ module Commendo
       file.read
     end
 
+    def pair_comparison_lua
+      @pair_comparison_lua ||= load_pair_comparison_lua
+    end
+
+    def load_pair_comparison_lua
+      file = File.open(File.expand_path('../pair_comparison.lua', __FILE__), "r")
+      file.read
+    end
+
     def similar_key_base
       "#{key_base}:similar"
     end
@@ -110,8 +124,13 @@ module Commendo
       "#{resource_key_base}:#{resource}"
     end
 
+    def group_key_base
+      "#{key_base}:groups"
+    end
+
+
     def group_key(group)
-      "#{key_base}:groups:#{group}"
+      "#{group_key_base}:#{group}"
     end
 
   end

data/lib/commendo/pair_comparison.lua

@@ -0,0 +1,29 @@
+local left_key = KEYS[1]
+local right_key = KEYS[2]
+local left_similarity_key = KEYS[3]
+local right_similarity_key = KEYS[4]
+
+local left = tonumber(ARGV[1])
+local right = tonumber(ARGV[2])
+local threshold = tonumber(ARGV[3])
+
+local function round(num, idp)
+  local mult = 10^(idp or 0)
+  return math.floor(num * mult + 0.5) / mult
+end
+
+redis.log(redis.LOG_NOTICE, 'Running pair comparison for ' .. left_key .. ' ' .. right_key)
+
+local intersect = table.getn(redis.call('SINTER', left_key, right_key))
+if intersect > 0 then
+    local union = table.getn(redis.call('SUNION', left_key, right_key))
+    local similarity = round(intersect / union, 3)
+    if similarity > threshold then
+        redis.call('ZADD', left_similarity_key, similarity, right)
+        redis.call('ZADD', right_similarity_key, similarity, left)
+    end
+end
+
+redis.log(redis.LOG_NOTICE, 'Finished running pair comparison for ' .. left_key .. ' ' .. right_key)
+
+return true

data/lib/commendo/similarity.lua

@@ -1,31 +1,56 @@
-local root_key = KEYS[1]
-local sim_key = KEYS[2]
-local set_key_base = ARGV[1]
-local threshold = tonumber(ARGV[2])
-
-redis.log(redis.LOG_NOTICE, 'Running similarity for ' .. root_key)
-local key_matches = redis.call('KEYS', set_key_base .. ':*')
-
-redis.call('DEL', sim_key)
-local count = 0
--- local similar = {}
-for _,key in ipairs(key_matches) do
-    if key ~= root_key then
-        count = count + 1
-        local intersect = table.getn(redis.call('SINTER', root_key, key))
-        if intersect > 0 then
-            local union = table.getn(redis.call('SUNION', root_key, key))
-            local similarity = intersect / union
-            if similarity > threshold then
-                -- table.insert(similar, key)
-                -- table.insert(similar, similarity)
-                local resource = key:gsub('%' .. set_key_base .. ':', '')
-                redis.call('ZADD', sim_key, similarity, resource)
+local resource_key = KEYS[1]
+local resource_key_base = ARGV[1]
+local sim_key_base = ARGV[2]
+local group_key_base = ARGV[3]
+local threshold = tonumber(ARGV[4])
+
+local function round(num, idp)
+  local mult = 10^(idp or 0)
+  return math.floor(num * mult + 0.5) / mult
+end
+
+redis.log(redis.LOG_NOTICE, 'Running complete similarity for ' .. resource_key)
+
+local resource = resource_key:gsub('%' .. resource_key_base .. ':', '')
+local groups = redis.call('smembers', resource_key)
+
+if table.getn(groups) > 999 then
+    redis.log(redis.LOG_NOTICE, 'Complete similarity too large for ' .. resource_key .. ', ' .. table.getn(groups))
+	return 999
+end
+
+local group_keys = {}
+for _,group in ipairs(groups) do
+    table.insert(group_keys, group_key_base .. ':' .. group)
+end
+--redis.log(redis.LOG_NOTICE, 'Found ' .. table.getn(group_keys) .. ' group keys')
+
+local resources = redis.call('sunion', unpack(group_keys))
+
+--local resources = redis.call('sunion', unpack(group_keys))
+--redis.log(redis.LOG_NOTICE, 'Found ' .. table.getn(resources) .. ' resources')
+
+local previous = 'foo'
+for _,to_compare in ipairs(resources) do
+--    redis.log(redis.LOG_NOTICE, 'Comparing ' .. resource .. ' and ' .. to_compare)
+    if to_compare ~= previous then
+        previous = to_compare
+        if resource > to_compare then
+--          redis.log(redis.LOG_NOTICE, 'Calculating similarity for ' .. resource .. ' and ' .. to_compare)
+            local intersect = table.getn(redis.call('SINTER', resource_key, resource_key_base .. ':' .. to_compare))
+            if intersect > 0 then
+                local union = table.getn(redis.call('SUNION', resource_key, resource_key_base .. ':' .. to_compare))
+                local similarity = round(intersect / union, 3)
+                if similarity > threshold then
+--                  redis.log(redis.LOG_NOTICE, resource .. ' and ' .. to_compare .. ' scored ' .. similarity)
+                    redis.call('ZADD', sim_key_base .. ':' .. resource, similarity, to_compare)
+                    redis.call('ZADD', sim_key_base .. ':' .. to_compare, similarity, resource)
+                end
             end
         end
     end
 end
 
--- redis.call('HMSET', sim_key, unpack(similar))
-redis.log(redis.LOG_NOTICE, 'Finished running similarity for ' .. root_key)
+redis.log(redis.LOG_NOTICE, 'Finished running complete similarity for ' .. resource_key)
+
 return true

data/lib/commendo/version.rb

@@ -1,3 +1,3 @@
 module Commendo
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/commendo/weighted_group.rb

@@ -21,7 +21,7 @@ module Commendo
       redis.del(tmp_key)
 
       similar_resources.map do |resource|
-        {resource: resource[0], similarity: resource[1].to_f}
+        {resource: resource[0], similarity: resource[1].to_f.round(3)}
       end
 
     end

data/test/content_set_test.rb

@@ -68,10 +68,10 @@ module Commendo
       expected = [
         {resource: '9', similarity: 0.5},
         {resource: '6', similarity: 0.5},
-        {resource: '12', similarity: 0.33333333333333},
+        {resource: '12', similarity: 0.333},
         {resource: '3', similarity: 0.25},
-        {resource: '21', similarity: 0.16666666666667},
-        {resource: '15', similarity: 0.16666666666667}
+        {resource: '21', similarity: 0.167},
+        {resource: '15', similarity: 0.167}
       ]
       assert_equal expected, cs.similar_to(18)
     end

data/test/weighted_group_test.rb

@@ -26,13 +26,12 @@ module Commendo
       weighted_group = WeightedGroup.new(redis, 'CommendoTests:WeightedGroup', { cs: cs1, weight: 1.0 },  { cs: cs2, weight: 10.0 },  { cs: cs3, weight: 100.0 } )
       expected = [
         {resource: '6', similarity: 55.5},
-        {resource: '12', similarity: 36.99999999999963},
+        {resource: '12', similarity: 36.963},
         {resource: '9', similarity: 5.0},
         {resource: '3', similarity: 2.5},
-        {resource: '21', similarity: 1.6666666666666998},
-        {resource: '15', similarity: 1.6666666666666998}
+        {resource: '21', similarity: 1.67},
+        {resource: '15', similarity: 1.67}
       ]
-
       assert_equal expected, weighted_group.similar_to(18)
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: commendo
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Rob Styles
@@ -110,6 +110,7 @@ files:
 - commendo.gemspec
 - lib/commendo.rb
 - lib/commendo/content_set.rb
+- lib/commendo/pair_comparison.lua
 - lib/commendo/similarity.lua
 - lib/commendo/tag_set.rb
 - lib/commendo/version.rb