commendo 0.0.9 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80baa49ab8364f70fd8dd4f2127f95d4f5ad0da2
-  data.tar.gz: 355812df077e1df1cb4d0425fb36fac8334d19f6
+  metadata.gz: f4d4570da8f16342e587461c8731982f4a82c8ba
+  data.tar.gz: bdde9ae21c9bb35c7c9b6073a82e651086132617
 SHA512:
-  metadata.gz: a1c2164296cfe3c47d0ef943594aeb6b4f20a007f421729ddd5b4328c3ffd2e90033f7315fac9451f98f39a088b504f0252cc3540a75560f6df9b4052bc4c9b6
-  data.tar.gz: 5d31d9f878a2eb7ab19e97b13887adb95b3df416866993b5c9ff8008a21f24b97c7acaf685a1baeb17d9003fa6716a5b1d5041efe057de1f2dc70327c8eba4df
+  metadata.gz: 31406c7ee3846a6046e8e149ca8ada66eaed799977ee615e778a5fb3a2c353165e246deb87b422bef7d5a544a4dddbac3788a4ed879a77a04b13b0f3ed9d7dbc
+  data.tar.gz: 15e58e641bd237466b80713c64d893ebd66877aa4c4a28241b584875d5e476fc28c45d68194eacd9b74cfef5a4e144c011e892185bd1ee9d27f38fc346c11032

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 1.0.0 / 2014-04-22
+* [FEATURE] Add limits to similarity requests and bump to production release 1.0.0 :)
+
 # 0.0.9 / 2014-04-09
 * [BUGFIX] Fix similarity calculation for resources in many sets
 

data/bin/commendo-load

@@ -1,21 +1,30 @@
 #!/usr/bin/env ruby
 
-redis_db = ARGV[0].to_i
-content_set_base_redis_key = ARGV[1]
-filename = ARGV[2]
-
 require 'redis'
 require 'commendo'
+require 'json'
 
-redis = Redis.new(db: 2)
-cs = Commendo::ContentSet.new(redis, content_set_base_redis_key)
-
-File.open(filename) do |f|
+filename = ARGV[0]
+redis_db = ARGV[1].to_i
+base_key = ARGV[2]
 
-  current_resource = nil
-  current_headings = []
+redis = Redis.new(db: redis_db, timeout: 60)
+cs = Commendo::ContentSet.new(redis, base_key)
 
-  f.each_line.with_index do |tsv, i|
-    next if i.zero?
+puts "Loading."
+File.open(filename) do |f|
+  f.each_line.with_index do |json, i|
+    print '.'
+    resource, scored_groups = JSON.parse(json)
+    cs.add(resource, *scored_groups)
+  end
+end
+puts "\nFinished loading"
 
-    resource, _classified, root, score, heading = tsv.split(/\t/)
+puts 'Calculating similarities'
+pbar = nil
+cs.calculate_similarity do |key, i, total|
+  pbar ||= ProgressBar.new('Calculating similarity', total)
+  pbar.inc
+  #puts "Calculating similarity for #{i}/#{total} — #{key}"
+end

data/commendo.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'redis'
+  spec.add_dependency 'progressbar'
 
   spec.add_development_dependency 'bundler', '~> 1.5'
   spec.add_development_dependency 'rake'

data/lib/commendo/content_set.rb

@@ -85,17 +85,18 @@ module Commendo
       end
     end
 
-    def similar_to(resource)
+    def similar_to(resource, limit = 0)
+      finish = limit -1
       if resource.kind_of? Array
         keys = resource.map do |res|
           similarity_key(res)
         end
         tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
         redis.zunionstore(tmp_key, keys)
-        similar_resources = redis.zrevrange(tmp_key, 0, -1, with_scores: true)
+        similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
         redis.del(tmp_key)
       else
-        similar_resources = redis.zrevrange(similarity_key(resource), 0, -1, with_scores: true)
+        similar_resources = redis.zrevrange(similarity_key(resource), 0, finish, with_scores: true)
       end
       similar_resources.map do |resource|
         {resource: resource[0], similarity: resource[1].to_f}
@@ -103,11 +104,18 @@ module Commendo
     end
 
     def filtered_similar_to(resource, options = {})
-      similar = similar_to(resource)
-      return similar if @tag_set.nil? || options[:include].nil? && options[:exclude].nil?
-      similar.delete_if { |s| !options[:exclude].nil? && @tag_set.matches(s[:resource], *options[:exclude]) }
-      similar.delete_if { |s| !options[:include].nil? && !@tag_set.matches(s[:resource], *options[:include]) }
-      similar
+      if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
+        return similar_to(resource, options[:limit] || 0)
+      else
+        similar = similar_to(resource)
+        limit = options[:limit] || similar.length
+        filtered = []
+        similar.each do |s|
+          return filtered if filtered.length >= limit
+          filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+        end
+        return filtered
+      end
     end
 
     def similarity_key(resource)

data/lib/commendo/tag_set.rb

@@ -21,9 +21,11 @@ module Commendo
       add(resource, *tags)
     end
 
-    def matches(resource, *tags)
+    def matches(resource, include, exclude = [])
       resource_tags = get(resource)
-      (resource_tags & tags).length > 0
+      can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
+      should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
+      return can_include && !should_exclude
     end
 
     def delete(resource)

data/lib/commendo/version.rb

@@ -1,3 +1,3 @@
 module Commendo
-  VERSION = '0.0.9'
+  VERSION = '1.0.0'
 end

data/lib/commendo/weighted_group.rb

@@ -8,7 +8,8 @@ module Commendo
       @content_sets, @redis, @key_base = content_sets, redis, key_base
     end
 
-    def similar_to(resource)
+    def similar_to(resource, limit = 0)
+      finish = limit -1
       resources = resource.kind_of?(Array) ? resource : [resource]
       keys = []
       weights = []
@@ -20,7 +21,7 @@ module Commendo
       end
       tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
       redis.zunionstore(tmp_key, keys, weights: weights)
-      similar_resources = redis.zrevrange(tmp_key, 0, -1, with_scores: true)
+      similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
       redis.del(tmp_key)
 
       similar_resources.map do |resource|
@@ -30,11 +31,18 @@ module Commendo
     end
 
     def filtered_similar_to(resource, options = {})
-      similar = similar_to(resource)
-      return similar if @tag_set.nil? || options[:include].nil? && options[:exclude].nil?
-      similar.delete_if { |s| !options[:exclude].nil? && @tag_set.matches(s[:resource], *options[:exclude]) }
-      similar.delete_if { |s| !options[:include].nil? && !@tag_set.matches(s[:resource], *options[:include]) }
-      similar
+      if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
+        return similar_to(resource, options[:limit] || 0)
+      else
+        similar = similar_to(resource)
+        limit = options[:limit] || similar.length
+        filtered = []
+        similar.each do |s|
+          return filtered if filtered.length >= limit
+          filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
+        end
+        return filtered
+      end
     end
 
   end

data/test/content_set_test.rb

@@ -34,6 +34,27 @@ module Commendo
       assert_equal expected, cs.similar_to('resource-1')
     end
 
+    def test_recommends_limited_by_number
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      key_base = 'CommendoTests'
+      cs = ContentSet.new(redis, key_base)
+      cs.add('resource-1', 'group-1', 'group-2')
+      cs.add('resource-2', 'group-1')
+      cs.add('resource-3', 'group-1', 'group-2')
+      cs.add('resource-4', 'group-2')
+      cs.calculate_similarity
+      expected = [
+        {resource: 'resource-3', similarity: 1.0},
+        {resource: 'resource-4', similarity: 0.667},
+        {resource: 'resource-2', similarity: 0.667}
+      ]
+      assert_equal expected[0..0], cs.similar_to('resource-1', 1)
+      assert_equal expected[0..1], cs.similar_to('resource-1', 2)
+      assert_equal expected, cs.similar_to('resource-1', 3)
+      assert_equal expected, cs.similar_to('resource-1', 99)
+    end
+
     def test_recommends_when_added_with_scores
       redis = Redis.new(db: 15)
       redis.flushdb
@@ -254,6 +275,29 @@ module Commendo
 
     end
 
+    def test_filters_include_by_tag_collection_and_limit
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      ts = TagSet.new(redis, 'CommendoTests:tags')
+      cs = ContentSet.new(redis, 'CommendoTests', ts)
+      (3..23).each do |group|
+        (3..23).each do |res|
+          cs.add(res, group) if res % group == 0
+          ts.add(res, 'mod3') if res.modulo(3).zero?
+          ts.add(res, 'mod4') if res.modulo(4).zero?
+          ts.add(res, 'mod5') if res.modulo(5).zero?
+        end
+      end
+      cs.calculate_similarity
+
+      actual = cs.filtered_similar_to(10, include: ['mod5'], limit: 2)
+      assert_equal 2, actual.length
+      assert contains_resource('5', actual)
+      #assert contains_resource('15', actual)
+      assert contains_resource('20', actual)
+
+    end
+
     def test_filters_exclude_by_tag_collection
       redis = Redis.new(db: 15)
       redis.flushdb

data/test/tag_set_test.rb

@@ -64,18 +64,63 @@ module Commendo
       ts.set(1, 'foo', 'bar', 'baz')
       ts.set(2, 'qux', 'qip')
 
-      assert ts.matches(1, 'foo')
-      assert ts.matches(1, 'bar', 'baz')
-      assert ts.matches(1, 'bar', 'baz', 'foo')
-      refute ts.matches(1, 'qux')
-      refute ts.matches(1, 'qip')
-
-      refute ts.matches(2, 'foo')
-      refute ts.matches(2, 'bar', 'baz')
-      refute ts.matches(2, 'bar', 'baz', 'foo')
-      assert ts.matches(2, 'qux', 'qip')
-      assert ts.matches(2, 'qux')
-      assert ts.matches(2, 'qip')
+      assert ts.matches(1, ['foo'])
+      assert ts.matches(1, ['bar', 'baz'])
+      assert ts.matches(1, ['bar', 'baz', 'foo'])
+      refute ts.matches(1, ['qux'])
+      refute ts.matches(1, ['qip'])
+
+      refute ts.matches(2, ['foo'])
+      refute ts.matches(2, ['bar', 'baz'])
+      refute ts.matches(2, ['bar', 'baz', 'foo'])
+      assert ts.matches(2, ['qux', 'qip'])
+      assert ts.matches(2, ['qux'])
+      assert ts.matches(2, ['qip'])
+    end
+
+    def test_matches_exclude_tags
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      ts = TagSet.new(redis, 'TagSetTest')
+      ts.set(1, 'foo', 'bar', 'baz')
+      ts.set(2, 'qux', 'qip')
+
+      refute ts.matches(1, nil, ['foo'])
+      refute ts.matches(1, [], ['foo'])
+      refute ts.matches(1, [], ['bar', 'baz'])
+      refute ts.matches(1, [], ['bar', 'baz', 'foo'])
+      assert ts.matches(1, [], ['qux'])
+      assert ts.matches(1, [], ['qip'])
+
+      assert ts.matches(2, nil, ['foo'])
+      assert ts.matches(2, [], ['foo'])
+      assert ts.matches(2, [], ['bar', 'baz'])
+      assert ts.matches(2, [], ['bar', 'baz', 'foo'])
+      refute ts.matches(2, [], ['qux', 'qip'])
+      refute ts.matches(2, [], ['qux'])
+      refute ts.matches(2, [], ['qip'])
+    end
+
+    def test_matches_include_and_exclude_tags
+      redis = Redis.new(db: 15)
+      redis.flushdb
+      ts = TagSet.new(redis, 'TagSetTest')
+      ts.set(1, 'foo', 'bar', 'baz')
+      ts.set(2, 'qux', 'qip')
+
+      refute ts.matches(1, ['foo'], ['bar'])
+      refute ts.matches(1, ['bar'], ['foo'])
+
+      assert ts.matches(1, ['foo'], [])
+      assert ts.matches(1, ['foo'], nil)
+      assert ts.matches(1, ['foo'], ['qux'])
+
+      assert ts.matches(2, ['qip'], ['foo'])
+      assert ts.matches(2, ['qux'], ['bar', 'baz'])
+      assert ts.matches(2, ['qip'], ['bar', 'baz', 'foo'])
+      refute ts.matches(2, ['qip'], ['qux', 'qip'])
+      refute ts.matches(2, ['qip'], ['qux'])
+      refute ts.matches(2, ['qux'], ['qip'])
     end
 
   end

data/test/weighted_group_test.rb

@@ -50,6 +50,28 @@ module Commendo
       assert_equal expected, weighted_group.similar_to(18)
     end
 
+    def test_calls_each_content_set_with_limits
+      weighted_group = WeightedGroup.new(
+        @redis,
+        'CommendoTests:WeightedGroup',
+        {cs: @cs1, weight: 1.0},
+        {cs: @cs2, weight: 10.0},
+        {cs: @cs3, weight: 100.0}
+      )
+      expected = [
+        {resource: '6', similarity: 74.037},
+        {resource: '12', similarity: 55.5},
+        {resource: '9', similarity: 6.67},
+        {resource: '3', similarity: 4.0},
+        {resource: '21', similarity: 2.86},
+        {resource: '15', similarity: 2.86}
+      ]
+      assert_equal expected[0..0], weighted_group.similar_to(18, 1)
+      assert_equal expected[0..2], weighted_group.similar_to(18, 3)
+      assert_equal expected, weighted_group.similar_to(18, 6)
+      assert_equal expected, weighted_group.similar_to(18, 99)
+    end
+
     def test_filters_include_recommendations
       weighted_group = WeightedGroup.new(
         @redis,
@@ -98,6 +120,26 @@ module Commendo
       assert_equal expected, weighted_group.filtered_similar_to(8, include: ['mod4'], exclude: ['mod5'])
     end
 
+    def test_filters_include_and_exclude_recommendations_and_limits
+      weighted_group = WeightedGroup.new(
+        @redis,
+        'CommendoTests:WeightedGroup',
+        {cs: @cs1, weight: 100.0},
+        {cs: @cs2, weight: 10.0},
+        {cs: @cs3, weight: 1.0}
+      )
+      expected = [
+        {resource: '16', similarity: 80.0},
+        {resource: '4', similarity: 66.7},
+        {resource: '12', similarity: 33.3}
+      ]
+      weighted_group.tag_set = @tag_set
+      assert_equal expected[0..0], weighted_group.filtered_similar_to(8, include: ['mod4'], exclude: ['mod5'], limit: 1)
+      assert_equal expected[0..1], weighted_group.filtered_similar_to(8, include: ['mod4'], exclude: ['mod5'], limit: 2)
+      assert_equal expected, weighted_group.filtered_similar_to(8, include: ['mod4'], exclude: ['mod5'], limit: 3)
+      assert_equal expected, weighted_group.filtered_similar_to(8, include: ['mod4'], exclude: ['mod5'], limit: 99)
+    end
+
     def test_similar_to_mutliple_items
       weighted_group = WeightedGroup.new(
         @redis,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: commendo
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 1.0.0
 platform: ruby
 authors:
 - Rob Styles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-11 00:00:00.000000000 Z
+date: 2014-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -24,6 +24,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement