commendo 1.2.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/bin/commendo-create-mysql-db +3 -0
  4. data/bin/commendo-create.sql +99 -0
  5. data/bin/commendo-load-tsv +11 -5
  6. data/bin/commendo-load-tsv-mysql.rb +43 -0
  7. data/bin/commendo-time-mysql.rb +31 -0
  8. data/commendo.gemspec +4 -2
  9. data/lib/commendo.rb +24 -0
  10. data/lib/commendo/configuration.rb +25 -0
  11. data/lib/commendo/content_set.rb +13 -182
  12. data/lib/commendo/mysql-backed/content_set.rb +152 -0
  13. data/lib/commendo/mysql-backed/tag_set.rb +81 -0
  14. data/lib/commendo/mysql-backed/weighted_group.rb +40 -0
  15. data/lib/commendo/redis-backed/content_set.rb +194 -0
  16. data/lib/commendo/{pair_comparison.lua → redis-backed/pair_comparison.lua} +0 -0
  17. data/lib/commendo/{similarity.lua → redis-backed/similarity.lua} +0 -0
  18. data/lib/commendo/redis-backed/tag_set.rb +54 -0
  19. data/lib/commendo/redis-backed/weighted_group.rb +54 -0
  20. data/lib/commendo/tag_set.rb +6 -42
  21. data/lib/commendo/version.rb +1 -1
  22. data/lib/commendo/weighted_group.rb +7 -41
  23. data/lib/mysql2/client.rb +17 -0
  24. data/model 2.mwb +0 -0
  25. data/sql_model.mwb +0 -0
  26. data/test/configuration_test.rb +71 -0
  27. data/test/mysql_content_set_test.rb +40 -0
  28. data/test/mysql_tag_set_test.rb +34 -0
  29. data/test/mysql_weighted_group_test.rb +54 -0
  30. data/test/redis_content_set_test.rb +57 -0
  31. data/test/redis_tag_set_test.rb +31 -0
  32. data/test/redis_weighted_group_test.rb +49 -0
  33. data/test/tests_for_content_sets.rb +379 -0
  34. data/test/tests_for_tag_sets.rb +130 -0
  35. data/test/tests_for_weighted_groups.rb +106 -0
  36. metadata +72 -12
  37. data/test/content_set_test.rb +0 -408
  38. data/test/tag_set_test.rb +0 -128
  39. data/test/weighted_group_test.rb +0 -191
@@ -0,0 +1,54 @@
1
+ module Commendo
2
+ module RedisBacked
3
+ class TagSet
4
+
5
+ attr_accessor :redis, :key_base
6
+
7
+ def initialize(key_base)
8
+ @redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
9
+ @key_base = key_base
10
+ end
11
+
12
+ def empty?
13
+ cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
14
+ cursor.to_i == 0 && keys.empty?
15
+ end
16
+
17
+ def get(resource)
18
+ redis.smembers(resource_key(resource)).sort
19
+ end
20
+
21
+ def add(resource, *tags)
22
+ redis.sadd(resource_key(resource), tags) unless tags.empty?
23
+ end
24
+
25
+ def set(resource, *tags)
26
+ delete(resource)
27
+ add(resource, *tags)
28
+ end
29
+
30
+ def matches(resource, include, exclude = [])
31
+ resource_tags = get(resource)
32
+ can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
33
+ should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
34
+ return can_include && !should_exclude
35
+ end
36
+
37
+ def delete(resource, *tags)
38
+ if tags.empty?
39
+ redis.del(resource_key(resource))
40
+ else
41
+ redis.srem(resource_key(resource), tags)
42
+ end
43
+ end
44
+
45
+ private
46
+
47
+ def resource_key(resource)
48
+ "#{key_base}:#{resource}"
49
+ end
50
+
51
+ end
52
+ end
53
+ end
54
+
@@ -0,0 +1,54 @@
1
+ module Commendo
2
+ module RedisBacked
3
+
4
+ class WeightedGroup
5
+
6
+ attr_accessor :content_sets, :redis, :key_base, :tag_set
7
+
8
+ def initialize(key_base, *content_sets)
9
+ @redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
10
+ @key_base = key_base
11
+ @content_sets = content_sets
12
+ end
13
+
14
+ def similar_to(resource, limit = 0)
15
+ finish = limit -1
16
+ resources = resource.kind_of?(Array) ? resource : [resource]
17
+ keys = []
18
+ weights = []
19
+ content_sets.each do |cs|
20
+ resources.each do |resource|
21
+ keys << cs[:cs].similarity_key(resource)
22
+ weights << cs[:weight]
23
+ end
24
+ end
25
+ tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
26
+ redis.zunionstore(tmp_key, keys, weights: weights)
27
+ similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
28
+ redis.del(tmp_key)
29
+
30
+ similar_resources.map do |resource|
31
+ {resource: resource[0], similarity: resource[1].to_f.round(3)}
32
+ end
33
+
34
+ end
35
+
36
+ def filtered_similar_to(resource, options = {})
37
+ if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
38
+ return similar_to(resource, options[:limit] || 0)
39
+ else
40
+ similar = similar_to(resource)
41
+ limit = options[:limit] || similar.length
42
+ filtered = []
43
+ similar.each do |s|
44
+ return filtered if filtered.length >= limit
45
+ filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
46
+ end
47
+ return filtered
48
+ end
49
+ end
50
+
51
+ end
52
+
53
+ end
54
+ end
@@ -1,51 +1,15 @@
1
1
  module Commendo
2
-
3
2
  class TagSet
3
+ extend Forwardable
4
4
 
5
- attr_accessor :redis, :key_base
6
-
7
- def initialize(redis, key_base)
8
- @redis, @key_base = redis, key_base
9
- end
10
-
11
- def empty?
12
- cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
13
- cursor.to_i == 0 && keys.empty?
14
- end
15
-
16
- def get(resource)
17
- redis.smembers(resource_key(resource)).sort
18
- end
5
+ def_delegators :@backend, :empty?, :get, :add, :set, :matches, :delete, :redis, :key_base
19
6
 
20
- def add(resource, *tags)
21
- redis.sadd(resource_key(resource), tags) unless tags.empty?
7
+ def initialize(args)
8
+ @backend = RedisBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :redis
9
+ @backend = MySqlBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :mysql
10
+ raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
22
11
  end
23
12
 
24
- def set(resource, *tags)
25
- delete(resource)
26
- add(resource, *tags)
27
- end
28
-
29
- def matches(resource, include, exclude = [])
30
- resource_tags = get(resource)
31
- can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
32
- should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
33
- return can_include && !should_exclude
34
- end
35
-
36
- def delete(resource, *tags)
37
- if tags.empty?
38
- redis.del(resource_key(resource))
39
- else
40
- redis.srem(resource_key(resource), tags)
41
- end
42
- end
43
-
44
- private
45
-
46
- def resource_key(resource)
47
- "#{key_base}:#{resource}"
48
- end
49
13
 
50
14
  end
51
15
  end
@@ -1,3 +1,3 @@
1
1
  module Commendo
2
- VERSION = '1.2.4'
2
+ VERSION = '2.0.0'
3
3
  end
@@ -1,50 +1,16 @@
1
1
  module Commendo
2
2
 
3
3
  class WeightedGroup
4
+ extend Forwardable
4
5
 
5
- attr_accessor :content_sets, :redis, :key_base, :tag_set
6
-
7
- def initialize(redis, key_base, *content_sets)
8
- @content_sets, @redis, @key_base = content_sets, redis, key_base
9
- end
10
-
11
- def similar_to(resource, limit = 0)
12
- finish = limit -1
13
- resources = resource.kind_of?(Array) ? resource : [resource]
14
- keys = []
15
- weights = []
16
- content_sets.each do |cs|
17
- resources.each do |resource|
18
- keys << cs[:cs].similarity_key(resource)
19
- weights << cs[:weight]
20
- end
21
- end
22
- tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
23
- redis.zunionstore(tmp_key, keys, weights: weights)
24
- similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
25
- redis.del(tmp_key)
26
-
27
- similar_resources.map do |resource|
28
- {resource: resource[0], similarity: resource[1].to_f.round(3)}
29
- end
30
-
6
+ def initialize(args)
7
+ @backend = RedisBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :redis
8
+ @backend = MySqlBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :mysql
9
+ raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
31
10
  end
32
11
 
33
- def filtered_similar_to(resource, options = {})
34
- if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
35
- return similar_to(resource, options[:limit] || 0)
36
- else
37
- similar = similar_to(resource)
38
- limit = options[:limit] || similar.length
39
- filtered = []
40
- similar.each do |s|
41
- return filtered if filtered.length >= limit
42
- filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
43
- end
44
- return filtered
45
- end
46
- end
12
+ def_delegators :@backend, :similar_to, :filtered_similar_to, :content_sets, :redis, :key_base, :tag_set, :tag_set=
47
13
 
48
14
  end
49
15
 
50
- end
16
+ end
@@ -0,0 +1,17 @@
1
+ module Mysql2
2
+ class Client
3
+ def transaction(&block)
4
+ raise ArgumentError, 'No block was given' unless block_given?
5
+ begin
6
+ query('BEGIN')
7
+ yield(self)
8
+ query('COMMIT')
9
+ return true # Successful Transaction
10
+ rescue
11
+ query('ROLLBACK')
12
+ raise
13
+ return false # Failed Transaction
14
+ end
15
+ end
16
+ end
17
+ end
Binary file
Binary file
@@ -0,0 +1,71 @@
1
+ gem 'minitest'
2
+ require 'minitest/autorun'
3
+ require 'minitest/pride'
4
+ require 'minitest/mock'
5
+ require 'mocha/setup'
6
+ require 'commendo'
7
+
8
+ module Commendo
9
+
10
+ class ConfigurationTest < Minitest::Test
11
+
12
+ def setup
13
+ Commendo.config = nil
14
+ end
15
+
16
+ def test_default_values_on_configuration_class
17
+ assert_equal :redis, Commendo::Configuration.new.backend
18
+ assert_equal 'localhost', Commendo::Configuration.new.host
19
+ assert_equal 6379, Commendo::Configuration.new.port
20
+ assert_equal 15, Commendo::Configuration.new.database
21
+ assert_nil Commendo::Configuration.new.username
22
+ assert_nil Commendo::Configuration.new.password
23
+ end
24
+
25
+ def test_returns_same_object_each_time
26
+ config1 = Commendo.config
27
+ config2 = Commendo.config
28
+ config3 = Commendo.config
29
+ assert config1.equal? config2
30
+ assert config2.equal? config3
31
+ end
32
+
33
+ def test_config_returns_default_configuration
34
+ config = Commendo.config
35
+ assert config.is_a? Commendo::Configuration
36
+ assert_equal :redis, config.backend
37
+ assert_equal 'localhost', config.host
38
+ assert_equal 6379, config.port
39
+ assert_equal 15, config.database
40
+ assert_nil config.username
41
+ assert_nil config.password
42
+ end
43
+
44
+ def test_configure_stores_settings
45
+ config = Commendo.config do |config|
46
+ config.backend = :mysql
47
+ config.host = 'mysql.example.com'
48
+ config.port = 9999
49
+ config.database = 'some_mysql_db'
50
+ config.username = 'root'
51
+ config.password = 'Passw0rd!!'
52
+ end
53
+
54
+ assert config.is_a? Commendo::Configuration
55
+ assert_equal :mysql, config.backend
56
+ assert_equal 'mysql.example.com', config.host
57
+ assert_equal 9999, config.port
58
+ assert_equal 'some_mysql_db', config.database
59
+ assert_equal 'root', config.username
60
+ assert_equal 'Passw0rd!!', config.password
61
+ end
62
+
63
+ def test_to_hash
64
+ expected = {backend: :redis, host: 'localhost', port: 6379, database: 15, username: nil, password: nil}
65
+ actual = Commendo::Configuration.new.to_hash
66
+ assert_equal expected, actual
67
+ end
68
+
69
+ end
70
+
71
+ end
@@ -0,0 +1,40 @@
1
+ require_relative 'tests_for_content_sets.rb'
2
+ gem 'minitest'
3
+ require 'minitest/autorun'
4
+ require 'minitest/pride'
5
+ require 'minitest/mock'
6
+ require 'mocha/setup'
7
+ require 'commendo'
8
+
9
+ module Commendo
10
+
11
+ class MySqlContentSetTest < Minitest::Test
12
+
13
+ def setup
14
+ Commendo.config do |config|
15
+ config.backend = :mysql
16
+ config.host = 'localhost'
17
+ config.port = 3306
18
+ config.database = 'commendo_test'
19
+ config.username = 'commendo'
20
+ config.password = 'commendo123'
21
+ end
22
+ client = Mysql2::Client.new(Commendo.config.to_hash)
23
+ %w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
24
+ @key_base = 'CommendoTests'
25
+ @cs = ContentSet.new(key_base: @key_base)
26
+ end
27
+
28
+ def create_tag_set(kb)
29
+ Commendo::TagSet.new(key_base: kb)
30
+ end
31
+
32
+ def create_content_set(key_base, ts = nil)
33
+ Commendo::ContentSet.new(key_base: key_base, tag_set: ts)
34
+ end
35
+
36
+ include TestsForContentSets
37
+
38
+ end
39
+
40
+ end
@@ -0,0 +1,34 @@
1
+ require_relative 'tests_for_tag_sets'
2
+ gem 'minitest'
3
+ require 'minitest/autorun'
4
+ require 'minitest/pride'
5
+ require 'minitest/mock'
6
+ require 'mocha/setup'
7
+ require 'commendo'
8
+
9
+ module Commendo
10
+
11
+ class MySqlTagSetTest < Minitest::Test
12
+
13
+ def setup
14
+ Commendo.config do |config|
15
+ config.backend = :mysql
16
+ config.host = 'localhost'
17
+ config.port = 3306
18
+ config.database = 'commendo_test'
19
+ config.username = 'commendo'
20
+ config.password = 'commendo123'
21
+ end
22
+ client = Mysql2::Client.new(Commendo.config.to_hash)
23
+ %w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
24
+ @ts = TagSet.new(key_base: 'TagSetTest')
25
+ end
26
+
27
+ def create_tag_set(kb)
28
+ Commendo::TagSet.new(key_base: kb)
29
+ end
30
+
31
+ include TestsForTagSets
32
+
33
+ end
34
+ end
@@ -0,0 +1,54 @@
1
+ require_relative 'tests_for_weighted_groups'
2
+ gem 'minitest'
3
+ require 'minitest/autorun'
4
+ require 'minitest/pride'
5
+ require 'minitest/mock'
6
+ require 'mocha/setup'
7
+ require 'commendo'
8
+
9
+ module Commendo
10
+
11
+ class MySqlWeightedGroupTest < Minitest::Test
12
+
13
+ def setup
14
+ super
15
+ Commendo.config do |config|
16
+ config.backend = :mysql
17
+ config.host = 'localhost'
18
+ config.port = 3306
19
+ config.database = 'commendo_test'
20
+ config.username = 'commendo'
21
+ config.password = 'commendo123'
22
+ end
23
+ client = Mysql2::Client.new(Commendo.config.to_hash)
24
+ %w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
25
+ @tag_set = TagSet.new(key_base: 'CommendoTests:Tags')
26
+ @cs1 = ContentSet.new(key_base: 'CommendoTests:ContentSet1', tag_set: @tag_set)
27
+ @cs2 = ContentSet.new(key_base: 'CommendoTests:ContentSet2', tag_set: @tag_set)
28
+ @cs3 = ContentSet.new(key_base: 'CommendoTests:ContentSet3', tag_set: @tag_set)
29
+ (3..23).each do |group|
30
+ (3..23).each do |res|
31
+ @cs1.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(2).zero?
32
+ @cs2.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(3).zero?
33
+ @cs3.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(6).zero?
34
+ @tag_set.add(res, 'mod3') if res.modulo(3).zero?
35
+ @tag_set.add(res, 'mod4') if res.modulo(4).zero?
36
+ @tag_set.add(res, 'mod5') if res.modulo(5).zero?
37
+ @tag_set.add(res, 'mod7') if res.modulo(7).zero?
38
+ end
39
+ end
40
+ [@cs1, @cs2, @cs3].each { |cs| cs.calculate_similarity }
41
+ @weighted_group = Commendo::WeightedGroup.new(key_base: 'CommendoTests:WeightedGroup',
42
+ content_sets: [{cs: @cs1, weight: 1.0},
43
+ {cs: @cs2, weight: 10.0},
44
+ {cs: @cs3, weight: 100.0}]
45
+ )
46
+ end
47
+
48
+ include TestsForWeightedGroups
49
+
50
+ end
51
+
52
+ end
53
+
54
+