commendo 1.2.4 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/bin/commendo-create-mysql-db +3 -0
  4. data/bin/commendo-create.sql +99 -0
  5. data/bin/commendo-load-tsv +11 -5
  6. data/bin/commendo-load-tsv-mysql.rb +43 -0
  7. data/bin/commendo-time-mysql.rb +31 -0
  8. data/commendo.gemspec +4 -2
  9. data/lib/commendo.rb +24 -0
  10. data/lib/commendo/configuration.rb +25 -0
  11. data/lib/commendo/content_set.rb +13 -182
  12. data/lib/commendo/mysql-backed/content_set.rb +152 -0
  13. data/lib/commendo/mysql-backed/tag_set.rb +81 -0
  14. data/lib/commendo/mysql-backed/weighted_group.rb +40 -0
  15. data/lib/commendo/redis-backed/content_set.rb +194 -0
  16. data/lib/commendo/{pair_comparison.lua → redis-backed/pair_comparison.lua} +0 -0
  17. data/lib/commendo/{similarity.lua → redis-backed/similarity.lua} +0 -0
  18. data/lib/commendo/redis-backed/tag_set.rb +54 -0
  19. data/lib/commendo/redis-backed/weighted_group.rb +54 -0
  20. data/lib/commendo/tag_set.rb +6 -42
  21. data/lib/commendo/version.rb +1 -1
  22. data/lib/commendo/weighted_group.rb +7 -41
  23. data/lib/mysql2/client.rb +17 -0
  24. data/model 2.mwb +0 -0
  25. data/sql_model.mwb +0 -0
  26. data/test/configuration_test.rb +71 -0
  27. data/test/mysql_content_set_test.rb +40 -0
  28. data/test/mysql_tag_set_test.rb +34 -0
  29. data/test/mysql_weighted_group_test.rb +54 -0
  30. data/test/redis_content_set_test.rb +57 -0
  31. data/test/redis_tag_set_test.rb +31 -0
  32. data/test/redis_weighted_group_test.rb +49 -0
  33. data/test/tests_for_content_sets.rb +379 -0
  34. data/test/tests_for_tag_sets.rb +130 -0
  35. data/test/tests_for_weighted_groups.rb +106 -0
  36. metadata +72 -12
  37. data/test/content_set_test.rb +0 -408
  38. data/test/tag_set_test.rb +0 -128
  39. data/test/weighted_group_test.rb +0 -191
@@ -0,0 +1,54 @@
1
+ module Commendo
2
+ module RedisBacked
3
+ class TagSet
4
+
5
+ attr_accessor :redis, :key_base
6
+
7
+ def initialize(key_base)
8
+ @redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
9
+ @key_base = key_base
10
+ end
11
+
12
+ def empty?
13
+ cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
14
+ cursor.to_i == 0 && keys.empty?
15
+ end
16
+
17
+ def get(resource)
18
+ redis.smembers(resource_key(resource)).sort
19
+ end
20
+
21
+ def add(resource, *tags)
22
+ redis.sadd(resource_key(resource), tags) unless tags.empty?
23
+ end
24
+
25
+ def set(resource, *tags)
26
+ delete(resource)
27
+ add(resource, *tags)
28
+ end
29
+
30
+ def matches(resource, include, exclude = [])
31
+ resource_tags = get(resource)
32
+ can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
33
+ should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
34
+ return can_include && !should_exclude
35
+ end
36
+
37
+ def delete(resource, *tags)
38
+ if tags.empty?
39
+ redis.del(resource_key(resource))
40
+ else
41
+ redis.srem(resource_key(resource), tags)
42
+ end
43
+ end
44
+
45
+ private
46
+
47
+ def resource_key(resource)
48
+ "#{key_base}:#{resource}"
49
+ end
50
+
51
+ end
52
+ end
53
+ end
54
+
@@ -0,0 +1,54 @@
1
+ module Commendo
2
+ module RedisBacked
3
+
4
+ class WeightedGroup
5
+
6
+ attr_accessor :content_sets, :redis, :key_base, :tag_set
7
+
8
+ def initialize(key_base, *content_sets)
9
+ @redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
10
+ @key_base = key_base
11
+ @content_sets = content_sets
12
+ end
13
+
14
+ def similar_to(resource, limit = 0)
15
+ finish = limit -1
16
+ resources = resource.kind_of?(Array) ? resource : [resource]
17
+ keys = []
18
+ weights = []
19
+ content_sets.each do |cs|
20
+ resources.each do |resource|
21
+ keys << cs[:cs].similarity_key(resource)
22
+ weights << cs[:weight]
23
+ end
24
+ end
25
+ tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
26
+ redis.zunionstore(tmp_key, keys, weights: weights)
27
+ similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
28
+ redis.del(tmp_key)
29
+
30
+ similar_resources.map do |resource|
31
+ {resource: resource[0], similarity: resource[1].to_f.round(3)}
32
+ end
33
+
34
+ end
35
+
36
+ def filtered_similar_to(resource, options = {})
37
+ if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
38
+ return similar_to(resource, options[:limit] || 0)
39
+ else
40
+ similar = similar_to(resource)
41
+ limit = options[:limit] || similar.length
42
+ filtered = []
43
+ similar.each do |s|
44
+ return filtered if filtered.length >= limit
45
+ filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
46
+ end
47
+ return filtered
48
+ end
49
+ end
50
+
51
+ end
52
+
53
+ end
54
+ end
@@ -1,51 +1,15 @@
1
1
  module Commendo
2
-
3
2
  class TagSet
3
+ extend Forwardable
4
4
 
5
- attr_accessor :redis, :key_base
6
-
7
- def initialize(redis, key_base)
8
- @redis, @key_base = redis, key_base
9
- end
10
-
11
- def empty?
12
- cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
13
- cursor.to_i == 0 && keys.empty?
14
- end
15
-
16
- def get(resource)
17
- redis.smembers(resource_key(resource)).sort
18
- end
5
+ def_delegators :@backend, :empty?, :get, :add, :set, :matches, :delete, :redis, :key_base
19
6
 
20
- def add(resource, *tags)
21
- redis.sadd(resource_key(resource), tags) unless tags.empty?
7
+ def initialize(args)
8
+ @backend = RedisBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :redis
9
+ @backend = MySqlBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :mysql
10
+ raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
22
11
  end
23
12
 
24
- def set(resource, *tags)
25
- delete(resource)
26
- add(resource, *tags)
27
- end
28
-
29
- def matches(resource, include, exclude = [])
30
- resource_tags = get(resource)
31
- can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
32
- should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
33
- return can_include && !should_exclude
34
- end
35
-
36
- def delete(resource, *tags)
37
- if tags.empty?
38
- redis.del(resource_key(resource))
39
- else
40
- redis.srem(resource_key(resource), tags)
41
- end
42
- end
43
-
44
- private
45
-
46
- def resource_key(resource)
47
- "#{key_base}:#{resource}"
48
- end
49
13
 
50
14
  end
51
15
  end
@@ -1,3 +1,3 @@
1
1
  module Commendo
2
- VERSION = '1.2.4'
2
+ VERSION = '2.0.0'
3
3
  end
@@ -1,50 +1,16 @@
1
1
  module Commendo
2
2
 
3
3
  class WeightedGroup
4
+ extend Forwardable
4
5
 
5
- attr_accessor :content_sets, :redis, :key_base, :tag_set
6
-
7
- def initialize(redis, key_base, *content_sets)
8
- @content_sets, @redis, @key_base = content_sets, redis, key_base
9
- end
10
-
11
- def similar_to(resource, limit = 0)
12
- finish = limit -1
13
- resources = resource.kind_of?(Array) ? resource : [resource]
14
- keys = []
15
- weights = []
16
- content_sets.each do |cs|
17
- resources.each do |resource|
18
- keys << cs[:cs].similarity_key(resource)
19
- weights << cs[:weight]
20
- end
21
- end
22
- tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
23
- redis.zunionstore(tmp_key, keys, weights: weights)
24
- similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
25
- redis.del(tmp_key)
26
-
27
- similar_resources.map do |resource|
28
- {resource: resource[0], similarity: resource[1].to_f.round(3)}
29
- end
30
-
6
+ def initialize(args)
7
+ @backend = RedisBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :redis
8
+ @backend = MySqlBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :mysql
9
+ raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
31
10
  end
32
11
 
33
- def filtered_similar_to(resource, options = {})
34
- if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
35
- return similar_to(resource, options[:limit] || 0)
36
- else
37
- similar = similar_to(resource)
38
- limit = options[:limit] || similar.length
39
- filtered = []
40
- similar.each do |s|
41
- return filtered if filtered.length >= limit
42
- filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
43
- end
44
- return filtered
45
- end
46
- end
12
+ def_delegators :@backend, :similar_to, :filtered_similar_to, :content_sets, :redis, :key_base, :tag_set, :tag_set=
47
13
 
48
14
  end
49
15
 
50
- end
16
+ end
@@ -0,0 +1,17 @@
1
+ module Mysql2
2
+ class Client
3
+ def transaction(&block)
4
+ raise ArgumentError, 'No block was given' unless block_given?
5
+ begin
6
+ query('BEGIN')
7
+ yield(self)
8
+ query('COMMIT')
9
+ return true # Successful Transaction
10
+ rescue
11
+ query('ROLLBACK')
12
+ raise
13
+ return false # Failed Transaction
14
+ end
15
+ end
16
+ end
17
+ end
Binary file
Binary file
@@ -0,0 +1,71 @@
1
+ gem 'minitest'
2
+ require 'minitest/autorun'
3
+ require 'minitest/pride'
4
+ require 'minitest/mock'
5
+ require 'mocha/setup'
6
+ require 'commendo'
7
+
8
+ module Commendo
9
+
10
+ class ConfigurationTest < Minitest::Test
11
+
12
+ def setup
13
+ Commendo.config = nil
14
+ end
15
+
16
+ def test_default_values_on_configuration_class
17
+ assert_equal :redis, Commendo::Configuration.new.backend
18
+ assert_equal 'localhost', Commendo::Configuration.new.host
19
+ assert_equal 6379, Commendo::Configuration.new.port
20
+ assert_equal 15, Commendo::Configuration.new.database
21
+ assert_nil Commendo::Configuration.new.username
22
+ assert_nil Commendo::Configuration.new.password
23
+ end
24
+
25
+ def test_returns_same_object_each_time
26
+ config1 = Commendo.config
27
+ config2 = Commendo.config
28
+ config3 = Commendo.config
29
+ assert config1.equal? config2
30
+ assert config2.equal? config3
31
+ end
32
+
33
+ def test_config_returns_default_configuration
34
+ config = Commendo.config
35
+ assert config.is_a? Commendo::Configuration
36
+ assert_equal :redis, config.backend
37
+ assert_equal 'localhost', config.host
38
+ assert_equal 6379, config.port
39
+ assert_equal 15, config.database
40
+ assert_nil config.username
41
+ assert_nil config.password
42
+ end
43
+
44
+ def test_configure_stores_settings
45
+ config = Commendo.config do |config|
46
+ config.backend = :mysql
47
+ config.host = 'mysql.example.com'
48
+ config.port = 9999
49
+ config.database = 'some_mysql_db'
50
+ config.username = 'root'
51
+ config.password = 'Passw0rd!!'
52
+ end
53
+
54
+ assert config.is_a? Commendo::Configuration
55
+ assert_equal :mysql, config.backend
56
+ assert_equal 'mysql.example.com', config.host
57
+ assert_equal 9999, config.port
58
+ assert_equal 'some_mysql_db', config.database
59
+ assert_equal 'root', config.username
60
+ assert_equal 'Passw0rd!!', config.password
61
+ end
62
+
63
+ def test_to_hash
64
+ expected = {backend: :redis, host: 'localhost', port: 6379, database: 15, username: nil, password: nil}
65
+ actual = Commendo::Configuration.new.to_hash
66
+ assert_equal expected, actual
67
+ end
68
+
69
+ end
70
+
71
+ end
@@ -0,0 +1,40 @@
1
+ require_relative 'tests_for_content_sets.rb'
2
+ gem 'minitest'
3
+ require 'minitest/autorun'
4
+ require 'minitest/pride'
5
+ require 'minitest/mock'
6
+ require 'mocha/setup'
7
+ require 'commendo'
8
+
9
+ module Commendo
10
+
11
+ class MySqlContentSetTest < Minitest::Test
12
+
13
+ def setup
14
+ Commendo.config do |config|
15
+ config.backend = :mysql
16
+ config.host = 'localhost'
17
+ config.port = 3306
18
+ config.database = 'commendo_test'
19
+ config.username = 'commendo'
20
+ config.password = 'commendo123'
21
+ end
22
+ client = Mysql2::Client.new(Commendo.config.to_hash)
23
+ %w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
24
+ @key_base = 'CommendoTests'
25
+ @cs = ContentSet.new(key_base: @key_base)
26
+ end
27
+
28
+ def create_tag_set(kb)
29
+ Commendo::TagSet.new(key_base: kb)
30
+ end
31
+
32
+ def create_content_set(key_base, ts = nil)
33
+ Commendo::ContentSet.new(key_base: key_base, tag_set: ts)
34
+ end
35
+
36
+ include TestsForContentSets
37
+
38
+ end
39
+
40
+ end
@@ -0,0 +1,34 @@
1
+ require_relative 'tests_for_tag_sets'
2
+ gem 'minitest'
3
+ require 'minitest/autorun'
4
+ require 'minitest/pride'
5
+ require 'minitest/mock'
6
+ require 'mocha/setup'
7
+ require 'commendo'
8
+
9
+ module Commendo
10
+
11
+ class MySqlTagSetTest < Minitest::Test
12
+
13
+ def setup
14
+ Commendo.config do |config|
15
+ config.backend = :mysql
16
+ config.host = 'localhost'
17
+ config.port = 3306
18
+ config.database = 'commendo_test'
19
+ config.username = 'commendo'
20
+ config.password = 'commendo123'
21
+ end
22
+ client = Mysql2::Client.new(Commendo.config.to_hash)
23
+ %w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
24
+ @ts = TagSet.new(key_base: 'TagSetTest')
25
+ end
26
+
27
+ def create_tag_set(kb)
28
+ Commendo::TagSet.new(key_base: kb)
29
+ end
30
+
31
+ include TestsForTagSets
32
+
33
+ end
34
+ end
@@ -0,0 +1,54 @@
1
+ require_relative 'tests_for_weighted_groups'
2
+ gem 'minitest'
3
+ require 'minitest/autorun'
4
+ require 'minitest/pride'
5
+ require 'minitest/mock'
6
+ require 'mocha/setup'
7
+ require 'commendo'
8
+
9
+ module Commendo
10
+
11
+ class MySqlWeightedGroupTest < Minitest::Test
12
+
13
+ def setup
14
+ super
15
+ Commendo.config do |config|
16
+ config.backend = :mysql
17
+ config.host = 'localhost'
18
+ config.port = 3306
19
+ config.database = 'commendo_test'
20
+ config.username = 'commendo'
21
+ config.password = 'commendo123'
22
+ end
23
+ client = Mysql2::Client.new(Commendo.config.to_hash)
24
+ %w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
25
+ @tag_set = TagSet.new(key_base: 'CommendoTests:Tags')
26
+ @cs1 = ContentSet.new(key_base: 'CommendoTests:ContentSet1', tag_set: @tag_set)
27
+ @cs2 = ContentSet.new(key_base: 'CommendoTests:ContentSet2', tag_set: @tag_set)
28
+ @cs3 = ContentSet.new(key_base: 'CommendoTests:ContentSet3', tag_set: @tag_set)
29
+ (3..23).each do |group|
30
+ (3..23).each do |res|
31
+ @cs1.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(2).zero?
32
+ @cs2.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(3).zero?
33
+ @cs3.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(6).zero?
34
+ @tag_set.add(res, 'mod3') if res.modulo(3).zero?
35
+ @tag_set.add(res, 'mod4') if res.modulo(4).zero?
36
+ @tag_set.add(res, 'mod5') if res.modulo(5).zero?
37
+ @tag_set.add(res, 'mod7') if res.modulo(7).zero?
38
+ end
39
+ end
40
+ [@cs1, @cs2, @cs3].each { |cs| cs.calculate_similarity }
41
+ @weighted_group = Commendo::WeightedGroup.new(key_base: 'CommendoTests:WeightedGroup',
42
+ content_sets: [{cs: @cs1, weight: 1.0},
43
+ {cs: @cs2, weight: 10.0},
44
+ {cs: @cs3, weight: 100.0}]
45
+ )
46
+ end
47
+
48
+ include TestsForWeightedGroups
49
+
50
+ end
51
+
52
+ end
53
+
54
+