commendo 1.2.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/bin/commendo-create-mysql-db +3 -0
- data/bin/commendo-create.sql +99 -0
- data/bin/commendo-load-tsv +11 -5
- data/bin/commendo-load-tsv-mysql.rb +43 -0
- data/bin/commendo-time-mysql.rb +31 -0
- data/commendo.gemspec +4 -2
- data/lib/commendo.rb +24 -0
- data/lib/commendo/configuration.rb +25 -0
- data/lib/commendo/content_set.rb +13 -182
- data/lib/commendo/mysql-backed/content_set.rb +152 -0
- data/lib/commendo/mysql-backed/tag_set.rb +81 -0
- data/lib/commendo/mysql-backed/weighted_group.rb +40 -0
- data/lib/commendo/redis-backed/content_set.rb +194 -0
- data/lib/commendo/{pair_comparison.lua → redis-backed/pair_comparison.lua} +0 -0
- data/lib/commendo/{similarity.lua → redis-backed/similarity.lua} +0 -0
- data/lib/commendo/redis-backed/tag_set.rb +54 -0
- data/lib/commendo/redis-backed/weighted_group.rb +54 -0
- data/lib/commendo/tag_set.rb +6 -42
- data/lib/commendo/version.rb +1 -1
- data/lib/commendo/weighted_group.rb +7 -41
- data/lib/mysql2/client.rb +17 -0
- data/model 2.mwb +0 -0
- data/sql_model.mwb +0 -0
- data/test/configuration_test.rb +71 -0
- data/test/mysql_content_set_test.rb +40 -0
- data/test/mysql_tag_set_test.rb +34 -0
- data/test/mysql_weighted_group_test.rb +54 -0
- data/test/redis_content_set_test.rb +57 -0
- data/test/redis_tag_set_test.rb +31 -0
- data/test/redis_weighted_group_test.rb +49 -0
- data/test/tests_for_content_sets.rb +379 -0
- data/test/tests_for_tag_sets.rb +130 -0
- data/test/tests_for_weighted_groups.rb +106 -0
- metadata +72 -12
- data/test/content_set_test.rb +0 -408
- data/test/tag_set_test.rb +0 -128
- data/test/weighted_group_test.rb +0 -191
File without changes
|
File without changes
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Commendo
|
2
|
+
module RedisBacked
|
3
|
+
class TagSet
|
4
|
+
|
5
|
+
attr_accessor :redis, :key_base
|
6
|
+
|
7
|
+
def initialize(key_base)
|
8
|
+
@redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
|
9
|
+
@key_base = key_base
|
10
|
+
end
|
11
|
+
|
12
|
+
def empty?
|
13
|
+
cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
|
14
|
+
cursor.to_i == 0 && keys.empty?
|
15
|
+
end
|
16
|
+
|
17
|
+
def get(resource)
|
18
|
+
redis.smembers(resource_key(resource)).sort
|
19
|
+
end
|
20
|
+
|
21
|
+
def add(resource, *tags)
|
22
|
+
redis.sadd(resource_key(resource), tags) unless tags.empty?
|
23
|
+
end
|
24
|
+
|
25
|
+
def set(resource, *tags)
|
26
|
+
delete(resource)
|
27
|
+
add(resource, *tags)
|
28
|
+
end
|
29
|
+
|
30
|
+
def matches(resource, include, exclude = [])
|
31
|
+
resource_tags = get(resource)
|
32
|
+
can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
|
33
|
+
should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
|
34
|
+
return can_include && !should_exclude
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete(resource, *tags)
|
38
|
+
if tags.empty?
|
39
|
+
redis.del(resource_key(resource))
|
40
|
+
else
|
41
|
+
redis.srem(resource_key(resource), tags)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def resource_key(resource)
|
48
|
+
"#{key_base}:#{resource}"
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Commendo
|
2
|
+
module RedisBacked
|
3
|
+
|
4
|
+
class WeightedGroup
|
5
|
+
|
6
|
+
attr_accessor :content_sets, :redis, :key_base, :tag_set
|
7
|
+
|
8
|
+
def initialize(key_base, *content_sets)
|
9
|
+
@redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
|
10
|
+
@key_base = key_base
|
11
|
+
@content_sets = content_sets
|
12
|
+
end
|
13
|
+
|
14
|
+
def similar_to(resource, limit = 0)
|
15
|
+
finish = limit -1
|
16
|
+
resources = resource.kind_of?(Array) ? resource : [resource]
|
17
|
+
keys = []
|
18
|
+
weights = []
|
19
|
+
content_sets.each do |cs|
|
20
|
+
resources.each do |resource|
|
21
|
+
keys << cs[:cs].similarity_key(resource)
|
22
|
+
weights << cs[:weight]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
|
26
|
+
redis.zunionstore(tmp_key, keys, weights: weights)
|
27
|
+
similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
|
28
|
+
redis.del(tmp_key)
|
29
|
+
|
30
|
+
similar_resources.map do |resource|
|
31
|
+
{resource: resource[0], similarity: resource[1].to_f.round(3)}
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
def filtered_similar_to(resource, options = {})
|
37
|
+
if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
|
38
|
+
return similar_to(resource, options[:limit] || 0)
|
39
|
+
else
|
40
|
+
similar = similar_to(resource)
|
41
|
+
limit = options[:limit] || similar.length
|
42
|
+
filtered = []
|
43
|
+
similar.each do |s|
|
44
|
+
return filtered if filtered.length >= limit
|
45
|
+
filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
|
46
|
+
end
|
47
|
+
return filtered
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
data/lib/commendo/tag_set.rb
CHANGED
@@ -1,51 +1,15 @@
|
|
1
1
|
module Commendo
|
2
|
-
|
3
2
|
class TagSet
|
3
|
+
extend Forwardable
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
def initialize(redis, key_base)
|
8
|
-
@redis, @key_base = redis, key_base
|
9
|
-
end
|
10
|
-
|
11
|
-
def empty?
|
12
|
-
cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
|
13
|
-
cursor.to_i == 0 && keys.empty?
|
14
|
-
end
|
15
|
-
|
16
|
-
def get(resource)
|
17
|
-
redis.smembers(resource_key(resource)).sort
|
18
|
-
end
|
5
|
+
def_delegators :@backend, :empty?, :get, :add, :set, :matches, :delete, :redis, :key_base
|
19
6
|
|
20
|
-
def
|
21
|
-
|
7
|
+
def initialize(args)
|
8
|
+
@backend = RedisBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :redis
|
9
|
+
@backend = MySqlBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :mysql
|
10
|
+
raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
|
22
11
|
end
|
23
12
|
|
24
|
-
def set(resource, *tags)
|
25
|
-
delete(resource)
|
26
|
-
add(resource, *tags)
|
27
|
-
end
|
28
|
-
|
29
|
-
def matches(resource, include, exclude = [])
|
30
|
-
resource_tags = get(resource)
|
31
|
-
can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
|
32
|
-
should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
|
33
|
-
return can_include && !should_exclude
|
34
|
-
end
|
35
|
-
|
36
|
-
def delete(resource, *tags)
|
37
|
-
if tags.empty?
|
38
|
-
redis.del(resource_key(resource))
|
39
|
-
else
|
40
|
-
redis.srem(resource_key(resource), tags)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
private
|
45
|
-
|
46
|
-
def resource_key(resource)
|
47
|
-
"#{key_base}:#{resource}"
|
48
|
-
end
|
49
13
|
|
50
14
|
end
|
51
15
|
end
|
data/lib/commendo/version.rb
CHANGED
@@ -1,50 +1,16 @@
|
|
1
1
|
module Commendo
|
2
2
|
|
3
3
|
class WeightedGroup
|
4
|
+
extend Forwardable
|
4
5
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
end
|
10
|
-
|
11
|
-
def similar_to(resource, limit = 0)
|
12
|
-
finish = limit -1
|
13
|
-
resources = resource.kind_of?(Array) ? resource : [resource]
|
14
|
-
keys = []
|
15
|
-
weights = []
|
16
|
-
content_sets.each do |cs|
|
17
|
-
resources.each do |resource|
|
18
|
-
keys << cs[:cs].similarity_key(resource)
|
19
|
-
weights << cs[:weight]
|
20
|
-
end
|
21
|
-
end
|
22
|
-
tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
|
23
|
-
redis.zunionstore(tmp_key, keys, weights: weights)
|
24
|
-
similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
|
25
|
-
redis.del(tmp_key)
|
26
|
-
|
27
|
-
similar_resources.map do |resource|
|
28
|
-
{resource: resource[0], similarity: resource[1].to_f.round(3)}
|
29
|
-
end
|
30
|
-
|
6
|
+
def initialize(args)
|
7
|
+
@backend = RedisBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :redis
|
8
|
+
@backend = MySqlBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :mysql
|
9
|
+
raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
|
31
10
|
end
|
32
11
|
|
33
|
-
|
34
|
-
if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
|
35
|
-
return similar_to(resource, options[:limit] || 0)
|
36
|
-
else
|
37
|
-
similar = similar_to(resource)
|
38
|
-
limit = options[:limit] || similar.length
|
39
|
-
filtered = []
|
40
|
-
similar.each do |s|
|
41
|
-
return filtered if filtered.length >= limit
|
42
|
-
filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
|
43
|
-
end
|
44
|
-
return filtered
|
45
|
-
end
|
46
|
-
end
|
12
|
+
def_delegators :@backend, :similar_to, :filtered_similar_to, :content_sets, :redis, :key_base, :tag_set, :tag_set=
|
47
13
|
|
48
14
|
end
|
49
15
|
|
50
|
-
end
|
16
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Mysql2
|
2
|
+
class Client
|
3
|
+
def transaction(&block)
|
4
|
+
raise ArgumentError, 'No block was given' unless block_given?
|
5
|
+
begin
|
6
|
+
query('BEGIN')
|
7
|
+
yield(self)
|
8
|
+
query('COMMIT')
|
9
|
+
return true # Successful Transaction
|
10
|
+
rescue
|
11
|
+
query('ROLLBACK')
|
12
|
+
raise
|
13
|
+
return false # Failed Transaction
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/model 2.mwb
ADDED
Binary file
|
data/sql_model.mwb
ADDED
Binary file
|
@@ -0,0 +1,71 @@
|
|
1
|
+
gem 'minitest'
|
2
|
+
require 'minitest/autorun'
|
3
|
+
require 'minitest/pride'
|
4
|
+
require 'minitest/mock'
|
5
|
+
require 'mocha/setup'
|
6
|
+
require 'commendo'
|
7
|
+
|
8
|
+
module Commendo
|
9
|
+
|
10
|
+
class ConfigurationTest < Minitest::Test
|
11
|
+
|
12
|
+
def setup
|
13
|
+
Commendo.config = nil
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_default_values_on_configuration_class
|
17
|
+
assert_equal :redis, Commendo::Configuration.new.backend
|
18
|
+
assert_equal 'localhost', Commendo::Configuration.new.host
|
19
|
+
assert_equal 6379, Commendo::Configuration.new.port
|
20
|
+
assert_equal 15, Commendo::Configuration.new.database
|
21
|
+
assert_nil Commendo::Configuration.new.username
|
22
|
+
assert_nil Commendo::Configuration.new.password
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_returns_same_object_each_time
|
26
|
+
config1 = Commendo.config
|
27
|
+
config2 = Commendo.config
|
28
|
+
config3 = Commendo.config
|
29
|
+
assert config1.equal? config2
|
30
|
+
assert config2.equal? config3
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_config_returns_default_configuration
|
34
|
+
config = Commendo.config
|
35
|
+
assert config.is_a? Commendo::Configuration
|
36
|
+
assert_equal :redis, config.backend
|
37
|
+
assert_equal 'localhost', config.host
|
38
|
+
assert_equal 6379, config.port
|
39
|
+
assert_equal 15, config.database
|
40
|
+
assert_nil config.username
|
41
|
+
assert_nil config.password
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_configure_stores_settings
|
45
|
+
config = Commendo.config do |config|
|
46
|
+
config.backend = :mysql
|
47
|
+
config.host = 'mysql.example.com'
|
48
|
+
config.port = 9999
|
49
|
+
config.database = 'some_mysql_db'
|
50
|
+
config.username = 'root'
|
51
|
+
config.password = 'Passw0rd!!'
|
52
|
+
end
|
53
|
+
|
54
|
+
assert config.is_a? Commendo::Configuration
|
55
|
+
assert_equal :mysql, config.backend
|
56
|
+
assert_equal 'mysql.example.com', config.host
|
57
|
+
assert_equal 9999, config.port
|
58
|
+
assert_equal 'some_mysql_db', config.database
|
59
|
+
assert_equal 'root', config.username
|
60
|
+
assert_equal 'Passw0rd!!', config.password
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_to_hash
|
64
|
+
expected = {backend: :redis, host: 'localhost', port: 6379, database: 15, username: nil, password: nil}
|
65
|
+
actual = Commendo::Configuration.new.to_hash
|
66
|
+
assert_equal expected, actual
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require_relative 'tests_for_content_sets.rb'
|
2
|
+
gem 'minitest'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'minitest/pride'
|
5
|
+
require 'minitest/mock'
|
6
|
+
require 'mocha/setup'
|
7
|
+
require 'commendo'
|
8
|
+
|
9
|
+
module Commendo
|
10
|
+
|
11
|
+
class MySqlContentSetTest < Minitest::Test
|
12
|
+
|
13
|
+
def setup
|
14
|
+
Commendo.config do |config|
|
15
|
+
config.backend = :mysql
|
16
|
+
config.host = 'localhost'
|
17
|
+
config.port = 3306
|
18
|
+
config.database = 'commendo_test'
|
19
|
+
config.username = 'commendo'
|
20
|
+
config.password = 'commendo123'
|
21
|
+
end
|
22
|
+
client = Mysql2::Client.new(Commendo.config.to_hash)
|
23
|
+
%w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
|
24
|
+
@key_base = 'CommendoTests'
|
25
|
+
@cs = ContentSet.new(key_base: @key_base)
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_tag_set(kb)
|
29
|
+
Commendo::TagSet.new(key_base: kb)
|
30
|
+
end
|
31
|
+
|
32
|
+
def create_content_set(key_base, ts = nil)
|
33
|
+
Commendo::ContentSet.new(key_base: key_base, tag_set: ts)
|
34
|
+
end
|
35
|
+
|
36
|
+
include TestsForContentSets
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require_relative 'tests_for_tag_sets'
|
2
|
+
gem 'minitest'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'minitest/pride'
|
5
|
+
require 'minitest/mock'
|
6
|
+
require 'mocha/setup'
|
7
|
+
require 'commendo'
|
8
|
+
|
9
|
+
module Commendo
|
10
|
+
|
11
|
+
class MySqlTagSetTest < Minitest::Test
|
12
|
+
|
13
|
+
def setup
|
14
|
+
Commendo.config do |config|
|
15
|
+
config.backend = :mysql
|
16
|
+
config.host = 'localhost'
|
17
|
+
config.port = 3306
|
18
|
+
config.database = 'commendo_test'
|
19
|
+
config.username = 'commendo'
|
20
|
+
config.password = 'commendo123'
|
21
|
+
end
|
22
|
+
client = Mysql2::Client.new(Commendo.config.to_hash)
|
23
|
+
%w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
|
24
|
+
@ts = TagSet.new(key_base: 'TagSetTest')
|
25
|
+
end
|
26
|
+
|
27
|
+
def create_tag_set(kb)
|
28
|
+
Commendo::TagSet.new(key_base: kb)
|
29
|
+
end
|
30
|
+
|
31
|
+
include TestsForTagSets
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require_relative 'tests_for_weighted_groups'
|
2
|
+
gem 'minitest'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'minitest/pride'
|
5
|
+
require 'minitest/mock'
|
6
|
+
require 'mocha/setup'
|
7
|
+
require 'commendo'
|
8
|
+
|
9
|
+
module Commendo
|
10
|
+
|
11
|
+
class MySqlWeightedGroupTest < Minitest::Test
|
12
|
+
|
13
|
+
def setup
|
14
|
+
super
|
15
|
+
Commendo.config do |config|
|
16
|
+
config.backend = :mysql
|
17
|
+
config.host = 'localhost'
|
18
|
+
config.port = 3306
|
19
|
+
config.database = 'commendo_test'
|
20
|
+
config.username = 'commendo'
|
21
|
+
config.password = 'commendo123'
|
22
|
+
end
|
23
|
+
client = Mysql2::Client.new(Commendo.config.to_hash)
|
24
|
+
%w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
|
25
|
+
@tag_set = TagSet.new(key_base: 'CommendoTests:Tags')
|
26
|
+
@cs1 = ContentSet.new(key_base: 'CommendoTests:ContentSet1', tag_set: @tag_set)
|
27
|
+
@cs2 = ContentSet.new(key_base: 'CommendoTests:ContentSet2', tag_set: @tag_set)
|
28
|
+
@cs3 = ContentSet.new(key_base: 'CommendoTests:ContentSet3', tag_set: @tag_set)
|
29
|
+
(3..23).each do |group|
|
30
|
+
(3..23).each do |res|
|
31
|
+
@cs1.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(2).zero?
|
32
|
+
@cs2.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(3).zero?
|
33
|
+
@cs3.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(6).zero?
|
34
|
+
@tag_set.add(res, 'mod3') if res.modulo(3).zero?
|
35
|
+
@tag_set.add(res, 'mod4') if res.modulo(4).zero?
|
36
|
+
@tag_set.add(res, 'mod5') if res.modulo(5).zero?
|
37
|
+
@tag_set.add(res, 'mod7') if res.modulo(7).zero?
|
38
|
+
end
|
39
|
+
end
|
40
|
+
[@cs1, @cs2, @cs3].each { |cs| cs.calculate_similarity }
|
41
|
+
@weighted_group = Commendo::WeightedGroup.new(key_base: 'CommendoTests:WeightedGroup',
|
42
|
+
content_sets: [{cs: @cs1, weight: 1.0},
|
43
|
+
{cs: @cs2, weight: 10.0},
|
44
|
+
{cs: @cs3, weight: 100.0}]
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
include TestsForWeightedGroups
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
|