commendo 1.2.4 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/bin/commendo-create-mysql-db +3 -0
- data/bin/commendo-create.sql +99 -0
- data/bin/commendo-load-tsv +11 -5
- data/bin/commendo-load-tsv-mysql.rb +43 -0
- data/bin/commendo-time-mysql.rb +31 -0
- data/commendo.gemspec +4 -2
- data/lib/commendo.rb +24 -0
- data/lib/commendo/configuration.rb +25 -0
- data/lib/commendo/content_set.rb +13 -182
- data/lib/commendo/mysql-backed/content_set.rb +152 -0
- data/lib/commendo/mysql-backed/tag_set.rb +81 -0
- data/lib/commendo/mysql-backed/weighted_group.rb +40 -0
- data/lib/commendo/redis-backed/content_set.rb +194 -0
- data/lib/commendo/{pair_comparison.lua → redis-backed/pair_comparison.lua} +0 -0
- data/lib/commendo/{similarity.lua → redis-backed/similarity.lua} +0 -0
- data/lib/commendo/redis-backed/tag_set.rb +54 -0
- data/lib/commendo/redis-backed/weighted_group.rb +54 -0
- data/lib/commendo/tag_set.rb +6 -42
- data/lib/commendo/version.rb +1 -1
- data/lib/commendo/weighted_group.rb +7 -41
- data/lib/mysql2/client.rb +17 -0
- data/model 2.mwb +0 -0
- data/sql_model.mwb +0 -0
- data/test/configuration_test.rb +71 -0
- data/test/mysql_content_set_test.rb +40 -0
- data/test/mysql_tag_set_test.rb +34 -0
- data/test/mysql_weighted_group_test.rb +54 -0
- data/test/redis_content_set_test.rb +57 -0
- data/test/redis_tag_set_test.rb +31 -0
- data/test/redis_weighted_group_test.rb +49 -0
- data/test/tests_for_content_sets.rb +379 -0
- data/test/tests_for_tag_sets.rb +130 -0
- data/test/tests_for_weighted_groups.rb +106 -0
- metadata +72 -12
- data/test/content_set_test.rb +0 -408
- data/test/tag_set_test.rb +0 -128
- data/test/weighted_group_test.rb +0 -191
File without changes
|
File without changes
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Commendo
|
2
|
+
module RedisBacked
|
3
|
+
class TagSet
|
4
|
+
|
5
|
+
attr_accessor :redis, :key_base
|
6
|
+
|
7
|
+
def initialize(key_base)
|
8
|
+
@redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
|
9
|
+
@key_base = key_base
|
10
|
+
end
|
11
|
+
|
12
|
+
def empty?
|
13
|
+
cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
|
14
|
+
cursor.to_i == 0 && keys.empty?
|
15
|
+
end
|
16
|
+
|
17
|
+
def get(resource)
|
18
|
+
redis.smembers(resource_key(resource)).sort
|
19
|
+
end
|
20
|
+
|
21
|
+
def add(resource, *tags)
|
22
|
+
redis.sadd(resource_key(resource), tags) unless tags.empty?
|
23
|
+
end
|
24
|
+
|
25
|
+
def set(resource, *tags)
|
26
|
+
delete(resource)
|
27
|
+
add(resource, *tags)
|
28
|
+
end
|
29
|
+
|
30
|
+
def matches(resource, include, exclude = [])
|
31
|
+
resource_tags = get(resource)
|
32
|
+
can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
|
33
|
+
should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
|
34
|
+
return can_include && !should_exclude
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete(resource, *tags)
|
38
|
+
if tags.empty?
|
39
|
+
redis.del(resource_key(resource))
|
40
|
+
else
|
41
|
+
redis.srem(resource_key(resource), tags)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def resource_key(resource)
|
48
|
+
"#{key_base}:#{resource}"
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Commendo
|
2
|
+
module RedisBacked
|
3
|
+
|
4
|
+
class WeightedGroup
|
5
|
+
|
6
|
+
attr_accessor :content_sets, :redis, :key_base, :tag_set
|
7
|
+
|
8
|
+
def initialize(key_base, *content_sets)
|
9
|
+
@redis = Redis.new(host: Commendo.config.host, port: Commendo.config.port, db: Commendo.config.database)
|
10
|
+
@key_base = key_base
|
11
|
+
@content_sets = content_sets
|
12
|
+
end
|
13
|
+
|
14
|
+
def similar_to(resource, limit = 0)
|
15
|
+
finish = limit -1
|
16
|
+
resources = resource.kind_of?(Array) ? resource : [resource]
|
17
|
+
keys = []
|
18
|
+
weights = []
|
19
|
+
content_sets.each do |cs|
|
20
|
+
resources.each do |resource|
|
21
|
+
keys << cs[:cs].similarity_key(resource)
|
22
|
+
weights << cs[:weight]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
|
26
|
+
redis.zunionstore(tmp_key, keys, weights: weights)
|
27
|
+
similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
|
28
|
+
redis.del(tmp_key)
|
29
|
+
|
30
|
+
similar_resources.map do |resource|
|
31
|
+
{resource: resource[0], similarity: resource[1].to_f.round(3)}
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
def filtered_similar_to(resource, options = {})
|
37
|
+
if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
|
38
|
+
return similar_to(resource, options[:limit] || 0)
|
39
|
+
else
|
40
|
+
similar = similar_to(resource)
|
41
|
+
limit = options[:limit] || similar.length
|
42
|
+
filtered = []
|
43
|
+
similar.each do |s|
|
44
|
+
return filtered if filtered.length >= limit
|
45
|
+
filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
|
46
|
+
end
|
47
|
+
return filtered
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
data/lib/commendo/tag_set.rb
CHANGED
@@ -1,51 +1,15 @@
|
|
1
1
|
module Commendo
|
2
|
-
|
3
2
|
class TagSet
|
3
|
+
extend Forwardable
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
def initialize(redis, key_base)
|
8
|
-
@redis, @key_base = redis, key_base
|
9
|
-
end
|
10
|
-
|
11
|
-
def empty?
|
12
|
-
cursor, keys = redis.scan(0, match: "#{key_base}:*", count: 1)
|
13
|
-
cursor.to_i == 0 && keys.empty?
|
14
|
-
end
|
15
|
-
|
16
|
-
def get(resource)
|
17
|
-
redis.smembers(resource_key(resource)).sort
|
18
|
-
end
|
5
|
+
def_delegators :@backend, :empty?, :get, :add, :set, :matches, :delete, :redis, :key_base
|
19
6
|
|
20
|
-
def
|
21
|
-
|
7
|
+
def initialize(args)
|
8
|
+
@backend = RedisBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :redis
|
9
|
+
@backend = MySqlBacked::TagSet.new(args[:key_base]) if Commendo.config.backend == :mysql
|
10
|
+
raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
|
22
11
|
end
|
23
12
|
|
24
|
-
def set(resource, *tags)
|
25
|
-
delete(resource)
|
26
|
-
add(resource, *tags)
|
27
|
-
end
|
28
|
-
|
29
|
-
def matches(resource, include, exclude = [])
|
30
|
-
resource_tags = get(resource)
|
31
|
-
can_include = include.nil? || include.empty? || (resource_tags & include).length > 0
|
32
|
-
should_exclude = !exclude.nil? && !exclude.empty? && (resource_tags & exclude).length > 0
|
33
|
-
return can_include && !should_exclude
|
34
|
-
end
|
35
|
-
|
36
|
-
def delete(resource, *tags)
|
37
|
-
if tags.empty?
|
38
|
-
redis.del(resource_key(resource))
|
39
|
-
else
|
40
|
-
redis.srem(resource_key(resource), tags)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
private
|
45
|
-
|
46
|
-
def resource_key(resource)
|
47
|
-
"#{key_base}:#{resource}"
|
48
|
-
end
|
49
13
|
|
50
14
|
end
|
51
15
|
end
|
data/lib/commendo/version.rb
CHANGED
@@ -1,50 +1,16 @@
|
|
1
1
|
module Commendo
|
2
2
|
|
3
3
|
class WeightedGroup
|
4
|
+
extend Forwardable
|
4
5
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
end
|
10
|
-
|
11
|
-
def similar_to(resource, limit = 0)
|
12
|
-
finish = limit -1
|
13
|
-
resources = resource.kind_of?(Array) ? resource : [resource]
|
14
|
-
keys = []
|
15
|
-
weights = []
|
16
|
-
content_sets.each do |cs|
|
17
|
-
resources.each do |resource|
|
18
|
-
keys << cs[:cs].similarity_key(resource)
|
19
|
-
weights << cs[:weight]
|
20
|
-
end
|
21
|
-
end
|
22
|
-
tmp_key = "#{key_base}:tmp:#{SecureRandom.uuid}"
|
23
|
-
redis.zunionstore(tmp_key, keys, weights: weights)
|
24
|
-
similar_resources = redis.zrevrange(tmp_key, 0, finish, with_scores: true)
|
25
|
-
redis.del(tmp_key)
|
26
|
-
|
27
|
-
similar_resources.map do |resource|
|
28
|
-
{resource: resource[0], similarity: resource[1].to_f.round(3)}
|
29
|
-
end
|
30
|
-
|
6
|
+
def initialize(args)
|
7
|
+
@backend = RedisBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :redis
|
8
|
+
@backend = MySqlBacked::WeightedGroup.new(args[:key_base], *args[:content_sets]) if Commendo.config.backend == :mysql
|
9
|
+
raise 'Unrecognised backend type, try :redis or :mysql' if @backend.nil?
|
31
10
|
end
|
32
11
|
|
33
|
-
|
34
|
-
if @tag_set.nil? || (options[:include].nil? && options[:exclude].nil?)
|
35
|
-
return similar_to(resource, options[:limit] || 0)
|
36
|
-
else
|
37
|
-
similar = similar_to(resource)
|
38
|
-
limit = options[:limit] || similar.length
|
39
|
-
filtered = []
|
40
|
-
similar.each do |s|
|
41
|
-
return filtered if filtered.length >= limit
|
42
|
-
filtered << s if @tag_set.matches(s[:resource], options[:include], options[:exclude])
|
43
|
-
end
|
44
|
-
return filtered
|
45
|
-
end
|
46
|
-
end
|
12
|
+
def_delegators :@backend, :similar_to, :filtered_similar_to, :content_sets, :redis, :key_base, :tag_set, :tag_set=
|
47
13
|
|
48
14
|
end
|
49
15
|
|
50
|
-
end
|
16
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Mysql2
|
2
|
+
class Client
|
3
|
+
def transaction(&block)
|
4
|
+
raise ArgumentError, 'No block was given' unless block_given?
|
5
|
+
begin
|
6
|
+
query('BEGIN')
|
7
|
+
yield(self)
|
8
|
+
query('COMMIT')
|
9
|
+
return true # Successful Transaction
|
10
|
+
rescue
|
11
|
+
query('ROLLBACK')
|
12
|
+
raise
|
13
|
+
return false # Failed Transaction
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/model 2.mwb
ADDED
Binary file
|
data/sql_model.mwb
ADDED
Binary file
|
@@ -0,0 +1,71 @@
|
|
1
|
+
gem 'minitest'
|
2
|
+
require 'minitest/autorun'
|
3
|
+
require 'minitest/pride'
|
4
|
+
require 'minitest/mock'
|
5
|
+
require 'mocha/setup'
|
6
|
+
require 'commendo'
|
7
|
+
|
8
|
+
module Commendo
|
9
|
+
|
10
|
+
class ConfigurationTest < Minitest::Test
|
11
|
+
|
12
|
+
def setup
|
13
|
+
Commendo.config = nil
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_default_values_on_configuration_class
|
17
|
+
assert_equal :redis, Commendo::Configuration.new.backend
|
18
|
+
assert_equal 'localhost', Commendo::Configuration.new.host
|
19
|
+
assert_equal 6379, Commendo::Configuration.new.port
|
20
|
+
assert_equal 15, Commendo::Configuration.new.database
|
21
|
+
assert_nil Commendo::Configuration.new.username
|
22
|
+
assert_nil Commendo::Configuration.new.password
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_returns_same_object_each_time
|
26
|
+
config1 = Commendo.config
|
27
|
+
config2 = Commendo.config
|
28
|
+
config3 = Commendo.config
|
29
|
+
assert config1.equal? config2
|
30
|
+
assert config2.equal? config3
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_config_returns_default_configuration
|
34
|
+
config = Commendo.config
|
35
|
+
assert config.is_a? Commendo::Configuration
|
36
|
+
assert_equal :redis, config.backend
|
37
|
+
assert_equal 'localhost', config.host
|
38
|
+
assert_equal 6379, config.port
|
39
|
+
assert_equal 15, config.database
|
40
|
+
assert_nil config.username
|
41
|
+
assert_nil config.password
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_configure_stores_settings
|
45
|
+
config = Commendo.config do |config|
|
46
|
+
config.backend = :mysql
|
47
|
+
config.host = 'mysql.example.com'
|
48
|
+
config.port = 9999
|
49
|
+
config.database = 'some_mysql_db'
|
50
|
+
config.username = 'root'
|
51
|
+
config.password = 'Passw0rd!!'
|
52
|
+
end
|
53
|
+
|
54
|
+
assert config.is_a? Commendo::Configuration
|
55
|
+
assert_equal :mysql, config.backend
|
56
|
+
assert_equal 'mysql.example.com', config.host
|
57
|
+
assert_equal 9999, config.port
|
58
|
+
assert_equal 'some_mysql_db', config.database
|
59
|
+
assert_equal 'root', config.username
|
60
|
+
assert_equal 'Passw0rd!!', config.password
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_to_hash
|
64
|
+
expected = {backend: :redis, host: 'localhost', port: 6379, database: 15, username: nil, password: nil}
|
65
|
+
actual = Commendo::Configuration.new.to_hash
|
66
|
+
assert_equal expected, actual
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require_relative 'tests_for_content_sets.rb'
|
2
|
+
gem 'minitest'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'minitest/pride'
|
5
|
+
require 'minitest/mock'
|
6
|
+
require 'mocha/setup'
|
7
|
+
require 'commendo'
|
8
|
+
|
9
|
+
module Commendo
|
10
|
+
|
11
|
+
class MySqlContentSetTest < Minitest::Test
|
12
|
+
|
13
|
+
def setup
|
14
|
+
Commendo.config do |config|
|
15
|
+
config.backend = :mysql
|
16
|
+
config.host = 'localhost'
|
17
|
+
config.port = 3306
|
18
|
+
config.database = 'commendo_test'
|
19
|
+
config.username = 'commendo'
|
20
|
+
config.password = 'commendo123'
|
21
|
+
end
|
22
|
+
client = Mysql2::Client.new(Commendo.config.to_hash)
|
23
|
+
%w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
|
24
|
+
@key_base = 'CommendoTests'
|
25
|
+
@cs = ContentSet.new(key_base: @key_base)
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_tag_set(kb)
|
29
|
+
Commendo::TagSet.new(key_base: kb)
|
30
|
+
end
|
31
|
+
|
32
|
+
def create_content_set(key_base, ts = nil)
|
33
|
+
Commendo::ContentSet.new(key_base: key_base, tag_set: ts)
|
34
|
+
end
|
35
|
+
|
36
|
+
include TestsForContentSets
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require_relative 'tests_for_tag_sets'
|
2
|
+
gem 'minitest'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'minitest/pride'
|
5
|
+
require 'minitest/mock'
|
6
|
+
require 'mocha/setup'
|
7
|
+
require 'commendo'
|
8
|
+
|
9
|
+
module Commendo
|
10
|
+
|
11
|
+
class MySqlTagSetTest < Minitest::Test
|
12
|
+
|
13
|
+
def setup
|
14
|
+
Commendo.config do |config|
|
15
|
+
config.backend = :mysql
|
16
|
+
config.host = 'localhost'
|
17
|
+
config.port = 3306
|
18
|
+
config.database = 'commendo_test'
|
19
|
+
config.username = 'commendo'
|
20
|
+
config.password = 'commendo123'
|
21
|
+
end
|
22
|
+
client = Mysql2::Client.new(Commendo.config.to_hash)
|
23
|
+
%w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
|
24
|
+
@ts = TagSet.new(key_base: 'TagSetTest')
|
25
|
+
end
|
26
|
+
|
27
|
+
def create_tag_set(kb)
|
28
|
+
Commendo::TagSet.new(key_base: kb)
|
29
|
+
end
|
30
|
+
|
31
|
+
include TestsForTagSets
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require_relative 'tests_for_weighted_groups'
|
2
|
+
gem 'minitest'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'minitest/pride'
|
5
|
+
require 'minitest/mock'
|
6
|
+
require 'mocha/setup'
|
7
|
+
require 'commendo'
|
8
|
+
|
9
|
+
module Commendo
|
10
|
+
|
11
|
+
class MySqlWeightedGroupTest < Minitest::Test
|
12
|
+
|
13
|
+
def setup
|
14
|
+
super
|
15
|
+
Commendo.config do |config|
|
16
|
+
config.backend = :mysql
|
17
|
+
config.host = 'localhost'
|
18
|
+
config.port = 3306
|
19
|
+
config.database = 'commendo_test'
|
20
|
+
config.username = 'commendo'
|
21
|
+
config.password = 'commendo123'
|
22
|
+
end
|
23
|
+
client = Mysql2::Client.new(Commendo.config.to_hash)
|
24
|
+
%w(Tags Resources).each {|table| client.query("DELETE FROM #{table};") }
|
25
|
+
@tag_set = TagSet.new(key_base: 'CommendoTests:Tags')
|
26
|
+
@cs1 = ContentSet.new(key_base: 'CommendoTests:ContentSet1', tag_set: @tag_set)
|
27
|
+
@cs2 = ContentSet.new(key_base: 'CommendoTests:ContentSet2', tag_set: @tag_set)
|
28
|
+
@cs3 = ContentSet.new(key_base: 'CommendoTests:ContentSet3', tag_set: @tag_set)
|
29
|
+
(3..23).each do |group|
|
30
|
+
(3..23).each do |res|
|
31
|
+
@cs1.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(2).zero?
|
32
|
+
@cs2.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(3).zero?
|
33
|
+
@cs3.add_by_group(group, res) if res.modulo(group).zero? && res.modulo(6).zero?
|
34
|
+
@tag_set.add(res, 'mod3') if res.modulo(3).zero?
|
35
|
+
@tag_set.add(res, 'mod4') if res.modulo(4).zero?
|
36
|
+
@tag_set.add(res, 'mod5') if res.modulo(5).zero?
|
37
|
+
@tag_set.add(res, 'mod7') if res.modulo(7).zero?
|
38
|
+
end
|
39
|
+
end
|
40
|
+
[@cs1, @cs2, @cs3].each { |cs| cs.calculate_similarity }
|
41
|
+
@weighted_group = Commendo::WeightedGroup.new(key_base: 'CommendoTests:WeightedGroup',
|
42
|
+
content_sets: [{cs: @cs1, weight: 1.0},
|
43
|
+
{cs: @cs2, weight: 10.0},
|
44
|
+
{cs: @cs3, weight: 100.0}]
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
include TestsForWeightedGroups
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
|