ryodo 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -1
- data/checks/matching.rb +6 -8
- data/data/suffix.dat +455 -22
- data/lib/ryodo.rb +4 -6
- data/lib/ryodo/convenience.rb +0 -4
- data/lib/ryodo/convenience/utf8.rb +6 -11
- data/lib/ryodo/domain.rb +21 -26
- data/lib/ryodo/ext/string.rb +1 -3
- data/lib/ryodo/ext/uri.rb +2 -5
- data/lib/ryodo/methods.rb +4 -7
- data/lib/ryodo/parser.rb +3 -10
- data/lib/ryodo/rule.rb +6 -8
- data/lib/ryodo/rule_set.rb +3 -12
- data/lib/ryodo/suffix_list.rb +6 -13
- data/lib/ryodo/suffix_list_fetcher.rb +27 -27
- data/lib/ryodo/version.rb +1 -1
- data/ryodo.gemspec +6 -7
- data/spec/_files/mozilla_effective_tld_names.dat +1672 -678
- data/spec/ryodo/suffix_list_fetcher_spec.rb +19 -31
- data/spec/ryodo/suffix_list_spec.rb +14 -18
- data/spec/ryodo_spec.rb +2 -3
- data/spec/spec_helper.rb +2 -19
- metadata +33 -33
data/lib/ryodo.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
RYODO_ROOT = File.expand_path("../..", __FILE__)
|
5
|
-
PUBLIC_SUFFIX_DATA_URI = "
|
3
|
+
PUBLIC_SUFFIX_DATA_URI = "https://publicsuffix.org/list/effective_tld_names.dat"
|
6
4
|
PUBLIC_SUFFIX_STORE = "#{RYODO_ROOT}/data/suffix.dat"
|
7
5
|
end
|
8
6
|
|
@@ -15,12 +13,12 @@ require "ryodo/rule_set"
|
|
15
13
|
require "ryodo/suffix_list"
|
16
14
|
|
17
15
|
require "ryodo/methods"
|
18
|
-
#require "ryodo/ext/string"
|
19
|
-
#require "ryodo/ext/uri"
|
16
|
+
# require "ryodo/ext/string"
|
17
|
+
# require "ryodo/ext/uri"
|
20
18
|
|
21
19
|
# Convenient shorthands
|
22
20
|
module Ryodo
|
23
21
|
extend Ryodo::Methods
|
24
22
|
require "ryodo/convenience"
|
25
23
|
end
|
26
|
-
#require "ryodo/convenience/utf8"
|
24
|
+
# require "ryodo/convenience/utf8"
|
data/lib/ryodo/convenience.rb
CHANGED
@@ -1,27 +1,22 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
module Convenience
|
5
3
|
module UTF8
|
4
|
+
alias_method :ryodo, :Ryodo
|
6
5
|
|
7
|
-
alias_method :ryodo
|
8
|
-
|
9
|
-
alias_method :ryodo?, :Ryodo?
|
6
|
+
alias_method :ryodo?, :Ryodo?
|
10
7
|
alias_method :valid_domain?, :Ryodo?
|
11
8
|
|
12
9
|
# Unicode junkie? ;o)
|
13
|
-
alias_method :"ryōdo",
|
14
|
-
alias_method :"ryōdo?",
|
10
|
+
alias_method :"ryōdo", :Ryodo
|
11
|
+
alias_method :"ryōdo?", :Ryodo?
|
15
12
|
|
16
|
-
alias_method :"領土",
|
13
|
+
alias_method :"領土", :Ryodo
|
17
14
|
alias_method :"りょうど", :Ryodo
|
18
15
|
|
19
|
-
alias_method :"領土?",
|
16
|
+
alias_method :"領土?", :Ryodo?
|
20
17
|
alias_method :"りょうどか", :Ryodo?
|
21
18
|
alias_method :"りょうどか。", :Ryodo?
|
22
|
-
|
23
19
|
end
|
24
|
-
|
25
20
|
end
|
26
21
|
end
|
27
22
|
|
data/lib/ryodo/domain.rb
CHANGED
@@ -1,8 +1,5 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
class Domain
|
5
|
-
|
6
3
|
# DomainString is a String with extended methods
|
7
4
|
class DomainString < String
|
8
5
|
|
@@ -11,7 +8,7 @@ module Ryodo
|
|
11
8
|
end
|
12
9
|
alias_method :r, :reverse
|
13
10
|
|
14
|
-
def to_a
|
11
|
+
def to_a(option = nil)
|
15
12
|
case option
|
16
13
|
when :reverse, :r
|
17
14
|
dsplit.reverse
|
@@ -20,59 +17,58 @@ module Ryodo
|
|
20
17
|
end
|
21
18
|
end
|
22
19
|
|
23
|
-
|
20
|
+
private
|
24
21
|
|
25
22
|
def dsplit
|
26
|
-
self.split("."
|
23
|
+
self.split(".", -1)
|
27
24
|
end
|
28
|
-
|
29
25
|
end
|
30
26
|
|
31
27
|
# remove own class comparison (we will use String#== via method_missing)
|
32
28
|
undef_method :==
|
33
29
|
|
34
|
-
def initialize
|
35
|
-
|
36
|
-
@domain_string
|
37
|
-
no_leading_dot
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
@suffix =
|
42
|
-
@domain =
|
43
|
-
@secondary =
|
44
|
-
@subdomain =
|
30
|
+
def initialize(domainStr)
|
31
|
+
fail TypeError, "Not a valid domain string!" unless domainStr.is_a?(String)
|
32
|
+
@domain_string = DomainString.new domainStr.downcase
|
33
|
+
no_leading_dot = @domain_string[0] != "."
|
34
|
+
parts = Ryodo::Parser.run(@domain_string)
|
35
|
+
no_dot_but_parts = no_leading_dot && parts
|
36
|
+
|
37
|
+
@suffix = parts[0].reverse.join(".") if no_dot_but_parts
|
38
|
+
@domain = (parts[0] + parts[1]).reverse.join(".") if no_dot_but_parts && !parts[1].empty?
|
39
|
+
@secondary = parts[1].first if no_dot_but_parts && !parts[1].empty?
|
40
|
+
@subdomain = (parts[2]).reverse.join(".") if no_dot_but_parts && !parts[2].empty?
|
45
41
|
end
|
46
42
|
|
47
43
|
def suffix
|
48
|
-
DomainString.new
|
44
|
+
DomainString.new(@suffix) if @suffix
|
49
45
|
end
|
50
46
|
alias_method :tld, :suffix
|
51
47
|
|
52
48
|
def domain
|
53
|
-
DomainString.new
|
49
|
+
DomainString.new(@domain) if @domain
|
54
50
|
end
|
55
51
|
alias_method :registered_domain, :domain
|
56
52
|
alias_method :regdomain, :domain
|
57
53
|
|
58
54
|
def second_level
|
59
|
-
DomainString.new
|
55
|
+
DomainString.new(@secondary) if @secondary
|
60
56
|
end
|
61
57
|
alias_method :sld, :second_level
|
62
58
|
alias_method :registered_name, :second_level
|
63
59
|
|
64
60
|
def subdomain
|
65
|
-
DomainString.new
|
61
|
+
DomainString.new(@subdomain) if @subdomain
|
66
62
|
end
|
67
63
|
|
68
64
|
def fqdn
|
69
|
-
DomainString.new
|
65
|
+
DomainString.new("#{to_s}.")
|
70
66
|
end
|
71
67
|
|
72
|
-
def
|
68
|
+
def valid?
|
73
69
|
!!@suffix && !!@secondary
|
74
70
|
end
|
75
|
-
alias_method :
|
71
|
+
alias_method :is_valid?, :valid?
|
76
72
|
|
77
73
|
def to_s
|
78
74
|
@domain_string
|
@@ -91,6 +87,5 @@ module Ryodo
|
|
91
87
|
def send(symbol, *args)
|
92
88
|
__send__(symbol, *args)
|
93
89
|
end
|
94
|
-
|
95
90
|
end
|
96
91
|
end
|
data/lib/ryodo/ext/string.rb
CHANGED
data/lib/ryodo/ext/uri.rb
CHANGED
@@ -1,12 +1,9 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
require "uri"
|
3
2
|
|
4
3
|
class URI::Generic
|
5
|
-
|
6
4
|
alias_method :set_host_string, :set_host
|
7
5
|
|
8
|
-
def set_host(
|
9
|
-
@host = Ryodo.parse(
|
6
|
+
def set_host(value)
|
7
|
+
@host = Ryodo.parse(set_host_string(value)) unless set_host_string(value).nil?
|
10
8
|
end
|
11
|
-
|
12
9
|
end
|
data/lib/ryodo/methods.rb
CHANGED
@@ -1,18 +1,15 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
module Methods
|
5
3
|
|
6
|
-
def parse
|
7
|
-
Ryodo::Domain.new
|
4
|
+
def parse(domain_string)
|
5
|
+
Ryodo::Domain.new(domain_string)
|
8
6
|
end
|
9
7
|
alias_method :[], :parse
|
10
8
|
|
11
|
-
def domain_valid?
|
12
|
-
|
9
|
+
def domain_valid?(domain_string)
|
10
|
+
parse(domain_string).valid?
|
13
11
|
end
|
14
12
|
alias_method :valid_domain?, :domain_valid?
|
15
13
|
alias_method :valid?, :domain_valid?
|
16
|
-
|
17
14
|
end
|
18
15
|
end
|
data/lib/ryodo/parser.rb
CHANGED
@@ -1,32 +1,25 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
class Parser
|
5
|
-
|
6
3
|
def initialize
|
7
4
|
@rules = Ryodo::RuleSet.new
|
8
5
|
end
|
9
6
|
|
10
|
-
def build_query
|
7
|
+
def build_query(domain)
|
11
8
|
domain.split(".").reverse
|
12
9
|
end
|
13
10
|
|
14
|
-
def parse
|
11
|
+
def parse(domain)
|
15
12
|
@rules.match build_query(domain)
|
16
13
|
end
|
17
14
|
|
18
15
|
class << self
|
19
|
-
|
20
|
-
def run domain
|
16
|
+
def run(domain)
|
21
17
|
instance.parse domain
|
22
18
|
end
|
23
19
|
|
24
|
-
|
25
20
|
def instance
|
26
21
|
@@instance ||= new
|
27
22
|
end
|
28
|
-
|
29
23
|
end
|
30
|
-
|
31
24
|
end
|
32
25
|
end
|
data/lib/ryodo/rule.rb
CHANGED
@@ -1,15 +1,13 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
|
-
class Rule < Struct.new(:exception, :
|
5
|
-
|
6
|
-
def has_children?
|
2
|
+
class Rule < Struct.new(:exception, :stop_ok, :children)
|
3
|
+
def children?
|
7
4
|
!children.empty?
|
8
5
|
end
|
6
|
+
alias :has_children? :children?
|
9
7
|
|
10
|
-
def
|
11
|
-
|
8
|
+
def suffix?
|
9
|
+
stop_ok
|
12
10
|
end
|
13
|
-
|
11
|
+
alias :is_suffix? :suffix?
|
14
12
|
end
|
15
13
|
end
|
data/lib/ryodo/rule_set.rb
CHANGED
@@ -1,8 +1,5 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
class RuleSet
|
5
|
-
|
6
3
|
def initialize
|
7
4
|
@tree = {}
|
8
5
|
build!
|
@@ -10,26 +7,21 @@ module Ryodo
|
|
10
7
|
|
11
8
|
def build!
|
12
9
|
Ryodo::SuffixList.list.each do |line|
|
13
|
-
|
14
10
|
line.each.with_index do |node_name, idx|
|
15
|
-
|
16
|
-
stopOK = node_name == line.last
|
11
|
+
stop_ok = node_name == line.last
|
17
12
|
exception = node_name[0] == "!"
|
18
13
|
node_name = node_name[1..-1] if exception
|
19
14
|
children = {}
|
20
|
-
node = Ryodo::Rule.new(exception,
|
15
|
+
node = Ryodo::Rule.new(exception, stop_ok, children)
|
21
16
|
|
22
17
|
if idx > 0
|
23
18
|
end_idx = idx - 1
|
24
|
-
parent
|
19
|
+
parent = select_rule(line[0..end_idx])
|
25
20
|
parent.children[node_name] = node unless parent.children[node_name]
|
26
|
-
|
27
21
|
else
|
28
22
|
@tree[node_name] = node unless @tree[node_name]
|
29
23
|
end
|
30
|
-
|
31
24
|
end
|
32
|
-
|
33
25
|
end
|
34
26
|
end
|
35
27
|
|
@@ -60,6 +52,5 @@ module Ryodo
|
|
60
52
|
[ suffix, [domain.shift], domain ]
|
61
53
|
end
|
62
54
|
end
|
63
|
-
|
64
55
|
end
|
65
56
|
end
|
data/lib/ryodo/suffix_list.rb
CHANGED
@@ -1,21 +1,18 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
|
-
|
5
2
|
class SuffixList
|
6
|
-
def initialize
|
3
|
+
def initialize(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
7
4
|
load_file(suffix_file)
|
8
5
|
end
|
9
6
|
|
10
7
|
def parse_data
|
11
8
|
# loads and converts to array
|
12
9
|
# "baz.bar.foo" => ["baz", "bar", "foo"]
|
13
|
-
File.readlines(@suffix_file).map{ |line| line.strip.split(".") }
|
10
|
+
File.readlines(@suffix_file).map { |line| line.strip.split(".") }
|
14
11
|
end
|
15
12
|
|
16
|
-
def load_file
|
13
|
+
def load_file(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
17
14
|
@suffix_file = suffix_file
|
18
|
-
@suffix_data = parse_data
|
15
|
+
@suffix_data = parse_data << ["example"]
|
19
16
|
end
|
20
17
|
|
21
18
|
def list
|
@@ -27,12 +24,11 @@ module Ryodo
|
|
27
24
|
end
|
28
25
|
|
29
26
|
class << self
|
30
|
-
|
31
|
-
def SuffixList suffix_file = Ryodo::PUBLIC_SUFFIX_STORE
|
27
|
+
def SuffixList(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
32
28
|
instance(suffix_file)
|
33
29
|
end
|
34
30
|
|
35
|
-
def reload
|
31
|
+
def reload(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
36
32
|
instance.load_file(suffix_file) && true
|
37
33
|
end
|
38
34
|
|
@@ -47,11 +43,8 @@ module Ryodo
|
|
47
43
|
def inspect
|
48
44
|
instance.inspect
|
49
45
|
end
|
50
|
-
|
51
46
|
end
|
52
47
|
|
53
48
|
private_class_method :new
|
54
|
-
|
55
49
|
end
|
56
|
-
|
57
50
|
end
|
@@ -1,33 +1,40 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
require "uri"
|
4
2
|
require "net/http"
|
5
3
|
|
6
4
|
module Ryodo
|
7
|
-
|
8
|
-
class FetchError < StandardError; end
|
5
|
+
FetchError = Class.new(StandardError)
|
9
6
|
|
10
7
|
class SuffixListFetcher
|
8
|
+
class << self
|
9
|
+
def fetch_and_save!(uri = Ryodo::PUBLIC_SUFFIX_DATA_URI, store = Ryodo::PUBLIC_SUFFIX_STORE)
|
10
|
+
fetcher = new(uri, store)
|
11
|
+
fetcher.fetch_data
|
12
|
+
fetcher.prepare_data
|
13
|
+
fetcher.save_data
|
14
|
+
true
|
15
|
+
rescue
|
16
|
+
false
|
17
|
+
end
|
18
|
+
end
|
11
19
|
|
12
|
-
def initialize
|
13
|
-
@uri = URI(uri)
|
20
|
+
def initialize(uri = Ryodo::PUBLIC_SUFFIX_DATA_URI, store = Ryodo::PUBLIC_SUFFIX_STORE)
|
21
|
+
@uri = URI.parse(uri)
|
14
22
|
@store = store
|
15
23
|
end
|
16
24
|
|
17
25
|
def fetch_data
|
18
|
-
|
19
|
-
|
20
|
-
|
26
|
+
http = Net::HTTP.new(@uri.host, @uri.port)
|
27
|
+
http.use_ssl = @uri.scheme == "https"
|
28
|
+
request = Net::HTTP::Get.new(@uri.request_uri)
|
29
|
+
response = http.request(request)
|
30
|
+
fail Ryodo::FetchError, "Could not fetch suffix data! (#{response})" unless response.is_a?(Net::HTTPSuccess)
|
31
|
+
@fetched_data = response.body.lines
|
21
32
|
end
|
22
33
|
|
23
34
|
def prepare_data
|
24
35
|
@prepared_data = @fetched_data.inject([]) do |acc, line|
|
25
|
-
next(acc) if line =~
|
26
|
-
|
27
|
-
split(".").
|
28
|
-
reverse.
|
29
|
-
join(".") # "foo.bar.baz" => "baz.bar.foo"
|
30
|
-
acc << dns_line
|
36
|
+
next(acc) if line =~ %r{\A//|\A\n}
|
37
|
+
acc << reverse_dn(line)
|
31
38
|
end.sort
|
32
39
|
end
|
33
40
|
|
@@ -37,18 +44,11 @@ module Ryodo
|
|
37
44
|
end if @prepared_data
|
38
45
|
end
|
39
46
|
|
40
|
-
|
41
|
-
def fetch_and_save! uri = Ryodo::PUBLIC_SUFFIX_DATA_URI, store = Ryodo::PUBLIC_SUFFIX_STORE
|
42
|
-
fetcher = self.new uri, store
|
43
|
-
fetcher.fetch_data
|
44
|
-
fetcher.prepare_data
|
45
|
-
fetcher.save_data
|
46
|
-
true
|
47
|
-
rescue
|
48
|
-
false
|
49
|
-
end
|
50
|
-
end
|
47
|
+
private
|
51
48
|
|
49
|
+
def reverse_dn(domain_name)
|
50
|
+
# "foo.bar.baz" => "baz.bar.foo"
|
51
|
+
domain_name.strip.split(".").reverse.join(".")
|
52
|
+
end
|
52
53
|
end
|
53
|
-
|
54
54
|
end
|