ryodo 0.2.7 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +2 -1
- data/checks/matching.rb +6 -8
- data/data/suffix.dat +455 -22
- data/lib/ryodo.rb +4 -6
- data/lib/ryodo/convenience.rb +0 -4
- data/lib/ryodo/convenience/utf8.rb +6 -11
- data/lib/ryodo/domain.rb +21 -26
- data/lib/ryodo/ext/string.rb +1 -3
- data/lib/ryodo/ext/uri.rb +2 -5
- data/lib/ryodo/methods.rb +4 -7
- data/lib/ryodo/parser.rb +3 -10
- data/lib/ryodo/rule.rb +6 -8
- data/lib/ryodo/rule_set.rb +3 -12
- data/lib/ryodo/suffix_list.rb +6 -13
- data/lib/ryodo/suffix_list_fetcher.rb +27 -27
- data/lib/ryodo/version.rb +1 -1
- data/ryodo.gemspec +6 -7
- data/spec/_files/mozilla_effective_tld_names.dat +1672 -678
- data/spec/ryodo/suffix_list_fetcher_spec.rb +19 -31
- data/spec/ryodo/suffix_list_spec.rb +14 -18
- data/spec/ryodo_spec.rb +2 -3
- data/spec/spec_helper.rb +2 -19
- metadata +33 -33
data/lib/ryodo.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
RYODO_ROOT = File.expand_path("../..", __FILE__)
|
5
|
-
PUBLIC_SUFFIX_DATA_URI = "
|
3
|
+
PUBLIC_SUFFIX_DATA_URI = "https://publicsuffix.org/list/effective_tld_names.dat"
|
6
4
|
PUBLIC_SUFFIX_STORE = "#{RYODO_ROOT}/data/suffix.dat"
|
7
5
|
end
|
8
6
|
|
@@ -15,12 +13,12 @@ require "ryodo/rule_set"
|
|
15
13
|
require "ryodo/suffix_list"
|
16
14
|
|
17
15
|
require "ryodo/methods"
|
18
|
-
#require "ryodo/ext/string"
|
19
|
-
#require "ryodo/ext/uri"
|
16
|
+
# require "ryodo/ext/string"
|
17
|
+
# require "ryodo/ext/uri"
|
20
18
|
|
21
19
|
# Convenient shorthands
|
22
20
|
module Ryodo
|
23
21
|
extend Ryodo::Methods
|
24
22
|
require "ryodo/convenience"
|
25
23
|
end
|
26
|
-
#require "ryodo/convenience/utf8"
|
24
|
+
# require "ryodo/convenience/utf8"
|
data/lib/ryodo/convenience.rb
CHANGED
@@ -1,27 +1,22 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
module Convenience
|
5
3
|
module UTF8
|
4
|
+
alias_method :ryodo, :Ryodo
|
6
5
|
|
7
|
-
alias_method :ryodo
|
8
|
-
|
9
|
-
alias_method :ryodo?, :Ryodo?
|
6
|
+
alias_method :ryodo?, :Ryodo?
|
10
7
|
alias_method :valid_domain?, :Ryodo?
|
11
8
|
|
12
9
|
# Unicode junkie? ;o)
|
13
|
-
alias_method :"ryōdo",
|
14
|
-
alias_method :"ryōdo?",
|
10
|
+
alias_method :"ryōdo", :Ryodo
|
11
|
+
alias_method :"ryōdo?", :Ryodo?
|
15
12
|
|
16
|
-
alias_method :"領土",
|
13
|
+
alias_method :"領土", :Ryodo
|
17
14
|
alias_method :"りょうど", :Ryodo
|
18
15
|
|
19
|
-
alias_method :"領土?",
|
16
|
+
alias_method :"領土?", :Ryodo?
|
20
17
|
alias_method :"りょうどか", :Ryodo?
|
21
18
|
alias_method :"りょうどか。", :Ryodo?
|
22
|
-
|
23
19
|
end
|
24
|
-
|
25
20
|
end
|
26
21
|
end
|
27
22
|
|
data/lib/ryodo/domain.rb
CHANGED
@@ -1,8 +1,5 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
class Domain
|
5
|
-
|
6
3
|
# DomainString is a String with extended methods
|
7
4
|
class DomainString < String
|
8
5
|
|
@@ -11,7 +8,7 @@ module Ryodo
|
|
11
8
|
end
|
12
9
|
alias_method :r, :reverse
|
13
10
|
|
14
|
-
def to_a
|
11
|
+
def to_a(option = nil)
|
15
12
|
case option
|
16
13
|
when :reverse, :r
|
17
14
|
dsplit.reverse
|
@@ -20,59 +17,58 @@ module Ryodo
|
|
20
17
|
end
|
21
18
|
end
|
22
19
|
|
23
|
-
|
20
|
+
private
|
24
21
|
|
25
22
|
def dsplit
|
26
|
-
self.split("."
|
23
|
+
self.split(".", -1)
|
27
24
|
end
|
28
|
-
|
29
25
|
end
|
30
26
|
|
31
27
|
# remove own class comparison (we will use String#== via method_missing)
|
32
28
|
undef_method :==
|
33
29
|
|
34
|
-
def initialize
|
35
|
-
|
36
|
-
@domain_string
|
37
|
-
no_leading_dot
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
@suffix =
|
42
|
-
@domain =
|
43
|
-
@secondary =
|
44
|
-
@subdomain =
|
30
|
+
def initialize(domainStr)
|
31
|
+
fail TypeError, "Not a valid domain string!" unless domainStr.is_a?(String)
|
32
|
+
@domain_string = DomainString.new domainStr.downcase
|
33
|
+
no_leading_dot = @domain_string[0] != "."
|
34
|
+
parts = Ryodo::Parser.run(@domain_string)
|
35
|
+
no_dot_but_parts = no_leading_dot && parts
|
36
|
+
|
37
|
+
@suffix = parts[0].reverse.join(".") if no_dot_but_parts
|
38
|
+
@domain = (parts[0] + parts[1]).reverse.join(".") if no_dot_but_parts && !parts[1].empty?
|
39
|
+
@secondary = parts[1].first if no_dot_but_parts && !parts[1].empty?
|
40
|
+
@subdomain = (parts[2]).reverse.join(".") if no_dot_but_parts && !parts[2].empty?
|
45
41
|
end
|
46
42
|
|
47
43
|
def suffix
|
48
|
-
DomainString.new
|
44
|
+
DomainString.new(@suffix) if @suffix
|
49
45
|
end
|
50
46
|
alias_method :tld, :suffix
|
51
47
|
|
52
48
|
def domain
|
53
|
-
DomainString.new
|
49
|
+
DomainString.new(@domain) if @domain
|
54
50
|
end
|
55
51
|
alias_method :registered_domain, :domain
|
56
52
|
alias_method :regdomain, :domain
|
57
53
|
|
58
54
|
def second_level
|
59
|
-
DomainString.new
|
55
|
+
DomainString.new(@secondary) if @secondary
|
60
56
|
end
|
61
57
|
alias_method :sld, :second_level
|
62
58
|
alias_method :registered_name, :second_level
|
63
59
|
|
64
60
|
def subdomain
|
65
|
-
DomainString.new
|
61
|
+
DomainString.new(@subdomain) if @subdomain
|
66
62
|
end
|
67
63
|
|
68
64
|
def fqdn
|
69
|
-
DomainString.new
|
65
|
+
DomainString.new("#{to_s}.")
|
70
66
|
end
|
71
67
|
|
72
|
-
def
|
68
|
+
def valid?
|
73
69
|
!!@suffix && !!@secondary
|
74
70
|
end
|
75
|
-
alias_method :
|
71
|
+
alias_method :is_valid?, :valid?
|
76
72
|
|
77
73
|
def to_s
|
78
74
|
@domain_string
|
@@ -91,6 +87,5 @@ module Ryodo
|
|
91
87
|
def send(symbol, *args)
|
92
88
|
__send__(symbol, *args)
|
93
89
|
end
|
94
|
-
|
95
90
|
end
|
96
91
|
end
|
data/lib/ryodo/ext/string.rb
CHANGED
data/lib/ryodo/ext/uri.rb
CHANGED
@@ -1,12 +1,9 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
require "uri"
|
3
2
|
|
4
3
|
class URI::Generic
|
5
|
-
|
6
4
|
alias_method :set_host_string, :set_host
|
7
5
|
|
8
|
-
def set_host(
|
9
|
-
@host = Ryodo.parse(
|
6
|
+
def set_host(value)
|
7
|
+
@host = Ryodo.parse(set_host_string(value)) unless set_host_string(value).nil?
|
10
8
|
end
|
11
|
-
|
12
9
|
end
|
data/lib/ryodo/methods.rb
CHANGED
@@ -1,18 +1,15 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
module Methods
|
5
3
|
|
6
|
-
def parse
|
7
|
-
Ryodo::Domain.new
|
4
|
+
def parse(domain_string)
|
5
|
+
Ryodo::Domain.new(domain_string)
|
8
6
|
end
|
9
7
|
alias_method :[], :parse
|
10
8
|
|
11
|
-
def domain_valid?
|
12
|
-
|
9
|
+
def domain_valid?(domain_string)
|
10
|
+
parse(domain_string).valid?
|
13
11
|
end
|
14
12
|
alias_method :valid_domain?, :domain_valid?
|
15
13
|
alias_method :valid?, :domain_valid?
|
16
|
-
|
17
14
|
end
|
18
15
|
end
|
data/lib/ryodo/parser.rb
CHANGED
@@ -1,32 +1,25 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
class Parser
|
5
|
-
|
6
3
|
def initialize
|
7
4
|
@rules = Ryodo::RuleSet.new
|
8
5
|
end
|
9
6
|
|
10
|
-
def build_query
|
7
|
+
def build_query(domain)
|
11
8
|
domain.split(".").reverse
|
12
9
|
end
|
13
10
|
|
14
|
-
def parse
|
11
|
+
def parse(domain)
|
15
12
|
@rules.match build_query(domain)
|
16
13
|
end
|
17
14
|
|
18
15
|
class << self
|
19
|
-
|
20
|
-
def run domain
|
16
|
+
def run(domain)
|
21
17
|
instance.parse domain
|
22
18
|
end
|
23
19
|
|
24
|
-
|
25
20
|
def instance
|
26
21
|
@@instance ||= new
|
27
22
|
end
|
28
|
-
|
29
23
|
end
|
30
|
-
|
31
24
|
end
|
32
25
|
end
|
data/lib/ryodo/rule.rb
CHANGED
@@ -1,15 +1,13 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
|
-
class Rule < Struct.new(:exception, :
|
5
|
-
|
6
|
-
def has_children?
|
2
|
+
class Rule < Struct.new(:exception, :stop_ok, :children)
|
3
|
+
def children?
|
7
4
|
!children.empty?
|
8
5
|
end
|
6
|
+
alias :has_children? :children?
|
9
7
|
|
10
|
-
def
|
11
|
-
|
8
|
+
def suffix?
|
9
|
+
stop_ok
|
12
10
|
end
|
13
|
-
|
11
|
+
alias :is_suffix? :suffix?
|
14
12
|
end
|
15
13
|
end
|
data/lib/ryodo/rule_set.rb
CHANGED
@@ -1,8 +1,5 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
2
|
class RuleSet
|
5
|
-
|
6
3
|
def initialize
|
7
4
|
@tree = {}
|
8
5
|
build!
|
@@ -10,26 +7,21 @@ module Ryodo
|
|
10
7
|
|
11
8
|
def build!
|
12
9
|
Ryodo::SuffixList.list.each do |line|
|
13
|
-
|
14
10
|
line.each.with_index do |node_name, idx|
|
15
|
-
|
16
|
-
stopOK = node_name == line.last
|
11
|
+
stop_ok = node_name == line.last
|
17
12
|
exception = node_name[0] == "!"
|
18
13
|
node_name = node_name[1..-1] if exception
|
19
14
|
children = {}
|
20
|
-
node = Ryodo::Rule.new(exception,
|
15
|
+
node = Ryodo::Rule.new(exception, stop_ok, children)
|
21
16
|
|
22
17
|
if idx > 0
|
23
18
|
end_idx = idx - 1
|
24
|
-
parent
|
19
|
+
parent = select_rule(line[0..end_idx])
|
25
20
|
parent.children[node_name] = node unless parent.children[node_name]
|
26
|
-
|
27
21
|
else
|
28
22
|
@tree[node_name] = node unless @tree[node_name]
|
29
23
|
end
|
30
|
-
|
31
24
|
end
|
32
|
-
|
33
25
|
end
|
34
26
|
end
|
35
27
|
|
@@ -60,6 +52,5 @@ module Ryodo
|
|
60
52
|
[ suffix, [domain.shift], domain ]
|
61
53
|
end
|
62
54
|
end
|
63
|
-
|
64
55
|
end
|
65
56
|
end
|
data/lib/ryodo/suffix_list.rb
CHANGED
@@ -1,21 +1,18 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
module Ryodo
|
4
|
-
|
5
2
|
class SuffixList
|
6
|
-
def initialize
|
3
|
+
def initialize(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
7
4
|
load_file(suffix_file)
|
8
5
|
end
|
9
6
|
|
10
7
|
def parse_data
|
11
8
|
# loads and converts to array
|
12
9
|
# "baz.bar.foo" => ["baz", "bar", "foo"]
|
13
|
-
File.readlines(@suffix_file).map{ |line| line.strip.split(".") }
|
10
|
+
File.readlines(@suffix_file).map { |line| line.strip.split(".") }
|
14
11
|
end
|
15
12
|
|
16
|
-
def load_file
|
13
|
+
def load_file(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
17
14
|
@suffix_file = suffix_file
|
18
|
-
@suffix_data = parse_data
|
15
|
+
@suffix_data = parse_data << ["example"]
|
19
16
|
end
|
20
17
|
|
21
18
|
def list
|
@@ -27,12 +24,11 @@ module Ryodo
|
|
27
24
|
end
|
28
25
|
|
29
26
|
class << self
|
30
|
-
|
31
|
-
def SuffixList suffix_file = Ryodo::PUBLIC_SUFFIX_STORE
|
27
|
+
def SuffixList(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
32
28
|
instance(suffix_file)
|
33
29
|
end
|
34
30
|
|
35
|
-
def reload
|
31
|
+
def reload(suffix_file = Ryodo::PUBLIC_SUFFIX_STORE)
|
36
32
|
instance.load_file(suffix_file) && true
|
37
33
|
end
|
38
34
|
|
@@ -47,11 +43,8 @@ module Ryodo
|
|
47
43
|
def inspect
|
48
44
|
instance.inspect
|
49
45
|
end
|
50
|
-
|
51
46
|
end
|
52
47
|
|
53
48
|
private_class_method :new
|
54
|
-
|
55
49
|
end
|
56
|
-
|
57
50
|
end
|
@@ -1,33 +1,40 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
|
3
1
|
require "uri"
|
4
2
|
require "net/http"
|
5
3
|
|
6
4
|
module Ryodo
|
7
|
-
|
8
|
-
class FetchError < StandardError; end
|
5
|
+
FetchError = Class.new(StandardError)
|
9
6
|
|
10
7
|
class SuffixListFetcher
|
8
|
+
class << self
|
9
|
+
def fetch_and_save!(uri = Ryodo::PUBLIC_SUFFIX_DATA_URI, store = Ryodo::PUBLIC_SUFFIX_STORE)
|
10
|
+
fetcher = new(uri, store)
|
11
|
+
fetcher.fetch_data
|
12
|
+
fetcher.prepare_data
|
13
|
+
fetcher.save_data
|
14
|
+
true
|
15
|
+
rescue
|
16
|
+
false
|
17
|
+
end
|
18
|
+
end
|
11
19
|
|
12
|
-
def initialize
|
13
|
-
@uri = URI(uri)
|
20
|
+
def initialize(uri = Ryodo::PUBLIC_SUFFIX_DATA_URI, store = Ryodo::PUBLIC_SUFFIX_STORE)
|
21
|
+
@uri = URI.parse(uri)
|
14
22
|
@store = store
|
15
23
|
end
|
16
24
|
|
17
25
|
def fetch_data
|
18
|
-
|
19
|
-
|
20
|
-
|
26
|
+
http = Net::HTTP.new(@uri.host, @uri.port)
|
27
|
+
http.use_ssl = @uri.scheme == "https"
|
28
|
+
request = Net::HTTP::Get.new(@uri.request_uri)
|
29
|
+
response = http.request(request)
|
30
|
+
fail Ryodo::FetchError, "Could not fetch suffix data! (#{response})" unless response.is_a?(Net::HTTPSuccess)
|
31
|
+
@fetched_data = response.body.lines
|
21
32
|
end
|
22
33
|
|
23
34
|
def prepare_data
|
24
35
|
@prepared_data = @fetched_data.inject([]) do |acc, line|
|
25
|
-
next(acc) if line =~
|
26
|
-
|
27
|
-
split(".").
|
28
|
-
reverse.
|
29
|
-
join(".") # "foo.bar.baz" => "baz.bar.foo"
|
30
|
-
acc << dns_line
|
36
|
+
next(acc) if line =~ %r{\A//|\A\n}
|
37
|
+
acc << reverse_dn(line)
|
31
38
|
end.sort
|
32
39
|
end
|
33
40
|
|
@@ -37,18 +44,11 @@ module Ryodo
|
|
37
44
|
end if @prepared_data
|
38
45
|
end
|
39
46
|
|
40
|
-
|
41
|
-
def fetch_and_save! uri = Ryodo::PUBLIC_SUFFIX_DATA_URI, store = Ryodo::PUBLIC_SUFFIX_STORE
|
42
|
-
fetcher = self.new uri, store
|
43
|
-
fetcher.fetch_data
|
44
|
-
fetcher.prepare_data
|
45
|
-
fetcher.save_data
|
46
|
-
true
|
47
|
-
rescue
|
48
|
-
false
|
49
|
-
end
|
50
|
-
end
|
47
|
+
private
|
51
48
|
|
49
|
+
def reverse_dn(domain_name)
|
50
|
+
# "foo.bar.baz" => "baz.bar.foo"
|
51
|
+
domain_name.strip.split(".").reverse.join(".")
|
52
|
+
end
|
52
53
|
end
|
53
|
-
|
54
54
|
end
|