wmap 2.7.2 → 2.7.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -16
- data/bin/refresh +5 -5
- data/lib/wmap.rb +1 -1
- data/lib/wmap/site_tracker.rb +1 -4
- data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
- data/lib/wmap/url_crawler.rb +1 -1
- data/version.txt +2 -2
- data/wmap.gemspec +2 -2
- metadata +4 -59
- data/ruby_whois_patches/base_cocca2.rb +0 -149
- data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
- data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
- data/ruby_whois_patches/whois.above.com.rb +0 -61
- data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
- data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
- data/ruby_whois_patches/whois.ai.rb +0 -112
- data/ruby_whois_patches/whois.arnes.si.rb +0 -121
- data/ruby_whois_patches/whois.ascio.com.rb +0 -91
- data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
- data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
- data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
- data/ruby_whois_patches/whois.denic.de.rb +0 -174
- data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
- data/ruby_whois_patches/whois.dns.be.rb +0 -134
- data/ruby_whois_patches/whois.dns.lu.rb +0 -129
- data/ruby_whois_patches/whois.dns.pl.rb +0 -150
- data/ruby_whois_patches/whois.dns.pt.rb +0 -119
- data/ruby_whois_patches/whois.domain.kg.rb +0 -126
- data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
- data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
- data/ruby_whois_patches/whois.dot.tk.rb +0 -140
- data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
- data/ruby_whois_patches/whois.isnic.is.rb +0 -130
- data/ruby_whois_patches/whois.je.rb +0 -119
- data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
- data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
- data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
- data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
- data/ruby_whois_patches/whois.nic.as.rb +0 -96
- data/ruby_whois_patches/whois.nic.at.rb +0 -109
- data/ruby_whois_patches/whois.nic.ch.rb +0 -141
- data/ruby_whois_patches/whois.nic.cl.rb +0 -117
- data/ruby_whois_patches/whois.nic.ec.rb +0 -157
- data/ruby_whois_patches/whois.nic.im.rb +0 -120
- data/ruby_whois_patches/whois.nic.it.rb +0 -170
- data/ruby_whois_patches/whois.nic.lv.rb +0 -116
- data/ruby_whois_patches/whois.nic.ly.rb +0 -127
- data/ruby_whois_patches/whois.nic.mu.rb +0 -27
- data/ruby_whois_patches/whois.nic.mx.rb +0 -123
- data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
- data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
- data/ruby_whois_patches/whois.nic.tel.rb +0 -129
- data/ruby_whois_patches/whois.nic.tr.rb +0 -133
- data/ruby_whois_patches/whois.nic.us.rb +0 -129
- data/ruby_whois_patches/whois.nic.ve.rb +0 -135
- data/ruby_whois_patches/whois.norid.no.rb +0 -127
- data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
- data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
- data/ruby_whois_patches/whois.registro.br.rb +0 -109
- data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
- data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
- data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
- data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
- data/ruby_whois_patches/whois.tucows.com.rb +0 -70
- data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,67 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Ruby Whois
|
3
|
-
#
|
4
|
-
# An intelligent pure Ruby WHOIS client and parser.
|
5
|
-
#
|
6
|
-
# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
|
7
|
-
#++
|
8
|
-
|
9
|
-
require 'whois/record/parser/base'
|
10
|
-
|
11
|
-
module Whois
|
12
|
-
class Record
|
13
|
-
class Parser
|
14
|
-
|
15
|
-
# Parser for the whois.corporatedomains.com server, added by Yang Li 02/05/2013.
|
16
|
-
class WhoisCorporatedomainsCom < Base
|
17
|
-
|
18
|
-
property_supported :registrar do
|
19
|
-
reg=Record::Registrar.new
|
20
|
-
content_for_scanner.scan(/Registrar(.+):\s+(.+)\n/).map do |entry|
|
21
|
-
reg["name"] = entry[1] if entry[0] =~ /Name/i
|
22
|
-
reg["organization"] = entry[1] if entry[0] =~ /Name/i
|
23
|
-
reg["url"] = entry[1] if entry[0] =~ /Homepage/i
|
24
|
-
end
|
25
|
-
return reg
|
26
|
-
end
|
27
|
-
|
28
|
-
property_supported :admin_contacts do
|
29
|
-
build_contact("Administrative contact", Whois::Record::Contact::TYPE_ADMIN)
|
30
|
-
end
|
31
|
-
|
32
|
-
property_supported :registrant_contacts do
|
33
|
-
build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
|
34
|
-
end
|
35
|
-
|
36
|
-
property_supported :technical_contacts do
|
37
|
-
build_contact("Technical contact", Whois::Record::Contact::TYPE_TECHNICAL)
|
38
|
-
end
|
39
|
-
|
40
|
-
property_not_supported :billing_contacts
|
41
|
-
|
42
|
-
# The following methods are implemented by Yang Li on 02/05/2013
|
43
|
-
private
|
44
|
-
|
45
|
-
def build_contact(element, type)
|
46
|
-
reg=Record::Contact.new(:type => type)
|
47
|
-
if content_for_scanner =~ /#{element}:\s*\n((.+\n)+)\n/i
|
48
|
-
line_num=1
|
49
|
-
$1.split(%r{\n}).each do |line|
|
50
|
-
reg["organization"]=line.strip if line_num==1
|
51
|
-
reg["name"]=line.strip if line_num==2
|
52
|
-
reg["address"]=line.strip if line_num==3
|
53
|
-
reg["city"]=line.strip if line_num==4
|
54
|
-
reg["country_code"]=line.strip if line_num==5
|
55
|
-
reg["email"]=line.split(':')[1].strip if line=~ /\w+\@\w+\.\w/
|
56
|
-
reg["phone"]=line.split(':')[1].strip if line=~ /Phone/
|
57
|
-
line_num=line_num+1
|
58
|
-
end
|
59
|
-
end
|
60
|
-
return reg
|
61
|
-
end
|
62
|
-
# ----------------------------------------------------------------------------
|
63
|
-
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
@@ -1,108 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Ruby Whois
|
3
|
-
#
|
4
|
-
# An intelligent pure Ruby WHOIS client and parser.
|
5
|
-
#
|
6
|
-
# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
|
7
|
-
#++
|
8
|
-
require 'whois/record/parser/base'
|
9
|
-
require 'whois/record/scanners/verisign'
|
10
|
-
|
11
|
-
|
12
|
-
module Whois
|
13
|
-
class Record
|
14
|
-
class Parser
|
15
|
-
|
16
|
-
# Parser for the whois.crsnic.net server.
|
17
|
-
class WhoisCrsnicNet < Base
|
18
|
-
include Scanners::Ast
|
19
|
-
|
20
|
-
property_supported :disclaimer do
|
21
|
-
node("Disclaimer")
|
22
|
-
end
|
23
|
-
|
24
|
-
property_supported :domain do
|
25
|
-
node("Domain Name") { |raw| raw.downcase }
|
26
|
-
end
|
27
|
-
|
28
|
-
property_not_supported :domain_id
|
29
|
-
|
30
|
-
property_supported :referral_whois do
|
31
|
-
node("Whois Server")
|
32
|
-
end
|
33
|
-
|
34
|
-
property_supported :referral_url do
|
35
|
-
node("Referral URL") do |raw|
|
36
|
-
last_useful_item(raw)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
property_supported :status do
|
41
|
-
node("Status")
|
42
|
-
end
|
43
|
-
|
44
|
-
property_supported :available? do
|
45
|
-
node("Registrar").nil?
|
46
|
-
end
|
47
|
-
|
48
|
-
property_supported :registered? do
|
49
|
-
!available?
|
50
|
-
end
|
51
|
-
|
52
|
-
property_supported :created_on do
|
53
|
-
node("Creation Date") { |raw| Time.parse(raw) }
|
54
|
-
end
|
55
|
-
|
56
|
-
property_supported :updated_on do
|
57
|
-
node("Updated Date") { |raw| Time.parse(raw) }
|
58
|
-
end
|
59
|
-
|
60
|
-
property_supported :expires_on do
|
61
|
-
node("Expiration Date") { |raw| Time.parse(raw) }
|
62
|
-
end
|
63
|
-
|
64
|
-
property_supported :registrar do
|
65
|
-
node("Registrar") do |raw|
|
66
|
-
Whois::Record::Registrar.new(
|
67
|
-
:name => last_useful_item(raw),
|
68
|
-
:organization => last_useful_item(raw),
|
69
|
-
:url => referral_url
|
70
|
-
)
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
property_supported :nameservers do
|
75
|
-
Array.wrap(node("Name Server")).reject { |value| value =~ / / }.map do |name|
|
76
|
-
Record::Nameserver.new(:name => name.downcase)
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
|
-
# Checks whether this response contains a message
|
81
|
-
# that can be reconducted to a "WHOIS Server Unavailable" status.
|
82
|
-
#
|
83
|
-
# @return [Boolean]
|
84
|
-
def response_unavailable?
|
85
|
-
!!node("response:unavailable")
|
86
|
-
end
|
87
|
-
|
88
|
-
# Initializes a new {Scanners::Verisign} instance
|
89
|
-
# passing the #{content_for_scanner}
|
90
|
-
# and calls +parse+ on it.
|
91
|
-
#
|
92
|
-
# @return [Hash]
|
93
|
-
def parse
|
94
|
-
Scanners::Verisign.new(content_for_scanner).parse
|
95
|
-
end
|
96
|
-
|
97
|
-
private
|
98
|
-
|
99
|
-
# In case of "SPAM Response", the response contains more than one item
|
100
|
-
# for the same value and the value becomes an Array.
|
101
|
-
def last_useful_item(values)
|
102
|
-
values.is_a?(Array) ? values.last : values
|
103
|
-
end
|
104
|
-
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
@@ -1,174 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Ruby Whois
|
3
|
-
#
|
4
|
-
# An intelligent pure Ruby WHOIS client and parser.
|
5
|
-
#
|
6
|
-
# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
|
7
|
-
#++
|
8
|
-
|
9
|
-
|
10
|
-
require 'whois/record/parser/base'
|
11
|
-
require 'whois/record/scanners/whois.denic.de.rb'
|
12
|
-
|
13
|
-
|
14
|
-
module Whois
|
15
|
-
class Record
|
16
|
-
class Parser
|
17
|
-
|
18
|
-
# Parser for the whois.denic.de server.
|
19
|
-
#
|
20
|
-
# @author Simone Carletti <weppos@weppos.net>
|
21
|
-
# @author Aaron Mueller <mail@aaron-mueller.de>
|
22
|
-
#
|
23
|
-
class WhoisDenicDe < Base
|
24
|
-
include Scanners::Nodable
|
25
|
-
|
26
|
-
property_supported :disclaimer do
|
27
|
-
node("Disclaimer")
|
28
|
-
end
|
29
|
-
|
30
|
-
|
31
|
-
property_supported :domain do
|
32
|
-
node("Domain")
|
33
|
-
end
|
34
|
-
|
35
|
-
property_not_supported :domain_id
|
36
|
-
|
37
|
-
|
38
|
-
property_supported :status do
|
39
|
-
if node("Status")
|
40
|
-
case node("Status")
|
41
|
-
when "connect" then :registered
|
42
|
-
when "free" then :available
|
43
|
-
when "invalid" then :invalid
|
44
|
-
# NEWSTATUS (inactive)
|
45
|
-
# The domain is registered, but there is not DNS entry for it.
|
46
|
-
when "failed" then :registered
|
47
|
-
else
|
48
|
-
Whois.bug!(ParserError, "Unknown status `#{node("Status")}'.")
|
49
|
-
end
|
50
|
-
else
|
51
|
-
if version < "2.0"
|
52
|
-
if invalid?
|
53
|
-
:invalid
|
54
|
-
else
|
55
|
-
:available
|
56
|
-
end
|
57
|
-
else
|
58
|
-
Whois.bug!(ParserError, "Unable to parse status.")
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
property_supported :available? do
|
64
|
-
!invalid? && (!!node("status:available") || node("Status") == "free")
|
65
|
-
end
|
66
|
-
|
67
|
-
property_supported :registered? do
|
68
|
-
!invalid? && !available?
|
69
|
-
end
|
70
|
-
|
71
|
-
|
72
|
-
property_not_supported :created_on
|
73
|
-
|
74
|
-
property_supported :updated_on do
|
75
|
-
node("Changed") { |raw| Time.parse(raw) }
|
76
|
-
end
|
77
|
-
|
78
|
-
property_not_supported :expires_on
|
79
|
-
|
80
|
-
|
81
|
-
property_supported :registrar do
|
82
|
-
node("Zone-C") do |raw|
|
83
|
-
Record::Registrar.new(
|
84
|
-
:id => nil,
|
85
|
-
:name => raw["name"],
|
86
|
-
:organization => raw["organization"],
|
87
|
-
:url => nil
|
88
|
-
)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
property_supported :registrant_contacts do
|
93
|
-
build_contact("Holder", Whois::Record::Contact::TYPE_REGISTRANT)
|
94
|
-
end
|
95
|
-
|
96
|
-
property_supported :admin_contacts do
|
97
|
-
build_contact("Admin-C", Whois::Record::Contact::TYPE_ADMIN)
|
98
|
-
end
|
99
|
-
|
100
|
-
# FIXME: check against different schema
|
101
|
-
|
102
|
-
property_supported :technical_contacts do
|
103
|
-
build_contact("Tech-C", Whois::Record::Contact::TYPE_TECHNICAL)
|
104
|
-
end
|
105
|
-
|
106
|
-
|
107
|
-
# Nameservers are listed in the following formats:
|
108
|
-
#
|
109
|
-
# Nserver: ns1.prodns.de. 213.160.64.75
|
110
|
-
# Nserver: ns1.prodns.de.
|
111
|
-
#
|
112
|
-
property_supported :nameservers do
|
113
|
-
node("Nserver") do |values|
|
114
|
-
values.map do |line|
|
115
|
-
name, ipv4 = line.split(/\s+/)
|
116
|
-
Record::Nameserver.new(:name => name.chomp("."), :ipv4 => ipv4)
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
|
122
|
-
# Checks whether the response has been throttled.
|
123
|
-
#
|
124
|
-
# @return [Boolean]
|
125
|
-
#
|
126
|
-
# @example
|
127
|
-
# % Error: 55000000002 Connection refused; access control limit reached.
|
128
|
-
#
|
129
|
-
def response_throttled?
|
130
|
-
!!node("response:throttled")
|
131
|
-
end
|
132
|
-
|
133
|
-
|
134
|
-
# NEWPROPERTY
|
135
|
-
def version
|
136
|
-
cached_properties_fetch :version do
|
137
|
-
if content_for_scanner =~ /^% Version: (.+)$/
|
138
|
-
$1
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
|
143
|
-
# NEWPROPERTY
|
144
|
-
def invalid?
|
145
|
-
cached_properties_fetch :invalid? do
|
146
|
-
!!node("status:invalid") || node("Status") == "invalid"
|
147
|
-
end
|
148
|
-
end
|
149
|
-
|
150
|
-
|
151
|
-
# Initializes a new {Scanners::WhoisDenicDe} instance
|
152
|
-
# passing the {#content_for_scanner}
|
153
|
-
# and calls +parse+ on it.
|
154
|
-
#
|
155
|
-
# @return [Hash]
|
156
|
-
def parse
|
157
|
-
Scanners::WhoisDenicDe.new(content_for_scanner).parse
|
158
|
-
end
|
159
|
-
|
160
|
-
|
161
|
-
private
|
162
|
-
|
163
|
-
def build_contact(element, type)
|
164
|
-
node(element) do |raw|
|
165
|
-
Record::Contact.new(raw) do |c|
|
166
|
-
c.type = type
|
167
|
-
end
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|
174
|
-
end
|
@@ -1,120 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Ruby Whois
|
3
|
-
#
|
4
|
-
# An intelligent pure Ruby WHOIS client and parser.
|
5
|
-
#
|
6
|
-
# Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
|
7
|
-
#++
|
8
|
-
|
9
|
-
|
10
|
-
require 'whois/record/parser/base'
|
11
|
-
|
12
|
-
|
13
|
-
module Whois
|
14
|
-
class Record
|
15
|
-
class Parser
|
16
|
-
|
17
|
-
# Parser for the whois.dk-hostmaster.dk server.
|
18
|
-
#
|
19
|
-
# @note This parser is just a stub and provides only a few basic methods
|
20
|
-
# to check for domain availability and get domain status.
|
21
|
-
# Please consider to contribute implementing missing methods.
|
22
|
-
#
|
23
|
-
# @see Whois::Record::Parser::Example
|
24
|
-
# The Example parser for the list of all available methods.
|
25
|
-
#
|
26
|
-
# @author Simone Carletti <weppos@weppos.net>
|
27
|
-
# @author Mikkel Kristensen <mikkel@tdx.dk>
|
28
|
-
#
|
29
|
-
class WhoisDkHostmasterDk < Base
|
30
|
-
|
31
|
-
property_supported :status do
|
32
|
-
if content_for_scanner =~ /Status:\s+(.+?)\n/
|
33
|
-
case $1.downcase
|
34
|
-
when "active"
|
35
|
-
:registered
|
36
|
-
when "deactivated"
|
37
|
-
:expired
|
38
|
-
else
|
39
|
-
Whois.bug!(ParserError, "Unknown status `#{$1}'.")
|
40
|
-
end
|
41
|
-
else
|
42
|
-
:available
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
property_supported :available? do
|
47
|
-
!!(content_for_scanner =~ /^No entries found for the selected source/)
|
48
|
-
end
|
49
|
-
|
50
|
-
property_supported :registered? do
|
51
|
-
!available?
|
52
|
-
end
|
53
|
-
|
54
|
-
|
55
|
-
property_supported :created_on do
|
56
|
-
if content_for_scanner =~ /Registered:\s+(.*)\n/
|
57
|
-
Time.parse($1)
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
property_not_supported :updated_on
|
62
|
-
|
63
|
-
property_supported :expires_on do
|
64
|
-
if content_for_scanner =~ /Expires:\s+(.*)\n/
|
65
|
-
Time.parse($1)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
property_supported :nameservers do
|
70
|
-
content_for_scanner.scan(/Hostname:\s+(.+)\n/).flatten.map do |name|
|
71
|
-
Record::Nameserver.new(:name => name)
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
# The following methods are implemented by Yang Li on 01/29/2013
|
76
|
-
# ----------------------------------------------------------------------------
|
77
|
-
property_supported :domain do
|
78
|
-
return $1 if content_for_scanner =~ /Domain:\s+(.+)\n/i
|
79
|
-
end
|
80
|
-
|
81
|
-
property_not_supported :domain_id
|
82
|
-
|
83
|
-
property_not_supported :registrar
|
84
|
-
|
85
|
-
property_supported :admin_contacts do
|
86
|
-
build_contact("Administrator", Whois::Record::Contact::TYPE_ADMIN)
|
87
|
-
end
|
88
|
-
|
89
|
-
property_supported :registrant_contacts do
|
90
|
-
build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
|
91
|
-
end
|
92
|
-
|
93
|
-
property_not_supported :technical_contacts
|
94
|
-
|
95
|
-
property_not_supported :billing_contacts
|
96
|
-
|
97
|
-
private
|
98
|
-
|
99
|
-
def build_contact(element, type)
|
100
|
-
reg=Record::Contact.new(:type => type)
|
101
|
-
if content_for_scanner =~ /^#{element}\n((.+\n)+)\n/i
|
102
|
-
$1.scan(/^(.+):\s+(.+)\n/).map do |entry|
|
103
|
-
reg["id"]=entry[1] if entry[0] =~ /Handle/i
|
104
|
-
reg["name"]=entry[1] if entry[0] =~ /Name/i
|
105
|
-
reg["organization"]=entry[1] if entry[0]=~ /Name/i
|
106
|
-
reg["address"]=entry[1] if entry[0]=~ /Address/i
|
107
|
-
reg["city"]= entry[1] if entry[0]=~ /City/i
|
108
|
-
reg["zip"]=entry[1] if entry[0]=~ /PostalCode/i
|
109
|
-
reg["country_code"]=entry[1] if entry[0]=~ /Country/i
|
110
|
-
reg["phone"]=entry[1] if entry[0]=~ /Phone/i
|
111
|
-
end
|
112
|
-
end
|
113
|
-
return reg
|
114
|
-
end
|
115
|
-
# ----------------------------------------------------------------------------
|
116
|
-
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|