wmap 2.7.2 → 2.7.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -16
  3. data/bin/refresh +5 -5
  4. data/lib/wmap.rb +1 -1
  5. data/lib/wmap/site_tracker.rb +1 -4
  6. data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
  7. data/lib/wmap/url_crawler.rb +1 -1
  8. data/version.txt +2 -2
  9. data/wmap.gemspec +2 -2
  10. metadata +4 -59
  11. data/ruby_whois_patches/base_cocca2.rb +0 -149
  12. data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
  13. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
  14. data/ruby_whois_patches/whois.above.com.rb +0 -61
  15. data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
  16. data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
  17. data/ruby_whois_patches/whois.ai.rb +0 -112
  18. data/ruby_whois_patches/whois.arnes.si.rb +0 -121
  19. data/ruby_whois_patches/whois.ascio.com.rb +0 -91
  20. data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
  21. data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
  22. data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
  23. data/ruby_whois_patches/whois.denic.de.rb +0 -174
  24. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
  25. data/ruby_whois_patches/whois.dns.be.rb +0 -134
  26. data/ruby_whois_patches/whois.dns.lu.rb +0 -129
  27. data/ruby_whois_patches/whois.dns.pl.rb +0 -150
  28. data/ruby_whois_patches/whois.dns.pt.rb +0 -119
  29. data/ruby_whois_patches/whois.domain.kg.rb +0 -126
  30. data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
  31. data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
  32. data/ruby_whois_patches/whois.dot.tk.rb +0 -140
  33. data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
  34. data/ruby_whois_patches/whois.isnic.is.rb +0 -130
  35. data/ruby_whois_patches/whois.je.rb +0 -119
  36. data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
  37. data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
  38. data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
  39. data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
  40. data/ruby_whois_patches/whois.nic.as.rb +0 -96
  41. data/ruby_whois_patches/whois.nic.at.rb +0 -109
  42. data/ruby_whois_patches/whois.nic.ch.rb +0 -141
  43. data/ruby_whois_patches/whois.nic.cl.rb +0 -117
  44. data/ruby_whois_patches/whois.nic.ec.rb +0 -157
  45. data/ruby_whois_patches/whois.nic.im.rb +0 -120
  46. data/ruby_whois_patches/whois.nic.it.rb +0 -170
  47. data/ruby_whois_patches/whois.nic.lv.rb +0 -116
  48. data/ruby_whois_patches/whois.nic.ly.rb +0 -127
  49. data/ruby_whois_patches/whois.nic.mu.rb +0 -27
  50. data/ruby_whois_patches/whois.nic.mx.rb +0 -123
  51. data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
  52. data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
  53. data/ruby_whois_patches/whois.nic.tel.rb +0 -129
  54. data/ruby_whois_patches/whois.nic.tr.rb +0 -133
  55. data/ruby_whois_patches/whois.nic.us.rb +0 -129
  56. data/ruby_whois_patches/whois.nic.ve.rb +0 -135
  57. data/ruby_whois_patches/whois.norid.no.rb +0 -127
  58. data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
  59. data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
  60. data/ruby_whois_patches/whois.registro.br.rb +0 -109
  61. data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
  62. data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
  63. data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
  64. data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
  65. data/ruby_whois_patches/whois.tucows.com.rb +0 -70
  66. data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,67 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
- require 'whois/record/parser/base'
10
-
11
- module Whois
12
- class Record
13
- class Parser
14
-
15
- # Parser for the whois.corporatedomains.com server, added by Yang Li 02/05/2013.
16
- class WhoisCorporatedomainsCom < Base
17
-
18
- property_supported :registrar do
19
- reg=Record::Registrar.new
20
- content_for_scanner.scan(/Registrar(.+):\s+(.+)\n/).map do |entry|
21
- reg["name"] = entry[1] if entry[0] =~ /Name/i
22
- reg["organization"] = entry[1] if entry[0] =~ /Name/i
23
- reg["url"] = entry[1] if entry[0] =~ /Homepage/i
24
- end
25
- return reg
26
- end
27
-
28
- property_supported :admin_contacts do
29
- build_contact("Administrative contact", Whois::Record::Contact::TYPE_ADMIN)
30
- end
31
-
32
- property_supported :registrant_contacts do
33
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
34
- end
35
-
36
- property_supported :technical_contacts do
37
- build_contact("Technical contact", Whois::Record::Contact::TYPE_TECHNICAL)
38
- end
39
-
40
- property_not_supported :billing_contacts
41
-
42
- # The following methods are implemented by Yang Li on 02/05/2013
43
- private
44
-
45
- def build_contact(element, type)
46
- reg=Record::Contact.new(:type => type)
47
- if content_for_scanner =~ /#{element}:\s*\n((.+\n)+)\n/i
48
- line_num=1
49
- $1.split(%r{\n}).each do |line|
50
- reg["organization"]=line.strip if line_num==1
51
- reg["name"]=line.strip if line_num==2
52
- reg["address"]=line.strip if line_num==3
53
- reg["city"]=line.strip if line_num==4
54
- reg["country_code"]=line.strip if line_num==5
55
- reg["email"]=line.split(':')[1].strip if line=~ /\w+\@\w+\.\w/
56
- reg["phone"]=line.split(':')[1].strip if line=~ /Phone/
57
- line_num=line_num+1
58
- end
59
- end
60
- return reg
61
- end
62
- # ----------------------------------------------------------------------------
63
-
64
- end
65
- end
66
- end
67
- end
@@ -1,108 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
- require 'whois/record/parser/base'
9
- require 'whois/record/scanners/verisign'
10
-
11
-
12
- module Whois
13
- class Record
14
- class Parser
15
-
16
- # Parser for the whois.crsnic.net server.
17
- class WhoisCrsnicNet < Base
18
- include Scanners::Ast
19
-
20
- property_supported :disclaimer do
21
- node("Disclaimer")
22
- end
23
-
24
- property_supported :domain do
25
- node("Domain Name") { |raw| raw.downcase }
26
- end
27
-
28
- property_not_supported :domain_id
29
-
30
- property_supported :referral_whois do
31
- node("Whois Server")
32
- end
33
-
34
- property_supported :referral_url do
35
- node("Referral URL") do |raw|
36
- last_useful_item(raw)
37
- end
38
- end
39
-
40
- property_supported :status do
41
- node("Status")
42
- end
43
-
44
- property_supported :available? do
45
- node("Registrar").nil?
46
- end
47
-
48
- property_supported :registered? do
49
- !available?
50
- end
51
-
52
- property_supported :created_on do
53
- node("Creation Date") { |raw| Time.parse(raw) }
54
- end
55
-
56
- property_supported :updated_on do
57
- node("Updated Date") { |raw| Time.parse(raw) }
58
- end
59
-
60
- property_supported :expires_on do
61
- node("Expiration Date") { |raw| Time.parse(raw) }
62
- end
63
-
64
- property_supported :registrar do
65
- node("Registrar") do |raw|
66
- Whois::Record::Registrar.new(
67
- :name => last_useful_item(raw),
68
- :organization => last_useful_item(raw),
69
- :url => referral_url
70
- )
71
- end
72
- end
73
-
74
- property_supported :nameservers do
75
- Array.wrap(node("Name Server")).reject { |value| value =~ / / }.map do |name|
76
- Record::Nameserver.new(:name => name.downcase)
77
- end
78
- end
79
-
80
- # Checks whether this response contains a message
81
- # that can be reconducted to a "WHOIS Server Unavailable" status.
82
- #
83
- # @return [Boolean]
84
- def response_unavailable?
85
- !!node("response:unavailable")
86
- end
87
-
88
- # Initializes a new {Scanners::Verisign} instance
89
- # passing the #{content_for_scanner}
90
- # and calls +parse+ on it.
91
- #
92
- # @return [Hash]
93
- def parse
94
- Scanners::Verisign.new(content_for_scanner).parse
95
- end
96
-
97
- private
98
-
99
- # In case of "SPAM Response", the response contains more than one item
100
- # for the same value and the value becomes an Array.
101
- def last_useful_item(values)
102
- values.is_a?(Array) ? values.last : values
103
- end
104
-
105
- end
106
- end
107
- end
108
- end
@@ -1,174 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
- require 'whois/record/scanners/whois.denic.de.rb'
12
-
13
-
14
- module Whois
15
- class Record
16
- class Parser
17
-
18
- # Parser for the whois.denic.de server.
19
- #
20
- # @author Simone Carletti <weppos@weppos.net>
21
- # @author Aaron Mueller <mail@aaron-mueller.de>
22
- #
23
- class WhoisDenicDe < Base
24
- include Scanners::Nodable
25
-
26
- property_supported :disclaimer do
27
- node("Disclaimer")
28
- end
29
-
30
-
31
- property_supported :domain do
32
- node("Domain")
33
- end
34
-
35
- property_not_supported :domain_id
36
-
37
-
38
- property_supported :status do
39
- if node("Status")
40
- case node("Status")
41
- when "connect" then :registered
42
- when "free" then :available
43
- when "invalid" then :invalid
44
- # NEWSTATUS (inactive)
45
- # The domain is registered, but there is not DNS entry for it.
46
- when "failed" then :registered
47
- else
48
- Whois.bug!(ParserError, "Unknown status `#{node("Status")}'.")
49
- end
50
- else
51
- if version < "2.0"
52
- if invalid?
53
- :invalid
54
- else
55
- :available
56
- end
57
- else
58
- Whois.bug!(ParserError, "Unable to parse status.")
59
- end
60
- end
61
- end
62
-
63
- property_supported :available? do
64
- !invalid? && (!!node("status:available") || node("Status") == "free")
65
- end
66
-
67
- property_supported :registered? do
68
- !invalid? && !available?
69
- end
70
-
71
-
72
- property_not_supported :created_on
73
-
74
- property_supported :updated_on do
75
- node("Changed") { |raw| Time.parse(raw) }
76
- end
77
-
78
- property_not_supported :expires_on
79
-
80
-
81
- property_supported :registrar do
82
- node("Zone-C") do |raw|
83
- Record::Registrar.new(
84
- :id => nil,
85
- :name => raw["name"],
86
- :organization => raw["organization"],
87
- :url => nil
88
- )
89
- end
90
- end
91
-
92
- property_supported :registrant_contacts do
93
- build_contact("Holder", Whois::Record::Contact::TYPE_REGISTRANT)
94
- end
95
-
96
- property_supported :admin_contacts do
97
- build_contact("Admin-C", Whois::Record::Contact::TYPE_ADMIN)
98
- end
99
-
100
- # FIXME: check against different schema
101
-
102
- property_supported :technical_contacts do
103
- build_contact("Tech-C", Whois::Record::Contact::TYPE_TECHNICAL)
104
- end
105
-
106
-
107
- # Nameservers are listed in the following formats:
108
- #
109
- # Nserver: ns1.prodns.de. 213.160.64.75
110
- # Nserver: ns1.prodns.de.
111
- #
112
- property_supported :nameservers do
113
- node("Nserver") do |values|
114
- values.map do |line|
115
- name, ipv4 = line.split(/\s+/)
116
- Record::Nameserver.new(:name => name.chomp("."), :ipv4 => ipv4)
117
- end
118
- end
119
- end
120
-
121
-
122
- # Checks whether the response has been throttled.
123
- #
124
- # @return [Boolean]
125
- #
126
- # @example
127
- # % Error: 55000000002 Connection refused; access control limit reached.
128
- #
129
- def response_throttled?
130
- !!node("response:throttled")
131
- end
132
-
133
-
134
- # NEWPROPERTY
135
- def version
136
- cached_properties_fetch :version do
137
- if content_for_scanner =~ /^% Version: (.+)$/
138
- $1
139
- end
140
- end
141
- end
142
-
143
- # NEWPROPERTY
144
- def invalid?
145
- cached_properties_fetch :invalid? do
146
- !!node("status:invalid") || node("Status") == "invalid"
147
- end
148
- end
149
-
150
-
151
- # Initializes a new {Scanners::WhoisDenicDe} instance
152
- # passing the {#content_for_scanner}
153
- # and calls +parse+ on it.
154
- #
155
- # @return [Hash]
156
- def parse
157
- Scanners::WhoisDenicDe.new(content_for_scanner).parse
158
- end
159
-
160
-
161
- private
162
-
163
- def build_contact(element, type)
164
- node(element) do |raw|
165
- Record::Contact.new(raw) do |c|
166
- c.type = type
167
- end
168
- end
169
- end
170
-
171
- end
172
- end
173
- end
174
- end
@@ -1,120 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dk-hostmaster.dk server.
18
- #
19
- # @note This parser is just a stub and provides only a few basic methods
20
- # to check for domain availability and get domain status.
21
- # Please consider to contribute implementing missing methods.
22
- #
23
- # @see Whois::Record::Parser::Example
24
- # The Example parser for the list of all available methods.
25
- #
26
- # @author Simone Carletti <weppos@weppos.net>
27
- # @author Mikkel Kristensen <mikkel@tdx.dk>
28
- #
29
- class WhoisDkHostmasterDk < Base
30
-
31
- property_supported :status do
32
- if content_for_scanner =~ /Status:\s+(.+?)\n/
33
- case $1.downcase
34
- when "active"
35
- :registered
36
- when "deactivated"
37
- :expired
38
- else
39
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
40
- end
41
- else
42
- :available
43
- end
44
- end
45
-
46
- property_supported :available? do
47
- !!(content_for_scanner =~ /^No entries found for the selected source/)
48
- end
49
-
50
- property_supported :registered? do
51
- !available?
52
- end
53
-
54
-
55
- property_supported :created_on do
56
- if content_for_scanner =~ /Registered:\s+(.*)\n/
57
- Time.parse($1)
58
- end
59
- end
60
-
61
- property_not_supported :updated_on
62
-
63
- property_supported :expires_on do
64
- if content_for_scanner =~ /Expires:\s+(.*)\n/
65
- Time.parse($1)
66
- end
67
- end
68
-
69
- property_supported :nameservers do
70
- content_for_scanner.scan(/Hostname:\s+(.+)\n/).flatten.map do |name|
71
- Record::Nameserver.new(:name => name)
72
- end
73
- end
74
-
75
- # The following methods are implemented by Yang Li on 01/29/2013
76
- # ----------------------------------------------------------------------------
77
- property_supported :domain do
78
- return $1 if content_for_scanner =~ /Domain:\s+(.+)\n/i
79
- end
80
-
81
- property_not_supported :domain_id
82
-
83
- property_not_supported :registrar
84
-
85
- property_supported :admin_contacts do
86
- build_contact("Administrator", Whois::Record::Contact::TYPE_ADMIN)
87
- end
88
-
89
- property_supported :registrant_contacts do
90
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
91
- end
92
-
93
- property_not_supported :technical_contacts
94
-
95
- property_not_supported :billing_contacts
96
-
97
- private
98
-
99
- def build_contact(element, type)
100
- reg=Record::Contact.new(:type => type)
101
- if content_for_scanner =~ /^#{element}\n((.+\n)+)\n/i
102
- $1.scan(/^(.+):\s+(.+)\n/).map do |entry|
103
- reg["id"]=entry[1] if entry[0] =~ /Handle/i
104
- reg["name"]=entry[1] if entry[0] =~ /Name/i
105
- reg["organization"]=entry[1] if entry[0]=~ /Name/i
106
- reg["address"]=entry[1] if entry[0]=~ /Address/i
107
- reg["city"]= entry[1] if entry[0]=~ /City/i
108
- reg["zip"]=entry[1] if entry[0]=~ /PostalCode/i
109
- reg["country_code"]=entry[1] if entry[0]=~ /Country/i
110
- reg["phone"]=entry[1] if entry[0]=~ /Phone/i
111
- end
112
- end
113
- return reg
114
- end
115
- # ----------------------------------------------------------------------------
116
-
117
- end
118
- end
119
- end
120
- end