wmap 2.7.2 → 2.7.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -16
  3. data/bin/refresh +5 -5
  4. data/lib/wmap.rb +1 -1
  5. data/lib/wmap/site_tracker.rb +1 -4
  6. data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
  7. data/lib/wmap/url_crawler.rb +1 -1
  8. data/version.txt +2 -2
  9. data/wmap.gemspec +2 -2
  10. metadata +4 -59
  11. data/ruby_whois_patches/base_cocca2.rb +0 -149
  12. data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
  13. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
  14. data/ruby_whois_patches/whois.above.com.rb +0 -61
  15. data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
  16. data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
  17. data/ruby_whois_patches/whois.ai.rb +0 -112
  18. data/ruby_whois_patches/whois.arnes.si.rb +0 -121
  19. data/ruby_whois_patches/whois.ascio.com.rb +0 -91
  20. data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
  21. data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
  22. data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
  23. data/ruby_whois_patches/whois.denic.de.rb +0 -174
  24. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
  25. data/ruby_whois_patches/whois.dns.be.rb +0 -134
  26. data/ruby_whois_patches/whois.dns.lu.rb +0 -129
  27. data/ruby_whois_patches/whois.dns.pl.rb +0 -150
  28. data/ruby_whois_patches/whois.dns.pt.rb +0 -119
  29. data/ruby_whois_patches/whois.domain.kg.rb +0 -126
  30. data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
  31. data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
  32. data/ruby_whois_patches/whois.dot.tk.rb +0 -140
  33. data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
  34. data/ruby_whois_patches/whois.isnic.is.rb +0 -130
  35. data/ruby_whois_patches/whois.je.rb +0 -119
  36. data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
  37. data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
  38. data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
  39. data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
  40. data/ruby_whois_patches/whois.nic.as.rb +0 -96
  41. data/ruby_whois_patches/whois.nic.at.rb +0 -109
  42. data/ruby_whois_patches/whois.nic.ch.rb +0 -141
  43. data/ruby_whois_patches/whois.nic.cl.rb +0 -117
  44. data/ruby_whois_patches/whois.nic.ec.rb +0 -157
  45. data/ruby_whois_patches/whois.nic.im.rb +0 -120
  46. data/ruby_whois_patches/whois.nic.it.rb +0 -170
  47. data/ruby_whois_patches/whois.nic.lv.rb +0 -116
  48. data/ruby_whois_patches/whois.nic.ly.rb +0 -127
  49. data/ruby_whois_patches/whois.nic.mu.rb +0 -27
  50. data/ruby_whois_patches/whois.nic.mx.rb +0 -123
  51. data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
  52. data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
  53. data/ruby_whois_patches/whois.nic.tel.rb +0 -129
  54. data/ruby_whois_patches/whois.nic.tr.rb +0 -133
  55. data/ruby_whois_patches/whois.nic.us.rb +0 -129
  56. data/ruby_whois_patches/whois.nic.ve.rb +0 -135
  57. data/ruby_whois_patches/whois.norid.no.rb +0 -127
  58. data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
  59. data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
  60. data/ruby_whois_patches/whois.registro.br.rb +0 -109
  61. data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
  62. data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
  63. data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
  64. data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
  65. data/ruby_whois_patches/whois.tucows.com.rb +0 -70
  66. data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,170 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
- require 'whois/record/scanners/whois.nic.it.rb'
12
-
13
-
14
- module Whois
15
- class Record
16
- class Parser
17
-
18
- # Parser for the whois.nic.it server.
19
- class WhoisNicIt < Base
20
- include Scanners::Ast
21
-
22
-
23
- property_supported :disclaimer do
24
- node("Disclaimer")
25
- end
26
-
27
-
28
- property_supported :domain do
29
- node("Domain") { |str| str.downcase }
30
- end
31
-
32
- property_not_supported :domain_id
33
-
34
-
35
- property_not_supported :referral_whois
36
-
37
- property_not_supported :referral_url
38
-
39
-
40
- property_supported :status do
41
- case s = node("Status").to_s.downcase
42
- when /^ok/, "active", /\bclient/
43
- :registered
44
- when "grace-period", "no-provider"
45
- :registered
46
- when /^pendingupdate/
47
- :registered
48
- when /^pendingtransfer/
49
- :registered
50
- when /redemption\-/
51
- :redemption
52
- when "pending-delete"
53
- :redemption
54
- # The domain will be deleted in 5 days
55
- when /^pendingdelete/
56
- :redemption
57
- when "unassignable"
58
- :reserved
59
- when "available"
60
- :available
61
- when /^inactive/
62
- :inactive
63
- else
64
- Whois.bug!(ParserError, "Unknown status `#{s}'.")
65
- end
66
- end
67
-
68
- property_supported :available? do
69
- status == :available
70
- end
71
-
72
- property_supported :registered? do
73
- status != :reserved &&
74
- !available?
75
- end
76
-
77
- # NEWPROPERTY
78
- def reserved?
79
- status == :reserved
80
- end
81
-
82
-
83
- property_supported :created_on do
84
- node("Created") { |str| Time.parse(str) }
85
- end
86
-
87
- property_supported :updated_on do
88
- node("Last Update") { |str| Time.parse(str) }
89
- end
90
-
91
- property_supported :expires_on do
92
- node("Expire Date") { |str| Time.parse(str) }
93
- end
94
-
95
-
96
- property_supported :registrar do
97
- node("Registrar") do |str|
98
- Record::Registrar.new(
99
- :id => str["Name"],
100
- :name => str["Name"],
101
- :organization => str["Organization"]
102
- )
103
- end
104
- end
105
-
106
- property_supported :registrant_contacts do
107
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
108
- end
109
-
110
- property_supported :admin_contacts do
111
- build_contact("Admin Contact", Whois::Record::Contact::TYPE_ADMIN)
112
- end
113
-
114
- property_supported :technical_contacts do
115
- build_contact("Technical Contacts", Whois::Record::Contact::TYPE_TECHNICAL)
116
- end
117
-
118
-
119
- property_supported :nameservers do
120
- Array.wrap(node("Nameservers")).map do |name|
121
- Record::Nameserver.new(:name => name)
122
- end
123
- end
124
-
125
-
126
- # Checks whether this response contains a message
127
- # that can be reconducted to a "WHOIS Server Unavailable" status.
128
- #
129
- # @return [Boolean]
130
- def response_unavailable?
131
- !!node("response:unavailable")
132
- end
133
-
134
- # Initializes a new {Scanners::WhoisNicIt} instance
135
- # passing the {#content_for_scanner}
136
- # and calls +parse+ on it.
137
- #
138
- # @return [Hash]
139
- def parse
140
- Scanners::WhoisNicIt.new(content_for_scanner).parse
141
- end
142
-
143
-
144
- private
145
-
146
- def build_contact(element, type)
147
- node(element) do |str|
148
- address = (str["Address"] || "").split("\n")
149
- company = address.size == 6 ? address.shift : nil
150
- Record::Contact.new(
151
- :id => str["ContactID"],
152
- :type => type,
153
- :name => str["Name"],
154
- :organization => str["Organization"] || company,
155
- :address => address[0],
156
- :city => address[1],
157
- :zip => address[2],
158
- :state => address[3],
159
- :country_code => address[4],
160
- :created_on => str["Created"] ? Time.parse(str["Created"]) : nil,
161
- :updated_on => str["Last Update"] ? Time.parse(str["Last Update"]) : nil
162
- )
163
- end
164
- end
165
-
166
- end
167
-
168
- end
169
- end
170
- end
@@ -1,116 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # = whois.nic.lv parser
18
- #
19
- # Parser for the whois.nic.lv server.
20
- #
21
- # NOTE: This parser is just a stub and provides only a few basic methods
22
- # to check for domain availability and get domain status.
23
- # Please consider to contribute implementing missing methods.
24
- # See WhoisNicIt parser for an explanation of all available methods
25
- # and examples.
26
- #
27
- class WhoisNicLv < Base
28
-
29
- property_supported :status do
30
- if available?
31
- :available
32
- else
33
- :registered
34
- end
35
- end
36
-
37
- property_supported :available? do
38
- !!(content_for_scanner =~ /Status: free/)
39
- end
40
-
41
- property_supported :registered? do
42
- !available?
43
- end
44
-
45
- property_not_supported :created_on
46
-
47
- property_supported :updated_on do
48
- if content_for_scanner =~ /Changed:\s+(.+)\n/
49
- Time.parse($1)
50
- end
51
- end
52
-
53
- property_not_supported :expires_on
54
-
55
- property_supported :nameservers do
56
- content_for_scanner.scan(/Nserver:\s+(.+)\n/).flatten.map do |name|
57
- Record::Nameserver.new(:name => name)
58
- end
59
- end
60
-
61
- # The following methods are implemented by Yang Li on 01/24/2013
62
- # ----------------------------------------------------------------------------
63
- property_supported :domain do
64
- return $1 if content_for_scanner =~ /Domain:\s+(.*)\n/i
65
- end
66
-
67
- property_not_supported :domain_id
68
-
69
- property_supported :registrar do
70
- reg=Record::Registrar.new
71
- if content_for_scanner =~ /^\[Registrar\]\n((.+\n)+)\n/
72
- contacts=$1
73
- contacts.scan(/(.+):(.+)\n/).map do |entry|
74
- reg["name"] = entry[1] if entry[0] =~ /Name/i
75
- reg["organization"] = entry[1] if entry[0] =~ /Name/i
76
- reg["url"] = entry[1] if entry[0] =~ /email/i
77
- end
78
- end
79
- return reg
80
- end
81
-
82
- property_not_supported :admin_contacts
83
-
84
- property_supported :registrant_contacts do
85
- build_contact("Holder", Whois::Record::Contact::TYPE_REGISTRANT)
86
- end
87
-
88
- property_supported :technical_contacts do
89
- build_contact("Tech", Whois::Record::Contact::TYPE_TECHNICAL)
90
- end
91
-
92
- property_not_supported :billing_contacts
93
-
94
- private
95
-
96
- def build_contact(element, type)
97
- reg=Record::Contact.new(:type => type)
98
- if content_for_scanner.gsub(/\//,'') =~ /^\[#{element}\]\n((.+\n)+)\n/i
99
- contacts=$1
100
- contacts.scan(/^(.+):\s+(.+)\n/).map do |entry|
101
- reg["name"]=entry[1] if entry[0] =~ /Name/i
102
- reg["organization"]=entry[1] if entry[0]=~ /Name/i
103
- reg["address"]=entry[1] if entry[0]=~ /Address/i
104
- reg["phone"]=entry[1] if entry[0]=~ /Phone/i
105
- reg["fax"]=entry[1] if entry[0]=~ /Fax/i
106
- reg["email"]=entry[1] if entry[0]=~ /Email/i
107
- end
108
- end
109
- return reg
110
- end
111
- # ----------------------------------------------------------------------------
112
-
113
- end
114
- end
115
- end
116
- end
@@ -1,127 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.nic.ly parser
19
- #
20
- # Parser for the whois.nic.ly server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisNicLy < Base
29
-
30
- property_supported :status do
31
- if available?
32
- :available
33
- else
34
- :registered
35
- end
36
- end
37
-
38
- property_supported :available? do
39
- (content_for_scanner.strip == "Not found")
40
- end
41
-
42
- property_supported :registered? do
43
- !available?
44
- end
45
-
46
-
47
- property_supported :created_on do
48
- if content_for_scanner =~ /Created:\s+(.*)\n/
49
- Time.parse($1)
50
- end
51
- end
52
-
53
- property_supported :updated_on do
54
- if content_for_scanner =~ /Updated:\s+(.*)\n/
55
- Time.parse($1)
56
- end
57
- end
58
-
59
- property_supported :expires_on do
60
- if content_for_scanner =~ /Expired:\s+(.*)\n/
61
- Time.parse($1)
62
- end
63
- end
64
-
65
-
66
- property_supported :nameservers do
67
- if content_for_scanner =~ /Domain servers in listed order:\n((.+\n)+)\n/
68
- $1.split("\n").map do |name|
69
- Record::Nameserver.new(:name => name.strip)
70
- end
71
- end
72
- end
73
-
74
- # The following methods are implemented by Yang Li on 01/24/2013
75
- # ----------------------------------------------------------------------------
76
- property_supported :domain do
77
- return $1.strip if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
78
- end
79
-
80
- property_not_supported :domain_id
81
-
82
- property_not_supported :registrar
83
-
84
- property_supported :admin_contacts do
85
- build_contact("Administrative Contact", Whois::Record::Contact::TYPE_REGISTRANT)
86
- end
87
-
88
- property_supported :registrant_contacts do
89
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
90
- end
91
-
92
- property_supported :technical_contacts do
93
- build_contact("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
94
- end
95
-
96
- property_supported :billing_contacts do
97
- build_contact("Billing Contact", Whois::Record::Contact::TYPE_BILLING)
98
- end
99
-
100
- private
101
-
102
- def build_contact(element, type)
103
- reg=Record::Contact.new(:type => type)
104
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
105
- line_num=1
106
- $1.split(%r{\n}).each do |line|
107
- reg["name"]=line.strip if line_num==1
108
- reg["organization"]=line.strip if line_num==2
109
- reg["address"]=line.strip if line_num==3
110
- reg["city"]=line.strip if line_num==4
111
- reg["state"]=line.strip if line_num==5
112
- reg["country"]=line.strip if line_num==6
113
- reg["zip"]=line.strip.split(':').last if line_num==7
114
- reg["phone"]=line.strip.split(':').last if line_num==8
115
- reg["fax"]=line.strip.split(':').last if line_num==9
116
- reg["email"]=line.strip if line_num==10
117
- line_num=line_num+1
118
- end
119
- end
120
- return reg
121
- end
122
- # ----------------------------------------------------------------------------
123
-
124
- end
125
- end
126
- end
127
- end
@@ -1,27 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base_cocca2'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.nic.mu server.
18
- #
19
- # @see Whois::Record::Parser::Example
20
- # The Example parser for the list of all available methods.
21
- #
22
- class WhoisNicMu < BaseCocca2
23
- end
24
-
25
- end
26
- end
27
- end
@@ -1,123 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.nic.mx parser
19
- #
20
- # Parser for the whois.nic.mx server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisNicMx < Base
29
-
30
- property_supported :status do
31
- if available?
32
- :available
33
- else
34
- :registered
35
- end
36
- end
37
-
38
- property_supported :available? do
39
- !!(content_for_scanner =~ /Object_Not_Found/)
40
- end
41
-
42
- property_supported :registered? do
43
- !available?
44
- end
45
-
46
-
47
- property_supported :created_on do
48
- if content_for_scanner =~ /Created On:\s+(.*)\n/
49
- Time.parse($1)
50
- end
51
- end
52
-
53
- # FIXME: the response contains localized data
54
- # Expiration Date: 10-may-2011
55
- # Last Updated On: 15-abr-2010 <--
56
- # property_supported :updated_on do
57
- # if content_for_scanner =~ /Last Updated On:\s+(.*)\n/
58
- # Time.parse($1)
59
- # end
60
- # end
61
-
62
- property_supported :expires_on do
63
- if content_for_scanner =~ /Expiration Date:\s+(.*)\n/
64
- Time.parse($1)
65
- end
66
- end
67
-
68
- property_supported :nameservers do
69
- if content_for_scanner =~ /Name Servers:\n((.+\n)+)\n/
70
- $1.scan(/DNS:\s+(.+)\n/).flatten.map do |line|
71
- name, ipv4 = line.strip.split(/\s+/)
72
- Record::Nameserver.new(:name => name, :ipv4 => ipv4)
73
- end
74
- end
75
- end
76
-
77
- # The following methods are implemented by Yang Li on 02/05/2013
78
- # ----------------------------------------------------------------------------
79
- property_supported :domain do
80
- return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
81
- end
82
-
83
- property_not_supported :domain_id
84
-
85
- property_not_supported :registrar
86
-
87
- property_supported :admin_contacts do
88
- build_contact("Administrative Contact", Whois::Record::Contact::TYPE_ADMIN)
89
- end
90
-
91
- property_supported :registrant_contacts do
92
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
93
- end
94
-
95
- property_supported :technical_contacts do
96
- build_contact("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
97
- end
98
-
99
- property_supported :billing_contacts do
100
- build_contact("Billing Contact", Whois::Record::Contact::TYPE_BILLING)
101
- end
102
-
103
- private
104
-
105
- def build_contact(element, type)
106
- reg=Record::Contact.new(:type => type)
107
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
108
- $1.scan(/^(.+):(.+)\n/).map do |entry|
109
- reg["name"]=entry[1].strip if entry[0] =~ /Name/i
110
- reg["organization"]=entry[1].strip if entry[0]=~ /Name/i
111
- reg["city"]= entry[1].strip if entry[0]=~ /City/i
112
- reg["state"]=entry[1].strip if entry[0]=~ /State/i
113
- reg["country"]=entry[1].strip if entry[0]=~ /Country/i
114
- end
115
- end
116
- return reg
117
- end
118
- # ----------------------------------------------------------------------------
119
-
120
- end
121
- end
122
- end
123
- end