wmap 2.7.2 → 2.7.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -16
  3. data/bin/refresh +5 -5
  4. data/lib/wmap.rb +1 -1
  5. data/lib/wmap/site_tracker.rb +1 -4
  6. data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
  7. data/lib/wmap/url_crawler.rb +1 -1
  8. data/version.txt +2 -2
  9. data/wmap.gemspec +2 -2
  10. metadata +4 -59
  11. data/ruby_whois_patches/base_cocca2.rb +0 -149
  12. data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
  13. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
  14. data/ruby_whois_patches/whois.above.com.rb +0 -61
  15. data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
  16. data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
  17. data/ruby_whois_patches/whois.ai.rb +0 -112
  18. data/ruby_whois_patches/whois.arnes.si.rb +0 -121
  19. data/ruby_whois_patches/whois.ascio.com.rb +0 -91
  20. data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
  21. data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
  22. data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
  23. data/ruby_whois_patches/whois.denic.de.rb +0 -174
  24. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
  25. data/ruby_whois_patches/whois.dns.be.rb +0 -134
  26. data/ruby_whois_patches/whois.dns.lu.rb +0 -129
  27. data/ruby_whois_patches/whois.dns.pl.rb +0 -150
  28. data/ruby_whois_patches/whois.dns.pt.rb +0 -119
  29. data/ruby_whois_patches/whois.domain.kg.rb +0 -126
  30. data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
  31. data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
  32. data/ruby_whois_patches/whois.dot.tk.rb +0 -140
  33. data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
  34. data/ruby_whois_patches/whois.isnic.is.rb +0 -130
  35. data/ruby_whois_patches/whois.je.rb +0 -119
  36. data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
  37. data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
  38. data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
  39. data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
  40. data/ruby_whois_patches/whois.nic.as.rb +0 -96
  41. data/ruby_whois_patches/whois.nic.at.rb +0 -109
  42. data/ruby_whois_patches/whois.nic.ch.rb +0 -141
  43. data/ruby_whois_patches/whois.nic.cl.rb +0 -117
  44. data/ruby_whois_patches/whois.nic.ec.rb +0 -157
  45. data/ruby_whois_patches/whois.nic.im.rb +0 -120
  46. data/ruby_whois_patches/whois.nic.it.rb +0 -170
  47. data/ruby_whois_patches/whois.nic.lv.rb +0 -116
  48. data/ruby_whois_patches/whois.nic.ly.rb +0 -127
  49. data/ruby_whois_patches/whois.nic.mu.rb +0 -27
  50. data/ruby_whois_patches/whois.nic.mx.rb +0 -123
  51. data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
  52. data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
  53. data/ruby_whois_patches/whois.nic.tel.rb +0 -129
  54. data/ruby_whois_patches/whois.nic.tr.rb +0 -133
  55. data/ruby_whois_patches/whois.nic.us.rb +0 -129
  56. data/ruby_whois_patches/whois.nic.ve.rb +0 -135
  57. data/ruby_whois_patches/whois.norid.no.rb +0 -127
  58. data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
  59. data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
  60. data/ruby_whois_patches/whois.registro.br.rb +0 -109
  61. data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
  62. data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
  63. data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
  64. data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
  65. data/ruby_whois_patches/whois.tucows.com.rb +0 -70
  66. data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,134 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dns.be server.
18
- #
19
- # @note This parser is just a stub and provides only a few basic methods
20
- # to check for domain availability and get domain status.
21
- # Please consider to contribute implementing missing methods.
22
- #
23
- # @see Whois::Record::Parser::Example
24
- # The Example parser for the list of all available methods.
25
- #
26
- class WhoisDnsBe < Base
27
-
28
- property_supported :status do
29
- if content_for_scanner =~ /Status:\s+(.+?)\n/
30
- case $1.downcase
31
- when "available" then :available
32
- when "not available" then :registered
33
- when "quarantine" then :redemption
34
- when "out of service" then :redemption
35
- # old response
36
- when "registered" then :registered
37
- when "free" then :available
38
- else
39
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
40
- end
41
- else
42
- Whois.bug!(ParserError, "Unable to parse status.")
43
- end
44
- end
45
-
46
- property_supported :available? do
47
- (status == :available)
48
- end
49
-
50
- property_supported :registered? do
51
- !available?
52
- end
53
-
54
- property_supported :created_on do
55
- if content_for_scanner =~ /Registered:\s+(.+)\n/
56
- Time.parse($1)
57
- end
58
- end
59
-
60
- property_not_supported :updated_on
61
-
62
- property_not_supported :expires_on
63
-
64
- property_supported :nameservers do
65
- if content_for_scanner =~ /Nameservers:\s((.+\n)+)\n/
66
- $1.split("\n").map do |line|
67
- if line.strip =~ /(.+) \((.+)\)/
68
- Record::Nameserver.new(:name => $1, :ipv4 => $2)
69
- else
70
- Record::Nameserver.new(:name => line.strip)
71
- end
72
- end
73
- end
74
- end
75
-
76
- # Checks whether the response has been throttled.
77
- #
78
- # @return [Boolean]
79
- def response_throttled?
80
- !!(content_for_scanner =~ /^% (Excessive querying|Maximum queries per hour reached)/)
81
- end
82
-
83
- # The following methods are implemented by Yang Li on 01/24/2013
84
- # ----------------------------------------------------------------------------
85
- property_supported :domain do
86
- return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
87
- end
88
-
89
- property_supported :domain_id do
90
- return $1 if content_for_scanner =~ /Domain ID:\s+(.*)\n/i
91
- end
92
-
93
- property_supported :registrar do
94
- reg=Record::Registrar.new
95
- if content_for_scanner =~ /Registrar Technical Contacts:\n((.+\n)+)\n/
96
- $1.scan(/^(.+?):(.+?)\n/).map do |entry|
97
- reg["name"] = entry[1].strip if entry[0] =~ /Name/i
98
- reg["organization"] = entry[1] if entry[0] =~ /Organisation/i
99
- reg["url"] = entry[1] if entry[0] =~ /Email/i
100
- end
101
- end
102
- return reg
103
- end
104
-
105
- property_supported :registrant_contacts do
106
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
107
- end
108
-
109
- property_not_supported :admin_contacts
110
-
111
- property_not_supported :technical_contacts
112
-
113
- property_not_supported :billing_contacts
114
-
115
- private
116
-
117
- def build_contact(element, type)
118
- reg=Record::Contact.new(:type => type)
119
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
120
- line_num=1
121
- $1.split(%r{\n}).each do |entry|
122
- reg["name"]=entry.strip if line_num==1
123
- reg["organization"]=entry.strip if line_num==1
124
- line_num=line_num+1
125
- end
126
- end
127
- return reg
128
- end
129
- # ----------------------------------------------------------------------------
130
-
131
- end
132
- end
133
- end
134
- end
@@ -1,129 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.dns.lu parser
19
- #
20
- # Parser for the whois.dns.lu server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisDnsLu < Base
29
-
30
- property_supported :status do
31
- if content_for_scanner =~ /domaintype:\s+(.+)\n/
32
- case $1.downcase
33
- when "active" then :registered
34
- else
35
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
36
- end
37
- else
38
- :available
39
- end
40
- end
41
-
42
- property_supported :available? do
43
- !!(content_for_scanner =~ /% No such domain/)
44
- end
45
-
46
- property_supported :registered? do
47
- !available?
48
- end
49
-
50
-
51
- property_supported :created_on do
52
- if content_for_scanner =~ /registered:\s+(.*)\n/
53
- # Force the parser to use the dd/mm/yyyy format.
54
- Time.utc(*$1.split("/").reverse)
55
- end
56
- end
57
-
58
- property_not_supported :updated_on
59
-
60
- property_not_supported :expires_on
61
-
62
-
63
- property_supported :nameservers do
64
- content_for_scanner.scan(/nserver:\s+(.+)\n/).flatten.map do |line|
65
- if line =~ /(.+) \[(.+)\]/
66
- Record::Nameserver.new(:name => $1, :ipv4 => $2)
67
- else
68
- Record::Nameserver.new(:name => line)
69
- end
70
- end
71
- end
72
-
73
- # The following methods are implemented by Yang Li on 01/29/2013
74
- # ----------------------------------------------------------------------------
75
- property_supported :domain do
76
- return $1 if content_for_scanner =~ /domainname:\s+(.*)\n/i
77
- end
78
-
79
- property_supported :domain_id do
80
- return $1 if content_for_scanner =~ /Domain ID:\s+(.*)\n/i
81
- end
82
-
83
- property_supported :registrar do
84
- reg=Record::Registrar.new
85
- content_for_scanner.scan(/^registrar-(.*):\s+(.+)\n/).map do |entry|
86
- reg["name"] = entry[1] if entry[0] =~ /name/i
87
- reg["url"] = entry[1] if entry[0] =~ /url/i
88
- reg["email"] = entry[1] if entry[0] =~ /email/i
89
- reg["country"] = entry[1] if entry[0] =~ /country/i
90
- end
91
- return reg
92
- end
93
-
94
- property_supported :registrant_contacts do
95
- build_contact("org", Whois::Record::Contact::TYPE_REGISTRANT)
96
- end
97
-
98
- property_supported :admin_contacts do
99
- build_contact("adm", Whois::Record::Contact::TYPE_ADMIN)
100
- end
101
-
102
- property_supported :technical_contacts do
103
- build_contact("tec", Whois::Record::Contact::TYPE_TECHNICAL)
104
- end
105
-
106
- property_not_supported :billing_contacts
107
-
108
- private
109
-
110
- def build_contact(element, type)
111
- reg=Record::Contact.new(:type => type)
112
- content_for_scanner.scan(/^#{element}-(.*):\s+(.+)\n/).map do |entry|
113
- reg["name"]=entry[1] if entry[0] =~ /name/i
114
- reg["address"]=entry[1] if entry[0]=~ /address/i
115
- reg["city"]= entry[1] if entry[0]=~ /city/i
116
- reg["zip"]=entry[1] if entry[0]=~ /ZipCode/i
117
- reg["country_code"]=entry[1] if entry[0]=~ /country/i
118
- reg["phone"]=entry[1] if entry[0]=~ /#{element}\sPhone\sNumber/i
119
- reg["fax"]=entry[1] if entry[0]=~ /#{element}\sFacsimile\sNumber/i
120
- reg["email"]=entry[1] if entry[0]=~ /#{element}\sEmail/i
121
- end
122
- return reg
123
- end
124
- # ----------------------------------------------------------------------------
125
-
126
- end
127
- end
128
- end
129
- end
@@ -1,150 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dns.pl server.
18
- #
19
- # @see Whois::Record::Parser::Example
20
- # The Example parser for the list of all available methods.
21
- #
22
- class WhoisDnsPl < Base
23
-
24
- property_supported :domain do
25
- if content_for_scanner =~ /DOMAIN NAME:\s+(.+)\n/
26
- $1
27
- end
28
- end
29
-
30
- property_not_supported :domain_id
31
-
32
-
33
- property_supported :status do
34
- if available?
35
- :available
36
- else
37
- :registered
38
- end
39
- end
40
-
41
- property_supported :available? do
42
- !!(content_for_scanner =~ /^No information available about domain name/)
43
- end
44
-
45
- property_supported :registered? do
46
- !available?
47
- end
48
-
49
-
50
- property_supported :created_on do
51
- if content_for_scanner =~ /created:\s+(.+?)\n/
52
- Time.parse($1)
53
- end
54
- end
55
-
56
- property_supported :updated_on do
57
- if content_for_scanner =~ /last modified:\s+(.+?)\n/
58
- Time.parse($1)
59
- end
60
- end
61
-
62
- property_supported :expires_on do
63
- if content_for_scanner =~ /renewal date:\s+(.+?)\n/
64
- Time.parse($1)
65
- end
66
- end
67
-
68
-
69
- property_supported :registrar do
70
- match = content_for_scanner.slice(/REGISTRAR:\n((.+\n)+)\n/, 1)
71
- return unless match
72
-
73
- lines = match.split("\n")
74
- Record::Registrar.new(
75
- :name => lines[0]
76
- )
77
- end
78
-
79
-
80
- property_supported :registrant_contacts do
81
- reg=Record::Contact.new(:type => Whois::Record::Contact::TYPE_REGISTRANT)
82
- if content_for_scanner =~ /^Registrant data (.+)/
83
- reg['name']=$1
84
- reg['organization']=$1
85
- end
86
- return reg
87
- end
88
-
89
- property_not_supported :admin_contacts
90
-
91
- property_supported :technical_contacts do
92
- build_contact("TECHNICAL CONTACT", Record::Contact::TYPE_TECHNICAL)
93
- end
94
-
95
-
96
- property_supported :nameservers do
97
- content_for_scanner.scan(/nameservers:\s+(.+)\n(.+)\n/).flatten.map do |line|
98
- line.strip!
99
- if line =~ /(.+) \[(.+)\]/
100
- Record::Nameserver.new(:name => $1.chomp("."), :ipv4 => $2)
101
- else
102
- Record::Nameserver.new(:name => line.chomp("."))
103
- end
104
- end
105
- end
106
-
107
-
108
- private
109
-
110
- def build_contact(element, type)
111
- match = content_for_scanner.slice(/#{element}:\n((.+\n)+)\n/, 1)
112
- return unless match
113
-
114
- values = parse_contact_block(match.split("\n"))
115
- zip, city = values["city"].match(/(.+?) (.+)/)[1..2]
116
-
117
- Record::Contact.new(
118
- :type => type,
119
- :id => values["handle"],
120
- :name => nil,
121
- :organization => values["company"],
122
- :address => values["street"],
123
- :city => city,
124
- :zip => zip,
125
- :state => nil,
126
- :country_code => values["location"],
127
- :phone => values["phone"],
128
- :fax => values["fax"],
129
- :email => nil
130
- )
131
- end
132
-
133
- def parse_contact_block(lines)
134
- key = nil
135
- hash = {}
136
- lines.each do |line|
137
- if line =~ /(.+):(.+)/
138
- hash[key = $1] = $2.strip
139
- else
140
- hash[key] += "\n#{line.strip}"
141
- end
142
- end
143
- hash
144
- end
145
-
146
- end
147
-
148
- end
149
- end
150
- end
@@ -1,119 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dns.pt server.
18
- #
19
- # @note This parser is just a stub and provides only a few basic methods
20
- # to check for domain availability and get domain status.
21
- # Please consider to contribute implementing missing methods.
22
- #
23
- # @see Whois::Record::Parser::Example
24
- # The Example parser for the list of all available methods.
25
- #
26
- class WhoisDnsPt < Base
27
-
28
- property_supported :status do
29
- if content_for_scanner =~ /^Estado \/ Status:\s+(.+)\n/
30
- case $1.downcase
31
- when "active" then :registered
32
- when "reserved" then :reserved
33
- when "tech-pro" then :inactive
34
- else
35
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
36
- end
37
- else
38
- :available
39
- end
40
- end
41
-
42
- property_supported :available? do
43
- !!(content_for_scanner =~ /^.* no match$/)
44
- end
45
-
46
- property_supported :registered? do
47
- !available?
48
- end
49
-
50
-
51
- property_supported :created_on do
52
- if content_for_scanner =~ / Creation Date .+?:\s+(.+)\n/
53
- Time.utc(*$1.split("/").reverse)
54
- end
55
- end
56
-
57
- property_not_supported :updated_on
58
-
59
- property_supported :expires_on do
60
- if content_for_scanner =~ / Expiration Date .+?:\s+(.+)\n/
61
- Time.utc(*$1.split("/").reverse)
62
- end
63
- end
64
-
65
- property_supported :nameservers do
66
- content_for_scanner.scan(/Nameserver:\s(?:.+\t)?(.+?)\.\n/).flatten.map do |name|
67
- Record::Nameserver.new(:name => name)
68
- end
69
- end
70
-
71
- # The following methods are implemented by Yang Li on 01/24/2013
72
- # ----------------------------------------------------------------------------
73
- property_supported :domain do
74
- return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
75
- end
76
-
77
- property_not_supported :domain_id
78
-
79
- property_not_supported :registrar
80
-
81
- property_not_supported :admin_contacts
82
-
83
- property_supported :registrant_contacts do
84
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
85
- end
86
-
87
- property_supported :technical_contacts do
88
- build_contact("Tech Contact", Whois::Record::Contact::TYPE_TECHNICAL)
89
- end
90
-
91
- property_supported :billing_contacts do
92
- build_contact("Billing Contact", Whois::Record::Contact::TYPE_BILLING)
93
- end
94
-
95
- private
96
-
97
- def build_contact(element, type)
98
- reg=Record::Contact.new(:type => type)
99
- if content_for_scanner.gsub(/\//,'') =~ /(.+#{element}\n(.+\n)+)\n/i
100
- line_num=1
101
- $1.split(%r{\n}).each do |line|
102
- reg["name"]=line.strip if line_num==2
103
- reg["organization"]=line.strip if line_num==2
104
- reg["address"]=line.strip if line_num==3
105
- reg["city"]=line.strip if line_num==4
106
- reg["zip"]=line.strip if line_num==5
107
- entry=line.split(':')
108
- reg["email"]=entry[1].strip if entry[0]=~ /Email/i
109
- line_num=line_num+1
110
- end
111
- end
112
- return reg
113
- end
114
- # ----------------------------------------------------------------------------
115
-
116
- end
117
- end
118
- end
119
- end