wmap 2.7.2 → 2.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -16
  3. data/bin/refresh +5 -5
  4. data/lib/wmap.rb +1 -1
  5. data/lib/wmap/site_tracker.rb +1 -4
  6. data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
  7. data/lib/wmap/url_crawler.rb +1 -1
  8. data/version.txt +2 -2
  9. data/wmap.gemspec +2 -2
  10. metadata +4 -59
  11. data/ruby_whois_patches/base_cocca2.rb +0 -149
  12. data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
  13. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
  14. data/ruby_whois_patches/whois.above.com.rb +0 -61
  15. data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
  16. data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
  17. data/ruby_whois_patches/whois.ai.rb +0 -112
  18. data/ruby_whois_patches/whois.arnes.si.rb +0 -121
  19. data/ruby_whois_patches/whois.ascio.com.rb +0 -91
  20. data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
  21. data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
  22. data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
  23. data/ruby_whois_patches/whois.denic.de.rb +0 -174
  24. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
  25. data/ruby_whois_patches/whois.dns.be.rb +0 -134
  26. data/ruby_whois_patches/whois.dns.lu.rb +0 -129
  27. data/ruby_whois_patches/whois.dns.pl.rb +0 -150
  28. data/ruby_whois_patches/whois.dns.pt.rb +0 -119
  29. data/ruby_whois_patches/whois.domain.kg.rb +0 -126
  30. data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
  31. data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
  32. data/ruby_whois_patches/whois.dot.tk.rb +0 -140
  33. data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
  34. data/ruby_whois_patches/whois.isnic.is.rb +0 -130
  35. data/ruby_whois_patches/whois.je.rb +0 -119
  36. data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
  37. data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
  38. data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
  39. data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
  40. data/ruby_whois_patches/whois.nic.as.rb +0 -96
  41. data/ruby_whois_patches/whois.nic.at.rb +0 -109
  42. data/ruby_whois_patches/whois.nic.ch.rb +0 -141
  43. data/ruby_whois_patches/whois.nic.cl.rb +0 -117
  44. data/ruby_whois_patches/whois.nic.ec.rb +0 -157
  45. data/ruby_whois_patches/whois.nic.im.rb +0 -120
  46. data/ruby_whois_patches/whois.nic.it.rb +0 -170
  47. data/ruby_whois_patches/whois.nic.lv.rb +0 -116
  48. data/ruby_whois_patches/whois.nic.ly.rb +0 -127
  49. data/ruby_whois_patches/whois.nic.mu.rb +0 -27
  50. data/ruby_whois_patches/whois.nic.mx.rb +0 -123
  51. data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
  52. data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
  53. data/ruby_whois_patches/whois.nic.tel.rb +0 -129
  54. data/ruby_whois_patches/whois.nic.tr.rb +0 -133
  55. data/ruby_whois_patches/whois.nic.us.rb +0 -129
  56. data/ruby_whois_patches/whois.nic.ve.rb +0 -135
  57. data/ruby_whois_patches/whois.norid.no.rb +0 -127
  58. data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
  59. data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
  60. data/ruby_whois_patches/whois.registro.br.rb +0 -109
  61. data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
  62. data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
  63. data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
  64. data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
  65. data/ruby_whois_patches/whois.tucows.com.rb +0 -70
  66. data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,134 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dns.be server.
18
- #
19
- # @note This parser is just a stub and provides only a few basic methods
20
- # to check for domain availability and get domain status.
21
- # Please consider to contribute implementing missing methods.
22
- #
23
- # @see Whois::Record::Parser::Example
24
- # The Example parser for the list of all available methods.
25
- #
26
- class WhoisDnsBe < Base
27
-
28
- property_supported :status do
29
- if content_for_scanner =~ /Status:\s+(.+?)\n/
30
- case $1.downcase
31
- when "available" then :available
32
- when "not available" then :registered
33
- when "quarantine" then :redemption
34
- when "out of service" then :redemption
35
- # old response
36
- when "registered" then :registered
37
- when "free" then :available
38
- else
39
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
40
- end
41
- else
42
- Whois.bug!(ParserError, "Unable to parse status.")
43
- end
44
- end
45
-
46
- property_supported :available? do
47
- (status == :available)
48
- end
49
-
50
- property_supported :registered? do
51
- !available?
52
- end
53
-
54
- property_supported :created_on do
55
- if content_for_scanner =~ /Registered:\s+(.+)\n/
56
- Time.parse($1)
57
- end
58
- end
59
-
60
- property_not_supported :updated_on
61
-
62
- property_not_supported :expires_on
63
-
64
- property_supported :nameservers do
65
- if content_for_scanner =~ /Nameservers:\s((.+\n)+)\n/
66
- $1.split("\n").map do |line|
67
- if line.strip =~ /(.+) \((.+)\)/
68
- Record::Nameserver.new(:name => $1, :ipv4 => $2)
69
- else
70
- Record::Nameserver.new(:name => line.strip)
71
- end
72
- end
73
- end
74
- end
75
-
76
- # Checks whether the response has been throttled.
77
- #
78
- # @return [Boolean]
79
- def response_throttled?
80
- !!(content_for_scanner =~ /^% (Excessive querying|Maximum queries per hour reached)/)
81
- end
82
-
83
- # The following methods are implemented by Yang Li on 01/24/2013
84
- # ----------------------------------------------------------------------------
85
- property_supported :domain do
86
- return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
87
- end
88
-
89
- property_supported :domain_id do
90
- return $1 if content_for_scanner =~ /Domain ID:\s+(.*)\n/i
91
- end
92
-
93
- property_supported :registrar do
94
- reg=Record::Registrar.new
95
- if content_for_scanner =~ /Registrar Technical Contacts:\n((.+\n)+)\n/
96
- $1.scan(/^(.+?):(.+?)\n/).map do |entry|
97
- reg["name"] = entry[1].strip if entry[0] =~ /Name/i
98
- reg["organization"] = entry[1] if entry[0] =~ /Organisation/i
99
- reg["url"] = entry[1] if entry[0] =~ /Email/i
100
- end
101
- end
102
- return reg
103
- end
104
-
105
- property_supported :registrant_contacts do
106
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
107
- end
108
-
109
- property_not_supported :admin_contacts
110
-
111
- property_not_supported :technical_contacts
112
-
113
- property_not_supported :billing_contacts
114
-
115
- private
116
-
117
- def build_contact(element, type)
118
- reg=Record::Contact.new(:type => type)
119
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
120
- line_num=1
121
- $1.split(%r{\n}).each do |entry|
122
- reg["name"]=entry.strip if line_num==1
123
- reg["organization"]=entry.strip if line_num==1
124
- line_num=line_num+1
125
- end
126
- end
127
- return reg
128
- end
129
- # ----------------------------------------------------------------------------
130
-
131
- end
132
- end
133
- end
134
- end
@@ -1,129 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.dns.lu parser
19
- #
20
- # Parser for the whois.dns.lu server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisDnsLu < Base
29
-
30
- property_supported :status do
31
- if content_for_scanner =~ /domaintype:\s+(.+)\n/
32
- case $1.downcase
33
- when "active" then :registered
34
- else
35
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
36
- end
37
- else
38
- :available
39
- end
40
- end
41
-
42
- property_supported :available? do
43
- !!(content_for_scanner =~ /% No such domain/)
44
- end
45
-
46
- property_supported :registered? do
47
- !available?
48
- end
49
-
50
-
51
- property_supported :created_on do
52
- if content_for_scanner =~ /registered:\s+(.*)\n/
53
- # Force the parser to use the dd/mm/yyyy format.
54
- Time.utc(*$1.split("/").reverse)
55
- end
56
- end
57
-
58
- property_not_supported :updated_on
59
-
60
- property_not_supported :expires_on
61
-
62
-
63
- property_supported :nameservers do
64
- content_for_scanner.scan(/nserver:\s+(.+)\n/).flatten.map do |line|
65
- if line =~ /(.+) \[(.+)\]/
66
- Record::Nameserver.new(:name => $1, :ipv4 => $2)
67
- else
68
- Record::Nameserver.new(:name => line)
69
- end
70
- end
71
- end
72
-
73
- # The following methods are implemented by Yang Li on 01/29/2013
74
- # ----------------------------------------------------------------------------
75
- property_supported :domain do
76
- return $1 if content_for_scanner =~ /domainname:\s+(.*)\n/i
77
- end
78
-
79
- property_supported :domain_id do
80
- return $1 if content_for_scanner =~ /Domain ID:\s+(.*)\n/i
81
- end
82
-
83
- property_supported :registrar do
84
- reg=Record::Registrar.new
85
- content_for_scanner.scan(/^registrar-(.*):\s+(.+)\n/).map do |entry|
86
- reg["name"] = entry[1] if entry[0] =~ /name/i
87
- reg["url"] = entry[1] if entry[0] =~ /url/i
88
- reg["email"] = entry[1] if entry[0] =~ /email/i
89
- reg["country"] = entry[1] if entry[0] =~ /country/i
90
- end
91
- return reg
92
- end
93
-
94
- property_supported :registrant_contacts do
95
- build_contact("org", Whois::Record::Contact::TYPE_REGISTRANT)
96
- end
97
-
98
- property_supported :admin_contacts do
99
- build_contact("adm", Whois::Record::Contact::TYPE_ADMIN)
100
- end
101
-
102
- property_supported :technical_contacts do
103
- build_contact("tec", Whois::Record::Contact::TYPE_TECHNICAL)
104
- end
105
-
106
- property_not_supported :billing_contacts
107
-
108
- private
109
-
110
- def build_contact(element, type)
111
- reg=Record::Contact.new(:type => type)
112
- content_for_scanner.scan(/^#{element}-(.*):\s+(.+)\n/).map do |entry|
113
- reg["name"]=entry[1] if entry[0] =~ /name/i
114
- reg["address"]=entry[1] if entry[0]=~ /address/i
115
- reg["city"]= entry[1] if entry[0]=~ /city/i
116
- reg["zip"]=entry[1] if entry[0]=~ /ZipCode/i
117
- reg["country_code"]=entry[1] if entry[0]=~ /country/i
118
- reg["phone"]=entry[1] if entry[0]=~ /#{element}\sPhone\sNumber/i
119
- reg["fax"]=entry[1] if entry[0]=~ /#{element}\sFacsimile\sNumber/i
120
- reg["email"]=entry[1] if entry[0]=~ /#{element}\sEmail/i
121
- end
122
- return reg
123
- end
124
- # ----------------------------------------------------------------------------
125
-
126
- end
127
- end
128
- end
129
- end
@@ -1,150 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dns.pl server.
18
- #
19
- # @see Whois::Record::Parser::Example
20
- # The Example parser for the list of all available methods.
21
- #
22
- class WhoisDnsPl < Base
23
-
24
- property_supported :domain do
25
- if content_for_scanner =~ /DOMAIN NAME:\s+(.+)\n/
26
- $1
27
- end
28
- end
29
-
30
- property_not_supported :domain_id
31
-
32
-
33
- property_supported :status do
34
- if available?
35
- :available
36
- else
37
- :registered
38
- end
39
- end
40
-
41
- property_supported :available? do
42
- !!(content_for_scanner =~ /^No information available about domain name/)
43
- end
44
-
45
- property_supported :registered? do
46
- !available?
47
- end
48
-
49
-
50
- property_supported :created_on do
51
- if content_for_scanner =~ /created:\s+(.+?)\n/
52
- Time.parse($1)
53
- end
54
- end
55
-
56
- property_supported :updated_on do
57
- if content_for_scanner =~ /last modified:\s+(.+?)\n/
58
- Time.parse($1)
59
- end
60
- end
61
-
62
- property_supported :expires_on do
63
- if content_for_scanner =~ /renewal date:\s+(.+?)\n/
64
- Time.parse($1)
65
- end
66
- end
67
-
68
-
69
- property_supported :registrar do
70
- match = content_for_scanner.slice(/REGISTRAR:\n((.+\n)+)\n/, 1)
71
- return unless match
72
-
73
- lines = match.split("\n")
74
- Record::Registrar.new(
75
- :name => lines[0]
76
- )
77
- end
78
-
79
-
80
- property_supported :registrant_contacts do
81
- reg=Record::Contact.new(:type => Whois::Record::Contact::TYPE_REGISTRANT)
82
- if content_for_scanner =~ /^Registrant data (.+)/
83
- reg['name']=$1
84
- reg['organization']=$1
85
- end
86
- return reg
87
- end
88
-
89
- property_not_supported :admin_contacts
90
-
91
- property_supported :technical_contacts do
92
- build_contact("TECHNICAL CONTACT", Record::Contact::TYPE_TECHNICAL)
93
- end
94
-
95
-
96
- property_supported :nameservers do
97
- content_for_scanner.scan(/nameservers:\s+(.+)\n(.+)\n/).flatten.map do |line|
98
- line.strip!
99
- if line =~ /(.+) \[(.+)\]/
100
- Record::Nameserver.new(:name => $1.chomp("."), :ipv4 => $2)
101
- else
102
- Record::Nameserver.new(:name => line.chomp("."))
103
- end
104
- end
105
- end
106
-
107
-
108
- private
109
-
110
- def build_contact(element, type)
111
- match = content_for_scanner.slice(/#{element}:\n((.+\n)+)\n/, 1)
112
- return unless match
113
-
114
- values = parse_contact_block(match.split("\n"))
115
- zip, city = values["city"].match(/(.+?) (.+)/)[1..2]
116
-
117
- Record::Contact.new(
118
- :type => type,
119
- :id => values["handle"],
120
- :name => nil,
121
- :organization => values["company"],
122
- :address => values["street"],
123
- :city => city,
124
- :zip => zip,
125
- :state => nil,
126
- :country_code => values["location"],
127
- :phone => values["phone"],
128
- :fax => values["fax"],
129
- :email => nil
130
- )
131
- end
132
-
133
- def parse_contact_block(lines)
134
- key = nil
135
- hash = {}
136
- lines.each do |line|
137
- if line =~ /(.+):(.+)/
138
- hash[key = $1] = $2.strip
139
- else
140
- hash[key] += "\n#{line.strip}"
141
- end
142
- end
143
- hash
144
- end
145
-
146
- end
147
-
148
- end
149
- end
150
- end
@@ -1,119 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.dns.pt server.
18
- #
19
- # @note This parser is just a stub and provides only a few basic methods
20
- # to check for domain availability and get domain status.
21
- # Please consider to contribute implementing missing methods.
22
- #
23
- # @see Whois::Record::Parser::Example
24
- # The Example parser for the list of all available methods.
25
- #
26
- class WhoisDnsPt < Base
27
-
28
- property_supported :status do
29
- if content_for_scanner =~ /^Estado \/ Status:\s+(.+)\n/
30
- case $1.downcase
31
- when "active" then :registered
32
- when "reserved" then :reserved
33
- when "tech-pro" then :inactive
34
- else
35
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
36
- end
37
- else
38
- :available
39
- end
40
- end
41
-
42
- property_supported :available? do
43
- !!(content_for_scanner =~ /^.* no match$/)
44
- end
45
-
46
- property_supported :registered? do
47
- !available?
48
- end
49
-
50
-
51
- property_supported :created_on do
52
- if content_for_scanner =~ / Creation Date .+?:\s+(.+)\n/
53
- Time.utc(*$1.split("/").reverse)
54
- end
55
- end
56
-
57
- property_not_supported :updated_on
58
-
59
- property_supported :expires_on do
60
- if content_for_scanner =~ / Expiration Date .+?:\s+(.+)\n/
61
- Time.utc(*$1.split("/").reverse)
62
- end
63
- end
64
-
65
- property_supported :nameservers do
66
- content_for_scanner.scan(/Nameserver:\s(?:.+\t)?(.+?)\.\n/).flatten.map do |name|
67
- Record::Nameserver.new(:name => name)
68
- end
69
- end
70
-
71
- # The following methods are implemented by Yang Li on 01/24/2013
72
- # ----------------------------------------------------------------------------
73
- property_supported :domain do
74
- return $1 if content_for_scanner =~ /Domain Name:\s+(.*)\n/i
75
- end
76
-
77
- property_not_supported :domain_id
78
-
79
- property_not_supported :registrar
80
-
81
- property_not_supported :admin_contacts
82
-
83
- property_supported :registrant_contacts do
84
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
85
- end
86
-
87
- property_supported :technical_contacts do
88
- build_contact("Tech Contact", Whois::Record::Contact::TYPE_TECHNICAL)
89
- end
90
-
91
- property_supported :billing_contacts do
92
- build_contact("Billing Contact", Whois::Record::Contact::TYPE_BILLING)
93
- end
94
-
95
- private
96
-
97
- def build_contact(element, type)
98
- reg=Record::Contact.new(:type => type)
99
- if content_for_scanner.gsub(/\//,'') =~ /(.+#{element}\n(.+\n)+)\n/i
100
- line_num=1
101
- $1.split(%r{\n}).each do |line|
102
- reg["name"]=line.strip if line_num==2
103
- reg["organization"]=line.strip if line_num==2
104
- reg["address"]=line.strip if line_num==3
105
- reg["city"]=line.strip if line_num==4
106
- reg["zip"]=line.strip if line_num==5
107
- entry=line.split(':')
108
- reg["email"]=entry[1].strip if entry[0]=~ /Email/i
109
- line_num=line_num+1
110
- end
111
- end
112
- return reg
113
- end
114
- # ----------------------------------------------------------------------------
115
-
116
- end
117
- end
118
- end
119
- end