wmap 2.7.2 → 2.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -16
  3. data/bin/refresh +5 -5
  4. data/lib/wmap.rb +1 -1
  5. data/lib/wmap/site_tracker.rb +1 -4
  6. data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
  7. data/lib/wmap/url_crawler.rb +1 -1
  8. data/version.txt +2 -2
  9. data/wmap.gemspec +2 -2
  10. metadata +4 -59
  11. data/ruby_whois_patches/base_cocca2.rb +0 -149
  12. data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
  13. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
  14. data/ruby_whois_patches/whois.above.com.rb +0 -61
  15. data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
  16. data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
  17. data/ruby_whois_patches/whois.ai.rb +0 -112
  18. data/ruby_whois_patches/whois.arnes.si.rb +0 -121
  19. data/ruby_whois_patches/whois.ascio.com.rb +0 -91
  20. data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
  21. data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
  22. data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
  23. data/ruby_whois_patches/whois.denic.de.rb +0 -174
  24. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
  25. data/ruby_whois_patches/whois.dns.be.rb +0 -134
  26. data/ruby_whois_patches/whois.dns.lu.rb +0 -129
  27. data/ruby_whois_patches/whois.dns.pl.rb +0 -150
  28. data/ruby_whois_patches/whois.dns.pt.rb +0 -119
  29. data/ruby_whois_patches/whois.domain.kg.rb +0 -126
  30. data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
  31. data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
  32. data/ruby_whois_patches/whois.dot.tk.rb +0 -140
  33. data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
  34. data/ruby_whois_patches/whois.isnic.is.rb +0 -130
  35. data/ruby_whois_patches/whois.je.rb +0 -119
  36. data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
  37. data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
  38. data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
  39. data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
  40. data/ruby_whois_patches/whois.nic.as.rb +0 -96
  41. data/ruby_whois_patches/whois.nic.at.rb +0 -109
  42. data/ruby_whois_patches/whois.nic.ch.rb +0 -141
  43. data/ruby_whois_patches/whois.nic.cl.rb +0 -117
  44. data/ruby_whois_patches/whois.nic.ec.rb +0 -157
  45. data/ruby_whois_patches/whois.nic.im.rb +0 -120
  46. data/ruby_whois_patches/whois.nic.it.rb +0 -170
  47. data/ruby_whois_patches/whois.nic.lv.rb +0 -116
  48. data/ruby_whois_patches/whois.nic.ly.rb +0 -127
  49. data/ruby_whois_patches/whois.nic.mu.rb +0 -27
  50. data/ruby_whois_patches/whois.nic.mx.rb +0 -123
  51. data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
  52. data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
  53. data/ruby_whois_patches/whois.nic.tel.rb +0 -129
  54. data/ruby_whois_patches/whois.nic.tr.rb +0 -133
  55. data/ruby_whois_patches/whois.nic.us.rb +0 -129
  56. data/ruby_whois_patches/whois.nic.ve.rb +0 -135
  57. data/ruby_whois_patches/whois.norid.no.rb +0 -127
  58. data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
  59. data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
  60. data/ruby_whois_patches/whois.registro.br.rb +0 -109
  61. data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
  62. data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
  63. data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
  64. data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
  65. data/ruby_whois_patches/whois.tucows.com.rb +0 -70
  66. data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,112 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.ai parser
19
- #
20
- # Parser for the whois.ai server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisAi < Base
29
-
30
- property_supported :status do
31
- if available?
32
- :available
33
- else
34
- :registered
35
- end
36
- end
37
-
38
- property_supported :available? do
39
- !!(content_for_scanner =~ /Domain (.+?) not registred/)
40
- end
41
-
42
- property_supported :registered? do
43
- !available?
44
- end
45
-
46
- property_not_supported :created_on
47
-
48
- property_not_supported :updated_on
49
-
50
- property_not_supported :expires_on
51
-
52
- property_supported :nameservers do
53
- if content_for_scanner =~ /Nameservers\n((.+\n)+)\n/
54
- $1.split("\n").select { |e| e =~ /Server Hostname/ }.map do |line|
55
- Record::Nameserver.new(:name => line.split(":").last.strip)
56
- end
57
- end
58
- end
59
-
60
- # The following methods are implemented by Yang Li on 02/04/2013
61
- # ----------------------------------------------------------------------------
62
- property_supported :domain do
63
- return $2.strip if content_for_scanner =~ /Complete Domain Name(.*):(.*)\n/i
64
- end
65
-
66
- property_not_supported :domain_id
67
-
68
- property_not_supported :registrar
69
-
70
- property_supported :admin_contacts do
71
- build_contact("Administrative Contact", Whois::Record::Contact::TYPE_ADMIN)
72
- end
73
-
74
- property_supported :registrant_contacts do
75
- build_contact("Organization Using Domain Name", Whois::Record::Contact::TYPE_REGISTRANT)
76
- end
77
-
78
- property_supported :technical_contacts do
79
- build_contact("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
80
- end
81
-
82
- property_supported :billing_contacts do
83
- build_contact("Billing Contact", Whois::Record::Contact::TYPE_BILLING)
84
- end
85
-
86
- private
87
-
88
- def build_contact(element, type)
89
- reg=Record::Contact.new(:type => type)
90
- if content_for_scanner =~ /#{element}\n((^\s.+\n)+)/i
91
- $1.scan(/^(.+):(.+)\n/).map do |entry|
92
- reg["id"]=entry[1].strip if entry[0] =~ /NIC Handle/i
93
- reg["name"]=entry[1].strip if entry[0] =~ /Name \(Last\, First\)/i
94
- reg["organization"]=entry[1].strip if entry[0]=~ /Organization Name/i
95
- reg["address"]=entry[1].strip if entry[0]=~ /Street Address/i
96
- reg["city"]= entry[1].strip if entry[0]=~ /City/i
97
- reg["zip"]=entry[1].strip if entry[0]=~ /Postal Code/i
98
- reg["state"]=entry[1].strip if entry[0]=~ /State/i
99
- reg["country"]=entry[1].strip if entry[0]=~ /Country/i
100
- reg["phone"]=entry[1].strip if entry[0]=~ /Phone\sNumber/i
101
- reg["fax"]=entry[1].strip if entry[0]=~ /Fax\sNumber/i
102
- reg["email"]=entry[1].strip if entry[0]=~ /E-Mailbox/i
103
- end
104
- end
105
- return reg
106
- end
107
- # ----------------------------------------------------------------------------
108
-
109
- end
110
- end
111
- end
112
- end
@@ -1,121 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.arnes.si parser
19
- #
20
- # Parser for the whois.arnes.si server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisArnesSi < Base
29
-
30
- property_supported :status do
31
- if content_for_scanner =~ /status:\s+(.+)\n/
32
- case $1.downcase
33
- when "ok" then :registered
34
- else
35
- Whois.bug!(ParserError, "Unknown status `#{$1}'.")
36
- end
37
- else
38
- :available
39
- end
40
- end
41
-
42
- property_supported :available? do
43
- !!(content_for_scanner =~ /% No entries found/)
44
- end
45
-
46
- property_supported :registered? do
47
- !available?
48
- end
49
-
50
-
51
- property_supported :created_on do
52
- if content_for_scanner =~ /created:\s+(.*)\n/
53
- Time.parse($1)
54
- end
55
- end
56
-
57
- property_not_supported :updated_on
58
-
59
- property_supported :expires_on do
60
- if content_for_scanner =~ /expire:\s+(.*)\n/
61
- Time.parse($1)
62
- end
63
- end
64
-
65
- property_supported :nameservers do
66
- content_for_scanner.scan(/nameserver:\s+(.+)\n/).flatten.map do |name|
67
- Record::Nameserver.new(:name => name)
68
- end
69
- end
70
-
71
- # The following methods are implemented by Yang Li on 01/24/2013
72
- # ----------------------------------------------------------------------------
73
- property_supported :domain do
74
- return $1 if content_for_scanner =~ /domain:\s+(.*)\n/i
75
- end
76
-
77
- property_not_supported :domain_id
78
-
79
- property_supported :registrar do
80
- reg=Record::Registrar.new
81
- content_for_scanner.scan(/^(registrar.*):\s+(.+)\n/).map do |entry|
82
- reg["name"] = entry[1] if entry[0] == "registrar"
83
- reg["organization"] = entry[1] if entry[0] == "registrar"
84
- reg["url"] = entry[1] if entry[0] =~ /url/i
85
- end
86
- return reg
87
- end
88
-
89
- property_not_supported :admin_contacts
90
-
91
- property_supported :registrant_contacts do
92
- build_contact("Domain holder", Whois::Record::Contact::TYPE_REGISTRANT)
93
- end
94
-
95
- property_supported :technical_contacts do
96
- build_contact("Tech", Whois::Record::Contact::TYPE_TECHNICAL)
97
- end
98
-
99
- property_not_supported :billing_contacts
100
-
101
- private
102
-
103
- def build_contact(element, type)
104
- reg=Record::Contact.new(:type => type)
105
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
106
- line_num=1
107
- $1.split(%r{\n}).each do |line|
108
- line=line.strip unless line.nil?
109
- reg["name"]=line if line_num==1
110
- reg["organization"]=line if line_num==1
111
- line_num=line_num+1
112
- end
113
- end
114
- return reg
115
- end
116
- # ----------------------------------------------------------------------------
117
-
118
- end
119
- end
120
- end
121
- end
@@ -1,91 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
- require 'whois/record/parser/base'
10
-
11
- module Whois
12
- class Record
13
- class Parser
14
-
15
- # Parser for the whois.ascio.com server, added by Yang Li 02/05/2013.
16
- class WhoisAscioCom < Base
17
-
18
- property_supported :created_on do
19
- if content_for_scanner =~ /^Record created:\s+(.+)\n/
20
- Time.parse($1)
21
- end
22
- end
23
-
24
- property_supported :updated_on do
25
- if content_for_scanner =~ /^Record last updated:\s+(.+)\n/
26
- Time.parse($1)
27
- end
28
- end
29
-
30
- property_supported :expires_on do
31
- if content_for_scanner =~ /Record expires:\s+(.+)\n/
32
- Time.parse($1)
33
- end
34
- end
35
-
36
- property_supported :nameservers do
37
- if content_for_scanner =~ /^Domain servers in listed order:\n((.+\n)+)/
38
- $1.split(%r{\n}).map do |line|
39
- Record::Nameserver.new(:name => line.strip.split(%r{\s+})[0])
40
- end
41
- end
42
- end
43
-
44
- property_supported :domain do
45
- return $1 if content_for_scanner =~ /Domain name:\s+(.+)\n/i
46
- end
47
-
48
- property_not_supported :domain_id
49
-
50
- property_supported :admin_contacts do
51
- build_contact("Administrative contact", Whois::Record::Contact::TYPE_ADMIN)
52
- end
53
-
54
- property_supported :registrant_contacts do
55
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
56
- end
57
-
58
- property_supported :technical_contacts do
59
- build_contact("Technical contact", Whois::Record::Contact::TYPE_TECHNICAL)
60
- end
61
-
62
- property_not_supported :billing_contacts
63
-
64
- private
65
-
66
- def build_contact(element, type)
67
- reg=Record::Contact.new(:type => type)
68
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
69
- line_num=1
70
- $1.split(%r{\n}).each do |line|
71
- line=line.strip unless line.nil?
72
- reg["id"]=line.split('(')[1].split(')')[0] if line_num==1
73
- reg["name"]=line.split('(')[0] if line_num==1
74
- reg["organization"]=line.strip if line_num==2
75
- reg["address"]=line if line_num==3
76
- reg["city"]=line if line_num==5
77
- reg["country_code"]=line if line_num==6
78
- reg["email"]=line if line=~ /.+\@.+\.\w/
79
- reg["phone"]=line.split('Fax:')[0] if line=~ /Fax/
80
- reg["fax"]=line.split('Fax:')[1] if line=~ /Fax/
81
- line_num=line_num+1
82
- end
83
- end
84
- return reg
85
- end
86
- # ----------------------------------------------------------------------------
87
-
88
- end
89
- end
90
- end
91
- end
@@ -1,123 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
- require 'whois/record/scanners/whois.cnnic.cn.rb'
12
-
13
-
14
- module Whois
15
- class Record
16
- class Parser
17
-
18
- # Parser for the whois.cnnic.cn server.
19
- class WhoisCnnicCn < Base
20
- include Scanners::Nodable
21
-
22
-
23
- property_not_supported :disclaimer
24
-
25
-
26
- property_supported :domain do
27
- node("Domain Name", &:downcase)
28
- end
29
-
30
- property_supported :domain_id do
31
- node("ROID")
32
- end
33
-
34
-
35
- property_not_supported :referral_whois
36
-
37
- property_not_supported :referral_url
38
-
39
-
40
- property_supported :status do
41
- Array.wrap node("Domain Status")
42
- end
43
-
44
- property_supported :available? do
45
- !!node("status:available")
46
- end
47
-
48
- property_supported :registered? do
49
- reserved? || !available?
50
- end
51
-
52
- # NEWPROPERTY
53
- def reserved?
54
- !!node("status:reserved")
55
- end
56
-
57
-
58
- property_supported :created_on do
59
- node("Registration Date") { |value| Time.parse(value) }
60
- end
61
-
62
- property_not_supported :updated_on
63
-
64
- property_supported :expires_on do
65
- node("Expiration Date") { |value| Time.parse(value) }
66
- end
67
-
68
-
69
- property_supported :registrar do
70
- node("Sponsoring Registrar") do |value|
71
- Record::Registrar.new(
72
- :id => value,
73
- :name => value
74
- )
75
- end
76
- end
77
-
78
- property_supported :registrant_contacts do
79
- build_contact("Registrant", Whois::Record::Contact::TYPE_REGISTRANT)
80
- end
81
-
82
- property_not_supported :admin_contacts
83
-
84
- property_not_supported :technical_contacts
85
-
86
-
87
- property_supported :nameservers do
88
- Array.wrap(node("Name Server")).map do |name|
89
- Nameserver.new(:name => name.downcase)
90
- end
91
- end
92
-
93
-
94
- # Initializes a new {Scanners::WhoisCnnicCn} instance
95
- # passing the {#content_for_scanner}
96
- # and calls +parse+ on it.
97
- #
98
- # @return [Hash]
99
- def parse
100
- Scanners::WhoisCnnicCn.new(content_for_scanner).parse
101
- end
102
-
103
-
104
- private
105
- # Modified by Yang Li 01/24/2013
106
- def build_contact(element, type)
107
- n = node("#{element}")
108
- o = node("#{element}")
109
- e = node("#{element} Contact Email")
110
- return if n.nil? && o.nil? && e.nil?
111
-
112
- Record::Contact.new(
113
- :type => type,
114
- :name => n,
115
- :organization => o,
116
- :email => e
117
- )
118
- end
119
-
120
- end
121
- end
122
- end
123
- end