wmap 2.7.2 → 2.7.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -16
  3. data/bin/refresh +5 -5
  4. data/lib/wmap.rb +1 -1
  5. data/lib/wmap/site_tracker.rb +1 -4
  6. data/lib/wmap/{wp_tracker.rb → site_tracker/wp_tracker.rb} +41 -4
  7. data/lib/wmap/url_crawler.rb +1 -1
  8. data/version.txt +2 -2
  9. data/wmap.gemspec +2 -2
  10. metadata +4 -59
  11. data/ruby_whois_patches/base_cocca2.rb +0 -149
  12. data/ruby_whois_patches/kero.yachay.pe.rb +0 -120
  13. data/ruby_whois_patches/whois.PublicDomainRegistry.com.rb +0 -124
  14. data/ruby_whois_patches/whois.above.com.rb +0 -61
  15. data/ruby_whois_patches/whois.adamsnames.tc.rb +0 -107
  16. data/ruby_whois_patches/whois.aeda.net.ae.rb +0 -105
  17. data/ruby_whois_patches/whois.ai.rb +0 -112
  18. data/ruby_whois_patches/whois.arnes.si.rb +0 -121
  19. data/ruby_whois_patches/whois.ascio.com.rb +0 -91
  20. data/ruby_whois_patches/whois.cnnic.cn.rb +0 -123
  21. data/ruby_whois_patches/whois.corporatedomains.com.rb +0 -67
  22. data/ruby_whois_patches/whois.crsnic.net.rb +0 -108
  23. data/ruby_whois_patches/whois.denic.de.rb +0 -174
  24. data/ruby_whois_patches/whois.dk-hostmaster.dk.rb +0 -120
  25. data/ruby_whois_patches/whois.dns.be.rb +0 -134
  26. data/ruby_whois_patches/whois.dns.lu.rb +0 -129
  27. data/ruby_whois_patches/whois.dns.pl.rb +0 -150
  28. data/ruby_whois_patches/whois.dns.pt.rb +0 -119
  29. data/ruby_whois_patches/whois.domain.kg.rb +0 -126
  30. data/ruby_whois_patches/whois.domainregistry.my.rb +0 -123
  31. data/ruby_whois_patches/whois.domreg.lt.rb +0 -110
  32. data/ruby_whois_patches/whois.dot.tk.rb +0 -140
  33. data/ruby_whois_patches/whois.hkirc.hk.rb +0 -121
  34. data/ruby_whois_patches/whois.isnic.is.rb +0 -130
  35. data/ruby_whois_patches/whois.je.rb +0 -119
  36. data/ruby_whois_patches/whois.jprs.jp.rb +0 -137
  37. data/ruby_whois_patches/whois.kenic.or.ke.rb +0 -140
  38. data/ruby_whois_patches/whois.markmonitor.com.rb +0 -118
  39. data/ruby_whois_patches/whois.melbourneit.com.rb +0 -58
  40. data/ruby_whois_patches/whois.nic.as.rb +0 -96
  41. data/ruby_whois_patches/whois.nic.at.rb +0 -109
  42. data/ruby_whois_patches/whois.nic.ch.rb +0 -141
  43. data/ruby_whois_patches/whois.nic.cl.rb +0 -117
  44. data/ruby_whois_patches/whois.nic.ec.rb +0 -157
  45. data/ruby_whois_patches/whois.nic.im.rb +0 -120
  46. data/ruby_whois_patches/whois.nic.it.rb +0 -170
  47. data/ruby_whois_patches/whois.nic.lv.rb +0 -116
  48. data/ruby_whois_patches/whois.nic.ly.rb +0 -127
  49. data/ruby_whois_patches/whois.nic.mu.rb +0 -27
  50. data/ruby_whois_patches/whois.nic.mx.rb +0 -123
  51. data/ruby_whois_patches/whois.nic.net.sa.rb +0 -111
  52. data/ruby_whois_patches/whois.nic.or.kr.rb +0 -101
  53. data/ruby_whois_patches/whois.nic.tel.rb +0 -129
  54. data/ruby_whois_patches/whois.nic.tr.rb +0 -133
  55. data/ruby_whois_patches/whois.nic.us.rb +0 -129
  56. data/ruby_whois_patches/whois.nic.ve.rb +0 -135
  57. data/ruby_whois_patches/whois.norid.no.rb +0 -127
  58. data/ruby_whois_patches/whois.pandi.or.id.rb +0 -118
  59. data/ruby_whois_patches/whois.psi-usa.info.rb +0 -63
  60. data/ruby_whois_patches/whois.registro.br.rb +0 -109
  61. data/ruby_whois_patches/whois.registrygate.com.rb +0 -55
  62. data/ruby_whois_patches/whois.rrpproxy.net.rb +0 -61
  63. data/ruby_whois_patches/whois.sgnic.sg.rb +0 -130
  64. data/ruby_whois_patches/whois.srs.net.nz.rb +0 -166
  65. data/ruby_whois_patches/whois.tucows.com.rb +0 -70
  66. data/ruby_whois_patches/whois.twnic.net.tw.rb +0 -133
@@ -1,126 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.domain.kg
19
- #
20
- # Parser for the whois.domain.kg server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisDomainKg < Base
29
-
30
- property_supported :status do
31
- if available?
32
- :available
33
- else
34
- :registered
35
- end
36
- end
37
-
38
- property_supported :available? do
39
- !!(content_for_scanner =~ /This domain is available for registration/)
40
- end
41
-
42
- property_supported :registered? do
43
- !available?
44
- end
45
-
46
-
47
- property_supported :created_on do
48
- if content_for_scanner =~ /Record created: (.+?)\n/
49
- Time.parse($1)
50
- end
51
- end
52
-
53
- property_supported :updated_on do
54
- if content_for_scanner =~ /Record last updated on (.+?)\n/
55
- Time.parse($1)
56
- end
57
- end
58
-
59
- property_supported :expires_on do
60
- if content_for_scanner =~ /Record expires on (.+?)\n/
61
- Time.parse($1)
62
- end
63
- end
64
-
65
- property_supported :nameservers do
66
- if content_for_scanner =~ /Name servers in the listed order:\n\n((.+\n)+)\n/
67
- $1.split("\n").map do |name|
68
- Record::Nameserver.new(:name => name.downcase)
69
- end
70
- end
71
- end
72
-
73
- # The following methods are implemented by Yang Li on 02/13/2013
74
- # ----------------------------------------------------------------------------
75
- property_supported :domain do
76
- return $1 if content_for_scanner =~ /^Domain\s+(.*)\n/i
77
- end
78
-
79
- property_not_supported :domain_id
80
-
81
- property_supported :registrar do
82
- reg=Record::Registrar.new
83
- content_for_scanner.scan(/^(.+):\s+(.+)\n/).map do |entry|
84
- reg["name"] = entry[1].strip if entry[0] =~ /Registrar/i
85
- reg["organization"] = entry[1].strip if entry[0] =~ /Registrar/i
86
- reg["url"] = entry[1].strip if entry[0] =~ /Referral URL/i
87
- end
88
- return reg
89
- end
90
-
91
- property_supported :admin_contacts do
92
- build_contact("Administrative Contact", Whois::Record::Contact::TYPE_ADMIN)
93
- end
94
-
95
- property_not_supported :registrant_contacts
96
-
97
- property_supported :technical_contacts do
98
- build_contact("Technical Contact", Whois::Record::Contact::TYPE_TECHNICAL)
99
- end
100
-
101
- property_supported :billing_contacts do
102
- build_contact("Billing Contact", Whois::Record::Contact::TYPE_BILLING)
103
- end
104
-
105
- private
106
-
107
- def build_contact(element, type)
108
- reg=Record::Contact.new(:type => type)
109
- if content_for_scanner =~ /^#{element}:\n((.+\n)+)\n/i
110
- $1.scan(/^(.+):(.+)\n/).map do |entry|
111
- reg["id"]=entry[1] if entry[0] =~ /PID/i
112
- reg["name"]=entry[1] if entry[0] =~ /Name/i
113
- reg["address"]=entry[1] if entry[0] =~ /Address/i
114
- reg["phone"]=entry[1] if entry[0] =~ /phone/i
115
- reg["fax"]=entry[1] if entry[0] =~ /fax/i
116
- reg["email"]=entry[1] if entry[0] =~ /Email/i
117
- end
118
- end
119
- return reg
120
- end
121
- # ----------------------------------------------------------------------------
122
-
123
- end
124
- end
125
- end
126
- end
@@ -1,123 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- # Parser for the whois.domainregistry.my server.
18
- #
19
- # @note This parser is just a stub and provides only a few basic methods
20
- # to check for domain availability and get domain status.
21
- # Please consider to contribute implementing missing methods.
22
- #
23
- # @see Whois::Record::Parser::Example
24
- # The Example parser for the list of all available methods.
25
- class WhoisDomainregistryMy < Base
26
-
27
- property_supported :status do
28
- if available?
29
- :available
30
- else
31
- :registered
32
- end
33
- end
34
-
35
- property_supported :available? do
36
- !!(content_for_scanner =~ /Domain Name [^ ]+ does not exist in data repository/)
37
- end
38
-
39
- property_supported :registered? do
40
- !available?
41
- end
42
-
43
-
44
- property_supported :created_on do
45
- if content_for_scanner =~ /\[Record Created\]\s+(.+?)\n/
46
- Time.parse($1)
47
- end
48
- end
49
-
50
- property_supported :updated_on do
51
- if content_for_scanner =~ /\[Record Last Modified\]\s+(.+?)\n/
52
- Time.parse($1)
53
- end
54
- end
55
-
56
- property_supported :expires_on do
57
- if content_for_scanner =~ /\[Record Expired\]\s+(.+?)\n/
58
- Time.parse($1)
59
- end
60
- end
61
-
62
- property_supported :nameservers do
63
- content_for_scanner.scan(/\[(?:Primary|Secondary) Name Server\](?:.+?)\n(.+\n)/).flatten.map do |line|
64
- name, ipv4 = line.strip.split(/\s+/)
65
- Record::Nameserver.new(:name => name, :ipv4 => ipv4)
66
- end
67
- end
68
-
69
- # The following methods are implemented by Yang Li on 02/05/2013
70
- # ----------------------------------------------------------------------------
71
- property_supported :domain do
72
- return $1 if content_for_scanner =~ /Domain Name\]\s+(.*)\n/i
73
- end
74
-
75
- property_supported :domain_id do
76
- return $1 if content_for_scanner =~ /Registration No\.\]\s+(.*)\n/i
77
- end
78
-
79
- property_not_supported :registrar
80
-
81
- property_supported :admin_contacts do
82
- build_contact("h", Whois::Record::Contact::TYPE_ADMIN)
83
- end
84
-
85
- property_supported :registrant_contacts do
86
- build_contact("g", Whois::Record::Contact::TYPE_REGISTRANT)
87
- end
88
-
89
- property_supported :technical_contacts do
90
- build_contact("j", Whois::Record::Contact::TYPE_TECHNICAL)
91
- end
92
-
93
- property_supported :billing_contacts do
94
- build_contact("i", Whois::Record::Contact::TYPE_BILLING)
95
- end
96
-
97
- private
98
-
99
- def build_contact(element, type)
100
- reg=Record::Contact.new(:type => type)
101
- if content_for_scanner =~ /^#{element}\s\[((.+\n)+)\n/i
102
- line_num=0
103
- $1.split(%r{\n}).each do |line|
104
- reg["name"]=line.strip if line_num==1
105
- reg["organization"]=line.strip if line_num==2
106
- reg["address"]=line.strip if line_num==3
107
- reg["city"]=line.strip if line_num==4
108
- reg["state"]=line.strip if line_num==5
109
- reg["country"]=line.strip if line_num==6
110
- reg["email"]=line.strip if line =~ /\w+\@\w+\.\w+/
111
- reg["phone"]=line.strip.split(' ')[1] if line =~ /\(Tel\)/i
112
- reg["fax"]=line.strip.split(' ')[1] if line =~ /\(Fax\)/i
113
- line_num=line_num+1
114
- end
115
- end
116
- return reg
117
- end
118
- # ----------------------------------------------------------------------------
119
-
120
- end
121
- end
122
- end
123
- end
@@ -1,110 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.domreg.lt parser
19
- #
20
- # Parser for the whois.domreg.lt server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisDomregLt < Base
29
-
30
- property_supported :status do
31
- if content_for_scanner =~ /Status:\s+(.*)\n/
32
- $1.to_sym
33
- end
34
- end
35
-
36
- property_supported :available? do
37
- (status == :available)
38
- end
39
-
40
- property_supported :registered? do
41
- !available?
42
- end
43
-
44
-
45
- property_supported :created_on do
46
- if content_for_scanner =~ /Registered:\s+(.*)\n/
47
- Time.parse($1)
48
- end
49
- end
50
-
51
- property_not_supported :updated_on
52
-
53
- property_not_supported :expires_on
54
-
55
-
56
- property_supported :nameservers do
57
- content_for_scanner.scan(/Nameserver:\s+(.+)\n/).flatten.map do |line|
58
- if line =~ /(.+)\t\[(.+)\]/
59
- Record::Nameserver.new(:name => $1, :ipv4 => $2)
60
- else
61
- Record::Nameserver.new(:name => line.strip)
62
- end
63
- end
64
- end
65
-
66
- # The following methods are implemented by Yang Li on 01/24/2013
67
- # ----------------------------------------------------------------------------
68
- property_supported :domain do
69
- return $1 if content_for_scanner =~ /^Domain:\s+(.*)\n/i
70
- end
71
-
72
- property_not_supported :domain_id
73
-
74
- property_supported :registrar do
75
- reg=Record::Registrar.new
76
- content_for_scanner.scan(/^(Registrar.*):\s+(.+)\n/).map do |entry|
77
- reg["name"] = entry[1] if entry[0] == "Registrar"
78
- reg["organization"] = entry[1] if entry[0] == "registrar"
79
- reg["url"] = entry[1] if entry[0] =~ /website/i
80
- end
81
- return reg
82
- end
83
-
84
- property_not_supported :admin_contacts
85
-
86
- property_supported :registrant_contacts do
87
- build_contact("Contact", Whois::Record::Contact::TYPE_REGISTRANT)
88
- end
89
-
90
- property_not_supported :technical_contacts
91
-
92
- property_not_supported :billing_contacts
93
-
94
- private
95
-
96
- def build_contact(element, type)
97
- reg=Record::Contact.new(:type => type)
98
- content_for_scanner.scan(/^#{element}(.+):\s+(.+)\n/).map do |entry|
99
- reg["name"] = entry[1] if entry[0] =~ /organization/i
100
- reg["organization"] = entry[1] if entry[0] =~ /organization/i
101
- reg["email"] = entry[1] if entry[0] =~ /email/i
102
- end
103
- return reg
104
- end
105
- # ---------------------------------------------------------------------------
106
-
107
- end
108
- end
109
- end
110
- end
@@ -1,140 +0,0 @@
1
- #--
2
- # Ruby Whois
3
- #
4
- # An intelligent pure Ruby WHOIS client and parser.
5
- #
6
- # Copyright (c) 2009-2012 Simone Carletti <weppos@weppos.net>
7
- #++
8
-
9
-
10
- require 'whois/record/parser/base'
11
-
12
-
13
- module Whois
14
- class Record
15
- class Parser
16
-
17
- #
18
- # = whois.dot.tk parser
19
- #
20
- # Parser for the whois.dot.tk server.
21
- #
22
- # NOTE: This parser is just a stub and provides only a few basic methods
23
- # to check for domain availability and get domain status.
24
- # Please consider to contribute implementing missing methods.
25
- # See WhoisNicIt parser for an explanation of all available methods
26
- # and examples.
27
- #
28
- class WhoisDotTk < Base
29
-
30
- property_supported :status do
31
- if available?
32
- :available
33
- else
34
- :registered
35
- end
36
- end
37
-
38
- property_supported :available? do
39
- !!(content_for_scanner =~ /domain name not known/)
40
- end
41
-
42
- property_supported :registered? do
43
- !available?
44
- end
45
-
46
-
47
- property_supported :created_on do
48
- if content_for_scanner =~ /Domain registered:\s+(.*)\n/
49
- DateTime.strptime($1, "%m/%d/%Y").to_time
50
- end
51
- end
52
-
53
- property_not_supported :updated_on
54
-
55
- property_supported :expires_on do
56
- if content_for_scanner =~ /Record will expire on:\s+(.*)\n/
57
- DateTime.strptime($1, "%m/%d/%Y").to_time
58
- end
59
- end
60
-
61
-
62
- property_supported :nameservers do
63
- if content_for_scanner =~ /Domain Nameservers:\n((.+\n)+)\s+\n/
64
- $1.split("\n").map do |name|
65
- Record::Nameserver.new(:name => name.strip.downcase)
66
- end
67
- end
68
- end
69
-
70
- # The following methods are implemented by Yang Li on 03/05/2013
71
- # ----------------------------------------------------------------------------
72
- property_supported :domain do
73
- if content_for_scanner =~ /Domain name:\n(.+)\n\n/
74
- return $1.strip
75
- end
76
- end
77
-
78
- property_not_supported :domain_id
79
-
80
- property_not_supported :registrar
81
-
82
- property_supported :registrant_contacts do
83
- build_contact("Organisation", Whois::Record::Contact::TYPE_REGISTRANT)
84
- end
85
-
86
- property_not_supported :admin_contacts
87
-
88
- property_not_supported :technical_contacts
89
-
90
- property_not_supported :billing_contacts
91
-
92
- private
93
-
94
- def build_contact(element, type)
95
- reg=Record::Contact.new(:type => type)
96
- if content_for_scanner =~ /#{element}:\n((.+\n)+)\n/
97
- con=$1
98
- contact=con.split(%r{\n})
99
- reg["organization"]=contact.shift.strip
100
- reg["name"]=contact.shift.strip
101
- reg["phone"]=build_node("Phone",con)
102
- reg["fax"]=build_node("Fax",con)
103
- reg["email"]=build_node("E-mail",con)
104
- myaddr=build_address(contact)
105
- reg["address"]=myaddr["address"]
106
- reg["city"]=myaddr["city"]
107
- reg["zip"]=myaddr["zip"]
108
- reg["state"]=myaddr["state"]
109
- reg["country"]=myaddr["country"]
110
- end
111
- return reg
112
- end
113
-
114
- def build_node(element,context)
115
- return $1.strip if context =~ /#{element}:(.+)\n/i
116
- end
117
-
118
- def build_address(context)
119
- addr=Hash.new
120
- line_num=2
121
- context.each do |line|
122
- line=line.chomp.strip
123
- addr["address"]=line if line_num==2
124
- addr["state"]=line if line_num==4
125
- addr["country"]=line if line_num==5
126
- if line_num==3 && !line.nil?
127
- zips=line.split(' ')
128
- addr["city"]=zips.pop
129
- addr["zip"]=zips.join(' ')
130
- end
131
- line_num=line_num+1
132
- end
133
- return addr
134
- end
135
- # ----------------------------------------------------------------------------
136
- end
137
-
138
- end
139
- end
140
- end