miketracy-wwmd 0.2.16 → 0.2.17
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +21 -0
- data/{README → README.rdoc} +27 -2
- data/lib/wwmd.rb +4 -4
- data/lib/wwmd/class_extensions.rb +2 -0
- data/lib/wwmd/{mixins.rb → class_extensions/extensions_base.rb} +25 -121
- data/lib/wwmd/class_extensions/extensions_encoding.rb +79 -0
- data/lib/wwmd/{mixins_external.rb → class_extensions/extensions_external.rb} +0 -0
- data/lib/wwmd/class_extensions/extensions_nilclass.rb +11 -0
- data/lib/wwmd/{mixins_extends.rb → class_extensions/extensions_rbkb.rb} +0 -0
- data/lib/wwmd/{encoding.rb → class_extensions/mixins_string_encoding.rb} +6 -6
- data/lib/wwmd/page.rb +3 -245
- data/lib/wwmd/page/auth.rb +0 -166
- data/lib/wwmd/page/constants.rb +7 -4
- data/lib/wwmd/page/form.rb +0 -15
- data/lib/wwmd/page/form_array.rb +96 -74
- data/lib/wwmd/page/headers.rb +25 -21
- data/lib/wwmd/page/helpers.rb +30 -0
- data/lib/wwmd/{hpricot_html2text.rb → page/html2text_hpricot.rb} +1 -1
- data/lib/wwmd/{nokogiri_html2text.rb → page/html2text_nokogiri.rb} +0 -0
- data/lib/wwmd/page/inputs.rb +1 -1
- data/lib/wwmd/page/irb_helpers.rb +37 -13
- data/lib/wwmd/page/page.rb +238 -0
- data/lib/wwmd/page/parsing_convenience.rb +8 -3
- data/lib/wwmd/page/scrape.rb +15 -19
- data/lib/wwmd/page/spider.rb +11 -11
- data/lib/wwmd/urlparse.rb +20 -5
- data/lib/wwmd/viewstate.rb +9 -112
- data/lib/wwmd/viewstate/viewstate.rb +101 -0
- data/lib/wwmd/viewstate/viewstate_deserializer_methods.rb +36 -36
- data/lib/wwmd/viewstate/viewstate_types.rb +0 -4
- data/lib/wwmd/viewstate/viewstate_utils.rb +6 -1
- data/lib/wwmd/viewstate/vs_stubs.rb +22 -0
- data/lib/wwmd/viewstate/{vs_array.rb → vs_stubs/vs_array.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_binary_serialized.rb → vs_stubs/vs_binary_serialized.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_hashtable.rb → vs_stubs/vs_hashtable.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_hybrid_dict.rb → vs_stubs/vs_hybrid_dict.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_indexed_string.rb → vs_stubs/vs_indexed_string.rb} +1 -1
- data/lib/wwmd/viewstate/{vs_indexed_string_ref.rb → vs_stubs/vs_indexed_string_ref.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_int_enum.rb → vs_stubs/vs_int_enum.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_list.rb → vs_stubs/vs_list.rb} +2 -1
- data/lib/wwmd/viewstate/{vs_pair.rb → vs_stubs/vs_pair.rb} +3 -1
- data/lib/wwmd/viewstate/vs_stubs/vs_read_types.rb +11 -0
- data/lib/wwmd/viewstate/{vs_read_value.rb → vs_stubs/vs_read_value.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_sparse_array.rb → vs_stubs/vs_sparse_array.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_string.rb → vs_stubs/vs_string.rb} +2 -1
- data/lib/wwmd/viewstate/{vs_string_array.rb → vs_stubs/vs_string_array.rb} +4 -2
- data/lib/wwmd/viewstate/{vs_string_formatted.rb → vs_stubs/vs_string_formatted.rb} +4 -2
- data/lib/wwmd/viewstate/{viewstate_class_helpers.rb → vs_stubs/vs_stub_helpers.rb} +2 -1
- data/lib/wwmd/viewstate/{vs_triplet.rb → vs_stubs/vs_triplet.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_type.rb → vs_stubs/vs_type.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_unit.rb → vs_stubs/vs_unit.rb} +3 -1
- data/lib/wwmd/viewstate/{vs_value.rb → vs_stubs/vs_value.rb} +4 -2
- data/lib/wwmd/wwmd_config.rb +44 -36
- data/lib/wwmd/wwmd_puts.rb +9 -0
- data/lib/wwmd/wwmd_utils.rb +22 -24
- data/tasks/setup.rb +1 -1
- metadata +41 -35
- data/README.txt +0 -62
- data/lib/wwmd/viewstate/vs_read_types.rb +0 -11
- data/wwmd.gemspec +0 -0
data/History.txt
CHANGED
@@ -1,3 +1,24 @@
|
|
1
|
+
== 0.2.17 / 2009-06-22
|
2
|
+
|
3
|
+
* lots happening between here and .9
|
4
|
+
* viewstate refactor complete
|
5
|
+
* clean up page/page.rb
|
6
|
+
* cleaning up page/headers.rb
|
7
|
+
* cleaning up page/scrape.rb
|
8
|
+
* FormArray refactor includes the form action (full URL)
|
9
|
+
* page.submit(page.get_form)
|
10
|
+
* still bugs in URLParse but hunting them down throw by throw
|
11
|
+
* remove broken NTLM (preserve auth header warnings)
|
12
|
+
* remove WWMDConfig in favor of WWMD module methods but preserve old class for back compat
|
13
|
+
* add some burp helpers
|
14
|
+
* Page#from_paste (take entire request into Page and turn off cookies)
|
15
|
+
* burp log parsing coming
|
16
|
+
* Curb includes http_put (with header munging bug so careful)
|
17
|
+
* internal monkey patch for Curb to do arbitrary verb tampering (not here yet)
|
18
|
+
* add String#pbcopy
|
19
|
+
* move lots of things around for clarity during refactor
|
20
|
+
* refactor progressing but still unstable (2.0.16 gem including viewstate is good to go)
|
21
|
+
|
1
22
|
== 0.2.9 / 2009-05-05
|
2
23
|
|
3
24
|
* bonesify
|
data/{README → README.rdoc}
RENAMED
@@ -1,5 +1,16 @@
|
|
1
|
+
== PARDON OUR DUST
|
1
2
|
|
2
|
-
|
3
|
+
WWMD is currently in the throes of major cleanup and refactoring.
|
4
|
+
|
5
|
+
0.2.17 should be stable.
|
6
|
+
|
7
|
+
The viewstate tools can be had by themselves by using:
|
8
|
+
|
9
|
+
require 'wwmd/viewstate'
|
10
|
+
|
11
|
+
We appreciate your patience.
|
12
|
+
|
13
|
+
<;'"}()[]>{ XSSFish says, "Swim wif me"
|
3
14
|
|
4
15
|
== DESCRIPTION:
|
5
16
|
|
@@ -33,11 +44,25 @@ check the wiki regularly.
|
|
33
44
|
* hpricot (for the time being)
|
34
45
|
* htmlentities
|
35
46
|
|
47
|
+
== INSTALL
|
48
|
+
|
49
|
+
==== gem installation
|
50
|
+
|
51
|
+
WWMD is available as a gem from github:
|
52
|
+
|
53
|
+
gem sources -a http://gems.github.com #(you only have to do this once)
|
54
|
+
gem install mtracy-wwmd
|
55
|
+
|
56
|
+
=== manual installation
|
57
|
+
|
58
|
+
fetch the repository from github and add path/to/wwmd/lib to your RUBYLIB
|
59
|
+
environment variable
|
60
|
+
|
36
61
|
== LICENSE:
|
37
62
|
|
38
63
|
(The MIT License)
|
39
64
|
|
40
|
-
Copyright (c) 2008
|
65
|
+
Copyright (c) 2008,2009 Michael Tracy <mtracy@matasano.com>
|
41
66
|
|
42
67
|
Permission is hereby granted, free of charge, to any person obtaining
|
43
68
|
a copy of this software and associated documentation files (the
|
data/lib/wwmd.rb
CHANGED
@@ -15,7 +15,7 @@ require 'rexml/document'
|
|
15
15
|
module WWMD
|
16
16
|
|
17
17
|
# :stopdoc:
|
18
|
-
VERSION = "0.2.
|
18
|
+
VERSION = "0.2.17"
|
19
19
|
PARSER = :nokogiri # :nokogiri || :hpricot
|
20
20
|
LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
|
21
21
|
PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
|
@@ -54,7 +54,7 @@ module WWMD
|
|
54
54
|
::File.join(::File.dirname(fname), dir, '**', '*.rb'))
|
55
55
|
|
56
56
|
Dir.glob(search_me).sort.each do |rb|
|
57
|
-
next if rb =~ /
|
57
|
+
next if rb =~ /html2text_/
|
58
58
|
require rb
|
59
59
|
end
|
60
60
|
end
|
@@ -68,11 +68,11 @@ WWMD.require_all_libs_relative_to(__FILE__)
|
|
68
68
|
if WWMD::PARSER == :nokogiri
|
69
69
|
require 'nokogiri'
|
70
70
|
WWMD::HDOC = Nokogiri::HTML
|
71
|
-
require 'wwmd/
|
71
|
+
require 'wwmd/page/html2text_nokogiri'
|
72
72
|
else
|
73
73
|
require 'hpricot'
|
74
74
|
WWMD::HDOC = Hpricot
|
75
|
-
require 'wwmd/
|
75
|
+
require 'wwmd/page/html2text_hpricot'
|
76
76
|
end
|
77
77
|
|
78
78
|
# EOF
|
@@ -1,27 +1,11 @@
|
|
1
1
|
require 'htmlentities'
|
2
2
|
|
3
3
|
=begin rdoc
|
4
|
-
|
4
|
+
let's re-open everything!
|
5
5
|
=end
|
6
6
|
|
7
7
|
require 'uri'
|
8
8
|
|
9
|
-
alias putd puts#:nodoc:
|
10
|
-
alias putx puts#:nodoc:
|
11
|
-
alias putw puts#:nodoc:
|
12
|
-
|
13
|
-
# I really hate this
|
14
|
-
class NilClass#:nodoc:
|
15
|
-
def empty?; return true; end
|
16
|
-
def size; return 0; end
|
17
|
-
def to_form; return FormArray.new([]); end
|
18
|
-
def clop; return nil; end
|
19
|
-
def inner_html; return nil; end
|
20
|
-
def get_attribute(*args); return nil; end
|
21
|
-
def grep(*args); return []; end
|
22
|
-
def escape(*args); return nil; end
|
23
|
-
end
|
24
|
-
|
25
9
|
class Numeric
|
26
10
|
# return binary representation of <tt>length</tt> size padded with \x00
|
27
11
|
# length: length in bytes to return (padded with least signficant \x00
|
@@ -50,8 +34,6 @@ end
|
|
50
34
|
|
51
35
|
class String
|
52
36
|
|
53
|
-
@@he = HTMLEntities.new
|
54
|
-
|
55
37
|
def strip_up
|
56
38
|
self.gsub(/[^\x20-\x7e,\n]/,"").gsub(/^\n/,"")
|
57
39
|
end
|
@@ -106,63 +88,7 @@ class String
|
|
106
88
|
return [self.clip,self.clop]
|
107
89
|
end
|
108
90
|
|
109
|
-
|
110
|
-
def b64d
|
111
|
-
self.unpack("m").first
|
112
|
-
end
|
113
|
-
|
114
|
-
# base 64 encode
|
115
|
-
def b64e
|
116
|
-
[self].pack("m").chomp
|
117
|
-
end
|
118
|
-
|
119
|
-
# URI.escape using defaults or passed regexp
|
120
|
-
def escape(reg=nil,unicodify=false)
|
121
|
-
if reg.nil?
|
122
|
-
ret = URI.escape(self)
|
123
|
-
elsif reg.kind_of?(Symbol)
|
124
|
-
case reg
|
125
|
-
when :none; return self
|
126
|
-
when :default; ret = URI.escape(self)
|
127
|
-
else; ret = URI.escape(self,WWMD::ESCAPE[reg])
|
128
|
-
end
|
129
|
-
else
|
130
|
-
ret = URI.escape(self,reg)
|
131
|
-
end
|
132
|
-
if unicodify
|
133
|
-
ret.gsub!(/%/,"%u00")
|
134
|
-
end
|
135
|
-
return ret
|
136
|
-
end
|
137
|
-
|
138
|
-
# URI.escape
|
139
|
-
def escape_url(reg=WWMD::ESCAPE[:url])#:nodoc:
|
140
|
-
self.escape(reg)
|
141
|
-
end
|
142
|
-
|
143
|
-
def escape_xss(reg=WWMD::ESCAPE[:xss])#:nodoc:
|
144
|
-
self.escape(reg)
|
145
|
-
end
|
146
|
-
|
147
|
-
def escape_default(reg=WWMD::ESCAPE[:default])
|
148
|
-
self.escape(reg)
|
149
|
-
end
|
150
|
-
# URI.escape all characters in string
|
151
|
-
def escape_all#:nodoc:
|
152
|
-
self.escape(/.*/)
|
153
|
-
end
|
154
|
-
|
155
|
-
# URI.unescape
|
156
|
-
def unescape
|
157
|
-
URI.unescape(self)
|
158
|
-
end
|
159
|
-
|
160
|
-
# encode the string using Encoding.to_utf7(self,false)
|
161
|
-
# (encode non [:alnum:] characters). Set <tt>all</tt> true
|
162
|
-
# to encode all characters in the string.
|
163
|
-
def to_utf7(all=false)
|
164
|
-
Encoding.to_utf7(self,all)
|
165
|
-
end
|
91
|
+
alias_method :clipa, :clopa
|
166
92
|
|
167
93
|
# File.dirname with a trailing slash
|
168
94
|
def dirname
|
@@ -171,8 +97,12 @@ class String
|
|
171
97
|
end
|
172
98
|
|
173
99
|
# File.basename
|
174
|
-
def basename
|
175
|
-
|
100
|
+
def basename(ext=nil)
|
101
|
+
if ext
|
102
|
+
File.basename(self,ext)
|
103
|
+
else
|
104
|
+
File.basename(self)
|
105
|
+
end
|
176
106
|
end
|
177
107
|
|
178
108
|
def extname
|
@@ -187,6 +117,19 @@ class String
|
|
187
117
|
return fname
|
188
118
|
end
|
189
119
|
|
120
|
+
# parse passed GET param string into a form and return the FormArray object
|
121
|
+
def to_form
|
122
|
+
if self.split("\n").size > 1
|
123
|
+
return self.to_form_from_show
|
124
|
+
end
|
125
|
+
ret = FormArray.new
|
126
|
+
self.split("&").each do |x|
|
127
|
+
y = x.split("=",2)
|
128
|
+
ret.extend!(y[0].to_s,y[1].to_s)
|
129
|
+
end
|
130
|
+
return ret
|
131
|
+
end
|
132
|
+
|
190
133
|
def to_form_from_show
|
191
134
|
self.split("\n").map { |a|
|
192
135
|
key,val = a.split("=",2)
|
@@ -200,18 +143,11 @@ class String
|
|
200
143
|
return self.gsub("\n","").to_form
|
201
144
|
end
|
202
145
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
return self.to_form_from_show
|
207
|
-
end
|
208
|
-
ret = FormArray.new
|
209
|
-
self.split("&").each do |x|
|
210
|
-
y = x.split("=",2)
|
211
|
-
ret.extend!(y[0].to_s,y[1].to_s)
|
212
|
-
end
|
213
|
-
return ret
|
146
|
+
def to_form_from_req
|
147
|
+
# self.split("\x0d\x0a\x0d\x0a")[1].to_form
|
148
|
+
self.split("\n\n")[1].to_form
|
214
149
|
end
|
150
|
+
alias_method :to_ffr, :to_form_from_req
|
215
151
|
|
216
152
|
# create filename from url changing "/" to "_"
|
217
153
|
def to_fn(ext=nil)
|
@@ -220,22 +156,6 @@ class String
|
|
220
156
|
return ret
|
221
157
|
end
|
222
158
|
|
223
|
-
# html entity encode string
|
224
|
-
# sym = :basic :named :decimal :hexadecimal
|
225
|
-
def eencode(sym=nil)
|
226
|
-
sym = :named if sym.nil?
|
227
|
-
@@he.encode(self,sym)
|
228
|
-
end
|
229
|
-
|
230
|
-
# decode html entities in string
|
231
|
-
def edecode
|
232
|
-
return @@he.decode(self)
|
233
|
-
end
|
234
|
-
|
235
|
-
def edecode!
|
236
|
-
self.replace(@@he.decode(self))
|
237
|
-
end
|
238
|
-
|
239
159
|
# strip html tags from string
|
240
160
|
def strip_html
|
241
161
|
self.gsub(/<\/?[^>]*>/, "")
|
@@ -284,14 +204,6 @@ class String
|
|
284
204
|
Digest::SHA512.digest(self).hexify
|
285
205
|
end
|
286
206
|
|
287
|
-
def to_qp
|
288
|
-
[self].pack("M")
|
289
|
-
end
|
290
|
-
|
291
|
-
def from_qp
|
292
|
-
self.unpack("M").first
|
293
|
-
end
|
294
|
-
|
295
207
|
def pbcopy
|
296
208
|
IO.popen('pbcopy', 'r+') { |c| c.print self }
|
297
209
|
end
|
@@ -313,14 +225,6 @@ class Array
|
|
313
225
|
end
|
314
226
|
end
|
315
227
|
|
316
|
-
class Hash#:nodoc:
|
317
|
-
# no idea what I was doing here
|
318
|
-
def to_f#:nodoc:
|
319
|
-
self.each_key { |l| puts "#{l} = " + self[l] }
|
320
|
-
return nil
|
321
|
-
end
|
322
|
-
end
|
323
|
-
|
324
228
|
class File
|
325
229
|
# write string to file
|
326
230
|
def self.write(filename,contents)
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'wwmd/class_extensions/mixins_string_encoding'
|
2
|
+
class String
|
3
|
+
include WWMD::Encoding
|
4
|
+
|
5
|
+
@@he = HTMLEntities.new
|
6
|
+
|
7
|
+
# base 64 decode
|
8
|
+
def b64d
|
9
|
+
self.unpack("m").first
|
10
|
+
end
|
11
|
+
|
12
|
+
# base 64 encode
|
13
|
+
def b64e
|
14
|
+
[self].pack("m").gsub("\n","")
|
15
|
+
end
|
16
|
+
|
17
|
+
# URI.escape using defaults or passed regexp
|
18
|
+
def escape(reg=nil,unicodify=false)
|
19
|
+
if reg.nil?
|
20
|
+
ret = URI.escape(self)
|
21
|
+
elsif reg.kind_of?(Symbol)
|
22
|
+
case reg
|
23
|
+
when :none; return self
|
24
|
+
when :default; ret = URI.escape(self)
|
25
|
+
else; ret = URI.escape(self,WWMD::ESCAPE[reg])
|
26
|
+
end
|
27
|
+
else
|
28
|
+
ret = URI.escape(self,reg)
|
29
|
+
end
|
30
|
+
if unicodify
|
31
|
+
ret.gsub!(/%/,"%u00")
|
32
|
+
end
|
33
|
+
return ret
|
34
|
+
end
|
35
|
+
|
36
|
+
# URI.escape
|
37
|
+
def escape_url(reg=WWMD::ESCAPE[:url])#:nodoc:
|
38
|
+
self.escape(reg)
|
39
|
+
end
|
40
|
+
|
41
|
+
def escape_xss(reg=WWMD::ESCAPE[:xss])#:nodoc:
|
42
|
+
self.escape(reg)
|
43
|
+
end
|
44
|
+
|
45
|
+
def escape_default(reg=WWMD::ESCAPE[:default])
|
46
|
+
self.escape(reg)
|
47
|
+
end
|
48
|
+
# URI.escape all characters in string
|
49
|
+
def escape_all#:nodoc:
|
50
|
+
self.escape(/.*/)
|
51
|
+
end
|
52
|
+
|
53
|
+
# URI.unescape
|
54
|
+
def unescape
|
55
|
+
URI.unescape(self)
|
56
|
+
end
|
57
|
+
|
58
|
+
# html entity encode string
|
59
|
+
# sym = :basic :named :decimal :hexadecimal
|
60
|
+
def eencode(sym=nil)
|
61
|
+
sym = :named if sym.nil?
|
62
|
+
@@he.encode(self,sym)
|
63
|
+
end
|
64
|
+
|
65
|
+
# decode html entities in string
|
66
|
+
def edecode
|
67
|
+
return @@he.decode(self)
|
68
|
+
end
|
69
|
+
|
70
|
+
# quoted printable
|
71
|
+
def to_qp
|
72
|
+
[self].pack("M")
|
73
|
+
end
|
74
|
+
|
75
|
+
def from_qp
|
76
|
+
self.unpack("M").first
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
File without changes
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# I really hate this
|
2
|
+
class NilClass#:nodoc:
|
3
|
+
def empty?; return true; end
|
4
|
+
def size; return 0; end
|
5
|
+
def to_form; return FormArray.new([]); end
|
6
|
+
def clop; return nil; end
|
7
|
+
def inner_html; return nil; end
|
8
|
+
def get_attribute(*args); return nil; end
|
9
|
+
def grep(*args); return []; end
|
10
|
+
def escape(*args); return nil; end
|
11
|
+
end
|
File without changes
|
@@ -3,11 +3,11 @@ Place methods to character encodings here
|
|
3
3
|
=end
|
4
4
|
|
5
5
|
module WWMD
|
6
|
-
# This is where character encodings should go as
|
6
|
+
# This is where character encodings should go as module methods
|
7
7
|
# to be used as mixins for the String class
|
8
|
-
|
8
|
+
module Encoding
|
9
9
|
|
10
|
-
#
|
10
|
+
# String.to_utf7 mixin
|
11
11
|
# (complete hack but it works)
|
12
12
|
#
|
13
13
|
# if all=true, encode all characters.
|
@@ -17,7 +17,7 @@ module WWMD
|
|
17
17
|
# used by:
|
18
18
|
# String.to_utf7
|
19
19
|
# String.to_utf7!
|
20
|
-
def
|
20
|
+
def to_utf7(all=nil)
|
21
21
|
if all.kind_of?(Regexp)
|
22
22
|
reg = all
|
23
23
|
elsif all.kind_of?(TrueClass)
|
@@ -25,9 +25,9 @@ module WWMD
|
|
25
25
|
else
|
26
26
|
reg = ESCAPE[:nalnum] || /[^a-zA-Z0-9]/
|
27
27
|
end
|
28
|
-
|
28
|
+
putd "DEBG:" + reg.inspect
|
29
29
|
ret = ''
|
30
|
-
|
30
|
+
self.each_byte do |b|
|
31
31
|
if b.chr.match(reg)
|
32
32
|
ret += "+" + Base64.encode64(b.chr.toutf16)[0..2] + "-"
|
33
33
|
else
|
data/lib/wwmd/page.rb
CHANGED
@@ -1,245 +1,3 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
attr_accessor :post_data
|
5
|
-
attr_accessor :header_data
|
6
|
-
attr_accessor :use_referer
|
7
|
-
attr_reader :forms
|
8
|
-
attr_reader :last_error
|
9
|
-
attr_reader :links # array of links (urls)
|
10
|
-
attr_reader :jlinks # array of included javascript files
|
11
|
-
attr_reader :spider # spider object
|
12
|
-
attr_reader :scrape # scrape object
|
13
|
-
attr_reader :urlparse # urlparse object
|
14
|
-
attr_reader :comments
|
15
|
-
|
16
|
-
attr_accessor :base_url # needed to properly munge relative urls into fq urls
|
17
|
-
attr_accessor :logged_in # are we logged in?
|
18
|
-
|
19
|
-
attr_accessor :opts
|
20
|
-
attr_accessor :inputs
|
21
|
-
|
22
|
-
# WWMD::Page is an extension of a Curl::Easy object which provides methods to
|
23
|
-
# enhance and ease the performance of web application penetration testing.
|
24
|
-
class Page
|
25
|
-
|
26
|
-
def initialize(opts={})
|
27
|
-
@opts = opts.clone
|
28
|
-
DEFAULTS.each { |k,v| @opts[k] = v if not opts[k] }
|
29
|
-
@spider = Spider.new(opts)
|
30
|
-
@scrape = Scrape.new
|
31
|
-
@base_url ||= opts[:base_url]
|
32
|
-
@scrape.warn = opts[:scrape_warn] if !opts[:scrape_warn].nil?
|
33
|
-
if opts.empty?
|
34
|
-
putw "Page initialized without opts"
|
35
|
-
@scrape.warn = false
|
36
|
-
end
|
37
|
-
@urlparse = URLParse.new()
|
38
|
-
@inputs = Inputs.new(self)
|
39
|
-
@logged_in = false
|
40
|
-
@body_data = ""
|
41
|
-
@post_data = ""
|
42
|
-
@comments = []
|
43
|
-
@header_data = FormArray.new
|
44
|
-
|
45
|
-
@curl_object = Curl::Easy.new
|
46
|
-
@opts.each do |k,v|
|
47
|
-
next if !(@curl_object.methods.include?("#{k}="))
|
48
|
-
next if k == :proxy_url
|
49
|
-
@curl_object.send("#{k}=",v)
|
50
|
-
end
|
51
|
-
@curl_object.on_body { |data| self._body_cb(data) }
|
52
|
-
@curl_object.on_header { |data| self._header_cb(data) }
|
53
|
-
|
54
|
-
# cookies?
|
55
|
-
@curl_object.enable_cookies = @opts[:enable_cookies]
|
56
|
-
if @curl_object.enable_cookies?
|
57
|
-
@curl_object.cookiejar = @opts[:cookiejar] || "./__cookiejar"
|
58
|
-
end
|
59
|
-
|
60
|
-
#proxy?
|
61
|
-
@curl_object.proxy_url = @opts[:proxy_url] if @opts[:use_proxy]
|
62
|
-
end
|
63
|
-
|
64
|
-
#:section: Heavy Lifting
|
65
|
-
|
66
|
-
# set reporting data for the page
|
67
|
-
#
|
68
|
-
# Scan for comments, anchors, links and javascript includes and
|
69
|
-
# set page flags. The heavy lifting for parsing is done in the
|
70
|
-
# scrape class.
|
71
|
-
#
|
72
|
-
# returns: <tt>array [ code, page_status, body_data.size ]</tt>
|
73
|
-
def set_data
|
74
|
-
# reset scrape and inputs object
|
75
|
-
# transparently gunzip
|
76
|
-
begin
|
77
|
-
io = StringIO.new(self.body_data)
|
78
|
-
gz = Zlib::GzipReader.new(io)
|
79
|
-
self.body_data.replace(gz.read)
|
80
|
-
rescue => e
|
81
|
-
end
|
82
|
-
@scrape.reset(self.body_data)
|
83
|
-
@inputs.set
|
84
|
-
|
85
|
-
@comments = @scrape.for_comments
|
86
|
-
# remove comments that are css selectors for IE silliness
|
87
|
-
@comments.reject! do |c|
|
88
|
-
c =~ /\[if IE\]/ ||
|
89
|
-
c =~ /\[if IE \d/ ||
|
90
|
-
c =~ /\[if lt IE \d/
|
91
|
-
end
|
92
|
-
@links = @scrape.for_links.map do |url|
|
93
|
-
@urlparse.parse(self.last_effective_url,url).to_s
|
94
|
-
end
|
95
|
-
@jlinks = @scrape.for_javascript_links
|
96
|
-
@forms = []
|
97
|
-
self.search("//form").each { |f| @forms << Form.new(f) }
|
98
|
-
@spider.add(self.last_effective_url,@links)
|
99
|
-
return [self.code,self.page_status,self.body_data.size]
|
100
|
-
end
|
101
|
-
|
102
|
-
# clear self.body_data and self.header_data
|
103
|
-
def clear_data
|
104
|
-
return false if self.opts[:parse] = false
|
105
|
-
@body_data = ""
|
106
|
-
@header_data.clear
|
107
|
-
@last_error = nil
|
108
|
-
end
|
109
|
-
|
110
|
-
# override Curl::Easy.perform to perform page actions,
|
111
|
-
# call <tt>self.set_data</tt>
|
112
|
-
#
|
113
|
-
# returns: <tt>array [ code, page_status, body_data.size ]</tt>
|
114
|
-
#
|
115
|
-
# don't call this directly if we are in console mode
|
116
|
-
# use get and submit respectively for GET and POST
|
117
|
-
def perform
|
118
|
-
self.clear_data
|
119
|
-
self.headers["Referer"] = self.cur if self.use_referer
|
120
|
-
begin
|
121
|
-
@curl_object.perform
|
122
|
-
rescue => e
|
123
|
-
@last_error = e
|
124
|
-
putw "WARN: #{e.class}" if e.class =~ /Curl::Err/
|
125
|
-
# self.logged_in = false
|
126
|
-
end
|
127
|
-
self.set_data
|
128
|
-
return [self.code,self.page_status,self.body_data.size]
|
129
|
-
end
|
130
|
-
|
131
|
-
# replacement for Curl::Easy.http_post
|
132
|
-
#
|
133
|
-
# post the form attempting to remove curl supplied headers (Expect, X-Forwarded-For
|
134
|
-
# call <tt>self.set_data</tt>
|
135
|
-
#
|
136
|
-
# if passed a regexp, escape values in the form using regexp before submitting
|
137
|
-
# if passed nil for the regexp arg, the form will not be escaped
|
138
|
-
# default: WWMD::ESCAPE[:url]
|
139
|
-
#
|
140
|
-
# returns: <tt>array [ code, body_data.size ]</tt>
|
141
|
-
def submit(iform=nil,reg=WWMD::ESCAPE[:default])
|
142
|
-
##### this is just getting worse and worse
|
143
|
-
if iform.class == "Symbol"
|
144
|
-
reg = iform
|
145
|
-
iform = nil
|
146
|
-
end
|
147
|
-
self.clear_data
|
148
|
-
["Expect","X-Forwarded-For","Content-length"].each { |s| self.clear_header(s) }
|
149
|
-
self.headers["Referer"] = self.cur if self.use_referer
|
150
|
-
if iform == nil
|
151
|
-
if not self.form.empty?
|
152
|
-
sform = self.form.clone
|
153
|
-
else
|
154
|
-
return "no form provided"
|
155
|
-
end
|
156
|
-
else
|
157
|
-
sform = iform.clone # clone the form so that we don't change the original
|
158
|
-
end
|
159
|
-
sform.escape_all!(reg)
|
160
|
-
if sform.empty?
|
161
|
-
self.http_post('')
|
162
|
-
else
|
163
|
-
self.http_post(self.post_data = sform.to_post)
|
164
|
-
end
|
165
|
-
begin
|
166
|
-
self.set_data
|
167
|
-
rescue => e
|
168
|
-
STDERR.puts "FATAL: could not parse page"
|
169
|
-
end
|
170
|
-
return [self.code, self.body_data.size]
|
171
|
-
end
|
172
|
-
|
173
|
-
# submit a form using POST string
|
174
|
-
def submit_string(post_string)
|
175
|
-
self.clear_data
|
176
|
-
self.http_post(post_string)
|
177
|
-
self.set_data
|
178
|
-
if self.ntlm?
|
179
|
-
putw "WARN: this page requires NTLM Authentication"
|
180
|
-
putw "WARN: use ntlm_get instead of get"
|
181
|
-
end
|
182
|
-
return [self.code, self.body_data.size]
|
183
|
-
end
|
184
|
-
|
185
|
-
# override for Curl::Easy.perform
|
186
|
-
#
|
187
|
-
# if the passed url string doesn't contain an fully qualified
|
188
|
-
# path, we'll guess and prepend opts[:base_url]
|
189
|
-
#
|
190
|
-
# returns: <tt>array [ code, body_data.size ]</tt>
|
191
|
-
def get(url=nil,parse=true)
|
192
|
-
if !(url =~ /[a-z]+:\/\//) && parse
|
193
|
-
self.url = @urlparse.parse(self.opts[:base_url],url).to_s if url
|
194
|
-
elsif url
|
195
|
-
self.url = url
|
196
|
-
end
|
197
|
-
self.perform
|
198
|
-
if self.ntlm?
|
199
|
-
putw "WARN: this page requires NTLM Authentication"
|
200
|
-
putw "use ntlm_get instead of get"
|
201
|
-
end
|
202
|
-
self.set_data
|
203
|
-
return [self.code, self.body_data.size]
|
204
|
-
end
|
205
|
-
|
206
|
-
# GET with params and POST it as a form
|
207
|
-
def post(url=nil)
|
208
|
-
ep = url.clip
|
209
|
-
self.url = @urlparse.parse(self.opts[:base_url],ep).to_s if ep
|
210
|
-
form = url.clop.to_form
|
211
|
-
self.submit(form)
|
212
|
-
end
|
213
|
-
|
214
|
-
# send arbitrary verb (only works with patch to taf2-curb
|
215
|
-
def verb(verb)
|
216
|
-
return false if !@curl_object.respond_to?(:http_verb)
|
217
|
-
self.clear_data
|
218
|
-
self.headers["Referer"] = self.cur if self.use_referer
|
219
|
-
self.http_verb(verb)
|
220
|
-
self.set_data
|
221
|
-
return [self.code, self.body_data.size]
|
222
|
-
end
|
223
|
-
|
224
|
-
#:section: Data callbacks and method_missing
|
225
|
-
|
226
|
-
# callback for <tt>self.on_body</tt>
|
227
|
-
def _body_cb(data)
|
228
|
-
@body_data << data if data
|
229
|
-
return data.length.to_i
|
230
|
-
end
|
231
|
-
|
232
|
-
# callback for <tt>self.on_header</tt>
|
233
|
-
def _header_cb(data)
|
234
|
-
myArr = Array.new(data.split(":",2))
|
235
|
-
@header_data.extend! myArr[0].to_s.strip,myArr[1].to_s.strip
|
236
|
-
return data.length.to_i
|
237
|
-
end
|
238
|
-
|
239
|
-
# send methods not defined here to <tt>@curl_object</tt>
|
240
|
-
def method_missing(methodname, *args)
|
241
|
-
@curl_object.send(methodname, *args)
|
242
|
-
end
|
243
|
-
|
244
|
-
end
|
245
|
-
end
|
1
|
+
require 'wwmd/wwmd_utils'
|
2
|
+
require 'wwmd/wwmd_config'
|
3
|
+
require 'wwmd/page/page'
|