miketracy-wwmd 0.2.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (70) hide show
  1. data/History.txt +3 -0
  2. data/README +62 -0
  3. data/README.txt +62 -0
  4. data/Rakefile +34 -0
  5. data/examples/config_example.yaml +24 -0
  6. data/examples/wwmd_example.rb +73 -0
  7. data/lib/wwmd.rb +78 -0
  8. data/lib/wwmd/encoding.rb +40 -0
  9. data/lib/wwmd/form.rb +110 -0
  10. data/lib/wwmd/form_array.rb +273 -0
  11. data/lib/wwmd/guid.rb +155 -0
  12. data/lib/wwmd/hpricot_html2text.rb +76 -0
  13. data/lib/wwmd/mixins.rb +318 -0
  14. data/lib/wwmd/mixins_extends.rb +188 -0
  15. data/lib/wwmd/mixins_external.rb +18 -0
  16. data/lib/wwmd/nokogiri_html2text.rb +41 -0
  17. data/lib/wwmd/page.rb +414 -0
  18. data/lib/wwmd/page/auth.rb +183 -0
  19. data/lib/wwmd/page/config.rb +44 -0
  20. data/lib/wwmd/page/constants.rb +60 -0
  21. data/lib/wwmd/page/headers.rb +107 -0
  22. data/lib/wwmd/page/inputs.rb +47 -0
  23. data/lib/wwmd/page/irb_helpers.rb +90 -0
  24. data/lib/wwmd/page/scrape.rb +202 -0
  25. data/lib/wwmd/page/spider.rb +127 -0
  26. data/lib/wwmd/page/urlparse.rb +79 -0
  27. data/lib/wwmd/page/utils.rb +30 -0
  28. data/lib/wwmd/viewstate.rb +118 -0
  29. data/lib/wwmd/viewstate/viewstate_class_helpers.rb +35 -0
  30. data/lib/wwmd/viewstate/viewstate_deserializer_methods.rb +213 -0
  31. data/lib/wwmd/viewstate/viewstate_from_xml.rb +126 -0
  32. data/lib/wwmd/viewstate/viewstate_types.rb +51 -0
  33. data/lib/wwmd/viewstate/viewstate_utils.rb +157 -0
  34. data/lib/wwmd/viewstate/viewstate_yaml.rb +25 -0
  35. data/lib/wwmd/viewstate/vs_array.rb +36 -0
  36. data/lib/wwmd/viewstate/vs_binary_serialized.rb +28 -0
  37. data/lib/wwmd/viewstate/vs_hashtable.rb +40 -0
  38. data/lib/wwmd/viewstate/vs_hybrid_dict.rb +40 -0
  39. data/lib/wwmd/viewstate/vs_indexed_string.rb +6 -0
  40. data/lib/wwmd/viewstate/vs_indexed_string_ref.rb +22 -0
  41. data/lib/wwmd/viewstate/vs_int_enum.rb +25 -0
  42. data/lib/wwmd/viewstate/vs_list.rb +32 -0
  43. data/lib/wwmd/viewstate/vs_pair.rb +27 -0
  44. data/lib/wwmd/viewstate/vs_read_types.rb +11 -0
  45. data/lib/wwmd/viewstate/vs_read_value.rb +33 -0
  46. data/lib/wwmd/viewstate/vs_sparse_array.rb +56 -0
  47. data/lib/wwmd/viewstate/vs_string.rb +29 -0
  48. data/lib/wwmd/viewstate/vs_string_array.rb +37 -0
  49. data/lib/wwmd/viewstate/vs_string_formatted.rb +30 -0
  50. data/lib/wwmd/viewstate/vs_triplet.rb +29 -0
  51. data/lib/wwmd/viewstate/vs_type.rb +21 -0
  52. data/lib/wwmd/viewstate/vs_unit.rb +28 -0
  53. data/lib/wwmd/viewstate/vs_value.rb +33 -0
  54. data/spec/README +3 -0
  55. data/spec/form_array.spec +49 -0
  56. data/spec/spider_csrf_test.spec +28 -0
  57. data/spec/urlparse_test.spec +89 -0
  58. data/tasks/ann.rake +80 -0
  59. data/tasks/bones.rake +20 -0
  60. data/tasks/gem.rake +201 -0
  61. data/tasks/git.rake +40 -0
  62. data/tasks/notes.rake +27 -0
  63. data/tasks/post_load.rake +34 -0
  64. data/tasks/rdoc.rake +51 -0
  65. data/tasks/rubyforge.rake +55 -0
  66. data/tasks/setup.rb +292 -0
  67. data/tasks/spec.rake +54 -0
  68. data/tasks/test.rake +40 -0
  69. data/tasks/zentest.rake +36 -0
  70. metadata +164 -0
@@ -0,0 +1,318 @@
1
+ =begin rdoc
2
+ mixins all around
3
+ =end
4
+
5
+ alias putd puts#:nodoc:
6
+ alias putx puts#:nodoc:
7
+ alias putw puts#:nodoc:
8
+
9
+ # I really hate this
10
+ class NilClass#:nodoc:
11
+ def empty?; return true; end
12
+ def size; return 0; end
13
+ def to_form; return FormArray.new([]); end
14
+ def clop; return nil; end
15
+ def inner_html; return nil; end
16
+ def get_attribute(*args); return nil; end
17
+ def grep(*args); return []; end
18
+ def escape(*args); return nil; end
19
+ end
20
+
21
+ class Numeric
22
+ # return binary representation of <tt>length</tt> size padded with \x00
23
+ # length: length in bytes to return (padded with least signficant \x00
24
+ # reverse: reverse the byte order
25
+ def to_bin (len,rev = false)
26
+ str = ""
27
+ bignum = self
28
+ 1.upto(len) do |i|
29
+ str << (bignum & 0xFF).to_n8
30
+ bignum = bignum >> 8
31
+ end
32
+ return str.reverse if rev
33
+ return str
34
+ end
35
+
36
+ # integer to ip address
37
+ def int_to_ip
38
+ [24, 16, 8, 0].map { |b| (self >> b) & 255 }.join('.')
39
+ end
40
+
41
+ # integer to mac address [uses ':' as delimiter]
42
+ def int_to_mac
43
+ [40,32,24,16,8,0].map { |b| ((self >> b) & 255).to_s(16).rjust(2,"0") }.join(":")
44
+ end
45
+ end
46
+
47
+ class String
48
+
49
+ @@he = HTMLEntities.new
50
+
51
+ # ip address to int
52
+ def ip_to_int
53
+ self.split('.').inject(0) { |a,e| (a << 8) + e.to_i }
54
+ end
55
+
56
+ # mac address to int [uses ':' as delimiter]
57
+ def mac_to_int
58
+ self.split(':').inject(0) { |a,e| (a << 8) + e.to_i(16) }
59
+ end
60
+
61
+ # return true or false for <tt>string.match</tt>
62
+ def contains?(rexp)
63
+ return !self.match(rexp).nil?
64
+ end
65
+
66
+ # strip the string and return true if empty
67
+ def empty?
68
+ return self.strip == ''
69
+ end
70
+
71
+ # return everything in the string (url) before the first get param
72
+ ## "http://foo.bar.com/page.asp?somearg=foo&otherarg=bar".clip
73
+ ## => "http://foo.bar.com/page.asp"
74
+ def clip(pref="?")
75
+ if (v = self.index(pref))
76
+ return self[0..(v-1)]
77
+ end
78
+ return self
79
+ end
80
+
81
+ # return everything in the string (url) after the first get parameter
82
+ # without the leading '?'
83
+ #
84
+ # pass true as the second param to also get back the ?
85
+ ## "http://foo.bar.com/page.asp?somearg=foo&otherarg=bar".clop
86
+ ## => "somearg=foo&otherarg=bar"
87
+ def clop(pref="?",preftoo=false)
88
+ (preftoo == false) ? add = "" : add = pref
89
+ if (v = self.index(pref))
90
+ return add + self[(v+1)..-1]
91
+ end
92
+ return nil
93
+ end
94
+
95
+ def clopp; self.clop("?",true); end #:nodoc:
96
+
97
+ def clopa
98
+ return [self.clip,self.clop]
99
+ end
100
+
101
+ # base 64 decode
102
+ def b64d
103
+ self.unpack("m").first
104
+ end
105
+
106
+ # base 64 encode
107
+ def b64e
108
+ [self].pack("m").chomp
109
+ end
110
+
111
+ # URI.escape using defaults or passed regexp
112
+ def escape(reg=WWMD::ESCAPE[:default],unicodify=false)
113
+ if reg == WWMD::ESCAPE[:none]
114
+ return self
115
+ elsif reg == WWMD::ESCAPE[:default]
116
+ ret = URI.escape(self)
117
+ elsif reg.kind_of?(Symbol)
118
+ ret = URI.escape(self,WWMD::ESCAPE[reg])
119
+ reg = WWMD::ESCAPE[reg]
120
+ else
121
+ ret = URI.escape(self,reg)
122
+ end
123
+ if unicodify
124
+ ret.gsub!(/%/,"%u00")
125
+ end
126
+ return ret
127
+ end
128
+
129
+ # URI.escape
130
+ def escape_url(reg=WWMD::ESCAPE[:url])#:nodoc:
131
+ self.escape(reg)
132
+ end
133
+
134
+ def escape_xss(reg=WWMD::ESCAPE[:xss])#:nodoc:
135
+ self.escape(reg)
136
+ end
137
+
138
+ def escape_default(reg=WWMD::ESCAPE[:default])
139
+ self.escape(reg)
140
+ end
141
+ # URI.escape all characters in string
142
+ def escape_all#:nodoc:
143
+ self.escape(/.*/)
144
+ end
145
+
146
+ # URI.unescape
147
+ def unescape
148
+ URI.unescape(self)
149
+ end
150
+
151
+ # encode the string using Encoding.to_utf7(self,false)
152
+ # (encode non [:alnum:] characters). Set <tt>all</tt> true
153
+ # to encode all characters in the string.
154
+ def to_utf7(all=false)
155
+ Encoding.to_utf7(self,all)
156
+ end
157
+
158
+ # File.dirname with a trailing slash
159
+ def dirname
160
+ return self if self.match(/\/$/)
161
+ File.dirname(self) + "/"
162
+ end
163
+
164
+ # File.basename
165
+ def basename
166
+ File.basename(self)
167
+ end
168
+
169
+ def extname
170
+ self.split('.').last
171
+ end
172
+
173
+ # write string to passed filename
174
+ # if filename is nil? will raise an error
175
+ def write(fname=nil)
176
+ raise "filename required" if fname.nil?
177
+ File.write(fname,self)
178
+ return fname
179
+ end
180
+
181
+ def to_form_from_show
182
+ self.split("\n").map { |a|
183
+ key,val = a.split("=",2)
184
+ key = key.split(" ")[-1]
185
+ val = val.strip if val
186
+ ["#{key}=#{val}"]
187
+ }.join("&").to_form.squeeze_keys!
188
+ end
189
+
190
+ def mform
191
+ return self.gsub("\n","").to_form
192
+ end
193
+
194
+ # parse passed GET param string into a form and return the FormArray object
195
+ def to_form
196
+ if self.split("\n").size > 1
197
+ return self.to_form_from_show
198
+ end
199
+ ret = FormArray.new
200
+ self.split("&").each do |x|
201
+ y = x.split("=",2)
202
+ ret.extend!(y[0].to_s,y[1].to_s)
203
+ end
204
+ return ret
205
+ end
206
+
207
+ # create filename from url changing "/" to "_"
208
+ def to_fn(ext=nil)
209
+ ret = self.clip.split("/")[3..-1].join("_")
210
+ ret += ".#{ext}" if not ext.nil?
211
+ return ret
212
+ end
213
+
214
+ # html entity encode string
215
+ # sym = :basic :named :decimal :hexadecimal
216
+ def eencode(sym=nil)
217
+ sym = :named if sym.nil?
218
+ @@he.encode(self,sym)
219
+ end
220
+
221
+ # decode html entities in string
222
+ def edecode
223
+ return @@he.decode(self)
224
+ end
225
+
226
+ def edecode!
227
+ self.replace(@@he.decode(self))
228
+ end
229
+
230
+ # strip html tags from string
231
+ def strip_html
232
+ self.gsub(/<\/?[^>]*>/, "")
233
+ end
234
+
235
+ # range or int
236
+ def head(c=5)
237
+ if c.kind_of?(Range) then
238
+ range = c
239
+ else
240
+ range = (0..(c - 1))
241
+ end
242
+ self.split("\n")[range].join("\n")
243
+ end
244
+
245
+ # return a literal regexp object for this string
246
+ #
247
+ # escape regexp operators
248
+ def to_regexp
249
+ return Regexp.new(self.gsub(/([\[\]\{\}\(\)\*\$\?])/) { |x| '\\' + x })
250
+ end
251
+
252
+ # check if this string is a guid
253
+ def is_guid?
254
+ begin
255
+ Guid.from_s(self)
256
+ rescue => e
257
+ return false
258
+ end
259
+ return true
260
+ end
261
+
262
+ def md5
263
+ Digest::MD5.digest(self).hexify
264
+ end
265
+
266
+ def sha1
267
+ Digest::SHA1.digest(self).hexify
268
+ end
269
+
270
+ def sha256
271
+ Digest::SHA256.digest(self).hexify
272
+ end
273
+
274
+ def sha512
275
+ Digest::SHA512.digest(self).hexify
276
+ end
277
+
278
+ def to_qp
279
+ [self].pack("M")
280
+ end
281
+
282
+ def from_qp
283
+ self.unpack("M").first
284
+ end
285
+ end
286
+
287
+ class Array
288
+ # grep each element of an array for the passed regular expression
289
+ # and return an Array of matches
290
+ # (only works one deep)
291
+ def each_grep(regex)
292
+ ret = []
293
+ self.each { |e| ret << e.grep(regex) }
294
+ return ret
295
+ end
296
+
297
+ # join the array with "\n" and write to a file
298
+ def to_file(filename)
299
+ File.write(filename,self.join("\n"))
300
+ end
301
+ end
302
+
303
+ class Hash#:nodoc:
304
+ # no idea what I was doing here
305
+ def to_f#:nodoc:
306
+ self.each_key { |l| puts "#{l} = " + self[l] }
307
+ return nil
308
+ end
309
+ end
310
+
311
+ class File
312
+ # write string to file
313
+ def self.write(filename,contents)
314
+ fout = File.open(filename,"w")
315
+ fout.print contents
316
+ fout.close
317
+ end
318
+ end
@@ -0,0 +1,188 @@
1
+ # Author Eric Monti (emonti@matasano.com)
2
+ require "base64"
3
+ require "stringio"
4
+ require 'zlib'
5
+ require 'open3'
6
+ require 'sha1'
7
+
8
+ HEXCHARS = [("0".."9").to_a, ("a".."f").to_a].flatten unless defined?(HEXCHARS)
9
+
10
+ #-----------------------------------------------------------------------------
11
+ # Mixins and class-specific items
12
+
13
+ class Fixnum#:nodoc:
14
+ def to_l32; [self].pack "L"; end
15
+ def to_b32; [self].pack "N"; end
16
+ def to_l16; [self].pack "v"; end
17
+ def to_b16; [self].pack "n"; end
18
+ def to_u8; [self].pack "C"; end
19
+ def to_l8; chr; end
20
+ def to_n8; chr; end
21
+ alias to_n32 to_b32
22
+ alias to_n16 to_b16
23
+ def to_n32; [self].pack "N"; end
24
+ def to_n16; [self].pack "n"; end
25
+
26
+ def self.from_l8(str); str[0]; end
27
+ def self.from_l16(str); (str.unpack "v")[0]; end
28
+ def self.from_l32(str); (str.unpack "V")[0]; end
29
+ def self.from_n8(str); str[0]; end
30
+ def self.from_n16(str); (str.unpack "n")[0]; end
31
+ def self.from_n32(str); (str.unpack "N")[0]; end
32
+ end
33
+
34
+ class String
35
+
36
+ # shortcut for hex sanity with regex
37
+ def ishex? ; (self =~ /^[a-f0-9]+$/i)? true : false ; end
38
+
39
+ # Convert a string to ASCII hex string
40
+ # supports a few options for format:
41
+ # :delim - delimter between each hex byte
42
+ # :prefix - prefix before each hex byte
43
+ # :suffix - suffix after each hex byte
44
+ #
45
+ def hexify(opts={})
46
+ s=self
47
+ delim = opts[:delim]
48
+ pre = (opts[:prefix] || "")
49
+ suf = (opts[:suffix] || "")
50
+
51
+ if (rx=opts[:rx]) and not rx.kind_of? Regexp
52
+ raise "rx must be a regular expression for a character class"
53
+ end
54
+
55
+ out=Array.new
56
+
57
+ s.each_byte do |c|
58
+ hc = if (rx and not rx.match c.chr)
59
+ c.chr
60
+ else
61
+ pre + (HEXCHARS[(c >> 4)] + HEXCHARS[(c & 0xf )]) + suf
62
+ end
63
+ out << (hc)
64
+ end
65
+ out.join(delim)
66
+ end
67
+
68
+
69
+ # Convert ASCII hex string to raw
70
+ # supports only 'delimiter' between hex bytes
71
+ def unhexify(d=/\s*/)
72
+ s=self.strip
73
+ out=StringIO.new
74
+ while m = s.match(/^([A-Fa-f0-9]{1,2})#{d}?/) do
75
+ out.write m[1].hex.chr
76
+ s = m.post_match
77
+ end
78
+ out.string
79
+ end
80
+ alias_method :dehexify, :unhexify
81
+
82
+ # ==========================================================================
83
+ # Extends String class to return a hexdump in the style of 'hexdump -C'
84
+ #
85
+ # :len => optionally specify a length other than 16 for a wider or thinner
86
+ # dump. If length is an odd number, it will be rounded up.
87
+ #
88
+ # :out => optionally specify an alternate IO object for output. By default,
89
+ # hexdump will output to STDOUT. Pass a StringIO object and it will return
90
+ # it as a string.
91
+ #
92
+ # Example:
93
+ # xxd = dat.hexdump(:len => 16, :out => StringIO.new)
94
+ # xxd => a hexdump
95
+ #
96
+ # xxd = dat.hexdump(:len => 16, :out => STDERR)
97
+ # xxd => nil
98
+ # ==========================================================================
99
+ def hexdump(opt={})
100
+ s=self
101
+ out = opt[:out] || StringIO.new
102
+ len = (opt[:len] and opt[:len] > 0)? opt[:len] + (opt[:len] % 2) : 16
103
+
104
+ off = opt[:start_addr] || 0
105
+ offlen = opt[:start_len] || 8
106
+
107
+ hlen=len/2
108
+
109
+ s.scan(/(?:.|\n){1,#{len}}/) do |m|
110
+ out.write(off.to_s(16).rjust(offlen, "0") + ' ')
111
+
112
+ i=0
113
+ m.each_byte do |c|
114
+ out.write c.to_s(16).rjust(2,"0") + " "
115
+ out.write(' ') if (i+=1) == hlen
116
+ end
117
+
118
+ out.write(" " * (len-i) ) # pad
119
+ out.write(" ") if i < hlen
120
+
121
+ out.write(" |" + m.tr("\0-\37\177-\377", '.') + "|\n")
122
+ off += m.length
123
+ end
124
+
125
+ out.write(off.to_s(16).rjust(offlen,'0') + "\n")
126
+
127
+ if out.class == StringIO
128
+ out.string
129
+ end
130
+ end
131
+
132
+
133
+ # ==========================================================================
134
+ # converts a hexdump back to binary - takes the same options as hexdump()
135
+ # fairly flexible. should work both with 'xxd' and 'hexdump -C' style dumps
136
+ def dehexdump(opt={})
137
+ s=self
138
+ out = opt[:out] || StringIO.new
139
+ len = (opt[:len] and opt[:len] > 0)? opt[:len] : 16
140
+
141
+ hcrx = /[A-Fa-f0-9]/
142
+ dumprx = /^(#{hcrx}+):?\s*((?:#{hcrx}{2}\s*){0,#{len}})/
143
+ off = opt[:start_addr] || 0
144
+
145
+ i=1
146
+ # iterate each line of hexdump
147
+ s.split(/\r?\n/).each do |hl|
148
+ # match and check offset
149
+ if m = dumprx.match(hl) and $1.hex == off
150
+ i+=1
151
+ # take the data chunk and unhexify it
152
+ raw = $2.unhexify
153
+ off += out.write(raw)
154
+ else
155
+ raise "Hexdump parse error on line #{i} #{s}"
156
+ end
157
+ end
158
+
159
+ if out.class == StringIO
160
+ out.string
161
+ end
162
+ end
163
+ alias_method :dedump, :dehexdump
164
+
165
+ # Does string "start with" dat?
166
+ # no clue whether/when this is faster than a regex, but it is easier
167
+ # than escaping regex characters
168
+ def starts_with?(dat)
169
+ self[0,dat.size] == dat
170
+ end
171
+
172
+ # returns CRC32 checksum for the string object
173
+ def crc32
174
+ Zlib.crc32 self
175
+ end
176
+
177
+ def swap16; unpack("v*").pack("n*"); end
178
+ def to_utf16; Kconv.kconv(self, NKF::UTF16, NKF::ASCII).swap16; end
179
+ def to_ascii; Kconv.kconv(swap16, NKF::ASCII, NKF::UTF16); end
180
+
181
+ end # class String
182
+
183
+
184
+ class Float
185
+ def log2; Math.log(self)/Math.log(2); end
186
+ end
187
+
188
+ __END__