miketracy-wwmd 0.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. data/History.txt +3 -0
  2. data/README +62 -0
  3. data/README.txt +62 -0
  4. data/Rakefile +34 -0
  5. data/examples/config_example.yaml +24 -0
  6. data/examples/wwmd_example.rb +73 -0
  7. data/lib/wwmd.rb +78 -0
  8. data/lib/wwmd/encoding.rb +40 -0
  9. data/lib/wwmd/form.rb +110 -0
  10. data/lib/wwmd/form_array.rb +273 -0
  11. data/lib/wwmd/guid.rb +155 -0
  12. data/lib/wwmd/hpricot_html2text.rb +76 -0
  13. data/lib/wwmd/mixins.rb +318 -0
  14. data/lib/wwmd/mixins_extends.rb +188 -0
  15. data/lib/wwmd/mixins_external.rb +18 -0
  16. data/lib/wwmd/nokogiri_html2text.rb +41 -0
  17. data/lib/wwmd/page.rb +414 -0
  18. data/lib/wwmd/page/auth.rb +183 -0
  19. data/lib/wwmd/page/config.rb +44 -0
  20. data/lib/wwmd/page/constants.rb +60 -0
  21. data/lib/wwmd/page/headers.rb +107 -0
  22. data/lib/wwmd/page/inputs.rb +47 -0
  23. data/lib/wwmd/page/irb_helpers.rb +90 -0
  24. data/lib/wwmd/page/scrape.rb +202 -0
  25. data/lib/wwmd/page/spider.rb +127 -0
  26. data/lib/wwmd/page/urlparse.rb +79 -0
  27. data/lib/wwmd/page/utils.rb +30 -0
  28. data/lib/wwmd/viewstate.rb +118 -0
  29. data/lib/wwmd/viewstate/viewstate_class_helpers.rb +35 -0
  30. data/lib/wwmd/viewstate/viewstate_deserializer_methods.rb +213 -0
  31. data/lib/wwmd/viewstate/viewstate_from_xml.rb +126 -0
  32. data/lib/wwmd/viewstate/viewstate_types.rb +51 -0
  33. data/lib/wwmd/viewstate/viewstate_utils.rb +157 -0
  34. data/lib/wwmd/viewstate/viewstate_yaml.rb +25 -0
  35. data/lib/wwmd/viewstate/vs_array.rb +36 -0
  36. data/lib/wwmd/viewstate/vs_binary_serialized.rb +28 -0
  37. data/lib/wwmd/viewstate/vs_hashtable.rb +40 -0
  38. data/lib/wwmd/viewstate/vs_hybrid_dict.rb +40 -0
  39. data/lib/wwmd/viewstate/vs_indexed_string.rb +6 -0
  40. data/lib/wwmd/viewstate/vs_indexed_string_ref.rb +22 -0
  41. data/lib/wwmd/viewstate/vs_int_enum.rb +25 -0
  42. data/lib/wwmd/viewstate/vs_list.rb +32 -0
  43. data/lib/wwmd/viewstate/vs_pair.rb +27 -0
  44. data/lib/wwmd/viewstate/vs_read_types.rb +11 -0
  45. data/lib/wwmd/viewstate/vs_read_value.rb +33 -0
  46. data/lib/wwmd/viewstate/vs_sparse_array.rb +56 -0
  47. data/lib/wwmd/viewstate/vs_string.rb +29 -0
  48. data/lib/wwmd/viewstate/vs_string_array.rb +37 -0
  49. data/lib/wwmd/viewstate/vs_string_formatted.rb +30 -0
  50. data/lib/wwmd/viewstate/vs_triplet.rb +29 -0
  51. data/lib/wwmd/viewstate/vs_type.rb +21 -0
  52. data/lib/wwmd/viewstate/vs_unit.rb +28 -0
  53. data/lib/wwmd/viewstate/vs_value.rb +33 -0
  54. data/spec/README +3 -0
  55. data/spec/form_array.spec +49 -0
  56. data/spec/spider_csrf_test.spec +28 -0
  57. data/spec/urlparse_test.spec +89 -0
  58. data/tasks/ann.rake +80 -0
  59. data/tasks/bones.rake +20 -0
  60. data/tasks/gem.rake +201 -0
  61. data/tasks/git.rake +40 -0
  62. data/tasks/notes.rake +27 -0
  63. data/tasks/post_load.rake +34 -0
  64. data/tasks/rdoc.rake +51 -0
  65. data/tasks/rubyforge.rake +55 -0
  66. data/tasks/setup.rb +292 -0
  67. data/tasks/spec.rake +54 -0
  68. data/tasks/test.rake +40 -0
  69. data/tasks/zentest.rake +36 -0
  70. metadata +164 -0
@@ -0,0 +1,318 @@
1
+ =begin rdoc
2
+ mixins all around
3
+ =end
4
+
5
+ alias putd puts#:nodoc:
6
+ alias putx puts#:nodoc:
7
+ alias putw puts#:nodoc:
8
+
9
+ # I really hate this
10
+ class NilClass#:nodoc:
11
+ def empty?; return true; end
12
+ def size; return 0; end
13
+ def to_form; return FormArray.new([]); end
14
+ def clop; return nil; end
15
+ def inner_html; return nil; end
16
+ def get_attribute(*args); return nil; end
17
+ def grep(*args); return []; end
18
+ def escape(*args); return nil; end
19
+ end
20
+
21
+ class Numeric
22
+ # return binary representation of <tt>length</tt> size padded with \x00
23
+ # length: length in bytes to return (padded with least signficant \x00
24
+ # reverse: reverse the byte order
25
+ def to_bin (len,rev = false)
26
+ str = ""
27
+ bignum = self
28
+ 1.upto(len) do |i|
29
+ str << (bignum & 0xFF).to_n8
30
+ bignum = bignum >> 8
31
+ end
32
+ return str.reverse if rev
33
+ return str
34
+ end
35
+
36
+ # integer to ip address
37
+ def int_to_ip
38
+ [24, 16, 8, 0].map { |b| (self >> b) & 255 }.join('.')
39
+ end
40
+
41
+ # integer to mac address [uses ':' as delimiter]
42
+ def int_to_mac
43
+ [40,32,24,16,8,0].map { |b| ((self >> b) & 255).to_s(16).rjust(2,"0") }.join(":")
44
+ end
45
+ end
46
+
47
+ class String
48
+
49
+ @@he = HTMLEntities.new
50
+
51
+ # ip address to int
52
+ def ip_to_int
53
+ self.split('.').inject(0) { |a,e| (a << 8) + e.to_i }
54
+ end
55
+
56
+ # mac address to int [uses ':' as delimiter]
57
+ def mac_to_int
58
+ self.split(':').inject(0) { |a,e| (a << 8) + e.to_i(16) }
59
+ end
60
+
61
+ # return true or false for <tt>string.match</tt>
62
+ def contains?(rexp)
63
+ return !self.match(rexp).nil?
64
+ end
65
+
66
+ # strip the string and return true if empty
67
+ def empty?
68
+ return self.strip == ''
69
+ end
70
+
71
+ # return everything in the string (url) before the first get param
72
+ ## "http://foo.bar.com/page.asp?somearg=foo&otherarg=bar".clip
73
+ ## => "http://foo.bar.com/page.asp"
74
+ def clip(pref="?")
75
+ if (v = self.index(pref))
76
+ return self[0..(v-1)]
77
+ end
78
+ return self
79
+ end
80
+
81
+ # return everything in the string (url) after the first get parameter
82
+ # without the leading '?'
83
+ #
84
+ # pass true as the second param to also get back the ?
85
+ ## "http://foo.bar.com/page.asp?somearg=foo&otherarg=bar".clop
86
+ ## => "somearg=foo&otherarg=bar"
87
+ def clop(pref="?",preftoo=false)
88
+ (preftoo == false) ? add = "" : add = pref
89
+ if (v = self.index(pref))
90
+ return add + self[(v+1)..-1]
91
+ end
92
+ return nil
93
+ end
94
+
95
+ def clopp; self.clop("?",true); end #:nodoc:
96
+
97
+ def clopa
98
+ return [self.clip,self.clop]
99
+ end
100
+
101
+ # base 64 decode
102
+ def b64d
103
+ self.unpack("m").first
104
+ end
105
+
106
+ # base 64 encode
107
+ def b64e
108
+ [self].pack("m").chomp
109
+ end
110
+
111
+ # URI.escape using defaults or passed regexp
112
+ def escape(reg=WWMD::ESCAPE[:default],unicodify=false)
113
+ if reg == WWMD::ESCAPE[:none]
114
+ return self
115
+ elsif reg == WWMD::ESCAPE[:default]
116
+ ret = URI.escape(self)
117
+ elsif reg.kind_of?(Symbol)
118
+ ret = URI.escape(self,WWMD::ESCAPE[reg])
119
+ reg = WWMD::ESCAPE[reg]
120
+ else
121
+ ret = URI.escape(self,reg)
122
+ end
123
+ if unicodify
124
+ ret.gsub!(/%/,"%u00")
125
+ end
126
+ return ret
127
+ end
128
+
129
+ # URI.escape
130
+ def escape_url(reg=WWMD::ESCAPE[:url])#:nodoc:
131
+ self.escape(reg)
132
+ end
133
+
134
+ def escape_xss(reg=WWMD::ESCAPE[:xss])#:nodoc:
135
+ self.escape(reg)
136
+ end
137
+
138
+ def escape_default(reg=WWMD::ESCAPE[:default])
139
+ self.escape(reg)
140
+ end
141
+ # URI.escape all characters in string
142
+ def escape_all#:nodoc:
143
+ self.escape(/.*/)
144
+ end
145
+
146
+ # URI.unescape
147
+ def unescape
148
+ URI.unescape(self)
149
+ end
150
+
151
+ # encode the string using Encoding.to_utf7(self,false)
152
+ # (encode non [:alnum:] characters). Set <tt>all</tt> true
153
+ # to encode all characters in the string.
154
+ def to_utf7(all=false)
155
+ Encoding.to_utf7(self,all)
156
+ end
157
+
158
+ # File.dirname with a trailing slash
159
+ def dirname
160
+ return self if self.match(/\/$/)
161
+ File.dirname(self) + "/"
162
+ end
163
+
164
+ # File.basename
165
+ def basename
166
+ File.basename(self)
167
+ end
168
+
169
+ def extname
170
+ self.split('.').last
171
+ end
172
+
173
+ # write string to passed filename
174
+ # if filename is nil? will raise an error
175
+ def write(fname=nil)
176
+ raise "filename required" if fname.nil?
177
+ File.write(fname,self)
178
+ return fname
179
+ end
180
+
181
+ def to_form_from_show
182
+ self.split("\n").map { |a|
183
+ key,val = a.split("=",2)
184
+ key = key.split(" ")[-1]
185
+ val = val.strip if val
186
+ ["#{key}=#{val}"]
187
+ }.join("&").to_form.squeeze_keys!
188
+ end
189
+
190
+ def mform
191
+ return self.gsub("\n","").to_form
192
+ end
193
+
194
+ # parse passed GET param string into a form and return the FormArray object
195
+ def to_form
196
+ if self.split("\n").size > 1
197
+ return self.to_form_from_show
198
+ end
199
+ ret = FormArray.new
200
+ self.split("&").each do |x|
201
+ y = x.split("=",2)
202
+ ret.extend!(y[0].to_s,y[1].to_s)
203
+ end
204
+ return ret
205
+ end
206
+
207
+ # create filename from url changing "/" to "_"
208
+ def to_fn(ext=nil)
209
+ ret = self.clip.split("/")[3..-1].join("_")
210
+ ret += ".#{ext}" if not ext.nil?
211
+ return ret
212
+ end
213
+
214
+ # html entity encode string
215
+ # sym = :basic :named :decimal :hexadecimal
216
+ def eencode(sym=nil)
217
+ sym = :named if sym.nil?
218
+ @@he.encode(self,sym)
219
+ end
220
+
221
+ # decode html entities in string
222
+ def edecode
223
+ return @@he.decode(self)
224
+ end
225
+
226
+ def edecode!
227
+ self.replace(@@he.decode(self))
228
+ end
229
+
230
+ # strip html tags from string
231
+ def strip_html
232
+ self.gsub(/<\/?[^>]*>/, "")
233
+ end
234
+
235
+ # range or int
236
+ def head(c=5)
237
+ if c.kind_of?(Range) then
238
+ range = c
239
+ else
240
+ range = (0..(c - 1))
241
+ end
242
+ self.split("\n")[range].join("\n")
243
+ end
244
+
245
+ # return a literal regexp object for this string
246
+ #
247
+ # escape regexp operators
248
+ def to_regexp
249
+ return Regexp.new(self.gsub(/([\[\]\{\}\(\)\*\$\?])/) { |x| '\\' + x })
250
+ end
251
+
252
+ # check if this string is a guid
253
+ def is_guid?
254
+ begin
255
+ Guid.from_s(self)
256
+ rescue => e
257
+ return false
258
+ end
259
+ return true
260
+ end
261
+
262
+ def md5
263
+ Digest::MD5.digest(self).hexify
264
+ end
265
+
266
+ def sha1
267
+ Digest::SHA1.digest(self).hexify
268
+ end
269
+
270
+ def sha256
271
+ Digest::SHA256.digest(self).hexify
272
+ end
273
+
274
+ def sha512
275
+ Digest::SHA512.digest(self).hexify
276
+ end
277
+
278
+ def to_qp
279
+ [self].pack("M")
280
+ end
281
+
282
+ def from_qp
283
+ self.unpack("M").first
284
+ end
285
+ end
286
+
287
+ class Array
288
+ # grep each element of an array for the passed regular expression
289
+ # and return an Array of matches
290
+ # (only works one deep)
291
+ def each_grep(regex)
292
+ ret = []
293
+ self.each { |e| ret << e.grep(regex) }
294
+ return ret
295
+ end
296
+
297
+ # join the array with "\n" and write to a file
298
+ def to_file(filename)
299
+ File.write(filename,self.join("\n"))
300
+ end
301
+ end
302
+
303
+ class Hash#:nodoc:
304
+ # no idea what I was doing here
305
+ def to_f#:nodoc:
306
+ self.each_key { |l| puts "#{l} = " + self[l] }
307
+ return nil
308
+ end
309
+ end
310
+
311
+ class File
312
+ # write string to file
313
+ def self.write(filename,contents)
314
+ fout = File.open(filename,"w")
315
+ fout.print contents
316
+ fout.close
317
+ end
318
+ end
@@ -0,0 +1,188 @@
1
+ # Author Eric Monti (emonti@matasano.com)
2
+ require "base64"
3
+ require "stringio"
4
+ require 'zlib'
5
+ require 'open3'
6
+ require 'sha1'
7
+
8
+ HEXCHARS = [("0".."9").to_a, ("a".."f").to_a].flatten unless defined?(HEXCHARS)
9
+
10
+ #-----------------------------------------------------------------------------
11
+ # Mixins and class-specific items
12
+
13
+ class Fixnum#:nodoc:
14
+ def to_l32; [self].pack "L"; end
15
+ def to_b32; [self].pack "N"; end
16
+ def to_l16; [self].pack "v"; end
17
+ def to_b16; [self].pack "n"; end
18
+ def to_u8; [self].pack "C"; end
19
+ def to_l8; chr; end
20
+ def to_n8; chr; end
21
+ alias to_n32 to_b32
22
+ alias to_n16 to_b16
23
+ def to_n32; [self].pack "N"; end
24
+ def to_n16; [self].pack "n"; end
25
+
26
+ def self.from_l8(str); str[0]; end
27
+ def self.from_l16(str); (str.unpack "v")[0]; end
28
+ def self.from_l32(str); (str.unpack "V")[0]; end
29
+ def self.from_n8(str); str[0]; end
30
+ def self.from_n16(str); (str.unpack "n")[0]; end
31
+ def self.from_n32(str); (str.unpack "N")[0]; end
32
+ end
33
+
34
+ class String
35
+
36
+ # shortcut for hex sanity with regex
37
+ def ishex? ; (self =~ /^[a-f0-9]+$/i)? true : false ; end
38
+
39
+ # Convert a string to ASCII hex string
40
+ # supports a few options for format:
41
+ # :delim - delimter between each hex byte
42
+ # :prefix - prefix before each hex byte
43
+ # :suffix - suffix after each hex byte
44
+ #
45
+ def hexify(opts={})
46
+ s=self
47
+ delim = opts[:delim]
48
+ pre = (opts[:prefix] || "")
49
+ suf = (opts[:suffix] || "")
50
+
51
+ if (rx=opts[:rx]) and not rx.kind_of? Regexp
52
+ raise "rx must be a regular expression for a character class"
53
+ end
54
+
55
+ out=Array.new
56
+
57
+ s.each_byte do |c|
58
+ hc = if (rx and not rx.match c.chr)
59
+ c.chr
60
+ else
61
+ pre + (HEXCHARS[(c >> 4)] + HEXCHARS[(c & 0xf )]) + suf
62
+ end
63
+ out << (hc)
64
+ end
65
+ out.join(delim)
66
+ end
67
+
68
+
69
+ # Convert ASCII hex string to raw
70
+ # supports only 'delimiter' between hex bytes
71
+ def unhexify(d=/\s*/)
72
+ s=self.strip
73
+ out=StringIO.new
74
+ while m = s.match(/^([A-Fa-f0-9]{1,2})#{d}?/) do
75
+ out.write m[1].hex.chr
76
+ s = m.post_match
77
+ end
78
+ out.string
79
+ end
80
+ alias_method :dehexify, :unhexify
81
+
82
+ # ==========================================================================
83
+ # Extends String class to return a hexdump in the style of 'hexdump -C'
84
+ #
85
+ # :len => optionally specify a length other than 16 for a wider or thinner
86
+ # dump. If length is an odd number, it will be rounded up.
87
+ #
88
+ # :out => optionally specify an alternate IO object for output. By default,
89
+ # hexdump will output to STDOUT. Pass a StringIO object and it will return
90
+ # it as a string.
91
+ #
92
+ # Example:
93
+ # xxd = dat.hexdump(:len => 16, :out => StringIO.new)
94
+ # xxd => a hexdump
95
+ #
96
+ # xxd = dat.hexdump(:len => 16, :out => STDERR)
97
+ # xxd => nil
98
+ # ==========================================================================
99
+ def hexdump(opt={})
100
+ s=self
101
+ out = opt[:out] || StringIO.new
102
+ len = (opt[:len] and opt[:len] > 0)? opt[:len] + (opt[:len] % 2) : 16
103
+
104
+ off = opt[:start_addr] || 0
105
+ offlen = opt[:start_len] || 8
106
+
107
+ hlen=len/2
108
+
109
+ s.scan(/(?:.|\n){1,#{len}}/) do |m|
110
+ out.write(off.to_s(16).rjust(offlen, "0") + ' ')
111
+
112
+ i=0
113
+ m.each_byte do |c|
114
+ out.write c.to_s(16).rjust(2,"0") + " "
115
+ out.write(' ') if (i+=1) == hlen
116
+ end
117
+
118
+ out.write(" " * (len-i) ) # pad
119
+ out.write(" ") if i < hlen
120
+
121
+ out.write(" |" + m.tr("\0-\37\177-\377", '.') + "|\n")
122
+ off += m.length
123
+ end
124
+
125
+ out.write(off.to_s(16).rjust(offlen,'0') + "\n")
126
+
127
+ if out.class == StringIO
128
+ out.string
129
+ end
130
+ end
131
+
132
+
133
+ # ==========================================================================
134
+ # converts a hexdump back to binary - takes the same options as hexdump()
135
+ # fairly flexible. should work both with 'xxd' and 'hexdump -C' style dumps
136
+ def dehexdump(opt={})
137
+ s=self
138
+ out = opt[:out] || StringIO.new
139
+ len = (opt[:len] and opt[:len] > 0)? opt[:len] : 16
140
+
141
+ hcrx = /[A-Fa-f0-9]/
142
+ dumprx = /^(#{hcrx}+):?\s*((?:#{hcrx}{2}\s*){0,#{len}})/
143
+ off = opt[:start_addr] || 0
144
+
145
+ i=1
146
+ # iterate each line of hexdump
147
+ s.split(/\r?\n/).each do |hl|
148
+ # match and check offset
149
+ if m = dumprx.match(hl) and $1.hex == off
150
+ i+=1
151
+ # take the data chunk and unhexify it
152
+ raw = $2.unhexify
153
+ off += out.write(raw)
154
+ else
155
+ raise "Hexdump parse error on line #{i} #{s}"
156
+ end
157
+ end
158
+
159
+ if out.class == StringIO
160
+ out.string
161
+ end
162
+ end
163
+ alias_method :dedump, :dehexdump
164
+
165
+ # Does string "start with" dat?
166
+ # no clue whether/when this is faster than a regex, but it is easier
167
+ # than escaping regex characters
168
+ def starts_with?(dat)
169
+ self[0,dat.size] == dat
170
+ end
171
+
172
+ # returns CRC32 checksum for the string object
173
+ def crc32
174
+ Zlib.crc32 self
175
+ end
176
+
177
+ def swap16; unpack("v*").pack("n*"); end
178
+ def to_utf16; Kconv.kconv(self, NKF::UTF16, NKF::ASCII).swap16; end
179
+ def to_ascii; Kconv.kconv(swap16, NKF::ASCII, NKF::UTF16); end
180
+
181
+ end # class String
182
+
183
+
184
+ class Float
185
+ def log2; Math.log(self)/Math.log(2); end
186
+ end
187
+
188
+ __END__