miketracy-wwmd 0.2.16 → 0.2.17

data/History.txt

@@ -1,3 +1,24 @@
+== 0.2.17 / 2009-06-22
+
+* lots happening between here and .9
+* viewstate refactor complete
+* clean up page/page.rb
+* cleaning up page/headers.rb
+* cleaning up page/scrape.rb
+* FormArray refactor includes the form action (full URL)
+  * page.submit(page.get_form)
+* still bugs in URLParse but hunting them down throw by throw
+* remove broken NTLM (preserve auth header warnings)
+* remove WWMDConfig in favor of WWMD module methods but preserve old class for back compat
+* add some burp helpers
+  * Page#from_paste (take entire request into Page and turn off cookies)
+  * burp log parsing coming
+* Curb includes http_put (with header munging bug so careful)
+* internal monkey patch for Curb to do arbitrary verb tampering (not here yet)
+* add String#pbcopy
+* move lots of things around for clarity during refactor
+* refactor progressing but still unstable (2.0.16 gem including viewstate is good to go)
+
 == 0.2.9 / 2009-05-05
 
 * bonesify

data/{README → README.rdoc}

@@ -1,5 +1,16 @@
+== PARDON OUR DUST
 
-	<;'"}()[]>{  XSSFish says, "Swim wif me"
+WWMD is currently in the throes of major cleanup and refactoring.
+
+0.2.17 should be stable.
+
+The viewstate tools can be had by themselves by using:
+
+  require 'wwmd/viewstate'
+
+We appreciate your patience.
+
+  <;'"}()[]>{  XSSFish says, "Swim wif me"
 
 == DESCRIPTION:
 
@@ -33,11 +44,25 @@ check the wiki regularly.
 * hpricot (for the time being)
 * htmlentities
 
+== INSTALL
+
+==== gem installation
+
+WWMD is available as a gem from github:
+
+    gem sources -a http://gems.github.com #(you only have to do this once)
+    gem install mtracy-wwmd
+
+=== manual installation
+
+fetch the repository from github and add path/to/wwmd/lib to your RUBYLIB 
+environment variable
+
 == LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2008
+Copyright (c) 2008,2009 Michael Tracy <mtracy@matasano.com>
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/wwmd.rb

@@ -15,7 +15,7 @@ require 'rexml/document'
 module WWMD
 
   # :stopdoc:
-  VERSION = "0.2.16"
+  VERSION = "0.2.17"
   PARSER = :nokogiri  # :nokogiri || :hpricot
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
@@ -54,7 +54,7 @@ module WWMD
         ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
 
     Dir.glob(search_me).sort.each do |rb|
-      next if rb =~ /_html2text/
+      next if rb =~ /html2text_/
       require rb
     end
   end
@@ -68,11 +68,11 @@ WWMD.require_all_libs_relative_to(__FILE__)
 if WWMD::PARSER == :nokogiri
   require 'nokogiri'
   WWMD::HDOC = Nokogiri::HTML
-  require 'wwmd/nokogiri_html2text'
+  require 'wwmd/page/html2text_nokogiri'
 else
   require 'hpricot'
   WWMD::HDOC = Hpricot
-  require 'wwmd/hpricot_html2text'
+  require 'wwmd/page/html2text_hpricot'
 end
 
 # EOF

data/lib/wwmd/class_extensions.rb

@@ -0,0 +1,2 @@
+Dir.glob(::File.join(::File.dirname(__FILE__),"class_extensions/","*.rb")).each { |rb| require rb }
+

data/lib/wwmd/{mixins.rb → class_extensions/extensions_base.rb}

@@ -1,27 +1,11 @@
 require 'htmlentities'
 
 =begin rdoc
-mixins all around
+let's re-open everything!
 =end
 
 require 'uri'
 
-alias putd puts#:nodoc:
-alias putx puts#:nodoc:
-alias putw puts#:nodoc:
-
-# I really hate this
-class NilClass#:nodoc:
-  def empty?; return true; end
-  def size;   return 0;    end
-  def to_form; return FormArray.new([]); end
-  def clop; return nil; end
-  def inner_html; return nil; end
-  def get_attribute(*args); return nil; end
-  def grep(*args); return []; end
-  def escape(*args); return nil; end
-end
-
 class Numeric
   # return binary representation of <tt>length</tt> size padded with \x00
   #  length:  length in bytes to return (padded with least signficant \x00
@@ -50,8 +34,6 @@ end
 
 class String
 
-  @@he = HTMLEntities.new
-
   def strip_up
     self.gsub(/[^\x20-\x7e,\n]/,"").gsub(/^\n/,"")
   end
@@ -106,63 +88,7 @@ class String
     return [self.clip,self.clop]
   end
 
-  # base 64 decode
-  def b64d
-    self.unpack("m").first
-  end
-
-  # base 64 encode
-  def b64e
-    [self].pack("m").chomp
-  end
-
-  # URI.escape using defaults or passed regexp
-  def escape(reg=nil,unicodify=false)
-    if reg.nil?
-      ret = URI.escape(self)
-    elsif reg.kind_of?(Symbol)
-      case reg
-        when :none; return self
-        when :default; ret =  URI.escape(self)
-        else; ret =  URI.escape(self,WWMD::ESCAPE[reg])
-      end
-    else
-      ret = URI.escape(self,reg)
-    end
-    if unicodify
-      ret.gsub!(/%/,"%u00")
-    end
-    return ret
-  end
-
-  # URI.escape
-  def escape_url(reg=WWMD::ESCAPE[:url])#:nodoc:
-    self.escape(reg)
-  end
-
-  def escape_xss(reg=WWMD::ESCAPE[:xss])#:nodoc:
-    self.escape(reg)
-  end
-
-  def escape_default(reg=WWMD::ESCAPE[:default])
-    self.escape(reg)
-  end
-  # URI.escape all characters in string
-  def escape_all#:nodoc:
-    self.escape(/.*/)
-  end
-
-  # URI.unescape
-  def unescape
-    URI.unescape(self)
-  end
-
-  # encode the string using Encoding.to_utf7(self,false)
-  # (encode non [:alnum:] characters).  Set <tt>all</tt> true
-  # to encode all characters in the string.
-  def to_utf7(all=false)
-    Encoding.to_utf7(self,all)
-  end
+  alias_method :clipa, :clopa
 
   # File.dirname with a trailing slash
   def dirname
@@ -171,8 +97,12 @@ class String
   end
 
   # File.basename
-  def basename
-    File.basename(self)
+  def basename(ext=nil)
+    if ext
+      File.basename(self,ext)
+    else
+      File.basename(self)
+    end
   end
 
   def extname
@@ -187,6 +117,19 @@ class String
     return fname
   end
 
+  # parse passed GET param string into a form and return the FormArray object
+  def to_form
+    if self.split("\n").size > 1
+      return self.to_form_from_show
+    end
+    ret = FormArray.new
+    self.split("&").each do |x|
+      y = x.split("=",2)
+      ret.extend!(y[0].to_s,y[1].to_s)
+    end
+    return ret
+  end
+
   def to_form_from_show
     self.split("\n").map { |a|
       key,val = a.split("=",2)
@@ -200,18 +143,11 @@ class String
     return self.gsub("\n","").to_form
   end
 
-  # parse passed GET param string into a form and return the FormArray object
-  def to_form
-    if self.split("\n").size > 1
-      return self.to_form_from_show
-    end
-    ret = FormArray.new
-    self.split("&").each do |x|
-      y = x.split("=",2)
-      ret.extend!(y[0].to_s,y[1].to_s)
-    end
-    return ret
+  def to_form_from_req
+#    self.split("\x0d\x0a\x0d\x0a")[1].to_form
+    self.split("\n\n")[1].to_form
   end
+  alias_method :to_ffr, :to_form_from_req
 
   # create filename from url changing "/" to "_"
   def to_fn(ext=nil)
@@ -220,22 +156,6 @@ class String
     return ret
   end
 
-  # html entity encode string
-  #  sym = :basic :named :decimal :hexadecimal
-  def eencode(sym=nil)
-    sym = :named if sym.nil?
-    @@he.encode(self,sym)
-  end
-
-  # decode html entities in string
-  def edecode
-    return @@he.decode(self)
-  end
-
-  def edecode!
-    self.replace(@@he.decode(self))
-  end
-
   # strip html tags from string
   def strip_html
     self.gsub(/<\/?[^>]*>/, "")
@@ -284,14 +204,6 @@ class String
     Digest::SHA512.digest(self).hexify
   end
 
-  def to_qp
-    [self].pack("M")
-  end
-
-  def from_qp
-    self.unpack("M").first
-  end
-
   def pbcopy
     IO.popen('pbcopy', 'r+') { |c| c.print self }
   end
@@ -313,14 +225,6 @@ class Array
   end
 end
 
-class Hash#:nodoc:
-  # no idea what I was doing here
-  def to_f#:nodoc:
-    self.each_key { |l| puts "#{l} = " + self[l] }
-    return nil
-  end
-end
-
 class File
   # write string to file
   def self.write(filename,contents)

data/lib/wwmd/class_extensions/extensions_encoding.rb

@@ -0,0 +1,79 @@
+require 'wwmd/class_extensions/mixins_string_encoding'
+class String
+  include WWMD::Encoding
+
+  @@he = HTMLEntities.new
+
+  # base 64 decode
+  def b64d
+    self.unpack("m").first
+  end
+
+  # base 64 encode
+  def b64e
+    [self].pack("m").gsub("\n","")
+  end
+
+  # URI.escape using defaults or passed regexp
+  def escape(reg=nil,unicodify=false)
+    if reg.nil?
+      ret = URI.escape(self)
+    elsif reg.kind_of?(Symbol)
+      case reg
+        when :none; return self
+        when :default; ret =  URI.escape(self)
+        else; ret =  URI.escape(self,WWMD::ESCAPE[reg])
+      end
+    else
+      ret = URI.escape(self,reg)
+    end
+    if unicodify
+      ret.gsub!(/%/,"%u00")
+    end
+    return ret
+  end
+
+  # URI.escape
+  def escape_url(reg=WWMD::ESCAPE[:url])#:nodoc:
+    self.escape(reg)
+  end
+
+  def escape_xss(reg=WWMD::ESCAPE[:xss])#:nodoc:
+    self.escape(reg)
+  end
+
+  def escape_default(reg=WWMD::ESCAPE[:default])
+    self.escape(reg)
+  end
+  # URI.escape all characters in string
+  def escape_all#:nodoc:
+    self.escape(/.*/)
+  end
+
+  # URI.unescape
+  def unescape
+    URI.unescape(self)
+  end
+
+  # html entity encode string
+  #  sym = :basic :named :decimal :hexadecimal
+  def eencode(sym=nil)
+    sym = :named if sym.nil?
+    @@he.encode(self,sym)
+  end
+
+  # decode html entities in string
+  def edecode
+    return @@he.decode(self)
+  end
+
+  # quoted printable
+  def to_qp
+    [self].pack("M")
+  end
+
+  def from_qp
+    self.unpack("M").first
+  end
+
+end

data/lib/wwmd/class_extensions/extensions_nilclass.rb

@@ -0,0 +1,11 @@
+# I really hate this
+class NilClass#:nodoc:
+  def empty?; return true; end
+  def size;   return 0;    end
+  def to_form; return FormArray.new([]); end
+  def clop; return nil; end
+  def inner_html; return nil; end
+  def get_attribute(*args); return nil; end
+  def grep(*args); return []; end
+  def escape(*args); return nil; end
+end

data/lib/wwmd/{encoding.rb → class_extensions/mixins_string_encoding.rb}

@@ -3,11 +3,11 @@ Place methods to character encodings here
 =end
 
 module WWMD
-  # This is where character encodings should go as singletons
+  # This is where character encodings should go as module methods
   # to be used as mixins for the String class
-  class Encoding
+  module Encoding
 
-    # Helper for String.to_utf7 mixin
+    # String.to_utf7 mixin
     # (complete hack but it works)
     #
     # if all=true, encode all characters.
@@ -17,7 +17,7 @@ module WWMD
     # used by:
     #  String.to_utf7
     #  String.to_utf7!
-    def self.to_utf7(str,all=nil)
+    def to_utf7(all=nil)
       if all.kind_of?(Regexp)
         reg = all
       elsif all.kind_of?(TrueClass)
@@ -25,9 +25,9 @@ module WWMD
       else
         reg = ESCAPE[:nalnum] || /[^a-zA-Z0-9]/
       end
-      puts reg.inspect
+      putd "DEBG:" + reg.inspect
       ret = ''
-      str.each_byte do |b|
+      self.each_byte do |b|
         if b.chr.match(reg)
           ret += "+" + Base64.encode64(b.chr.toutf16)[0..2] + "-"
         else

data/lib/wwmd/page.rb

@@ -1,245 +1,3 @@
-module WWMD
-  attr_accessor :curl_object
-  attr_accessor :body_data
-  attr_accessor :post_data
-  attr_accessor :header_data
-  attr_accessor :use_referer
-  attr_reader   :forms
-  attr_reader   :last_error
-  attr_reader   :links         # array of links (urls)
-  attr_reader   :jlinks        # array of included javascript files
-  attr_reader   :spider        # spider object
-  attr_reader   :scrape        # scrape object
-  attr_reader   :urlparse      # urlparse object
-  attr_reader   :comments
-
-  attr_accessor :base_url      # needed to properly munge relative urls into fq urls
-  attr_accessor :logged_in     # are we logged in?
-
-  attr_accessor :opts
-  attr_accessor :inputs
-
-  # WWMD::Page is an extension of a Curl::Easy object which provides methods to
-  # enhance and ease the performance of web application penetration testing.
-  class Page
-
-    def initialize(opts={})
-      @opts = opts.clone
-      DEFAULTS.each { |k,v| @opts[k] = v if not opts[k] }
-      @spider = Spider.new(opts)
-      @scrape = Scrape.new
-      @base_url ||= opts[:base_url]
-      @scrape.warn = opts[:scrape_warn] if !opts[:scrape_warn].nil?
-      if opts.empty?
-        putw "Page initialized without opts"
-        @scrape.warn = false
-      end
-      @urlparse = URLParse.new()
-      @inputs = Inputs.new(self)
-      @logged_in = false
-      @body_data = ""
-      @post_data = ""
-      @comments = []
-      @header_data = FormArray.new
-
-      @curl_object = Curl::Easy.new
-      @opts.each do |k,v|
-        next if !(@curl_object.methods.include?("#{k}="))
-        next if k == :proxy_url
-        @curl_object.send("#{k}=",v)
-      end
-      @curl_object.on_body   { |data| self._body_cb(data) }
-      @curl_object.on_header { |data| self._header_cb(data) }
-
-      # cookies?
-      @curl_object.enable_cookies = @opts[:enable_cookies]
-      if @curl_object.enable_cookies?
-        @curl_object.cookiejar = @opts[:cookiejar] || "./__cookiejar"
-      end
-
-      #proxy?
-      @curl_object.proxy_url = @opts[:proxy_url] if @opts[:use_proxy]
-    end
-
-#:section: Heavy Lifting
-
-    # set reporting data for the page
-    #
-    # Scan for comments, anchors, links and javascript includes and
-    # set page flags.  The heavy lifting for parsing is done in the
-    # scrape class.
-    #
-    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
-    def set_data
-      # reset scrape and inputs object
-      # transparently gunzip
-      begin
-        io = StringIO.new(self.body_data)
-        gz = Zlib::GzipReader.new(io)
-        self.body_data.replace(gz.read)
-      rescue => e
-      end
-      @scrape.reset(self.body_data)
-      @inputs.set
-
-      @comments = @scrape.for_comments
-      # remove comments that are css selectors for IE silliness
-      @comments.reject! do |c|
-        c =~ /\[if IE\]/ ||
-        c =~ /\[if IE \d/ ||
-        c =~ /\[if lt IE \d/
-      end
-      @links = @scrape.for_links.map do |url|
-        @urlparse.parse(self.last_effective_url,url).to_s
-      end
-      @jlinks = @scrape.for_javascript_links
-      @forms = []
-      self.search("//form").each { |f| @forms << Form.new(f) }
-      @spider.add(self.last_effective_url,@links)
-      return [self.code,self.page_status,self.body_data.size]
-    end
-
-    # clear self.body_data and self.header_data
-    def clear_data
-      return false if self.opts[:parse] = false
-      @body_data = ""
-      @header_data.clear
-      @last_error = nil
-    end
-
-    # override Curl::Easy.perform to perform page actions,
-    #  call <tt>self.set_data</tt>
-    #
-    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
-    #
-    # don't call this directly if we are in console mode
-    # use get and submit respectively for GET and POST
-    def perform
-      self.clear_data
-      self.headers["Referer"] = self.cur if self.use_referer
-      begin
-        @curl_object.perform
-      rescue => e
-        @last_error = e
-        putw "WARN: #{e.class}" if e.class =~ /Curl::Err/
-#        self.logged_in = false
-      end
-      self.set_data
-      return [self.code,self.page_status,self.body_data.size]
-    end
-
-    # replacement for Curl::Easy.http_post
-    #
-    # post the form attempting to remove curl supplied headers (Expect, X-Forwarded-For
-    # call <tt>self.set_data</tt>
-    #
-    # if passed a regexp, escape values in the form using regexp before submitting
-    # if passed nil for the regexp arg, the form will not be escaped
-    # default: WWMD::ESCAPE[:url]
-    #
-    # returns: <tt>array [ code, body_data.size ]</tt>
-    def submit(iform=nil,reg=WWMD::ESCAPE[:default])
-##### this is just getting worse and worse
-      if iform.class == "Symbol"
-        reg = iform
-        iform = nil
-      end
-      self.clear_data
-      ["Expect","X-Forwarded-For","Content-length"].each { |s| self.clear_header(s) }
-      self.headers["Referer"] = self.cur if self.use_referer
-      if iform == nil
-        if not self.form.empty?
-          sform = self.form.clone
-        else
-          return "no form provided"
-        end
-      else
-        sform = iform.clone             # clone the form so that we don't change the original
-      end
-      sform.escape_all!(reg)
-      if sform.empty?
-        self.http_post('')
-      else
-        self.http_post(self.post_data = sform.to_post)
-      end
-      begin
-        self.set_data
-      rescue => e
-        STDERR.puts "FATAL: could not parse page"
-      end
-      return [self.code, self.body_data.size]
-    end
-
-    # submit a form using POST string
-    def submit_string(post_string)
-      self.clear_data
-      self.http_post(post_string)
-      self.set_data
-      if self.ntlm?
-        putw "WARN: this page requires NTLM Authentication"
-        putw "WARN: use ntlm_get instead of get"
-      end
-      return [self.code, self.body_data.size]
-    end
-
-    # override for Curl::Easy.perform
-    #
-    # if the passed url string doesn't contain an fully qualified
-    # path, we'll guess and prepend opts[:base_url]
-    #
-    # returns: <tt>array [ code, body_data.size ]</tt>
-    def get(url=nil,parse=true)
-      if !(url =~ /[a-z]+:\/\//) && parse
-        self.url = @urlparse.parse(self.opts[:base_url],url).to_s if url
-      elsif url
-        self.url = url
-      end
-      self.perform
-      if self.ntlm?
-        putw "WARN: this page requires NTLM Authentication"
-        putw "use ntlm_get instead of get"
-      end
-      self.set_data
-      return [self.code, self.body_data.size]
-    end
-
-    # GET with params and POST it as a form
-    def post(url=nil)
-      ep = url.clip
-      self.url = @urlparse.parse(self.opts[:base_url],ep).to_s if ep
-      form = url.clop.to_form
-      self.submit(form)
-    end
-
-    # send arbitrary verb (only works with patch to taf2-curb
-    def verb(verb)
-      return false if !@curl_object.respond_to?(:http_verb)
-      self.clear_data
-      self.headers["Referer"] = self.cur if self.use_referer
-      self.http_verb(verb)
-      self.set_data
-      return [self.code, self.body_data.size]
-    end
-
-#:section: Data callbacks and method_missing
-
-    # callback for <tt>self.on_body</tt>
-    def _body_cb(data)
-      @body_data << data if data
-      return data.length.to_i
-    end
-
-    # callback for <tt>self.on_header</tt>
-    def _header_cb(data)
-      myArr = Array.new(data.split(":",2))
-      @header_data.extend! myArr[0].to_s.strip,myArr[1].to_s.strip
-      return data.length.to_i
-    end
-
-    # send methods not defined here to <tt>@curl_object</tt>
-    def method_missing(methodname, *args)
-      @curl_object.send(methodname, *args)
-    end
-
-  end
-end
+require 'wwmd/wwmd_utils'
+require 'wwmd/wwmd_config'
+require 'wwmd/page/page'