wwmd 0.2.20.3

data/lib/wwmd/page/html2text_hpricot.rb

@@ -0,0 +1,76 @@
+# Geoff Davis geoff at geoffdavis.net
+# Wed May 2 20:08:44 EDT 2007
+# http://rubyforge.org/pipermail/raleigh-rb-members/2007-May/000789.html
+# modified by mtracy at matasano.com for WWMD
+ 
+module WWMD
+   InlineTags = ['a','abbr','acronym','address','b','bdo','big','cite','code','del','dfn','em','font','i','ins','kbd','label','noframes','noscript','q','s','samp','small','span','strike','strong','sub','sup','td','th','tt','u','html','body','table']
+   BlockTags =  ['blockquote','br','center','dd','div','fieldset','form','h1','h2','h3', 'h4','h5','h6','hr','p','pre','tr','var',]
+   ListTags =   ['dir','dl','menu','ol','ul']
+   ItemTags =   ['li','dt']
+#   AsciiEquivalents =  {"amp"=>"&","bull"=>"*","copy"=>"(c)","laquo"=>"<<","raquo"=>">>","ge"=> ">=","le"=>"<=","mdash"=>"-","ndash"=>"-","plusmn"=>"+/-","times"=>"x"}
+ 
+#   NamedCharRegex = Regexp.new("(&("+Hpricot::NamedCharacters.keys.join("|")+");)")
+ 
+  class Page
+    def element_to_text(n)
+      tag = n.etag || n.stag
+      name = tag.name.downcase
+      s = ""
+      is_block  = BlockTags.include?(name)
+      is_list   = ListTags.include?(name)
+      is_item   = ItemTags.include?(name)
+      is_inline = InlineTags.include?(name)
+      if is_block or is_list or is_item or is_inline
+        n.each_child do |c|
+          s += node_to_text(c)
+        end
+        if is_block or is_list
+          s += "\n"
+        elsif is_item
+          s = "* " + s + "\n"
+        end
+      end
+      s
+    end
+ 
+    def node_to_text(n)
+      return "" if n.comment?
+      return element_to_text(n) if n.elem?
+      return n.inner_text if n.text?
+  
+      s = ""
+      begin
+        n.each_child do |c|
+          s += node_to_text(c)
+        end
+      rescue => e
+        putw "WARN: #{e.inspect}"
+      end
+      return s
+    end
+ 
+ #   def lookup_named_char(s)
+ #     c = Hpricot::NamedCharacters[s[1...-1]]
+ #     c.chr if c
+ #   end
+ 
+    def html2text
+      doc = self.scrape.hdoc
+      text = node_to_text(doc)
+#      text.gsub!(NamedCharRegex){|s| "#{lookup_named_char(s)}"}
+      # clean up white space
+      text.gsub!("\r"," ")
+      text.squeeze!(" ")
+      text.strip!
+      ret = ''
+      text.split(/\n/).each do |l|
+        l.strip!
+        next if l == ''
+        next if l =~ /^\?+$/
+        ret += "#{l}\n"
+      end
+      return ret
+    end
+  end
+end

data/lib/wwmd/page/html2text_nokogiri.rb

@@ -0,0 +1,42 @@
+=begin rdoc
+html2text that works with Nokogiri
+=end
+module WWMD
+
+  INLINETAGS =  ['a','abbr','acronym','address','b','bdo','big','cite',
+                 'code','del','dfn','em','font','i','ins','kbd','label',
+                 'noframes','noscript','q','s','samp','small','span',
+                 'strike','strong','sub','sup','td','th','tt','u',
+                 'html','body','table']
+  BLOCKTAGS =   ['blockquote','center','dd','div','fieldset','form',
+                 'h1','h2','h3','h4','h5','h6','p','pre','tr','var',]
+  LISTTAGS =    ['dir','dl','menu','ol','ul']
+  ITEMTAGS =    ['li','dt']
+  SPECIALTAGS = ['br','hr']
+
+  class Page
+    def html2text
+      arr = []
+      self.scrape.hdoc.traverse do |x|
+        arr << [x.parent.name,x.text] if x.text?
+        if x.elem?
+          arr << [x.name,""] if SPECIALTAGS.include?(x.name)
+        end
+      end
+      ret = ""
+      arr.each do |name,str|
+        (ret += "\n"; next ) if name == "br"
+        (ret += "\n" + ("-" * 72) + "\n"; next) if name == "hr"
+        s = str.strip
+        if BLOCKTAGS.include?(name) or LISTTAGS.include?(name)
+          s += "\n"
+        elsif ITEMTAGS.include?(name)
+          s = "* " + s + "\n"
+        end
+        ret += s
+      end
+      ret.gsub(/\n+/) { "\n" }
+      ret.gsub(/[^\x20-\x7e,\n]/,"").gsub(/^\n/,"")
+    end
+  end
+end

data/lib/wwmd/page/inputs.rb

@@ -0,0 +1,47 @@
+module WWMD
+  class Inputs
+    attr_accessor :elems
+
+    @cobj  = '' # wwmd object
+    @elems = '' # array of elems parse out by self.new()
+
+    def initialize(*args)
+      @cobj = args.shift
+    end
+
+    def show
+      putx @elems
+    end
+
+    # call me from Page.set_data
+    def set
+      @elems = [@cobj.search("//input").map,@cobj.search("//select").map].flatten
+    end
+
+    def get(attr=nil)
+      @elems.map { |x| x[attr] }.reject { |y| y.nil? }
+    end
+
+    #
+    # return: FormArray containing all page inputs
+    def form
+      ret = {}
+      @elems.map do |x|
+        name  = x['name']
+        id    = x['id']
+        next if (name.nil? && id.nil?)
+        value = x['value']
+        type  = x['type']
+        ret[name] = value
+        ret[id] = value if ((id || name) != name)
+      end
+      return FormArray.new(ret)
+    end
+
+    #
+    # return: FormArray containing get params
+    def params
+      return FormArray.new(@cobj.cur.clop.to_form)
+    end
+  end
+end

data/lib/wwmd/page/irb_helpers.rb

@@ -0,0 +1,114 @@
+=begin rdoc
+this file contains methods to help operations in irb (display methods etc.).
+=end
+module WWMD
+  class Page
+
+#:section: IRB helper methods
+
+    def head(i=1)
+      if i.kind_of?(Range)
+        puts self.body_data.split("\n")[i].join("\n")
+        return nil
+      end
+      puts self.body_data.head(i)
+    end
+
+    # IRB: text report what has been parsed from this page
+    def report(short=nil)
+      puts "-------------------------------------------------"
+      self.summary
+      puts "---- links found [#{self.has_links?.to_s} | #{self.links.size}]"
+      self.links.each_index { |i| puts "#{i.to_s} :: #{@links[i]}" } if short.nil?
+      puts "---- javascript found [#{self.has_jlinks?.to_s} | #{self.jlinks.size}]"
+      self.jlinks.each { |url| puts url } if short.nil?
+      puts "---- forms found [#{self.has_form?.to_s} | #{self.forms.size}]"
+      puts "---- comments found [#{self.has_comments?.to_s}]"
+      return nil
+    end
+
+    alias_method :show, :report#:nodoc:
+
+    # IRB: display summary of what has been parsed from this page
+    def summary
+      status = self.page_status
+      puts "XXXX[#{self.report_flags}] | #{self.response_code.to_s} | #{status} | #{self.url} | #{self.size}"
+      return nil
+    end
+
+    # IRB: display current headers
+    def request_headers
+      self.headers.each_pair { |k,v| puts "#{k}: #{v}" }
+      return nil
+    end
+
+    alias_method :show_headers, :request_headers#:nodoc:
+    alias_method :req_headers, :request_headers#:nodoc:
+
+    # IRB: display response headers
+    def response_headers
+      self.header_data.each { |x| puts "#{x[0]} :: #{x[1]}" }
+      return nil
+    end
+
+    alias_method :resp_headers, :response_headers#:nodoc:
+
+    # display self.body_data
+    def dump_body
+      puts self.body_data
+    end
+
+    alias_method :dump, :dump_body#:nodoc:
+
+    # IRB: puts the page filtered through html2text
+    def to_text; puts self.html2text; end
+    def text; self.html2text; end
+
+    # IRB: display a human readable report of all forms contained in page.body_data
+    def all_forms
+      self.forms.each_index { |x| puts "[#{x.to_s}]-------"; self.forms[x].report }
+      nil
+    end
+
+    def onclicks
+      self.search("//*[@onclick]").each { |x| puts x[:onclick] }
+      nil
+    end
+
+    # hexdump self.body_data
+    def hexdump
+      puts self.body_data.hexdump
+    end
+
+    # this only works on a mac so get a mac
+    def open #:nodoc:
+      fn = "wwmdtmp_#{Guid.new}.html"
+      self.write(fn)
+      %x[open #{fn}]
+    end
+  end
+
+  class Form
+    def report
+      return nil if not WWMD::console
+      puts "action = #{self.action}"
+      self.fields.each { |field| puts field.to_text }
+      return nil
+    end
+    alias_method :show, :report
+  end
+
+  class FormArray
+    # IRB: puts the form in human readable format
+    # if you <tt>form.show(true)</tt> it will show unescaped values
+    def show(unescape=false)
+      if unescape
+        self.each_index { |i| puts i.to_s + " :: " + self[i][0].to_s + " = " + self[i][1].to_s.unescape }
+      else
+        self.each_index { |i| puts i.to_s + " :: " + self[i][0].to_s + " = " + self[i][1].to_s }
+      end
+      return nil
+    end
+
+  end
+end

data/lib/wwmd/page/page.rb

@@ -0,0 +1,257 @@
+module WWMD
+  # WWMD::Page is an extension of a Curl::Easy object which provides methods to
+  # enhance and ease the performance of web application penetration testing.
+  class Page
+    attr_accessor :curl_object
+    attr_accessor :body_data
+    attr_accessor :post_data
+    attr_accessor :header_data
+    attr_accessor :use_referer
+    attr_reader   :forms
+    attr_reader   :last_error
+    attr_reader   :links         # array of links (urls)
+    attr_reader   :jlinks        # array of included javascript files
+    attr_reader   :spider        # spider object
+    attr_reader   :scrape        # scrape object
+    attr_reader   :urlparse      # urlparse object
+    attr_reader   :comments
+
+    attr_reader   :header_file
+
+    attr_accessor :base_url      # needed to properly munge relative urls into fq urls
+    attr_accessor :logged_in     # are we logged in?
+
+    attr_accessor :opts
+    attr_accessor :inputs
+
+
+    include WWMDUtils
+
+    def inspect
+      # hack
+      return "Page"
+    end
+
+    def initialize(opts={}, &block)
+      @opts = opts.clone
+      DEFAULTS.each { |k,v| @opts[k] = v unless @opts.has_key?(k) }
+      @spider = Spider.new(@opts)
+      @scrape = Scrape.new
+      @base_url ||= opts[:base_url]
+      @scrape.warn = opts[:scrape_warn] if !opts[:scrape_warn].nil? # yeah yeah... bool false
+      @urlparse = URLParse.new()
+      @inputs = Inputs.new(self)
+      @logged_in = false
+      @body_data = ""
+      @post_data = ""
+      @comments = []
+      @header_data = FormArray.new
+      @header_file = nil
+
+      @curl_object = Curl::Easy.new
+      @opts.each do |k,v|
+        next if k == :proxy_url
+        if (@curl_object.respond_to?("#{k}=".intern))
+          @curl_object.send("#{k}=".intern,v)
+        else
+          self.instance_variable_set("@#{k.to_s}".intern,v)
+        end
+      end
+      @curl_object.on_body   { |data| self._body_cb(data) }
+      @curl_object.on_header { |data| self._header_cb(data) }
+
+      # cookies?
+      @curl_object.enable_cookies = @opts[:enable_cookies]
+      if @curl_object.enable_cookies?
+        @curl_object.cookiejar = @opts[:cookiejar] || "./__cookiejar"
+      end
+
+      #proxy?
+      @curl_object.proxy_url = @opts[:proxy_url] if @opts[:use_proxy]
+      instance_eval(&block) if block_given?
+      if opts.empty? && @scrape.warn
+        putw "Page initialized without opts"
+        @scrape.warn = false
+      end
+
+      if @header_file
+        begin
+          headers_from_file(@header_file)
+          @curl_object.enable_cookies = false
+        rescue => e
+          puts "ERROR: #{e}"
+        end
+      end
+    end
+
+#:section: Heavy Lifting
+
+    # set reporting data for the page
+    #
+    # Scan for comments, anchors, links and javascript includes and
+    # set page flags.  The heavy lifting for parsing is done in the
+    # scrape class.
+    #
+    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
+    def set_data
+      # reset scrape and inputs object
+      # transparently gunzip
+      begin
+        io = StringIO.new(self.body_data)
+        gz = Zlib::GzipReader.new(io)
+        self.body_data.replace(gz.read)
+      rescue => e
+      end
+      @scrape.reset(self.body_data)
+      @inputs.set
+
+      # remove comments that are css selectors for IE silliness
+      @comments = @scrape.for_comments.reject do |c|
+        c =~ /\[if IE\]/ ||
+        c =~ /\[if IE \d/ ||
+        c =~ /\[if lt IE \d/
+      end
+      @links = @scrape.for_links.map do |url|
+        l = @urlparse.parse(self.last_effective_url,url).to_s
+      end
+      @jlinks = @scrape.for_javascript_links
+      @forms = @scrape.for_forms
+      @spider.add(self.last_effective_url,@links)
+      return [self.code,self.body_data.size]
+    end
+
+    # clear self.body_data and self.header_data
+    def clear_data
+      return false if self.opts[:parse] = false
+      @body_data = ""
+      @post_data = nil
+      @header_data.clear
+      @last_error = nil
+    end
+
+    # override Curl::Easy.perform to perform page actions,
+    #  call <tt>self.set_data</tt>
+    #
+    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
+    #
+    # don't call this directly if we are in console mode
+    # use get and submit respectively for GET and POST
+    def perform
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      begin
+        @curl_object.perform
+      rescue => e
+        @last_error = e
+        putw "WARN: #{e.class}" if e.class =~ /Curl::Err/
+      end
+      self.set_data
+    end
+
+    # replacement for Curl::Easy.http_post
+    #
+    # post the form attempting to remove curl supplied headers (Expect, X-Forwarded-For
+    # call <tt>self.set_data</tt>
+    #
+    # if passed a regexp, escape values in the form using regexp before submitting
+    # if passed nil for the regexp arg, the form will not be escaped
+    # default: WWMD::ESCAPE[:url]
+    #
+    # returns: <tt>array [ code, body_data.size ]</tt>
+    def submit(iform=nil,reg=WWMD::ESCAPE[:default])
+    ## this is just getting worse and worse
+      if iform.class == "Symbol"
+        reg = iform
+        iform = nil
+      end
+      reg = WWMD::ESCAPE[reg] if reg.class == Symbol
+      self.clear_data
+      ["Expect","X-Forwarded-For","Content-length"].each { |s| self.clear_header(s) }
+      self.headers["Referer"] = self.cur if self.use_referer
+      unless iform
+        unless self.form.empty?
+          sform = self.form.clone
+        else
+          return "no form provided"
+        end
+      else
+        sform = iform.clone             # clone the form so that we don't change the original
+      end
+      sform.escape_all!(reg)
+      self.url = sform.action if sform.action
+      if sform.empty?
+        self.http_post('')
+      else
+        self.http_post(self.post_data = sform.to_post)
+      end
+      self.set_data
+    end
+
+    # submit a form using POST string
+    def submit_string(post_string)
+      self.clear_data
+      self.http_post(post_string)
+      putw "WARN: authentication headers in response" if self.auth?
+      self.set_data
+    end
+
+    # override for Curl::Easy.perform
+    #
+    # if the passed url string doesn't contain an fully qualified
+    # path, we'll guess and prepend opts[:base_url]
+    #
+    # returns: <tt>array [ code, body_data.size ]</tt>
+    def get(url=nil,parse=true)
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      if !(url =~ /[a-z]+:\/\//) && parse
+        self.url = @urlparse.parse(self.base_url,url).to_s if url
+      elsif url
+        self.url = url
+      end
+      self.http_get
+      putw "WARN: authentication headers in response" if self.auth?
+      self.set_data
+    end
+
+    # GET with params and POST it as a form
+    def post(url=nil)
+      ep = url.clip
+      self.url = @urlparse.parse(self.opts[:base_url],ep).to_s if ep
+      form = url.clop.to_form
+      self.submit(form)
+    end
+
+    # send arbitrary verb (only works with patch to taf2-curb)
+    def verb(verb,url=nil)
+      return false if !@curl_object.respond_to?(:http_verb)
+      self.url = url if url
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      self.http_verb(verb)
+      self.set_data
+    end
+
+#:section: Data callbacks and method_missing
+
+    # callback for <tt>self.on_body</tt>
+    def _body_cb(data)
+      @body_data << data if data
+      return data.length.to_i
+    end
+
+    # callback for <tt>self.on_header</tt>
+    def _header_cb(data)
+      myArr = Array.new(data.split(":",2))
+      @header_data.add(myArr[0].to_s.strip,myArr[1].to_s.strip)
+#      @header_data[myArr[0].to_s.strip] = myArr[1].to_s.strip
+      return data.length.to_i
+    end
+
+    # send methods not defined here to <tt>@curl_object</tt>
+    def method_missing(methodname, *args)
+      @curl_object.send(methodname, *args)
+    end
+
+  end
+end