iZsh-wwmd 0.2.19

data/lib/wwmd/page/headers.rb

@@ -0,0 +1,111 @@
+module WWMD
+  class Page
+
+#:section: Header helper methods
+
+    # clear header at <key>
+    def clear_header(key)
+      self.headers.delete_if { |k,v| k.upcase == key.upcase }
+      return nil
+    end
+
+    alias_method :delete_header, :clear_header#:nodoc:
+
+    # clear all headers
+    def clear_headers
+      self.headers.delete_if { |k,v| true }
+      "headers cleared"
+    end
+
+    # set headers from passed argument
+    #  Nil:    set headers from WWMD::DEFAULT_HEADERS
+    #  Symbol: entry in WWMD::HEADERS to set from
+    #  Hash:   hash to set headers from
+    #  String: filename (NOT IMPLEMENTED)
+    #
+    #  if clear == true then headers will be cleared before setting
+    def set_headers(arg=nil,clear=false)
+      clear_headers if clear
+      if arg.nil?
+        begin
+          clear_headers
+          WWMD::DEFAULT_HEADERS.each { |k,v| self.headers[k] = v }
+          return "headers set from default"
+        rescue => e
+          putw "WARN: " + e
+          return false
+        end
+      elsif arg.class == Symbol
+        set_headers(WWMD::HEADERS[arg])
+        putw "headers set from #{arg}"
+        return true
+      elsif arg.class == Hash
+        arg.each { |k,v| self.headers[k] = v }
+        putw "headers set from hash"
+        return true
+      end
+      putw "error setting headers"
+      return false
+    end
+
+    # set headers back to default headers
+    def default_headers(arg=nil)
+      set_headers
+    end
+
+    alias_method :set_default, :default_headers
+
+    # set headers from text
+    def headers_from_array(arr)
+      clear_headers
+      arr.each do |line|
+        next if (line.empty? || line =~ /^(GET|POST)/)
+        k,v = line.split(":",2)
+        self.headers[k.strip] = v.strip
+      end
+      nil
+    end
+
+    # set headers from paste
+    def headers_from_paste
+      headers_from_array(%x[pbpaste])
+    end
+
+    # set headers from file
+    def headers_from_file(fn)
+      headers_from_array(File.read(fn).split("\n"))
+      return "headers set from #{fn}"
+    end
+
+    # set headers to utf7 encoding post
+    def set_utf7_headers
+      self.headers["Content-Type"] = "application/x-www-form-urlencoded;charset=UTF-7"
+      return "headers set to utf7"
+    end
+
+    # set headers to ajax
+    def set_ajax_headers
+      self.headers["X-Requested-With"] = "XMLHttpRequest"
+      self.headers["X-Prototype-Version"] = "1.5.0"
+      return "headers set to ajax"
+    end
+
+    # set headers to SOAP request headers
+    def set_soap_headers
+      self.headers['Content-Type'] = "text/xml;charset=utf-8"
+      self.headers['SOAPAction'] = "\"\""
+      return "headers set to soap"
+    end
+
+    # get the current Cookie header
+    def get_cookie
+      self.headers["Cookie"]
+    end
+
+    # set the Cookie header
+    def set_cookie(cookie=nil)
+      self.headers["Cookie"] = cookie
+    end
+
+  end
+end

data/lib/wwmd/page/helpers.rb

@@ -0,0 +1,30 @@
+module WWMD
+  class Page
+    # copy and paste from burp request windows
+    # page object gets set with headers and url (not correct)
+    # returns [headers,form]
+    #   form = page.from_paste
+    def from_paste
+      self.enable_cookies = false
+      req = %x[pbpaste]
+      return false if not req
+      h,b = req.chomp.split("\r\n\r\n",2)
+      oh = h
+      h = h.split("\r\n")
+      m,u,p = h.shift.split(" ")
+      return nil unless m =~ (/^(POST|GET)/)
+      self.url = self.base_url + u
+      self.headers_from_array(h)
+      self.body_data = b
+      self.set_data
+      form = b.to_form
+      form.action = @urlparse.parse(self.base_url, u).to_s
+      [oh,form]
+    end
+
+    def resp_paste
+      self.body_data = %x[pbpaste].split("\r\n\r\n",2)[1]
+      self.set_data
+    end
+  end
+end

data/lib/wwmd/page/html2text_hpricot.rb

@@ -0,0 +1,76 @@
+# Geoff Davis geoff at geoffdavis.net
+# Wed May 2 20:08:44 EDT 2007
+# http://rubyforge.org/pipermail/raleigh-rb-members/2007-May/000789.html
+# modified by mtracy at matasano.com for WWMD
+ 
+module WWMD
+   InlineTags = ['a','abbr','acronym','address','b','bdo','big','cite','code','del','dfn','em','font','i','ins','kbd','label','noframes','noscript','q','s','samp','small','span','strike','strong','sub','sup','td','th','tt','u','html','body','table']
+   BlockTags =  ['blockquote','br','center','dd','div','fieldset','form','h1','h2','h3', 'h4','h5','h6','hr','p','pre','tr','var',]
+   ListTags =   ['dir','dl','menu','ol','ul']
+   ItemTags =   ['li','dt']
+#   AsciiEquivalents =  {"amp"=>"&","bull"=>"*","copy"=>"(c)","laquo"=>"<<","raquo"=>">>","ge"=> ">=","le"=>"<=","mdash"=>"-","ndash"=>"-","plusmn"=>"+/-","times"=>"x"}
+ 
+#   NamedCharRegex = Regexp.new("(&("+Hpricot::NamedCharacters.keys.join("|")+");)")
+ 
+  class Page
+    def element_to_text(n)
+      tag = n.etag || n.stag
+      name = tag.name.downcase
+      s = ""
+      is_block  = BlockTags.include?(name)
+      is_list   = ListTags.include?(name)
+      is_item   = ItemTags.include?(name)
+      is_inline = InlineTags.include?(name)
+      if is_block or is_list or is_item or is_inline
+        n.each_child do |c|
+          s += node_to_text(c)
+        end
+        if is_block or is_list
+          s += "\n"
+        elsif is_item
+          s = "* " + s + "\n"
+        end
+      end
+      s
+    end
+ 
+    def node_to_text(n)
+      return "" if n.comment?
+      return element_to_text(n) if n.elem?
+      return n.inner_text if n.text?
+  
+      s = ""
+      begin
+        n.each_child do |c|
+          s += node_to_text(c)
+        end
+      rescue => e
+        putw "WARN: #{e.inspect}"
+      end
+      return s
+    end
+ 
+ #   def lookup_named_char(s)
+ #     c = Hpricot::NamedCharacters[s[1...-1]]
+ #     c.chr if c
+ #   end
+ 
+    def html2text
+      doc = self.scrape.hdoc
+      text = node_to_text(doc)
+#      text.gsub!(NamedCharRegex){|s| "#{lookup_named_char(s)}"}
+      # clean up white space
+      text.gsub!("\r"," ")
+      text.squeeze!(" ")
+      text.strip!
+      ret = ''
+      text.split(/\n/).each do |l|
+        l.strip!
+        next if l == ''
+        next if l =~ /^\?+$/
+        ret += "#{l}\n"
+      end
+      return ret
+    end
+  end
+end

data/lib/wwmd/page/html2text_nokogiri.rb

@@ -0,0 +1,42 @@
+=begin rdoc
+html2text that works with Nokogiri
+=end
+module WWMD
+
+  INLINETAGS =  ['a','abbr','acronym','address','b','bdo','big','cite',
+                 'code','del','dfn','em','font','i','ins','kbd','label',
+                 'noframes','noscript','q','s','samp','small','span',
+                 'strike','strong','sub','sup','td','th','tt','u',
+                 'html','body','table']
+  BLOCKTAGS =   ['blockquote','center','dd','div','fieldset','form',
+                 'h1','h2','h3','h4','h5','h6','p','pre','tr','var',]
+  LISTTAGS =    ['dir','dl','menu','ol','ul']
+  ITEMTAGS =    ['li','dt']
+  SPECIALTAGS = ['br','hr']
+
+  class Page
+    def html2text
+      arr = []
+      self.scrape.hdoc.traverse do |x|
+        arr << [x.parent.name,x.text] if x.text?
+        if x.elem?
+          arr << [x.name,""] if SPECIALTAGS.include?(x.name)
+        end
+      end
+      ret = ""
+      arr.each do |name,str|
+        (ret += "\n"; next ) if name == "br"
+        (ret += "\n" + ("-" * 72) + "\n"; next) if name == "hr"
+        s = str.strip
+        if BLOCKTAGS.include?(name) or LISTTAGS.include?(name)
+          s += "\n"
+        elsif ITEMTAGS.include?(name)
+          s = "* " + s + "\n"
+        end
+        ret += s
+      end
+      ret.gsub(/\n+/) { "\n" }
+      ret.gsub(/[^\x20-\x7e,\n]/,"").gsub(/^\n/,"")
+    end
+  end
+end

data/lib/wwmd/page/inputs.rb

@@ -0,0 +1,47 @@
+module WWMD
+  class Inputs
+    attr_accessor :elems
+
+    @cobj  = '' # wwmd object
+    @elems = '' # array of elems parse out by self.new()
+
+    def initialize(*args)
+      @cobj = args.shift
+    end
+
+    def show
+      putx @elems
+    end
+
+    # call me from Page.set_data
+    def set
+      @elems = [@cobj.search("//input").map,@cobj.search("//select").map].flatten
+    end
+
+    def get(attr=nil)
+      @elems.map { |x| x[attr] }.reject { |y| y.nil? }
+    end
+
+    #
+    # return: FormArray containing all page inputs
+    def form
+      ret = {}
+      @elems.map do |x|
+        name  = x['name']
+        id    = x['id']
+        next if (name.nil? && id.nil?)
+        value = x['value']
+        type  = x['type']
+        ret[name] = value
+        ret[id] = value if ((id || name) != name)
+      end
+      return FormArray.new(ret)
+    end
+
+    #
+    # return: FormArray containing get params
+    def params
+      return FormArray.new(@cobj.cur.clop.to_form)
+    end
+  end
+end

data/lib/wwmd/page/irb_helpers.rb

@@ -0,0 +1,114 @@
+=begin rdoc
+this file contains methods to help operations in irb (display methods etc.).
+=end
+module WWMD
+  class Page
+
+#:section: IRB helper methods
+
+    def head(i=1)
+      if i.kind_of?(Range)
+        puts self.body_data.split("\n")[i].join("\n")
+        return nil
+      end
+      puts self.body_data.head(i)
+    end
+
+    # IRB: text report what has been parsed from this page
+    def report(short=nil)
+      puts "-------------------------------------------------"
+      self.summary
+      puts "---- links found [#{self.has_links?.to_s} | #{self.links.size}]"
+      self.links.each_index { |i| puts "#{i.to_s} :: #{@links[i]}" } if short.nil?
+      puts "---- javascript found [#{self.has_jlinks?.to_s} | #{self.jlinks.size}]"
+      self.jlinks.each { |url| puts url } if short.nil?
+      puts "---- forms found [#{self.has_form?.to_s} | #{self.forms.size}]"
+      puts "---- comments found [#{self.has_comments?.to_s}]"
+      return nil
+    end
+
+    alias_method :show, :report#:nodoc:
+
+    # IRB: display summary of what has been parsed from this page
+    def summary
+      status = self.page_status
+      puts "XXXX[#{self.report_flags}] | #{self.response_code.to_s} | #{status} | #{self.url} | #{self.size}"
+      return nil
+    end
+
+    # IRB: display current headers
+    def request_headers
+      self.headers.each_pair { |k,v| puts "#{k}: #{v}" }
+      return nil
+    end
+
+    alias_method :show_headers, :request_headers#:nodoc:
+    alias_method :req_headers, :request_headers#:nodoc:
+
+    # IRB: display response headers
+    def response_headers
+      self.header_data.each { |x| puts "#{x[0]} :: #{x[1]}" }
+      return nil
+    end
+
+    alias_method :resp_headers, :response_headers#:nodoc:
+
+    # display self.body_data
+    def dump_body
+      puts self.body_data
+    end
+
+    alias_method :dump, :dump_body#:nodoc:
+
+    # IRB: puts the page filtered through html2text
+    def to_text; puts self.html2text; end
+    def text; self.html2text; end
+
+    # IRB: display a human readable report of all forms contained in page.body_data
+    def all_forms
+      self.forms.each_index { |x| puts "[#{x.to_s}]-------"; self.forms[x].report }
+      nil
+    end
+
+    def onclicks
+      self.search("//*[@onclick]").each { |x| puts x[:onclick] }
+      nil
+    end
+
+    # hexdump self.body_data
+    def hexdump
+      puts self.body_data.hexdump
+    end
+
+    # this only works on a mac so get a mac
+    def open #:nodoc:
+      fn = "wwmdtmp_#{Guid.new}.html"
+      self.write(fn)
+      %x[open #{fn}]
+    end
+  end
+
+  class Form
+    def report
+      return nil if not WWMD::console
+      puts "action = #{self.action}"
+      self.fields.each { |field| puts field.to_text }
+      return nil
+    end
+    alias_method :show, :report
+  end
+
+  class FormArray
+    # IRB: puts the form in human readable format
+    # if you <tt>form.show(true)</tt> it will show unescaped values
+    def show(unescape=false)
+      if unescape
+        self.each_index { |i| puts i.to_s + " :: " + self[i][0].to_s + " = " + self[i][1].to_s.unescape }
+      else
+        self.each_index { |i| puts i.to_s + " :: " + self[i][0].to_s + " = " + self[i][1].to_s }
+      end
+      return nil
+    end
+
+  end
+end

data/lib/wwmd/page/page.rb

@@ -0,0 +1,241 @@
+module WWMD
+  # WWMD::Page is an extension of a Curl::Easy object which provides methods to
+  # enhance and ease the performance of web application penetration testing.
+  class Page
+
+    attr_accessor :curl_object
+    attr_accessor :body_data
+    attr_accessor :post_data
+    attr_accessor :header_data
+    attr_accessor :use_referer
+    attr_reader   :forms
+    attr_reader   :last_error
+    attr_reader   :links         # array of links (urls)
+    attr_reader   :jlinks        # array of included javascript files
+    attr_reader   :spider        # spider object
+    attr_reader   :scrape        # scrape object
+    attr_reader   :urlparse      # urlparse object
+    attr_reader   :comments
+
+    attr_accessor :base_url      # needed to properly munge relative urls into fq urls
+    attr_accessor :logged_in     # are we logged in?
+
+    attr_accessor :opts
+    attr_accessor :inputs
+
+    include WWMDUtils
+
+    def initialize(opts={}, &block)
+      @opts = opts.clone
+      DEFAULTS.each { |k,v| @opts[k] = v unless opts[k] }
+      @spider = Spider.new(opts)
+      @scrape = Scrape.new
+      @base_url ||= opts[:base_url]
+      @scrape.warn = opts[:scrape_warn] if !opts[:scrape_warn].nil? # yeah yeah... bool false
+      @urlparse = URLParse.new()
+      @inputs = Inputs.new(self)
+      @logged_in = false
+      @body_data = ""
+      @post_data = ""
+      @comments = []
+      @header_data = FormArray.new
+
+      @curl_object = Curl::Easy.new
+      @opts.each do |k,v|
+        next if k == :proxy_url
+        self.instance_variable_set("@#{k.to_s}".intern,v)
+        if (@curl_object.methods.include?("#{k}="))
+          @curl_object.send("#{k}=",v)
+        end
+      end
+      @curl_object.on_body   { |data| self._body_cb(data) }
+      @curl_object.on_header { |data| self._header_cb(data) }
+
+      # cookies?
+      @curl_object.enable_cookies = @opts[:enable_cookies]
+      if @curl_object.enable_cookies?
+        @curl_object.cookiejar = @opts[:cookiejar] || "./__cookiejar"
+      end
+
+      #proxy?
+      @curl_object.proxy_url = @opts[:proxy_url] if @opts[:use_proxy]
+      instance_eval(&block) if block_given?
+      if opts.empty? && @scrape.warn
+        putw "Page initialized without opts"
+        @scrape.warn = false
+      end
+    end
+
+#:section: Heavy Lifting
+
+    # set reporting data for the page
+    #
+    # Scan for comments, anchors, links and javascript includes and
+    # set page flags.  The heavy lifting for parsing is done in the
+    # scrape class.
+    #
+    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
+    def set_data
+      # reset scrape and inputs object
+      # transparently gunzip
+      begin
+        io = StringIO.new(self.body_data)
+        gz = Zlib::GzipReader.new(io)
+        self.body_data.replace(gz.read)
+      rescue => e
+      end
+      @scrape.reset(self.body_data)
+      @inputs.set
+
+      # remove comments that are css selectors for IE silliness
+      @comments = @scrape.for_comments.reject do |c|
+        c =~ /\[if IE\]/ ||
+        c =~ /\[if IE \d/ ||
+        c =~ /\[if lt IE \d/
+      end
+      @links = @scrape.for_links.map do |url|
+        @urlparse.parse(self.last_effective_url,url).to_s
+      end
+      @jlinks = @scrape.for_javascript_links
+      @forms = @scrape.for_forms
+      @spider.add(self.last_effective_url,@links)
+      return [self.code,self.body_data.size]
+    end
+
+    # clear self.body_data and self.header_data
+    def clear_data
+      return false if self.opts[:parse] = false
+      @body_data = ""
+      @header_data.clear
+      @post_data = ""
+      @last_error = nil
+    end
+
+    # override Curl::Easy.perform to perform page actions,
+    #  call <tt>self.set_data</tt>
+    #
+    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
+    #
+    # don't call this directly if we are in console mode
+    # use get and submit respectively for GET and POST
+    def perform
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      begin
+        @curl_object.perform
+      rescue => e
+        @last_error = e
+        putw "WARN: #{e.class}" if e.class =~ /Curl::Err/
+      end
+      self.set_data
+    end
+
+    # replacement for Curl::Easy.http_post
+    #
+    # post the form attempting to remove curl supplied headers (Expect, X-Forwarded-For
+    # call <tt>self.set_data</tt>
+    #
+    # if passed a regexp, escape values in the form using regexp before submitting
+    # if passed nil for the regexp arg, the form will not be escaped
+    # default: WWMD::ESCAPE[:url]
+    #
+    # returns: <tt>array [ code, body_data.size ]</tt>
+    def submit(iform=nil,reg=WWMD::ESCAPE[:default])
+    ## this is just getting worse and worse
+      if iform.class == "Symbol"
+        reg = iform
+        iform = nil
+      end
+      self.clear_data
+      ["Expect","X-Forwarded-For","Content-length"].each { |s| self.clear_header(s) }
+      self.headers["Referer"] = self.cur if self.use_referer
+      unless iform
+        unless self.form.empty?
+          sform = self.form.clone
+        else
+          return "no form provided"
+        end
+      else
+        sform = iform.clone             # clone the form so that we don't change the original
+      end
+      sform.escape_all!(reg)
+      self.url = sform.action if sform.action
+      if sform.empty?
+        self.http_post('')
+      else
+        self.http_post(self.post_data = sform.to_post)
+      end
+      self.set_data
+    end
+
+    # submit a form using POST string
+    def submit_string(post_string)
+      self.clear_data
+      self.http_post(post_string)
+      putw "WARN: authentication headers in response" if self.auth?
+      self.set_data
+    end
+
+    # override for Curl::Easy.perform
+    #
+    # if the passed url string doesn't contain an fully qualified
+    # path, we'll guess and prepend opts[:base_url]
+    #
+    # returns: <tt>array [ code, body_data.size ]</tt>
+    def get(url=nil,parse=true)
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      if !(url =~ /[a-z]+:\/\//) && parse
+        self.url = @urlparse.parse(self.opts[:base_url],url).to_s if url
+      elsif url
+        self.url = url
+      end
+      self.http_get
+      putw "WARN: authentication headers in response" if self.auth?
+      self.set_data
+    end
+
+    # GET with params and POST it as a form
+    def post(url=nil)
+      ep = url.clip
+      self.url = @urlparse.parse(self.opts[:base_url],ep).to_s if ep
+      form = url.clop.to_form
+      self.submit(form)
+    end
+
+    # send arbitrary verb (only works with patch to taf2-curb)
+    def verb(verb,url=nil)
+      return false if !@curl_object.respond_to?(:http_verb)
+      self.url = url if url
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      self.http_verb(verb)
+      self.set_data
+    end
+
+#:section: Data callbacks and method_missing
+
+    # callback for <tt>self.on_body</tt>
+    def _body_cb(data)
+      @body_data << data if data
+      return data.length.to_i
+    end
+
+    # callback for <tt>self.on_header</tt>
+    def _header_cb(data)
+      myArr = Array.new(data.split(":",2))
+      @header_data.extend! myArr[0].to_s.strip,myArr[1].to_s.strip
+      return data.length.to_i
+    end
+
+    # send methods not defined here to <tt>@curl_object</tt>
+    def method_missing(methodname, *args)
+      if self.respond_to?(methodname)
+        self.send(methodname, *args)
+      else
+        @curl_object.send(methodname, *args)
+      end
+    end
+
+  end
+end