miketracy-wwmd 0.2.16 → 0.2.17

data/lib/wwmd/page/headers.rb

@@ -25,58 +25,62 @@ module WWMD
     #
     #  if clear == true then headers will be cleared before setting
     def set_headers(arg=nil,clear=false)
-      self.clear_headers if clear
+      clear_headers if clear
       if arg.nil?
         begin
-          self.clear_headers
+          clear_headers
           WWMD::DEFAULT_HEADERS.each { |k,v| self.headers[k] = v }
           return "headers set from default"
         rescue => e
-          puts e
-          return "error setting headers"
+          putw "WARN: " + e
+          return false
         end
       elsif arg.class == Symbol
-        self.set_headers(WWMD::HEADERS[arg])
-        return "headers set from #{arg}"
+        set_headers(WWMD::HEADERS[arg])
+        putw "headers set from #{arg}"
+        return true
       elsif arg.class == Hash
         arg.each { |k,v| self.headers[k] = v }
-        return "headers set from hash"
+        putw "headers set from hash"
+        return true
       end
-      "error setting headers"
+      putw "error setting headers"
+      return false
     end
 
     # set headers back to default headers
     def default_headers(arg=nil)
-      self.set_headers
+      set_headers
     end
 
     alias_method :set_default, :default_headers
 
     # set headers from text
     def headers_from_array(arr)
-      self.clear_headers
+      clear_headers
       arr.each do |line|
-        h = line.split(":",2)
-        next if h[1].class != String
-        self.headers[h[0]] = h[1].strip
+        next if (line.empty? || line =~ /^(GET|POST)/)
+        k,v = line.split(":",2)
+        self.headers[k.strip] = v.strip
       end
+      nil
+    end
+
+    # set headers from paste
+    def headers_from_paste
+      headers_from_array(%x[pbpaste])
     end
 
     # set headers from file
     def headers_from_file(fn)
-      self.clear_headers
-      File.read(fn).each do |line|
-        h = line.split(":",2)
-        next if h[1].class != String
-        self.headers[h[0]] = h[1].strip
-      end
-      return nil
+      headers_from_array(File.read(fn).split("\n"))
+      return "headers set from #{fn}"
     end
 
     # set headers to utf7 encoding post
     def set_utf7_headers
       self.headers["Content-Type"] = "application/x-www-form-urlencoded;charset=UTF-7"
-      "headers set to utf7"
+      return "headers set to utf7"
     end
 
     # set headers to ajax

data/lib/wwmd/page/helpers.rb

@@ -0,0 +1,30 @@
+module WWMD
+  class Page
+    # copy and paste from burp request windows
+    # page object gets set with headers and url (not correct)
+    # returns [headers,form]
+    #   form = page.from_paste
+    def from_paste
+      self.enable_cookies = false
+      req = %x[pbpaste]
+      return false if not req
+      h,b = req.chomp.split("\r\n\r\n",2)
+      oh = h
+      h = h.split("\r\n")
+      m,u,p = h.shift.split(" ")
+      return nil unless m =~ (/^(POST|GET)/)
+      self.url = self.base_url + u
+      self.headers_from_array(h)
+      self.body_data = b
+      self.set_data
+      form = b.to_form
+      form.action = @urlparse.parse(self.base_url, u).to_s
+      [oh,form]
+    end
+
+    def resp_paste
+      self.body_data = %x[pbpaste].split("\r\n\r\n",2)[1]
+      self.set_data
+    end
+  end
+end

data/lib/wwmd/{hpricot_html2text.rb → page/html2text_hpricot.rb}

@@ -45,7 +45,7 @@ module WWMD
           s += node_to_text(c)
         end
       rescue => e
-#        puts "WARNING: #{e.inspect}"
+        putw "WARN: #{e.inspect}"
       end
       return s
     end

data/lib/wwmd/page/inputs.rb

@@ -10,7 +10,7 @@ module WWMD
     end
 
     def show
-      puts @elems
+      putx @elems
     end
 
     # call me from Page.set_data

data/lib/wwmd/page/irb_helpers.rb

@@ -16,14 +16,14 @@ module WWMD
 
     # IRB: text report what has been parsed from this page
     def report(short=nil)
-      putx "-------------------------------------------------"
+      puts "-------------------------------------------------"
       self.summary
-      putx "---- links found [#{self.has_links?.to_s} | #{self.links.size}]"
-      self.links.each_index { |i| putx "#{i.to_s} :: #{@links[i]}" } if short.nil?
-      putx "---- javascript found [#{self.has_jlinks?.to_s} | #{self.jlinks.size}]"
-      self.jlinks.each { |url| putx url } if short.nil?
-      putx "---- forms found [#{self.has_form?.to_s} | #{self.forms.size}]"
-      putx "---- comments found [#{self.has_comments?.to_s}]"
+      puts "---- links found [#{self.has_links?.to_s} | #{self.links.size}]"
+      self.links.each_index { |i| puts "#{i.to_s} :: #{@links[i]}" } if short.nil?
+      puts "---- javascript found [#{self.has_jlinks?.to_s} | #{self.jlinks.size}]"
+      self.jlinks.each { |url| puts url } if short.nil?
+      puts "---- forms found [#{self.has_form?.to_s} | #{self.forms.size}]"
+      puts "---- comments found [#{self.has_comments?.to_s}]"
       return nil
     end
 
@@ -32,13 +32,13 @@ module WWMD
     # IRB: display summary of what has been parsed from this page
     def summary
       status = self.page_status
-      putx "XXXX[#{self.report_flags}] | #{self.response_code.to_s} | #{status} | #{self.url} | #{self.size}"
+      puts "XXXX[#{self.report_flags}] | #{self.response_code.to_s} | #{status} | #{self.url} | #{self.size}"
       return nil
     end
 
     # IRB: display current headers
     def request_headers
-      self.headers.each_pair { |k,v| putx "#{k}: #{v}" }
+      self.headers.each_pair { |k,v| puts "#{k}: #{v}" }
       return nil
     end
 
@@ -47,7 +47,7 @@ module WWMD
 
     # IRB: display response headers
     def response_headers
-      self.header_data.each { |x| putx "#{x[0]} :: #{x[1]}" }
+      self.header_data.each { |x| puts "#{x[0]} :: #{x[1]}" }
       return nil
     end
 
@@ -55,18 +55,18 @@ module WWMD
 
     # display self.body_data
     def dump_body
-      putx self.body_data
+      puts self.body_data
     end
 
     alias_method :dump, :dump_body#:nodoc:
 
     # IRB: puts the page filtered through html2text
-    def to_text; putx self.html2text; end
+    def to_text; puts self.html2text; end
     def text; self.html2text; end
 
     # IRB: display a human readable report of all forms contained in page.body_data
     def all_forms
-      self.forms.each_index { |x| putx "[#{x.to_s}]-------"; self.forms[x].report }
+      self.forms.each_index { |x| puts "[#{x.to_s}]-------"; self.forms[x].report }
       nil
     end
 
@@ -87,4 +87,28 @@ module WWMD
       %x[open #{fn}]
     end
   end
+
+  class Form
+    def report
+      return nil if not WWMD::console
+      puts "action = #{self.action}"
+      self.fields.each { |field| puts field.to_text }
+      return nil
+    end
+    alias_method :show, :report
+  end
+
+  class FormArray
+    # IRB: puts the form in human readable format
+    # if you <tt>form.show(true)</tt> it will show unescaped values
+    def show(unescape=false)
+      if unescape
+        self.each_index { |i| puts i.to_s + " :: " + self[i][0].to_s + " = " + self[i][1].to_s.unescape }
+      else
+        self.each_index { |i| puts i.to_s + " :: " + self[i][0].to_s + " = " + self[i][1].to_s }
+      end
+      return nil
+    end
+
+  end
 end

data/lib/wwmd/page/page.rb

@@ -0,0 +1,238 @@
+module WWMD
+  attr_accessor :curl_object
+  attr_accessor :body_data
+  attr_accessor :post_data
+  attr_accessor :header_data
+  attr_accessor :use_referer
+  attr_reader   :forms
+  attr_reader   :last_error
+  attr_reader   :links         # array of links (urls)
+  attr_reader   :jlinks        # array of included javascript files
+  attr_reader   :spider        # spider object
+  attr_reader   :scrape        # scrape object
+  attr_reader   :urlparse      # urlparse object
+  attr_reader   :comments
+
+  attr_accessor :base_url      # needed to properly munge relative urls into fq urls
+  attr_accessor :logged_in     # are we logged in?
+
+  attr_accessor :opts
+  attr_accessor :inputs
+
+  # WWMD::Page is an extension of a Curl::Easy object which provides methods to
+  # enhance and ease the performance of web application penetration testing.
+  class Page
+
+    include WWMDUtils
+
+    def initialize(opts={}, &block)
+      @opts = opts.clone
+      DEFAULTS.each { |k,v| @opts[k] = v unless opts[k] }
+      @spider = Spider.new(opts)
+      @scrape = Scrape.new
+      @base_url ||= opts[:base_url]
+      @scrape.warn = opts[:scrape_warn] if !opts[:scrape_warn].nil? # yeah yeah... bool false
+      @urlparse = URLParse.new()
+      @inputs = Inputs.new(self)
+      @logged_in = false
+      @body_data = ""
+      @post_data = ""
+      @comments = []
+      @header_data = FormArray.new
+
+      @curl_object = Curl::Easy.new
+      @opts.each do |k,v|
+        next if k == :proxy_url
+        self.instance_variable_set("@#{k.to_s}".intern,v)
+        if (@curl_object.methods.include?("#{k}="))
+          @curl_object.send("#{k}=",v)
+        end
+      end
+      @curl_object.on_body   { |data| self._body_cb(data) }
+      @curl_object.on_header { |data| self._header_cb(data) }
+
+      # cookies?
+      @curl_object.enable_cookies = @opts[:enable_cookies]
+      if @curl_object.enable_cookies?
+        @curl_object.cookiejar = @opts[:cookiejar] || "./__cookiejar"
+      end
+
+      #proxy?
+      @curl_object.proxy_url = @opts[:proxy_url] if @opts[:use_proxy]
+      instance_eval(&block) if block_given?
+      if opts.empty? && @scrape.warn
+        putw "Page initialized without opts"
+        @scrape.warn = false
+      end
+    end
+
+#:section: Heavy Lifting
+
+    # set reporting data for the page
+    #
+    # Scan for comments, anchors, links and javascript includes and
+    # set page flags.  The heavy lifting for parsing is done in the
+    # scrape class.
+    #
+    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
+    def set_data
+      # reset scrape and inputs object
+      # transparently gunzip
+      begin
+        io = StringIO.new(self.body_data)
+        gz = Zlib::GzipReader.new(io)
+        self.body_data.replace(gz.read)
+      rescue => e
+      end
+      @scrape.reset(self.body_data)
+      @inputs.set
+
+      # remove comments that are css selectors for IE silliness
+      @comments = @scrape.for_comments.reject do |c|
+        c =~ /\[if IE\]/ ||
+        c =~ /\[if IE \d/ ||
+        c =~ /\[if lt IE \d/
+      end
+      @links = @scrape.for_links.map do |url|
+        @urlparse.parse(self.last_effective_url,url).to_s
+      end
+      @jlinks = @scrape.for_javascript_links
+      @forms = @scrape.for_forms
+      @spider.add(self.last_effective_url,@links)
+      return [self.code,self.body_data.size]
+    end
+
+    # clear self.body_data and self.header_data
+    def clear_data
+      return false if self.opts[:parse] = false
+      @body_data = ""
+      @header_data.clear
+      @last_error = nil
+    end
+
+    # override Curl::Easy.perform to perform page actions,
+    #  call <tt>self.set_data</tt>
+    #
+    # returns: <tt>array [ code, page_status, body_data.size ]</tt>
+    #
+    # don't call this directly if we are in console mode
+    # use get and submit respectively for GET and POST
+    def perform
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      begin
+        @curl_object.perform
+      rescue => e
+        @last_error = e
+        putw "WARN: #{e.class}" if e.class =~ /Curl::Err/
+      end
+      self.set_data
+    end
+
+    # replacement for Curl::Easy.http_post
+    #
+    # post the form attempting to remove curl supplied headers (Expect, X-Forwarded-For
+    # call <tt>self.set_data</tt>
+    #
+    # if passed a regexp, escape values in the form using regexp before submitting
+    # if passed nil for the regexp arg, the form will not be escaped
+    # default: WWMD::ESCAPE[:url]
+    #
+    # returns: <tt>array [ code, body_data.size ]</tt>
+    def submit(iform=nil,reg=WWMD::ESCAPE[:default])
+    ## this is just getting worse and worse
+      if iform.class == "Symbol"
+        reg = iform
+        iform = nil
+      end
+      self.clear_data
+      ["Expect","X-Forwarded-For","Content-length"].each { |s| self.clear_header(s) }
+      self.headers["Referer"] = self.cur if self.use_referer
+      unless iform
+        unless self.form.empty?
+          sform = self.form.clone
+        else
+          return "no form provided"
+        end
+      else
+        sform = iform.clone             # clone the form so that we don't change the original
+      end
+      sform.escape_all!(reg)
+      self.url = sform.action if sform.action
+      if sform.empty?
+        self.http_post('')
+      else
+        self.http_post(self.post_data = sform.to_post)
+      end
+      self.set_data
+    end
+
+    # submit a form using POST string
+    def submit_string(post_string)
+      self.clear_data
+      self.http_post(post_string)
+      putw "WARN: authentication headers in response" if self.auth?
+      self.set_data
+    end
+
+    # override for Curl::Easy.perform
+    #
+    # if the passed url string doesn't contain an fully qualified
+    # path, we'll guess and prepend opts[:base_url]
+    #
+    # returns: <tt>array [ code, body_data.size ]</tt>
+    def get(url=nil,parse=true)
+      if !(url =~ /[a-z]+:\/\//) && parse
+        self.url = @urlparse.parse(self.opts[:base_url],url).to_s if url
+      elsif url
+        self.url = url
+      end
+      self.perform
+      putw "WARN: authentication headers in response" if self.auth?
+      self.set_data
+    end
+
+    # GET with params and POST it as a form
+    def post(url=nil)
+      ep = url.clip
+      self.url = @urlparse.parse(self.opts[:base_url],ep).to_s if ep
+      form = url.clop.to_form
+      self.submit(form)
+    end
+
+    # send arbitrary verb (only works with patch to taf2-curb
+    def verb(verb,url=nil)
+      return false if !@curl_object.respond_to?(:http_verb)
+      self.url = url if url
+      self.clear_data
+      self.headers["Referer"] = self.cur if self.use_referer
+      self.http_verb(verb)
+      self.set_data
+    end
+
+#:section: Data callbacks and method_missing
+
+    # callback for <tt>self.on_body</tt>
+    def _body_cb(data)
+      @body_data << data if data
+      return data.length.to_i
+    end
+
+    # callback for <tt>self.on_header</tt>
+    def _header_cb(data)
+      myArr = Array.new(data.split(":",2))
+      @header_data.extend! myArr[0].to_s.strip,myArr[1].to_s.strip
+      return data.length.to_i
+    end
+
+    # send methods not defined here to <tt>@curl_object</tt>
+    def method_missing(methodname, *args)
+      if WWMD.respond_to?(methodname)
+        WWMD.send(methodname, *args)
+      else
+        @curl_object.send(methodname, *args)
+      end
+    end
+
+  end
+end