miketracy-wwmd 0.2.11

data/lib/wwmd/page/auth.rb

@@ -0,0 +1,183 @@
+=begin rdoc
+This is where we do all the undocumented auth stuff.  NTLM is here and 
+hooked in.
+
+WWMDNTLM is an incredibly naive NTLM implementation (used to get 
+around NTLM for one project ahwile back
+=end
+
+module WWMD
+  class Page
+
+#:stopdoc:
+
+#:section: Authentication helpers
+
+    # check if this request requires NTLM
+    def ntlm?
+      return false if self.code != 401
+      count = 0
+      self.header_data.each do |i|
+        if i[0] =~ /www-authenticate/i
+          count += 1 if (i[1] == "Negotiate" || i[1] == "NTLM")
+        end
+      end
+      return (count > 0)
+    end
+
+    # does this request have an authenticate header?
+    def auth?
+      return false if self.code != 401
+      count = 0
+      self.header_data.each do |i|
+        if i[0] =~ /www-authenticate/i
+          count += 1
+        end
+      end
+      return (count > 0)
+    end
+
+    # not sure why this is here
+    def ntlm_perform(exp=nil)#:nodoc:
+      self.perform
+      return (self.code == exp)
+    end
+
+    # perform a get usig NTLM
+    def ntlm_get(url=nil?,debug=false)
+      self.clear_header('Authorization')
+      nobj = WWMDNTLM.new(self.opts)
+      self.url = @urlparse.parse(self.opts[:base_url],url) if not url.nil?
+      self.perform
+      return "This request does not appear to require NTLM" if not self.ntlm?
+      self.headers['Authorization'] = nobj.type_1_msg
+      self.perform
+      type2 = self.header_data.get_value('WWW-Authenticate')
+      nonce = nobj.get_nonce(type2)
+      type3 = nobj.type_3_msg(nonce)
+      self.headers['Authorization'] = type3
+      self.perform
+      self.clear_header('Authorization')
+      return self.code
+    end
+
+#:startdoc:
+
+  end
+
+  class WWMDNTLM#:nodoc:
+    attr_accessor :hostname
+    attr_accessor :domain
+    attr_accessor :username
+    attr_accessor :password
+    attr_accessor :opts
+    attr_accessor :negotiate_flags
+    attr_accessor :debug
+
+    def initialize(opts,debug=false)
+      @opts = opts
+      @hostname = self.opts[:hostname]
+      @domain   = self.opts[:domain]
+      @username = self.opts[:username]
+      @password = self.opts[:password]
+      @hostname = "LOCALHOST" if self.hostname.nil?
+      @negotiate_flags = 0x00002201.to_l32
+      @debug = debug
+    end
+
+    def type_1_msg
+# do not add domain for now here as it doesn't seem to be needed
+# it does need to be set for type3 messages
+      host_len = self.hostname.size
+      host_off = 0x20
+#      if self.domain.nil?
+      if true
+        dom_off = 0
+        dom_len = 0
+      else
+        dom_off = (host_off + hostname.size)
+        dom_len = self.domain.size
+      end
+      msg = ""
+      msg << "NTLMSSP\x00"        # signature[8]
+      msg << 0x01.to_l32          # type[4]
+      msg << self.negotiate_flags # NegotiateFlags[4]
+      msg << dom_len.to_l16       # domain string length[2]
+      msg << dom_len.to_l16       # domain string length[2]
+      msg << dom_off.to_l32       # domain string offset[4]
+      msg << host_len.to_l16      # host string length[2]
+      msg << host_len.to_l16      # host string length[2]
+      msg << host_off.to_l32      # host string offset[4]
+#      msg << self.domain if not self.domain.nil? # domain[var]
+      msg << self.hostname        # host name[var]
+      return "NTLM " + msg.b64e
+    end
+
+    def get_nonce(t2msg)
+      # Signature[8]
+      # MessageType[4]
+      # TargetNameFields[8]
+      # NegotiateFlags[4]
+      # ServerChallenge[8]
+      # Reserved[8] ! 0x00
+      # TargetInfoFields[8]
+      # Version[8]
+      # Payload[var]
+      msg = t2msg.split[1].b64d
+      return msg[24..31]
+    end
+
+    def type_3_msg(nonce)
+      hlen = 0x40
+      poff = hlen
+      domain = self.domain.to_utf16
+      username = self.username.to_utf16
+      hostname = self.hostname.to_utf16
+      lmresp = NTLM.lm_response(self.opts[:password],nonce,:lmhash)
+      ntresp = NTLM.lm_response(self.opts[:password],nonce,:nthash)
+      msg = ""
+      msg << "NTLMSSP\x00"            # Signature[8]
+      msg << 0x03.to_l32              # MessageType[4]
+                                      # LmChallengeResonseFields[8]
+      msg << lmresp.size.to_l16           # LmChallengeResponseLen[2]
+      msg << lmresp.size.to_l16           # LmChallengeResponseMaxLen[2]
+      msg << poff.to_l32                  # LmChallengeResponseBufferOffset[4]
+      poff += lmresp.size
+#      msg << 0x40.to_l32                  # LmChallengeResponseBufferOffset[4]
+                                      # NtChallengeResponseFields[8]
+      msg << ntresp.size.to_l16           # NtChallengeResponseLen[2]
+      msg << ntresp.size.to_l16           # NtChallengeResponseMaxLen[2]
+#      msg << 0x58.to_l32                  # NtChallengeResponseBufferOffset[4]
+      msg << poff.to_l32                  # NtChallengeResponseBufferOffset[4]
+      poff += ntresp.size
+                                      # DomainNameFields[8]
+      msg << domain.size.to_l16           # DomainNameLen[2]
+      msg << domain.size.to_l16           # DomainNameMaxLen[2]
+      msg << poff.to_l32                  # DomainNameBufferOffset[4]
+      poff += domain.size
+                                      # UserNameFields[8]
+      msg << username.size.to_l16         # UserNameLen[2]
+      msg << username.size.to_l16         # UserNameMaxLen[2]
+      msg << poff.to_l32                  # UserNameBufferOffset[4]
+      poff += username.size
+                                      # WorkstationFields[8]
+      msg << hostname.size.to_l16         # WorkstationLen[2]
+      msg << hostname.size.to_l16         # WorkstationMaxLen[2]
+      msg << poff.to_l32                  # WorkstationBufferOffset[4]
+      poff += hostname.size
+                                      # EncryptedRandomSessionKeyFields[8]
+      msg << 0x00.to_l16                  # EncryptedRandomSessionKeyLen[2]
+      msg << 0x00.to_l16                  # EncryptedRandomSessionKeyMaxLen[2]
+      msg << 0x00.to_l32                  # EncryptedRandomSessionKeyBufferOffset[4]
+      msg << self.negotiate_flags     # NegotiateFlags[4]
+                                      # Version[8] (optional do not add)
+                                      # Payload[var]
+      msg << lmresp                       # LmChallenge[var]
+      msg << ntresp                       # NtChallenge[var]
+      msg << domain                       # DomainName[var]
+      msg << username                     # UserName[var]
+      msg << hostname                     # Workstation[var]
+      return "NTLM " + msg.b64e
+    end
+  end
+end

data/lib/wwmd/page/config.rb

@@ -0,0 +1,44 @@
+module WWMD
+  class WWMDConfig
+
+    def self.load_config(file)
+      begin
+        config = YAML.load_file(file)
+      rescue => e
+        putw "config file not found #{file}"
+        putw e.inspect
+        exit
+      end
+      return config
+    end
+
+    def self.parse_opts(args)
+      inopts = Hash.new
+      opts = OptionParser.new do |opts|
+        # set defaults
+        opts.on("-p", "--password PASSWORD", "Password")     { |v| inopts[:password] = v }
+        opts.on("-u", "--username USERNAME", "Username")     { |v| inopts[:username] = v }
+        opts.on("--header_file HEADER_FILE","Header file")   { |v| inopts[:header_file] = v }
+        opts.on("--base_url BASE_URL","Base url")            { |v| inopts[:base_url] = v }
+        opts.on("--use_proxy PROXY_URL", "Use proxy at url") do |v|
+          ENV['HTTP_PROXY'] = "http://" + v.to_s
+          inopts[:use_proxy] = true
+          inopts[:proxy_url] = v
+        end
+        opts.on("--no_proxy","do not use proxy") do |v|
+          inopts[:use_proxy] = false
+          inopts[:proxy_url] = nil
+        end
+        opts.on("--use_auth","login before getting url")     { |v| inopts[:use_auth] = true }
+        opts.on("--no_auth","no login before getting url")   { |v| inopts[:use_auth] = false }
+        opts.on("--debug","debugging really doesn't work")   { |v| inopts[:debug] = true }
+        opts.on_tail("-h", "--help", "Show this message") do
+          putx opts
+          exit
+        end
+      end
+      opts.parse!(args)
+      return inopts
+    end
+  end
+end

data/lib/wwmd/page/constants.rb

@@ -0,0 +1,60 @@
+module WWMD
+  XSSFISH = "<;'\"}()[]>{"
+
+  DEFAULTS = {
+    :base_url => "",
+    :use_auth => true,
+    :enable_cookies => true,
+    :cookiejar => "./__cookiejar",
+    :follow_location => true,
+    :max_redirects => 20,
+    :use_proxy => false,
+    :debug => false,
+    :scrape_warn => true,
+    :parse => true,
+    :timeout => 20,
+  }
+
+  ESCAPE = {
+    :url     => /[^a-zA-Z0-9\-_%]/,
+    :nalnum  => /[^a-zA-Z0-9]/,
+    :xss     => /[^a-zA-Z0-9=?&()']/,
+    :ltgt    => /[<>]/,
+    :all     => /.*/,
+    :b64     => /[=+\/]/,
+    :none    => :none,
+    :default => :default,
+  }
+
+  UA = {
+    :mozilla => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.16) Gecko/20080702 Firefox/2.0.0.16",
+    :moz3 => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.0.1) Gecko/2008070206 Firefox/3.0.1",
+    :ie6 => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
+    :ie7 => "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
+    :ie8 => "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)",
+    :opera => "Opera/9.20 (Windows NT 6.0; U; en)",
+    :safari => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_4_11; en) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22"
+  }
+
+  DEFAULT_HEADERS = {
+    "User-Agent" => UA[:moz3],
+    "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language" => "en-US,en;q=0.8,en-au;q=0.6,en-us;q=0.4,en;q=0.2",
+    "Accept-Encoding" => "gzip,deflate",
+    "Accept-Charset" => "SO-8859-1,utf-8;q=0.7,*;q=0.7",
+    "Keep-Alive" => "300",
+    "Connection" => "keep-alive",
+  }
+
+  HEADERS = {
+    :default => nil,
+    :utf7 => {
+      "Content-Type" => "application/x-www-form-urlencoded;charset=UTF-7",
+      "Content-Transfer-Encoding" => "7bit",
+    },
+    :ajax => {
+      "X-Requested-With" => "XMLHttpRequest",
+      "X-Prototype-Version" => "1.5.0",
+    },
+  }
+end

data/lib/wwmd/page/headers.rb

@@ -0,0 +1,107 @@
+module WWMD
+  class Page
+
+#:section: Header helper methods
+
+    # clear header at <key>
+    def clear_header(key)
+      self.headers.delete_if { |k,v| k.upcase == key.upcase }
+      return nil
+    end
+
+    alias_method :delete_header, :clear_header#:nodoc:
+
+    # clear all headers
+    def clear_headers
+      self.headers.delete_if { |k,v| true }
+      "headers cleared"
+    end
+
+    # set headers from passed argument
+    #  Nil:    set headers from WWMD::DEFAULT_HEADERS
+    #  Symbol: entry in WWMD::HEADERS to set from
+    #  Hash:   hash to set headers from
+    #  String: filename (NOT IMPLEMENTED)
+    #
+    #  if clear == true then headers will be cleared before setting
+    def set_headers(arg=nil,clear=false)
+      self.clear_headers if clear
+      if arg.nil?
+        begin
+          self.clear_headers
+          WWMD::DEFAULT_HEADERS.each { |k,v| self.headers[k] = v }
+          return "headers set from default"
+        rescue => e
+          puts e
+          return "error setting headers"
+        end
+      elsif arg.class == Symbol
+        self.set_headers(WWMD::HEADERS[arg])
+        return "headers set from #{arg}"
+      elsif arg.class == Hash
+        arg.each { |k,v| self.headers[k] = v }
+        return "headers set from hash"
+      end
+      "error setting headers"
+    end
+
+    # set headers back to default headers
+    def default_headers(arg=nil)
+      self.set_headers
+    end
+
+    alias_method :set_default, :default_headers
+
+    # set headers from text
+    def headers_from_array(arr)
+      self.clear_headers
+      arr.each do |line|
+        h = line.split(":",2)
+        next if h[1].class != String
+        self.headers[h[0]] = h[1].strip
+      end
+    end
+
+    # set headers from file
+    def headers_from_file(fn)
+      self.clear_headers
+      File.read(fn).each do |line|
+        h = line.split(":",2)
+        next if h[1].class != String
+        self.headers[h[0]] = h[1].strip
+      end
+      return nil
+    end
+
+    # set headers to utf7 encoding post
+    def set_utf7_headers
+      self.headers["Content-Type"] = "application/x-www-form-urlencoded;charset=UTF-7"
+      "headers set to utf7"
+    end
+
+    # set headers to ajax
+    def set_ajax_headers
+      self.headers["X-Requested-With"] = "XMLHttpRequest"
+      self.headers["X-Prototype-Version"] = "1.5.0"
+      return "headers set to ajax"
+    end
+
+    # set headers to SOAP request headers
+    def set_soap_headers
+      self.headers['Content-Type'] = "text/xml;charset=utf-8"
+      self.headers['SOAPAction'] = "\"\""
+      return "headers set to soap"
+    end
+
+    # get the current Cookie header
+    def get_cookie
+      self.headers["Cookie"]
+    end
+
+    # set the Cookie header
+    def set_cookie(cookie=nil)
+      self.headers["Cookie"] = cookie
+    end
+
+  end
+end

data/lib/wwmd/page/inputs.rb

@@ -0,0 +1,47 @@
+module WWMD
+  class Inputs
+    attr_accessor :elems
+
+    @cobj  = '' # wwmd object
+    @elems = '' # array of elems parse out by self.new()
+
+    def initialize(*args)
+      @cobj = args.shift
+    end
+
+    def show
+      puts @elems
+    end
+
+    # call me from Page.set_data
+    def set
+      @elems = [@cobj.search("//input").map,@cobj.search("//select").map].flatten
+    end
+
+    def get(attr=nil)
+      @elems.map { |x| x[attr] }.reject { |y| y.nil? }
+    end
+
+    #
+    # return: FormArray containing all page inputs
+    def form
+      ret = {}
+      @elems.map do |x|
+        name  = x['name']
+        id    = x['id']
+        next if (name.nil? && id.nil?)
+        value = x['value']
+        type  = x['type']
+        ret[name] = value
+        ret[id] = value if ((id || name) != name)
+      end
+      return FormArray.new(ret)
+    end
+
+    #
+    # return: FormArray containing get params
+    def params
+      return FormArray.new(@cobj.cur.clop.to_form)
+    end
+  end
+end

data/lib/wwmd/page/irb_helpers.rb

@@ -0,0 +1,90 @@
+=begin rdoc
+this file contains methods to help operations in irb (display methods etc.).
+=end
+module WWMD
+  class Page
+
+#:section: IRB helper methods
+
+    def head(i=1)
+      if i.kind_of?(Range)
+        puts self.body_data.split("\n")[i].join("\n")
+        return nil
+      end
+      puts self.body_data.head(i)
+    end
+
+    # IRB: text report what has been parsed from this page
+    def report(short=nil)
+      putx "-------------------------------------------------"
+      self.summary
+      putx "---- links found [#{self.has_links?.to_s} | #{self.links.size}]"
+      self.links.each_index { |i| putx "#{i.to_s} :: #{@links[i]}" } if short.nil?
+      putx "---- javascript found [#{self.has_jlinks?.to_s} | #{self.jlinks.size}]"
+      self.jlinks.each { |url| putx url } if short.nil?
+      putx "---- forms found [#{self.has_form?.to_s} | #{self.forms.size}]"
+      putx "---- comments found [#{self.has_comments?.to_s}]"
+      return nil
+    end
+
+    alias_method :show, :report#:nodoc:
+
+    # IRB: display summary of what has been parsed from this page
+    def summary
+      status = self.page_status
+      putx "XXXX[#{self.report_flags}] | #{self.response_code.to_s} | #{status} | #{self.url} | #{self.size}"
+      return nil
+    end
+
+    # IRB: display current headers
+    def request_headers
+      self.headers.each_pair { |k,v| putx "#{k}: #{v}" }
+      return nil
+    end
+
+    alias_method :show_headers, :request_headers#:nodoc:
+    alias_method :req_headers, :request_headers#:nodoc:
+
+    # IRB: display response headers
+    def response_headers
+      self.header_data.each { |x| putx "#{x[0]} :: #{x[1]}" }
+      return nil
+    end
+
+    alias_method :resp_headers, :response_headers#:nodoc:
+
+    # display self.body_data
+    def dump_body
+      putx self.body_data
+    end
+
+    alias_method :dump, :dump_body#:nodoc:
+
+    # IRB: puts the page filtered through html2text
+    def to_text; putx self.html2text; end
+    def text; self.html2text; end
+
+    # IRB: display a human readable report of all forms contained in page.body_data
+    def all_forms
+      self.forms.each_index { |x| putx "[#{x.to_s}]-------"; self.forms[x].report }
+      nil
+    end
+
+    def onclicks
+      self.search("//*[@onclick]").each { |x| puts x[:onclick] }
+      nil
+    end
+
+    # hexdump self.body_data
+    def hexdump
+      puts self.body_data.hexdump
+    end
+
+    # this only works on a mac so get a mac
+    def open #:nodoc:
+      fn = "wwmdtmp_#{Guid.new}.html"
+      self.write(fn)
+      %x[open #{fn}]
+    end
+  end
+end