wwmd 0.2.20.3

data/lib/wwmd/page/parsing_convenience.rb

@@ -0,0 +1,98 @@
+module WWMD
+  class Page
+#:section: Parsing convenience methods
+# methods that help parse and find information on a page including
+# access to forms etc.
+
+    # grep for regexp and remove leading whitespace
+    def grep(reg)
+      self.body_data.grep(reg).map { |i| i.gsub(/^\s+/, "") }
+    end
+
+    # return this page's form (at index id) as a FormArray
+    def get_form(id=nil)
+      id = 0 if not id
+      return nil if forms.empty? || !forms[id]
+      f = @forms[id]
+      action = f.action
+      action ||= action
+      action ||= cur
+      action ||= "PARSE_ERROR"
+      url_action = @urlparse.parse(self.cur,action).to_s
+      type = f.type
+      FormArray.new do |x|
+        x.set_fields(f.fields)
+        x.action = url_action
+        x.type   = type
+      end
+    end
+
+    # return the complete url to the form action on this page
+    def action(id=nil)
+      id = 0 if not id
+      act = self.forms[id].action
+      return self.last_effective_url if (act.nil? || act.empty?)
+      return @urlparse.parse(self.last_effective_url,act).to_s
+    end
+
+    # return an array of Element objects for an xpath search
+    def search(xpath)
+      self.scrape.hdoc.search(xpath)
+    end
+
+    # return an array of inner_html for each <script> tag encountered
+    def dump_scripts
+      self.get_tags("//script").map { |s| s.inner_html if s.inner_html.strip != '' }
+    end
+
+    alias_method :scripts, :dump_scripts
+
+    # set link using an integer link from self.report
+    #--
+    # NOTE: I always use page.get(page.l(1)) anyway.
+    #++
+    def set_link(index)
+      self.url = @links[index]
+    end
+
+    # return link at index from @links array
+    def get_link(index)
+      @links[index]
+    end
+
+    alias_method :link, :get_link #:nodoc:
+    alias_method :l, :get_link #:nodoc:
+
+    def all_tags#:nodoc:
+      return self.search("*").map { |x| x.name }
+    end                                        
+
+    def furl(url)
+      self.url = @urlparse.parse(self.base_url,url).to_s
+    end
+
+    # set self.opts[:base_url]
+    def setbase(url=nil)
+      return nil if not url
+      self.opts[:base_url] = url
+      self.base_url = url
+    end
+
+    # write self.body_data to file
+    def write(filename)
+      File.write(filename,self.body_data)
+      return "wrote to " + filename
+    end
+
+    # read self.body_data from file
+    def read(filename)
+      self.body_data = File.read(filename)
+      self.set_data
+    end
+
+    # alias_method for body_data 
+    def raw 
+      self.body_data
+    end
+  end
+end

data/lib/wwmd/page/reporting_helpers.rb

@@ -0,0 +1,89 @@
+module WWMD
+  class Page
+    attr_accessor :status
+#:section: Reporting helper methods
+# These are methods that generate data for a parsed page
+
+    # return text representation of page code
+    #
+    # override with specific statuses in helper depending on page text
+    # etc to include statuses outside 200 = OK and other = ERR
+    def page_status
+      @status = "OK"
+      @status = "ERR" if self.response_code > 399
+    end
+
+#    alias_method :status, :page_status#:nodoc:
+
+    # return value of @logged_in
+    def logged_in?
+      return @logged_in
+    end
+
+    # return a string of flags:
+    # Ll links
+    # Jj javascript includes
+    # Ff forms
+    # Cc comments
+    def report_flags
+      self.has_links?      ? ret  = "L" : ret  = "l"
+      self.has_jlinks?     ? ret += "J" : ret += "j"
+      self.has_form?       ? ret += "F" : ret += "f"
+      self.has_comments?   ? ret += "C" : ret += "c"
+      return ret
+    end
+
+    def has_links?;    return !@links.empty?;     end
+    def has_jlinks?;   return !@jlinks.empty?;    end
+    def has_form?;     return !(@forms.size < 1); end
+    def has_comments?; return !@comments.empty?;  end
+
+    # return page size in bytes
+    def size
+      return self.body_data.size
+    end
+
+    # return md5sum for self.body_data
+    def md5
+      return self.body_data.md5
+    end
+
+    # does this response have SET-COOKIE headers?
+    def set_cookies?
+      ret = FormArray.new()
+      self.header_data.each do |x|
+        if x[0].upcase == "SET-COOKIE"
+          ret << x[1].split(";").first.split("=",2)
+        end
+      end
+      ret
+    end
+    alias_method :set_cookies, :set_cookies?
+
+    def time
+      self.total_time
+    end
+
+    # return MD5 for DOM fingerprint
+    # take all tag names in page.to_s.md5
+    def fingerprint
+      self.all_tags.to_s.md5
+    end
+    alias_method :fp, :fingerprint #:nodoc:
+
+    # alias_method for last_effective_url
+    def current_url
+      self.last_effective_url
+    end
+
+    alias_method :current, :current_url
+    alias_method :cur, :current_url
+    alias_method :now, :current_url
+
+    # the last http response code
+    def code
+      self.response_code # .to_s
+    end
+
+  end
+end

data/lib/wwmd/page/scrape.rb

@@ -0,0 +1,196 @@
+module WWMD
+  LINKS_REGEXP = [
+    /window\.open\s*\(([^\)]+)/i,
+    /open_window\s*\(([^\)]+)/i,
+    /window\.location\s*=\s*(['"][^'"]+['"])/i,
+    /.*location.href\s*=\s*(['"][^'"]+['"])/i,
+    /document.forms.*action\s*=\s*(['"][^'"]+['"])/i,
+    /Ajax\.Request\s*\((['"][^'"]+['"])/i,
+  ]
+
+  class Scrape
+
+    attr_accessor :debug
+    attr_accessor :warn
+    attr_accessor :links  # links found on page
+    attr_accessor :jlinks # links to javascript includes
+    attr_reader :hdoc
+
+    # create a new scrape object using passed HTML
+    def initialize(page='<>')
+      @page = page
+      @hdoc = HDOC.parse(@page)
+      @links = Array.new
+      @debug = false
+      @warn = false
+    end
+
+    # reset this scrape object (called by WWMD::Page)
+    def reset(page)
+      @page = page
+      @hdoc = HDOC.parse(@page)
+      @links = Array.new
+    end
+
+    # scan the passed string for the configured regular expressions
+    # and return them as an array
+    def urls_from_regexp(content,re,split=0)
+      ret = []
+      scrape = content.scan(re)
+      scrape.each do |url|
+        # cheat and take split string(,)[split]
+        add = url.to_s.split(',')[split].gsub(/['"]/, '')
+        next if (add == '' || add.nil?)
+        ret << add
+      end
+      return ret
+    end
+
+    # xpath search for tags and return the passed attribute
+    #  urls_from_xpath("//a","href")
+    def urls_from_xpath(xpath,attr)
+      ret = []
+      @hdoc.search(xpath).each do |elem|
+        url = elem[attr]
+        next if url.empty?
+        ret << url.strip
+      end
+      return ret
+    end
+
+    # <b>NEED</b> to move this to external configuration
+    #
+    # list of urls we don't care to store in our links list
+    def reject_links
+      putw "WARN: override reject_links in helper script" if @warn
+      default_reject_links
+    end
+
+    # default reject links (override using reject_links in helper script)
+    def default_reject_links
+      @links.reject! do |url|
+        url.nil? ||
+        url.extname == ".css" ||
+        url.extname == ".pdf" ||
+        url =~ /javascript:/i ||
+        url =~ /mailto:/i ||
+        url =~ /[\[\]]/ ||
+        url =~ /^#/
+      end
+    end
+
+    # return an array of Form objects for forms on page
+    def for_forms
+      ret = []
+      @hdoc.search("//form").each { |f| ret << Form.new(f) }
+      ret
+    end
+
+    # use xpath searches to get
+    # * //a href
+    # * //area href
+    # * //frame src
+    # * //iframe src
+    # * //form action
+    # * //meta refresh content urls
+    # then get //script tags and regexp out links in javascript function calls
+    # from elem.inner_html
+    def for_links(reject=true)
+      self.urls_from_xpath("//a","href").each      { |url| @links << url } # get <a href=""> elements
+      self.urls_from_xpath("//area","href").each   { |url| @links << url } # get <area href=""> elements
+      self.urls_from_xpath("//frame","src").each   { |url| @links << url } # get <frame src=""> elements
+      self.urls_from_xpath("//iframe","src").each  { |url| @links << url } # get <iframe src=""> elements
+      self.urls_from_xpath("//form","action").each { |url| @links << url } # get <form action=""> elements
+
+      # <meta> refresh
+      @hdoc.search("//meta").each do |meta|
+        next if meta['http-equiv'] != "refresh"
+        next if not (content = meta['content'].split(/=/)[1])
+        @links << content.strip
+      end
+
+      # add urls from onclick handlers
+      @hdoc.search("*[@onclick]").each do |onclick|
+        LINKS_REGEXP.each do |re|
+          self.urls_from_regexp(onclick['onclick'],re).each do |url|
+            @links << url
+          end
+        end
+      end
+
+      # add urls_from_regexp (limit to <script> tags (elem.inner_html))
+      @hdoc.search("//script").each do |scr|
+        LINKS_REGEXP.each do |re|
+          self.urls_from_regexp(scr.inner_html,re).each { |url| @links << url }
+        end
+      end
+
+      # re-define urls_from_helper in what you mix in
+      begin
+        self.urls_from_helper
+      end
+
+      self.reject_links; # reject links we don't care about
+      return @links
+    end
+
+    # scrape the page for <script src=""> tags
+    def for_javascript_links
+      urls = []
+      @hdoc.search("//script[@src]").each { |tag| urls << tag['src'] }
+      urls.reject! { |url| File.extname(url).clip != ".js" }
+      return urls
+    end
+
+    # scan page for comment fields
+    def for_comments
+      @page.scan(/\<!\s*--(.*?)--\s*\>/m).map { |x| x.to_s }
+    end
+
+    # scrape the page for a meta refresh tag and return the url from the contents attribute or nil
+    def for_meta_refresh
+      has_mr = @hdoc.search("//meta").map { |x| x['http-equiv'] }.include?('Refresh')
+      if has_mr
+        urls = @hdoc.search("//meta[@content]").map { |x| x['content'].split(";",2)[1] }
+        if urls.size > 1
+          STDERR.puts "PARSE ERROR: more than one meta refresh tag"
+          return "ERR"
+        end
+        k,v = urls.first.split("=",2)
+        if k.upcase.strip != "URL"
+          STDERR.puts "PARSE ERROR: content attribute of meta refresh does not contain url"
+          return "ERR"
+        end
+        return v.strip
+      else
+        return nil
+      end
+    end
+
+    # scrape the page for a script tag that contains a bare location.href tag (to redirect the page)
+    def for_javascript_redirect
+      redirs = []
+      @hdoc.search("//script").each do |scr|
+        scr.inner_html.scan(/.*location.href\s*=\s*['"]([^'"]+)['"]/i).each { |x| redirs += x }
+      end
+      if redirs.size > 1
+        STDERR.puts "PARSE ERROR: more than one javascript redirect"
+        return "ERR"
+      end
+      return redirs.first if not redirs.empty?
+      return nil
+    end
+
+    # renamed class variable (for backward compat)
+    def warnings#:nodoc:
+      return @warn
+    end
+
+    # define an urls_from_helper method in your task specific script
+    def urls_from_helper
+      putw "WARN: Please set an urls_from_helper override in your helper script" if @warn
+      return nil
+    end
+
+  end
+end

data/lib/wwmd/page/spider.rb

@@ -0,0 +1,127 @@
+module WWMD
+  # when a WWMD::Page object is created, it created its own WWMD::Spider object
+  # which can be accessed using <tt>page.spider.method</tt>.  The <tt>page.set_data</tt>
+  # method calls <tt>page.spider.add</tt> with the current url and a list of scraped
+  # links from the page.  This class doesn't do any real heavy lifting.
+  #
+  # a simple spider can be written just by recursing through page.spider.next until
+  # it's empty.  
+  class Spider
+
+    attr_accessor :queued
+    attr_accessor :visited
+    attr_accessor :bypass
+    attr_accessor :local_only
+    attr_reader   :opts
+    attr_accessor :ignore
+    attr_accessor :csrf_token
+
+    DEFAULT_IGNORE = [
+      /logoff/i,
+      /logout/i,
+    ]
+
+    # pass me opts and an array of regexps to ignore
+    # we have a set of sane(ish) defaults here
+    def initialize(opts={},ignore=nil)
+      @opts    = opts
+      @visited = []
+      @queued  = []
+      @local_only = true
+      @csrf_token = nil
+      if !opts[:spider_local_only].nil?
+        @local_only = opts[:spider_local_only]
+      end
+      @ignore = ignore || DEFAULT_IGNORE
+    end
+
+    # push an url onto the queue
+    def push_url(url)
+      return false if _check_ignore(url)
+      if @local_only
+        return false if !(url =~ /#{@opts[:base_url]}/)
+      end
+      return false if (@visited.include?(url) or @queued.include?(url))
+      @queued.push(url)
+      true
+    end
+    alias_method :push, :push_url
+
+    # skip items in the queue
+    def skip(tim=1)
+      tim.times { |i| @queued.shift }
+      true
+    end
+
+    # get the next url in the queue
+    def get_next
+      queued.shift
+    end
+
+    alias_method :next, :get_next
+
+    # more elements in the queue?
+    def next?
+      !queued.empty?
+    end
+
+    # get the last ul we visited?  this doesn't look right
+    def get_last(url)
+      tmp =  @visited.reject { |v| v =~ /#{url}/ }
+      return tmp[-1]
+    end
+
+    # show the visited list (or the entry in the list at [id])
+    def show_visited(id=nil)
+      if id.nil?
+        @visited.each_index { |i| putx i.to_s + " :: " + @visited[i].to_s }
+        return nil
+      else
+        return @visited[id]
+      end
+    end
+
+    alias_method :v, :show_visited
+
+    # return the current queue (or the entry in the queue at [id]
+    def show_queue(id=nil)
+      if id.nil?
+        @queued.each_index { |i| putx i.to_s + " :: " + @queued[i].to_s }
+        return nil
+      else
+        return @queued[id]
+      end
+    end
+
+    alias_method :q, :show_queue
+
+    # add url to queue
+    def add(url='',links=[])
+      return nil if @visited.include?(url)
+      @visited.push(url)
+      links.each { |l| self.push_url l }
+      nil
+    end
+
+    # set up the ignore list
+    # ignore list is an array of regexp objects
+    # remember to set this up before calling any Page methods
+    def set_ignore(arr)
+      @ignore = arr
+    end
+
+    def _de_csrf(url)
+      return url if @csrf_token.nil?
+      act,params = url.clopa
+      form = params.to_form
+      return url if !form.has_key?(@csrf_token)
+      form[@csrf_token] = ''
+      url = act + form.to_get
+    end
+
+    def _check_ignore(url)
+      @ignore.each { |x| return true if (url =~ x) }
+      return false
+    end
+  end
+end