miketracy-wwmd 0.2.11

data/lib/wwmd/page/scrape.rb

@@ -0,0 +1,202 @@
+module WWMD
+  LINKS_REGEXP = [
+    /window\.open\s*\(([^\)]+)/i,
+    /open_window\s*\(([^\)]+)/i,
+    /window\.location\s*=\s*(['"][^'"]+['"])/i,
+    /.*location.href\s*=\s*(['"][^'"]+['"])/i,
+    /document.forms.*action\s*=\s*(['"][^'"]+['"])/i,
+    /Ajax\.Request\s*\((['"][^'"]+['"])/i,
+  ]
+
+  AJAX_REGEXP = [
+    /Ajax\.Request\s*\((['"][^'"]+['"])/i,
+  ]
+
+  SRC_REGEXP = [
+    /src=\s*(['"][^'"]+['"])/i
+  ]
+
+#  NOT_URL_CHAR = "[^0-9a-zA-Z\:\/\+\\-\%\#]"
+
+  class Scrape
+
+    attr_accessor :debug
+    attr_accessor :warn
+    attr_accessor :links  # links found on page
+    attr_accessor :jlinks # links to javascript includes
+
+    attr_reader :hdoc
+
+    @debug    = false
+    @warn     = true
+
+    # create a new scrape object using passed HTML
+    def initialize(page='<>')
+      @page = page
+      @hdoc = HDOC.parse(@page)
+      @links = Array.new
+      @debug = false
+      @warn  = true
+    end
+
+    # reset this scrape object (called by WWMD::Page)
+    def reset(page)
+      @page = page
+      @hdoc = HDOC.parse(@page)
+      @links = Array.new
+    end
+
+    # scan the passed string for the configured regular expressions
+    # and return them as an array
+    def urls_from_regexp(content,re,split=0)
+      ret = []
+      scrape = content.scan(re)
+      scrape.each do |url|
+        # cheat and take split string(,)[split]
+        add = url.to_s.split(',')[split].gsub(/['"]/, '')
+        next if (add == '' || add.nil?)
+        ret << add
+      end
+      return ret
+    end
+
+    # xpath search for tags and return the passed attribute
+    #  urls_from_xpath("//a","href")
+    def urls_from_xpath(xpath,attr)
+      ret = []
+      @hdoc.search(xpath).each do |elem|
+        url = elem[attr]
+        next if url.empty?
+        ret << url.strip
+      end
+      return ret
+    end
+
+    # <b>NEED</b> to move this to external configuration
+    #
+    # list of urls we don't care to store in our links list
+    def reject_links
+      putw "WARN: override reject_links in helper script" if @warn
+      default_reject_links
+    end
+
+    # default reject links (override using reject_links in helper script)
+    def default_reject_links
+      @links.reject! do |url|
+        url.nil? ||
+        url.extname == ".css" ||
+        url.extname == ".pdf" ||
+        url =~ /javascript:/i ||
+        url =~ /mailto:/i ||
+        url =~ /[\[\]]/ ||
+        url =~ /^#/
+      end
+    end
+
+    # define an urls_from_helper method in your task specific script
+    def urls_from_helper
+      putw "WARN: Please set an urls_from_helper override in your helper script" if @warn
+      return nil
+    end
+
+    # use xpath searches to get
+    # * //a href
+    # * //area href
+    # * //frame src
+    # * //iframe src
+    # * //form action
+    # * //meta refresh content urls
+    # then get //script tags and regexp out links in javascript function calls
+    # from elem.inner_html
+    def for_links(reject=true)
+      self.urls_from_xpath("//a","href").each { |url| @links << url };      # get <a href=""> elements
+      self.urls_from_xpath("//area","href").each { |url| @links << url };   # get <area href=""> elements
+      self.urls_from_xpath("//frame","src").each { |url| @links << url };   # get <frame src=""> elements
+      self.urls_from_xpath("//iframe","src").each { |url| @links << url };  # get <iframe src=""> elements
+      self.urls_from_xpath("//form","action").each { |url| @links << url }; # get <form action=""> elements
+
+      # <meta> refresh
+      @hdoc.search("//meta").each do |meta|
+        next if meta['http-equiv'] != "refresh"
+        next if not (content = meta['content'].split(/=/)[1])
+        @links << content.strip
+      end
+
+      # add urls from onclick handlers
+      @hdoc.search("*[@onclick]").each do |onclick|
+        LINKS_REGEXP.each do |re|
+          self.urls_from_regexp(onclick['onclick'],re).each do |url|
+            @links << url
+          end
+        end
+      end
+
+      # add urls_from_regexp (limit to <script> tags (elem.inner_html))
+      @hdoc.search("//script").each do |scr|
+        LINKS_REGEXP.each do |re|
+          self.urls_from_regexp(scr.inner_html,re).each { |url| @links << url }
+        end
+      end
+
+      # re-define urls_from_helper in what you mix in
+      begin
+        self.urls_from_helper
+      end
+
+      self.reject_links; # reject links we don't care about
+      return @links
+    end
+
+    # scrape the page for <script src=""> tags
+    def for_javascript_links
+      urls = []
+      @hdoc.search("//script[@src]").each { |tag| urls << tag['src'] }
+      urls.reject! { |url| File.extname(url).clip != ".js" }
+      return urls
+    end
+
+    # scan page for comment fields
+    def for_comments
+      @page.scan(/\<!\s*--(.*?)--\s*\>/m).map { |x| x.to_s }
+    end
+
+    # scrape the page for a meta refresh tag and return the url from the contents attribute or nil
+    def for_meta_refresh
+      has_mr = @hdoc.search("//meta").map { |x| x['http-equiv'] }.include?('Refresh')
+      if has_mr
+        urls = @hdoc.search("//meta[@content]").map { |x| x['content'].split(";",2)[1] }
+        if urls.size > 1
+          STDERR.puts "PARSE ERROR: more than one meta refresh tag"
+          return "ERR"
+        end
+        k,v = urls.first.split("=",2)
+        if k.upcase.strip != "URL"
+          STDERR.puts "PARSE ERROR: content attribute of meta refresh does not contain url"
+          return "ERR"
+        end
+        return v.strip
+      else
+        return nil
+      end
+    end
+
+    # scrape the page for a script tag that contains a bare location.href tag (to redirect the page)
+    def for_javascript_redirect
+      redirs = []
+      @hdoc.search("//script").each do |scr|
+        scr.inner_html.scan(/.*location.href\s*=\s*['"]([^'"]+)['"]/i).each { |x| redirs += x }
+      end
+      if redirs.size > 1
+        STDERR.puts "PARSE ERROR: more than one javascript redirect"
+        return "ERR"
+      end
+      return redirs.first if not redirs.empty?
+      return nil
+    end
+
+    # renamed class variable (for backward compat)
+    def warnings#:nodoc:
+      return @warn
+    end
+  end
+end

data/lib/wwmd/page/spider.rb

@@ -0,0 +1,127 @@
+module WWMD
+  # when a WWMD::Page object is created, it created its own WWMD::Spider object
+  # which can be accessed using <tt>page.spider.method</tt>.  The <tt>page.set_data</tt>
+  # method calls <tt>page.spider.add</tt> with the current url and a list of scraped
+  # links from the page.  This class doesn't do any real heavy lifting.
+  #
+  # a simple spider can be written just by recursing through page.spider.next until
+  # it's empty.  
+  class Spider
+
+    attr_accessor :queued
+    attr_accessor :visited
+    attr_accessor :bypass
+    attr_accessor :local_only
+    attr_reader   :opts
+    attr_accessor :ignore
+    attr_accessor :csrf_token
+
+    DEFAULT_IGNORE = [
+      /logoff/i,
+      /logout/i,
+    ]
+
+    # pass me opts and an array of regexps to ignore
+    # we have a set of sane(ish) defaults here
+    def initialize(opts={},ignore=nil)
+      @opts    = opts
+      @visited = []
+      @queued  = []
+      @bypass  = []
+      @local_only = true
+      @csrf_token = nil
+      if !opts[:spider_local_only].nil?
+        @local_only = opts[:spider_local_only]
+      end
+      @ignore = ignore || DEFAULT_IGNORE
+    end
+
+    # push an url onto the queue
+    def push_url(url)
+      return false if _check_ignore(url)
+      url = _de_csrf(url)
+      if @local_only
+        return false if !(url =~ /#{@opts[:base_url]}/)
+      end
+      @bypass.each { |b| return true if (url =~ b) }
+      @queued.push(url) if (@visited.find { |v| v == url }.nil? and @queued.find { |q| q == url }.nil?)
+      return true
+    end
+
+    # skip items in the queue
+    def skip(tim=1)
+      tim.times { |i| @queued.shift }
+      return true
+    end
+
+    # get the next url in the queue
+    def get_next
+      return queued.shift
+    end
+
+    alias_method :next, :get_next
+
+    # more elements in the queue?
+    def next?
+      return !queued.empty?
+    end
+
+    # get the last ul we visited?  this doesn't look right
+    def get_last(url)
+      tmp =  @visited.reject { |v| v =~ /#{url}/ }
+      return tmp[-1]
+    end
+
+    # show the visited list (or the entry in the list at [id])
+    def show_visited(id=nil)
+      if id.nil?
+        @visited.each_index { |i| putx i.to_s + " :: " + @visited[i].to_s }
+        return nil
+      else
+        return @visited[id]
+      end
+    end
+
+    alias_method :v, :show_visited
+
+    # return the current queue (or the entry in the queue at [id]
+    def show_queue(id=nil)
+      if id.nil?
+        @queued.each_index { |i| putx i.to_s + " :: " + @queued[i].to_s }
+        return nil
+      else
+        return @queued[id]
+      end
+    end
+
+    alias_method :q, :show_queue
+
+    # add url to queue
+    def add(url='',links=[])
+      @visited.push(_de_csrf(url))
+      links.each { |l| self.push_url l }
+      return nil
+    end
+
+    # set up the ignore list
+    # ignore list is an array of regexp objects
+    # remember to set this up before calling any Page methods
+    def set_ignore(arr)
+      @ignore = arr
+    end
+
+    def _de_csrf(url)
+      return url if @csrf_token.nil?
+      act,params = url.clopa
+      form = params.to_form
+      return url if !form.has_key?(@csrf_token)
+      form[@csrf_token] = ''
+      url = act + form.to_get
+    end
+
+    def _check_ignore(url)
+      @ignore.each { |x| return true if (url =~ x) }
+      return false
+    end
+  end
+end

data/lib/wwmd/page/urlparse.rb

@@ -0,0 +1,79 @@
+module WWMD
+  # yay for experiments in re-inventing the wheel
+  class URLParse
+    HANDLERS = [:https,:http,:ftp,:file]
+    attr_reader :proto,:location,:path,:script,:rpath,:params,:base_url,:fqpath
+
+    def initialize()
+      # nothing to see here, move along
+    end
+
+    def parse(*args)
+      if args.size == 1
+        base = ""
+        actual = args.shift
+      else
+        base = args.shift
+        actual = args.shift
+      end
+      @proto = @location = @path = @script = @rpath = nil
+      @base = base.to_s
+      @actual = actual
+      if self.has_proto?
+        @base = @actual
+        @actual = ""
+      end
+# does this work for http://location/?  probably not
+      @base += "/" if (!@base.has_ext? || @base.split("/").size == 3)
+      @rpath = make_me_path.join("/")
+      @params = @rpath.clop
+      @path = "/" + @rpath
+      if @rpath.has_ext?
+        @path = "/" + @rpath.dirname
+        @script = @rpath.basename.clip
+      end
+      @script = "" if @script.nil?
+      @base_url = @proto + "://" + @location
+      @fqpath = @path + @script
+      self
+    end
+
+    def make_me_path
+      @proto,tpath = @base.split(":",2)
+      tpath ||= ""
+      if @actual.empty?
+        a_path = tpath.split("/").reject { |x| x.empty? }
+      else
+        a_path = tpath.dirname.split("/").reject { |x| x.empty? }
+      end
+      @location = a_path.shift
+      a_path = [] if (@actual =~ (/^\//))
+      b_path = @actual.split("/").reject { |x| x.empty? }
+      a_path.pop if (a_path[-1] =~ /^\?/).kind_of?(Fixnum) && !b_path.empty?
+      c_path = (a_path + @actual.split("/").reject { |x| x.empty? }).flatten
+      d_path = []
+      c_path.each do |x|
+        (d_path.pop;next) if x == ".."
+        next if x == "."
+        d_path << x
+      end
+      return d_path
+    end
+
+    def has_proto?
+      return true if HANDLERS.include?(@actual.split(":").first.downcase.to_sym)
+      return false
+    end
+
+    def to_s
+      return "#{@proto}://#{@location}/#{rpath}"
+    end
+  end
+end
+
+class String
+  def has_ext? #:nodoc:
+    return false if self.basename.split(".",2)[1].empty?
+    return true
+  end
+end

data/lib/wwmd/page/utils.rb

@@ -0,0 +1,30 @@
+module WWMD
+  class WWMDUtils
+
+    def self.header_array_from_file(filename)
+      ret = Hash.new
+      File.readlines(filename).each do |line|
+        a = line.chomp.split(/\t/,2)
+        ret[a[0]] = a[1]
+      end
+      return ret
+    end
+
+    def self.ranstr(len=8,digits=false)
+      chars = ("a".."z").to_a
+      chars += ("0".."9").to_a if digits
+      ret = ""
+      1.upto(len) { |i| ret << chars[rand(chars.size-1)] }
+      return ret
+    end
+
+    def self.rannum(len=8,hex=false)
+      chars = ("0".."9").to_a
+      chars += ("A".."F").to_a if hex
+      ret = ""
+      1.upto(len) { |i| ret << chars[rand(chars.size-1)] }
+      return ret
+    end
+
+  end
+end

data/lib/wwmd/viewstate.rb

@@ -0,0 +1,118 @@
+require 'wwmd/viewstate/viewstate_utils'
+module WWMD
+  class ViewState < ViewStateUtils
+  end
+end
+require 'rubygems'
+require 'nokogiri'
+require 'htmlentities'
+require 'rexml/document'
+require 'wwmd/mixins'
+require 'wwmd/mixins_extends'
+require 'wwmd/viewstate/viewstate_types'
+require 'wwmd/viewstate/viewstate_class_helpers'
+require 'wwmd/viewstate/viewstate_yaml'
+require 'wwmd/viewstate/viewstate_deserializer_methods'
+require 'wwmd/viewstate/viewstate_from_xml'
+require 'wwmd/viewstate/vs_read_value'
+require 'wwmd/viewstate/vs_read_types'
+require 'wwmd/viewstate/vs_value'
+require 'wwmd/viewstate/vs_array'
+require 'wwmd/viewstate/vs_binary_serialized'
+require 'wwmd/viewstate/vs_int_enum'
+require 'wwmd/viewstate/vs_hashtable'
+require 'wwmd/viewstate/vs_hybrid_dict'
+require 'wwmd/viewstate/vs_list'
+require 'wwmd/viewstate/vs_pair'
+require 'wwmd/viewstate/vs_sparse_array'
+require 'wwmd/viewstate/vs_string'
+require 'wwmd/viewstate/vs_string_array'
+require 'wwmd/viewstate/vs_string_formatted'
+require 'wwmd/viewstate/vs_triplet'
+require 'wwmd/viewstate/vs_type'
+require 'wwmd/viewstate/vs_unit'
+require 'wwmd/viewstate/vs_indexed_string'
+require 'wwmd/viewstate/vs_indexed_string_ref'
+module WWMD
+  class ViewState
+    attr_accessor :b64
+    attr_accessor :obj_queue
+    attr_accessor :mac
+    attr_accessor :debug
+    attr_reader   :raw
+    attr_reader   :stack
+    attr_reader   :bufarr
+    attr_reader   :magic
+    attr_reader   :size
+    attr_reader   :indexed_strings
+    attr_reader   :last_offset
+    attr_reader   :xml
+
+    def initialize(b64=nil)
+      @b64 = b64
+      @raw = ""
+      @stack = ""
+      @obj_queue = []
+      @bufarr = []
+      @size = 0
+      @indexed_strings = []
+      @mac = nil
+      @debug = false
+    end
+
+    # mac_enabled?
+    def mac_enabled?
+      return !@mac.nil?
+    end
+
+    # deserialize
+    def deserialize(b64=nil)
+      @obj_queue = []
+      @b64 = b64 if b64
+      @raw = @b64.b64d
+      @bufarr = @raw.scan(/./m)
+      @size = @bufarr.size
+      raise "Invalid ViewState" if not self.magic?
+      @obj_queue << self.deserialize_value
+      if @bufarr.size == 20 then
+        @mac = bufarr.slice!(0..19).join("")
+        dlog(0x00,"MAC = #{@mac.hexify}")
+      end
+      raise "Error Parsing Viewstate (left: #{@buffarr.size})" if not @bufarr.size == 0
+      return !self.raw.nil?
+    end
+
+    def serialize(objs=nil,version=2)
+      @obj_queue = objs if objs
+      @stack << "\xFF\x01"
+      @stack << @obj_queue.first.serialize
+      @stack << @mac if @mac
+      return !self.stack.nil?
+    end
+
+    def to_xml
+      @xml = REXML::Document.new()
+      header = REXML::Element.new("ViewState")
+      header.add_attribute("version", @magic.b64e)
+      header.add_attribute("version_string", @magic.hexify)
+      header.add_element(@obj_queue.first.to_xml)
+      if self.mac_enabled?
+        max = REXML::Element.new("Mac")
+        max.add_attribute("encoding","hexify")
+        max.add_text(@mac.hexify)
+        header.add_element(max)
+      end
+      @xml.add_element(header)
+      @xml
+    end
+
+    def from_yaml(yaml)
+      @obj_queue = YAML.load(yaml)
+    end
+
+    def to_yaml
+      @obj_queue.to_yaml
+    end
+
+ end
+end