miketracy-wwmd 0.2.16 → 0.2.17

data/lib/wwmd/page/parsing_convenience.rb

@@ -12,8 +12,13 @@ module WWMD
     # return this page's form (at index id) as a FormArray
     def get_form(id=nil)
       id = 0 if not id
-      return nil if forms.empty?
-      @forms[id].to_form_array
+      return nil if forms.empty? || !forms[id]
+      f = @forms[id]
+      url_action = @urlparse.parse(self.cur,f.action).to_s
+      FormArray.new do |x|
+        x.set_fields(f.fields)
+        x.action = url_action
+      end
     end
 
     # return the complete url to the form action on this page
@@ -57,7 +62,7 @@ module WWMD
     end                                        
 
     def furl(url)
-      self.url = @urlparse.parse(self.opts[:base_url],url).to_s
+      self.url = @urlparse.parse(self.base_url,url).to_s
     end
 
     # set self.opts[:base_url]

data/lib/wwmd/page/scrape.rb

@@ -1,3 +1,5 @@
+# o hai!  I need your help.
+
 module WWMD
   LINKS_REGEXP = [
     /window\.open\s*\(([^\)]+)/i,
@@ -8,35 +10,21 @@ module WWMD
     /Ajax\.Request\s*\((['"][^'"]+['"])/i,
   ]
 
-  AJAX_REGEXP = [
-    /Ajax\.Request\s*\((['"][^'"]+['"])/i,
-  ]
-
-  SRC_REGEXP = [
-    /src=\s*(['"][^'"]+['"])/i
-  ]
-
-#  NOT_URL_CHAR = "[^0-9a-zA-Z\:\/\+\\-\%\#]"
-
   class Scrape
 
     attr_accessor :debug
     attr_accessor :warn
     attr_accessor :links  # links found on page
     attr_accessor :jlinks # links to javascript includes
-
     attr_reader :hdoc
 
-    @debug    = false
-    @warn     = true
-
     # create a new scrape object using passed HTML
     def initialize(page='<>')
       @page = page
       @hdoc = HDOC.parse(@page)
       @links = Array.new
       @debug = false
-      @warn  = true
+      @warn = false
     end
 
     # reset this scrape object (called by WWMD::Page)
@@ -93,10 +81,11 @@ module WWMD
       end
     end
 
-    # define an urls_from_helper method in your task specific script
-    def urls_from_helper
-      putw "WARN: Please set an urls_from_helper override in your helper script" if @warn
-      return nil
+    # return an array of Form objects for forms on page
+    def for_forms
+      ret = []
+      @hdoc.search("//form").each { |f| ret << Form.new(f) }
+      ret
     end
 
     # use xpath searches to get
@@ -198,5 +187,12 @@ module WWMD
     def warnings#:nodoc:
       return @warn
     end
+
+    # define an urls_from_helper method in your task specific script
+    def urls_from_helper
+      putw "WARN: Please set an urls_from_helper override in your helper script" if @warn
+      return nil
+    end
+
   end
 end

data/lib/wwmd/page/spider.rb

@@ -23,11 +23,11 @@ module WWMD
 
     # pass me opts and an array of regexps to ignore
     # we have a set of sane(ish) defaults here
-    def initialize(opts={},ignore=nil)
+    def initialize(opts={},ignore=nil,&block)
+      @block ||= block
       @opts    = opts
       @visited = []
       @queued  = []
-      @bypass  = []
       @local_only = true
       @csrf_token = nil
       if !opts[:spider_local_only].nil?
@@ -39,31 +39,30 @@ module WWMD
     # push an url onto the queue
     def push_url(url)
       return false if _check_ignore(url)
-      url = _de_csrf(url)
       if @local_only
         return false if !(url =~ /#{@opts[:base_url]}/)
       end
-      @bypass.each { |b| return true if (url =~ b) }
-      @queued.push(url) if (@visited.find { |v| v == url }.nil? and @queued.find { |q| q == url }.nil?)
-      return true
+      return false if (@visited.include?(url) or @queued.include?(url))
+      @queued.push(url)
+      true
     end
 
     # skip items in the queue
     def skip(tim=1)
       tim.times { |i| @queued.shift }
-      return true
+      true
     end
 
     # get the next url in the queue
     def get_next
-      return queued.shift
+      queued.shift
     end
 
     alias_method :next, :get_next
 
     # more elements in the queue?
     def next?
-      return !queued.empty?
+      !queued.empty?
     end
 
     # get the last ul we visited?  this doesn't look right
@@ -98,9 +97,10 @@ module WWMD
 
     # add url to queue
     def add(url='',links=[])
-      @visited.push(_de_csrf(url))
+      return nil if @visited.include?(url)
+      @visited.push(url)
       links.each { |l| self.push_url l }
-      return nil
+      nil
     end
 
     # set up the ignore list

data/lib/wwmd/urlparse.rb

@@ -1,6 +1,5 @@
 require 'htmlentities'
-require 'wwmd/mixins'
-require 'wwmd/mixins_extends'
+require 'wwmd/class_extensions'
 module WWMD
 
   # yay for experiments in re-inventing the wheel
@@ -37,7 +36,20 @@ module WWMD
         @script = @rpath.basename.clip
       end
       @script = "" if @script.nil?
-      @base_url = @proto + "://" + @location
+      begin
+        @base_url = @proto + "://" + @location
+      rescue => e
+        dgb = WWMD.debug
+        WWMD.debug = true
+        putd "ERROR in urlparse TRACE:"
+        pp *args
+        putd @proto
+        putd @base
+        putd @actual
+        putd @location
+        WWMD.debug = dbg
+        raise e
+      end
       @fqpath = @path + @script
       self
     end
@@ -71,8 +83,11 @@ module WWMD
     end
 
     def has_proto?
-      return true if HANDLERS.include?(@actual.split(":").first.downcase.to_sym)
-      return false
+      begin
+        return true if HANDLERS.include?(@actual.split(":").first.downcase.to_sym)
+      rescue
+        return false
+      end
     end
 
     def to_s

data/lib/wwmd/viewstate.rb

@@ -1,120 +1,17 @@
-require 'wwmd/viewstate/viewstate_utils'
-module WWMD
-  class ViewState < ViewStateUtils
-  end
-end
 require 'rubygems'
 require 'nokogiri'
 require 'rexml/document'
 require 'htmlentities'
-require 'wwmd/mixins'
-require 'wwmd/mixins_extends'
-require 'wwmd/mixins_external'
+require 'wwmd/viewstate/viewstate_utils'
+module WWMD
+  class ViewState
+    include ViewStateUtils
+  end
+end
+require 'wwmd/class_extensions'
+require 'wwmd/viewstate/viewstate'
 require 'wwmd/viewstate/viewstate_types'
-require 'wwmd/viewstate/viewstate_class_helpers'
 require 'wwmd/viewstate/viewstate_yaml'
 require 'wwmd/viewstate/viewstate_deserializer_methods'
 require 'wwmd/viewstate/viewstate_from_xml'
-require 'wwmd/viewstate/vs_read_value'
-require 'wwmd/viewstate/vs_read_types'
-require 'wwmd/viewstate/vs_value'
-require 'wwmd/viewstate/vs_array'
-require 'wwmd/viewstate/vs_binary_serialized'
-require 'wwmd/viewstate/vs_int_enum'
-require 'wwmd/viewstate/vs_hashtable'
-require 'wwmd/viewstate/vs_hybrid_dict'
-require 'wwmd/viewstate/vs_list'
-require 'wwmd/viewstate/vs_pair'
-require 'wwmd/viewstate/vs_sparse_array'
-require 'wwmd/viewstate/vs_string'
-require 'wwmd/viewstate/vs_string_array'
-require 'wwmd/viewstate/vs_string_formatted'
-require 'wwmd/viewstate/vs_triplet'
-require 'wwmd/viewstate/vs_type'
-require 'wwmd/viewstate/vs_unit'
-require 'wwmd/viewstate/vs_indexed_string'
-require 'wwmd/viewstate/vs_indexed_string_ref'
-module WWMD
-  class ViewState
-    attr_accessor :b64
-    attr_accessor :obj_queue
-    attr_accessor :mac
-    attr_accessor :debug
-    attr_reader   :raw
-    attr_reader   :stack
-    attr_reader   :bufarr
-    attr_reader   :magic
-    attr_reader   :size
-    attr_reader   :indexed_strings
-    attr_reader   :last_offset
-    attr_reader   :xml
-
-    def initialize(b64=nil)
-      @b64 = b64
-      @raw = ""
-      @stack = ""
-      @obj_queue = []
-      @bufarr = []
-      @size = 0
-      @indexed_strings = []
-      @mac = nil
-      @debug = false
-      self.deserialize if b64
-    end
-
-    # mac_enabled?
-    def mac_enabled?
-      return !@mac.nil?
-    end
-
-    # deserialize
-    def deserialize(b64=nil)
-      @obj_queue = []
-      @b64 = b64 if b64
-      @raw = @b64.b64d
-      @bufarr = @raw.scan(/./m)
-      @size = @bufarr.size
-      raise "Invalid ViewState" if not self.magic?
-      @obj_queue << self.deserialize_value
-      if @bufarr.size == 20 then
-        @mac = bufarr.slice!(0..19).join("")
-        dlog(0x00,"MAC = #{@mac.hexify}")
-      end
-      raise "Error Parsing Viewstate (left: #{@buffarr.size})" if not @bufarr.size == 0
-      return !self.raw.nil?
-    end
-
-    def serialize(objs=nil,version=2)
-      @obj_queue = objs if objs
-      @stack << "\xFF\x01"
-      @stack << @obj_queue.first.serialize
-      @stack << @mac if @mac
-      return !self.stack.nil?
-    end
-
-    def to_xml
-      @xml = REXML::Document.new()
-      header = REXML::Element.new("ViewState")
-      header.add_attribute("version", @magic.b64e)
-      header.add_attribute("version_string", @magic.hexify)
-      header.add_element(@obj_queue.first.to_xml)
-      if self.mac_enabled?
-        max = REXML::Element.new("Mac")
-        max.add_attribute("encoding","hexify")
-        max.add_text(@mac.hexify)
-        header.add_element(max)
-      end
-      @xml.add_element(header)
-      @xml
-    end
-
-    def from_yaml(yaml)
-      @obj_queue = YAML.load(yaml)
-    end
-
-    def to_yaml
-      @obj_queue.to_yaml
-    end
-
- end
-end
+require 'wwmd/viewstate/vs_stubs'

data/lib/wwmd/viewstate/viewstate.rb

@@ -0,0 +1,101 @@
+module WWMD
+  class ViewState
+    attr_accessor :b64
+    attr_accessor :obj_queue
+    attr_accessor :mac
+    attr_accessor :debug
+    attr_reader   :raw
+    attr_reader   :stack
+    attr_reader   :bufarr
+    attr_reader   :magic
+    attr_reader   :size
+    attr_reader   :indexed_strings
+    attr_reader   :last_offset
+    attr_reader   :xml
+    attr_reader   :ndoc
+    attr_reader   :obj_counts
+
+    def initialize(b64=nil)
+      @b64 = b64
+      @raw = ""
+      @stack = ""
+      @obj_queue = []
+      @bufarr = []
+      @size = 0
+      @indexed_strings = []
+      @mac = nil
+      @debug = false
+      @obj_counts = {}
+      self.deserialize if b64
+    end
+
+    # mac_enabled?
+    def mac_enabled?
+      return !@mac.nil?
+    end
+
+    # deserialize
+    def deserialize(b64=nil)
+      @obj_queue = []
+      @b64 = b64 if b64
+      @raw = @b64.b64d
+      @bufarr = @raw.scan(/./m)
+      @size = @bufarr.size
+      raise "Invalid ViewState" if not self.magic?
+      @obj_queue << self.deserialize_value
+      if @bufarr.size == 20 then
+        @mac = bufarr.slice!(0..19).join("")
+        dlog(0x00,"MAC = #{@mac.hexify}")
+      end
+      raise "Error Parsing Viewstate (left: #{@buffarr.size})" if not @bufarr.size == 0
+      return !self.raw.nil?
+    end
+
+    def serialize(objs=nil,version=2)
+      @obj_queue = objs if objs
+      @stack << "\xFF\x01"
+      @stack << @obj_queue.first.serialize
+      @stack << @mac if @mac
+      return !self.stack.nil?
+    end
+
+    def to_xml
+      @xml = REXML::Document.new()
+      header = REXML::Element.new("ViewState")
+      header.add_attribute("version", @magic.b64e)
+      header.add_attribute("version_string", @magic.hexify)
+      header.add_element(@obj_queue.first.to_xml)
+      if self.mac_enabled?
+        max = REXML::Element.new("Mac")
+        max.add_attribute("encoding","hexify")
+        max.add_text(@mac.hexify)
+        header.add_element(max)
+      end
+      @xml.add_element(header)
+      @ndoc = Nokogiri::XML.parse(@xml.to_s)
+      self
+    end
+
+    # xpath search the nokogiri doc if we have one
+    def search(*args)
+      return "No XML" if !@ndoc
+      @ndoc.search(*args)
+    end
+
+    # move pp due to to_xml returning self
+    # this is all for the sake of getting #search to work
+    def pp(*args)
+      return "Undefined" if !@xml
+      @xml.pp(*args)
+    end
+
+    def from_yaml(yaml)
+      @obj_queue = YAML.load(yaml)
+    end
+
+    def to_yaml
+      @obj_queue.to_yaml
+    end
+
+ end
+end

data/lib/wwmd/viewstate/viewstate_deserializer_methods.rb

@@ -3,39 +3,39 @@ module WWMD
     # complex types
     def pair(t=nil)
       dlog t,"next = #{next_type}"
-      VSPair.new(self.deserialize_value,self.deserialize_value)
+      VSStubs::VSPair.new(self.deserialize_value,self.deserialize_value)
     end
 
     def triplet(t=nil)
       dlog t,"next = #{next_type}"
-      VSTriplet.new(self.deserialize_value,self.deserialize_value,self.deserialize_value)
+      VSStubs::VSTriplet.new(self.deserialize_value,self.deserialize_value,self.deserialize_value)
     end
 
     def type(t=nil)
       typeref,typeval = self.deserialize_type
       dlog(t,"typeref = 0x#{typeref.to_s(16)} typeval = #{typeval}")
-      VSType.new(typeref,typeval)
+      VSStubs::VSType.new(typeref,typeval)
     end
 
     def string_formatted(t=nil)
       typeref,typeval = self.deserialize_type
       str = self.read_string
       dlog(t,"typeref = 0x#{typeref.to_s(16)} typeval = #{typeval} string = #{str}")
-      VSStringFormatted.new(typeref,typeval,str)
+      VSStubs::VSStringFormatted.new(typeref,typeval,str)
     end
 
     def int_enum(t=nil)
       typeref,typeval = self.deserialize_type
       index = self.read_7bit_encoded_int
       dlog(t,"typeref = 0x#{typeref.to_s(16)} typeval = #{typeval} index = #{index}")
-      VSIntEnum.new(typeref,typeval,index)
+      VSStubs::VSIntEnum.new(typeref,typeval,index)
     end
 
     def binary_serialized(t=nil)
       count = self.read_7bit_encoded_int
       dlog(t,count)
       bin = self.read(count)
-      me = VSBinarySerialized.new()
+      me = VSStubs::VSBinarySerialized.new()
       me.set(bin)
       return me
     end
@@ -45,7 +45,7 @@ module WWMD
       size  = read_7bit_encoded_int
       elems = read_7bit_encoded_int
       dlog(t,"typeref = 0x#{typeref.to_s(16)} typeval = #{typeval} size = #{size} elems = #{elems}")
-      me = VSSparseArray.new(typeref,typeval,size,elems)
+      me = VSStubs::VSSparseArray.new(typeref,typeval,size,elems)
       if elems > size
         raise "Invalid sparse_array"
       end
@@ -59,7 +59,7 @@ module WWMD
     def hashtable(t=nil)
       len = read_7bit_encoded_int
       dlog(t,"len = #{len}")
-      me = VSHashtable.new()
+      me = VSStubs::VSHashtable.new()
       (1..len).each do |i|
         me.add(self.deserialize_value,self.deserialize_value)
       end
@@ -69,7 +69,7 @@ module WWMD
     def hybrid_dict(t=nil)
       len = read_7bit_encoded_int
       dlog(t,"len = #{len}")
-      me = VSHybridDict.new()
+      me = VSStubs::VSHybridDict.new()
       (1..len).each do |i|
         me.add(self.deserialize_value,self.deserialize_value)
       end
@@ -80,7 +80,7 @@ module WWMD
       typeref,typeval = self.deserialize_type
       len = read_7bit_encoded_int
       dlog(t,"typeref = 0x#{typeref.to_s(16)} typeval = #{typeval} len = #{len}")
-      me = VSArray.new(typeref,typeval)
+      me = VSStubs::VSArray.new(typeref,typeval)
       (1..len).each do |i|
         me.add(self.deserialize_value)
       end
@@ -90,7 +90,7 @@ module WWMD
     def string_array(t=nil)
       len = read_7bit_encoded_int
       dlog(t,"len = #{len}")
-      me = VSStringArray.new()
+      me = VSStubs::VSStringArray.new()
       (1..len).each do |i|
         str = self.read_string
         me.add(str)
@@ -102,7 +102,7 @@ module WWMD
     def list(t=nil)
       len = read_7bit_encoded_int
       dlog(t,"len = #{len}")
-      me = VSList.new()
+      me = VSStubs::VSList.new()
       (1..len).each do |i|
         me.add(self.deserialize_value)
       end
@@ -113,99 +113,99 @@ module WWMD
       s1 = read_double
       s2 = read_int32
       dlog(t,"#{s1.to_s(16).rjust(16,"0")},#{s2.to_s(16).rjust(8,"0")}")
-      VSUnit.new(s1,s2)
+      VSStubs::VSUnit.new(s1,s2)
     end
 
     def indexed_string(t=nil)
       str = self.read_string
       @indexed_strings << str
       dlog(t,"idx = #{@indexed_strings.size - 1} val = #{str}")
-      VSIndexedString.new(str)
+      VSStubs::VSIndexedString.new(str)
     end
 
     def indexed_string_ref(t=nil)
       ref = self.read_int
       dlog(t,"ref = #{ref} val = #{@indexed_strings[ref]}")
-      VSIndexedStringRef.new(ref)
+      VSStubs::VSIndexedStringRef.new(ref)
     end
 
     def string(t=nil)
       str = self.read_string
       dlog(t,str)
-      VSString.new(str)
+      VSStubs::VSString.new(str)
     end
 
-    # VSReadValue types
+    # VSStubs::VSReadValue types
     def color(t=nil)
       val = self.read_int32
       dlog(t,val.to_s(16))
-      VSColor.new(val)
+      VSStubs::VSColor.new(val)
     end
 
     def known_color(t=nil)
       index = self.read_7bit_encoded_int
       dlog(t,"index = #{index.to_s(16)}")
-      VSKnownColor.new(index)
+      VSStubs::VSKnownColor.new(index)
     end
 
     def int16(t=nil)
       val = read_short
       dlog(t,val)
-      VSInt16.new(val)
+      VSStubs::VSInt16.new(val)
     end
 
     def int32(t=nil)
       val = self.read_7bit_encoded_int
       dlog(t,val)
-      VSInt32.new(val)
+      VSStubs::VSInt32.new(val)
     end
 
     def byte(t=nil)
       val = self.read_byte
       dlog(t,val)
-      VSByte.new(val)
+      VSStubs::VSByte.new(val)
     end
 
     def char(t=nil)
       val = self.read_byte
       dlog(t,val)
-      VSChar.new(val)
+      VSStubs::VSChar.new(val)
     end
 
     def date_time(t=nil)
       val = self.read_double
       dlog(t,val)
-      VSDateTime.new(val)
+      VSStubs::VSDateTime.new(val)
     end
 
     def double(t=nil)
       val = self.read_double
       dlog(t,val)
-      VSDouble.new(val)
+      VSStubs::VSDouble.new(val)
     end
 
     def single(t=nil)
       val = self.read_single
       dlog(t,val)
-      VSSingle.new(val)
+      VSStubs::VSSingle.new(val)
   end
 
-    # VSValue types
-    def null(t=nil);        dlog(t,nil); return VSValue.new(t); end
-    def empty_byte(t=nil);  dlog(t,nil); return VSValue.new(t); end
-    def zeroint32(t=nil);   dlog(t,nil); return VSValue.new(t); end
-    def bool_true(t=nil);   dlog(t,nil); return VSValue.new(t); end
-    def bool_false(t=nil);  dlog(t,nil); return VSValue.new(t); end
-    def empty_color(t=nil); dlog(t,nil); return VSValue.new(t); end
-    def empty_unit(t=nil);  dlog(t,nil); return VSValue.new(t); end
+    # VSStubs::VSValue types
+    def null(t=nil);        dlog(t,nil); return VSStubs::VSValue.new(t); end
+    def empty_byte(t=nil);  dlog(t,nil); return VSStubs::VSValue.new(t); end
+    def zeroint32(t=nil);   dlog(t,nil); return VSStubs::VSValue.new(t); end
+    def bool_true(t=nil);   dlog(t,nil); return VSStubs::VSValue.new(t); end
+    def bool_false(t=nil);  dlog(t,nil); return VSStubs::VSValue.new(t); end
+    def empty_color(t=nil); dlog(t,nil); return VSStubs::VSValue.new(t); end
+    def empty_unit(t=nil);  dlog(t,nil); return VSStubs::VSValue.new(t); end
 
     # deserialize_value
     def deserialize_value
       @last_offset = self.offset
       token = self.read_byte # self.read_raw_byte
       if not (tsym = VIEWSTATE_TYPES[token])
-        puts "TOKEN: [0x#{token.to_s(16)}] at #{last_offset}"
-        puts @bufarr.slice(0..31).join("").hexdump
+        STDERR.puts "TOKEN: [0x#{token.to_s(16)}] at #{last_offset}"
+        STDERR.puts @bufarr.slice(0..31).join("").hexdump
         raise "Invalid Type [0x#{token.to_s(16)}] at #{last_offset}" if not (tsym = VIEWSTATE_TYPES[token])
       end
       nobj = self.send(tsym,token)