stratagem 0.2.0 → 0.2.2

data/lib/stratagem/client.rb

@@ -9,7 +9,7 @@ module Stratagem
     def send(snapshot)
       Stratagem.logger.debug "Sending report to server"
       url = URI.parse("#{@authentication.base_url}/snapshots")
-      req = Net::HTTPS::Post.new(url.path)
+      req = Stratagem.ssl? ? Net::HTTPS::Post.new(url.path) : Net::HTTP::Post.new(url.path)
       
       req.set_form_data({
         'auth_token' => @authentication.credentials[:token],
@@ -17,7 +17,8 @@ module Stratagem
         'timestamp' => snapshot.timestamp.to_i,
         'model' => snapshot.model.export.to_json
       }, ';')
-      res = Net::HTTPS.new(url.host, url.port).start {|http| http.request(req) }
+      client = Stratagem.ssl? ? Net::HTTPS.new(url.host, url.port) : Net::HTTP.new(url.host, url.port)
+      res = client.start {|http| http.request(req) }
       puts "response:"
       case res
       when Net::HTTPSuccess, Net::HTTPRedirection

data/lib/stratagem/crawler/authentication.rb

@@ -75,9 +75,15 @@ module Stratagem::Crawler
       if authentication.login_page.nil?
         puts "locating login page"
         puts "testing #{site_models.first.pages.size} pages"
-        site_models.first.pages.sort {|a,b| b.inbound_edges(:redirect).size <=> a.inbound_edges(:redirect).size }.each do |page|
-          if (page.login_form)
-            puts "\tfound login form - #{page.login_form}"
+        possibilities = site_models.first.pages.select {|page| page.login_form != nil }
+        possibilities.sort! {|a,b| b.inbound_edges(:redirect).size <=> a.inbound_edges(:redirect).size }
+
+        # if the first page has one or more redirects to it then use it; otherwise select with page with the fewest inputs
+        if (possibilities.first.inbound_edges(:redirect).size > 0)
+          return possibilities.first
+        else
+          page = possibilities.sort {|a,b| a.login_form.inputs.size <=> b.login_form.inputs.size }.first
+          if (page)
             authentication.login_page = page
             return page
           end
@@ -94,6 +100,7 @@ module Stratagem::Crawler
 
     def login(user)
       populate_login_form(user).submit {|action,params| 
+        p params
         post(action, params)
       }
     end

data/lib/stratagem/crawler/form.rb

@@ -1,7 +1,7 @@
 # Primarily used to fill out login forms rather than trying to fudge the before_filters
 module Stratagem::Crawler
   class Form
-    attr_accessor :action, :method, :fields, :buttons, :page
+    attr_accessor :action, :method, :buttons, :page
     attr_reader :inputs, :buttons
 
     def initialize
@@ -70,7 +70,7 @@ module Stratagem::Crawler
 
     def guess_alternate_attribute
       if (name =~ /(.*)\[(.*)\]/)
-        $1.to_sym
+        $2.to_sym
       else
         name.to_sym
       end

data/lib/stratagem/crawler/html_utils.rb

@@ -13,12 +13,22 @@ module Stratagem::Crawler
         # this maps to the form action, not the controller action
         form.action =~ /log[-]*in/ ||
         form.action =~ /sign[-]*in/ ||
-        !form.inputs.find {|input| input.type == 'password' }.nil?
+        begin
+          password_field = form.inputs.find {|input| input.type == 'password' }
+          if (password_field)
+            confirmation_field = form.inputs.find {|input| input.name.include?('_confirmation') && (input.name.to_s.sub('_confirmation','') == password_field.name) }
+            confirmation_field.nil?
+          else
+            false
+          end
+        end
       }.sort {|a,b| a.inputs.size <=> b.inputs.size }
       possibilities.first
     end
 
     def parse_forms(document)
+      return [] if document.nil?
+      
       document.xpath('//form').map do |form_tag|
         form = Form.new()
         form.action = form_load_attribute(form_tag, 'action')
@@ -49,6 +59,7 @@ module Stratagem::Crawler
         value = (value ? value.value : option_tag.inner_html).strip
         input << value unless value.empty?
       end
+      form << input
     end
 
     def form_add_input(form, input_tag)

data/lib/stratagem/crawler/parameter_resolver.rb

@@ -10,7 +10,7 @@ module Stratagem::Crawler
       if (unknown_params.size > 0)
         resolve_with_convention(unknown_params, resolved_params)
         resolve_with_instrumentation(route_container, resolved_params)
-
+        resolve_id_with_convention(route_container, unknown_params, resolved_params)
         log "\tresolved parameter types - #{resolved_params.inspect}"
       end
       
@@ -67,13 +67,27 @@ module Stratagem::Crawler
             unknown_params.delete(param)
             puts "\t\tresolved #{param} to #{model} using convention"
           rescue NameError
-            puts $!
-            puts $!.backtrace
-            # not a model
+            puts "ERROR: #{$!.message}"
           end
         end
       end
     end
 
+    # resolve the ID parameter by looking at the previous segment of the URL
+    def resolve_id_with_convention(route_container, unknown_params, resolved_params)
+      if (unknown_params.include?('id'))
+        route_container.path.split('/').inject(nil) {|prev,part|
+          if (prev && part == ':id')
+            begin
+              resolved_params['id'] = prev.singularize.camelize.constantize
+              unknown_params.delete('id')
+            rescue
+              puts $!.message
+            end
+          end
+          part
+        }
+      end
+    end
   end
 end

data/lib/stratagem/crawler/route_invoker.rb

@@ -14,16 +14,17 @@ module Stratagem::Crawler
     def call_route(route_info, track_invocations=true)
       begin
         call_route!(route_info, track_invocations)
-      rescue
+      rescue Exception
         # TODO - add exception as a response page
         puts "ERROR: #{$!.message}"
+        puts $!.backtrace
       end
     end
 
     def call_route!(route_info, track_invocations=true)
       return if route_info.nil?
 
-      puts route_info[:verb].downcase+" "+route_info[:path]
+      puts route_info[:verb].downcase+" "+route_info[:path]+" - #{route_info[:route_container].path}"
       verb = route_info[:verb].downcase
       verb = 'get' if verb == '' || verb == 'any'
 
@@ -33,6 +34,7 @@ module Stratagem::Crawler
             do_get(route_info)
             puts "\tresponse code: #{response.code}" if response
           when 'post'
+            do_post(route_info)
           when 'put'
             do_put(route_info)
           when 'delete'
@@ -48,8 +50,10 @@ module Stratagem::Crawler
           invocations.each do |i|
             puts "\t\t#{i.controller_action} -> #{i.model_class}"
           end
-          puts "\tchanges: #{changes.values.inspect}" if changes.size > 0
+          
+          puts "\tchanges values: #{changes.values.inspect}" if changes.size > 0
           site_model.add(route_info[:route_container], controller, request, response, invocations, changes) {|redirect_url| redirect_proc.call(redirect_url) }
+          puts "\tadded to site model"
         end
       else
         puts "ERROR: did not call #{route_info.inspect}"
@@ -61,9 +65,12 @@ module Stratagem::Crawler
     end
 
     def do_put(route_info)
-      raise "unable to invoke PUT requests, application must first be crawled with GET requests for phase #{phase}." unless site_model.pages.size > 0
+      # raise "unable to invoke PUT requests, application must first be crawled with GET requests for phase #{site_model.name}." unless site_model.pages.size > 0
 
       form = guess_form_for_route(route_info)
+      if (form.nil?)
+        puts "WARNING: could not locate form for route #{route_info[:route_container].path}"
+      end
       
       params = {}
 
@@ -77,15 +84,11 @@ module Stratagem::Crawler
         end
       end
       
-      p hash_reads
-      
       # let's find out what the method is looking for in the params object
       models_by_hash_key = infer_models_for_param_reads(route_info[:route_container],hash_reads)
       params = map_models_to_attributes(models_by_hash_key)
       
-      p params
-      
-      guess_unknown_params(models_by_hash_key, params, form)
+      guess_unknown_params(models_by_hash_key, params, form) if form
 
       # run again with the params
       puts "PUTTING: #{route_info[:path]} with #{params.inspect}"
@@ -95,13 +98,47 @@ module Stratagem::Crawler
       end
     end
 
+    def do_post(route_info)
+      raise "unable to invoke PUT requests, application must first be crawled with GET requests for phase #{phase}." unless site_model.pages.size > 0
+
+      form = guess_form_for_route(route_info)
+      if (form.nil?)
+        puts "WARNING: could not locate form for route #{route_info[:route_container].path}"
+      end
+      
+      params = {}
+
+      # note: this should fail to generate anything meaningful, as we have not yet set up the parameters
+      hash_reads = Hash.track_parameter_reads do
+        begin
+          post route_info[:path], params
+        rescue
+          # TODO - log error as page response
+          puts "ERROR: #{response.code}"
+        end
+      end
+      
+      # let's find out what the method is looking for in the params object
+      models_by_hash_key = infer_models_for_param_reads(route_info[:route_container],hash_reads)
+      params = map_models_to_attributes(models_by_hash_key)
+      
+      guess_unknown_params(models_by_hash_key, params, form) if form
+
+      # run again with the params
+      puts "POSTING: #{route_info[:path]} with #{params.inspect}"
+
+      invocation_delta = model_invocations_for_request(:write) do
+        post route_info[:path], params
+      end
+    end
+
     private
     
     def guess_unknown_params(models_by_hash_key, known_params, form)
       (form.parameter_keys - IGNORE_PARAMETERS) .each {|path_s|
         path = path_s.split('[')
         path.last.gsub!(']', '')
-        value = known_params
+        value = known_params || {}
         model = nil
         path.each do |key|
           key = key.to_sym
@@ -137,8 +174,11 @@ module Stratagem::Crawler
         site_model.pages.each do |page|
           page.forms.each do |form|
             usable = route_info[:route_container].responds_to?(form.action, form.implied_method || form.method)
-            # puts "\t#{form.action} - #{form.method} - #{form.implied_method}" 
-            # puts "\tUSABLE" if usable
+            usable ||= begin
+              (route_info[:path].sub(/\.\(\:format\)*/, '') == form.action) &&
+              ((form.implied_method || form.method).to_s.downcase == route_info[:verb].to_s.downcase)
+            end
+            puts "\tform: #{form.action} - #{form.implied_method || form.method} - #{usable}" 
             forms << form if (usable)
           end
         end
@@ -183,9 +223,12 @@ module Stratagem::Crawler
         if (invocation.model_instance)
           model = application_model.models.find {|m| m.klass == invocation.model_instance.class }
 
-          puts "\t\t#{invocation.model_class}.#{invocation.method} - #{invocation.model_instance}"
+          puts "\t\t#{invocation.model_class}.#{invocation.method} - #{invocation.model_instance} - #{invocation.model_instance.id}"
+          ids = aquifer.instances_of(invocation.model_instance.class).map {|i| i.id }
+          puts "\t\tinstances: #{ids.inspect}"
           prior = aquifer.instances_of(invocation.model_instance.class).find {|m| m.id == invocation.model_instance.id }
           post = invocation.model_instance
+          prior ||= post
           attribute_names = (prior.stratagem.attribute_names + prior.stratagem.foreign_keys)
           changes[model] =  attribute_names.select {|an|
             begin
@@ -216,7 +259,7 @@ module Stratagem::Crawler
         yield path
       else
         combinations = segment_values[0].product(*segment_values.slice(1,segment_values.size-1))
-        combinations.each do |combination|
+        combinations.uniq.each do |combination|
           url = path.clone
           route.segment_keys.each_with_index {|segment_key,i| url.gsub!(':'+segment_key.to_s, combination[i].to_s) }
           yield url
@@ -232,7 +275,7 @@ module Stratagem::Crawler
 
 
       i = 12345
-      insert_values = (route.segment_keys - [:format]).map {|segment_key|
+      insert_values = (route_container.segment_keys - [:format]).map {|segment_key|
         model = parameter_types[segment_key.to_s]
         value = nil
         if (model)
@@ -250,7 +293,6 @@ module Stratagem::Crawler
         permutation = {:verb => verb, :path => path, :route_container => route_container}
         routes << permutation
       end
-      
       [routes, params]
     end
 

data/lib/stratagem/crawler/session.rb

@@ -91,21 +91,29 @@ module Stratagem::Crawler::Session
 
   def print
     # print out pages and inbound / outbound links
+    # debugger
     site_model.pages.each do |page|
       title = page.title
       title ||= page.redirected_to.url if page.redirected_to
       title ||= page.response.code
       log "Page: #{page.url} - #{title} - #{page.response.code}"
-      page.outbound_edges.each do |edge|
-        log "\tout: #{edge.to.url} - #{edge.to.title} - #{edge.to.route}"
-      end
-      page.inbound_edges.each do |edge|
-        log "\tin: #{edge.from.url} - #{edge.from.title}"
+      begin
+        page.outbound_edges.each do |edge|
+         log "\tout: #{edge.to.url} - #{edge.to.title} - #{edge.to.route}"
+        end
+        page.inbound_edges.each do |edge|
+         log "\tin: #{edge.from.url} - #{edge.from.title}"
+        end
+      rescue Exception
+        puts $!.message
+        puts $!.backtrace
       end
     end
+    puts "end of site model"
   end
 
   def crawl(verbs=[:any,:get])
+    puts "crawling site for verbs #{verbs.inspect}"
     verbs = [verbs] unless verbs.kind_of?(Array)
     
     # grab all pages independently

data/lib/stratagem/crawler/site_model.rb

@@ -17,6 +17,7 @@ module Stratagem::Crawler
         :pages => @pages.map {|page| page.export },
         :edges => @edges.map {|edge| edge.export },
         :authentication => authentication.nil? ? nil : {
+          :authenticated_user_id => authentication.authenticated_with.object_id,
           :success => authentication.success, 
           :login_page_external_id => authentication.login_page.object_id, 
           :response_page_external_id => authentication.response_page.object_id,
@@ -74,12 +75,13 @@ module Stratagem::Crawler
     attr_accessor :method
     attr_accessor :redirected_to
     attr_accessor :document
+    attr_accessor :response_body
 
     def initialize(site_model, controller, request, response, invocations, model_changes, &block)
       @site_model = site_model
       @invocations = invocations
       @model_changes = model_changes
-      @authenticity_checked = controller.authenticity_checked?
+      @authenticity_checked = (controller && controller.methods.include?(:authenticity_checked?)) ? controller.authenticity_checked? : true
       init(request, response, &block)
     end
 
@@ -98,7 +100,8 @@ module Stratagem::Crawler
         :references => @invocations.map {|i| i.to_reference.export },
         :model_changes => Hash[@model_changes.map {|model,changes| [model.object_id, changes] }].to_json,
         :authenticity_checked => @authenticity_checked,
-        :parameters => @request.parameters.to_json
+        :parameters => @request.parameters.to_json,
+        :response_body => @response_body
       }
       h
     end
@@ -109,7 +112,12 @@ module Stratagem::Crawler
       @url = request.url
       @path = request.path
       @method = request.method
-      @document = Nokogiri::HTML(response.body)
+      begin
+        @document = Nokogiri::HTML(response.body)
+      rescue
+        puts "ERROR: Could not parse html: #{$!.message} - #{response.body}"
+      end
+      @response_body = response.body
       self.redirected_to = block.call(response.redirect_url) if response.redirect?
     end
 
@@ -126,11 +134,15 @@ module Stratagem::Crawler
 
     def forms
       @forms ||= begin
-        forms = self.parse_forms(@document)
-        forms.each do |form|
-          form.page = self
+        if (@document)
+          forms = self.parse_forms(@document)
+          forms.each do |form|
+            form.page = self
+          end
+          forms
+        else
+          []
         end
-        forms
       end
     end
 
@@ -151,7 +163,7 @@ module Stratagem::Crawler
     end
 
     def title
-      unless @title
+      if ((@document) && !(@title))
         title = (@document/'head title').first
         @title = title.inner_html if title
       end

data/lib/stratagem/extensions/object.rb

@@ -2,8 +2,8 @@ class Object
   def methods_include?(name)
     methods.include?(name.to_sym) || methods.include?(name.to_s)
   end
-
-  def self.subclasses
+  
+  def self.sg_subclasses
     classes = []
     ObjectSpace.each_object(Class) do |c|
       next unless c.superclass == self || c.ancestors.include?(self)

data/lib/stratagem/extensions/string.rb

@@ -7,8 +7,8 @@ class String
     match = false
     [
       Regexp.compile("^#{token}$", true),
-      Regexp.compile("^#{token}[^A-Za-z0-9]", true),
-      Regexp.compile("[^A-Za-z0-9]#{token}$", true),
+      Regexp.compile("^#{token}[^A-Za-z0-9_\-]*", true),
+      Regexp.compile("[^A-Za-z0-9_\-]*#{token}$", true),
     ].each do |regex|
       if (regex.match(self))
         match = true

data/lib/stratagem/instrumentation.rb

@@ -0,0 +1,18 @@
+module Stratagem::Instrumentation; end
+
+require 'stratagem/instrumentation/rails'
+require 'stratagem/instrumentation/method_invocation'
+require 'stratagem/instrumentation/models'
+require 'stratagem/instrumentation/request_forgery_protection'
+
+if (Stratagem.rails_3?)
+  require 'stratagem/instrumentation/rails3/parameters'
+elsif (Stratagem.rails_2?)
+  require 'stratagem/instrumentation/rails2/action_controller'
+  require 'stratagem/instrumentation/rails2/action_mailer'
+else
+  raise "Unsupported Rails version #{Stratagem.rails_version}"
+end
+
+
+