stratagem 0.2.3 → 0.2.4

data/lib/stratagem/crawler/route_invoker.rb

@@ -15,9 +15,7 @@ module Stratagem::Crawler
       begin
         call_route!(route_info, track_invocations)
       rescue Exception
-        # TODO - add exception as a response page
-        puts "ERROR: #{$!.message}"
-        puts $!.backtrace
+        Stratagem.logger.error($!)
       end
     end
 
@@ -31,8 +29,8 @@ module Stratagem::Crawler
       invocations = model_invocations_for_request do
         case verb
           when 'get'
+            puts "\t#{route_info[:path].to_s == '/connections'} - #{route_info[:path]}"
             do_get(route_info)
-            puts "\tresponse code: #{response.code}" if response
           when 'post'
             do_post(route_info)
           when 'put'
@@ -43,20 +41,20 @@ module Stratagem::Crawler
         end
       end
 
-      if (response)
-        if (track_invocations)
-          changes = detect_attribute_changes_in_models(invocations)
+      if (response && track_invocations)
+        changes = detect_attribute_changes_in_models(invocations)
+        if (![500].include?(response.code) || (changes.size > 0) || (invocations.size > 0))
           puts "\tfound #{invocations.size} invocations"
           invocations.each do |i|
             puts "\t\t#{i.controller_action} -> #{i.model_class}"
           end
-          
-          puts "\tchanges values: #{changes.values.inspect}" if changes.size > 0
-          site_model.add(route_info[:route_container], controller, request, response, invocations, changes) {|redirect_url| redirect_proc.call(redirect_url) }
+          puts "\tchanges values: #{changes.values.inspect}"
+          page = site_model.add(route_info[:route_container], controller, request, response, invocations, changes) {|redirect_url| redirect_proc.call(redirect_url) }
+          puts "\tresponse code: #{response.code} - body size: #{page.response_body.size}"
           puts "\tadded to site model"
         end
       else
-        puts "ERROR: did not call #{route_info.inspect}"
+        puts "ERROR: did not call #{route_info[:path]}"
       end
     end
 
@@ -169,7 +167,6 @@ module Stratagem::Crawler
     
     def guess_form_for_route(route_info)
       forms = []
-      pages = site_models.map {|sm| sm.pages }.flatten
       site_models.each do |site_model|
         site_model.pages.each do |page|
           page.forms.each do |form|
@@ -178,7 +175,7 @@ module Stratagem::Crawler
               (route_info[:path].sub(/\.\(\:format\)*/, '') == form.action) &&
               ((form.implied_method || form.method).to_s.downcase == route_info[:verb].to_s.downcase)
             end
-            puts "\tform: #{form.action} - #{form.implied_method || form.method} - #{usable}" 
+            puts "\tpageset: #{site_model.name} - form: #{form.action} - #{form.implied_method || form.method} - #{usable} - #{form.inputs.map{|i| i.name}}" 
             forms << form if (usable)
           end
         end

data/lib/stratagem/crawler/session.rb

@@ -56,6 +56,7 @@ module Stratagem::Crawler::Session
   end
   
   def crawler_session(application_model=nil)
+    aquifer # force fill
     @application_model = application_model if application_model
     open_session do |session|
       @session = session
@@ -68,7 +69,16 @@ module Stratagem::Crawler::Session
   end
   
   def aquifer(initial_capacity=6)
-    @aquifer ||= Stratagem::AutoMock::Aquifer.init(application_model).fill(initial_capacity)
+    @aquifer ||= begin
+      Stratagem.logger.phase "mocking_models"
+
+      if Stratagem.configuration.use_automatic_mocking
+        Stratagem::AutoMock::Aquifer.init(application_model).fill_by_automock(initial_capacity)
+      else
+        # TODO - refactor out into another method
+        Stratagem::AutoMock::Aquifer.init(application_model).fill_by_configuration(Stratagem.configuration.credentials)
+      end
+    end
   end
   
   def site_models
@@ -81,11 +91,13 @@ module Stratagem::Crawler::Session
   end
 
   def page_set(name, &block)
+    Stratagem.logger.phase "traversing_site"
+    
     log "---------------------------------------"
     log "Crawling page set #{name}"
     log "---------------------------------------"
     reset!
-    site_models << Stratagem::Crawler::SiteModel.new(name)
+    site_models << Stratagem::Crawler::SiteModel::PageSet.new(name)
     yield site_model
   end
 

data/lib/stratagem/crawler/site_model.rb

@@ -1,174 +1,5 @@
-module Stratagem::Crawler
-  class SiteModel
-    include Stratagem::Crawler::HtmlUtils
+module Stratagem::Crawler::SiteModel; end
 
-    attr_reader :pages, :edges, :name
-    attr_accessor :authentication
-
-    def initialize(name)
-      @name = name
-      @pages = []
-      @edges = []
-    end
-
-    def export
-      {
-        :name => name,
-        :pages => @pages.map {|page| page.export },
-        :edges => @edges.map {|edge| edge.export },
-        :authentication => authentication.nil? ? nil : {
-          :authenticated_user_id => authentication.authenticated_with.object_id,
-          :success => authentication.success, 
-          :login_page_external_id => authentication.login_page.object_id, 
-          :response_page_external_id => authentication.response_page.object_id,
-          :ssl => authentication.ssl
-        },
-      }
-    end
-
-    def add_edge(from,to,type)
-      self.edges << Edge.new(from,to,type)
-    end
-
-    def add(route, controller, request, response, invocations=[], model_changes={}, &block)
-      page = Page.new(self, controller, request, response, invocations, model_changes, &block)
-      self.pages << page
-      page
-    end
-
-    def pages_for(id)
-      if (id.kind_of?(String))
-        pages.select {|page| page.url == id }
-      else
-        pages.select {|page| page.route == id }
-      end
-    end
-
-  end
-
-  class Edge
-    # type -> :link, :redirect,
-    attr_accessor :from, :to, :type
-
-    def initialize(from, to, type)
-      @from = from
-      @to = to
-      @type = type
-    end
-    
-    def export
-      {
-        :from => from.object_id,
-        :to => to.object_id,
-        :relation_type => type
-      }
-    end
-  end
-
-  class Page
-    include Stratagem::Crawler::HtmlUtils
-
-    attr_reader :response
-    
-    attr_accessor :url
-    attr_accessor :path
-    attr_accessor :method
-    attr_accessor :redirected_to
-    attr_accessor :document
-    attr_accessor :response_body
-
-    def initialize(site_model, controller, request, response, invocations, model_changes, &block)
-      @site_model = site_model
-      @invocations = invocations
-      @model_changes = model_changes
-      @authenticity_checked = (controller && controller.methods.include?(:authenticity_checked?)) ? controller.authenticity_checked? : true
-      init(request, response, &block)
-    end
-
-    def route
-      @route ||= Stratagem::Model::Application.instance.routes.recognize(self)
-    end
-
-    def export
-      h = {
-        :external_id => self.object_id,
-        :url => url,
-        :path => path,
-        :request_method => method,
-        :redirected_to_page_external_id => redirected_to ? redirected_to.object_id : nil,
-        :route_external_id => route ? route.object_id : nil,
-        :references => @invocations.map {|i| i.to_reference.export },
-        :model_changes => Hash[@model_changes.map {|model,changes| [model.object_id, changes] }].to_json,
-        :authenticity_checked => @authenticity_checked,
-        :parameters => @request.parameters.to_json,
-        :response_body => @response_body
-      }
-      h
-    end
-
-    def init(request, response, &block)
-      @request = request.clone
-      @response = response.clone
-      @url = request.url
-      @path = request.path
-      @method = request.method
-      begin
-        @document = Nokogiri::HTML(response.body)
-      rescue
-        puts "ERROR: Could not parse html: #{$!.message} - #{response.body}"
-      end
-      @response_body = response.body
-      self.redirected_to = block.call(response.redirect_url) if response.redirect?
-    end
-
-    def reload(&block)
-      # TODO - should support all the verbs and params, but
-      # hack together for now to reload the authenticity token
-      request,response = yield url
-      init(request, response) {|redirected_to| }
-    end
-
-    def redirected?
-      !self.redirected_to.nil?
-    end
-
-    def forms
-      @forms ||= begin
-        if (@document)
-          forms = self.parse_forms(@document)
-          forms.each do |form|
-            form.page = self
-          end
-          forms
-        else
-          []
-        end
-      end
-    end
-
-    def login_form
-      self.find_login_form(@document)
-    end
-
-    def to_html
-      @document.to_html
-    end
-
-    def outbound_edges(type=nil)
-      @site_model.edges.select {|edge| edge.from == self && (type.nil? || (type == edge.type)) }
-    end
-
-    def inbound_edges(type=nil)
-      @site_model.edges.select {|edge| (edge.to == self) && (type.nil? || (type == edge.type)) }
-    end
-
-    def title
-      if ((@document) && !(@title))
-        title = (@document/'head title').first
-        @title = title.inner_html if title
-      end
-      @title
-    end
-  end
-
-end
+require 'stratagem/crawler/site_model/page_set'
+require 'stratagem/crawler/site_model/page'
+require 'stratagem/crawler/site_model/edge'

data/lib/stratagem/crawler/site_model/edge.rb

@@ -0,0 +1,20 @@
+module Stratagem::Crawler::SiteModel
+  class Edge
+    # type -> :link, :redirect,
+    attr_accessor :from, :to, :type
+
+    def initialize(from, to, type)
+      @from = from
+      @to = to
+      @type = type
+    end
+  
+    def export
+      {
+        :from_page_external_id => from.object_id,
+        :to_page_external_id => to.object_id,
+        :relation_type => type
+      }
+    end
+  end
+end

data/lib/stratagem/crawler/site_model/page.rb

@@ -0,0 +1,121 @@
+module Stratagem::Crawler::SiteModel
+
+  class Page
+    include Stratagem::Crawler::HtmlUtils
+
+    attr_reader :response
+  
+    attr_accessor :url
+    attr_accessor :path
+    attr_accessor :method
+    attr_accessor :redirected_to
+    attr_accessor :document
+    attr_accessor :response_body
+
+    def initialize(site_model, controller, request, response, invocations, model_changes, &block)
+      @site_model = site_model
+      @invocations = invocations
+      @model_changes = model_changes
+      @authenticity_checked = (controller && controller.methods.include?(:authenticity_checked?)) ? controller.authenticity_checked? : true
+      init(request, response, &block)
+    end
+
+    def route
+      @route ||= Stratagem::Model::Application.instance.routes.recognize(self)
+    end
+
+    def export
+      h = {
+        :external_id => self.object_id,
+        :url => url,
+        :path => path,
+        :request_method => method,
+        :redirected_to_page_external_id => redirected_to ? redirected_to.object_id : nil,
+        :route_external_id => route ? route.object_id : nil,
+        :references_attributes => @invocations.map {|i| i.to_reference.export },
+        :model_changes => Hash[@model_changes.map {|model,changes| [model.object_id, changes] }].to_json,
+        :authenticity_checked => @authenticity_checked,
+        :parameters => @request.parameters.to_json,
+        :response_body => @response_body,
+        :response_code => @response.code
+      }
+      h
+    end
+
+    def init(request, response, &block)
+      @request = request.clone
+      @response = response.clone
+      @url = request.url
+      @path = request.path
+      @method = request.method
+      body = response.body
+      begin
+        if [500, 400, 302].include?(response.code)
+          body = '' 
+        elsif 
+          body.kind_of?(Array)
+          body = body.join
+          body = body.slice(0,20000)
+        end
+        
+        @document = Nokogiri::HTML(body)
+      rescue
+        puts "ERROR: Could not parse html: #{$!.message} - #{body}"
+      end
+    
+      @response_body = body
+    
+      self.redirected_to = block.call(response.redirect_url) if response.redirect?
+    end
+
+    def reload(&block)
+      # TODO - should support all the verbs and params, but
+      # hack together for now to reload the authenticity token
+      request,response = yield url
+      init(request, response) {|redirected_to| }
+    end
+
+    def redirected?
+      !self.redirected_to.nil?
+    end
+
+    def forms
+      @forms ||= begin
+        if (@document)
+          forms = self.parse_forms(@document)
+          forms.each do |form|
+            form.page = self
+          end
+          forms
+        else
+          []
+        end
+      end
+    end
+
+    def login_form
+      self.find_login_form(@document)
+    end
+
+    def to_html
+      @document.to_html
+    end
+
+    def outbound_edges(type=nil)
+      @site_model.edges.select {|edge| edge.from == self && (type.nil? || (type == edge.type)) }
+    end
+
+    def inbound_edges(type=nil)
+      @site_model.edges.select {|edge| (edge.to == self) && (type.nil? || (type == edge.type)) }
+    end
+
+    def title
+      if ((@document) && !(@title))
+        title = (@document/'head title').first
+        @title = title.inner_html if title
+      end
+      @title
+    end
+  end
+
+end

data/lib/stratagem/crawler/site_model/page_set.rb

@@ -0,0 +1,58 @@
+module Stratagem::Crawler::SiteModel
+
+  class PageSet
+    include Stratagem::Crawler::HtmlUtils
+
+    attr_reader :pages, :edges, :name
+    attr_accessor :authentication
+
+    def initialize(name)
+      @name = name
+      @pages = []
+      @edges = []
+    end
+
+    def export
+      puts "Exporting page set #{name} with #{@pages.size} initial pages"
+      pages = @pages.select {|page| page.response_body && page.response_body.size > 0 }
+      puts "\t#{pages.size} filtered pages"
+      pages = pages.map {|page| page.export }
+
+      pages << authentication.login_page.export if authentication && authentication.login_page
+      pages << authentication.response_page.export if authentication && authentication.response_page
+      
+      puts "\t#{pages.size} exported pages"
+      {
+        :name => name,
+        :pages_attributes => pages,
+        :edges_attributes => @edges.map {|edge| edge.export },
+        :authentication_attributes => authentication.nil? ? nil : {
+          :authenticated_user_id => authentication.authenticated_with.object_id,
+          :success => authentication.success, 
+          :login_page_external_id => authentication.login_page ? authentication.login_page.object_id : nil, 
+          :response_page_external_id => authentication.response_page ? authentication.response_page.object_id : nil,
+          :ssl => authentication.ssl
+        },
+      }
+    end
+
+    def add_edge(from,to,type)
+      self.edges << Edge.new(from,to,type)
+    end
+
+    def add(route, controller, request, response, invocations=[], model_changes={}, &block)
+      page = Page.new(self, controller, request, response, invocations, model_changes, &block)
+      self.pages << page
+      page
+    end
+
+    def pages_for(id)
+      if (id.kind_of?(String))
+        pages.select {|page| page.url == id }
+      else
+        pages.select {|page| page.route == id }
+      end
+    end
+
+  end
+end