stratagem 0.1.8 → 0.1.9

data/lib/stratagem/auto_mock/factory.rb

@@ -51,12 +51,12 @@ module Stratagem::AutoMock
       return [object,valid] if valid
     end
 
-    protected
-
     def add_mocked(instance)
       (mocked(instance.class)) << instance
     end
 
+    protected
+
     def mocked(model=nil)
       @mocked ||= {}
       if (model)

data/lib/stratagem/client.rb

@@ -11,7 +11,7 @@ module Stratagem
       url = URI.parse("#{@authentication.base_url}/snapshots")
       req = Net::HTTP::Post.new(url.path)
       req.set_form_data({
-        'api_key' => @authentication.credentials[:token],
+        'auth_token' => @authentication.credentials[:token],
         'project_id' => @authentication.credentials[:project],
         'timestamp' => snapshot.timestamp.to_i,
         'model' => snapshot.model.export.to_json

data/lib/stratagem/crawler.rb

@@ -5,5 +5,7 @@ require 'stratagem/crawler/form'
 require 'stratagem/crawler/trace_utils'
 require 'stratagem/crawler/html_utils'
 require 'stratagem/crawler/authentication'
+require 'stratagem/crawler/parameter_resolver'
+require 'stratagem/crawler/route_invoker'
 require 'stratagem/crawler/session'
 require 'stratagem/crawler/site_model'

data/lib/stratagem/crawler/authentication.rb

@@ -41,14 +41,14 @@ module Stratagem::Crawler
       reset_authentication
 
       login(user)
-      route = application_model.routes.recognize(response.request.path, :post)
+      route = application_model.routes.recognize(request.path, :post)
 
       redirected_to = nil
-      page = site_model.add(route, response) {|redirect_url| redirected_to = redirect_url }
+      page = site_model.add(route, request, response) {|redirect_url| redirected_to = redirect_url }
       authentication.response_page = page
 
       begin
-        if (response.request.url == (redirected_to || '')) || (![200,302].include?(response.code.to_i))
+        if (request.url == (redirected_to || '')) || (![200,302].include?(response.code.to_i))
           authentication.success = false
         else
           authentication.success = authentication.response_page.login_form.nil?
@@ -60,7 +60,7 @@ module Stratagem::Crawler
 
       puts "authenticated? #{authentication.success}"
       if (response && authentication.success)
-        authentication.ssl = response.request.ssl?
+        authentication.ssl = request.ssl?
         yield 
         logout
       else
@@ -90,15 +90,13 @@ module Stratagem::Crawler
     end
 
     def logout
-      get "/signout"
-      get "/logout"
-      delete "/user_sessions/1"
+      reset!
     end
 
     def login(user)
       populate_login_form(user).submit {|action,params| 
         post(action, params)
-        puts response.body
+        # puts response.body
       }
     end
 
@@ -124,7 +122,10 @@ module Stratagem::Crawler
     def populate_login_form(user)
       # set up the form
       page = find_login_form
-      page.reload {|url| get url; response }
+      p page.login_form
+      page.reload {|url| get url; [request,response] }
+      p page.login_form
+      p page.response.body
       form = page.login_form
 
       # map the input values

data/lib/stratagem/crawler/parameter_resolver.rb

@@ -0,0 +1,83 @@
+module Stratagem::Crawler
+  module ParameterResolver
+
+    def resolve_parameter_types(route_container)
+      log "\tresolving parameter types"
+      resolved_params = {}
+      route_infos, params = build_url(route_container, resolved_params)
+      route_info = route_infos.first
+      unknown_params = params.keys
+      log "\tunknown params: #{unknown_params.inspect} - #{unknown_params.size}"
+
+      resolve_with_convention(unknown_params, resolved_params)
+      log "\tunknown params after convention: #{unknown_params.inspect} - #{unknown_params.size}"
+
+      resolve_with_instrumentation(route_container, resolved_params)
+      log "\tunknown params after instrumentation: #{unknown_params.inspect} - #{unknown_params.size}"
+      
+      p resolved_params
+
+      if (resolved_params.size > 0)
+        resolved_params
+      else
+        nil
+      end
+    end
+
+    def resolve_with_instrumentation(route_container, resolved_params)
+      route_infos, params = build_url(route_container, resolved_params)
+      route_info = route_infos.first
+      unknown_params = params.keys
+
+      progress = nil
+      while ((unknown_params.size > 0) && (progress.nil? || (progress > 0)))
+        progress = 0
+
+        puts "\tloading model invocations for request"
+        delta = model_invocations_for_request do
+          call_route(route_info, false)
+        end
+
+        puts "\tcalled route, found #{delta.size} invocations"
+
+        unknown_params.clone.each do |key|
+          value = params[key]
+          value_s = params[key].map {|v| v.to_s }
+          delta.each do |invocation|
+            # TODO inspect is a hack, refactor
+            if (invocation.args.include?(value.first)) || (invocation.args.inspect.include?('"'+value.first.to_s+'"'))
+              # found match
+              puts "\t\tresolved #{key} to #{invocation.model_class}"
+              unknown_params.delete(key)
+              resolved_params[key] = invocation.model_class
+              progress += 1
+              break
+            end
+          end
+        end
+
+        route_infos, params = build_url(route_container, resolved_params)
+        route_info = route_infos.first
+      end
+    end
+
+    def resolve_with_convention(unknown_params, resolved_params)
+      unknown_params.each do |param|
+        if (param =~ /_id$/)
+          begin
+            model = param.gsub(/^:/,'').gsub(/_id$/, '').camelize.constantize
+            #success
+            resolved_params[param] = model
+            unknown_params.delete(param)
+            puts "\t\tresolved #{param} to #{model} using convention"
+          rescue NameError
+            puts $!
+            puts $!.backtrace
+            # not a model
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/stratagem/crawler/route_invoker.rb

@@ -0,0 +1,187 @@
+module Stratagem::Crawler
+  module RouteInvoker
+    include Stratagem::Crawler::ParameterResolver
+
+    def visit(route_container)
+      puts "Visiting #{route_container.route}"
+      build_urls(route_container).each do |route_info|
+        call_route(route_info)
+      end
+    end
+
+    def call_route(route_info, track_invocations=true)
+      begin
+        call_route!(route_info, track_invocations)
+      rescue
+        # TODO - add exception as a response page
+        puts "ERROR: #{$!.message}"
+      end
+    end
+
+    def call_route!(route_info, track_invocations=true)
+      return if route_info.nil?
+
+      puts 'CALLING: .'+route_info[:verb].downcase+". - "+route_info[:path]
+      verb = route_info[:verb].downcase
+      verb = 'get' if verb == '' || verb == 'any'
+
+      invocations = model_invocations_for_request do
+        case verb
+          when 'get'
+            do_get(route_info)
+            puts "\tresponse code: #{response.code}" if response
+          when 'post'
+          when 'put'
+            do_put(route_info)
+          when 'delete'
+          else
+            raise "Unsupported verb: #{route[:verb]}"
+        end
+      end
+
+      if (response)
+        if (track_invocations)
+          changes = detect_attribute_changes_in_models(invocations)
+          puts "\tfound #{invocations.size} invocations"
+          puts "\tchanges: #{changes.values.inspect}" if changes.size > 0
+          site_model.add(route_info[:route_container], request, response, invocations, changes) {|redirect_url| redirect_proc.call(redirect_url) }
+        end
+      else
+        puts "ERROR: did not call #{route_info.inspect}"
+      end
+    end
+
+    def do_get(route_info)
+      get route_info[:path]
+    end
+
+    def do_put(route_info)
+
+      # note: this should fail to generate anything meaningful, as we have not yet set up the parameters
+      put route_info[:path]
+
+      # let's find out what the method is looking for in the params object
+      params = map_models_to_attributes(infer_models_for_param_reads(route_info[:route_container],controller.params.hash_reads))
+
+      # run again with the params
+      puts "PUTTING: #{route_info[:path]} with #{params.inspect}"
+
+      invocation_delta = model_invocations_for_request(:write) do
+        put route_info[:path], params
+      end
+    end
+
+    private
+    
+
+    def map_models_to_attributes(models)
+      result = {}
+      models.each {|param_read,model|
+        result[param_read] = aquifer.mock_attributes(model.klass)
+      }
+      result
+    end
+
+    def infer_models_for_param_reads(route,param_reads)
+      param_reads = param_reads.select {|read| read.to_s !~ /_id$/ }
+      param_reads -= [:action,'action',:controller,'controller',:id,'id',:format,'format']
+
+      result = {}
+
+      # resolve param reads to model types
+      # only support simple expected mappings at the moment
+      param_reads.each do |param_read|
+        class_name = param_read.to_s.camelize
+        if (class_name.singularize == class_name)
+          # this looks promising
+          model = application_model.models.find {|model| model.klass.name == class_name }
+          if (model)
+            puts "\t\tresolved param #{param_read} to #{model.klass.name}"
+            result[param_read] = model
+          end
+        end
+      end
+
+      result
+    end
+
+    def detect_attribute_changes_in_models(invocations)
+      changes = {}
+      invocations.select {|invocation| invocation.type == :write }.each do |invocation|
+        if (invocation.model_instance)
+          model = application_model.models.find {|m| m.klass == invocation.model_instance.class }
+
+          puts "\t\t#{invocation.model_class}.#{invocation.method} - #{invocation.model_instance}"
+          prior = aquifer.instances_of(invocation.model_instance.class).find {|m| m.id == invocation.model_instance.id }
+          post = invocation.model_instance
+          attribute_names = (prior.stratagem.attribute_names + prior.stratagem.foreign_keys)
+          changes[model] =  attribute_names.select {|an|
+            begin
+              prior.send(an) != post.send(an)
+            rescue
+              puts "\t\t\t#{an} cannot be determined - #{$!.message}"
+            end
+          }
+        end
+      end
+      changes
+    end
+
+    # Builds a list of string URLs for a given route. This is done
+    # by replacing :xyz_id segments in the route with known values
+    # from the well
+    def build_urls(route_container)
+      urls = []
+      route = route_container.route
+      param_types = (self.parameter_types[route_container] ||= resolve_parameter_types(route_container))
+      route_infos, params = build_url(route_container, param_types)
+      route_infos
+    end
+
+    def url_permutations(route, segment_values, &block)
+      path = route.path.sub('(.:format)', '')
+      if (segment_values.size == 0)
+        yield path
+      else
+        combinations = segment_values[0].product(*segment_values.slice(1,segment_values.size-1))
+        combinations.each do |combination|
+          url = path.clone
+          route.segment_keys.each_with_index {|segment_key,i| url.gsub!(':'+segment_key.to_s, combination[i].to_s) }
+          yield url
+        end
+      end
+    end
+
+    def build_url(route_container, parameter_types={})
+      parameter_types ||= {}
+      params = {}
+      route = route_container.route
+      verb = route_container.verb.to_s
+
+
+      i = 12345
+      insert_values = (route.segment_keys - [:format]).map {|segment_key|
+        model = parameter_types[segment_key.to_s]
+        value = nil
+        if (model)
+          value = (aquifer.instances_of model).map {|inst| 
+            inst.methods_include?(segment_key) ? inst.send(segment_key) : inst.to_param
+          }
+        else
+          value = [i += 1]
+        end
+        params[segment_key.to_s] = value
+      }
+
+      routes = []
+      url_permutations(route_container,insert_values) do |path|
+        puts "yielded: #{path}"
+        permutation = {:verb => verb, :path => path, :route_container => route_container}
+        routes << permutation
+      end
+      
+      [routes, params]
+    end
+
+  end
+end

data/lib/stratagem/crawler/session.rb

@@ -29,6 +29,20 @@ module Stratagem::Crawler::Session
   include Stratagem::Crawler::HtmlUtils
   include Stratagem::Crawler::TraceUtils
   include Stratagem::Crawler::Authentication
+  include Stratagem::Crawler::RouteInvoker
+
+  attr_writer :aquifer
+
+  # def self.app
+  #   # DEPRECATE Rails application fallback
+  #   # This should be set by the initializer
+  #   (defined?(Rails.application) && Rails.application) || nil
+  # end
+  # 
+
+  def app
+    (defined?(Rails.application) && Rails.application) || nil
+  end
 
   def log(msg)
     Stratagem.logger.debug msg
@@ -68,14 +82,18 @@ module Stratagem::Crawler::Session
   end
 
   def page_set(name, &block)
+    reset!
     site_models << Stratagem::Crawler::SiteModel.new(name)
     yield site_model
   end
 
-  def display
+  def print
     # print out pages and inbound / outbound links
     site_model.pages.each do |page|
-      log "Page: #{page.url} - #{page.title} - #{page.response.code}"
+      title = page.title
+      title ||= page.redirected_to.url if page.redirected_to
+      title ||= page.response.code
+      log "Page: #{page.url} - #{title} - #{page.response.code}"
       page.outbound_edges.each do |edge|
         log "\tout: #{edge.to.url} - #{edge.to.title} - #{edge.to.route}"
       end
@@ -102,10 +120,10 @@ module Stratagem::Crawler::Session
       else
         if (application_model.routes.invalid.include?(route_container))
           log "Skipping invalid route #{route_container.route.to_s}"
-        elsif (verbs.include?(route_container.route.conditions[:method]))
+        elsif (verbs.include?(route_container.verb))
           visit(route_container)
         else
-          log "Skipping route with verb #{route_container.route.conditions[:method]} - #{route_container.route.to_s}"
+          log "Skipping route with verb #{route_container.verb} - #{route_container.route.to_s}"
         end
       end
     }
@@ -120,203 +138,6 @@ module Stratagem::Crawler::Session
 
   private
 
-  def visit(route_container)
-    puts "Visiting #{route_container.route}"
-    build_urls(route_container).each do |route_info|
-      call_route(route_container, route_info)
-    end
-  end
-
-  def call_route(route, route_info, track_invocations=true)
-    return if route_info.nil?
-
-    puts 'CALLING: .'+route_info[:verb].downcase+". - "+route_info[:path]
-    verb = route_info[:verb].downcase
-    verb = 'get' if verb == '' || verb == 'any'
-
-    begin
-      invocations = model_invocations_for_request do
-        case verb
-          when 'get'
-            do_get(route, route_info[:path])
-            puts "\tresponse code: #{response.code}" if response
-          when 'post'
-          when 'put'
-            do_put(route, route_info[:path])
-          when 'delete'
-          else
-            raise "Unsupported verb: #{route[:verb]}"
-        end
-      end
-
-      if (response)
-        changes = detect_attribute_changes_in_models(invocations)
-        puts "\tfound #{invocations.size} invocations"
-        puts "\tchanges: #{changes.values.inspect}" if changes.size > 0
-        site_model.add(route, response, invocations, changes) {|redirect_url| redirect_proc.call(redirect_url) }
-      else
-        puts "ERROR: did not call #{route_info.inspect}"
-      end
-    rescue
-      puts $!.message
-      puts $!.backtrace
-    end
-  end
-
-  def do_get(route,path)
-    get path
-  end
-
-  def do_put(route,path)
-
-    # note: this should fail to generate anything meaningful, as we have not yet set up the parameters
-    put path
-    
-    # let's find out what the method is looking for in the params object
-    params = map_models_to_attributes(infer_models_for_param_reads(route,controller.params.hash_reads))
-
-    # run again with the params
-    puts "PUTTING: #{path} with #{params.inspect}"
-
-    invocation_delta = model_invocations_for_request(:write) do
-      put path, params
-    end
-  end
-
-  def map_models_to_attributes(models)
-    result = {}
-    models.each {|param_read,model|
-      result[param_read] = aquifer.mock_attributes(model.klass)
-    }
-    result
-  end
-
-  def infer_models_for_param_reads(route,param_reads)
-    param_reads = param_reads.select {|read| read.to_s !~ /_id$/ }
-    param_reads -= [:action,'action',:controller,'controller',:id,'id',:format,'format']
-
-    result = {}
-    
-    # resolve param reads to model types
-    # only support simple expected mappings at the moment
-    param_reads.each do |param_read|
-      class_name = param_read.to_s.camelize
-      if (class_name.singularize == class_name)
-        # this looks promising
-        model = application_model.models.find {|model| model.klass.name == class_name }
-        if (model)
-          puts "\t\tresolved param #{param_read} to #{model.klass.name}"
-          result[param_read] = model
-        end
-      end
-    end
-    
-    result
-  end
-
-  def detect_attribute_changes_in_models(invocations)
-    changes = {}
-    invocations.select {|invocation| invocation.type == :write }.each do |invocation|
-      if (invocation.model_instance)
-        model = application_model.models.find {|m| m.klass == invocation.model_instance.class }
-        
-        puts "\t\t#{invocation.model_class}.#{invocation.method} - #{invocation.model_instance}"
-        prior = aquifer.instances_of(invocation.model_instance.class).find {|m| m.id == invocation.model_instance.id }
-        post = invocation.model_instance
-        attribute_names = (prior.stratagem.attribute_names + prior.stratagem.foreign_keys)
-        changes[model] =  attribute_names.select {|an|
-          begin
-            prior.send(an) != post.send(an)
-          rescue
-            puts "\t\t\t#{an} cannot be determined - #{$!.message}"
-          end
-        }
-      end
-    end
-    changes
-  end
-
-  # Builds a list of string URLs for a given route. This is done
-  # by replacing :xyz_id segments in the route with known values
-  # from the well
-  def build_urls(route_container)
-    urls = []
-    route = route_container.route
-    param_types = (self.parameter_types[route_container] ||= resolve_parameter_types(route_container))
-    route_infos, params = build_url(route_container, param_types)
-    # puts "route: #{route_container.route.to_s} - #{parameter_types.inspect} - permutations:"
-    # route_infos.each do |info|
-    #   puts "\t#{info[:path]}"
-    # end
-    route_infos
-  end
-
-  def url_permutations(meta_segments, segment_stack=[], &block)
-    if (segment_stack.size == meta_segments.size)
-      yield segment_stack
-    else
-      cursor = segment_stack.size
-      options = meta_segments[cursor]
-      if (options.kind_of?(Array))
-        options.each do |option|
-          url_permutations(meta_segments, segment_stack + [option], &block)
-        end
-      else
-        url_permutations(meta_segments, segment_stack + [options], &block)
-      end
-    end
-  end
-
-  def build_url(route_container, parameter_types={})
-    params = {}
-    route = route_container.route
-    name = route.to_s
-    verb = route.conditions[:method].to_s
-
-    parameter_types ||= {}
-
-    i = 12345
-    segs = route.segments.inject([]) {|accumulated,segment|
-      s = segment.to_s
-      if (s =~ /^:/)
-        model = parameter_types[s]
-        value = nil
-        if (model)
-          if (aquifer.instances_of(model).size == 0)
-            aquifer.print
-          end
-          value = (aquifer.instances_of model).map {|inst| 
-            attr_name = s.gsub(/^:/, '').to_sym
-            if inst.methods_include?(attr_name)
-              inst.send(attr_name)
-            else
-              inst.id
-            end
-          }
-        else
-          i += 1
-          value = [i]
-        end
-        accumulated << value
-        params[s] = value
-      else
-        accumulated << s
-      end
-      accumulated
-    }
-
-    routes = []
-    reqs = route.requirements.empty? ? "" : route.requirements.inspect
-    url_permutations(segs) do |segments|
-      path = segments.join('').gsub('(.:format)', '').gsub(/\/$/, '')
-      puts "\t\tbuilt url: #{path}"
-      permutation = {:name => name, :verb => verb, :segs => segs, :reqs => reqs, :path => path}
-      routes << permutation
-    end
-
-    [routes, params]
-  end
-
   def handle_redirect(redirect_url)
     existing_pages = site_model.pages_for(response.redirect_url)
     if (existing_pages.size > 0)
@@ -326,59 +147,10 @@ module Stratagem::Crawler::Session
         get redirect_url
       end
 
-      site_model.add(nil, response) {|redirect_url|
+      site_model.add(nil, request, response) {|redirect_url|
         # TODO - record as bug!
         puts "recursive redirect #{redirect_url}"
       }
     end
   end
-
-  def resolve_parameter_types(route_container)
-    puts "\tresolving parameter types"
-    resolved_parameters = {}
-    route_infos, params = build_url(route_container, resolved_parameters)
-    route_info = route_infos.first
-    unknown_params = params.keys
-    log "\tunknown params: #{unknown_params.inspect} - #{unknown_params.size}"
-    progress = nil
-    while ((unknown_params.size > 0) && (progress.nil? || (progress > 0)))
-      progress = 0
-
-      puts "\tloading model invocations for request"
-      delta = model_invocations_for_request do
-        call_route(route_container, route_info, false)
-      end
-
-      puts "\tcalled route, found #{delta.size} invocations"
-
-      unknown_params.clone.each do |key|
-        value = params[key]
-        value_s = params[key].map {|v| v.to_s }
-        delta.each do |invocation|
-          # puts "\t#{route_info[:path]} - #{invocation.model_class.name} - #{invocation.method} - #{invocation.args.inspect} - #{value_s}"
-          # TODO inspect is a hack, refactor
-          if (invocation.args.include?(value.first)) || (invocation.args.inspect.include?('"'+value.first.to_s+'"'))
-            # found match
-
-            puts "\t\tresolved #{key} to #{invocation.model_class}"
-            unknown_params.delete(key)
-            resolved_parameters[key] = invocation.model_class
-            progress += 1
-
-            break
-          end
-        end
-      end
-
-      route_infos, params = build_url(route_container, resolved_parameters)
-      route_info = route_infos.first
-    end
-
-    if (resolved_parameters.size > 0)
-      resolved_parameters
-    else
-      nil
-    end
-  end
-
 end