stratagem 0.1.7 → 0.1.8

data/lib/stratagem/crawler/session.rb

@@ -24,8 +24,6 @@ class CrawlError < StratagemError; end
 #   end
 # end
 
-PHASES = [:unauthenticated,:authenticated]
-
 module Stratagem::Crawler::Session
   include ActionController::Integration::Runner
   include Stratagem::Crawler::HtmlUtils
@@ -36,38 +34,42 @@ module Stratagem::Crawler::Session
     Stratagem.logger.debug msg
   end
   
+  def parameter_types
+    @parameter_types ||= {}
+  end
+  
+  def redirect_proc
+    @redirect_proc ||= Proc.new {|redirect_url| handle_redirect(redirect_url) }
+  end
+  
   def crawler_session(application_model=nil)
-    @model = application_model
-    @model ||= Stratagem::ModelBuilder.new.run
-    @redirect_proc = Proc.new {|redirect_url| handle_redirect(redirect_url) }
-    @parameter_types = {} # routecontainer -> {:route_segment => Model, :route_segment => Model}
+    @application_model = application_model if application_model
     open_session do |session|
       @session = session
-      phase(:unauthenticated)
       yield
     end
   end
   
   def application_model
-    @model
+    @application_model ||= Stratagem::ModelBuilder.new.run
+  end
+  
+  def aquifer(initial_capacity=6)
+    @aquifer ||= Stratagem::AutoMock::Aquifer.init(application_model).fill(initial_capacity)
   end
   
   def site_models
-    @site_models ||= {} # builds models of the site for various phases of analysis, see PHASES
+    @site_models ||= []
   end
 
   def site_model
-    raise Stratagem::Crawler::CrawlError.new("Phase not specified") unless @current_model
-    @current_model
-  end
-
-  def phase(new_phase)
-    @current_phase = new_phase
-    @current_model = site_models[@current_phase] ||= Stratagem::Crawler::SiteModel.new
+    raise Stratagem::Crawler::CrawlError.new("Not within page set") unless site_models.last
+    site_models.last
   end
 
-  def phases
-    site_models
+  def page_set(name, &block)
+    site_models << Stratagem::Crawler::SiteModel.new(name)
+    yield site_model
   end
 
   def display
@@ -83,23 +85,28 @@ module Stratagem::Crawler::Session
     end
   end
 
-  def crawl
+  def crawl(verbs=[:any,:get])
+    verbs = [verbs] unless verbs.kind_of?(Array)
+    
     # grab all pages independently
-    @model.routes.invalid.each {|route_container|
-      puts "skipping invalid route #{route_container.route.to_s}"
-    }
 
     authentication_controller = nil
-    if (@current_phase == :authenticated)
-      route = @model.routes.recognize(authentication.login_page)
-      authentication_controller = route.controller
+    if (site_model.authentication)
+      route = application_model.routes.recognize(authentication.login_page)
+      authentication_controller = route.controller if route
     end
     
-    @model.routes.each {|route_container|
+    application_model.routes.each {|route_container|
       if authentication_controller && route_container.controller && (route_container.controller.klass == authentication_controller.klass)
         log "Skipping authentication routes #{route_container.route.to_s}"
       else
-        visit(route_container) unless @model.routes.invalid.include?(route_container)
+        if (application_model.routes.invalid.include?(route_container))
+          log "Skipping invalid route #{route_container.route.to_s}"
+        elsif (verbs.include?(route_container.route.conditions[:method]))
+          visit(route_container)
+        else
+          log "Skipping route with verb #{route_container.route.conditions[:method]} - #{route_container.route.to_s}"
+        end
       end
     }
 
@@ -114,40 +121,133 @@ module Stratagem::Crawler::Session
   private
 
   def visit(route_container)
-    puts "visiting #{route_container.route}"
+    puts "Visiting #{route_container.route}"
     build_urls(route_container).each do |route_info|
       call_route(route_container, route_info)
     end
   end
 
-  def call_route(route, route_info)
+  def call_route(route, route_info, track_invocations=true)
+    return if route_info.nil?
+
     puts 'CALLING: .'+route_info[:verb].downcase+". - "+route_info[:path]
     verb = route_info[:verb].downcase
     verb = 'get' if verb == '' || verb == 'any'
-    case verb
-      when 'get'
-        do_get(route, route_info[:path])
-        puts response.code if response
-      when 'post'
-      when 'put'
-      when 'delete'
+
+    begin
+      invocations = model_invocations_for_request do
+        case verb
+          when 'get'
+            do_get(route, route_info[:path])
+            puts "\tresponse code: #{response.code}" if response
+          when 'post'
+          when 'put'
+            do_put(route, route_info[:path])
+          when 'delete'
+          else
+            raise "Unsupported verb: #{route[:verb]}"
+        end
+      end
+
+      if (response)
+        changes = detect_attribute_changes_in_models(invocations)
+        puts "\tfound #{invocations.size} invocations"
+        puts "\tchanges: #{changes.values.inspect}" if changes.size > 0
+        site_model.add(route, response, invocations, changes) {|redirect_url| redirect_proc.call(redirect_url) }
       else
-        raise "Unsupported verb: #{route[:verb]}"
+        puts "ERROR: did not call #{route_info.inspect}"
+      end
+    rescue
+      puts $!.message
+      puts $!.backtrace
     end
   end
 
+  def do_get(route,path)
+    get path
+  end
+
+  def do_put(route,path)
+
+    # note: this should fail to generate anything meaningful, as we have not yet set up the parameters
+    put path
+    
+    # let's find out what the method is looking for in the params object
+    params = map_models_to_attributes(infer_models_for_param_reads(route,controller.params.hash_reads))
+
+    # run again with the params
+    puts "PUTTING: #{path} with #{params.inspect}"
+
+    invocation_delta = model_invocations_for_request(:write) do
+      put path, params
+    end
+  end
+
+  def map_models_to_attributes(models)
+    result = {}
+    models.each {|param_read,model|
+      result[param_read] = aquifer.mock_attributes(model.klass)
+    }
+    result
+  end
+
+  def infer_models_for_param_reads(route,param_reads)
+    param_reads = param_reads.select {|read| read.to_s !~ /_id$/ }
+    param_reads -= [:action,'action',:controller,'controller',:id,'id',:format,'format']
+
+    result = {}
+    
+    # resolve param reads to model types
+    # only support simple expected mappings at the moment
+    param_reads.each do |param_read|
+      class_name = param_read.to_s.camelize
+      if (class_name.singularize == class_name)
+        # this looks promising
+        model = application_model.models.find {|model| model.klass.name == class_name }
+        if (model)
+          puts "\t\tresolved param #{param_read} to #{model.klass.name}"
+          result[param_read] = model
+        end
+      end
+    end
+    
+    result
+  end
+
+  def detect_attribute_changes_in_models(invocations)
+    changes = {}
+    invocations.select {|invocation| invocation.type == :write }.each do |invocation|
+      if (invocation.model_instance)
+        model = application_model.models.find {|m| m.klass == invocation.model_instance.class }
+        
+        puts "\t\t#{invocation.model_class}.#{invocation.method} - #{invocation.model_instance}"
+        prior = aquifer.instances_of(invocation.model_instance.class).find {|m| m.id == invocation.model_instance.id }
+        post = invocation.model_instance
+        attribute_names = (prior.stratagem.attribute_names + prior.stratagem.foreign_keys)
+        changes[model] =  attribute_names.select {|an|
+          begin
+            prior.send(an) != post.send(an)
+          rescue
+            puts "\t\t\t#{an} cannot be determined - #{$!.message}"
+          end
+        }
+      end
+    end
+    changes
+  end
+
   # Builds a list of string URLs for a given route. This is done
   # by replacing :xyz_id segments in the route with known values
   # from the well
   def build_urls(route_container)
     urls = []
     route = route_container.route
-    parameter_types = (@parameter_types[route_container] ||= resolve_parameter_types(route_container))
-    route_infos, params = build_url(route_container, parameter_types)
-    puts "route: #{route_container.route.to_s} permutations:"
-    route_infos.each do |info|
-      puts "\t#{info[:path]}"
-    end
+    param_types = (self.parameter_types[route_container] ||= resolve_parameter_types(route_container))
+    route_infos, params = build_url(route_container, param_types)
+    # puts "route: #{route_container.route.to_s} - #{parameter_types.inspect} - permutations:"
+    # route_infos.each do |info|
+    #   puts "\t#{info[:path]}"
+    # end
     route_infos
   end
 
@@ -182,13 +282,14 @@ module Stratagem::Crawler::Session
         model = parameter_types[s]
         value = nil
         if (model)
-          value = (Stratagem::AutoMock::Aquifer.instance.instances_of model).map {|inst| 
+          if (aquifer.instances_of(model).size == 0)
+            aquifer.print
+          end
+          value = (aquifer.instances_of model).map {|inst| 
             attr_name = s.gsub(/^:/, '').to_sym
             if inst.methods_include?(attr_name)
-              puts "#{attr_name} is a method on the object"
               inst.send(attr_name)
             else
-              puts "#{attr_name} is being mapped to the id on the object"
               inst.id
             end
           }
@@ -208,6 +309,7 @@ module Stratagem::Crawler::Session
     reqs = route.requirements.empty? ? "" : route.requirements.inspect
     url_permutations(segs) do |segments|
       path = segments.join('').gsub('(.:format)', '').gsub(/\/$/, '')
+      puts "\t\tbuilt url: #{path}"
       permutation = {:name => name, :verb => verb, :segs => segs, :reqs => reqs, :path => path}
       routes << permutation
     end
@@ -231,48 +333,34 @@ module Stratagem::Crawler::Session
     end
   end
 
-  def do_get(route,path)
-    begin
-      get path
-      site_model.add(route, response) {|redirect_url| @redirect_proc.call(redirect_url) }
-    rescue
-      puts $!.message
-      puts path
-      #puts $!.backtrace
-    end
-  end
-
   def resolve_parameter_types(route_container)
-    puts "resolving parameter types for #{route_container.route}"
+    puts "\tresolving parameter types"
     resolved_parameters = {}
     route_infos, params = build_url(route_container, resolved_parameters)
     route_info = route_infos.first
     unknown_params = params.keys
-    puts "unknown params: #{unknown_params}"
+    log "\tunknown params: #{unknown_params.inspect} - #{unknown_params.size}"
     progress = nil
     while ((unknown_params.size > 0) && (progress.nil? || (progress > 0)))
       progress = 0
 
-      # p route_infos
-      # puts "---"
+      puts "\tloading model invocations for request"
       delta = model_invocations_for_request do
-        call_route(route_container, route_info)
+        call_route(route_container, route_info, false)
       end
 
-      puts "delta - #{delta.size}"
-      #p delta
+      puts "\tcalled route, found #{delta.size} invocations"
 
       unknown_params.clone.each do |key|
         value = params[key]
         value_s = params[key].map {|v| v.to_s }
         delta.each do |invocation|
-          puts "#{route_info[:path]} - #{invocation.model_class.name} - #{invocation.method} - #{invocation.args.inspect} - #{value_s}"
-
+          # puts "\t#{route_info[:path]} - #{invocation.model_class.name} - #{invocation.method} - #{invocation.args.inspect} - #{value_s}"
           # TODO inspect is a hack, refactor
           if (invocation.args.include?(value.first)) || (invocation.args.inspect.include?('"'+value.first.to_s+'"'))
             # found match
 
-            puts "\tresolved #{key} to #{invocation.model_class}"
+            puts "\t\tresolved #{key} to #{invocation.model_class}"
             unknown_params.delete(key)
             resolved_parameters[key] = invocation.model_class
             progress += 1
@@ -285,6 +373,7 @@ module Stratagem::Crawler::Session
       route_infos, params = build_url(route_container, resolved_parameters)
       route_info = route_infos.first
     end
+
     if (resolved_parameters.size > 0)
       resolved_parameters
     else
@@ -292,5 +381,4 @@ module Stratagem::Crawler::Session
     end
   end
 
-
 end

data/lib/stratagem/crawler/site_model.rb

@@ -2,17 +2,26 @@ module Stratagem::Crawler
   class SiteModel
     include Stratagem::Crawler::HtmlUtils
 
-    attr_reader :pages, :edges
+    attr_reader :pages, :edges, :name
+    attr_accessor :authentication
 
-    def initialize
+    def initialize(name)
+      @name = name
       @pages = []
       @edges = []
     end
 
     def export
       {
+        :name => name,
         :pages => @pages.map {|page| page.export },
-        :edges => @edges.map {|edge| edge.export }
+        :edges => @edges.map {|edge| edge.export },
+        :authentication => authentication.nil? ? nil : {
+          :success => authentication.success, 
+          :login_page_external_id => authentication.login_page.object_id, 
+          :response_page_external_id => authentication.response_page.object_id,
+          :ssl => authentication.ssl
+        },
       }
     end
 
@@ -20,8 +29,8 @@ module Stratagem::Crawler
       self.edges << Edge.new(from,to,type)
     end
 
-    def add(route, response, &block)
-      page = Page.new(self, response, &block)
+    def add(route, response, invocations=[], model_changes={}, &block)
+      page = Page.new(self, response, invocations, model_changes, &block)
       self.pages << page
       page
     end
@@ -66,8 +75,10 @@ module Stratagem::Crawler
     attr_accessor :redirected_to
     attr_accessor :document
 
-    def initialize(site_model, response, &block)
+    def initialize(site_model, response, invocations, model_changes, &block)
       @site_model = site_model
+      @invocations = invocations
+      @model_changes = model_changes
       init(response, &block)
     end
 
@@ -82,7 +93,10 @@ module Stratagem::Crawler
         :path => path,
         :method => method,
         :redirected_to_page_external_id => redirected_to ? redirected_to.object_id : nil,
-        :route_external_id => route.object_id
+        :route_external_id => route.object_id,
+        :references => @invocations.map {|i| i.to_reference.export },
+        :model_changes => Hash[@model_changes.map {|model,changes| [model.object_id, changes] }].to_json,
+        :parameters => response.request.parameters.to_json
       }
     end
 

data/lib/stratagem/crawler/trace_utils.rb

@@ -1,10 +1,12 @@
 module Stratagem::Crawler
   module TraceUtils
-    def model_invocations_for_request()
+    def model_invocations_for_request(type=nil)
       prior_invocations = ActiveRecord::Base.stratagem.invocations_audit.clone
       yield
       post_invocations = ActiveRecord::Base.stratagem.invocations_audit.clone
       delta = post_invocations - prior_invocations
+      delta = delta.select {|i| i.type == type } if (type)
+      delta
     end
   end
 end

data/lib/stratagem/extensions/trace_compression.rb

@@ -0,0 +1,52 @@
+# it's not elegant, but it compresses stack traces in a json friendly manner
+class TraceDeflator
+  class << self
+    def create_word_map(compressed)
+      word_counts = {}
+      compressed.each do |column|
+        column.each do |row|
+          word_counts[row[0]] ||= 0
+          word_counts[row[0]] += 1
+        end
+      end
+
+      map = word_counts.keys.select {|key| (key.length) > 2 && (word_counts[key] > 1) }.sort {|a,b| word_counts[b] <=> word_counts[a]}
+    end
+
+    def sub_in_word_map(compressed,word_map)
+      compressed.map! do |column|
+        column.map! do |row|
+          sub_index = word_map.index(row[0])
+          sub_index ? [sub_index,row[1]] : row
+        end
+      end
+    end
+
+    def deflate(trace)
+      tokenized = trace.map {|s| s.split('/') }
+      longest = tokenized.inject(0) {|memo,a| a.size > memo ? a.size : memo }
+      tokenized.each {|a|
+        a << "" until a.size >= longest
+      }
+      compressed = tokenized.transpose.map do |column|
+        compressed_column = column.inject([]) {|new_col,col|
+          new_col << [col,0] if (new_col.size == 0)
+          prev = new_col.last
+          prev[0] == col ? prev[1] += 1 : new_col << [col,1]
+          new_col
+        }
+      end
+  
+      word_map = create_word_map(compressed)
+      sub_in_word_map(compressed, word_map)
+
+      compressed = compressed.map {|column|
+        column.map {|row|
+          "#{row[1]}*#{row[0]}"
+        }.join("\n")
+      }
+  
+      {:a => word_map, :b => compressed}
+    end
+  end
+end

data/lib/stratagem/extensions.rb

@@ -4,3 +4,4 @@ require 'stratagem/extensions/string'
 require 'stratagem/extensions/hash'
 require 'stratagem/extensions/object'
 require 'stratagem/extensions/module'
+require 'stratagem/extensions/trace_compression'

data/lib/stratagem/framework_extensions/models/adapters/active_model/metadata.rb

@@ -13,7 +13,7 @@ module Stratagem::ApplicationExtensions::Models::Adapters::ActiveModel
     def relations(relation_type=nil) # :belongs_to, :has_many
       @relations ||= {}
       @relations[relation_type || :all] ||= model.reflect_on_all_associations(relation_type).map {|a|
-        Stratagem::ApplicationExtensions::Models::Metadata::StratagemAssociation.new(a.name.to_sym, a.association_foreign_key.to_sym, a.klass, a.macro)
+        Stratagem::ApplicationExtensions::Models::Metadata::StratagemAssociation.new(a.name.to_sym, a.association_foreign_key.to_sym, a.klass, a.macro, a.options)
       }
     end
 
@@ -53,12 +53,7 @@ module Stratagem::ApplicationExtensions::Models::Adapters::ActiveModel
     end
   
     def attribute_names
-      instance.attribute_names.map {|a| a.to_sym}  - ignore_attributes
-    end
-
-    # junk attributes
-    def ignore_attributes
-      ["!".to_sym, :[]]
+      instance.attribute_names.map {|a| a.to_sym}  - model.stratagem.ignore_attributes
     end
 
     # Attributes generally used by the persistence mechanism that should not be human writable
@@ -83,7 +78,7 @@ module Stratagem::ApplicationExtensions::Models::Adapters::ActiveModel
           column.type
         end
       else
-        if (name =~ /password/)
+        if (name.to_s =~ /password/)
           :string
         else
           types = [:string, :boolean, :integer]

data/lib/stratagem/framework_extensions/models/adapters/active_model/tracing.rb

@@ -39,10 +39,29 @@ module Stratagem::ApplicationExtensions::Models::Adapters::ActiveModel
         # add logging of save methods
         
         def create_or_update(*args)
-          path,action,line = stratagem.controller_trace(/active_record\/base\.rb/)
-          stratagem.write_invocation(self, action.to_sym, args)
+          alternate_model = nil
+          path,action,line,trace,index = stratagem.controller_trace(/active_record\/base\.rb/)
+          if (index)
+            model_path,model_action,model_line = find_model_path(trace,index)
+            if (model_path)
+              puts "WRITE INVOCATION - USING: #{model_path},#{model_action},#{model_line}"
+              alternate_model = Stratagem::Model::Application.instance.models.find {|m| m.path == model_path }.klass
+              action = model_action
+              puts alternate_model
+            end
+          end
+
+          stratagem.write_invocation(self, alternate_model || self.class, action.to_sym, args)
           old_create_or_update(*args)
         end
+        
+        def find_model_path(trace,controller_index)
+          0.upto(controller_index) do |i|
+            line = trace[i]
+            return stratagem.parse_trace_line(line) if (line =~ /\/app\/models\//)
+          end
+          []
+        end
       end
     end
 

data/lib/stratagem/framework_extensions/models/adapters/common/detect.rb

@@ -0,0 +1,7 @@
+module Stratagem::ApplicationExtensions::Models::Adapters::Common
+  class Detect < Stratagem::ApplicationExtensions::Models::Detect
+    def self.supports?(model)
+      true
+    end
+  end
+end

data/lib/stratagem/framework_extensions/models/adapters/common/metadata.rb

@@ -0,0 +1,36 @@
+module Stratagem::ApplicationExtensions::Models::Adapters::Common
+
+  # prefix method names with  to avoid collision
+  class Metadata
+
+    attr_reader :model, :instance
+
+    def initialize(model)
+      @model = model
+      @instance = @model.new unless (@model == ActiveRecord::Base)
+    end
+    
+    def attribute_names
+      @attribute_names ||= begin
+        names = model.new.methods.select {|m| 
+          s = m.to_s
+          if (s.to_s =~ /=$/)
+            name = s.gsub(/=$/, '')
+            name.singularize == name
+          else
+            false
+          end
+        }.map {|m| 
+          m.to_s.gsub(/=$/, '').to_sym 
+        }
+        names - (model.stratagem.internal_attributes + model.stratagem.ignore_attributes + model.stratagem.relation_names)
+      end
+    end
+
+    # junk attributes
+    def ignore_attributes
+      @ignore_attributes ||= ["!", "[]", "===", "==", "=", "taguri"].map {|a| a.to_sym }
+    end
+
+  end
+end

data/lib/stratagem/framework_extensions/models/adapters/common/tracing.rb

@@ -0,0 +1,4 @@
+module Stratagem::ApplicationExtensions::Models::Adapters::Common
+  module Tracing
+  end
+end

data/lib/stratagem/framework_extensions/models/annotations.rb

@@ -1,8 +1,30 @@
 # Defines the stratagem namespace attached to the model
 module Stratagem::ApplicationExtensions::Models
-  MethodInvocation = Struct.new(:method, :controller_path, :controller_action, :line_number, :model_instance, :model_class, :stack_trace, :args)
+  MethodInvocation = Struct.new(:method, :controller_path, :controller_action, :line_number, :model_instance, :model_class, :stack_trace, :args, :type)
   ValidatorDefinition = Struct.new(:validation, :field, :args, :model_class)
 
+  # Ability to convert a MethodInvocation object to a Reference object. Sort of a hack.
+  module MethodInvocationToReference
+    def to_reference
+      app = Stratagem::Model::Application.instance
+      model = model_class ? app.models.find {|model| model.klass == model_class } : nil
+      controller = controller_path ? app.controllers.find {|controller| controller.path == controller_path } : nil
+      Stratagem::Model::Component::Reference.new(
+        :from_component => controller,
+        :to_component => model,
+        :function => controller_action,
+        :method => method, 
+        :line_number => line_number,
+        :reference_type => type,
+        :stack_trace => stack_trace
+      )
+    end
+  end
+  MethodInvocation.class_eval do
+    include MethodInvocationToReference
+  end
+  
+  
   class InstanceAnnotations
     include Mocking
 

data/lib/stratagem/framework_extensions/models/metadata.rb

@@ -1,7 +1,7 @@
 
 module Stratagem::ApplicationExtensions::Models
   module Metadata
-    StratagemAssociation = Struct.new(:name, :foreign_key, :klass, :macro)
+    StratagemAssociation = Struct.new(:name, :foreign_key, :klass, :macro, :options)
 
     INSTANCE_ENUMERATION_METHODS = [:relations, :attribute_names, :ignore_attributes, :internal_attributes, :unaccessible_attributes, :invalid_columns, :exclude_attributes_for_mocking]
     INSTANCE_ENTITY_METHODS = [:attribute_type, :column_from_error, :authenticates?, :whitelists_attributes?, :blacklists_attributes?]
@@ -74,11 +74,11 @@ module Stratagem::ApplicationExtensions::Models
             memory << callback.send(method, *args) if callback.methods_include?(method) || callback.methods_include?(method.to_s)
             memory
           rescue
-            puts $!.message
+            puts "error running callbacks: #{$!.message}"
             puts $!.backtrace
           end
         }
-        (results || []).flatten.compact
+        (results || []).flatten.compact.uniq
       end
 
   end