stratagem 0.1.7

data/lib/stratagem/crawler/session.rb

@@ -0,0 +1,296 @@
+require 'nokogiri'
+
+require 'action_pack'
+require 'action_controller'
+
+class CrawlError < StratagemError; end
+
+# require 'rack/lint'
+# module Rack
+#   # Rack::Lint validates your application and the requests and
+#   # responses according to the Rack spec.
+# 
+#   class Lint
+#     alias_method :old_call, :call
+# 
+#     def call(env)
+#       data = env['rack.input']
+#       data.set_encoding(Encoding::ASCII_8BIT) if data.respond_to?(:set_encoding)
+# 
+#       status, headers, body = old_call(env)
+#       body.close # fix for Rack
+#       [status,headers,body]
+#     end
+#   end
+# end
+
+PHASES = [:unauthenticated,:authenticated]
+
+module Stratagem::Crawler::Session
+  include ActionController::Integration::Runner
+  include Stratagem::Crawler::HtmlUtils
+  include Stratagem::Crawler::TraceUtils
+  include Stratagem::Crawler::Authentication
+
+  def log(msg)
+    Stratagem.logger.debug msg
+  end
+  
+  def crawler_session(application_model=nil)
+    @model = application_model
+    @model ||= Stratagem::ModelBuilder.new.run
+    @redirect_proc = Proc.new {|redirect_url| handle_redirect(redirect_url) }
+    @parameter_types = {} # routecontainer -> {:route_segment => Model, :route_segment => Model}
+    open_session do |session|
+      @session = session
+      phase(:unauthenticated)
+      yield
+    end
+  end
+  
+  def application_model
+    @model
+  end
+  
+  def site_models
+    @site_models ||= {} # builds models of the site for various phases of analysis, see PHASES
+  end
+
+  def site_model
+    raise Stratagem::Crawler::CrawlError.new("Phase not specified") unless @current_model
+    @current_model
+  end
+
+  def phase(new_phase)
+    @current_phase = new_phase
+    @current_model = site_models[@current_phase] ||= Stratagem::Crawler::SiteModel.new
+  end
+
+  def phases
+    site_models
+  end
+
+  def display
+    # print out pages and inbound / outbound links
+    site_model.pages.each do |page|
+      log "Page: #{page.url} - #{page.title} - #{page.response.code}"
+      page.outbound_edges.each do |edge|
+        log "\tout: #{edge.to.url} - #{edge.to.title} - #{edge.to.route}"
+      end
+      page.inbound_edges.each do |edge|
+        log "\tin: #{edge.from.url} - #{edge.from.title}"
+      end
+    end
+  end
+
+  def crawl
+    # grab all pages independently
+    @model.routes.invalid.each {|route_container|
+      puts "skipping invalid route #{route_container.route.to_s}"
+    }
+
+    authentication_controller = nil
+    if (@current_phase == :authenticated)
+      route = @model.routes.recognize(authentication.login_page)
+      authentication_controller = route.controller
+    end
+    
+    @model.routes.each {|route_container|
+      if authentication_controller && route_container.controller && (route_container.controller.klass == authentication_controller.klass)
+        log "Skipping authentication routes #{route_container.route.to_s}"
+      else
+        visit(route_container) unless @model.routes.invalid.include?(route_container)
+      end
+    }
+
+    # establish edges on site graph
+    site_model.pages.each do |page|
+      site_model.add_edge(page, page.redirected_to, :redirect) if page.redirected?
+    end
+
+    site_model
+  end
+
+  private
+
+  def visit(route_container)
+    puts "visiting #{route_container.route}"
+    build_urls(route_container).each do |route_info|
+      call_route(route_container, route_info)
+    end
+  end
+
+  def call_route(route, route_info)
+    puts 'CALLING: .'+route_info[:verb].downcase+". - "+route_info[:path]
+    verb = route_info[:verb].downcase
+    verb = 'get' if verb == '' || verb == 'any'
+    case verb
+      when 'get'
+        do_get(route, route_info[:path])
+        puts response.code if response
+      when 'post'
+      when 'put'
+      when 'delete'
+      else
+        raise "Unsupported verb: #{route[:verb]}"
+    end
+  end
+
+  # Builds a list of string URLs for a given route. This is done
+  # by replacing :xyz_id segments in the route with known values
+  # from the well
+  def build_urls(route_container)
+    urls = []
+    route = route_container.route
+    parameter_types = (@parameter_types[route_container] ||= resolve_parameter_types(route_container))
+    route_infos, params = build_url(route_container, parameter_types)
+    puts "route: #{route_container.route.to_s} permutations:"
+    route_infos.each do |info|
+      puts "\t#{info[:path]}"
+    end
+    route_infos
+  end
+
+  def url_permutations(meta_segments, segment_stack=[], &block)
+    if (segment_stack.size == meta_segments.size)
+      yield segment_stack
+    else
+      cursor = segment_stack.size
+      options = meta_segments[cursor]
+      if (options.kind_of?(Array))
+        options.each do |option|
+          url_permutations(meta_segments, segment_stack + [option], &block)
+        end
+      else
+        url_permutations(meta_segments, segment_stack + [options], &block)
+      end
+    end
+  end
+
+  def build_url(route_container, parameter_types={})
+    params = {}
+    route = route_container.route
+    name = route.to_s
+    verb = route.conditions[:method].to_s
+
+    parameter_types ||= {}
+
+    i = 12345
+    segs = route.segments.inject([]) {|accumulated,segment|
+      s = segment.to_s
+      if (s =~ /^:/)
+        model = parameter_types[s]
+        value = nil
+        if (model)
+          value = (Stratagem::AutoMock::Aquifer.instance.instances_of model).map {|inst| 
+            attr_name = s.gsub(/^:/, '').to_sym
+            if inst.methods_include?(attr_name)
+              puts "#{attr_name} is a method on the object"
+              inst.send(attr_name)
+            else
+              puts "#{attr_name} is being mapped to the id on the object"
+              inst.id
+            end
+          }
+        else
+          i += 1
+          value = [i]
+        end
+        accumulated << value
+        params[s] = value
+      else
+        accumulated << s
+      end
+      accumulated
+    }
+
+    routes = []
+    reqs = route.requirements.empty? ? "" : route.requirements.inspect
+    url_permutations(segs) do |segments|
+      path = segments.join('').gsub('(.:format)', '').gsub(/\/$/, '')
+      permutation = {:name => name, :verb => verb, :segs => segs, :reqs => reqs, :path => path}
+      routes << permutation
+    end
+
+    [routes, params]
+  end
+
+  def handle_redirect(redirect_url)
+    existing_pages = site_model.pages_for(response.redirect_url)
+    if (existing_pages.size > 0)
+      existing_pages.first
+    else
+      invocation_delta = model_invocations_for_request do
+        get redirect_url
+      end
+
+      site_model.add(nil, response) {|redirect_url|
+        # TODO - record as bug!
+        puts "recursive redirect #{redirect_url}"
+      }
+    end
+  end
+
+  def do_get(route,path)
+    begin
+      get path
+      site_model.add(route, response) {|redirect_url| @redirect_proc.call(redirect_url) }
+    rescue
+      puts $!.message
+      puts path
+      #puts $!.backtrace
+    end
+  end
+
+  def resolve_parameter_types(route_container)
+    puts "resolving parameter types for #{route_container.route}"
+    resolved_parameters = {}
+    route_infos, params = build_url(route_container, resolved_parameters)
+    route_info = route_infos.first
+    unknown_params = params.keys
+    puts "unknown params: #{unknown_params}"
+    progress = nil
+    while ((unknown_params.size > 0) && (progress.nil? || (progress > 0)))
+      progress = 0
+
+      # p route_infos
+      # puts "---"
+      delta = model_invocations_for_request do
+        call_route(route_container, route_info)
+      end
+
+      puts "delta - #{delta.size}"
+      #p delta
+
+      unknown_params.clone.each do |key|
+        value = params[key]
+        value_s = params[key].map {|v| v.to_s }
+        delta.each do |invocation|
+          puts "#{route_info[:path]} - #{invocation.model_class.name} - #{invocation.method} - #{invocation.args.inspect} - #{value_s}"
+
+          # TODO inspect is a hack, refactor
+          if (invocation.args.include?(value.first)) || (invocation.args.inspect.include?('"'+value.first.to_s+'"'))
+            # found match
+
+            puts "\tresolved #{key} to #{invocation.model_class}"
+            unknown_params.delete(key)
+            resolved_parameters[key] = invocation.model_class
+            progress += 1
+
+            break
+          end
+        end
+      end
+
+      route_infos, params = build_url(route_container, resolved_parameters)
+      route_info = route_infos.first
+    end
+    if (resolved_parameters.size > 0)
+      resolved_parameters
+    else
+      nil
+    end
+  end
+
+
+end

data/lib/stratagem/crawler/site_model.rb

@@ -0,0 +1,138 @@
+module Stratagem::Crawler
+  class SiteModel
+    include Stratagem::Crawler::HtmlUtils
+
+    attr_reader :pages, :edges
+
+    def initialize
+      @pages = []
+      @edges = []
+    end
+
+    def export
+      {
+        :pages => @pages.map {|page| page.export },
+        :edges => @edges.map {|edge| edge.export }
+      }
+    end
+
+    def add_edge(from,to,type)
+      self.edges << Edge.new(from,to,type)
+    end
+
+    def add(route, response, &block)
+      page = Page.new(self, response, &block)
+      self.pages << page
+      page
+    end
+
+    def pages_for(id)
+      if (id.kind_of?(String))
+        pages.select {|page| page.url == id }
+      else
+        pages.select {|page| page.route == id }
+      end
+    end
+
+  end
+
+  class Edge
+    # type -> :link, :redirect,
+    attr_accessor :from, :to, :type
+
+    def initialize(from, to, type)
+      @from = from
+      @to = to
+      @type = type
+    end
+    
+    def export
+      {
+        :from => from.object_id,
+        :to => to.object_id,
+        :type => type
+      }
+    end
+  end
+
+  class Page
+    include Stratagem::Crawler::HtmlUtils
+
+    attr_reader :response
+    
+    attr_accessor :url
+    attr_accessor :path
+    attr_accessor :method
+    attr_accessor :redirected_to
+    attr_accessor :document
+
+    def initialize(site_model, response, &block)
+      @site_model = site_model
+      init(response, &block)
+    end
+
+    def route
+      Stratagem::Model::Application.instance.routes.recognize(self)
+    end
+
+    def export
+      {
+        :external_id => self.object_id,
+        :url => url,
+        :path => path,
+        :method => method,
+        :redirected_to_page_external_id => redirected_to ? redirected_to.object_id : nil,
+        :route_external_id => route.object_id
+      }
+    end
+
+    def init(response, &block)
+      @response = response.clone
+      @url = response.request.url
+      @path = response.request.path
+      @method = response.request.method
+      @document = Nokogiri::HTML(response.body)
+      self.redirected_to = block.call(response.redirect_url) if response.redirect?
+    end
+
+    def reload(&block)
+      # TODO - should support all the verbs and params, but
+      # hack together for now to reload the authenticity token
+      response = yield url
+      init(response) {|redirected_to| }
+    end
+
+    def redirected?
+      !self.redirected_to.nil?
+    end
+
+    def forms
+      self.parse_forms(@document)
+    end
+
+    def login_form
+      self.find_login_form(@document)
+    end
+
+    def to_html
+      @document.to_html
+    end
+
+    def outbound_edges(type=nil)
+      @site_model.edges.select {|edge| edge.from == self && (type.nil? || (type == edge.type)) }
+    end
+
+    def inbound_edges(type=nil)
+      @site_model.edges.select {|edge| (edge.to == self) && (type.nil? || (type == edge.type)) }
+    end
+
+    def title
+      unless @title
+        title = (@document/'head title').first
+        @title = title.inner_html if title
+      end
+      @title
+    end
+  end
+
+end

data/lib/stratagem/crawler/trace_utils.rb

@@ -0,0 +1,10 @@
+module Stratagem::Crawler
+  module TraceUtils
+    def model_invocations_for_request()
+      prior_invocations = ActiveRecord::Base.stratagem.invocations_audit.clone
+      yield
+      post_invocations = ActiveRecord::Base.stratagem.invocations_audit.clone
+      delta = post_invocations - prior_invocations
+    end
+  end
+end

data/lib/stratagem/crawler.rb

@@ -0,0 +1,9 @@
+module Stratagem::Crawler
+end
+
+require 'stratagem/crawler/form'
+require 'stratagem/crawler/trace_utils'
+require 'stratagem/crawler/html_utils'
+require 'stratagem/crawler/authentication'
+require 'stratagem/crawler/session'
+require 'stratagem/crawler/site_model'

data/lib/stratagem/extensions/class.rb

@@ -0,0 +1,9 @@
+class Class
+  def classes
+    list = [self]
+    c = self
+    list << c while ((c = c.superclass) != nil)
+    list
+  end
+end
+

data/lib/stratagem/extensions/hash.rb

@@ -0,0 +1,16 @@
+class Hash
+  alias_method :ruby_get, :[]
+  attr_reader :hash_reads, :hash_writes
+
+  def [](name)
+    (@hash_reads ||= []) << name if (@auditing)
+    ruby_get name
+  end
+
+  def enable_auditing
+    @auditing = true
+  end
+  
+  # def []=
+  # end
+end

data/lib/stratagem/extensions/module.rb

@@ -0,0 +1,11 @@
+class Module
+  def subclasses
+    classes = []
+    ObjectSpace.each_object(Class) do |c|
+      next unless c.superclass == self || c.ancestors.include?(self)
+      next if c == self
+      classes << c
+    end
+    classes
+  end
+end

data/lib/stratagem/extensions/object.rb

@@ -0,0 +1,15 @@
+class Object
+  def methods_include?(name)
+    methods.include?(name.to_sym) || methods.include?(name.to_s)
+  end
+
+  def self.subclasses
+    classes = []
+    ObjectSpace.each_object(Class) do |c|
+      next unless c.superclass == self || c.ancestors.include?(self)
+      next if c == self
+      classes << c
+    end
+    classes
+  end
+end

data/lib/stratagem/extensions/red_parse.rb

@@ -0,0 +1,86 @@
+class RedParse::Node
+  def talking_about?(model)
+    talking_about = false
+
+    if self.receiver.methods_include?(:name)
+      if ((self.receiver) && self.receiver.name == model.klass.name)
+#        puts "\treceiver name #{self.receiver.name} is equal to #{model.klass.name}"
+        talking_about = true
+      elsif (self.receiver)
+        name = self.receiver.name
+        # if it's lowercase then it's possibly referencing an association
+        if (name.downcase == name)
+          # look through the models to see if one has an association with this name and class
+          Stratagem::Model::Application.instance.models.each do |app_model|
+            talking_about = app_model.association_match?(name, model)
+            break if talking_about
+          end
+        end
+      end
+    end
+    
+    talking_about
+  end
+
+  # iterates the node and figures out the class of the leaf that a call is talking about
+  def dereference
+    # recursion is evil
+    path = []
+    current_node = self
+    # puts "current:#{current_node.inspect}"
+    if (current_node.kind_of?(RedParse::VarNode))
+      path << current_node.name if current_node.methods_include?(:name)
+    else
+      while (current_node.methods_include?(:receiver) && (current_node = current_node.receiver) != nil)
+        path << current_node.name if current_node.methods_include?(:name)
+      end
+    end
+    # p path
+    previous = nil
+    path.reverse!
+    path.map! {|element|
+      new_val = element
+      if (element.downcase != element)
+        # class?
+        begin
+          new_val = Class.class_eval(element)
+        rescue
+          logger.error "Unable to load class #{element} in #{path.inspect}"
+        end
+      elsif (element.pluralize == element)
+        # collection?
+        if (previous)
+          if (previous.kind_of?(Class))
+            # look up the loaded model of the class
+            model = Stratagem::Model::Application.instance.models.find {|m| m.klass == previous }
+            if (model)
+              match = previous.reflect_on_all_associations.find {|assoc| assoc.name.to_sym == element.to_sym }
+              if (match)
+                new_val = Class.class_eval(match.class_name)
+              else
+                logger.error "I give up, cannot determine what #{element} is in #{path.inspect}"
+              end
+            end
+          else
+            logger.error "Unknown previous type when trying to dereference a collection #{element} in #{path.inspect}"
+          end
+        else
+          logger.error "Unable to determine prior element for #{element} in #{path.inspect}"
+        end
+      elsif previous
+        # call to the class / object in the previous statement
+        if (['find','all','first','last'].include?(element))
+          # then expect to receive an instance of the previous type
+          new_val = previous
+        end
+      end
+      previous = new_val
+      new_val
+    }
+    path.last
+  end
+
+  def logger
+    Rails.logger
+  end
+end

data/lib/stratagem/extensions/string.rb

@@ -0,0 +1,20 @@
+class String
+  def stratagem_strip_erb
+    self.gsub(/^[<%=]+/, '').gsub(/[-%>]+$/,'').strip
+  end
+
+  def stratagem_contains_token?(token)
+    match = false
+    [
+      Regexp.compile("^#{token}$", true),
+      Regexp.compile("^#{token}[^A-Za-z0-9]", true),
+      Regexp.compile("[^A-Za-z0-9]#{token}$", true),
+    ].each do |regex|
+      if (regex.match(self))
+        match = true
+        break
+      end
+    end
+    match
+  end
+end

data/lib/stratagem/extensions.rb

@@ -0,0 +1,6 @@
+require 'stratagem/extensions/red_parse'
+require 'stratagem/extensions/class'
+require 'stratagem/extensions/string'
+require 'stratagem/extensions/hash'
+require 'stratagem/extensions/object'
+require 'stratagem/extensions/module'

data/lib/stratagem/framework_extensions/controllers/action_controller.rb

@@ -0,0 +1,10 @@
+class ActionController::Request
+  alias_method :rails_parameters, :parameters
+  undef_method :parameters
+
+  def parameters
+    params = rails_parameters
+    params.enable_auditing
+    params
+  end
+end

data/lib/stratagem/framework_extensions/controllers/action_mailer.rb

@@ -0,0 +1,12 @@
+module ActionMailer
+  class Base
+    def create!(method_name, *parameters)
+      puts "Skipping action mailer #{method_name}, #{parameters.inspect}"
+    end
+
+    def deliver!(mail = @mail)
+      puts "Skipping delivery in action mailer"
+    end
+  end
+end
+

data/lib/stratagem/framework_extensions/controllers.rb

@@ -0,0 +1,5 @@
+module Stratagem::ApplicationExtensions::Controllers; end
+
+require 'stratagem/framework_extensions/controllers/action_controller'
+require 'stratagem/framework_extensions/controllers/action_mailer'
+

data/lib/stratagem/framework_extensions/models/adapters/active_model/detect.rb

@@ -0,0 +1,7 @@
+module Stratagem::ApplicationExtensions::Models::Adapters::ActiveModel
+  class Detect < Stratagem::ApplicationExtensions::Models::Detect
+    def self.supports?(model)
+      model.ancestors.include?(ActiveRecord::Base)
+    end
+  end
+end

data/lib/stratagem/framework_extensions/models/adapters/active_model/extensions.rb

@@ -0,0 +1,35 @@
+class ActiveRecord::Base
+  class << self
+    def removed_methods=(methods)
+      @@removed_methods = methods
+    end
+
+    def removed_methods
+      @@removed_methods
+    end
+
+    def removed_validators=(validators)
+      @@removed_validators = validators
+    end
+  end
+
+  Stratagem::ApplicationExtensions::Models::Annotations.configure(self)
+
+end
+
+module ActiveRecord::Validations::ClassMethods
+  @@removed_validators = []
+  
+  instance_methods.each do |m|
+    if (m =~ /^validates_/) && (m !~ /validates_each/)
+      alias_method "old_#{m}", m
+      undef_method m
+      puts "removing validator #{m}"
+      @@removed_validators << m.to_sym
+    end
+  end
+
+  def removed_validators
+    @@removed_validators
+  end
+end