stratagem 0.1.7 → 0.1.8

data/lib/stratagem/framework_extensions/models/tracing.rb

@@ -3,6 +3,10 @@ module Stratagem::ApplicationExtensions::Models
 
     @@invocations_audit = []
 
+    def self.invocations_audit
+      @@invocations_audit
+    end
+
     def invocations_audit
       @@invocations_audit
     end
@@ -28,26 +32,44 @@ module Stratagem::ApplicationExtensions::Models
       invocations_audit.clear
     end
 
-    def write_invocation(model_instance, method, args)
-      invocation(method, args, write_invocations, model_instance)
+    def write_invocation(object, model, method, args)
+      invocation(method, args, write_invocations, :write, object, model)
     end
 
     def read_invocation(method, *args)
       # ensure that the read did not stem from a write operation
-      path,action,line = controller_trace(/active_record\/base\.rb/)
-      invocation(method, args, read_invocations) unless (action =~ /create/) || (action =~ /update/) || (action =~ /save/)
+      path,action,line,trace,index = controller_trace(/active_record\/base\.rb/)
+      invocation(method, args, read_invocations, :read) unless (action =~ /create/) || (action =~ /update/) || (action =~ /save/)
     end
 
-    def invocation(method, args, enumeration, model_instance=nil)
-      path,action,line = controller_trace
+    def invocation(method, args, enumeration, type, object=nil, alternate_model=nil)
+      path,action,line,trace,index = controller_trace
       args = args.first if args && (args.size == 1) && (args.first.kind_of?(Array))
-      add_invocation enumeration, MethodInvocation.new(method, path, action, line, model_instance, model, caller, args) if (path)
+      add_invocation enumeration, MethodInvocation.new(method, path, action, line, object, alternate_model || model, caller, args, type) if (path)
     end
 
     def controller_trace(regex = /_controller\.rb/)
-      controller_trace = caller.select {|c| c =~ regex }.last
-      if controller_trace
-        path,line,action = controller_trace.split(':')
+      trace_index = nil
+      trace_line = nil
+      caller.reverse.each_with_index do |line,i|
+        if (line =~ regex)
+          trace_index = i
+          trace_line = line
+          break
+        end
+      end
+
+      if trace_line
+        path,action,line = parse_trace_line(trace_line)
+        [path,action,line,caller,trace_index]
+      else
+        []
+      end
+    end
+
+    def parse_trace_line(trace_line)
+      path,line,action = trace_line.split(':')
+      if (action)
         action.gsub!(/[`']/, '').gsub!('in ', '')
         line = line.to_i
         [path,action,line]

data/lib/stratagem/framework_extensions/models.rb

@@ -7,13 +7,13 @@ require 'stratagem/framework_extensions/models/tracing'
 require 'stratagem/framework_extensions/models/annotations'
 require 'stratagem/framework_extensions/models/detect'
 
-base = File.join(File.dirname(__FILE__), 'models', 'adapters', 'common')
+base = File.join(File.dirname(__FILE__), 'models', 'adapters', 'util')
 Dir.entries(base).select {|s| s =~ /\.rb$/}.each {|helper|
   require File.join(base, helper.gsub(/\.rb/, ''))
 }
 
 base = File.join(File.dirname(__FILE__), 'models', 'adapters')
-Dir.entries(base).select {|s| s !~ /^\./ && s != 'common' }.each {|adapter_dir|
+Dir.entries(base).select {|s| s !~ /^\./ && s != 'util' }.each {|adapter_dir|
   require File.join(base, adapter_dir, 'detect')
   require File.join(base, adapter_dir, 'tracing')
   require File.join(base, adapter_dir, 'metadata')

data/lib/stratagem/model/application.rb

@@ -31,8 +31,7 @@ module Stratagem::Model
     
     def export
       puts "exporting site model"
-      p crawler.export
-      puts "done."
+      references = @controllers.map {|c| c.references }.flatten.map {|r| r.export }.uniq
       h = {
         :rails_version => rails_version,
         :models => @models.export, 
@@ -41,9 +40,10 @@ module Stratagem::Model
         :views => @views.export,
         :gems => @gems.export,
         :plugins => @plugins.export,
-        :site_model => crawler.export,
-        :references => []
+        :site_model => crawler ? crawler.export : nil,
+        :references => references
       }
+      p h
       h
     end
 
@@ -114,6 +114,10 @@ module Stratagem::Model
       @components.each {|e| yield e }
     end
 
+    def map
+      @components.map {|e| yield e }
+    end
+
     def << (component)
       if (component.kind_of?(Array))
         component.each {|e| 

data/lib/stratagem/model/components/base.rb

@@ -30,11 +30,14 @@ module Stratagem::Model::Component
         source = File.read(path)
         begin
           parse_tree = RedParse.new(source).parse
+          puts "parsed.."
           Stratagem::Model::ParseUtil.find_classes(parse_tree).map {|klass|
             self.new(path, parse_tree, klass)
           }
         rescue
+          puts "error loading #{path}"
           puts $!.message
+          puts $!.backtrace
           logger.fatal "Unable to load parse tree for #{path}"
           []
         end

data/lib/stratagem/model/components/controller.rb

@@ -2,26 +2,19 @@ require 'set'
 
 module Stratagem::Model::Component
   class Action
+    attr_reader :name, :controller
+
     def initialize(controller, name, method)
+      @controller = controller
       @name = name
       @method = method # :put, :get, :post, :delete
       @models_rendered = {} # model => [MethodInvocation]
     end
 
-    def models_read
-      models(ActiveRecord::Base.stratagem.read_invocations)
-    end
-    
-    def models_modified
-      models(ActiveRecord::Base.stratagem.write_invocations)
-    end
-
-    def models(invocations)
-      invocations.values.select {|invocation|
+    def model_invocations
+      Stratagem::ApplicationExtensions::Models::Tracing.invocations_audit.select {|invocation|
         (invocation.controller_path == controller.path) &&
-          (invocation.controller_action == self.name)
-      }.map {|invocation|
-        invocation.model_class
+          (invocation.controller_action.to_s == self.name.to_s)
       }.uniq
     end
 
@@ -103,16 +96,22 @@ module Stratagem::Model::Component
       []
     end
    
-    private
-    
-    def create_reference(node, method, model)
-      Reference.new(
-        :from_class => self.klass, 
-        :to_class => model.klass, 
-        :method => method, 
-        :line_number => node.linerange.first,
-        :reference_type => :write
-      )
+    def references
+      actions.map {|action| 
+        action.model_invocations.map {|inv| inv.to_reference }
+      }.flatten
     end
+   
+    private
+
+    # def create_reference(node, method, model)
+    #   Reference.new(
+    #     :from_class => self.klass, 
+    #     :to_class => model.klass, 
+    #     :method => method, 
+    #     :line_number => node.linerange.first,
+    #     :reference_type => :write
+    #   )
+    # end
   end
 end

data/lib/stratagem/model/components/model.rb

@@ -130,6 +130,7 @@ module Stratagem::Model::Component
     def export
       adapters = stratagem? ? klass.stratagem.callbacks.map {|c| c.class.name } : []
       {
+        :external_id => self.object_id,
         :type => :model, 
         :path => @path.gsub(RAILS_ROOT+'/', ''), 
         :class_name => @klass.name, 
@@ -143,7 +144,6 @@ module Stratagem::Model::Component
         :whitelists_attributes => stratagem? ? @klass.stratagem.whitelists_attributes? : nil,
         :blacklists_attributes => stratagem? ? @klass.stratagem.blacklists_attributes? : nil,
         :instance_methods => @model_instance_methods,
-        :referenced_by => @model_referenced_by.map {|r| r.export },
         :relations => relations.map {|r| r.export },
         :adapters => adapters
       }
@@ -154,7 +154,8 @@ module Stratagem::Model::Component
     def relations
       if (stratagem?)
         @relations ||= klass.stratagem.relations.map {|relation|
-          Reference.new(:reference_type => relation.macro, :from_class => self.klass, :to_class => relation.klass)
+          to = app_model.models.find {|m| m.klass == relation.klass }
+          Reference.new(:reference_type => relation.macro, :from_component => self, :to_component => to, :options => relation.options)
         }
       else
         []

data/lib/stratagem/model/components/reference.rb

@@ -1,30 +1,41 @@
 module Stratagem::Model::Component
   class Reference
     attr_accessor :reference_type # :read, :write
-    attr_accessor :from_class, :to_class
-    attr_accessor :line_number
-    attr_accessor :method
+    attr_accessor :from_component, :to_component
+    attr_accessor :line_number, :method, :function, :options, :stack_trace
 
-    Vars = [:reference_type, :from_class, :to_class, :line_number, :method]
+    Vars = [:reference_type, :from_component, :to_component, :line_number, :method, :function, :options, :stack_trace]
     
     def initialize(args={})
       args.each {|key,val| self.send("#{key}=", val) }
     end
 
     def ==(other)
-      Vars.each do |attribute|
-        self.send(attribute) == other.send(attribute) 
-      end
+      Vars.find {|attribute|
+        self.send(attribute) != other.send(attribute) 
+      }.nil?
     end
     
     def export
-      h = {}
-      Vars.each do |key|
-        h[key.to_s] = self.send(key).to_s
+      h = {
+        :external_id => self.object_id,
+        :reference_type => reference_type,
+        :from_component_external_id => from_component.object_id,
+        :to_component_external_id => to_component.object_id,
+        :line_number => line_number,
+        :method => method,
+        :function => function,
+        :options => options ? options.to_json : nil,
+        :stack_trace => reference_type == :write ? compressed_stack_trace : nil
+      }
+    end
+    
+    def compressed_stack_trace
+      if (stack_trace)
+        TraceDeflator.deflate(stack_trace)
+      else
+        nil
       end
-      h[:from_class.to_s] = from_class.name
-      h[:to_class.to_s] = to_class.name
-      h
     end
   end
 end

data/lib/stratagem/model/components/route.rb

@@ -36,9 +36,6 @@ module Stratagem::Model::Component
     end
 
     def export
-      p @route.requirements
-      p action
-      puts "--"
       {
         :external_id => self.object_id,
         :type => :route, 

data/lib/stratagem/model/components/view.rb

@@ -32,6 +32,7 @@ module Stratagem::Model::Component
     def export
       begin
         {
+          :external_id => self.object_id,
           :type => :view, 
           :path => @path, 
           :render_path => @render_path, 

data/lib/stratagem/model_builder.rb

@@ -1,10 +1,9 @@
 module Stratagem
   class ModelBuilder
-    attr_reader :parsed_models, :parsed_controllers, :aquifer 
+    attr_reader :parsed_models, :parsed_controllers
 
     def initialize
       @model = Stratagem::Model::Application.instance
-      @aquifer = Stratagem::AutoMock::Aquifer.init(@model)
     end
 
     def run
@@ -16,7 +15,6 @@ module Stratagem
 
       print_errors
       
-      @aquifer.fill
       @model
     end
 
@@ -49,12 +47,12 @@ module Stratagem
         models.each do |c|
           log "\t#{c.klass.name} loaded from #{model}"
 
-          references = []
-          @model.controllers.each do |controller|
-            references += controller.modifies(c)
-          end
-          log "\t\t#{references.size} references from controllers"
-          c.model_referenced_by = references
+          # references = []
+          # @model.controllers.each do |controller|
+          #   references += controller.modifies(c)
+          # end
+          # log "\t\t#{references.size} references from controllers"
+          # c.model_referenced_by = references
         end
         @model.models << models
       }
@@ -101,9 +99,9 @@ module Stratagem
           controller_name << 'Controller'
           controller_class = controllers.find {|controller| controller.klass.name == controller_name }
           controller_object = controller_class ? controller_class.klass.new : nil
-          controller_action = route.parameter_shell[:action].to_sym
+          controller_action = route.parameter_shell[:action] ? route.parameter_shell[:action].to_sym : nil
 
-          if (controller_object) && (controller_object.methods_include?(controller_action))
+          if (controller_object && controller_action) && (controller_object.methods_include?(controller_action))
             controllers.each do |controller|
               controller.add_routable_action(controller_action, route.conditions[:method] || :get)
             end

data/lib/stratagem/site_crawler.rb

@@ -9,35 +9,30 @@ module Stratagem
     def run
       crawler_session(@application_model) do
         log "crawling site"
-        phase(:unauthenticated)
-        crawl
-        display
-        authenticated = authenticate
-        
-        if (authenticated)
-          phase(:authenticated)
+        page_set(:unauthenticated) do |pages|
+          puts "SET: #{pages.object_id}"
           crawl
           display
         end
+
+        users.each do |user|
+          page_set("user_#{user.id}") do |pages|
+            authenticate(user) do
+              puts "authenticated with #{user.stratagem.mock_attributes.inspect}"
+              crawl
+              crawl(:put)
+            end
+          end
+        end
+
       end
       
       self
     end  
     
     def export
-      phases = site_models.map {|phase,model| 
-        h = model.export
-        h[:name] = phase
-        h
-      }
       {
-        :authentication => {
-          :success => authentication.success, 
-          :login_page_external_id => authentication.login_page.object_id, 
-          :response_page_external_id => authentication.response_page.object_id,
-          :ssl => authentication.ssl
-        },
-        :phases => phases
+        :page_sets => site_models.map {|site_model| site_model.export }
       }
     end
 

data/lib/stratagem.rb

@@ -2,7 +2,7 @@ class StratagemError < RuntimeError
   attr_accessor :target
 
   def initialize(*args)
-    super(*args)
+    super(args.first)
     (@@all ||= []) << self
   end
 end

data/spec/model/component_spec.rb

@@ -0,0 +1,43 @@
+require 'spec/spec_helper'
+require 'lib/security'
+
+module Security
+  module Model
+    module Component
+      class View
+        def full_path
+          File.join(RAILS_ROOT, 'spec', 'samples', @render_path+"."+@extension)
+        end
+      end
+    end
+  end
+end
+
+
+describe Security::Model::Component::Model do
+  before do
+    path = File.join(RAILS_ROOT, 'spec', 'samples', 'sample_model.rb')
+    models = Security::::Model::Component::Model.load_all(path)
+    @model = models.first
+  end
+  
+  it "should not error on serialize" do
+    lambda { @model.export.to_json }.should_not raise_exception
+  end
+  
+  it "should export a hash" do
+    @model.export.should be_kind_of(Hash)
+  end
+end
+
+describe Security::::Model::Component::View do
+  before do
+    @view = Security::::Model::Component::View.new('index.html.erb')
+    @template = @view.read
+  end
+  
+  it "should read the template from disk" do
+    @template.should_not be_nil
+    @template.size.should > 0
+  end
+end

data/spec/model/components/view_spec.rb

@@ -0,0 +1,43 @@
+require 'spec/spec_helper'
+require 'lib/security'
+
+describe Security::Model::Component::View do
+  before do
+    @view = Security::Model::Component::View.new('signup.html.erb')
+    @template = @view.read
+  end
+
+  describe :file_system_pointers do
+    it "should give the correct full path" do
+      File.exists?(@view.full_path).should be_true
+    end
+  
+    it "should give the correct directory" do
+      @view.directory.should eql(File.join(RAILS_ROOT, 'spec', 'samples'))
+    end
+  end
+
+  describe :loading do
+    it "should read the template from disk" do
+      @template.should_not be_nil
+      @template.size.should > 0
+    end
+  end
+
+  describe :html_extraction do
+    it "should identify the models that the forms are talking about" do
+      @view.forms.first.model.should eql(User)
+    end
+
+    it "should have 2 forms" do
+      @view.forms.each {|f| p f.export }
+      @view.forms.size.should eql(2)
+    end
+
+    it "should have 3 fields in each form" do
+      @view.forms.each {|form|
+        form.fields.size.should eql(3)
+      }
+    end
+  end
+end

data/spec/model/test_spec.rb

@@ -0,0 +1,10 @@
+require 'spec/spec_helper'
+require 'lib/security'
+
+describe Object do
+  it "should test" do
+    source = File.open(File.join(RAILS_ROOT, "spec","samples","sample_model.rb")).readlines.join("\n")
+    tree = RedParse.new(source).parse
+    p tree.first.linerange.first
+  end
+end

data/spec/samples/404.html.erb

@@ -0,0 +1,30 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+       "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+  <title>The page you were looking for doesn't exist (404)</title>
+	<style type="text/css">
+		body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+		div.dialog {
+			width: 25em;
+			padding: 0 4em;
+			margin: 4em auto 0 auto;
+			border: 1px solid #ccc;
+			border-right-color: #999;
+			border-bottom-color: #999;
+		}
+		h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+	</style>
+</head>
+
+<body>
+  <!-- This file lives in public/404.html -->
+  <div class="dialog">
+    <h1>The page you were looking for doesn't exist.</h1>
+    <p>You may have mistyped the address or the page may have moved.</p>
+  </div>
+</body>
+</html>

data/spec/samples/_form.html.erb

@@ -0,0 +1,8 @@
+<%= form.label :login %><br />
+<%= form.text_field :login %><br />
+<br />
+<%= form.label :password, form.object.new_record? ? nil : "Change password" %><br />
+<%= form.password_field :password %><br />
+<br />
+<%= form.label :password_confirmation %><br />
+<%= form.password_field :password_confirmation %><br />

data/spec/samples/index.html.erb

@@ -0,0 +1,77 @@
+<div class="contact">email1@clearnetsec.com</div>
+<div class="contact">&email2@clearnetsec.com-</div>
+
+	<strong>About</strong>
+	<p>
+	We are a team of security engineers, software developers and machine learning experts with a unique perspective on security.
+	ClearNet Security is a security services and development firm offering penetration testing, vulnerability assessments, and software development expertise since 2004.
+	</p>
+
+	<br />
+	
+	<strong>
+	Expertise for less
+	</strong>
+	<p>
+	We accept direct work and project based work from partner companies.  Our direct à la carte rate is $135 per hour.  
+	</p>
+</div>
+
+<div class="profiles">
+
+	<h1>ClearNet Security Principals</h1>
+	
+	<ul id="profiles">
+		<li>
+			<%= image_tag 'photos/tate.png', :class => 'profile' %>
+			<h3>Tate Hansen, Owner, Principal</h3>
+			<p>
+				Tate has 15+ years of engineering experience. He has specialized in security, including security
+				product development, security assessments, penetration testing, and building defensively strong 
+				systems.  Prior to ClearNet Security, Tate worked as a security engineer on both the 
+				Intrusion Detection and Vulnerability Assessment product teams at StillSecure, did a stint at Sun 
+				Microsystems where he solved critical networking problems for Sun’s customers, and was a 
+				member of Sun’s CCC Security Team.  Tate has performed well over 100 security assessments 
+				and is ClearNet Security’s PCI DSS engineer.  
+			</p>
+			<p>
+				<%= link_to image_tag("icons/linkedin_s.png"), "http://www.linkedin.com/in/tatehansen", :target => '_blank', :rel => 'nofollow', :class => :facebook  %>
+				<%= link_to image_tag("icons/twitter_s.png"), "http://www.twitter.com/tatehansen", :target => '_blank', :rel => 'nofollow', :class => :twitter  %>
+				<%= link_to image_tag("icons/blog_s.png"), "http://blog.clearnetsec.com", :target => '_blank', :rel => 'nofollow', :class => :blog  %>		
+			</p>
+			<div class="cf"></div>
+		</li>
+		<li>
+			<%= image_tag 'photos/cj.png', :class => 'profile' %>
+			<h3>Charles Grimes II, Owner, Principal</h3>
+			<p>
+				Charles has 12+ years of software design and engineering experience. He has a 
+				proven track record of successfully bringing new products and intellectual property to market. 
+				His teams have produced new data analysis technologies for Social Media, Business Activity 
+				Monitoring, Business Process Monitoring, Log Analysis and Security Information and Event 
+				Management. He has patented work in the area of adaptive, distributed data collection. 
+				Charles is ClearNet Security's principal technologist. 
+			<p>
+			<%= link_to image_tag("icons/linkedin_s.png"), "http://www.linkedin.com/in/charlesgrimes", :target => '_blank', :rel => 'nofollow', :class => :facebook  %>
+			<%= link_to image_tag("icons/twitter_s.png"), "http://www.twitter.com/cj2", :target => '_blank', :rel => 'nofollow', :class => :twitter  %>
+			</p>
+			<div class="cf"></div>
+		</li>
+	</ul>
+
+	<div class="cf"></div>
+
+	<br />
+
+	<h1>Our Team of Specialists</h1>
+	<p>
+	We have a great network of security and software specialist.  We frequently pull in specialist to provide 
+	specific expertise for the job at hand.  Our network includes team leaders of commercial intrusion detection
+	products and commercial vulnerability assessment products.  We work with AI (artificial intelligence) and 
+	machine learning experts and engineers with strong mathematics and cryptography experience.  Our software 
+	expertise covers Java, .NET, Ruby, Ruby on Rails, C, C++.  
+	</p>
+	<div class="cf">email1@clearnetsec.com&nbsp;</div>
+</div>
+<br />
+<div class="cf">&nbsp;</div>

data/spec/samples/sample_model.rb

@@ -0,0 +1,5 @@
+class SampleModel
+  def show
+    puts "hi"
+  end
+end

data/spec/samples/signup.html.erb

@@ -0,0 +1,14 @@
+<h1>Register</h1>
+ 
+<% form_for @user, :url => account_path do |f| %>
+  <%= f.error_messages %>
+  <%= render :partial => "form", :object => f, :locals => {:a => 'a'} %>
+  <%= f.submit "Register" %>
+<% end %>
+
+<% form_for @user, :url => account_path do |f| %>
+  <%= f.error_messages %>
+  <%= render :partial => "form", :object => f %>
+  <%= f.submit "Register" %>
+<% end %>
+