veracode 1.0.0.alpha3 → 1.0.0.alpha4

data/bin/veracode

@@ -16,12 +16,10 @@ require 'veracode'
 require 'veracode/version'
 
 $options =  {
-              :phase1 => false, 
-              :phase2 => false, 
-              :phase3 => true,
               :archive_source => true,
               :include_inherited => false,
               :jruby => false,
+              :environment => false,
             }
 
 subcommand = ARGV.shift
@@ -34,22 +32,13 @@ case subcommand
         $options[:verbose] = true
       end
 
-      opts.on("-a", "--all", "Archive objects at all stages") do
-        $options[:phase1] = true
-        $options[:phase2] = true
-      end
-
-      opts.on("-f", "--file", "Disassemble .rb files") do
-        $options[:disasm] = true
-      end
-
       opts.on("-j", "--jruby", "Force JRuby mode") do
         $options[:jruby] = true
       end
 
-      opts.on("--[no-]source", "[Don't] Include source code in archive") do |s|
-        $options[:archive_source] = s
-      end
+      # opts.on("--[no-]source", "[Don't] Include source code in archive") do |s|
+      #   $options[:archive_source] = s
+      # end
 
       opts.on("-D", "--debug", "Enable debug output") do
         $DEBUG = true
@@ -73,6 +62,23 @@ case subcommand
                     "       #{opts.program_name} help"
     end.parse!
 
+  when "environment", "env"
+    $options[:environment] = true
+
+    OptionParser.new do |opts|
+      opts.banner = "Usage: veracode environment [options]"
+
+      opts.on("-v", "--verbose", "Run verbosely") do
+        $options[:verbose] = true
+      end
+
+      opts.on("-D", "--debug", "Enable debug output") do
+        $DEBUG = true
+      end
+
+    end.parse!
+    Veracode.prepare
+
   else
     $stderr.puts "#{subcommand.dump} is not a valid subcommand"
 

data/lib/veracode/schema.rb

@@ -67,17 +67,20 @@ module Veracode
       schema = 'Veracode::' + File.read(schema_file).each_line.reject {|l| l =~ /^\s*#/}.join      
     rescue Exception => e
       puts "Unable to retrieve schema information from 'db/schema.rb'. Are your migrations up to date?"
-      log_error "Unable to archive 'db/schema.rb' (#{e.message})"      
+      log_error "Unable to retrieve schema from 'db/schema.rb' (#{e.message})"
+      add_to_archive  %Q|module "Veracode::Schema"\n|
+      add_to_archive  %Q|endmodule\n\n|
+      return
     end
-    
+
+    add_to_archive  %Q|module "Veracode::Schema"\n|
     begin
-      add_to_archive  %Q|module "Veracode::Schema"\n|
       eval(schema)
-      add_to_archive  %Q|endmodule\n\n|
     rescue Exception => e
       puts "Unable to evaluate schema information from 'db/schema.rb'. (#{e.message})"
       log_error "Unable to evaluate 'db/schema.rb' (#{e.message})"
     end
+    add_to_archive  %Q|endmodule\n\n|
   end
 
-end
+end

data/lib/veracode/version.rb

@@ -1,4 +1,4 @@
 module Veracode
-  VERSION = '1.0.0.alpha3'
+  VERSION = '1.0.0.alpha4'
   ARCHIVE_VERSION = '2012-07-04'
 end

data/lib/veracode.rb

@@ -1,3 +1,5 @@
+require 'pathname'
+require 'set'
 require 'zlib'
 require 'zip/zip'
 require 'veracode/version'
@@ -6,6 +8,7 @@ require 'veracode/gems'
 
 module Veracode
   @run_id = nil
+  @required_libs = Set.new
 
   # Metadata and method disassemblies for all Modules (.txt.gz)
   @disasmlog = nil
@@ -37,6 +40,7 @@ module Veracode
   def self.init
     @run_id = Time.now.strftime("%Y%m%d%H%M%S")
     @archive_dirname = File.join("tmp","veracode-#{@run_id}")
+    @required_libs.merge(["pathname", "set", "zlib", "zip/zip", "veracode"])
 
     if !Dir.exists?("tmp")
       begin
@@ -221,20 +225,33 @@ module Veracode
 
   ##############################################################################
   # Helpers
+  def self.cond_require(lib)
+    if @required_libs.add?(lib)
+      return require lib
+    end
+    return false
+  end
+
   def self.glob_require(files)
+    any_new = false
+    total, count = 0, 0
     Dir.glob(files) do |f|
       print "Requiring #{f.to_s} " if $options[:verbose]
       
       begin
-        require File.expand_path(f)
+        required = require File.expand_path(f)
       rescue Exception => e
         puts "(failed: #{e.message})" if $options[:verbose]
         log_error "Unable to require #{File.expand_path(f).to_s.dump} (#{e.message})"
       else
-        puts "(OK)" if $options[:verbose]
+        puts "(OK: #{(required ? "required" : "already required")})" if $options[:verbose]
       end
-
+      any_new |= required
+      total += 1
+      count += 1 if required
     end
+    puts "#{count}/#{total} files were required" if $options[:verbose]
+    any_new
   end
 
   def self.safe_name(o)
@@ -242,7 +259,11 @@ module Veracode
     when o == ActiveSupport::TimeWithZone
       "ActiveSupport::TimeWithZone"
     when o.is_a?(Module)
-      ( o.name.nil? ? o.to_s : o.name )
+      begin
+        ( o.name.nil? ? o.to_s : o.name.to_s )
+      rescue
+        o.to_s
+      end
     when o.is_a?(Method), o.is_a?(UnboundMethod)
       o.name.to_s
     else
@@ -307,7 +328,13 @@ module Veracode
   def self.prepare_archive
     @disasmlog = Zlib::GzipWriter.new(File.open(@disasmlog_filename, "wb"), nil, nil)
     @disasmlog.puts "#{RUBY_ENGINE}-#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}"
-    @disasmlog.puts "# " + `rails --version`.chomp
+    if $options[:environment]
+      @disasmlog.puts "# EnvironmentDef %s-%s_rails-%s" % [RUBY_ENGINE, RUBY_VERSION, Rails.version]
+    else
+      @disasmlog.puts "# Environment %s-%s_rails-%s" % [RUBY_ENGINE, RUBY_VERSION, Rails.version]
+    end
+    @disasmlog.puts "# Ruby #{RUBY_ENGINE}-#{RUBY_VERSION}"
+    @disasmlog.puts "# Rails #{Rails.version}"
     @disasmlog.puts
   end
 
@@ -384,7 +411,7 @@ module Veracode
         m.included_modules.map {|m| "include #{m.inspect.dump}\n" }.join : 
         ""
     ) + 
-    ( m.singleton_class.included_modules.count > 0 ? 
+    ( m.respond_to?(:singleton_class) && m.singleton_class.included_modules.count > 0 ? 
         m.singleton_class.included_modules.map {|m| "extend #{m.inspect.dump}\n" }.join : 
         ""
     )
@@ -420,23 +447,26 @@ module Veracode
       end
     end
 
-    if m.respond_to?(:global_variables)
-      m.global_variables.each do |v_symbol|
-        begin
-          v = eval(v_symbol.to_s)
-          formatted_contents += format_variable(v_symbol, v, "global")
-        rescue Exception => e
-          log_error "Error archiving global variable #{v_symbol.to_s.dump}: #{e.message}"
-          formatted_contents += format_variable(v_symbol, :veracode_nil, "global")
-        end
-        
+    begin
+      if m == Kernel
+        m.global_variables.each do |v_symbol|
+          begin
+            v = eval(v_symbol.to_s)
+            formatted_contents += format_variable(v_symbol, v, "global")
+          rescue Exception => e
+            log_error "Error archiving global variable #{v_symbol.to_s.dump}: #{e.message}"
+            formatted_contents += format_variable(v_symbol, :veracode_nil, "global")
+          end
+        end        
       end
+    rescue Exception => e
+      # m.respond_to?(:global_variables) was throwing exceptions
     end
 
     %w[ public protected private ].each {|p|
       get_methods = (p + "_instance_methods").to_sym
-      if m.respond_to?(get_methods) && m.send(get_methods, $options[:include_inherited]).count > 0
-        m.send(get_methods, $options[:include_inherited]).each do |m_symbol|
+      if m.respond_to?(get_methods) && m.__send__(get_methods, $options[:include_inherited]).count > 0
+        m.__send__(get_methods, $options[:include_inherited]).each do |m_symbol|
           begin
             method = m.instance_method(m_symbol)
             formatted_contents += format_method(method, "#{p.to_s}_instance", with_disasm)
@@ -488,7 +518,12 @@ module Veracode
   # Archiving Objects
   def self.archive(objects, with_disasm=true)
 
-    objects = objects - [Veracode]
+    objects = objects - [
+                          Veracode, 
+                          Veracode::ActiveRecord, 
+                          Veracode::ActiveRecord::Model, 
+                          Veracode::ActiveRecord::Schema,
+                         ]
 
     if $options[:verbose]
       puts "Archiving #{objects.count.to_s} objects" + (with_disasm ? " with disassembly" : "")
@@ -517,8 +552,8 @@ module Veracode
   def self.compile_templates
 
     begin
-      require 'action_view' unless defined? ActionView
-      require 'action_controller' unless defined? ActionController
+      cond_require 'action_view' unless defined? ActionView
+      cond_require 'action_controller' unless defined? ActionController
     rescue Exception => e
       log_error "Unable to satisfy haml dependencies (#{e.message})"
       return
@@ -541,12 +576,13 @@ module Veracode
     return unless templates.count > 0
 
     puts "Found #{templates.count} templates" if $options[:verbose]
+    log_error "Found #{templates.count} templates"
 
     haml_templates = templates.grep(/\.haml$/)
     if haml_templates.any?
       begin
-        require 'haml' unless defined? Haml
-        require 'haml/template/plugin' unless defined? Haml::Plugin
+        cond_require 'haml' unless defined? Haml
+        cond_require 'haml/template/plugin' unless defined? Haml::Plugin
       rescue Exception => e
         puts "Unable to satisfy haml dependencies"
         log_error "Unable to satisfy haml dependencies (#{e.message})"
@@ -563,13 +599,16 @@ module Veracode
       puts "Compiling template #{template}" if $options[:verbose]
 
       begin
+        # This render will fail, but will trigger compilation of template
         view.render(:file => template)
       rescue Exception => e
         log_error "Compiled template #{template} #{e.message}"
       end
     }
 
-    puts "Compiled #{ActionView::CompiledTemplates.instance_methods.count.to_s} templates " if $options[:verbose]
+    puts "Compiled #{ActionView::CompiledTemplates.instance_methods.count.to_s} templates" if $options[:verbose]
+    log_error "Compiled #{ActionView::CompiledTemplates.instance_methods.count.to_s} templates"
+    log_error "Not all templates were compiled" if ActionView::CompiledTemplates.instance_methods.count < templates.count
   end
 
   def self.compile_erb_templates
@@ -644,9 +683,9 @@ module Veracode
     return unless templates.count > 0
 
     begin
-      require 'action_view'
-      require 'haml'
-      require 'haml/template/plugin'
+      cond_require 'action_view'
+      cond_require 'haml'
+      cond_require 'haml/template/plugin'
     rescue Exception => e
       log_error "Unable to satisfy haml dependencies (#{e.message})"
       return
@@ -685,6 +724,59 @@ module Veracode
 
   end
 
+  def self.require_libs(lib_paths)
+    for lib_path in lib_paths
+      dirsToProcess = [Pathname(lib_path)]
+      until dirsToProcess.count == 0 || !Dir.exists?(dirsToProcess[0])
+        currentDir = dirsToProcess.delete_at(0)
+        for child in currentDir.children
+          if child.directory?
+            dirsToProcess[dirsToProcess.count] = child
+            base = child.to_s.partition("#{lib_path}/")[2]
+            lib = ""
+            for part in base.split('/').reverse
+              lib = "#{part}/#{lib}"
+              lib = lib[0..lib.length-2] if lib[lib.length-1] == '/'
+            begin
+              if cond_require lib
+                puts "requiring #{lib}" if $options[:verbose]
+              end
+            rescue Exception => e
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+def self.require_rails(gemdir)
+  dirsToProcess = [Pathname(gemdir)]
+  until dirsToProcess.count == 0
+    currentDir = dirsToProcess.delete_at(0)
+    for child in currentDir.children
+      if child.directory?
+        dirsToProcess[dirsToProcess.count] = child
+      end
+      base = child.to_s.partition("#{gemdir}/")[2]
+      if base.index("action_controller") != nil || base.index("action_view") != nil || base.index("active_record") != nil
+        lib = ""
+        for part in base.split('/').reverse
+          lib = "#{part}/#{lib}"
+          lib = lib[0..lib.length-2] if lib[lib.length-1] == '/'
+          lib.chomp!(File.extname(lib))
+          begin
+            if cond_require lib
+              puts "requiring #{lib}" if $options[:verbose]
+            end
+          rescue Exception => e
+          end
+        end
+      end
+    end
+  end
+end
+
 
 
 
@@ -702,16 +794,6 @@ module Veracode
       puts
     end
 
-    if $options[:disasm]
-      rbfiles = File.join("**", "*.rb")
-      Dir[rbfiles].each do |f|
-        puts RubyVM::InstructionSequence.compile_file(f).disasm
-        puts
-      end
-      exit
-    end
-
-    prepare_archive
 
     ################################################################
     ## phase 1 - Create baseline
@@ -720,10 +802,6 @@ module Veracode
     puts "Phase 1 - Initial State" if $options[:verbose]
     self.stats if $options[:verbose]
     
-    if $options[:phase1]
-      puts "Processing and disassembling Ruby standard classes and modules"
-      archive(@modules)
-    end
     ## /phase 1 - Create baseline
     ################################################################
 
@@ -734,10 +812,7 @@ module Veracode
 
     puts "Phase 2 - Load Rails" if $options[:verbose]
     begin
-      require "rails"
-      require 'action_controller'
-      require 'action_view'
-      require 'active_record'
+      require "rails/all"
     rescue Exception => e
       puts "Unable to require rails: #{e.message}"
       log_error "Unable to require rails: #{e.message}"
@@ -746,26 +821,6 @@ module Veracode
       puts "Required rails" if $options[:verbose]
     end
 
-    self.update
-
-    self.stats if $options[:verbose]
-
-    if $options[:phase2]
-      puts "Processing and disassembling Rails classes and modules"
-      archive(@modules)
-    end
-    
-    self.rebaseline
-    ## /phase 2 - Require rails
-    ################################################################
-    
-    
-    
-    ################################################################
-    # phase 3 - require app
-
-    puts "Phase 3 - Imitate Rails" if $options[:verbose]
-
     ## Imitate script/rails
     # APP_PATH = File.expand_path('config/application')
       # APP_PATH is already set in bin/veracode
@@ -779,7 +834,7 @@ module Veracode
       # require 'rails/commands'
 
     ## Imitate rails/commands when console
-    glob_require 'rails/commands/console'
+    cond_require 'rails/commands/console.rb'
     # require APP_PATH # => config/application.rb
 
     glob_require "config/application.rb"
@@ -790,27 +845,62 @@ module Veracode
     
     # Imitate Rails::Console.initialize_console
     # require "pp"
-    glob_require "rails/console/app"
-    glob_require "rails/console/helpers"
+    cond_require "rails/console/app.rb"
+    cond_require "rails/console/helpers.rb"
+
+    if $options[:environment]
+      @stdlib = $:
+      @gemdir = Gem.dir
 
-    glob_require "app/models/**/*.rb"
-    glob_require "app/helpers/**/*.rb"
-    glob_require "app/controllers/application_controller.rb"
-    glob_require "app/controllers/**/*.rb"
+      require_libs(@stdlib)
+      require_rails(@gemdir)
+    end
+
+    self.rebaseline
+
+    self.stats if $options[:verbose]
+
+    ## /phase 2 - Require rails
+    ################################################################
+    
+    
+    
+    ################################################################
+    # phase 3 - require app
+
+    puts "Phase 3 - Imitate Rails" if $options[:verbose]
+
+    any_new = true
+    while any_new
+      any_new = false
+      any_new |= glob_require "lib/**/*.rb"
+      any_new |= glob_require "app/**/*.rb"
+      puts "new successful requires? #{any_new.to_s}" if $options[:verbose]
+    end
 
     compile_templates
 
     self.update
     self.stats if $options[:verbose]
 
-    if $options[:phase3]
-      puts "Processing and disassembling #{APP_NAME} classes and modules"
+    # Ensure compiled templates are fully disassembled in archive
+    @baseline_modules.delete(ActionView::CompiledTemplates)
+
+    if $options[:environment]
+      puts "Processing and disassembling environment"
+      archive(@modules.reject  {|o| safe_name(o) =~ /^#<(Class|Module):0x[0-9a-f]+>/i }
+                      .reject  {|o| safe_name(o) =~ /^Veracode/ }
+                      .reject  {|o| safe_name(o) =~ /^EmptyRails/ }
+                      .reject  {|o| safe_name(o) =~ /^ActionView::CompiledTemplates$/ }, false)
+    else
+      puts "Processing Ruby and Rails classes and modules"
       archive(@baseline_modules, false)
+      add_to_archive "\n# Phase 3 - App disassembly\n"
+      puts "Processing and disassembling #{APP_NAME} classes and modules"
       archive(@modules - @baseline_modules, true)
+      archive_schema
     end
 
-    archive_schema
-
     ## /phase 3 - require app
     ################################################################
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: veracode
 version: !ruby/object:Gem::Version
-  version: 1.0.0.alpha3
+  version: 1.0.0.alpha4
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-27 00:00:00.000000000 Z
+date: 2012-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70156231653560 !ruby/object:Gem::Requirement
+  requirement: &70110652152040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70156231653560
+  version_requirements: *70110652152040
 description: Prepares your Ruby on Rails app for submission to Veracode.
 email: devcontact@veracode.com
 executables: