navo 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3adda44205c06f6444cdc4969561ce867498cc6f
-  data.tar.gz: a8261f98b2e072aab9451df37b558c4b2376308a
+  metadata.gz: 6e19118f65db1a7db3490eb3ce679b05d9c93dd4
+  data.tar.gz: 28fdb3567c3f9b883e54a88c258e3e5776f6f5b3
 SHA512:
-  metadata.gz: ec8ecc5aee52a8c00aeaab4ffed1062f613f71a85b808327d1c0dd62e9d0326793a5e1cbe4f49a1628d46038b0b050e8ddc7884b90a1d9c9cc6dae39a4fa02a3
-  data.tar.gz: 940a3980f2acae81e2353e7c57d2a79d523bb7fa2d572f6b28a9c28bc8bdc1d96e02339aa55e0e5e0b97fca54387c1ee2bcffd793a2b2e4f5175bad72cc178e4
+  metadata.gz: 8df3881c54fbba49a9bcabd1e1fce93728407f7302f8a115091cf89f25897fdfbd189c24212b237207cd5bc3d8499fe69d1dc48bd8f45583890990486ec4bb0c
+  data.tar.gz: 59849b6754981ccf3526f10ec0f893b1f8f52018fb29fa5826a163c5a051a4681ef297a49d92354933302dc8b46468e323792e0b5f6b3860d77424473095154d

data/lib/navo.rb

@@ -1,8 +1,26 @@
 require 'navo/constants'
 require 'navo/errors'
 require 'navo/configuration'
+require 'navo/logger'
 require 'navo/sandbox'
 require 'navo/suite'
-require 'navo/suite_state'
+require 'navo/berksfile'
+require 'navo/state_file'
 require 'navo/utils'
 require 'navo/version'
+
+module Navo
+  class << self
+    attr_accessor :mutex
+
+    def synchronize
+      mutex.synchronize do
+        yield
+      end
+    end
+  end
+end
+
+Navo.mutex = Mutex.new
+
+Excon.defaults[:read_timeout] = 600

data/lib/navo/berksfile.rb

@@ -0,0 +1,58 @@
+require 'fileutils'
+
+module Navo
+  # A global Berksfile to be shared amongst all threads.
+  #
+  # This synchronizes access so we don't have multiple threads doing the same
+  # work resolving cookbooks.
+  class Berksfile
+    class << self
+      attr_accessor :path
+      attr_accessor :config
+
+      def load
+        require 'berkshelf' # Lazily require so we don't have to load for every command
+        berksfile
+      end
+
+      def install(logger: nil)
+        if @installed
+          logger.info 'Berksfile cookbooks already resolved'
+          return
+        end
+
+        logger.info 'Installing Berksfile...'
+        Berkshelf.logger = Celluloid.logger = logger
+        Berkshelf.ui.mute { Berkshelf::Installer.new(berksfile).run }
+        Celluloid.logger = nil # Ignore annoying shutdown messages
+
+        @installed = true
+      end
+
+      def vendor(logger:)
+          Berkshelf.logger = Celluloid.logger = logger
+          Berkshelf.ui.mute { berksfile.vendor(vendor_directory) }
+          Celluloid.logger = nil # Ignore annoying shutdown messages
+      end
+
+      def cache_directory
+        Berkshelf::CookbookStore.default_path
+      end
+
+      def vendor_directory
+        @vendor_directory ||=
+          FileUtils.mkdir_p(File.join(config.repo_root, %w[.navo vendored-cookbooks])).first
+      end
+
+      def lockfile_path
+        berksfile.lockfile.filepath
+      end
+
+      private
+
+      def berksfile
+        @berksfile ||= Berkshelf::Berksfile.from_file(@path)
+      end
+    end
+  end
+end

data/lib/navo/chef_formatter.rb

@@ -0,0 +1,278 @@
+class Chef
+  module Formatters
+    class NavoFormatter < Formatters::Base
+      cli_name(:navo)
+
+      def initialize(out, err)
+        super
+
+        @up_to_date_resources = 0
+        @updated_resources = 0
+        @skipped_resources = 0
+        @resource_stack = []
+        @resource_action_times = Hash.new { |hash, key| hash[key] = [] }
+
+        @deprecations = {}
+      end
+
+      def total_resources
+        @up_to_date_resources + @updated_resources + @skipped_resources
+      end
+
+      def print_deprecations
+        return if @deprecations.empty?
+        puts_line 'Deprecated features used!'
+
+        @deprecations.each do |message, locations|
+          if locations.size == 1
+            puts_line "  #{message} at one location:"
+          else
+            puts_line "  #{message} at #{locations.size} locations:"
+          end
+          locations.each do |location|
+            prefix = '    - '
+            Array(location).each do |line|
+              puts_line "#{prefix}#{line}"
+              prefix = '      '
+            end
+          end
+        end
+        puts_line ''
+      end
+
+      def run_start(version)
+        @start_time = Time.now
+        puts_line "Starting Chef client #{version}...", :cyan
+      end
+
+      def ohai_completed(node)
+        puts_line ''
+        puts_line 'Ohai run completed', :cyan
+      end
+
+      def library_load_start(file_count)
+        @load_start_time = Time.now
+        puts_line ''
+        puts_line 'Loading cookbook libraries...', :cyan
+      end
+
+      def library_load_complete
+        elapsed = Time.now - @load_start_time
+        puts_line "Loaded cookbook libraries (#{elapsed}s)", :cyan
+      end
+
+      def attribute_load_start(file_count)
+        @load_start_time = Time.now
+        puts_line ''
+        puts_line 'Loading cookbook attributes...', :cyan
+      end
+
+      def attribute_load_complete
+        elapsed = Time.now - @load_start_time
+        puts_line "Loaded cookbook attributes (#{elapsed}s)", :cyan
+      end
+
+      def lwrp_load_start(file_count)
+        @load_start_time = Time.now
+        puts_line ''
+        puts_line 'Loading custom resources...', :cyan
+      end
+
+      def lwrp_load_complete
+        elapsed = Time.now - @load_start_time
+        puts_line "Loaded custom resources (#{elapsed}s)", :cyan
+      end
+
+      def definition_load_start(file_count)
+        @load_start_time = Time.now
+        puts_line ''
+        puts_line 'Loading definitions...', :cyan
+      end
+
+      def definition_load_complete
+        elapsed = Time.now - @load_start_time
+        puts_line "Loaded definitions (#{elapsed}s)", :cyan
+      end
+
+      def recipe_load_start(recipes)
+        @load_start_time = Time.now
+        puts_line ''
+        puts_line "Loading recipes...", :cyan
+      end
+
+      def recipe_load_complete
+        elapsed = Time.now - @load_start_time
+        puts_line "Recipes loaded (#{elapsed}s)", :cyan
+      end
+
+      def file_loaded(path)
+        puts_line "Loaded #{path}"
+      end
+
+      def converge_start(run_context)
+        puts_line ''
+        puts_line "Converging #{run_context.resource_collection.all_resources.size} resources..."
+      end
+
+      def converge_complete
+        unindent while @resource_stack.pop
+        puts_line ''
+        puts_line 'Converge completed', :green
+      end
+
+      def converge_failed(e)
+        unindent while @resource_stack.pop
+        puts_line ''
+        puts_line "Converge failed: #{e}", :red
+      end
+
+      def resource_action_start(resource, action, notification_type = nil, notifier = nil)
+        # Track the current recipe so we update it only when it changes
+        # (i.e. when descending into another recipe via include_recipe)
+        if resource.cookbook_name && resource.recipe_name
+          current_recipe = "#{resource.cookbook_name}::#{resource.recipe_name}"
+
+          unless current_recipe == @current_recipe
+            @current_recipe = current_recipe
+            puts_line current_recipe, :magenta
+          end
+        end
+
+        # Record the resource and the time we started so we can figure out how
+        # long it took to complete
+        @resource_stack << [resource, Time.now]
+        indent
+
+        puts_line "#{resource} action #{action}"
+      end
+
+      def resource_failed_retriable(resource, action, retry_count, exception)
+        puts_line "#{resource} action #{action} FAILED; retrying...", :yellow
+      end
+
+      # Called when a resource fails and will not be retried.
+      def resource_failed(resource, action, exception)
+        _, start_time = @resource_stack.pop
+        elapsed = Time.now - start_time
+        @resource_action_times[[resource.to_s, action.to_s]] << elapsed
+        puts_line "#{resource} action #{action} (#{elapsed}s) FAILED: #{exception}", :red
+        unindent
+      end
+
+      def resource_skipped(resource, action, conditional)
+        @skipped_resources += 1
+        _, start_time = @resource_stack.pop
+        elapsed = Time.now - start_time
+        @resource_action_times[[resource.to_s, action.to_s]] << elapsed
+        puts_line "#{resource} action #{action} (#{elapsed}s) SKIPPED due to: #{conditional.short_description}"
+        unindent
+      end
+
+      def resource_up_to_date(resource, action)
+        @up_to_date_resources += 1
+        _, start_time = @resource_stack.pop
+        elapsed = Time.now - start_time
+        @resource_action_times[[resource.to_s, action.to_s]] << elapsed
+        puts_line "#{resource} action #{action} up-to-date (#{elapsed}s)"
+        unindent
+      end
+
+      def resource_update_applied(resource, action, update)
+        indent
+
+        Array(update).compact.each do |line|
+          if line.is_a?(String)
+            puts_line "- #{line}", :green
+          elsif line.is_a?(Array)
+            # Expanded output delta
+            line.each do |detail|
+              if detail =~ /^\+(?!\+\+ )/
+                color = :green
+              elsif detail =~ /^-(?!-- )/
+                color = :red
+              else
+                color = :cyan
+              end
+              puts_line detail, color
+            end
+          end
+        end
+
+        unindent
+      end
+
+      def resource_updated(resource, action)
+        @updated_resources += 1
+        _, start_time = @resource_stack.pop
+        elapsed = Time.now - start_time
+        @resource_action_times[[resource.to_s, action.to_s]] << elapsed
+        puts_line "#{resource} action #{action} updated (#{elapsed}s)"
+        unindent
+      end
+
+      def handlers_start(handler_count)
+        @handler_count = handler_count
+        puts_line ''
+        if @handler_count > 0
+          puts_line "Running #{handler_count} handlers:", :cyan
+        else
+          puts_line 'No registered handlers to run', :cyan
+        end
+      end
+
+      def handler_executed(handler)
+        puts_line "- #{handler.class.name}"
+      end
+
+      def handlers_completed
+        puts_line 'Running handlers complete' unless @handler_count == 0
+      end
+
+       def deprecation(message, location=caller(2..2)[0])
+        if Chef::Config[:treat_deprecation_warnings_as_errors]
+          super
+        end
+
+        # Save deprecations to the screen until the end
+        @deprecations[message] ||= Set.new
+        @deprecations[message] << location
+      end
+
+      def run_completed(node)
+        @end_time = Time.now
+
+        print_deprecations
+
+        puts_line ''
+        print_resource_summary
+        puts_line "Chef client finished in #{@end_time - @start_time} seconds", :cyan
+      end
+
+      private
+
+      def print_resource_summary
+        puts_line "#{@updated_resources}/#{total_resources} resources updated"
+
+        slowest_resources = @resource_action_times.sort_by do |key, values|
+          -values.inject(:+)
+        end
+
+        puts_line 'Slowest resource actions:'
+        slowest_resources[0...10].each do |key, values|
+          elapsed = '%-.3fs' % values.inject(:+)
+          puts_line "#{elapsed.ljust(8)} #{key.first} (#{key[1]})"
+        end
+
+        puts_line ''
+      end
+
+      def indent
+        indent_by 2
+      end
+
+      def unindent
+        indent_by -2
+      end
+    end
+  end
+end

data/lib/navo/cli.rb

@@ -1,40 +1,58 @@
-require 'berkshelf'
 require 'navo'
+require 'parallel'
 require 'thor'
 
 module Navo
   # Command line application interface.
   class CLI < Thor
-    desc 'create', 'create a container for test suite(s) to run within'
-    def create(pattern = nil)
-      exit suites_for(pattern).map(&:create).all? ? 0 : 1
+    def initialize(*args)
+      super
+      Navo::Logger.output = STDOUT
+      STDOUT.sync = true
+      Navo::Logger.level = config['log-level']
     end
 
-    desc 'converge', 'run Chef for test suite(s)'
-    def converge(pattern = nil)
-      exit suites_for(pattern).map(&:converge).all? ? 0 : 1
-    end
-
-    desc 'verify', 'run test suite(s)'
-    def verify(pattern = nil)
-      exit suites_for(pattern).map(&:verify).all? ? 0 : 1
-    end
+    {
+      create: 'create a container for test suite(s) to run within',
+      converge: 'run Chef for test suite(s)',
+      verify: 'run test suites(s)',
+      test: 'converge and run test suites(s)',
+      destroy: 'clean up test suite(s)',
+    }.each do |action, description|
+      desc "#{action} [suite|regexp]", description
+      option :concurrency,
+             aliases: '-c',
+             type: :numeric,
+             default: Parallel.processor_count,
+             desc: 'Execute up to the specified number of test suites concurrently'
+      option 'log-level',
+             aliases: '-l',
+             type: :string,
+             desc: 'Set the log output verbosity level'
 
-    desc 'test', 'converge and run test suite(s)'
-    def test(pattern = nil)
-      exit suites_for(pattern).map(&:test).all? ? 0 : 1
-    end
+      if action == :test
+        option 'destroy',
+               aliases: '-d',
+               type: :string,
+               desc: 'Destroy strategy to use after testing (passing, always, never)'
+      end
 
-    desc 'destroy', 'clean up test suite(s)'
-    def destroy(pattern = nil)
-      exit suites_for(pattern).map(&:destroy).all? ? 0 : 1
+      define_method(action) do |*args|
+        apply_flags_to_config!
+        execute(action, *args)
+      end
     end
 
     desc 'login', "open a shell inside a suite's container"
     def login(pattern)
+      apply_flags_to_config!
+
       suites = suites_for(pattern)
-      if suites.size > 1
-        puts 'Pattern matched more than one test suite'
+      if suites.size == 0
+        logger.console "Pattern '#{pattern}' matched no test suites", severity: :error
+        exit 1
+      elsif suites.size > 1
+        logger.console "Pattern '#{pattern}' matched more than one test suite", severity: :error
         exit 1
       else
         suites.first.login
@@ -47,13 +65,52 @@ module Navo
       @config ||= Configuration.load_applicable
     end
 
+    def logger
+      @logger ||= Navo::Logger.new
+    end
+
     def suites_for(pattern)
       suite_names = config['suites'].keys
       suite_names.select! { |name| name =~ /#{pattern}/ } if pattern
 
       suite_names.map do |suite_name|
-        Suite.new(name: suite_name, config: config)
+        Suite.new(name: suite_name, config: config, global_state: @global_state)
       end
     end
+
+    def apply_flags_to_config!
+      config['log-level'] = options['log-level'] if options['log-level']
+      Navo::Logger.level = config['log-level']
+      config['concurrency'] = options['concurrency'] if options['concurrency']
+      config['destroy'] = options.fetch('destroy', 'passing')
+
+      # Initialize here so config is correctly set
+      Berksfile.path = File.expand_path(config['chef']['berksfile'], config.repo_root)
+      Berksfile.config = config
+      @global_state = StateFile.new(file: File.join(config.repo_root, %w[.navo global-state.yaml]),
+                                    logger: logger).tap(&:load)
+    end
+
+    def execute(action, pattern = nil)
+      suites = suites_for(pattern)
+      results = Parallel.map(suites, in_threads: config['concurrency']) do |suite|
+        succeeded = suite.send(action)
+        [succeeded, suite]
+      end
+
+      failures = results.reject { |succeeded, result| succeeded }
+      failures.each do |_, suite|
+        logger.console("Failed to #{action} #{suite.name}", severity: :error)
+        logger.console("See #{suite.log_file} for full log output", severity: :error)
+      end
+
+      exit failures.any? ? 1 : 0
+    rescue Interrupt
+      # Handle Ctrl-C
+      logger.console('INTERRUPTED', severity: :warn)
+    rescue => ex
+      logger.console("#{ex.class}: #{ex.message}", severity: :fatal)
+      logger.console(ex.backtrace.join("\n"), severity: :fatal)
+    end
   end
 end