overcommit 0.31.0 → 0.32.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f165cfdcb2a7a4293e715b273489473ed22d4a3
-  data.tar.gz: 3aac84eccfbc7f7096cb7a7216118eac072996b7
+  metadata.gz: b45a46c1290a6ddd0fd418f28012e7138f01470e
+  data.tar.gz: 1443470dae06fb17b84fe99823d8e2c655bdb792
 SHA512:
-  metadata.gz: 39205b47a623e3ed2377af1305290c138e35c1926f12b98c2be2e18533ad808b138c666f40b4ab59393b0ca5ef16ecca971279d65f3b51b8a56209ba791e2669
-  data.tar.gz: e129c516f6953be502dc60737c263487a38ab1ecc0a8866a28e8edf980026c45aec3c340675e8508f9ce59981f72893af946b13491882990e1746b325e3ab898
+  metadata.gz: 133dc22840e178a2473670780a385fb0b4c46a48d175c5ab7ad391eccfb0fc1517a1f4318f8ad91e1ef53ca40cd2925d3e3912fb203b91e8aeeb8a9285892df1
+  data.tar.gz: 0847b5f3045de5b413be8704a76fbd6fdef908a34e11b64414a58628eadc3f961d77efa1e2257ef0a7b77b6e861b2496a1ec97f5752a02cbe1f9a17fe1cd1adb

data/config/default.yml

@@ -39,6 +39,13 @@ gemfile: false
 # to the root of the repository.
 plugin_directory: '.git-hooks'
 
+# Number of hooks that can be run concurrently. Typically this won't need to be
+# adjusted, but if you know that some of your hooks themselves use multiple
+# processors you can lower this value accordingly. You can define
+# single-operator mathematical expressions, e.g. '%{processors} * 2', or
+# '%{processors} / 2'.
+concurrency: '%{processors}'
+
 # Whether to check if a hook plugin has changed since Overcommit last ran it.
 # This is a defense mechanism when working with repositories which can contain
 # untrusted code (e.g. when you fetch a pull request from a third party).

data/lib/overcommit/configuration.rb

@@ -13,7 +13,10 @@ module Overcommit
     def initialize(hash, options = {})
       @options = options.dup
       @options[:logger] ||= Overcommit::Logger.silent
-      @hash = Overcommit::ConfigurationValidator.new.validate(hash, options)
+      @hash = hash # Assign so validator can read original values
+      unless options[:validate] == false
+        @hash = Overcommit::ConfigurationValidator.new.validate(self, hash, options)
+      end
     end
 
     def ==(other)
@@ -34,6 +37,26 @@ module Overcommit
       File.join(Overcommit::Utils.repo_root, @hash['plugin_directory'] || '.git-hooks')
     end
 
+    def concurrency
+      @concurrency ||=
+        begin
+          cores = Overcommit::Utils.processor_count
+          content = @hash.fetch('concurrency', '%{processors}')
+          if content.is_a?(String)
+            concurrency_expr = content % { processors: cores }
+
+            a, op, b = concurrency_expr.scan(%r{(\d+)\s*([+\-*\/])\s*(\d+)})[0]
+            if a
+              a.to_i.send(op, b.to_i)
+            else
+              concurrency_expr.to_i
+            end
+          else
+            content.to_i
+          end
+        end
+    end
+
     # Returns configuration for all hooks in each hook type.
     #
     # @return [Hash]

data/lib/overcommit/configuration_validator.rb

@@ -3,12 +3,13 @@ module Overcommit
   class ConfigurationValidator
     # Validates hash for any invalid options, normalizing where possible.
     #
+    # @param config [Overcommit::Configuration]
     # @param hash [Hash] hash representation of YAML config
     # @param options[Hash]
     # @option default [Boolean] whether hash represents the default built-in config
     # @option logger [Overcommit::Logger] logger to output warnings to
     # @return [Hash] validated hash (potentially modified)
-    def validate(hash, options)
+    def validate(config, hash, options)
       @options = options.dup
       @log = options[:logger]
 
@@ -16,6 +17,7 @@ module Overcommit
       ensure_hook_type_sections_exist(hash)
       check_hook_name_format(hash)
       check_for_missing_enabled_option(hash) unless @options[:default]
+      check_for_too_many_processors(config, hash)
       check_for_verify_plugin_signatures_option(hash)
 
       hash
@@ -96,6 +98,31 @@ module Overcommit
       @log.newline if any_warnings
     end
 
+    # Prints a warning if any hook has a number of processors larger than the
+    # global `concurrency` setting.
+    def check_for_too_many_processors(config, hash)
+      concurrency = config.concurrency
+
+      errors = []
+      Overcommit::Utils.supported_hook_type_classes.each do |hook_type|
+        hash.fetch(hook_type, {}).each do |hook_name, hook_config|
+          processors = hook_config.fetch('processors', 1)
+          if processors > concurrency
+            errors << "#{hook_type}::#{hook_name} `processors` value " \
+                      "(#{processors}) is larger than the global `concurrency` " \
+                      "option (#{concurrency})"
+          end
+        end
+      end
+
+      if errors.any?
+        @log.error errors.join("\n") if @log
+        @log.newline if @log
+        raise Overcommit::Exceptions::ConfigurationError,
+              'One or more hooks had invalid `processor` value configured'
+      end
+    end
+
     # Prints a warning if the `verify_plugin_signatures` option is used instead
     # of the new `verify_signatures` option.
     def check_for_verify_plugin_signatures_option(hash)

data/lib/overcommit/hook/base.rb

@@ -61,6 +61,14 @@ module Overcommit::Hook
       @config['required']
     end
 
+    def parallelize?
+      @config['parallelize'] != false
+    end
+
+    def processors
+      @config.fetch('processors', 1)
+    end
+
     def quiet?
       @config['quiet']
     end

data/lib/overcommit/hook_runner.rb

@@ -1,7 +1,7 @@
 module Overcommit
   # Responsible for loading the hooks the repository has configured and running
   # them, collecting and displaying the results.
-  class HookRunner
+  class HookRunner # rubocop:disable Metrics/ClassLength
     # @param config [Overcommit::Configuration]
     # @param logger [Overcommit::Logger]
     # @param context [Overcommit::HookContext]
@@ -12,6 +12,10 @@ module Overcommit
       @context = context
       @printer = printer
       @hooks = []
+
+      @lock = Mutex.new
+      @resource = ConditionVariable.new
+      @slots_available = @config.concurrency
     end
 
     # Loads and runs the hooks registered for this {HookRunner}.
@@ -51,78 +55,120 @@ module Overcommit
       if @hooks.any?(&:enabled?)
         @printer.start_run
 
-        interrupted = false
-        run_failed = false
-        run_warned = false
-
-        @hooks.each do |hook|
-          hook_status = run_hook(hook)
+        # Sort so hooks requiring fewer processors get queued first. This
+        # ensures we make better use of our available processors
+        @hooks_left = @hooks.sort_by { |hook| processors_for_hook(hook) }
+        @threads = Array.new(@config.concurrency) { Thread.new(&method(:consume)) }
 
-          run_failed = true if hook_status == :fail
-          run_warned = true if hook_status == :warn
-
-          if hook_status == :interrupt
-            # Stop running any more hooks and assume a bad result
-            interrupted = true
-            break
+        begin
+          InterruptHandler.disable_until_finished_or_interrupted do
+            @threads.each(&:join)
           end
+        rescue Interrupt
+          @printer.interrupt_triggered
+          # We received an interrupt on the main thread, so alert the
+          # remaining workers that an exception occurred
+          @interrupted = true
+          @threads.each { |thread| thread.raise Interrupt }
         end
 
-        print_results(run_failed, run_warned, interrupted)
+        print_results
 
-        !(run_failed || interrupted)
+        !(@failed || @interrupted)
       else
         @printer.nothing_to_run
         true # Run was successful
       end
     end
 
-    # @param failed [Boolean]
-    # @param warned [Boolean]
-    # @param interrupted [Boolean]
-    def print_results(failed, warned, interrupted)
-      if interrupted
+    def consume
+      loop do
+        hook = @lock.synchronize { @hooks_left.pop }
+        break unless hook
+        run_hook(hook)
+      end
+    end
+
+    def wait_for_slot(hook)
+      @lock.synchronize do
+        slots_needed = processors_for_hook(hook)
+
+        loop do
+          if @slots_available >= slots_needed
+            @slots_available -= slots_needed
+
+            # Give another thread a chance since there are still slots available
+            @resource.signal if @slots_available > 0
+            break
+          elsif @slots_available > 0
+            # It's possible that another hook that requires fewer slots can be
+            # served, so give another a chance
+            @resource.signal
+
+            # Wait for a signal from another thread to try again
+            @resource.wait(@lock)
+          end
+        end
+      end
+    end
+
+    def release_slot(hook)
+      @lock.synchronize do
+        slots_released = processors_for_hook(hook)
+        @slots_available += slots_released
+
+        if @hooks_left.any?
+          # Signal once. `wait_for_slot` will perform additional signals if
+          # there are still slots available. This prevents us from sending out
+          # useless signals
+          @resource.signal
+        end
+      end
+    end
+
+    def processors_for_hook(hook)
+      hook.parallelize? ? hook.processors : @config.concurrency
+    end
+
+    def print_results
+      if @interrupted
         @printer.run_interrupted
-      elsif failed
+      elsif @failed
         @printer.run_failed
-      elsif warned
+      elsif @warned
         @printer.run_warned
       else
         @printer.run_succeeded
       end
     end
 
-    def run_hook(hook)
-      return if should_skip?(hook)
-
-      @printer.start_hook(hook)
-
+    def run_hook(hook) # rubocop:disable Metrics/CyclomaticComplexity
       status, output = nil, nil
 
       begin
-        # Disable the interrupt handler during individual hook run so that
-        # Ctrl-C actually stops the current hook from being run, but doesn't
-        # halt the entire process.
-        InterruptHandler.disable_until_finished_or_interrupted do
-          status, output = hook.run_and_transform
-        end
+        wait_for_slot(hook)
+        return if should_skip?(hook)
+
+        status, output = hook.run_and_transform
       rescue => ex
         status = :fail
         output = "Hook raised unexpected error\n#{ex.message}\n#{ex.backtrace.join("\n")}"
-      rescue Interrupt
-        # At this point, interrupt has been handled and protection is back in
-        # effect thanks to the InterruptHandler.
-        status = :interrupt
-        output = 'Hook was interrupted by Ctrl-C; restoring repo state...'
       end
 
-      @printer.end_hook(hook, status, output)
+      @failed = true if status == :fail
+      @warned = true if status == :warn
+
+      @printer.end_hook(hook, status, output) unless @interrupted
 
       status
+    rescue Interrupt
+      @interrupted = true
+    ensure
+      release_slot(hook)
     end
 
     def should_skip?(hook)
-      return true unless hook.enabled?
+      return true if @interrupted || !hook.enabled?
 
       if hook.skip?
         if hook.required?

data/lib/overcommit/printer.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'monitor'
+
 module Overcommit
   # Provide a set of callbacks which can be executed as events occur during the
   # course of {HookRunner#run}.
@@ -9,6 +11,8 @@ module Overcommit
     def initialize(logger, context)
       @log = logger
       @context = context
+      @lock = Monitor.new # Need to use monitor so we can have re-entrant locks
+      synchronize_all_methods
     end
 
     # Executed at the very beginning of running the collection of hooks.
@@ -20,13 +24,6 @@ module Overcommit
       log.debug "✓ No applicable #{hook_script_name} hooks to run"
     end
 
-    # Executed at the start of an individual hook run.
-    def start_hook(hook)
-      unless hook.quiet?
-        print_header(hook)
-      end
-    end
-
     def hook_skipped(hook)
       log.warning "Skipping #{hook.name}"
     end
@@ -39,11 +36,16 @@ module Overcommit
     def end_hook(hook, status, output)
       # Want to print the header for quiet hooks only if the result wasn't good
       # so that the user knows what failed
-      print_header(hook) if hook.quiet? && status != :pass
+      print_header(hook) if !hook.quiet? || status != :pass
 
       print_result(hook, status, output)
     end
 
+    def interrupt_triggered
+      log.newline
+      log.error 'Interrupt signal received. Stopping hooks...'
+    end
+
     # Executed when a hook run was interrupted/cancelled by user.
     def run_interrupted
       log.newline
@@ -108,5 +110,27 @@ module Overcommit
     def hook_script_name
       @context.hook_script_name
     end
+
+    # Get all public methods that were defined on this class and wrap them with
+    # synchronization locks so we ensure the output isn't interleaved amongst
+    # the various threads.
+    def synchronize_all_methods
+      methods = self.class.instance_methods - self.class.superclass.instance_methods
+
+      methods.each do |method_name|
+        old_method = :"old_#{method_name}"
+        new_method = :"synchronized_#{method_name}"
+
+        self.class.__send__(:alias_method, old_method, method_name)
+
+        self.class.send(:define_method, new_method) do |*args|
+          @lock.synchronize do
+            __send__(old_method, *args)
+          end
+        end
+
+        self.class.__send__(:alias_method, method_name, new_method)
+      end
+    end
   end
 end

data/lib/overcommit/utils.rb

@@ -211,6 +211,42 @@ module Overcommit
         Subprocess.spawn_detached(args)
       end
 
+      # Return the number of processors used by the OS for process scheduling.
+      #
+      # @see https://github.com/grosser/parallel/blob/v1.6.1/lib/parallel/processor_count.rb#L17-L51
+      def processor_count # rubocop:disable all
+        @processor_count ||=
+          begin
+            if Overcommit::OS.windows?
+              require 'win32ole'
+              result = WIN32OLE.connect('winmgmts://').ExecQuery(
+                'select NumberOfLogicalProcessors from Win32_Processor')
+              result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
+            elsif File.readable?('/proc/cpuinfo')
+              IO.read('/proc/cpuinfo').scan(/^processor/).size
+            elsif File.executable?('/usr/bin/hwprefs')
+              IO.popen('/usr/bin/hwprefs thread_count').read.to_i
+            elsif File.executable?('/usr/sbin/psrinfo')
+              IO.popen('/usr/sbin/psrinfo').read.scan(/^.*on-*line/).size
+            elsif File.executable?('/usr/sbin/ioscan')
+              IO.popen('/usr/sbin/ioscan -kC processor') do |out|
+                out.read.scan(/^.*processor/).size
+              end
+            elsif File.executable?('/usr/sbin/pmcycles')
+              IO.popen('/usr/sbin/pmcycles -m').read.count("\n")
+            elsif File.executable?('/usr/sbin/lsdev')
+              IO.popen('/usr/sbin/lsdev -Cc processor -S 1').read.count("\n")
+            elsif File.executable?('/usr/sbin/sysctl')
+              IO.popen('/usr/sbin/sysctl -n hw.ncpu').read.to_i
+            elsif File.executable?('/sbin/sysctl')
+              IO.popen('/sbin/sysctl -n hw.ncpu').read.to_i
+            else
+              # Unknown platform; assume 1 processor
+              1
+            end
+          end
+      end
+
       # Calls a block of code with a modified set of environment variables,
       # restoring them once the code has executed.
       def with_environment(env)

data/lib/overcommit/version.rb

@@ -2,5 +2,5 @@
 
 # Defines the gem version.
 module Overcommit
-  VERSION = '0.31.0'.freeze
+  VERSION = '0.32.0.rc1'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: overcommit
 version: !ruby/object:Gem::Version
-  version: 0.31.0
+  version: 0.32.0.rc1
 platform: ruby
 authors:
 - Brigade Engineering
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: childprocess
@@ -263,9 +263,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.5.1