gpack 2.0.0 → 2.0.1

data/lib/gpack/core/parallel.rb

@@ -0,0 +1,570 @@
+
+require 'rbconfig'
+#require 'parallel/version'
+#require 'parallel/processor_count'
+
+
+module Parallel
+  module ProcessorCount
+    # Number of processors seen by the OS and used for process scheduling.
+    #
+    # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
+    # * BSD: /sbin/sysctl
+    # * Cygwin: /proc/cpuinfo
+    # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
+    # * HP-UX: /usr/sbin/ioscan
+    # * IRIX: /usr/sbin/sysconf
+    # * Linux: /proc/cpuinfo
+    # * Minix 3+: /proc/cpuinfo
+    # * Solaris: /usr/sbin/psrinfo
+    # * Tru64 UNIX: /usr/sbin/psrinfo
+    # * UnixWare: /usr/sbin/psrinfo
+    #
+    def processor_count
+      @processor_count ||= begin
+        os_name = RbConfig::CONFIG["target_os"]
+        if os_name =~ /mingw|mswin/
+          require 'win32ole'
+          result = WIN32OLE.connect("winmgmts://").ExecQuery(
+            "select NumberOfLogicalProcessors from Win32_Processor")
+          result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
+        elsif File.readable?("/proc/cpuinfo")
+          IO.read("/proc/cpuinfo").scan(/^processor/).size
+        elsif File.executable?("/usr/bin/hwprefs")
+          IO.popen("/usr/bin/hwprefs thread_count").read.to_i
+        elsif File.executable?("/usr/sbin/psrinfo")
+          IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
+        elsif File.executable?("/usr/sbin/ioscan")
+          IO.popen("/usr/sbin/ioscan -kC processor") do |out|
+            out.read.scan(/^.*processor/).size
+          end
+        elsif File.executable?("/usr/sbin/pmcycles")
+          IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
+        elsif File.executable?("/usr/sbin/lsdev")
+          IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
+        elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
+          IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
+        elsif File.executable?("/usr/sbin/sysctl")
+          IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
+        elsif File.executable?("/sbin/sysctl")
+          IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
+        else
+          $stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"]
+          $stderr.puts "Assuming 1 processor."
+          1
+        end
+      end
+    end
+
+    # Number of physical processor cores on the current system.
+    #
+    def physical_processor_count
+      @physical_processor_count ||= begin
+        ppc = case RbConfig::CONFIG["target_os"]
+        when /darwin1/
+          IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
+        when /linux/
+          cores = {}  # unique physical ID / core ID combinations
+          phy = 0
+          IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
+            if ln.start_with?("physical")
+              phy = ln[/\d+/]
+            elsif ln.start_with?("core")
+              cid = phy + ":" + ln[/\d+/]
+              cores[cid] = true if not cores[cid]
+            end
+          end
+          cores.count
+        when /mswin|mingw/
+          require 'win32ole'
+          result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
+            "select NumberOfCores from Win32_Processor")
+          result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
+        else
+          processor_count
+        end
+        # fall back to logical count if physical info is invalid
+        ppc > 0 ? ppc : processor_count
+      end
+    end
+  end
+end
+
+module Parallel
+  VERSION = Version = '1.10.0'
+end
+
+module Parallel
+  extend Parallel::ProcessorCount
+
+  class DeadWorker < StandardError
+  end
+
+  class Break < StandardError
+  end
+
+  class Kill < StandardError
+  end
+
+  class UndumpableException < StandardError
+    def initialize(original)
+      super "#{original.class}: #{original.message}"
+      @bracktrace = original.backtrace
+    end
+
+    def backtrace
+      @bracktrace
+    end
+  end
+
+  Stop = Object.new
+
+  class ExceptionWrapper
+    attr_reader :exception
+    def initialize(exception)
+      @exception =
+        begin
+          Marshal.dump(exception) && exception
+        rescue
+          UndumpableException.new(exception)
+        end
+    end
+  end
+
+  class Worker
+    attr_reader :pid, :read, :write
+    attr_accessor :thread
+    def initialize(read, write, pid)
+      @read, @write, @pid = read, write, pid
+    end
+
+    def stop
+      close_pipes
+      wait # if it goes zombie, rather wait here to be able to debug
+    end
+
+    # might be passed to started_processes and simultaneously closed by another thread
+    # when running in isolation mode, so we have to check if it is closed before closing
+    def close_pipes
+      read.close unless read.closed?
+      write.close unless write.closed?
+    end
+
+    def work(data)
+      begin
+        Marshal.dump(data, write)
+      rescue Errno::EPIPE
+        raise DeadWorker
+      end
+
+      result = begin
+        Marshal.load(read)
+      rescue EOFError
+        raise DeadWorker
+      end
+      raise result.exception if ExceptionWrapper === result
+      result
+    end
+
+    private
+
+    def wait
+      Process.wait(pid)
+    rescue Interrupt
+      # process died
+    end
+  end
+
+  class JobFactory
+    def initialize(source, mutex)
+      @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
+      @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
+      @mutex = mutex
+      @index = -1
+      @stopped = false
+    end
+
+    def next
+      if producer?
+        # - index and item stay in sync
+        # - do not call lambda after it has returned Stop
+        item, index = @mutex.synchronize do
+          return if @stopped
+          item = @lambda.call
+          @stopped = (item == Parallel::Stop)
+          return if @stopped
+          [item, @index += 1]
+        end
+      else
+        index = @mutex.synchronize { @index += 1 }
+        return if index >= size
+        item = @source[index]
+      end
+      [item, index]
+    end
+
+    def size
+      if producer?
+        Float::INFINITY
+      else
+        @source.size
+      end
+    end
+
+    # generate item that is sent to workers
+    # just index is faster + less likely to blow up with unserializable errors
+    def pack(item, index)
+      producer? ? [item, index] : index
+    end
+
+    # unpack item that is sent to workers
+    def unpack(data)
+      producer? ? data : [@source[data], data]
+    end
+
+    private
+
+    def producer?
+      @lambda
+    end
+
+    def queue_wrapper(array)
+      array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
+    end
+  end
+
+  class UserInterruptHandler
+    INTERRUPT_SIGNAL = :SIGINT
+
+    class << self
+      # kill all these pids or threads if user presses Ctrl+c
+      def kill_on_ctrl_c(pids, options)
+        @to_be_killed ||= []
+        old_interrupt = nil
+        signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
+
+        if @to_be_killed.empty?
+          old_interrupt = trap_interrupt(signal) do
+            $stderr.puts 'Parallel execution interrupted, exiting ...'
+            @to_be_killed.flatten.each { |pid| kill(pid) }
+          end
+        end
+
+        @to_be_killed << pids
+
+        yield
+      ensure
+        @to_be_killed.pop # do not kill pids that could be used for new processes
+        restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
+      end
+
+      def kill(thing)
+        Process.kill(:KILL, thing)
+      rescue Errno::ESRCH
+        # some linux systems already automatically killed the children at this point
+        # so we just ignore them not being there
+      end
+
+      private
+
+      def trap_interrupt(signal)
+        old = Signal.trap signal, 'IGNORE'
+
+        Signal.trap signal do
+          yield
+          if old == "DEFAULT"
+            raise Interrupt
+          else
+            old.call
+          end
+        end
+
+        old
+      end
+
+      def restore_interrupt(old, signal)
+        Signal.trap signal, old
+      end
+    end
+  end
+
+  class << self
+    def in_threads(options={:count => 2})
+      count, _ = extract_count_from_options(options)
+      Array.new(count) do |i|
+        Thread.new { yield(i) }
+      end.map!(&:value)
+    end
+
+    def in_processes(options = {}, &block)
+      count, options = extract_count_from_options(options)
+      count ||= processor_count
+      map(0...count, options.merge(:in_processes => count), &block)
+    end
+
+    def each(array, options={}, &block)
+      map(array, options.merge(:preserve_results => false), &block)
+      array
+    end
+
+    def each_with_index(array, options={}, &block)
+      each(array, options.merge(:with_index => true), &block)
+    end
+
+    def map(source, options = {}, &block)
+      options[:mutex] = Mutex.new
+
+      if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
+        method = :in_threads
+        size = options[method] || processor_count
+      elsif options[:in_threads]
+        method = :in_threads
+        size = options[method]
+      else
+        method = :in_processes
+        if Process.respond_to?(:fork)
+          size = options[method] || processor_count
+        else
+          warn "Process.fork is not supported by this Ruby"
+          size = 0
+        end
+      end
+
+      job_factory = JobFactory.new(source, options[:mutex])
+      size = [job_factory.size, size].min
+
+      options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
+      add_progress_bar!(job_factory, options)
+
+      if size == 0
+        work_direct(job_factory, options, &block)
+      elsif method == :in_threads
+        work_in_threads(job_factory, options.merge(:count => size), &block)
+      else
+        work_in_processes(job_factory, options.merge(:count => size), &block)
+      end
+    end
+
+    def map_with_index(array, options={}, &block)
+      map(array, options.merge(:with_index => true), &block)
+    end
+
+    def worker_number
+      Thread.current[:parallel_worker_number]
+    end
+
+    def worker_number=(worker_num)
+      Thread.current[:parallel_worker_number] = worker_num
+    end
+
+    private
+
+    def add_progress_bar!(job_factory, options)
+      if progress_options = options[:progress]
+        raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
+        require 'ruby-progressbar'
+
+        if progress_options == true
+          progress_options = { title: "Progress" }
+        elsif progress_options.respond_to? :to_str
+          progress_options = { title: progress_options.to_str }
+        end
+
+        progress_options = {
+          total: job_factory.size,
+          format: '%t |%E | %B | %a'
+        }.merge(progress_options)
+
+        progress = ProgressBar.create(progress_options)
+        old_finish = options[:finish]
+        options[:finish] = lambda do |item, i, result|
+          old_finish.call(item, i, result) if old_finish
+          progress.increment
+        end
+      end
+    end
+
+    def work_direct(job_factory, options, &block)
+      self.worker_number = 0
+      results = []
+      while set = job_factory.next
+        item, index = set
+        results << with_instrumentation(item, index, options) do
+          call_with_index(item, index, options, &block)
+        end
+      end
+      results
+    ensure
+      self.worker_number = nil
+    end
+
+    def work_in_threads(job_factory, options, &block)
+      raise "interrupt_signal is no longer supported for threads" if options[:interrupt_signal]
+      results = []
+      results_mutex = Mutex.new # arrays are not thread-safe on jRuby
+      exception = nil
+
+      in_threads(options) do |worker_num|
+        self.worker_number = worker_num
+        # as long as there are more jobs, work on one of them
+        while !exception && set = job_factory.next
+          begin
+            item, index = set
+            result = with_instrumentation item, index, options do
+              call_with_index(item, index, options, &block)
+            end
+            results_mutex.synchronize { results[index] = result }
+          rescue StandardError => e
+            exception = e
+          end
+        end
+      end
+
+      handle_exception(exception, results)
+    end
+
+    def work_in_processes(job_factory, options, &blk)
+      workers = if options[:isolation]
+        [] # we create workers per job and not beforehand
+      else
+        create_workers(job_factory, options, &blk)
+      end
+      results = []
+      results_mutex = Mutex.new # arrays are not thread-safe
+      exception = nil
+
+      UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
+        in_threads(options) do |i|
+          worker = workers[i]
+
+          begin
+            loop do
+              break if exception
+              item, index = job_factory.next
+              break unless index
+
+              if options[:isolation]
+                worker = replace_worker(job_factory, workers, i, options, blk)
+              end
+
+              worker.thread = Thread.current
+
+              begin
+                result = with_instrumentation item, index, options do
+                  worker.work(job_factory.pack(item, index))
+                end
+                results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
+              rescue StandardError => e
+                exception = e
+                if Parallel::Kill === exception
+                  (workers - [worker]).each do |w|
+                    w.thread.kill unless w.thread.nil?
+                    UserInterruptHandler.kill(w.pid)
+                  end
+                end
+              end
+            end
+          ensure
+            worker.stop if worker
+          end
+        end
+      end
+
+      handle_exception(exception, results)
+    end
+
+    def replace_worker(job_factory, workers, i, options, blk)
+      # old worker is no longer used ... stop it
+      worker = workers[i]
+      worker.stop if worker
+
+      # create a new replacement worker
+      running = workers - [worker]
+      workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
+    end
+
+    def create_workers(job_factory, options, &block)
+      workers = []
+      Array.new(options[:count]).each_with_index do |_, i|
+        workers << worker(job_factory, options.merge(started_workers: workers, worker_number: i), &block)
+      end
+      workers
+    end
+
+    def worker(job_factory, options, &block)
+      child_read, parent_write = IO.pipe
+      parent_read, child_write = IO.pipe
+
+      pid = Process.fork do
+        self.worker_number = options[:worker_number]
+
+        begin
+          options.delete(:started_workers).each(&:close_pipes)
+
+          parent_write.close
+          parent_read.close
+
+          process_incoming_jobs(child_read, child_write, job_factory, options, &block)
+        ensure
+          child_read.close
+          child_write.close
+        end
+      end
+
+      child_read.close
+      child_write.close
+
+      Worker.new(parent_read, parent_write, pid)
+    end
+
+    def process_incoming_jobs(read, write, job_factory, options, &block)
+      until read.eof?
+        data = Marshal.load(read)
+        item, index = job_factory.unpack(data)
+        result = begin
+          call_with_index(item, index, options, &block)
+        rescue StandardError => e
+          ExceptionWrapper.new(e)
+        end
+        Marshal.dump(result, write)
+      end
+    end
+
+    def handle_exception(exception, results)
+      return nil if [Parallel::Break, Parallel::Kill].include? exception.class
+      raise exception if exception
+      results
+    end
+
+    # options is either a Integer or a Hash with :count
+    def extract_count_from_options(options)
+      if options.is_a?(Hash)
+        count = options[:count]
+      else
+        count = options
+        options = {}
+      end
+      [count, options]
+    end
+
+    def call_with_index(item, index, options, &block)
+      args = [item]
+      args << index if options[:with_index]
+      if options[:return_results]
+        block.call(*args)
+      else
+        block.call(*args)
+        nil # avoid GC overhead of passing large results around
+      end
+    end
+
+    def with_instrumentation(item, index, options)
+      on_start = options[:start]
+      on_finish = options[:finish]
+      options[:mutex].synchronize { on_start.call(item, index) } if on_start
+      result = yield
+      options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
+      result unless options[:preserve_results] == false
+    end
+  end
+end

data/lib/gpack/core/parse_repos.rb

@@ -0,0 +1,73 @@
+
+## Parse the GpackRepose file
+
+def parse_gpackrepos()
+
+grepos_file = $SETTINGS["core"]["repofile"]
+
+## Options for YAML File
+required_keys = ["url","localdir","branch"]
+valid_config = ["remote_key","ssh_command"]
+
+
+grepos = GitCollection.new()
+
+if !File.exist?(grepos_file)
+   raise "File does not exist #{grepos_file}"
+end
+
+unlocked = File.exists?(".gpackunlock")
+
+yml_file = YAML.load_file(grepos_file)
+
+yml_file.each do |key,entry|
+   if key == "config"
+      # Read in config settings
+      # Check if the config option is valid
+      entry.each do |ckey,centry|
+         if !valid_config.index(ckey)
+            raise "Error in file '#{grepos_file}'.\n\tError in configuration entry #{key}\n\tConfig option must be one of #{valid_config}"
+         end
+
+         case ckey
+            when "lock"
+               # TODO implement this
+            when "remote_key"
+               #SSH KEY stuff
+               $SETTINGS["ssh"]["key_url"] = centry
+            when "ssh_command"
+               # Arguements to ssh
+               $SETTINGS["ssh"]["cmd"] = centry
+         end
+         
+      end
+      
+   else
+      reponame = key
+      
+      # Check required keys exist
+      if !required_keys.all? {|s| entry.key? s}
+         raise "Error in file '#{grepos_file}'.\n\tEntry #{key}\n\tFor a repository these properties are required #{required_keys}"
+      end
+      
+      # Optional Key Parsing
+      if entry.key?("lock")
+         readonly = entry["lock"]
+      else
+         readonly = true
+      end
+      
+      new_repo = GitReference.new :url=>entry["url"], :localdir=>entry["localdir"], :branch=>entry["branch"]
+      
+      if unlocked
+         new_repo.readonly = false
+      end
+      grepos.add_ref(new_repo)
+      
+   end
+end
+
+
+return grepos
+
+end

data/lib/gpack/core/ssh.rb

@@ -0,0 +1,33 @@
+
+def download_ssh_key()
+   key_url = $SETTINGS["ssh"]["key_url"]
+   if key_url
+      remote_key = Tempfile.new('gpack_ssh') # TODO make this readable only by user      
+      begin
+         download = open(key_url)
+         IO.copy_stream(download, remote_key.path)
+      rescue
+         puts "Error with URL #{key_url}\nEnsure this is a valid url and can be reached"
+         raise
+      end
+      $SETTINGS["ssh"]["key"] = remote_key
+
+   end
+end
+
+def set_ssh_cmd()
+   remote_key = $SETTINGS["ssh"]["key"] 
+   
+   if remote_key
+      id_arg = " -i #{remote_key.path}" 
+
+      ssh_cmd=$SETTINGS["ssh"]["cmd"]
+      if $SETTINGS["ssh"]["cmd"]
+         ssh_cmd="#{ssh_cmd}#{id_arg}" 
+      else
+         ssh_cmd="ssh #{id_arg}"
+      end
+      $SETTINGS["ssh"]["cmd"] = ssh_cmd
+   end
+   
+end