rack2aws 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b4761ec5789bc29a4d5fc2d56799a32b149be32
-  data.tar.gz: 63c2dadc08a92138f0e249732aa870d250c41636
+  metadata.gz: 72ae29bf646a70f2cc5731fbad6ba8a1c61f2115
+  data.tar.gz: 9464c44d339a1a1587795efde4b0d2f622aaebd8
 SHA512:
-  metadata.gz: acb2647fd5e88e139bce7e1eff15aac44333ac6753b555ffa607702b90975a151f00e502301557bfc1a2596578416cdd15d6825286674cb0bbca545f550e7b5f
-  data.tar.gz: c3d38eea9c6ab2591022869c828e791c63666c5e8d5d35a06ed46f04c7834c8f12026c2915113db4766425ab7bcd819365bab7f18e34bfa392700efdc5fd151b
+  metadata.gz: 621b2fbfd728276c06cf70e45c5b777b84b740c36cc7ea33ab789a3064c56f51b92562bc4a7017a481aed2565cd692544619f5a084bb40ce670284a2d0e2dfaf
+  data.tar.gz: 903d33fa596f8106bb4d5d2d6a6a1a1d0955135d4918119f078063ffd1450ff2cf68aeae8cb3a6e0d155b556201517d709dff772330e94f12c70957be2edbfe1

data/lib/rack2aws.rb

@@ -2,13 +2,14 @@ require 'fog'
 require 'commander'
 require 'rack2aws/config'
 require 'rack2aws/version'
+require 'rack2aws/processor_count'
 
 
 module Rack2Aws
-  class FileCopy
+  class FileManager
     include Rack2Aws::Configuration
 
-    attr_reader :per_page, :rackspace, :aws, :rackspace_directory, :aws_directory, :verbose_mode, :files, :total
+    attr_reader :per_page, :rackspace, :aws, :rackspace_directory, :aws_directory, :nproc, :verbose_mode, :files, :total
 
     def initialize(options={})
       options = default_options.merge(options)
@@ -18,6 +19,7 @@ module Rack2Aws
 
       @rackspace_directory = rackspace.directories.get(options[:rackspace_container])
       @aws_directory = aws.directories.get(options[:aws_bucket])
+      @nproc = options[:nproc]
       @verbose_mode = options[:verbose]
       @files = []
       @total = 0
@@ -57,15 +59,15 @@ module Rack2Aws
     def copy_files(page, files)
       puts "  [#{Process.pid}] Page #{page+1}: Copying #{files.size} files..." if verbose_mode
       total = files.size
-      max_processes = 4
       process_pids = {}
       time = Time.now
 
       while !files.empty? or !process_pids.empty?
-        while process_pids.size < max_processes and files.any? do
+        while process_pids.size < nproc and files.any? do
           file = files.pop
           pid = Process.fork do
             copy_file(file)
+            exit!(0)
           end
           process_pids[pid] = { :file => file }
         end
@@ -92,6 +94,7 @@ module Rack2Aws
 
   class CLI
     include Commander::Methods
+    include Rack2Aws::ProcessorCount
 
     def run
       program :name, 'rack2aws'
@@ -107,6 +110,7 @@ module Rack2Aws
 
         cmd.option '--container CONTAINER_NAME', String, 'Rackspace Cloud Files container name'
         cmd.option '--bucket BUCKET_NAME', String, 'AWS S3 bucket name'
+        cmd.option '--nproc NUM_PROC', Integer, 'Number of processes to fork'
         cmd.action do |args, options|
           if options.container.nil?
             options.container = ask('Rackspace Cloud Files container: ')
@@ -116,9 +120,14 @@ module Rack2Aws
             options.bucket = ask('AWS S3 bucket: ')
           end
 
-          FileCopy.new({
+          if options.nproc.nil?
+            options.nproc = processor_count()
+          end
+
+          FileManager.new({
                          :rackspace_container => options.container,
                          :aws_bucket => options.bucket,
+                         :nproc => options.nproc,
                          :verbose => $verbose
                        }).copy
         end

data/lib/rack2aws/processor_count.rb

@@ -0,0 +1,88 @@
+module Rack2Aws
+	# Taken from parallel lib on Github:
+	# 	https://github.com/grosser/parallel/blob/master/lib/parallel/processor_count.rb
+	module ProcessorCount
+    # Number of processors seen by the OS and used for process scheduling.
+    #
+    # * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
+    # * BSD: /sbin/sysctl
+    # * Cygwin: /proc/cpuinfo
+    # * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
+    # * HP-UX: /usr/sbin/ioscan
+    # * IRIX: /usr/sbin/sysconf
+    # * Linux: /proc/cpuinfo
+    # * Minix 3+: /proc/cpuinfo
+    # * Solaris: /usr/sbin/psrinfo
+    # * Tru64 UNIX: /usr/sbin/psrinfo
+    # * UnixWare: /usr/sbin/psrinfo
+    #
+    def processor_count
+      @processor_count ||= begin
+        os_name = RbConfig::CONFIG["target_os"]
+        if os_name =~ /mingw|mswin/
+          require 'win32ole'
+          result = WIN32OLE.connect("winmgmts://").ExecQuery(
+            "select NumberOfLogicalProcessors from Win32_Processor")
+          result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
+        elsif File.readable?("/proc/cpuinfo")
+          IO.read("/proc/cpuinfo").scan(/^processor/).size
+        elsif File.executable?("/usr/bin/hwprefs")
+          IO.popen("/usr/bin/hwprefs thread_count").read.to_i
+        elsif File.executable?("/usr/sbin/psrinfo")
+          IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
+        elsif File.executable?("/usr/sbin/ioscan")
+          IO.popen("/usr/sbin/ioscan -kC processor") do |out|
+            out.read.scan(/^.*processor/).size
+          end
+        elsif File.executable?("/usr/sbin/pmcycles")
+          IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
+        elsif File.executable?("/usr/sbin/lsdev")
+          IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
+        elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
+          IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
+        elsif File.executable?("/usr/sbin/sysctl")
+          IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
+        elsif File.executable?("/sbin/sysctl")
+          IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
+        else
+          $stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"]
+          $stderr.puts "Assuming 1 processor."
+          1
+        end
+      end
+    end
+
+    #
+    # Number of physical processor cores on the current system.
+    #
+    def physical_processor_count
+      @physical_processor_count ||= begin
+        ppc = case RbConfig::CONFIG["target_os"]
+        when /darwin1/
+          IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
+        when /linux/
+          cores = {}  # unique physical ID / core ID combinations
+          phy = 0
+          IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
+            if ln.start_with?("physical")
+              phy = ln[/\d+/]
+            elsif ln.start_with?("core")
+              cid = phy + ":" + ln[/\d+/]
+              cores[cid] = true if not cores[cid]
+            end
+          end
+          cores.count
+        when /mswin|mingw/
+          require 'win32ole'
+          result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
+            "select NumberOfCores from Win32_Processor")
+          result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
+        else
+          processor_count
+        end
+        # fall back to logical count if physical info is invalid
+        ppc > 0 ? ppc : processor_count
+      end
+    end
+  end
+end

data/lib/rack2aws/version.rb

@@ -1,3 +1,3 @@
 module Rack2Aws
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack2aws
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Faissal Elamraoui
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-11 00:00:00.000000000 Z
+date: 2016-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -78,6 +78,7 @@ files:
 - lib/rack2aws.rb
 - lib/rack2aws/config.rb
 - lib/rack2aws/errors.rb
+- lib/rack2aws/processor_count.rb
 - lib/rack2aws/props_reader.rb
 - lib/rack2aws/version.rb
 homepage: https://amrfaissal.github.io/rack2aws