redsnapper 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 062acc09584fa69d3d578a9e3c31f8e43c894e6f
-  data.tar.gz: 802d7dcd2beeef69c6d36e12f81d69717f22137a
+  metadata.gz: c309291faf31352831f25043200a135caf7bcfec
+  data.tar.gz: 5257b0a6599e3a6b651d19774fac3b3d6cf89d11
 SHA512:
-  metadata.gz: c11dfc51cd54450e5e9e1c5304021e06130997e98fa9c8a07539e499dac07b1b4cce05e0d0e519d01fa4d41e005d2743eb4c782f55d26074333982453a2bb5fa
-  data.tar.gz: c09deb701584d816271ad581d276ee58ab472af3329ab0f2badad4c00f7c38f3684013d042b7146d9df992c753b2b49a3b2bcfd22457d6a0817ea89ef1546004
+  metadata.gz: 5c6ef06d9e9a0c067cfd4dfcf19125d2a4420f877fa38985d6db376379c7f98df3e8da6b21ce0943b8890e21fd3da7f289a867051c5b0539a89da359a332ef84
+  data.tar.gz: 39b6eca49cc73cf4f4771b6d0b6b22f78e2fdbe220e8ff1d6509c036eb050ebe513573487eb14f6e5be25204e7bf26fddbc8feaafa702215374c212743a2daac

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

data/bin/redsnapper

@@ -10,6 +10,13 @@ OptionParser.new do |opts|
   opts.on('-d', '--directory DIR', 'Extract files from this directory of the archive') do |dir|
     options[:directory] = dir
   end
+  opts.on('-j', '--jobs COUNT', Integer, 'Number of workers to use') do |jobs|
+    options[:thread_pool_size] = jobs
+  end
+  opts.on('-m', '--max-files-per-job COUNT', Integer,
+          'Maximum number of files to extract in one job') do |count|
+    options[:max_files_per_job] = count
+  end
 end.parse!
 
 RedSnapper.new(ARGV.shift, options.merge(:tarsnap_options => ARGV)).run

data/lib/redsnapper.rb

@@ -2,6 +2,21 @@ require 'thread/pool'
 require 'open3'
 require 'set'
 
+class Array
+  def interleaved_slices(max_per_slice)
+    raise ArgumentError unless max_per_slice >= 1
+
+    count = (size.to_f / max_per_slice).ceil
+
+    slices = (1..count).map { [] }
+    each_with_index do |v, i|
+      slices[i % count].push(v)
+    end
+
+    slices
+  end
+end
+
 class RedSnapper
   TARSNAP = 'tarsnap'
   THREAD_POOL_SIZE = 10
@@ -10,6 +25,8 @@ class RedSnapper
   def initialize(archive, options = {})
     @archive = archive
     @options = options
+    @max_files_per_job = options[:max_files_per_job] || MAX_FILES_PER_JOB
+    @thread_pool_size = options[:thread_pool_size] || THREAD_POOL_SIZE
   end
 
   def file_groups
@@ -40,11 +57,12 @@ class RedSnapper
     end
 
     files.push(*empty_dirs)
-    files.each_slice([ (files.size.to_f / THREAD_POOL_SIZE).ceil, MAX_FILES_PER_JOB ].min).to_a
+    files_per_slice = [ (files.size.to_f / @thread_pool_size).ceil, @max_files_per_job ].min
+    files.interleaved_slices(files_per_slice)
   end
 
   def run
-    pool = Thread.pool(THREAD_POOL_SIZE)
+    pool = Thread.pool(@thread_pool_size)
     mutex = Mutex.new
 
     file_groups.each do |chunk|

data/redsnapper.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: redsnapper 0.2.0 ruby lib
+# stub: redsnapper 0.2.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "redsnapper"
-  s.version = "0.2.0"
+  s.version = "0.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Scott Wheeler"]
-  s.date = "2016-12-16"
+  s.date = "2016-12-30"
   s.description = "Faster extraction of large tarsnap archives using a pool of parallel tarsnap clients"
   s.email = "scott@directededge.com"
   s.executables = ["redsnapper"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redsnapper
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Scott Wheeler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-16 00:00:00.000000000 Z
+date: 2016-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thread