redsnapper 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 437b104c05b11ec9161133017a941e47a7ffb346
-  data.tar.gz: 3d573ff4757231f174467a84750d59e820b38300
+  metadata.gz: ad704b383d106faf8371c582c8ac01674268e8b3
+  data.tar.gz: ff80041ec11718c48302a367a8a7a48bc59ca654
 SHA512:
-  metadata.gz: 0aad4e248e1e002e7d5a29910726b64939f7f1a9bfbfc330bf603e7d1de6f6777d07d0028ce47cf186d70cafebde5732b856b0cd9eb3fbdb1ea12d247ed25699
-  data.tar.gz: 553e1c4415de048636a5fab39219da562c11e4a77afd6a8a21e40c06efa153d32ae87e0560dbb2567edb3359365e65053885bb90834f270bed3dedb7133a9058
+  metadata.gz: f21f67d23ddd54adeabcca421dff2ccbaf031544605a6d336cc903fc5e170a0831abe7e076532317b65831017835ad8e2de84a6b5d3413b40b0432accf60690d
+  data.tar.gz: 067eb80b79bd5baa06229802cfde89ef0f6b3382ac83005746929af890a47536147ab05ea25182be5d9642242355f35aed12f2fdede4f262d4be8b03cb5eca0e

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/lib/redsnapper.rb

@@ -4,7 +4,10 @@ require 'set'
 
 class RedSnapper
   TARSNAP = 'tarsnap'
-  THREAD_POOL_SIZE = 10
+  THREAD_POOL_DEFAULT_SIZE = 10
+  EXIT_ERROR = "tarsnap: Error exit delayed from previous errors.\n"
+
+  @@output_mutex = Mutex.new
 
   class Group
     attr_reader :files, :size
@@ -24,14 +27,17 @@ class RedSnapper
   def initialize(archive, options = {})
     @archive = archive
     @options = options
-    @thread_pool_size = options[:thread_pool_size] || THREAD_POOL_SIZE
+    @thread_pool = Thread.pool(options[:thread_pool_size] || THREAD_POOL_DEFAULT_SIZE)
+    @error = false
   end
 
-  def files
+  def file_sizes
+    return @sizes if @sizes
+    
     command = [ TARSNAP, '-tvf', @archive, *@options[:tarsnap_options] ]
     command.push(@options[:directory]) if @options[:directory]
 
-    files = {}
+    @sizes = {}
     dirs = Set.new
 
     Open3.popen3(*command) do |_, out, _|
@@ -40,50 +46,55 @@ class RedSnapper
         if name.end_with?('/')
           dirs.add(name)
         else
-          files[name] = size.to_i
+          @sizes[name] = size.to_i
         end
       end
     end
 
-    files.each { |f, _| dirs.delete(File.dirname(f) + '/') }
-
     empty_dirs = dirs.clone
+    @sizes.each { |f, _| empty_dirs.delete(File.dirname(f) + '/') }
 
     dirs.each do |dir|
-      components = dir.split('/')
-      components.each_with_index do |component, i|
-        empty_dirs.delete(components[0, i].join('/') + '/')
+      components = dir.split('/')[0..-2]
+      components.each_with_index do |_, i|
+        empty_dirs.delete(components[0, i + 1].join('/') + '/')
       end
     end
 
-    empty_dirs.each { |dir| files[dir] = 0 }
+    empty_dirs.each { |dir| @sizes[dir] = 0 }
 
-    files
+    @sizes
+  end
+
+  def files
+    @files ||= file_sizes.keys
   end
 
   def file_groups
-    groups = (1..@thread_pool_size).map { Group.new }
-    files.sort { |a, b| b.last <=> a.last }.each do |file|
+    groups = (1..@thread_pool.max).map { Group.new }
+    file_sizes.sort { |a, b| b.last <=> a.last }.each do |file|
       groups.sort.last.add(*file)
     end
     groups.map(&:files)
   end
 
   def run
-    pool = Thread.pool(@thread_pool_size)
-    mutex = Mutex.new
-
     file_groups.each do |chunk|
-      pool.process do
+      @thread_pool.process do
         command = [ TARSNAP, '-xvf', @archive, *(@options[:tarsnap_options] + chunk) ]
         Open3.popen3(*command) do |_, _, err|
           while line = err.gets
-            mutex.synchronize { warn line.chomp }
+            if line == EXIT_ERROR
+              @error = true
+              next
+            end
+            @@output_mutex.synchronize { warn line.chomp }
           end
         end
       end
     end
 
-    pool.shutdown
+    @thread_pool.shutdown
+    @@output_mutex.synchronize { warn EXIT_ERROR } if @error
   end
 end

data/redsnapper.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: redsnapper 0.3.0 ruby lib
+# stub: redsnapper 0.3.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "redsnapper"
-  s.version = "0.3.0"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Scott Wheeler"]
-  s.date = "2016-12-30"
+  s.date = "2016-12-31"
   s.description = "Faster extraction of large tarsnap archives using a pool of parallel tarsnap clients"
   s.email = "scott@directededge.com"
   s.executables = ["redsnapper"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redsnapper
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Scott Wheeler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-30 00:00:00.000000000 Z
+date: 2016-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thread