redsnapper 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad704b383d106faf8371c582c8ac01674268e8b3
-  data.tar.gz: ff80041ec11718c48302a367a8a7a48bc59ca654
+  metadata.gz: 20921b02baf5c314923959df7f3eeb540cfdcb2d
+  data.tar.gz: 59c37e1602ade44ad3745f6ff42e0cac358f2a91
 SHA512:
-  metadata.gz: f21f67d23ddd54adeabcca421dff2ccbaf031544605a6d336cc903fc5e170a0831abe7e076532317b65831017835ad8e2de84a6b5d3413b40b0432accf60690d
-  data.tar.gz: 067eb80b79bd5baa06229802cfde89ef0f6b3382ac83005746929af890a47536147ab05ea25182be5d9642242355f35aed12f2fdede4f262d4be8b03cb5eca0e
+  metadata.gz: f6b52020b660064c763be81cea95dc917619f5c9d7be7792d62a8c0f4fc3f383ea1febf78f08ef959ddf0f1375049319c81ddc80943ffa258904a254a7d57689
+  data.tar.gz: 13217548cdaf7c211052feb77c8849ab5c365040b0ffd0be5e9a60cb271b754853a3d6f683ec061c9e158ba1634980c167ba3dba15430934441b9cf53374721e

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

data/lib/redsnapper.rb

@@ -1,11 +1,14 @@
 require 'thread/pool'
 require 'open3'
 require 'set'
+require 'date'
 
 class RedSnapper
   TARSNAP = 'tarsnap'
   THREAD_POOL_DEFAULT_SIZE = 10
+
   EXIT_ERROR = "tarsnap: Error exit delayed from previous errors.\n"
+  NOT_OLDER_ERROR = "File on disk is not older; skipping.\n"
 
   @@output_mutex = Mutex.new
 
@@ -31,51 +34,65 @@ class RedSnapper
     @error = false
   end
 
-  def file_sizes
-    return @sizes if @sizes
-    
+  def files
+    return @files if @files
+
     command = [ TARSNAP, '-tvf', @archive, *@options[:tarsnap_options] ]
     command.push(@options[:directory]) if @options[:directory]
 
-    @sizes = {}
-    dirs = Set.new
+    @files = {}
 
     Open3.popen3(*command) do |_, out, _|
       out.gets(nil).split("\n").each do |entry|
-        (_, _, _, _, size, _, _, _, name) = entry.split(/\s+/, 9)
-        if name.end_with?('/')
-          dirs.add(name)
-        else
-          @sizes[name] = size.to_i
-        end
+        (_, _, _, _, size, month, day, year_or_time, name) = entry.split(/\s+/, 9)
+
+        date = DateTime.parse("#{month} #{day}, #{year_or_time}")
+        date = date.prev_year if date < DateTime.now
+
+        @files[name] = {
+          :size => size.to_i,
+          :date => date
+        }
       end
     end
 
-    empty_dirs = dirs.clone
-    @sizes.each { |f, _| empty_dirs.delete(File.dirname(f) + '/') }
+    @files
+  end
 
+  def empty_dirs(files, dirs)
+    empty_dirs = dirs.clone
+    files.each { |f| empty_dirs.delete(File.dirname(f) + '/') }
     dirs.each do |dir|
       components = dir.split('/')[0..-2]
       components.each_with_index do |_, i|
         empty_dirs.delete(components[0, i + 1].join('/') + '/')
       end
     end
-
-    empty_dirs.each { |dir| @sizes[dir] = 0 }
-
-    @sizes
+    empty_dirs
   end
 
-  def files
-    @files ||= file_sizes.keys
+  def files_to_extract
+    files_to_extract, dirs = files.partition { |f| !f.first.end_with?('/') }.map(&:to_h)
+    empty_dirs(files_to_extract.keys, dirs.keys).each do |dir|
+      files_to_extract[dir] = { :size => 0 }
+    end
+    files_to_extract
   end
 
   def file_groups
     groups = (1..@thread_pool.max).map { Group.new }
-    file_sizes.sort { |a, b| b.last <=> a.last }.each do |file|
-      groups.sort.last.add(*file)
+    files_to_extract.sort { |a, b| b.last[:size] <=> a.last[:size] }.each do |name, props|
+
+      # If the previous batch of files had an entry with the same size and date,
+      # assume that this is a duplicate and assign it zero weight.  There may be
+      # some false positives here since the granularity of the data we have from
+      # tarsnap is only "same day".  However, a false positive just affects the
+      # queing scheme, not which files get queued.
+
+      size = (@options[:previous] && @options[:previous][name] == props) ? 0 : props[:size]
+      groups.sort.last.add(name, size)
     end
-    groups.map(&:files)
+    groups.map(&:files).reject(&:empty?)
   end
 
   def run
@@ -84,6 +101,7 @@ class RedSnapper
         command = [ TARSNAP, '-xvf', @archive, *(@options[:tarsnap_options] + chunk) ]
         Open3.popen3(*command) do |_, _, err|
           while line = err.gets
+            next if line.end_with?(NOT_OLDER_ERROR)
             if line == EXIT_ERROR
               @error = true
               next

data/redsnapper.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: redsnapper 0.3.1 ruby lib
+# stub: redsnapper 0.4.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "redsnapper"
-  s.version = "0.3.1"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Scott Wheeler"]
-  s.date = "2016-12-31"
+  s.date = "2017-01-05"
   s.description = "Faster extraction of large tarsnap archives using a pool of parallel tarsnap clients"
   s.email = "scott@directededge.com"
   s.executables = ["redsnapper"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redsnapper
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Scott Wheeler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-31 00:00:00.000000000 Z
+date: 2017-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thread