redsnapper 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c309291faf31352831f25043200a135caf7bcfec
-  data.tar.gz: 5257b0a6599e3a6b651d19774fac3b3d6cf89d11
+  metadata.gz: 437b104c05b11ec9161133017a941e47a7ffb346
+  data.tar.gz: 3d573ff4757231f174467a84750d59e820b38300
 SHA512:
-  metadata.gz: 5c6ef06d9e9a0c067cfd4dfcf19125d2a4420f877fa38985d6db376379c7f98df3e8da6b21ce0943b8890e21fd3da7f289a867051c5b0539a89da359a332ef84
-  data.tar.gz: 39b6eca49cc73cf4f4771b6d0b6b22f78e2fdbe220e8ff1d6509c036eb050ebe513573487eb14f6e5be25204e7bf26fddbc8feaafa702215374c212743a2daac
+  metadata.gz: 0aad4e248e1e002e7d5a29910726b64939f7f1a9bfbfc330bf603e7d1de6f6777d07d0028ce47cf186d70cafebde5732b856b0cd9eb3fbdb1ea12d247ed25699
+  data.tar.gz: 553e1c4415de048636a5fab39219da562c11e4a77afd6a8a21e40c06efa153d32ae87e0560dbb2567edb3359365e65053885bb90834f270bed3dedb7133a9058

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

data/bin/redsnapper

@@ -13,10 +13,6 @@ OptionParser.new do |opts|
   opts.on('-j', '--jobs COUNT', Integer, 'Number of workers to use') do |jobs|
     options[:thread_pool_size] = jobs
   end
-  opts.on('-m', '--max-files-per-job COUNT', Integer,
-          'Maximum number of files to extract in one job') do |count|
-    options[:max_files_per_job] = count
-  end
 end.parse!
 
 RedSnapper.new(ARGV.shift, options.merge(:tarsnap_options => ARGV)).run

data/lib/redsnapper.rb

@@ -2,51 +2,51 @@ require 'thread/pool'
 require 'open3'
 require 'set'
 
-class Array
-  def interleaved_slices(max_per_slice)
-    raise ArgumentError unless max_per_slice >= 1
-
-    count = (size.to_f / max_per_slice).ceil
-
-    slices = (1..count).map { [] }
-    each_with_index do |v, i|
-      slices[i % count].push(v)
-    end
-
-    slices
-  end
-end
-
 class RedSnapper
   TARSNAP = 'tarsnap'
   THREAD_POOL_SIZE = 10
-  MAX_FILES_PER_JOB = 1000
+
+  class Group
+    attr_reader :files, :size
+    def initialize
+      @files = []
+      @size = 0
+    end
+    def add(name, size)
+      @files << name
+      @size += size
+    end
+    def <=>(other)
+      other.size <=> size
+    end
+  end
 
   def initialize(archive, options = {})
     @archive = archive
     @options = options
-    @max_files_per_job = options[:max_files_per_job] || MAX_FILES_PER_JOB
     @thread_pool_size = options[:thread_pool_size] || THREAD_POOL_SIZE
   end
 
-  def file_groups
-    command = [ TARSNAP, '-tf', @archive, *@options[:tarsnap_options] ]
+  def files
+    command = [ TARSNAP, '-tvf', @archive, *@options[:tarsnap_options] ]
     command.push(@options[:directory]) if @options[:directory]
 
-    files = []
+    files = {}
     dirs = Set.new
 
     Open3.popen3(*command) do |_, out, _|
       out.gets(nil).split("\n").each do |entry|
-        if entry.end_with?('/')
-          dirs.add(entry)
+        (_, _, _, _, size, _, _, _, name) = entry.split(/\s+/, 9)
+        if name.end_with?('/')
+          dirs.add(name)
         else
-          files.push(entry)
+          files[name] = size.to_i
         end
       end
     end
 
-    files.each { |f| dirs.delete(File.dirname(f) + '/') }
+    files.each { |f, _| dirs.delete(File.dirname(f) + '/') }
+
     empty_dirs = dirs.clone
 
     dirs.each do |dir|
@@ -56,9 +56,17 @@ class RedSnapper
       end
     end
 
-    files.push(*empty_dirs)
-    files_per_slice = [ (files.size.to_f / @thread_pool_size).ceil, @max_files_per_job ].min
-    files.interleaved_slices(files_per_slice)
+    empty_dirs.each { |dir| files[dir] = 0 }
+
+    files
+  end
+
+  def file_groups
+    groups = (1..@thread_pool_size).map { Group.new }
+    files.sort { |a, b| b.last <=> a.last }.each do |file|
+      groups.sort.last.add(*file)
+    end
+    groups.map(&:files)
   end
 
   def run

data/redsnapper.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: redsnapper 0.2.1 ruby lib
+# stub: redsnapper 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "redsnapper"
-  s.version = "0.2.1"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redsnapper
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Scott Wheeler