redsnapper 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/bin/redsnapper +7 -0
- data/lib/redsnapper.rb +20 -2
- data/redsnapper.gemspec +3 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c309291faf31352831f25043200a135caf7bcfec
|
4
|
+
data.tar.gz: 5257b0a6599e3a6b651d19774fac3b3d6cf89d11
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5c6ef06d9e9a0c067cfd4dfcf19125d2a4420f877fa38985d6db376379c7f98df3e8da6b21ce0943b8890e21fd3da7f289a867051c5b0539a89da359a332ef84
|
7
|
+
data.tar.gz: 39b6eca49cc73cf4f4771b6d0b6b22f78e2fdbe220e8ff1d6509c036eb050ebe513573487eb14f6e5be25204e7bf26fddbc8feaafa702215374c212743a2daac
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.1
|
data/bin/redsnapper
CHANGED
@@ -10,6 +10,13 @@ OptionParser.new do |opts|
|
|
10
10
|
opts.on('-d', '--directory DIR', 'Extract files from this directory of the archive') do |dir|
|
11
11
|
options[:directory] = dir
|
12
12
|
end
|
13
|
+
opts.on('-j', '--jobs COUNT', Integer, 'Number of workers to use') do |jobs|
|
14
|
+
options[:thread_pool_size] = jobs
|
15
|
+
end
|
16
|
+
opts.on('-m', '--max-files-per-job COUNT', Integer,
|
17
|
+
'Maximum number of files to extract in one job') do |count|
|
18
|
+
options[:max_files_per_job] = count
|
19
|
+
end
|
13
20
|
end.parse!
|
14
21
|
|
15
22
|
RedSnapper.new(ARGV.shift, options.merge(:tarsnap_options => ARGV)).run
|
data/lib/redsnapper.rb
CHANGED
@@ -2,6 +2,21 @@ require 'thread/pool'
|
|
2
2
|
require 'open3'
|
3
3
|
require 'set'
|
4
4
|
|
5
|
+
class Array
|
6
|
+
def interleaved_slices(max_per_slice)
|
7
|
+
raise ArgumentError unless max_per_slice >= 1
|
8
|
+
|
9
|
+
count = (size.to_f / max_per_slice).ceil
|
10
|
+
|
11
|
+
slices = (1..count).map { [] }
|
12
|
+
each_with_index do |v, i|
|
13
|
+
slices[i % count].push(v)
|
14
|
+
end
|
15
|
+
|
16
|
+
slices
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
5
20
|
class RedSnapper
|
6
21
|
TARSNAP = 'tarsnap'
|
7
22
|
THREAD_POOL_SIZE = 10
|
@@ -10,6 +25,8 @@ class RedSnapper
|
|
10
25
|
def initialize(archive, options = {})
|
11
26
|
@archive = archive
|
12
27
|
@options = options
|
28
|
+
@max_files_per_job = options[:max_files_per_job] || MAX_FILES_PER_JOB
|
29
|
+
@thread_pool_size = options[:thread_pool_size] || THREAD_POOL_SIZE
|
13
30
|
end
|
14
31
|
|
15
32
|
def file_groups
|
@@ -40,11 +57,12 @@ class RedSnapper
|
|
40
57
|
end
|
41
58
|
|
42
59
|
files.push(*empty_dirs)
|
43
|
-
|
60
|
+
files_per_slice = [ (files.size.to_f / @thread_pool_size).ceil, @max_files_per_job ].min
|
61
|
+
files.interleaved_slices(files_per_slice)
|
44
62
|
end
|
45
63
|
|
46
64
|
def run
|
47
|
-
pool = Thread.pool(
|
65
|
+
pool = Thread.pool(@thread_pool_size)
|
48
66
|
mutex = Mutex.new
|
49
67
|
|
50
68
|
file_groups.each do |chunk|
|
data/redsnapper.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: redsnapper 0.2.
|
5
|
+
# stub: redsnapper 0.2.1 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "redsnapper"
|
9
|
-
s.version = "0.2.
|
9
|
+
s.version = "0.2.1"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Scott Wheeler"]
|
14
|
-
s.date = "2016-12-
|
14
|
+
s.date = "2016-12-30"
|
15
15
|
s.description = "Faster extraction of large tarsnap archives using a pool of parallel tarsnap clients"
|
16
16
|
s.email = "scott@directededge.com"
|
17
17
|
s.executables = ["redsnapper"]
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redsnapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Scott Wheeler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thread
|