redsnapper 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/redsnapper.rb +31 -20
- data/redsnapper.gemspec +3 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad704b383d106faf8371c582c8ac01674268e8b3
|
4
|
+
data.tar.gz: ff80041ec11718c48302a367a8a7a48bc59ca654
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f21f67d23ddd54adeabcca421dff2ccbaf031544605a6d336cc903fc5e170a0831abe7e076532317b65831017835ad8e2de84a6b5d3413b40b0432accf60690d
|
7
|
+
data.tar.gz: 067eb80b79bd5baa06229802cfde89ef0f6b3382ac83005746929af890a47536147ab05ea25182be5d9642242355f35aed12f2fdede4f262d4be8b03cb5eca0e
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.1
|
data/lib/redsnapper.rb
CHANGED
@@ -4,7 +4,10 @@ require 'set'
|
|
4
4
|
|
5
5
|
class RedSnapper
|
6
6
|
TARSNAP = 'tarsnap'
|
7
|
-
|
7
|
+
THREAD_POOL_DEFAULT_SIZE = 10
|
8
|
+
EXIT_ERROR = "tarsnap: Error exit delayed from previous errors.\n"
|
9
|
+
|
10
|
+
@@output_mutex = Mutex.new
|
8
11
|
|
9
12
|
class Group
|
10
13
|
attr_reader :files, :size
|
@@ -24,14 +27,17 @@ class RedSnapper
|
|
24
27
|
def initialize(archive, options = {})
|
25
28
|
@archive = archive
|
26
29
|
@options = options
|
27
|
-
@
|
30
|
+
@thread_pool = Thread.pool(options[:thread_pool_size] || THREAD_POOL_DEFAULT_SIZE)
|
31
|
+
@error = false
|
28
32
|
end
|
29
33
|
|
30
|
-
def
|
34
|
+
def file_sizes
|
35
|
+
return @sizes if @sizes
|
36
|
+
|
31
37
|
command = [ TARSNAP, '-tvf', @archive, *@options[:tarsnap_options] ]
|
32
38
|
command.push(@options[:directory]) if @options[:directory]
|
33
39
|
|
34
|
-
|
40
|
+
@sizes = {}
|
35
41
|
dirs = Set.new
|
36
42
|
|
37
43
|
Open3.popen3(*command) do |_, out, _|
|
@@ -40,50 +46,55 @@ class RedSnapper
|
|
40
46
|
if name.end_with?('/')
|
41
47
|
dirs.add(name)
|
42
48
|
else
|
43
|
-
|
49
|
+
@sizes[name] = size.to_i
|
44
50
|
end
|
45
51
|
end
|
46
52
|
end
|
47
53
|
|
48
|
-
files.each { |f, _| dirs.delete(File.dirname(f) + '/') }
|
49
|
-
|
50
54
|
empty_dirs = dirs.clone
|
55
|
+
@sizes.each { |f, _| empty_dirs.delete(File.dirname(f) + '/') }
|
51
56
|
|
52
57
|
dirs.each do |dir|
|
53
|
-
components = dir.split('/')
|
54
|
-
components.each_with_index do |
|
55
|
-
empty_dirs.delete(components[0, i].join('/') + '/')
|
58
|
+
components = dir.split('/')[0..-2]
|
59
|
+
components.each_with_index do |_, i|
|
60
|
+
empty_dirs.delete(components[0, i + 1].join('/') + '/')
|
56
61
|
end
|
57
62
|
end
|
58
63
|
|
59
|
-
empty_dirs.each { |dir|
|
64
|
+
empty_dirs.each { |dir| @sizes[dir] = 0 }
|
60
65
|
|
61
|
-
|
66
|
+
@sizes
|
67
|
+
end
|
68
|
+
|
69
|
+
def files
|
70
|
+
@files ||= file_sizes.keys
|
62
71
|
end
|
63
72
|
|
64
73
|
def file_groups
|
65
|
-
groups = (1..@
|
66
|
-
|
74
|
+
groups = (1..@thread_pool.max).map { Group.new }
|
75
|
+
file_sizes.sort { |a, b| b.last <=> a.last }.each do |file|
|
67
76
|
groups.sort.last.add(*file)
|
68
77
|
end
|
69
78
|
groups.map(&:files)
|
70
79
|
end
|
71
80
|
|
72
81
|
def run
|
73
|
-
pool = Thread.pool(@thread_pool_size)
|
74
|
-
mutex = Mutex.new
|
75
|
-
|
76
82
|
file_groups.each do |chunk|
|
77
|
-
|
83
|
+
@thread_pool.process do
|
78
84
|
command = [ TARSNAP, '-xvf', @archive, *(@options[:tarsnap_options] + chunk) ]
|
79
85
|
Open3.popen3(*command) do |_, _, err|
|
80
86
|
while line = err.gets
|
81
|
-
|
87
|
+
if line == EXIT_ERROR
|
88
|
+
@error = true
|
89
|
+
next
|
90
|
+
end
|
91
|
+
@@output_mutex.synchronize { warn line.chomp }
|
82
92
|
end
|
83
93
|
end
|
84
94
|
end
|
85
95
|
end
|
86
96
|
|
87
|
-
|
97
|
+
@thread_pool.shutdown
|
98
|
+
@@output_mutex.synchronize { warn EXIT_ERROR } if @error
|
88
99
|
end
|
89
100
|
end
|
data/redsnapper.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: redsnapper 0.3.
|
5
|
+
# stub: redsnapper 0.3.1 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "redsnapper"
|
9
|
-
s.version = "0.3.
|
9
|
+
s.version = "0.3.1"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Scott Wheeler"]
|
14
|
-
s.date = "2016-12-
|
14
|
+
s.date = "2016-12-31"
|
15
15
|
s.description = "Faster extraction of large tarsnap archives using a pool of parallel tarsnap clients"
|
16
16
|
s.email = "scott@directededge.com"
|
17
17
|
s.executables = ["redsnapper"]
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redsnapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Scott Wheeler
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thread
|