uricp 0.0.13 → 0.0.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +2 -1
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +20 -23
- data/Jenkinsfile +113 -0
- data/README.md +2 -2
- data/Rakefile +26 -31
- data/bionic/Dockerfile +20 -0
- data/centos7/Dockerfile +18 -0
- data/features/step_definitions/orbit_steps.rb +19 -13
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +21 -0
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp.rb +3 -29
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +11 -11
- data/lib/uricp/strategy/cached_get.rb +14 -20
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +27 -19
- data/lib/uricp/strategy/local_convert.rb +10 -17
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +7 -13
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +46 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +14 -13
- data/lib/uricp/version.rb +4 -2
- data/uricp.gemspec +3 -4
- data/xenial/Dockerfile +20 -0
- metadata +39 -51
- data/Dockerfile_centos6.6 +0 -39
- data/Dockerfile_centos7 +0 -39
- data/Dockerfile_trusty-ruby193 +0 -32
- data/README.rdoc +0 -23
- data/spec/something_spec.rb +0 -5
@@ -1,13 +1,10 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class Cleaner
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
if options['clean'] && sequence_complete?
|
9
|
-
|
10
|
-
end
|
6
|
+
return proposal if options['clean'] && sequence_complete?
|
7
|
+
|
11
8
|
debug "#{self.class.name}: not appropriate"
|
12
9
|
false
|
13
10
|
end
|
@@ -21,8 +18,5 @@ module Uricp::Strategy
|
|
21
18
|
@proposed_options.delete('clean')
|
22
19
|
self
|
23
20
|
end
|
24
|
-
|
25
21
|
end
|
26
|
-
|
27
22
|
end
|
28
|
-
|
@@ -1,7 +1,5 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
module Common
|
4
|
-
|
5
3
|
include Methadone::CLILogging
|
6
4
|
include Uricp::CurlPrimitives
|
7
5
|
|
@@ -14,10 +12,10 @@ module Uricp::Strategy
|
|
14
12
|
|
15
13
|
def unsupported_transfer
|
16
14
|
raise Uricp::UnsupportedURLtype,
|
17
|
-
|
15
|
+
"Unsupported transfer from #{from} to #{to}"
|
18
16
|
end
|
19
17
|
|
20
|
-
alias
|
18
|
+
alias command unsupported_transfer
|
21
19
|
|
22
20
|
def all_local_files?
|
23
21
|
!sequence_complete? && file_source? && to.scheme == 'file'
|
@@ -44,24 +42,23 @@ module Uricp::Strategy
|
|
44
42
|
end
|
45
43
|
|
46
44
|
def qcow2?(magic)
|
47
|
-
magic.unpack('a3C') == ['QFI',0xfb]
|
45
|
+
magic.unpack('a3C') == ['QFI', 0xfb]
|
48
46
|
end
|
49
47
|
|
50
48
|
def encoding(io)
|
51
49
|
magic = io.read(4).to_s
|
52
|
-
|
53
|
-
when lz4?(magic)
|
50
|
+
if lz4?(magic)
|
54
51
|
:lz4
|
55
|
-
|
52
|
+
elsif qcow2?(magic)
|
56
53
|
version = io.read(4)
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
54
|
+
case version.unpack('N')
|
55
|
+
when [2]
|
56
|
+
:qcow2
|
57
|
+
when [3]
|
58
|
+
:qcow3
|
59
|
+
else
|
60
|
+
:qcow2un
|
61
|
+
end
|
65
62
|
else
|
66
63
|
:raw
|
67
64
|
end
|
@@ -73,6 +70,10 @@ module Uricp::Strategy
|
|
73
70
|
)
|
74
71
|
end
|
75
72
|
|
73
|
+
def dry_run?
|
74
|
+
options['dry-run']
|
75
|
+
end
|
76
|
+
|
76
77
|
def lz4_source?
|
77
78
|
options['source-format'] == :lz4
|
78
79
|
end
|
@@ -91,8 +92,9 @@ module Uricp::Strategy
|
|
91
92
|
|
92
93
|
PIPE_URI = URI('pipe:/')
|
93
94
|
|
94
|
-
def get_temp_filename(
|
95
|
-
|
95
|
+
def get_temp_filename(base_dir)
|
96
|
+
t = Time.now.strftime('%Y%m%d')
|
97
|
+
File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
|
96
98
|
end
|
97
99
|
|
98
100
|
def proposed_path
|
@@ -107,6 +109,12 @@ module Uricp::Strategy
|
|
107
109
|
options['source-format'] && !lz4_source?
|
108
110
|
end
|
109
111
|
|
110
|
-
|
112
|
+
def rbd_target(uri)
|
113
|
+
uri.path[1..-1]
|
114
|
+
end
|
111
115
|
|
116
|
+
def rbd_id
|
117
|
+
'libvirt'
|
118
|
+
end
|
119
|
+
end
|
112
120
|
end
|
@@ -1,17 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalConvert
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
if conversion_required? && file_source? && supported_source?
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
"qemu-img does not support required conversion"
|
14
|
-
end
|
7
|
+
return proposal if new_qemu? || supported_conversion?
|
8
|
+
|
9
|
+
raise Uricp::UnsupportedConversion,
|
10
|
+
'qemu-img does not support required conversion'
|
15
11
|
end
|
16
12
|
debug "#{self.class.name}: not appropriate"
|
17
13
|
false
|
@@ -24,11 +20,11 @@ module Uricp::Strategy
|
|
24
20
|
def proposal
|
25
21
|
@proposed_options = options.dup
|
26
22
|
if to.scheme == 'file'
|
27
|
-
|
23
|
+
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
28
24
|
else
|
29
25
|
@proposed_options['from_uri'] = temp_uri
|
30
|
-
|
31
|
-
|
26
|
+
@proposed_options['clean'] ||= []
|
27
|
+
@proposed_options['clean'] << proposed_path
|
32
28
|
end
|
33
29
|
@proposed_options.delete('source-format')
|
34
30
|
@proposed_options.delete('target-format')
|
@@ -36,11 +32,11 @@ module Uricp::Strategy
|
|
36
32
|
end
|
37
33
|
|
38
34
|
def new_qemu?
|
39
|
-
@new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~
|
35
|
+
@new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
|
40
36
|
end
|
41
37
|
|
42
38
|
def supported_conversion?
|
43
|
-
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
|
39
|
+
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
|
44
40
|
end
|
45
41
|
|
46
42
|
def qemu_img_command(target)
|
@@ -59,14 +55,11 @@ module Uricp::Strategy
|
|
59
55
|
end
|
60
56
|
|
61
57
|
def compat_option
|
62
|
-
|
58
|
+
'-o compat=0.10' if new_qemu?
|
63
59
|
end
|
64
60
|
|
65
61
|
def compress_option
|
66
62
|
options['compress'] && '-c'
|
67
63
|
end
|
68
|
-
|
69
64
|
end
|
70
|
-
|
71
65
|
end
|
72
|
-
|
@@ -1,11 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalLink
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
return proposal if
|
6
|
+
return proposal if !compression_required? &&
|
7
|
+
all_local_files? &&
|
8
|
+
!format_change? &&
|
9
|
+
linkable?
|
10
|
+
|
9
11
|
debug "#{self.class.name}: not appropriate"
|
10
12
|
false
|
11
13
|
end
|
@@ -21,11 +23,11 @@ module Uricp::Strategy
|
|
21
23
|
end
|
22
24
|
|
23
25
|
def linkable?
|
24
|
-
|
25
|
-
|
26
|
+
from_path_dir = File.dirname(from.path)
|
27
|
+
to_path_dir = File.dirname(to.path)
|
28
|
+
File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
|
29
|
+
File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
|
30
|
+
(!File.exist?(to.path) || File.file?(to.path))
|
26
31
|
end
|
27
|
-
|
28
32
|
end
|
29
|
-
|
30
33
|
end
|
31
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCache
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
9
|
if cache_root
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache!
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no cacheing requested"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -32,9 +30,5 @@ module Uricp::Strategy
|
|
32
30
|
@proposed_options.delete('cache_name')
|
33
31
|
self
|
34
32
|
end
|
35
|
-
|
36
33
|
end
|
37
|
-
|
38
34
|
end
|
39
|
-
|
40
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCacheConvert
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
|
-
if conversion_required? && supported_source?
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
if conversion_required? && supported_source?
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache! if cache_root
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no non-stream conversion detected"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -28,18 +26,16 @@ module Uricp::Strategy
|
|
28
26
|
def proposal
|
29
27
|
@proposed_options = options.dup
|
30
28
|
if cache_root
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
@proposed_options['from_uri'] = temp_cache_uri
|
30
|
+
@proposed_options['sweep'] = [temp_cache_file, cache_file]
|
31
|
+
@proposed_options.delete('cache')
|
32
|
+
@proposed_options.delete('cache_name')
|
35
33
|
else
|
36
34
|
@proposed_options['from_uri'] = temp_uri
|
37
|
-
|
38
|
-
|
35
|
+
@proposed_options['clean'] ||= []
|
36
|
+
@proposed_options['clean'] << proposed_path
|
39
37
|
end
|
40
38
|
self
|
41
39
|
end
|
42
|
-
|
43
40
|
end
|
44
|
-
|
45
41
|
end
|
@@ -1,30 +1,25 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedCompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
|
-
|
8
5
|
def appropriate?
|
9
6
|
case from.scheme
|
10
7
|
when 'pipe'
|
11
|
-
|
8
|
+
return proposal if compression_required?
|
12
9
|
end
|
13
10
|
debug "#{self.class.name}: not appropriate"
|
14
11
|
false
|
15
12
|
end
|
16
13
|
|
17
14
|
def command
|
18
|
-
|
15
|
+
'lz4 |'
|
19
16
|
end
|
20
17
|
|
21
18
|
def proposal
|
22
19
|
@proposed_options = options.dup
|
23
20
|
@proposed_options.delete('compress')
|
24
|
-
@proposed_options['encoding']='lz4'
|
21
|
+
@proposed_options['encoding'] = 'lz4'
|
25
22
|
self
|
26
23
|
end
|
27
|
-
|
28
24
|
end
|
29
|
-
|
30
25
|
end
|
@@ -1,35 +1,31 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedDecompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
case from.scheme
|
9
7
|
when 'pipe'
|
10
|
-
|
8
|
+
return proposal if lz4_source?
|
11
9
|
end
|
12
10
|
debug "#{self.class.name}: not appropriate"
|
13
11
|
false
|
14
12
|
end
|
15
13
|
|
16
14
|
def command
|
17
|
-
|
15
|
+
'lz4 -d |'
|
18
16
|
end
|
19
17
|
|
20
18
|
def proposal
|
21
19
|
@proposed_options = options.dup
|
22
20
|
@proposed_options.delete('source-format')
|
23
21
|
if @proposed_options['target-format']
|
24
|
-
|
25
|
-
if @proposed_options['source-format'] == @proposed_options['target-format']
|
26
|
-
|
27
|
-
|
28
|
-
|
22
|
+
@proposed_options['source-format'] = :raw
|
23
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
24
|
+
@proposed_options.delete('source-format')
|
25
|
+
@proposed_options.delete('target-format')
|
26
|
+
end
|
29
27
|
end
|
30
28
|
self
|
31
29
|
end
|
32
|
-
|
33
30
|
end
|
34
|
-
|
35
31
|
end
|
@@ -1,19 +1,15 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedLocalDecompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
case from.scheme
|
9
7
|
when 'pipe'
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
16
|
-
end
|
8
|
+
if raw_target? && lz4_source? && to.scheme == 'file'
|
9
|
+
return proposal if always_write_sparse?
|
10
|
+
|
11
|
+
debug "#{self.class.name}: using safe sparse expansion via stream"
|
12
|
+
end
|
17
13
|
end
|
18
14
|
debug "#{self.class.name}: not appropriate"
|
19
15
|
false
|
@@ -27,16 +23,14 @@ module Uricp::Strategy
|
|
27
23
|
@proposed_options = options.dup
|
28
24
|
@proposed_options.delete('source-format')
|
29
25
|
if @proposed_options['target-format']
|
30
|
-
|
31
|
-
if @proposed_options['source-format'] == @proposed_options['target-format']
|
32
|
-
|
33
|
-
|
34
|
-
|
26
|
+
@proposed_options['source-format'] = :raw
|
27
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
28
|
+
@proposed_options.delete('source-format')
|
29
|
+
@proposed_options.delete('target-format')
|
30
|
+
end
|
35
31
|
end
|
36
32
|
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
37
33
|
self
|
38
34
|
end
|
39
|
-
|
40
35
|
end
|
41
|
-
|
42
36
|
end
|