uricp 0.0.13 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +2 -1
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +20 -23
- data/Jenkinsfile +113 -0
- data/README.md +2 -2
- data/Rakefile +26 -31
- data/bionic/Dockerfile +20 -0
- data/centos7/Dockerfile +18 -0
- data/features/step_definitions/orbit_steps.rb +19 -13
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +21 -0
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp.rb +3 -29
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +11 -11
- data/lib/uricp/strategy/cached_get.rb +14 -20
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +27 -19
- data/lib/uricp/strategy/local_convert.rb +10 -17
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +7 -13
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +46 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +14 -13
- data/lib/uricp/version.rb +4 -2
- data/uricp.gemspec +3 -4
- data/xenial/Dockerfile +20 -0
- metadata +39 -51
- data/Dockerfile_centos6.6 +0 -39
- data/Dockerfile_centos7 +0 -39
- data/Dockerfile_trusty-ruby193 +0 -32
- data/README.rdoc +0 -23
- data/spec/something_spec.rb +0 -5
@@ -1,13 +1,10 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class Cleaner
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
if options['clean'] && sequence_complete?
|
9
|
-
|
10
|
-
end
|
6
|
+
return proposal if options['clean'] && sequence_complete?
|
7
|
+
|
11
8
|
debug "#{self.class.name}: not appropriate"
|
12
9
|
false
|
13
10
|
end
|
@@ -21,8 +18,5 @@ module Uricp::Strategy
|
|
21
18
|
@proposed_options.delete('clean')
|
22
19
|
self
|
23
20
|
end
|
24
|
-
|
25
21
|
end
|
26
|
-
|
27
22
|
end
|
28
|
-
|
@@ -1,7 +1,5 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
module Common
|
4
|
-
|
5
3
|
include Methadone::CLILogging
|
6
4
|
include Uricp::CurlPrimitives
|
7
5
|
|
@@ -14,10 +12,10 @@ module Uricp::Strategy
|
|
14
12
|
|
15
13
|
def unsupported_transfer
|
16
14
|
raise Uricp::UnsupportedURLtype,
|
17
|
-
|
15
|
+
"Unsupported transfer from #{from} to #{to}"
|
18
16
|
end
|
19
17
|
|
20
|
-
alias
|
18
|
+
alias command unsupported_transfer
|
21
19
|
|
22
20
|
def all_local_files?
|
23
21
|
!sequence_complete? && file_source? && to.scheme == 'file'
|
@@ -44,24 +42,23 @@ module Uricp::Strategy
|
|
44
42
|
end
|
45
43
|
|
46
44
|
def qcow2?(magic)
|
47
|
-
magic.unpack('a3C') == ['QFI',0xfb]
|
45
|
+
magic.unpack('a3C') == ['QFI', 0xfb]
|
48
46
|
end
|
49
47
|
|
50
48
|
def encoding(io)
|
51
49
|
magic = io.read(4).to_s
|
52
|
-
|
53
|
-
when lz4?(magic)
|
50
|
+
if lz4?(magic)
|
54
51
|
:lz4
|
55
|
-
|
52
|
+
elsif qcow2?(magic)
|
56
53
|
version = io.read(4)
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
54
|
+
case version.unpack('N')
|
55
|
+
when [2]
|
56
|
+
:qcow2
|
57
|
+
when [3]
|
58
|
+
:qcow3
|
59
|
+
else
|
60
|
+
:qcow2un
|
61
|
+
end
|
65
62
|
else
|
66
63
|
:raw
|
67
64
|
end
|
@@ -73,6 +70,10 @@ module Uricp::Strategy
|
|
73
70
|
)
|
74
71
|
end
|
75
72
|
|
73
|
+
def dry_run?
|
74
|
+
options['dry-run']
|
75
|
+
end
|
76
|
+
|
76
77
|
def lz4_source?
|
77
78
|
options['source-format'] == :lz4
|
78
79
|
end
|
@@ -91,8 +92,9 @@ module Uricp::Strategy
|
|
91
92
|
|
92
93
|
PIPE_URI = URI('pipe:/')
|
93
94
|
|
94
|
-
def get_temp_filename(
|
95
|
-
|
95
|
+
def get_temp_filename(base_dir)
|
96
|
+
t = Time.now.strftime('%Y%m%d')
|
97
|
+
File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
|
96
98
|
end
|
97
99
|
|
98
100
|
def proposed_path
|
@@ -107,6 +109,12 @@ module Uricp::Strategy
|
|
107
109
|
options['source-format'] && !lz4_source?
|
108
110
|
end
|
109
111
|
|
110
|
-
|
112
|
+
def rbd_target(uri)
|
113
|
+
uri.path[1..-1]
|
114
|
+
end
|
111
115
|
|
116
|
+
def rbd_id
|
117
|
+
'libvirt'
|
118
|
+
end
|
119
|
+
end
|
112
120
|
end
|
@@ -1,17 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalConvert
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
if conversion_required? && file_source? && supported_source?
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
"qemu-img does not support required conversion"
|
14
|
-
end
|
7
|
+
return proposal if new_qemu? || supported_conversion?
|
8
|
+
|
9
|
+
raise Uricp::UnsupportedConversion,
|
10
|
+
'qemu-img does not support required conversion'
|
15
11
|
end
|
16
12
|
debug "#{self.class.name}: not appropriate"
|
17
13
|
false
|
@@ -24,11 +20,11 @@ module Uricp::Strategy
|
|
24
20
|
def proposal
|
25
21
|
@proposed_options = options.dup
|
26
22
|
if to.scheme == 'file'
|
27
|
-
|
23
|
+
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
28
24
|
else
|
29
25
|
@proposed_options['from_uri'] = temp_uri
|
30
|
-
|
31
|
-
|
26
|
+
@proposed_options['clean'] ||= []
|
27
|
+
@proposed_options['clean'] << proposed_path
|
32
28
|
end
|
33
29
|
@proposed_options.delete('source-format')
|
34
30
|
@proposed_options.delete('target-format')
|
@@ -36,11 +32,11 @@ module Uricp::Strategy
|
|
36
32
|
end
|
37
33
|
|
38
34
|
def new_qemu?
|
39
|
-
@new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~
|
35
|
+
@new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
|
40
36
|
end
|
41
37
|
|
42
38
|
def supported_conversion?
|
43
|
-
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
|
39
|
+
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
|
44
40
|
end
|
45
41
|
|
46
42
|
def qemu_img_command(target)
|
@@ -59,14 +55,11 @@ module Uricp::Strategy
|
|
59
55
|
end
|
60
56
|
|
61
57
|
def compat_option
|
62
|
-
|
58
|
+
'-o compat=0.10' if new_qemu?
|
63
59
|
end
|
64
60
|
|
65
61
|
def compress_option
|
66
62
|
options['compress'] && '-c'
|
67
63
|
end
|
68
|
-
|
69
64
|
end
|
70
|
-
|
71
65
|
end
|
72
|
-
|
@@ -1,11 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalLink
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
return proposal if
|
6
|
+
return proposal if !compression_required? &&
|
7
|
+
all_local_files? &&
|
8
|
+
!format_change? &&
|
9
|
+
linkable?
|
10
|
+
|
9
11
|
debug "#{self.class.name}: not appropriate"
|
10
12
|
false
|
11
13
|
end
|
@@ -21,11 +23,11 @@ module Uricp::Strategy
|
|
21
23
|
end
|
22
24
|
|
23
25
|
def linkable?
|
24
|
-
|
25
|
-
|
26
|
+
from_path_dir = File.dirname(from.path)
|
27
|
+
to_path_dir = File.dirname(to.path)
|
28
|
+
File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
|
29
|
+
File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
|
30
|
+
(!File.exist?(to.path) || File.file?(to.path))
|
26
31
|
end
|
27
|
-
|
28
32
|
end
|
29
|
-
|
30
33
|
end
|
31
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCache
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
9
|
if cache_root
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache!
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no cacheing requested"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -32,9 +30,5 @@ module Uricp::Strategy
|
|
32
30
|
@proposed_options.delete('cache_name')
|
33
31
|
self
|
34
32
|
end
|
35
|
-
|
36
33
|
end
|
37
|
-
|
38
34
|
end
|
39
|
-
|
40
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCacheConvert
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
|
-
if conversion_required? && supported_source?
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
if conversion_required? && supported_source?
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache! if cache_root
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no non-stream conversion detected"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -28,18 +26,16 @@ module Uricp::Strategy
|
|
28
26
|
def proposal
|
29
27
|
@proposed_options = options.dup
|
30
28
|
if cache_root
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
@proposed_options['from_uri'] = temp_cache_uri
|
30
|
+
@proposed_options['sweep'] = [temp_cache_file, cache_file]
|
31
|
+
@proposed_options.delete('cache')
|
32
|
+
@proposed_options.delete('cache_name')
|
35
33
|
else
|
36
34
|
@proposed_options['from_uri'] = temp_uri
|
37
|
-
|
38
|
-
|
35
|
+
@proposed_options['clean'] ||= []
|
36
|
+
@proposed_options['clean'] << proposed_path
|
39
37
|
end
|
40
38
|
self
|
41
39
|
end
|
42
|
-
|
43
40
|
end
|
44
|
-
|
45
41
|
end
|
@@ -1,30 +1,25 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedCompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
|
-
|
8
5
|
def appropriate?
|
9
6
|
case from.scheme
|
10
7
|
when 'pipe'
|
11
|
-
|
8
|
+
return proposal if compression_required?
|
12
9
|
end
|
13
10
|
debug "#{self.class.name}: not appropriate"
|
14
11
|
false
|
15
12
|
end
|
16
13
|
|
17
14
|
def command
|
18
|
-
|
15
|
+
'lz4 |'
|
19
16
|
end
|
20
17
|
|
21
18
|
def proposal
|
22
19
|
@proposed_options = options.dup
|
23
20
|
@proposed_options.delete('compress')
|
24
|
-
@proposed_options['encoding']='lz4'
|
21
|
+
@proposed_options['encoding'] = 'lz4'
|
25
22
|
self
|
26
23
|
end
|
27
|
-
|
28
24
|
end
|
29
|
-
|
30
25
|
end
|
@@ -1,35 +1,31 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedDecompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
case from.scheme
|
9
7
|
when 'pipe'
|
10
|
-
|
8
|
+
return proposal if lz4_source?
|
11
9
|
end
|
12
10
|
debug "#{self.class.name}: not appropriate"
|
13
11
|
false
|
14
12
|
end
|
15
13
|
|
16
14
|
def command
|
17
|
-
|
15
|
+
'lz4 -d |'
|
18
16
|
end
|
19
17
|
|
20
18
|
def proposal
|
21
19
|
@proposed_options = options.dup
|
22
20
|
@proposed_options.delete('source-format')
|
23
21
|
if @proposed_options['target-format']
|
24
|
-
|
25
|
-
if @proposed_options['source-format'] == @proposed_options['target-format']
|
26
|
-
|
27
|
-
|
28
|
-
|
22
|
+
@proposed_options['source-format'] = :raw
|
23
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
24
|
+
@proposed_options.delete('source-format')
|
25
|
+
@proposed_options.delete('target-format')
|
26
|
+
end
|
29
27
|
end
|
30
28
|
self
|
31
29
|
end
|
32
|
-
|
33
30
|
end
|
34
|
-
|
35
31
|
end
|
@@ -1,19 +1,15 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedLocalDecompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
case from.scheme
|
9
7
|
when 'pipe'
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
16
|
-
end
|
8
|
+
if raw_target? && lz4_source? && to.scheme == 'file'
|
9
|
+
return proposal if always_write_sparse?
|
10
|
+
|
11
|
+
debug "#{self.class.name}: using safe sparse expansion via stream"
|
12
|
+
end
|
17
13
|
end
|
18
14
|
debug "#{self.class.name}: not appropriate"
|
19
15
|
false
|
@@ -27,16 +23,14 @@ module Uricp::Strategy
|
|
27
23
|
@proposed_options = options.dup
|
28
24
|
@proposed_options.delete('source-format')
|
29
25
|
if @proposed_options['target-format']
|
30
|
-
|
31
|
-
if @proposed_options['source-format'] == @proposed_options['target-format']
|
32
|
-
|
33
|
-
|
34
|
-
|
26
|
+
@proposed_options['source-format'] = :raw
|
27
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
28
|
+
@proposed_options.delete('source-format')
|
29
|
+
@proposed_options.delete('target-format')
|
30
|
+
end
|
35
31
|
end
|
36
32
|
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
37
33
|
self
|
38
34
|
end
|
39
|
-
|
40
35
|
end
|
41
|
-
|
42
36
|
end
|