uricp 0.0.15 → 0.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +16 -21
- data/Jenkinsfile +90 -33
- data/Rakefile +24 -36
- data/almalinux8/Dockerfile +26 -0
- data/bin/uricp +25 -11
- data/bionic/Dockerfile +4 -3
- data/centos7/Dockerfile +17 -12
- data/cucumber.yml +1 -0
- data/features/check_uri_path.feature +14 -0
- data/features/rbd_access.feature +226 -0
- data/features/step_definitions/orbit_steps.rb +11 -4
- data/features/step_definitions/rbd_steps.rb +8 -0
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +28 -0
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +30 -12
- data/lib/uricp/strategy/cached_get.rb +19 -22
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +76 -18
- data/lib/uricp/strategy/local_convert.rb +10 -17
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +11 -16
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +30 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
- data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
- data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
- data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
- data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
- data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
- data/lib/uricp/strategy/rbd_flattener.rb +23 -0
- data/lib/uricp/strategy/rbd_put.rb +38 -0
- data/lib/uricp/strategy/rbd_snap.rb +56 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +23 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +22 -13
- data/lib/uricp/version.rb +4 -2
- data/lib/uricp.rb +27 -19
- data/uricp.gemspec +3 -3
- data/xenial/Dockerfile +4 -3
- metadata +41 -25
- data/spec/something_spec.rb +0 -5
- data/trusty/Dockerfile +0 -20
@@ -1,17 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalConvert
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
if conversion_required? && file_source? && supported_source?
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
"qemu-img does not support required conversion"
|
14
|
-
end
|
7
|
+
return proposal if new_qemu? || supported_conversion?
|
8
|
+
|
9
|
+
raise Uricp::UnsupportedConversion,
|
10
|
+
'qemu-img does not support required conversion'
|
15
11
|
end
|
16
12
|
debug "#{self.class.name}: not appropriate"
|
17
13
|
false
|
@@ -24,11 +20,11 @@ module Uricp::Strategy
|
|
24
20
|
def proposal
|
25
21
|
@proposed_options = options.dup
|
26
22
|
if to.scheme == 'file'
|
27
|
-
|
23
|
+
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
28
24
|
else
|
29
25
|
@proposed_options['from_uri'] = temp_uri
|
30
|
-
|
31
|
-
|
26
|
+
@proposed_options['clean'] ||= []
|
27
|
+
@proposed_options['clean'] << proposed_path
|
32
28
|
end
|
33
29
|
@proposed_options.delete('source-format')
|
34
30
|
@proposed_options.delete('target-format')
|
@@ -36,11 +32,11 @@ module Uricp::Strategy
|
|
36
32
|
end
|
37
33
|
|
38
34
|
def new_qemu?
|
39
|
-
@new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~
|
35
|
+
@new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
|
40
36
|
end
|
41
37
|
|
42
38
|
def supported_conversion?
|
43
|
-
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
|
39
|
+
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
|
44
40
|
end
|
45
41
|
|
46
42
|
def qemu_img_command(target)
|
@@ -59,14 +55,11 @@ module Uricp::Strategy
|
|
59
55
|
end
|
60
56
|
|
61
57
|
def compat_option
|
62
|
-
|
58
|
+
'-o compat=0.10' if new_qemu?
|
63
59
|
end
|
64
60
|
|
65
61
|
def compress_option
|
66
62
|
options['compress'] && '-c'
|
67
63
|
end
|
68
|
-
|
69
64
|
end
|
70
|
-
|
71
65
|
end
|
72
|
-
|
@@ -1,11 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalLink
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
return proposal if
|
6
|
+
return proposal if !compression_required? &&
|
7
|
+
all_local_files? &&
|
8
|
+
!format_change? &&
|
9
|
+
linkable?
|
10
|
+
|
9
11
|
debug "#{self.class.name}: not appropriate"
|
10
12
|
false
|
11
13
|
end
|
@@ -21,11 +23,11 @@ module Uricp::Strategy
|
|
21
23
|
end
|
22
24
|
|
23
25
|
def linkable?
|
24
|
-
|
25
|
-
|
26
|
+
from_path_dir = File.dirname(from.path)
|
27
|
+
to_path_dir = File.dirname(to.path)
|
28
|
+
File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
|
29
|
+
File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
|
30
|
+
(!File.exist?(to.path) || File.file?(to.path))
|
26
31
|
end
|
27
|
-
|
28
32
|
end
|
29
|
-
|
30
33
|
end
|
31
|
-
|
@@ -1,24 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCache
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
debug "#{self.class.name}: no cacheing requested"
|
9
|
+
with_active_cache do
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache!
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
16
|
+
false
|
20
17
|
end
|
21
|
-
false
|
22
18
|
end
|
23
19
|
|
24
20
|
def command
|
@@ -28,13 +24,12 @@ module Uricp::Strategy
|
|
28
24
|
def proposal
|
29
25
|
@proposed_options = options.dup
|
30
26
|
@proposed_options['sweep'] = [temp_cache_file, cache_file]
|
27
|
+
if to.scheme == 'rbd'
|
28
|
+
@proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
|
29
|
+
end
|
31
30
|
@proposed_options.delete('cache')
|
32
31
|
@proposed_options.delete('cache_name')
|
33
32
|
self
|
34
33
|
end
|
35
|
-
|
36
34
|
end
|
37
|
-
|
38
35
|
end
|
39
|
-
|
40
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCacheConvert
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
|
-
if conversion_required? && supported_source?
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
if conversion_required? && supported_source?
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache! if cache_root
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no non-stream conversion detected"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -28,18 +26,16 @@ module Uricp::Strategy
|
|
28
26
|
def proposal
|
29
27
|
@proposed_options = options.dup
|
30
28
|
if cache_root
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
@proposed_options['from_uri'] = temp_cache_uri
|
30
|
+
@proposed_options['sweep'] = [temp_cache_file, cache_file]
|
31
|
+
@proposed_options.delete('cache')
|
32
|
+
@proposed_options.delete('cache_name')
|
35
33
|
else
|
36
34
|
@proposed_options['from_uri'] = temp_uri
|
37
|
-
|
38
|
-
|
35
|
+
@proposed_options['clean'] ||= []
|
36
|
+
@proposed_options['clean'] << proposed_path
|
39
37
|
end
|
40
38
|
self
|
41
39
|
end
|
42
|
-
|
43
40
|
end
|
44
|
-
|
45
41
|
end
|
@@ -1,30 +1,25 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedCompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
|
-
|
8
5
|
def appropriate?
|
9
6
|
case from.scheme
|
10
7
|
when 'pipe'
|
11
|
-
|
8
|
+
return proposal if compression_required?
|
12
9
|
end
|
13
10
|
debug "#{self.class.name}: not appropriate"
|
14
11
|
false
|
15
12
|
end
|
16
13
|
|
17
14
|
def command
|
18
|
-
|
15
|
+
'lz4 |'
|
19
16
|
end
|
20
17
|
|
21
18
|
def proposal
|
22
19
|
@proposed_options = options.dup
|
23
20
|
@proposed_options.delete('compress')
|
24
|
-
@proposed_options['encoding']='lz4'
|
21
|
+
@proposed_options['encoding'] = 'lz4'
|
25
22
|
self
|
26
23
|
end
|
27
|
-
|
28
24
|
end
|
29
|
-
|
30
25
|
end
|
@@ -1,35 +1,31 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedDecompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
case from.scheme
|
9
7
|
when 'pipe'
|
10
|
-
|
8
|
+
return proposal if lz4_source?
|
11
9
|
end
|
12
10
|
debug "#{self.class.name}: not appropriate"
|
13
11
|
false
|
14
12
|
end
|
15
13
|
|
16
14
|
def command
|
17
|
-
|
15
|
+
'lz4 -d |'
|
18
16
|
end
|
19
17
|
|
20
18
|
def proposal
|
21
19
|
@proposed_options = options.dup
|
22
20
|
@proposed_options.delete('source-format')
|
23
21
|
if @proposed_options['target-format']
|
24
|
-
|
25
|
-
if @proposed_options['source-format'] == @proposed_options['target-format']
|
26
|
-
|
27
|
-
|
28
|
-
|
22
|
+
@proposed_options['source-format'] = :raw
|
23
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
24
|
+
@proposed_options.delete('source-format')
|
25
|
+
@proposed_options.delete('target-format')
|
26
|
+
end
|
29
27
|
end
|
30
28
|
self
|
31
29
|
end
|
32
|
-
|
33
30
|
end
|
34
|
-
|
35
31
|
end
|
@@ -1,19 +1,15 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedLocalDecompress
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
case from.scheme
|
9
7
|
when 'pipe'
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
16
|
-
end
|
8
|
+
if raw_target? && lz4_source? && to.scheme == 'file'
|
9
|
+
return proposal if always_write_sparse?
|
10
|
+
|
11
|
+
debug "#{self.class.name}: using safe sparse expansion via stream"
|
12
|
+
end
|
17
13
|
end
|
18
14
|
debug "#{self.class.name}: not appropriate"
|
19
15
|
false
|
@@ -27,16 +23,14 @@ module Uricp::Strategy
|
|
27
23
|
@proposed_options = options.dup
|
28
24
|
@proposed_options.delete('source-format')
|
29
25
|
if @proposed_options['target-format']
|
30
|
-
|
31
|
-
if @proposed_options['source-format'] == @proposed_options['target-format']
|
32
|
-
|
33
|
-
|
34
|
-
|
26
|
+
@proposed_options['source-format'] = :raw
|
27
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
28
|
+
@proposed_options.delete('source-format')
|
29
|
+
@proposed_options.delete('target-format')
|
30
|
+
end
|
35
31
|
end
|
36
32
|
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
37
33
|
self
|
38
34
|
end
|
39
|
-
|
40
35
|
end
|
41
|
-
|
42
36
|
end
|
@@ -1,14 +1,11 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class PipedLocalPut
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
end
|
6
|
+
return proposal if to.scheme == 'file' &&
|
7
|
+
from.scheme == 'pipe'
|
8
|
+
|
12
9
|
debug "#{self.class.name}: not appropriate"
|
13
10
|
false
|
14
11
|
end
|
@@ -22,8 +19,5 @@ module Uricp::Strategy
|
|
22
19
|
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
23
20
|
self
|
24
21
|
end
|
25
|
-
|
26
22
|
end
|
27
|
-
|
28
23
|
end
|
29
|
-
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'json'
|
2
|
+
module Uricp::Strategy
|
3
|
+
class PipedRbdGet
|
4
|
+
include Uricp::Strategy::Common
|
5
|
+
include Uricp::Strategy::CacheCommon
|
6
|
+
include Methadone::SH
|
7
|
+
|
8
|
+
def appropriate?
|
9
|
+
without_active_cache do
|
10
|
+
if from.scheme == 'rbd' &&
|
11
|
+
rbd_snapshot_spec?(from) &&
|
12
|
+
to.scheme != 'rbd'
|
13
|
+
return proposal unless sequence_complete?
|
14
|
+
end
|
15
|
+
end
|
16
|
+
debug "#{self.class.name}: not appropriate"
|
17
|
+
false
|
18
|
+
end
|
19
|
+
|
20
|
+
def command
|
21
|
+
"rbd export --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' - |"
|
22
|
+
end
|
23
|
+
|
24
|
+
def proposal
|
25
|
+
@proposed_options = options.dup
|
26
|
+
@proposed_options['from_uri'] = PIPE_URI
|
27
|
+
self
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -1,8 +1,6 @@
|
|
1
1
|
require 'open-uri'
|
2
2
|
module Uricp::Strategy
|
3
|
-
|
4
3
|
class PipedRemoteGet
|
5
|
-
|
6
4
|
include Uricp::Strategy::Common
|
7
5
|
|
8
6
|
def appropriate?
|
@@ -10,33 +8,35 @@ module Uricp::Strategy
|
|
10
8
|
when 'http', 'https'
|
11
9
|
return proposal unless sequence_complete?
|
12
10
|
else
|
13
|
-
|
14
|
-
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
false
|
15
13
|
end
|
16
14
|
end
|
17
15
|
|
18
|
-
alias
|
16
|
+
alias command curl_download_to_pipe
|
19
17
|
|
20
18
|
def proposal
|
21
19
|
@proposed_options = options.dup
|
22
20
|
@proposed_options['from_uri'] = PIPE_URI
|
23
21
|
if conversion_required?
|
24
22
|
@proposed_options['source-format'] = format_peek
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
23
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
24
|
+
@proposed_options.delete('source-format')
|
25
|
+
@proposed_options.delete('target-format')
|
26
|
+
end
|
29
27
|
end
|
30
28
|
if options['max-cache'] &&
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
size_peek.to_i > options['max-cache'].to_i
|
30
|
+
@proposed_options.delete('cache')
|
31
|
+
@proposed_options.delete('cache_name')
|
32
|
+
@proposed_options.delete('max-cache')
|
35
33
|
end
|
36
34
|
self
|
37
35
|
end
|
38
36
|
|
39
37
|
def format_peek
|
38
|
+
return options['target-format'] || 'raw' if dry_run?
|
39
|
+
|
40
40
|
options['from_uri'].open(headers) do |u|
|
41
41
|
encoding(u)
|
42
42
|
end
|
@@ -48,14 +48,16 @@ module Uricp::Strategy
|
|
48
48
|
raise
|
49
49
|
end
|
50
50
|
rescue SocketError => e
|
51
|
-
raise SocketError, options['from_uri'].to_s+
|
51
|
+
raise SocketError, options['from_uri'].to_s + ' inaccessible: ' + e.message
|
52
52
|
end
|
53
53
|
|
54
54
|
def size_peek
|
55
|
-
|
55
|
+
return options['max-cache'] if dry_run?
|
56
|
+
|
57
|
+
size_headers = headers
|
56
58
|
size_headers['Range'] = 'bytes=0-0'
|
57
59
|
options['from_uri'].open(headers) do |u|
|
58
|
-
|
60
|
+
match = %r{bytes\s+(\d+)-(\d+)/(\d+|\*)}i.match(u.meta['content-range'])
|
59
61
|
match && match[3].to_i
|
60
62
|
end
|
61
63
|
rescue OpenURI::HTTPError => e
|
@@ -66,15 +68,13 @@ module Uricp::Strategy
|
|
66
68
|
raise
|
67
69
|
end
|
68
70
|
rescue SocketError => e
|
69
|
-
raise SocketError, options['from_uri'].to_s+
|
71
|
+
raise SocketError, options['from_uri'].to_s + ' inaccessible: ' + e.message
|
70
72
|
end
|
71
73
|
|
72
74
|
def headers
|
73
|
-
headers={'Range' => 'bytes=0-7'}
|
75
|
+
headers = { 'Range' => 'bytes=0-7' }
|
74
76
|
headers['X-Auth-Token'] = options['authenticator'].call if http_authentication?
|
75
77
|
headers
|
76
78
|
end
|
77
|
-
|
78
79
|
end
|
79
|
-
|
80
80
|
end
|