uricp 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +1 -8
- data/Jenkinsfile +27 -0
- data/Rakefile +19 -36
- data/bionic/Dockerfile +2 -2
- data/centos7/Dockerfile +2 -2
- data/features/step_definitions/orbit_steps.rb +11 -4
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp.rb +3 -3
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +11 -11
- data/lib/uricp/strategy/cached_get.rb +14 -20
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +25 -18
- data/lib/uricp/strategy/local_convert.rb +9 -16
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +7 -13
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +44 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +14 -13
- data/lib/uricp/version.rb +2 -2
- data/trusty/Dockerfile +2 -2
- data/uricp.gemspec +0 -1
- data/xenial/Dockerfile +2 -2
- metadata +7 -20
- data/spec/something_spec.rb +0 -5
data/lib/uricp/segmenter.rb
CHANGED
@@ -1,9 +1,7 @@
|
|
1
1
|
require 'pathname'
|
2
2
|
|
3
3
|
module Uricp
|
4
|
-
|
5
4
|
class Segmenter
|
6
|
-
|
7
5
|
include Methadone::CLILogging
|
8
6
|
include Methadone::SH
|
9
7
|
include Uricp::CurlPrimitives
|
@@ -15,7 +13,7 @@ module Uricp
|
|
15
13
|
@stream = File.pipe?(source) || File.chardev?(source)
|
16
14
|
else
|
17
15
|
@source = STDIN
|
18
|
-
|
16
|
+
@stream = true
|
19
17
|
end
|
20
18
|
split_path
|
21
19
|
end
|
@@ -23,9 +21,9 @@ module Uricp
|
|
23
21
|
def split_path
|
24
22
|
elements = Pathname.new(to.path).enum_for(:each_filename).to_a
|
25
23
|
elements.shift
|
26
|
-
@account=elements.shift
|
27
|
-
@container=elements.shift
|
28
|
-
@object_path=File.join(elements)
|
24
|
+
@account = elements.shift
|
25
|
+
@container = elements.shift
|
26
|
+
@object_path = File.join(elements)
|
29
27
|
end
|
30
28
|
|
31
29
|
def upload
|
@@ -33,7 +31,7 @@ module Uricp
|
|
33
31
|
segmented_upload
|
34
32
|
else
|
35
33
|
sh! curl_upload_from(options['from']),
|
36
|
-
|
34
|
+
on_fail: "Upload to #{to} failed"
|
37
35
|
end
|
38
36
|
end
|
39
37
|
|
@@ -54,16 +52,16 @@ module Uricp
|
|
54
52
|
end
|
55
53
|
|
56
54
|
def object_manifest
|
57
|
-
@object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
|
55
|
+
@object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/'
|
58
56
|
end
|
59
57
|
|
60
58
|
def segmented_upload
|
61
59
|
debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
|
62
60
|
suffix = 0
|
63
61
|
until @source.eof?
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
debug "#{self.class.name}: Uploading segment #{suffix}"
|
63
|
+
upload_segment(suffix)
|
64
|
+
suffix = suffix.next
|
67
65
|
end
|
68
66
|
add_manifest
|
69
67
|
end
|
@@ -71,18 +69,16 @@ module Uricp
|
|
71
69
|
def upload_segment(segment_number)
|
72
70
|
segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
|
73
71
|
debug "Uploading with #{curl_upload_from('-', segment_name)}"
|
74
|
-
open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
|
72
|
+
open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
|
75
73
|
copy_length = IO.copy_stream(@source, destination, segment_size)
|
76
|
-
|
74
|
+
debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
|
77
75
|
end
|
78
76
|
end
|
79
77
|
|
80
78
|
def add_manifest
|
81
79
|
debug "Adding DLO object_manifest #{object_manifest}"
|
82
80
|
sh! curl_manifest(object_manifest),
|
83
|
-
|
81
|
+
on_fail: "Upload to #{to} failed"
|
84
82
|
end
|
85
|
-
|
86
83
|
end
|
87
|
-
|
88
84
|
end
|
@@ -1,16 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
module CacheCommon
|
6
|
-
|
7
5
|
def validate_cache!
|
6
|
+
return if dry_run?
|
7
|
+
|
8
|
+
unless cache_exists?
|
9
|
+
raise Uricp::MissingCache,
|
10
|
+
"no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory"
|
11
|
+
end
|
12
|
+
return if cache_name
|
13
|
+
|
8
14
|
raise Uricp::MissingCache,
|
9
|
-
|
10
|
-
raise Uricp::MissingCache,
|
11
|
-
"no cache name found" unless cache_name
|
15
|
+
'no cache name found'
|
12
16
|
end
|
13
|
-
|
17
|
+
|
14
18
|
def in_cache?
|
15
19
|
File.readable? cache_file
|
16
20
|
end
|
@@ -24,7 +28,7 @@ module Uricp::Strategy
|
|
24
28
|
end
|
25
29
|
|
26
30
|
def cache_exists?
|
27
|
-
cache_root && %w
|
31
|
+
cache_root && %w[temp cache].all? do |d|
|
28
32
|
File.directory?(File.join(cache_root, d))
|
29
33
|
end
|
30
34
|
end
|
@@ -40,9 +44,5 @@ module Uricp::Strategy
|
|
40
44
|
def cache_file
|
41
45
|
@cache_file ||= File.join(cache_root, 'cache', cache_name)
|
42
46
|
end
|
43
|
-
|
44
47
|
end
|
45
|
-
|
46
48
|
end
|
47
|
-
|
48
|
-
|
@@ -1,48 +1,42 @@
|
|
1
1
|
require 'uri'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class CachedGet
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
9
|
if cache_root
|
12
10
|
validate_cache!
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
|
17
|
-
end
|
11
|
+
return proposal if in_cache? || file_source?
|
12
|
+
|
13
|
+
debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
|
18
14
|
else
|
19
|
-
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
20
16
|
end
|
21
17
|
false
|
22
18
|
end
|
23
19
|
|
24
20
|
def command
|
25
|
-
|
21
|
+
':;'
|
26
22
|
end
|
27
23
|
|
28
24
|
def proposal
|
29
25
|
@proposed_options = options.dup
|
30
|
-
unless file_source?
|
31
|
-
@proposed_options['from_uri'] = URI.join('file:///', cache_file)
|
32
|
-
end
|
26
|
+
@proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
|
33
27
|
@proposed_options.delete('cache')
|
34
28
|
@proposed_options.delete('cache_name')
|
35
29
|
if conversion_required?
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
30
|
+
unless dry_run?
|
31
|
+
@proposed_options['source-format'] =
|
32
|
+
File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
|
33
|
+
end
|
34
|
+
if @proposed_options['source-format'] == @proposed_options['target-format']
|
35
|
+
@proposed_options.delete('source-format')
|
36
|
+
@proposed_options.delete('target-format')
|
37
|
+
end
|
42
38
|
end
|
43
39
|
self
|
44
40
|
end
|
45
|
-
|
46
41
|
end
|
47
|
-
|
48
42
|
end
|
@@ -1,13 +1,10 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class Cleaner
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
if options['clean'] && sequence_complete?
|
9
|
-
|
10
|
-
end
|
6
|
+
return proposal if options['clean'] && sequence_complete?
|
7
|
+
|
11
8
|
debug "#{self.class.name}: not appropriate"
|
12
9
|
false
|
13
10
|
end
|
@@ -21,8 +18,5 @@ module Uricp::Strategy
|
|
21
18
|
@proposed_options.delete('clean')
|
22
19
|
self
|
23
20
|
end
|
24
|
-
|
25
21
|
end
|
26
|
-
|
27
22
|
end
|
28
|
-
|
@@ -1,7 +1,5 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
module Common
|
4
|
-
|
5
3
|
include Methadone::CLILogging
|
6
4
|
include Uricp::CurlPrimitives
|
7
5
|
|
@@ -14,10 +12,10 @@ module Uricp::Strategy
|
|
14
12
|
|
15
13
|
def unsupported_transfer
|
16
14
|
raise Uricp::UnsupportedURLtype,
|
17
|
-
|
15
|
+
"Unsupported transfer from #{from} to #{to}"
|
18
16
|
end
|
19
17
|
|
20
|
-
alias
|
18
|
+
alias command unsupported_transfer
|
21
19
|
|
22
20
|
def all_local_files?
|
23
21
|
!sequence_complete? && file_source? && to.scheme == 'file'
|
@@ -44,24 +42,23 @@ module Uricp::Strategy
|
|
44
42
|
end
|
45
43
|
|
46
44
|
def qcow2?(magic)
|
47
|
-
magic.unpack('a3C') == ['QFI',0xfb]
|
45
|
+
magic.unpack('a3C') == ['QFI', 0xfb]
|
48
46
|
end
|
49
47
|
|
50
48
|
def encoding(io)
|
51
49
|
magic = io.read(4).to_s
|
52
|
-
|
53
|
-
when lz4?(magic)
|
50
|
+
if lz4?(magic)
|
54
51
|
:lz4
|
55
|
-
|
52
|
+
elsif qcow2?(magic)
|
56
53
|
version = io.read(4)
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
54
|
+
case version.unpack('N')
|
55
|
+
when [2]
|
56
|
+
:qcow2
|
57
|
+
when [3]
|
58
|
+
:qcow3
|
59
|
+
else
|
60
|
+
:qcow2un
|
61
|
+
end
|
65
62
|
else
|
66
63
|
:raw
|
67
64
|
end
|
@@ -73,6 +70,10 @@ module Uricp::Strategy
|
|
73
70
|
)
|
74
71
|
end
|
75
72
|
|
73
|
+
def dry_run?
|
74
|
+
options['dry-run']
|
75
|
+
end
|
76
|
+
|
76
77
|
def lz4_source?
|
77
78
|
options['source-format'] == :lz4
|
78
79
|
end
|
@@ -92,7 +93,7 @@ module Uricp::Strategy
|
|
92
93
|
PIPE_URI = URI('pipe:/')
|
93
94
|
|
94
95
|
def get_temp_filename(base_dir)
|
95
|
-
t = Time.now.strftime(
|
96
|
+
t = Time.now.strftime('%Y%m%d')
|
96
97
|
File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
|
97
98
|
end
|
98
99
|
|
@@ -108,6 +109,12 @@ module Uricp::Strategy
|
|
108
109
|
options['source-format'] && !lz4_source?
|
109
110
|
end
|
110
111
|
|
111
|
-
|
112
|
+
def rbd_target(uri)
|
113
|
+
uri.path[1..-1]
|
114
|
+
end
|
112
115
|
|
116
|
+
def rbd_id
|
117
|
+
'libvirt'
|
118
|
+
end
|
119
|
+
end
|
113
120
|
end
|
@@ -1,17 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalConvert
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
if conversion_required? && file_source? && supported_source?
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
"qemu-img does not support required conversion"
|
14
|
-
end
|
7
|
+
return proposal if new_qemu? || supported_conversion?
|
8
|
+
|
9
|
+
raise Uricp::UnsupportedConversion,
|
10
|
+
'qemu-img does not support required conversion'
|
15
11
|
end
|
16
12
|
debug "#{self.class.name}: not appropriate"
|
17
13
|
false
|
@@ -24,11 +20,11 @@ module Uricp::Strategy
|
|
24
20
|
def proposal
|
25
21
|
@proposed_options = options.dup
|
26
22
|
if to.scheme == 'file'
|
27
|
-
|
23
|
+
@proposed_options['from_uri'] = @proposed_options['to_uri']
|
28
24
|
else
|
29
25
|
@proposed_options['from_uri'] = temp_uri
|
30
|
-
|
31
|
-
|
26
|
+
@proposed_options['clean'] ||= []
|
27
|
+
@proposed_options['clean'] << proposed_path
|
32
28
|
end
|
33
29
|
@proposed_options.delete('source-format')
|
34
30
|
@proposed_options.delete('target-format')
|
@@ -40,7 +36,7 @@ module Uricp::Strategy
|
|
40
36
|
end
|
41
37
|
|
42
38
|
def supported_conversion?
|
43
|
-
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
|
39
|
+
([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
|
44
40
|
end
|
45
41
|
|
46
42
|
def qemu_img_command(target)
|
@@ -59,14 +55,11 @@ module Uricp::Strategy
|
|
59
55
|
end
|
60
56
|
|
61
57
|
def compat_option
|
62
|
-
|
58
|
+
'-o compat=0.10' if new_qemu?
|
63
59
|
end
|
64
60
|
|
65
61
|
def compress_option
|
66
62
|
options['compress'] && '-c'
|
67
63
|
end
|
68
|
-
|
69
64
|
end
|
70
|
-
|
71
65
|
end
|
72
|
-
|
@@ -1,11 +1,13 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class LocalLink
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
return proposal if
|
6
|
+
return proposal if !compression_required? &&
|
7
|
+
all_local_files? &&
|
8
|
+
!format_change? &&
|
9
|
+
linkable?
|
10
|
+
|
9
11
|
debug "#{self.class.name}: not appropriate"
|
10
12
|
false
|
11
13
|
end
|
@@ -21,11 +23,11 @@ module Uricp::Strategy
|
|
21
23
|
end
|
22
24
|
|
23
25
|
def linkable?
|
24
|
-
|
25
|
-
|
26
|
+
from_path_dir = File.dirname(from.path)
|
27
|
+
to_path_dir = File.dirname(to.path)
|
28
|
+
File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
|
29
|
+
File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
|
30
|
+
(!File.exist?(to.path) || File.file?(to.path))
|
26
31
|
end
|
27
|
-
|
28
32
|
end
|
29
|
-
|
30
33
|
end
|
31
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCache
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
9
|
if cache_root
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache!
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no cacheing requested"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -32,9 +30,5 @@ module Uricp::Strategy
|
|
32
30
|
@proposed_options.delete('cache_name')
|
33
31
|
self
|
34
32
|
end
|
35
|
-
|
36
33
|
end
|
37
|
-
|
38
34
|
end
|
39
|
-
|
40
|
-
|
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
3
|
module Uricp::Strategy
|
4
|
-
|
5
4
|
class PipedCacheConvert
|
6
|
-
|
7
5
|
include Uricp::Strategy::Common
|
8
6
|
include Uricp::Strategy::CacheCommon
|
9
7
|
|
10
8
|
def appropriate?
|
11
|
-
if conversion_required? && supported_source?
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
if conversion_required? && supported_source?
|
10
|
+
case from.scheme
|
11
|
+
when 'pipe'
|
12
|
+
validate_cache! if cache_root
|
13
|
+
return proposal
|
14
|
+
end
|
15
|
+
debug "#{self.class.name}: not appropriate"
|
18
16
|
else
|
19
|
-
|
17
|
+
debug "#{self.class.name}: no non-stream conversion detected"
|
20
18
|
end
|
21
19
|
false
|
22
20
|
end
|
@@ -28,18 +26,16 @@ module Uricp::Strategy
|
|
28
26
|
def proposal
|
29
27
|
@proposed_options = options.dup
|
30
28
|
if cache_root
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
@proposed_options['from_uri'] = temp_cache_uri
|
30
|
+
@proposed_options['sweep'] = [temp_cache_file, cache_file]
|
31
|
+
@proposed_options.delete('cache')
|
32
|
+
@proposed_options.delete('cache_name')
|
35
33
|
else
|
36
34
|
@proposed_options['from_uri'] = temp_uri
|
37
|
-
|
38
|
-
|
35
|
+
@proposed_options['clean'] ||= []
|
36
|
+
@proposed_options['clean'] << proposed_path
|
39
37
|
end
|
40
38
|
self
|
41
39
|
end
|
42
|
-
|
43
40
|
end
|
44
|
-
|
45
41
|
end
|