uricp 0.0.15 → 0.0.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/Gemfile.lock +1 -8
  4. data/Jenkinsfile +27 -0
  5. data/Rakefile +19 -36
  6. data/bionic/Dockerfile +2 -2
  7. data/centos7/Dockerfile +2 -2
  8. data/features/step_definitions/orbit_steps.rb +11 -4
  9. data/lib/segment_upload.rb +20 -22
  10. data/lib/uricp.rb +3 -3
  11. data/lib/uricp/curl_primitives.rb +31 -33
  12. data/lib/uricp/orbit_auth.rb +23 -27
  13. data/lib/uricp/segmenter.rb +12 -16
  14. data/lib/uricp/strategy/cache_common.rb +11 -11
  15. data/lib/uricp/strategy/cached_get.rb +14 -20
  16. data/lib/uricp/strategy/cleaner.rb +2 -8
  17. data/lib/uricp/strategy/common.rb +25 -18
  18. data/lib/uricp/strategy/local_convert.rb +9 -16
  19. data/lib/uricp/strategy/local_link.rb +10 -8
  20. data/lib/uricp/strategy/piped_cache.rb +7 -13
  21. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  22. data/lib/uricp/strategy/piped_compress.rb +3 -8
  23. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  24. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  25. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  26. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  27. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  28. data/lib/uricp/strategy/piped_rbd_get.rb +44 -0
  29. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  30. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  31. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  32. data/lib/uricp/strategy/remote_put.rb +11 -17
  33. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  34. data/lib/uricp/strategy/sweeper.rb +2 -8
  35. data/lib/uricp/uri_strategy.rb +14 -13
  36. data/lib/uricp/version.rb +2 -2
  37. data/trusty/Dockerfile +2 -2
  38. data/uricp.gemspec +0 -1
  39. data/xenial/Dockerfile +2 -2
  40. metadata +7 -20
  41. data/spec/something_spec.rb +0 -5
@@ -1,9 +1,7 @@
1
1
  require 'pathname'
2
2
 
3
3
  module Uricp
4
-
5
4
  class Segmenter
6
-
7
5
  include Methadone::CLILogging
8
6
  include Methadone::SH
9
7
  include Uricp::CurlPrimitives
@@ -15,7 +13,7 @@ module Uricp
15
13
  @stream = File.pipe?(source) || File.chardev?(source)
16
14
  else
17
15
  @source = STDIN
18
- @stream = true
16
+ @stream = true
19
17
  end
20
18
  split_path
21
19
  end
@@ -23,9 +21,9 @@ module Uricp
23
21
  def split_path
24
22
  elements = Pathname.new(to.path).enum_for(:each_filename).to_a
25
23
  elements.shift
26
- @account=elements.shift
27
- @container=elements.shift
28
- @object_path=File.join(elements)
24
+ @account = elements.shift
25
+ @container = elements.shift
26
+ @object_path = File.join(elements)
29
27
  end
30
28
 
31
29
  def upload
@@ -33,7 +31,7 @@ module Uricp
33
31
  segmented_upload
34
32
  else
35
33
  sh! curl_upload_from(options['from']),
36
- :on_fail => "Upload to #{to} failed"
34
+ on_fail: "Upload to #{to} failed"
37
35
  end
38
36
  end
39
37
 
@@ -54,16 +52,16 @@ module Uricp
54
52
  end
55
53
 
56
54
  def object_manifest
57
- @object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
55
+ @object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/'
58
56
  end
59
57
 
60
58
  def segmented_upload
61
59
  debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
62
60
  suffix = 0
63
61
  until @source.eof?
64
- debug "#{self.class.name}: Uploading segment #{suffix}"
65
- upload_segment(suffix)
66
- suffix = suffix.next
62
+ debug "#{self.class.name}: Uploading segment #{suffix}"
63
+ upload_segment(suffix)
64
+ suffix = suffix.next
67
65
  end
68
66
  add_manifest
69
67
  end
@@ -71,18 +69,16 @@ module Uricp
71
69
  def upload_segment(segment_number)
72
70
  segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
73
71
  debug "Uploading with #{curl_upload_from('-', segment_name)}"
74
- open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
72
+ open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
75
73
  copy_length = IO.copy_stream(@source, destination, segment_size)
76
- debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
74
+ debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
77
75
  end
78
76
  end
79
77
 
80
78
  def add_manifest
81
79
  debug "Adding DLO object_manifest #{object_manifest}"
82
80
  sh! curl_manifest(object_manifest),
83
- :on_fail => "Upload to #{to} failed"
81
+ on_fail: "Upload to #{to} failed"
84
82
  end
85
-
86
83
  end
87
-
88
84
  end
@@ -1,16 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  module CacheCommon
6
-
7
5
  def validate_cache!
6
+ return if dry_run?
7
+
8
+ unless cache_exists?
9
+ raise Uricp::MissingCache,
10
+ "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory"
11
+ end
12
+ return if cache_name
13
+
8
14
  raise Uricp::MissingCache,
9
- "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory" unless cache_exists?
10
- raise Uricp::MissingCache,
11
- "no cache name found" unless cache_name
15
+ 'no cache name found'
12
16
  end
13
-
17
+
14
18
  def in_cache?
15
19
  File.readable? cache_file
16
20
  end
@@ -24,7 +28,7 @@ module Uricp::Strategy
24
28
  end
25
29
 
26
30
  def cache_exists?
27
- cache_root && %w{temp cache}.all? do |d|
31
+ cache_root && %w[temp cache].all? do |d|
28
32
  File.directory?(File.join(cache_root, d))
29
33
  end
30
34
  end
@@ -40,9 +44,5 @@ module Uricp::Strategy
40
44
  def cache_file
41
45
  @cache_file ||= File.join(cache_root, 'cache', cache_name)
42
46
  end
43
-
44
47
  end
45
-
46
48
  end
47
-
48
-
@@ -1,48 +1,42 @@
1
1
  require 'uri'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class CachedGet
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
10
  validate_cache!
13
- if in_cache? || file_source?
14
- return proposal
15
- else
16
- debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
17
- end
11
+ return proposal if in_cache? || file_source?
12
+
13
+ debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
18
14
  else
19
- debug "#{self.class.name}: not appropriate"
15
+ debug "#{self.class.name}: not appropriate"
20
16
  end
21
17
  false
22
18
  end
23
19
 
24
20
  def command
25
- ":;"
21
+ ':;'
26
22
  end
27
23
 
28
24
  def proposal
29
25
  @proposed_options = options.dup
30
- unless file_source?
31
- @proposed_options['from_uri'] = URI.join('file:///', cache_file)
32
- end
26
+ @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
33
27
  @proposed_options.delete('cache')
34
28
  @proposed_options.delete('cache_name')
35
29
  if conversion_required?
36
- @proposed_options['source-format'] =
37
- File.open(@proposed_options['from_uri'].path) {|f| encoding(f)}
38
- if @proposed_options['source-format'] == @proposed_options['target-format']
39
- @proposed_options.delete('source-format')
40
- @proposed_options.delete('target-format')
41
- end
30
+ unless dry_run?
31
+ @proposed_options['source-format'] =
32
+ File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
33
+ end
34
+ if @proposed_options['source-format'] == @proposed_options['target-format']
35
+ @proposed_options.delete('source-format')
36
+ @proposed_options.delete('target-format')
37
+ end
42
38
  end
43
39
  self
44
40
  end
45
-
46
41
  end
47
-
48
42
  end
@@ -1,13 +1,10 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class Cleaner
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- if options['clean'] && sequence_complete?
9
- return proposal
10
- end
6
+ return proposal if options['clean'] && sequence_complete?
7
+
11
8
  debug "#{self.class.name}: not appropriate"
12
9
  false
13
10
  end
@@ -21,8 +18,5 @@ module Uricp::Strategy
21
18
  @proposed_options.delete('clean')
22
19
  self
23
20
  end
24
-
25
21
  end
26
-
27
22
  end
28
-
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  module Common
4
-
5
3
  include Methadone::CLILogging
6
4
  include Uricp::CurlPrimitives
7
5
 
@@ -14,10 +12,10 @@ module Uricp::Strategy
14
12
 
15
13
  def unsupported_transfer
16
14
  raise Uricp::UnsupportedURLtype,
17
- "Unsupported transfer from #{from.to_s} to #{to.to_s}"
15
+ "Unsupported transfer from #{from} to #{to}"
18
16
  end
19
17
 
20
- alias :command :unsupported_transfer
18
+ alias command unsupported_transfer
21
19
 
22
20
  def all_local_files?
23
21
  !sequence_complete? && file_source? && to.scheme == 'file'
@@ -44,24 +42,23 @@ module Uricp::Strategy
44
42
  end
45
43
 
46
44
  def qcow2?(magic)
47
- magic.unpack('a3C') == ['QFI',0xfb]
45
+ magic.unpack('a3C') == ['QFI', 0xfb]
48
46
  end
49
47
 
50
48
  def encoding(io)
51
49
  magic = io.read(4).to_s
52
- case
53
- when lz4?(magic)
50
+ if lz4?(magic)
54
51
  :lz4
55
- when qcow2?(magic)
52
+ elsif qcow2?(magic)
56
53
  version = io.read(4)
57
- case version.unpack('N')
58
- when [2]
59
- :qcow2
60
- when [3]
61
- :qcow3
62
- else
63
- :qcow2un
64
- end
54
+ case version.unpack('N')
55
+ when [2]
56
+ :qcow2
57
+ when [3]
58
+ :qcow3
59
+ else
60
+ :qcow2un
61
+ end
65
62
  else
66
63
  :raw
67
64
  end
@@ -73,6 +70,10 @@ module Uricp::Strategy
73
70
  )
74
71
  end
75
72
 
73
+ def dry_run?
74
+ options['dry-run']
75
+ end
76
+
76
77
  def lz4_source?
77
78
  options['source-format'] == :lz4
78
79
  end
@@ -92,7 +93,7 @@ module Uricp::Strategy
92
93
  PIPE_URI = URI('pipe:/')
93
94
 
94
95
  def get_temp_filename(base_dir)
95
- t = Time.now.strftime("%Y%m%d")
96
+ t = Time.now.strftime('%Y%m%d')
96
97
  File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
97
98
  end
98
99
 
@@ -108,6 +109,12 @@ module Uricp::Strategy
108
109
  options['source-format'] && !lz4_source?
109
110
  end
110
111
 
111
- end
112
+ def rbd_target(uri)
113
+ uri.path[1..-1]
114
+ end
112
115
 
116
+ def rbd_id
117
+ 'libvirt'
118
+ end
119
+ end
113
120
  end
@@ -1,17 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalConvert
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if conversion_required? && file_source? && supported_source?
9
- if new_qemu? || supported_conversion?
10
- return proposal
11
- else
12
- raise Uricp::UnsupportedConversion,
13
- "qemu-img does not support required conversion"
14
- end
7
+ return proposal if new_qemu? || supported_conversion?
8
+
9
+ raise Uricp::UnsupportedConversion,
10
+ 'qemu-img does not support required conversion'
15
11
  end
16
12
  debug "#{self.class.name}: not appropriate"
17
13
  false
@@ -24,11 +20,11 @@ module Uricp::Strategy
24
20
  def proposal
25
21
  @proposed_options = options.dup
26
22
  if to.scheme == 'file'
27
- @proposed_options['from_uri'] = @proposed_options['to_uri']
23
+ @proposed_options['from_uri'] = @proposed_options['to_uri']
28
24
  else
29
25
  @proposed_options['from_uri'] = temp_uri
30
- @proposed_options['clean'] ||= []
31
- @proposed_options['clean'] << proposed_path
26
+ @proposed_options['clean'] ||= []
27
+ @proposed_options['clean'] << proposed_path
32
28
  end
33
29
  @proposed_options.delete('source-format')
34
30
  @proposed_options.delete('target-format')
@@ -40,7 +36,7 @@ module Uricp::Strategy
40
36
  end
41
37
 
42
38
  def supported_conversion?
43
- ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
39
+ ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
44
40
  end
45
41
 
46
42
  def qemu_img_command(target)
@@ -59,14 +55,11 @@ module Uricp::Strategy
59
55
  end
60
56
 
61
57
  def compat_option
62
- "-o compat=0.10" if new_qemu?
58
+ '-o compat=0.10' if new_qemu?
63
59
  end
64
60
 
65
61
  def compress_option
66
62
  options['compress'] && '-c'
67
63
  end
68
-
69
64
  end
70
-
71
65
  end
72
-
@@ -1,11 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalLink
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- return proposal if all_local_files? && !format_change? && linkable?
6
+ return proposal if !compression_required? &&
7
+ all_local_files? &&
8
+ !format_change? &&
9
+ linkable?
10
+
9
11
  debug "#{self.class.name}: not appropriate"
10
12
  false
11
13
  end
@@ -21,11 +23,11 @@ module Uricp::Strategy
21
23
  end
22
24
 
23
25
  def linkable?
24
- File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
25
- (!File.exist?(to.path) || File.file?(to.path))
26
+ from_path_dir = File.dirname(from.path)
27
+ to_path_dir = File.dirname(to.path)
28
+ File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
29
+ File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
30
+ (!File.exist?(to.path) || File.file?(to.path))
26
31
  end
27
-
28
32
  end
29
-
30
33
  end
31
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCache
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache!
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache!
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no cacheing requested"
17
+ debug "#{self.class.name}: no cacheing requested"
20
18
  end
21
19
  false
22
20
  end
@@ -32,9 +30,5 @@ module Uricp::Strategy
32
30
  @proposed_options.delete('cache_name')
33
31
  self
34
32
  end
35
-
36
33
  end
37
-
38
34
  end
39
-
40
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCacheConvert
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if conversion_required? && supported_source?
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache! if cache_root
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
9
+ if conversion_required? && supported_source?
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache! if cache_root
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no non-stream conversion detected"
17
+ debug "#{self.class.name}: no non-stream conversion detected"
20
18
  end
21
19
  false
22
20
  end
@@ -28,18 +26,16 @@ module Uricp::Strategy
28
26
  def proposal
29
27
  @proposed_options = options.dup
30
28
  if cache_root
31
- @proposed_options['from_uri'] = temp_cache_uri
32
- @proposed_options['sweep'] = [temp_cache_file, cache_file]
33
- @proposed_options.delete('cache')
34
- @proposed_options.delete('cache_name')
29
+ @proposed_options['from_uri'] = temp_cache_uri
30
+ @proposed_options['sweep'] = [temp_cache_file, cache_file]
31
+ @proposed_options.delete('cache')
32
+ @proposed_options.delete('cache_name')
35
33
  else
36
34
  @proposed_options['from_uri'] = temp_uri
37
- @proposed_options['clean'] ||= []
38
- @proposed_options['clean'] << proposed_path
35
+ @proposed_options['clean'] ||= []
36
+ @proposed_options['clean'] << proposed_path
39
37
  end
40
38
  self
41
39
  end
42
-
43
40
  end
44
-
45
41
  end