uricp 0.0.15 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/Gemfile.lock +1 -8
  4. data/Jenkinsfile +27 -0
  5. data/Rakefile +19 -36
  6. data/bionic/Dockerfile +2 -2
  7. data/centos7/Dockerfile +2 -2
  8. data/features/step_definitions/orbit_steps.rb +11 -4
  9. data/lib/segment_upload.rb +20 -22
  10. data/lib/uricp.rb +3 -3
  11. data/lib/uricp/curl_primitives.rb +31 -33
  12. data/lib/uricp/orbit_auth.rb +23 -27
  13. data/lib/uricp/segmenter.rb +12 -16
  14. data/lib/uricp/strategy/cache_common.rb +11 -11
  15. data/lib/uricp/strategy/cached_get.rb +14 -20
  16. data/lib/uricp/strategy/cleaner.rb +2 -8
  17. data/lib/uricp/strategy/common.rb +25 -18
  18. data/lib/uricp/strategy/local_convert.rb +9 -16
  19. data/lib/uricp/strategy/local_link.rb +10 -8
  20. data/lib/uricp/strategy/piped_cache.rb +7 -13
  21. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  22. data/lib/uricp/strategy/piped_compress.rb +3 -8
  23. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  24. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  25. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  26. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  27. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  28. data/lib/uricp/strategy/piped_rbd_get.rb +44 -0
  29. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  30. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  31. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  32. data/lib/uricp/strategy/remote_put.rb +11 -17
  33. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  34. data/lib/uricp/strategy/sweeper.rb +2 -8
  35. data/lib/uricp/uri_strategy.rb +14 -13
  36. data/lib/uricp/version.rb +2 -2
  37. data/trusty/Dockerfile +2 -2
  38. data/uricp.gemspec +0 -1
  39. data/xenial/Dockerfile +2 -2
  40. metadata +7 -20
  41. data/spec/something_spec.rb +0 -5
@@ -1,9 +1,7 @@
1
1
  require 'pathname'
2
2
 
3
3
  module Uricp
4
-
5
4
  class Segmenter
6
-
7
5
  include Methadone::CLILogging
8
6
  include Methadone::SH
9
7
  include Uricp::CurlPrimitives
@@ -15,7 +13,7 @@ module Uricp
15
13
  @stream = File.pipe?(source) || File.chardev?(source)
16
14
  else
17
15
  @source = STDIN
18
- @stream = true
16
+ @stream = true
19
17
  end
20
18
  split_path
21
19
  end
@@ -23,9 +21,9 @@ module Uricp
23
21
  def split_path
24
22
  elements = Pathname.new(to.path).enum_for(:each_filename).to_a
25
23
  elements.shift
26
- @account=elements.shift
27
- @container=elements.shift
28
- @object_path=File.join(elements)
24
+ @account = elements.shift
25
+ @container = elements.shift
26
+ @object_path = File.join(elements)
29
27
  end
30
28
 
31
29
  def upload
@@ -33,7 +31,7 @@ module Uricp
33
31
  segmented_upload
34
32
  else
35
33
  sh! curl_upload_from(options['from']),
36
- :on_fail => "Upload to #{to} failed"
34
+ on_fail: "Upload to #{to} failed"
37
35
  end
38
36
  end
39
37
 
@@ -54,16 +52,16 @@ module Uricp
54
52
  end
55
53
 
56
54
  def object_manifest
57
- @object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
55
+ @object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/'
58
56
  end
59
57
 
60
58
  def segmented_upload
61
59
  debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
62
60
  suffix = 0
63
61
  until @source.eof?
64
- debug "#{self.class.name}: Uploading segment #{suffix}"
65
- upload_segment(suffix)
66
- suffix = suffix.next
62
+ debug "#{self.class.name}: Uploading segment #{suffix}"
63
+ upload_segment(suffix)
64
+ suffix = suffix.next
67
65
  end
68
66
  add_manifest
69
67
  end
@@ -71,18 +69,16 @@ module Uricp
71
69
  def upload_segment(segment_number)
72
70
  segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
73
71
  debug "Uploading with #{curl_upload_from('-', segment_name)}"
74
- open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
72
+ open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
75
73
  copy_length = IO.copy_stream(@source, destination, segment_size)
76
- debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
74
+ debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
77
75
  end
78
76
  end
79
77
 
80
78
  def add_manifest
81
79
  debug "Adding DLO object_manifest #{object_manifest}"
82
80
  sh! curl_manifest(object_manifest),
83
- :on_fail => "Upload to #{to} failed"
81
+ on_fail: "Upload to #{to} failed"
84
82
  end
85
-
86
83
  end
87
-
88
84
  end
@@ -1,16 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  module CacheCommon
6
-
7
5
  def validate_cache!
6
+ return if dry_run?
7
+
8
+ unless cache_exists?
9
+ raise Uricp::MissingCache,
10
+ "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory"
11
+ end
12
+ return if cache_name
13
+
8
14
  raise Uricp::MissingCache,
9
- "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory" unless cache_exists?
10
- raise Uricp::MissingCache,
11
- "no cache name found" unless cache_name
15
+ 'no cache name found'
12
16
  end
13
-
17
+
14
18
  def in_cache?
15
19
  File.readable? cache_file
16
20
  end
@@ -24,7 +28,7 @@ module Uricp::Strategy
24
28
  end
25
29
 
26
30
  def cache_exists?
27
- cache_root && %w{temp cache}.all? do |d|
31
+ cache_root && %w[temp cache].all? do |d|
28
32
  File.directory?(File.join(cache_root, d))
29
33
  end
30
34
  end
@@ -40,9 +44,5 @@ module Uricp::Strategy
40
44
  def cache_file
41
45
  @cache_file ||= File.join(cache_root, 'cache', cache_name)
42
46
  end
43
-
44
47
  end
45
-
46
48
  end
47
-
48
-
@@ -1,48 +1,42 @@
1
1
  require 'uri'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class CachedGet
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
10
  validate_cache!
13
- if in_cache? || file_source?
14
- return proposal
15
- else
16
- debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
17
- end
11
+ return proposal if in_cache? || file_source?
12
+
13
+ debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
18
14
  else
19
- debug "#{self.class.name}: not appropriate"
15
+ debug "#{self.class.name}: not appropriate"
20
16
  end
21
17
  false
22
18
  end
23
19
 
24
20
  def command
25
- ":;"
21
+ ':;'
26
22
  end
27
23
 
28
24
  def proposal
29
25
  @proposed_options = options.dup
30
- unless file_source?
31
- @proposed_options['from_uri'] = URI.join('file:///', cache_file)
32
- end
26
+ @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
33
27
  @proposed_options.delete('cache')
34
28
  @proposed_options.delete('cache_name')
35
29
  if conversion_required?
36
- @proposed_options['source-format'] =
37
- File.open(@proposed_options['from_uri'].path) {|f| encoding(f)}
38
- if @proposed_options['source-format'] == @proposed_options['target-format']
39
- @proposed_options.delete('source-format')
40
- @proposed_options.delete('target-format')
41
- end
30
+ unless dry_run?
31
+ @proposed_options['source-format'] =
32
+ File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
33
+ end
34
+ if @proposed_options['source-format'] == @proposed_options['target-format']
35
+ @proposed_options.delete('source-format')
36
+ @proposed_options.delete('target-format')
37
+ end
42
38
  end
43
39
  self
44
40
  end
45
-
46
41
  end
47
-
48
42
  end
@@ -1,13 +1,10 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class Cleaner
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- if options['clean'] && sequence_complete?
9
- return proposal
10
- end
6
+ return proposal if options['clean'] && sequence_complete?
7
+
11
8
  debug "#{self.class.name}: not appropriate"
12
9
  false
13
10
  end
@@ -21,8 +18,5 @@ module Uricp::Strategy
21
18
  @proposed_options.delete('clean')
22
19
  self
23
20
  end
24
-
25
21
  end
26
-
27
22
  end
28
-
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  module Common
4
-
5
3
  include Methadone::CLILogging
6
4
  include Uricp::CurlPrimitives
7
5
 
@@ -14,10 +12,10 @@ module Uricp::Strategy
14
12
 
15
13
  def unsupported_transfer
16
14
  raise Uricp::UnsupportedURLtype,
17
- "Unsupported transfer from #{from.to_s} to #{to.to_s}"
15
+ "Unsupported transfer from #{from} to #{to}"
18
16
  end
19
17
 
20
- alias :command :unsupported_transfer
18
+ alias command unsupported_transfer
21
19
 
22
20
  def all_local_files?
23
21
  !sequence_complete? && file_source? && to.scheme == 'file'
@@ -44,24 +42,23 @@ module Uricp::Strategy
44
42
  end
45
43
 
46
44
  def qcow2?(magic)
47
- magic.unpack('a3C') == ['QFI',0xfb]
45
+ magic.unpack('a3C') == ['QFI', 0xfb]
48
46
  end
49
47
 
50
48
  def encoding(io)
51
49
  magic = io.read(4).to_s
52
- case
53
- when lz4?(magic)
50
+ if lz4?(magic)
54
51
  :lz4
55
- when qcow2?(magic)
52
+ elsif qcow2?(magic)
56
53
  version = io.read(4)
57
- case version.unpack('N')
58
- when [2]
59
- :qcow2
60
- when [3]
61
- :qcow3
62
- else
63
- :qcow2un
64
- end
54
+ case version.unpack('N')
55
+ when [2]
56
+ :qcow2
57
+ when [3]
58
+ :qcow3
59
+ else
60
+ :qcow2un
61
+ end
65
62
  else
66
63
  :raw
67
64
  end
@@ -73,6 +70,10 @@ module Uricp::Strategy
73
70
  )
74
71
  end
75
72
 
73
+ def dry_run?
74
+ options['dry-run']
75
+ end
76
+
76
77
  def lz4_source?
77
78
  options['source-format'] == :lz4
78
79
  end
@@ -92,7 +93,7 @@ module Uricp::Strategy
92
93
  PIPE_URI = URI('pipe:/')
93
94
 
94
95
  def get_temp_filename(base_dir)
95
- t = Time.now.strftime("%Y%m%d")
96
+ t = Time.now.strftime('%Y%m%d')
96
97
  File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
97
98
  end
98
99
 
@@ -108,6 +109,12 @@ module Uricp::Strategy
108
109
  options['source-format'] && !lz4_source?
109
110
  end
110
111
 
111
- end
112
+ def rbd_target(uri)
113
+ uri.path[1..-1]
114
+ end
112
115
 
116
+ def rbd_id
117
+ 'libvirt'
118
+ end
119
+ end
113
120
  end
@@ -1,17 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalConvert
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if conversion_required? && file_source? && supported_source?
9
- if new_qemu? || supported_conversion?
10
- return proposal
11
- else
12
- raise Uricp::UnsupportedConversion,
13
- "qemu-img does not support required conversion"
14
- end
7
+ return proposal if new_qemu? || supported_conversion?
8
+
9
+ raise Uricp::UnsupportedConversion,
10
+ 'qemu-img does not support required conversion'
15
11
  end
16
12
  debug "#{self.class.name}: not appropriate"
17
13
  false
@@ -24,11 +20,11 @@ module Uricp::Strategy
24
20
  def proposal
25
21
  @proposed_options = options.dup
26
22
  if to.scheme == 'file'
27
- @proposed_options['from_uri'] = @proposed_options['to_uri']
23
+ @proposed_options['from_uri'] = @proposed_options['to_uri']
28
24
  else
29
25
  @proposed_options['from_uri'] = temp_uri
30
- @proposed_options['clean'] ||= []
31
- @proposed_options['clean'] << proposed_path
26
+ @proposed_options['clean'] ||= []
27
+ @proposed_options['clean'] << proposed_path
32
28
  end
33
29
  @proposed_options.delete('source-format')
34
30
  @proposed_options.delete('target-format')
@@ -40,7 +36,7 @@ module Uricp::Strategy
40
36
  end
41
37
 
42
38
  def supported_conversion?
43
- ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
39
+ ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
44
40
  end
45
41
 
46
42
  def qemu_img_command(target)
@@ -59,14 +55,11 @@ module Uricp::Strategy
59
55
  end
60
56
 
61
57
  def compat_option
62
- "-o compat=0.10" if new_qemu?
58
+ '-o compat=0.10' if new_qemu?
63
59
  end
64
60
 
65
61
  def compress_option
66
62
  options['compress'] && '-c'
67
63
  end
68
-
69
64
  end
70
-
71
65
  end
72
-
@@ -1,11 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalLink
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- return proposal if all_local_files? && !format_change? && linkable?
6
+ return proposal if !compression_required? &&
7
+ all_local_files? &&
8
+ !format_change? &&
9
+ linkable?
10
+
9
11
  debug "#{self.class.name}: not appropriate"
10
12
  false
11
13
  end
@@ -21,11 +23,11 @@ module Uricp::Strategy
21
23
  end
22
24
 
23
25
  def linkable?
24
- File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
25
- (!File.exist?(to.path) || File.file?(to.path))
26
+ from_path_dir = File.dirname(from.path)
27
+ to_path_dir = File.dirname(to.path)
28
+ File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
29
+ File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
30
+ (!File.exist?(to.path) || File.file?(to.path))
26
31
  end
27
-
28
32
  end
29
-
30
33
  end
31
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCache
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache!
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache!
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no cacheing requested"
17
+ debug "#{self.class.name}: no cacheing requested"
20
18
  end
21
19
  false
22
20
  end
@@ -32,9 +30,5 @@ module Uricp::Strategy
32
30
  @proposed_options.delete('cache_name')
33
31
  self
34
32
  end
35
-
36
33
  end
37
-
38
34
  end
39
-
40
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCacheConvert
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if conversion_required? && supported_source?
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache! if cache_root
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
9
+ if conversion_required? && supported_source?
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache! if cache_root
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no non-stream conversion detected"
17
+ debug "#{self.class.name}: no non-stream conversion detected"
20
18
  end
21
19
  false
22
20
  end
@@ -28,18 +26,16 @@ module Uricp::Strategy
28
26
  def proposal
29
27
  @proposed_options = options.dup
30
28
  if cache_root
31
- @proposed_options['from_uri'] = temp_cache_uri
32
- @proposed_options['sweep'] = [temp_cache_file, cache_file]
33
- @proposed_options.delete('cache')
34
- @proposed_options.delete('cache_name')
29
+ @proposed_options['from_uri'] = temp_cache_uri
30
+ @proposed_options['sweep'] = [temp_cache_file, cache_file]
31
+ @proposed_options.delete('cache')
32
+ @proposed_options.delete('cache_name')
35
33
  else
36
34
  @proposed_options['from_uri'] = temp_uri
37
- @proposed_options['clean'] ||= []
38
- @proposed_options['clean'] << proposed_path
35
+ @proposed_options['clean'] ||= []
36
+ @proposed_options['clean'] << proposed_path
39
37
  end
40
38
  self
41
39
  end
42
-
43
40
  end
44
-
45
41
  end