uricp 0.0.15 → 0.0.20

Sign up to get free protection for your applications and to get access to all the features.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/Gemfile.lock +16 -21
  4. data/Jenkinsfile +90 -33
  5. data/Rakefile +24 -36
  6. data/almalinux8/Dockerfile +26 -0
  7. data/bin/uricp +25 -11
  8. data/bionic/Dockerfile +4 -3
  9. data/centos7/Dockerfile +17 -12
  10. data/cucumber.yml +1 -0
  11. data/features/check_uri_path.feature +14 -0
  12. data/features/rbd_access.feature +226 -0
  13. data/features/step_definitions/orbit_steps.rb +11 -4
  14. data/features/step_definitions/rbd_steps.rb +8 -0
  15. data/features/step_definitions/uricp_steps.rb +8 -4
  16. data/focal/Dockerfile +28 -0
  17. data/lib/segment_upload.rb +20 -22
  18. data/lib/uricp/curl_primitives.rb +31 -33
  19. data/lib/uricp/orbit_auth.rb +23 -27
  20. data/lib/uricp/segmenter.rb +12 -16
  21. data/lib/uricp/strategy/cache_common.rb +30 -12
  22. data/lib/uricp/strategy/cached_get.rb +19 -22
  23. data/lib/uricp/strategy/cleaner.rb +2 -8
  24. data/lib/uricp/strategy/common.rb +76 -18
  25. data/lib/uricp/strategy/local_convert.rb +10 -17
  26. data/lib/uricp/strategy/local_link.rb +10 -8
  27. data/lib/uricp/strategy/piped_cache.rb +11 -16
  28. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  29. data/lib/uricp/strategy/piped_compress.rb +3 -8
  30. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  31. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  32. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  33. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  34. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  35. data/lib/uricp/strategy/piped_rbd_get.rb +30 -0
  36. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  37. data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
  38. data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
  39. data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
  40. data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
  41. data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
  42. data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
  43. data/lib/uricp/strategy/rbd_flattener.rb +23 -0
  44. data/lib/uricp/strategy/rbd_put.rb +38 -0
  45. data/lib/uricp/strategy/rbd_snap.rb +56 -0
  46. data/lib/uricp/strategy/rbd_sweeper.rb +23 -0
  47. data/lib/uricp/strategy/remote_put.rb +11 -17
  48. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  49. data/lib/uricp/strategy/sweeper.rb +2 -8
  50. data/lib/uricp/uri_strategy.rb +22 -13
  51. data/lib/uricp/version.rb +4 -2
  52. data/lib/uricp.rb +27 -19
  53. data/uricp.gemspec +3 -3
  54. data/xenial/Dockerfile +4 -3
  55. metadata +41 -25
  56. data/spec/something_spec.rb +0 -5
  57. data/trusty/Dockerfile +0 -20
@@ -1,17 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalConvert
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if conversion_required? && file_source? && supported_source?
9
- if new_qemu? || supported_conversion?
10
- return proposal
11
- else
12
- raise Uricp::UnsupportedConversion,
13
- "qemu-img does not support required conversion"
14
- end
7
+ return proposal if new_qemu? || supported_conversion?
8
+
9
+ raise Uricp::UnsupportedConversion,
10
+ 'qemu-img does not support required conversion'
15
11
  end
16
12
  debug "#{self.class.name}: not appropriate"
17
13
  false
@@ -24,11 +20,11 @@ module Uricp::Strategy
24
20
  def proposal
25
21
  @proposed_options = options.dup
26
22
  if to.scheme == 'file'
27
- @proposed_options['from_uri'] = @proposed_options['to_uri']
23
+ @proposed_options['from_uri'] = @proposed_options['to_uri']
28
24
  else
29
25
  @proposed_options['from_uri'] = temp_uri
30
- @proposed_options['clean'] ||= []
31
- @proposed_options['clean'] << proposed_path
26
+ @proposed_options['clean'] ||= []
27
+ @proposed_options['clean'] << proposed_path
32
28
  end
33
29
  @proposed_options.delete('source-format')
34
30
  @proposed_options.delete('target-format')
@@ -36,11 +32,11 @@ module Uricp::Strategy
36
32
  end
37
33
 
38
34
  def new_qemu?
39
- @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /^compat/m)
35
+ @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
40
36
  end
41
37
 
42
38
  def supported_conversion?
43
- ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
39
+ ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
44
40
  end
45
41
 
46
42
  def qemu_img_command(target)
@@ -59,14 +55,11 @@ module Uricp::Strategy
59
55
  end
60
56
 
61
57
  def compat_option
62
- "-o compat=0.10" if new_qemu?
58
+ '-o compat=0.10' if new_qemu?
63
59
  end
64
60
 
65
61
  def compress_option
66
62
  options['compress'] && '-c'
67
63
  end
68
-
69
64
  end
70
-
71
65
  end
72
-
@@ -1,11 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalLink
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- return proposal if all_local_files? && !format_change? && linkable?
6
+ return proposal if !compression_required? &&
7
+ all_local_files? &&
8
+ !format_change? &&
9
+ linkable?
10
+
9
11
  debug "#{self.class.name}: not appropriate"
10
12
  false
11
13
  end
@@ -21,11 +23,11 @@ module Uricp::Strategy
21
23
  end
22
24
 
23
25
  def linkable?
24
- File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
25
- (!File.exist?(to.path) || File.file?(to.path))
26
+ from_path_dir = File.dirname(from.path)
27
+ to_path_dir = File.dirname(to.path)
28
+ File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
29
+ File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
30
+ (!File.exist?(to.path) || File.file?(to.path))
26
31
  end
27
-
28
32
  end
29
-
30
33
  end
31
-
@@ -1,24 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCache
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if cache_root
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache!
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
18
- else
19
- debug "#{self.class.name}: no cacheing requested"
9
+ with_active_cache do
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache!
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
16
+ false
20
17
  end
21
- false
22
18
  end
23
19
 
24
20
  def command
@@ -28,13 +24,12 @@ module Uricp::Strategy
28
24
  def proposal
29
25
  @proposed_options = options.dup
30
26
  @proposed_options['sweep'] = [temp_cache_file, cache_file]
27
+ if to.scheme == 'rbd'
28
+ @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
29
+ end
31
30
  @proposed_options.delete('cache')
32
31
  @proposed_options.delete('cache_name')
33
32
  self
34
33
  end
35
-
36
34
  end
37
-
38
35
  end
39
-
40
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCacheConvert
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if conversion_required? && supported_source?
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache! if cache_root
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
9
+ if conversion_required? && supported_source?
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache! if cache_root
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no non-stream conversion detected"
17
+ debug "#{self.class.name}: no non-stream conversion detected"
20
18
  end
21
19
  false
22
20
  end
@@ -28,18 +26,16 @@ module Uricp::Strategy
28
26
  def proposal
29
27
  @proposed_options = options.dup
30
28
  if cache_root
31
- @proposed_options['from_uri'] = temp_cache_uri
32
- @proposed_options['sweep'] = [temp_cache_file, cache_file]
33
- @proposed_options.delete('cache')
34
- @proposed_options.delete('cache_name')
29
+ @proposed_options['from_uri'] = temp_cache_uri
30
+ @proposed_options['sweep'] = [temp_cache_file, cache_file]
31
+ @proposed_options.delete('cache')
32
+ @proposed_options.delete('cache_name')
35
33
  else
36
34
  @proposed_options['from_uri'] = temp_uri
37
- @proposed_options['clean'] ||= []
38
- @proposed_options['clean'] << proposed_path
35
+ @proposed_options['clean'] ||= []
36
+ @proposed_options['clean'] << proposed_path
39
37
  end
40
38
  self
41
39
  end
42
-
43
40
  end
44
-
45
41
  end
@@ -1,30 +1,25 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
-
8
5
  def appropriate?
9
6
  case from.scheme
10
7
  when 'pipe'
11
- return proposal if compression_required?
8
+ return proposal if compression_required?
12
9
  end
13
10
  debug "#{self.class.name}: not appropriate"
14
11
  false
15
12
  end
16
13
 
17
14
  def command
18
- "lz4 |"
15
+ 'lz4 |'
19
16
  end
20
17
 
21
18
  def proposal
22
19
  @proposed_options = options.dup
23
20
  @proposed_options.delete('compress')
24
- @proposed_options['encoding']='lz4'
21
+ @proposed_options['encoding'] = 'lz4'
25
22
  self
26
23
  end
27
-
28
24
  end
29
-
30
25
  end
@@ -1,35 +1,31 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- return proposal if lz4_source?
8
+ return proposal if lz4_source?
11
9
  end
12
10
  debug "#{self.class.name}: not appropriate"
13
11
  false
14
12
  end
15
13
 
16
14
  def command
17
- "lz4 -d |"
15
+ 'lz4 -d |'
18
16
  end
19
17
 
20
18
  def proposal
21
19
  @proposed_options = options.dup
22
20
  @proposed_options.delete('source-format')
23
21
  if @proposed_options['target-format']
24
- @proposed_options['source-format'] = :raw
25
- if @proposed_options['source-format'] == @proposed_options['target-format']
26
- @proposed_options.delete('source-format')
27
- @proposed_options.delete('target-format')
28
- end
22
+ @proposed_options['source-format'] = :raw
23
+ if @proposed_options['source-format'] == @proposed_options['target-format']
24
+ @proposed_options.delete('source-format')
25
+ @proposed_options.delete('target-format')
26
+ end
29
27
  end
30
28
  self
31
29
  end
32
-
33
30
  end
34
-
35
31
  end
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
@@ -24,7 +22,5 @@ module Uricp::Strategy
24
22
  @proposed_options['from_uri'] = PIPE_URI
25
23
  self
26
24
  end
27
-
28
25
  end
29
-
30
26
  end
@@ -1,19 +1,15 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- if raw_target? && lz4_source? && to.scheme == 'file'
11
- if always_write_sparse?
12
- return proposal
13
- else
14
- debug "#{self.class.name}: using safe sparse expansion via stream"
15
- end
16
- end
8
+ if raw_target? && lz4_source? && to.scheme == 'file'
9
+ return proposal if always_write_sparse?
10
+
11
+ debug "#{self.class.name}: using safe sparse expansion via stream"
12
+ end
17
13
  end
18
14
  debug "#{self.class.name}: not appropriate"
19
15
  false
@@ -27,16 +23,14 @@ module Uricp::Strategy
27
23
  @proposed_options = options.dup
28
24
  @proposed_options.delete('source-format')
29
25
  if @proposed_options['target-format']
30
- @proposed_options['source-format'] = :raw
31
- if @proposed_options['source-format'] == @proposed_options['target-format']
32
- @proposed_options.delete('source-format')
33
- @proposed_options.delete('target-format')
34
- end
26
+ @proposed_options['source-format'] = :raw
27
+ if @proposed_options['source-format'] == @proposed_options['target-format']
28
+ @proposed_options.delete('source-format')
29
+ @proposed_options.delete('target-format')
30
+ end
35
31
  end
36
32
  @proposed_options['from_uri'] = @proposed_options['to_uri']
37
33
  self
38
34
  end
39
-
40
35
  end
41
-
42
36
  end
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalGet
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
@@ -22,8 +20,5 @@ module Uricp::Strategy
22
20
  @proposed_options['from_uri'] = PIPE_URI
23
21
  self
24
22
  end
25
-
26
23
  end
27
-
28
24
  end
29
-
@@ -1,14 +1,11 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalPut
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- case from.scheme
9
- when 'pipe'
10
- return proposal if to.scheme == 'file'
11
- end
6
+ return proposal if to.scheme == 'file' &&
7
+ from.scheme == 'pipe'
8
+
12
9
  debug "#{self.class.name}: not appropriate"
13
10
  false
14
11
  end
@@ -22,8 +19,5 @@ module Uricp::Strategy
22
19
  @proposed_options['from_uri'] = @proposed_options['to_uri']
23
20
  self
24
21
  end
25
-
26
22
  end
27
-
28
23
  end
29
-
@@ -0,0 +1,30 @@
1
+ require 'json'
2
+ module Uricp::Strategy
3
+ class PipedRbdGet
4
+ include Uricp::Strategy::Common
5
+ include Uricp::Strategy::CacheCommon
6
+ include Methadone::SH
7
+
8
+ def appropriate?
9
+ without_active_cache do
10
+ if from.scheme == 'rbd' &&
11
+ rbd_snapshot_spec?(from) &&
12
+ to.scheme != 'rbd'
13
+ return proposal unless sequence_complete?
14
+ end
15
+ end
16
+ debug "#{self.class.name}: not appropriate"
17
+ false
18
+ end
19
+
20
+ def command
21
+ "rbd export --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' - |"
22
+ end
23
+
24
+ def proposal
25
+ @proposed_options = options.dup
26
+ @proposed_options['from_uri'] = PIPE_URI
27
+ self
28
+ end
29
+ end
30
+ end
@@ -1,8 +1,6 @@
1
1
  require 'open-uri'
2
2
  module Uricp::Strategy
3
-
4
3
  class PipedRemoteGet
5
-
6
4
  include Uricp::Strategy::Common
7
5
 
8
6
  def appropriate?
@@ -10,33 +8,35 @@ module Uricp::Strategy
10
8
  when 'http', 'https'
11
9
  return proposal unless sequence_complete?
12
10
  else
13
- debug "#{self.class.name}: not appropriate"
14
- false
11
+ debug "#{self.class.name}: not appropriate"
12
+ false
15
13
  end
16
14
  end
17
15
 
18
- alias :command :curl_download_to_pipe
16
+ alias command curl_download_to_pipe
19
17
 
20
18
  def proposal
21
19
  @proposed_options = options.dup
22
20
  @proposed_options['from_uri'] = PIPE_URI
23
21
  if conversion_required?
24
22
  @proposed_options['source-format'] = format_peek
25
- if @proposed_options['source-format'] == @proposed_options['target-format']
26
- @proposed_options.delete('source-format')
27
- @proposed_options.delete('target-format')
28
- end
23
+ if @proposed_options['source-format'] == @proposed_options['target-format']
24
+ @proposed_options.delete('source-format')
25
+ @proposed_options.delete('target-format')
26
+ end
29
27
  end
30
28
  if options['max-cache'] &&
31
- size_peek.to_i > options['max-cache'].to_i
32
- @proposed_options.delete('cache')
33
- @proposed_options.delete('cache_name')
34
- @proposed_options.delete('max-cache')
29
+ size_peek.to_i > options['max-cache'].to_i
30
+ @proposed_options.delete('cache')
31
+ @proposed_options.delete('cache_name')
32
+ @proposed_options.delete('max-cache')
35
33
  end
36
34
  self
37
35
  end
38
36
 
39
37
  def format_peek
38
+ return options['target-format'] || 'raw' if dry_run?
39
+
40
40
  options['from_uri'].open(headers) do |u|
41
41
  encoding(u)
42
42
  end
@@ -48,14 +48,16 @@ module Uricp::Strategy
48
48
  raise
49
49
  end
50
50
  rescue SocketError => e
51
- raise SocketError, options['from_uri'].to_s+" inaccessible: "+e.message
51
+ raise SocketError, options['from_uri'].to_s + ' inaccessible: ' + e.message
52
52
  end
53
53
 
54
54
  def size_peek
55
- size_headers=headers
55
+ return options['max-cache'] if dry_run?
56
+
57
+ size_headers = headers
56
58
  size_headers['Range'] = 'bytes=0-0'
57
59
  options['from_uri'].open(headers) do |u|
58
- match = %r<bytes\s+(\d+)-(\d+)/(\d+|\*)>i.match(u.meta['content-range'])
60
+ match = %r{bytes\s+(\d+)-(\d+)/(\d+|\*)}i.match(u.meta['content-range'])
59
61
  match && match[3].to_i
60
62
  end
61
63
  rescue OpenURI::HTTPError => e
@@ -66,15 +68,13 @@ module Uricp::Strategy
66
68
  raise
67
69
  end
68
70
  rescue SocketError => e
69
- raise SocketError, options['from_uri'].to_s+" inaccessible: "+e.message
71
+ raise SocketError, options['from_uri'].to_s + ' inaccessible: ' + e.message
70
72
  end
71
73
 
72
74
  def headers
73
- headers={'Range' => 'bytes=0-7'}
75
+ headers = { 'Range' => 'bytes=0-7' }
74
76
  headers['X-Auth-Token'] = options['authenticator'].call if http_authentication?
75
77
  headers
76
78
  end
77
-
78
79
  end
79
-
80
80
  end