uricp 0.0.15 → 0.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/Gemfile.lock +16 -21
  4. data/Jenkinsfile +90 -33
  5. data/Rakefile +24 -36
  6. data/almalinux8/Dockerfile +26 -0
  7. data/bin/uricp +25 -11
  8. data/bionic/Dockerfile +4 -3
  9. data/centos7/Dockerfile +17 -12
  10. data/cucumber.yml +1 -0
  11. data/features/check_uri_path.feature +14 -0
  12. data/features/rbd_access.feature +226 -0
  13. data/features/step_definitions/orbit_steps.rb +11 -4
  14. data/features/step_definitions/rbd_steps.rb +8 -0
  15. data/features/step_definitions/uricp_steps.rb +8 -4
  16. data/focal/Dockerfile +28 -0
  17. data/lib/segment_upload.rb +20 -22
  18. data/lib/uricp/curl_primitives.rb +31 -33
  19. data/lib/uricp/orbit_auth.rb +23 -27
  20. data/lib/uricp/segmenter.rb +12 -16
  21. data/lib/uricp/strategy/cache_common.rb +30 -12
  22. data/lib/uricp/strategy/cached_get.rb +19 -22
  23. data/lib/uricp/strategy/cleaner.rb +2 -8
  24. data/lib/uricp/strategy/common.rb +76 -18
  25. data/lib/uricp/strategy/local_convert.rb +10 -17
  26. data/lib/uricp/strategy/local_link.rb +10 -8
  27. data/lib/uricp/strategy/piped_cache.rb +11 -16
  28. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  29. data/lib/uricp/strategy/piped_compress.rb +3 -8
  30. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  31. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  32. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  33. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  34. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  35. data/lib/uricp/strategy/piped_rbd_get.rb +30 -0
  36. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  37. data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
  38. data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
  39. data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
  40. data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
  41. data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
  42. data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
  43. data/lib/uricp/strategy/rbd_flattener.rb +23 -0
  44. data/lib/uricp/strategy/rbd_put.rb +38 -0
  45. data/lib/uricp/strategy/rbd_snap.rb +56 -0
  46. data/lib/uricp/strategy/rbd_sweeper.rb +23 -0
  47. data/lib/uricp/strategy/remote_put.rb +11 -17
  48. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  49. data/lib/uricp/strategy/sweeper.rb +2 -8
  50. data/lib/uricp/uri_strategy.rb +22 -13
  51. data/lib/uricp/version.rb +4 -2
  52. data/lib/uricp.rb +27 -19
  53. data/uricp.gemspec +3 -3
  54. data/xenial/Dockerfile +4 -3
  55. metadata +41 -25
  56. data/spec/something_spec.rb +0 -5
  57. data/trusty/Dockerfile +0 -20
@@ -1,17 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalConvert
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if conversion_required? && file_source? && supported_source?
9
- if new_qemu? || supported_conversion?
10
- return proposal
11
- else
12
- raise Uricp::UnsupportedConversion,
13
- "qemu-img does not support required conversion"
14
- end
7
+ return proposal if new_qemu? || supported_conversion?
8
+
9
+ raise Uricp::UnsupportedConversion,
10
+ 'qemu-img does not support required conversion'
15
11
  end
16
12
  debug "#{self.class.name}: not appropriate"
17
13
  false
@@ -24,11 +20,11 @@ module Uricp::Strategy
24
20
  def proposal
25
21
  @proposed_options = options.dup
26
22
  if to.scheme == 'file'
27
- @proposed_options['from_uri'] = @proposed_options['to_uri']
23
+ @proposed_options['from_uri'] = @proposed_options['to_uri']
28
24
  else
29
25
  @proposed_options['from_uri'] = temp_uri
30
- @proposed_options['clean'] ||= []
31
- @proposed_options['clean'] << proposed_path
26
+ @proposed_options['clean'] ||= []
27
+ @proposed_options['clean'] << proposed_path
32
28
  end
33
29
  @proposed_options.delete('source-format')
34
30
  @proposed_options.delete('target-format')
@@ -36,11 +32,11 @@ module Uricp::Strategy
36
32
  end
37
33
 
38
34
  def new_qemu?
39
- @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /^compat/m)
35
+ @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
40
36
  end
41
37
 
42
38
  def supported_conversion?
43
- ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
39
+ ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
44
40
  end
45
41
 
46
42
  def qemu_img_command(target)
@@ -59,14 +55,11 @@ module Uricp::Strategy
59
55
  end
60
56
 
61
57
  def compat_option
62
- "-o compat=0.10" if new_qemu?
58
+ '-o compat=0.10' if new_qemu?
63
59
  end
64
60
 
65
61
  def compress_option
66
62
  options['compress'] && '-c'
67
63
  end
68
-
69
64
  end
70
-
71
65
  end
72
-
@@ -1,11 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalLink
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- return proposal if all_local_files? && !format_change? && linkable?
6
+ return proposal if !compression_required? &&
7
+ all_local_files? &&
8
+ !format_change? &&
9
+ linkable?
10
+
9
11
  debug "#{self.class.name}: not appropriate"
10
12
  false
11
13
  end
@@ -21,11 +23,11 @@ module Uricp::Strategy
21
23
  end
22
24
 
23
25
  def linkable?
24
- File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
25
- (!File.exist?(to.path) || File.file?(to.path))
26
+ from_path_dir = File.dirname(from.path)
27
+ to_path_dir = File.dirname(to.path)
28
+ File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
29
+ File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
30
+ (!File.exist?(to.path) || File.file?(to.path))
26
31
  end
27
-
28
32
  end
29
-
30
33
  end
31
-
@@ -1,24 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCache
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if cache_root
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache!
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
18
- else
19
- debug "#{self.class.name}: no cacheing requested"
9
+ with_active_cache do
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache!
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
16
+ false
20
17
  end
21
- false
22
18
  end
23
19
 
24
20
  def command
@@ -28,13 +24,12 @@ module Uricp::Strategy
28
24
  def proposal
29
25
  @proposed_options = options.dup
30
26
  @proposed_options['sweep'] = [temp_cache_file, cache_file]
27
+ if to.scheme == 'rbd'
28
+ @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
29
+ end
31
30
  @proposed_options.delete('cache')
32
31
  @proposed_options.delete('cache_name')
33
32
  self
34
33
  end
35
-
36
34
  end
37
-
38
35
  end
39
-
40
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCacheConvert
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if conversion_required? && supported_source?
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache! if cache_root
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
9
+ if conversion_required? && supported_source?
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache! if cache_root
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no non-stream conversion detected"
17
+ debug "#{self.class.name}: no non-stream conversion detected"
20
18
  end
21
19
  false
22
20
  end
@@ -28,18 +26,16 @@ module Uricp::Strategy
28
26
  def proposal
29
27
  @proposed_options = options.dup
30
28
  if cache_root
31
- @proposed_options['from_uri'] = temp_cache_uri
32
- @proposed_options['sweep'] = [temp_cache_file, cache_file]
33
- @proposed_options.delete('cache')
34
- @proposed_options.delete('cache_name')
29
+ @proposed_options['from_uri'] = temp_cache_uri
30
+ @proposed_options['sweep'] = [temp_cache_file, cache_file]
31
+ @proposed_options.delete('cache')
32
+ @proposed_options.delete('cache_name')
35
33
  else
36
34
  @proposed_options['from_uri'] = temp_uri
37
- @proposed_options['clean'] ||= []
38
- @proposed_options['clean'] << proposed_path
35
+ @proposed_options['clean'] ||= []
36
+ @proposed_options['clean'] << proposed_path
39
37
  end
40
38
  self
41
39
  end
42
-
43
40
  end
44
-
45
41
  end
@@ -1,30 +1,25 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
-
8
5
  def appropriate?
9
6
  case from.scheme
10
7
  when 'pipe'
11
- return proposal if compression_required?
8
+ return proposal if compression_required?
12
9
  end
13
10
  debug "#{self.class.name}: not appropriate"
14
11
  false
15
12
  end
16
13
 
17
14
  def command
18
- "lz4 |"
15
+ 'lz4 |'
19
16
  end
20
17
 
21
18
  def proposal
22
19
  @proposed_options = options.dup
23
20
  @proposed_options.delete('compress')
24
- @proposed_options['encoding']='lz4'
21
+ @proposed_options['encoding'] = 'lz4'
25
22
  self
26
23
  end
27
-
28
24
  end
29
-
30
25
  end
@@ -1,35 +1,31 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- return proposal if lz4_source?
8
+ return proposal if lz4_source?
11
9
  end
12
10
  debug "#{self.class.name}: not appropriate"
13
11
  false
14
12
  end
15
13
 
16
14
  def command
17
- "lz4 -d |"
15
+ 'lz4 -d |'
18
16
  end
19
17
 
20
18
  def proposal
21
19
  @proposed_options = options.dup
22
20
  @proposed_options.delete('source-format')
23
21
  if @proposed_options['target-format']
24
- @proposed_options['source-format'] = :raw
25
- if @proposed_options['source-format'] == @proposed_options['target-format']
26
- @proposed_options.delete('source-format')
27
- @proposed_options.delete('target-format')
28
- end
22
+ @proposed_options['source-format'] = :raw
23
+ if @proposed_options['source-format'] == @proposed_options['target-format']
24
+ @proposed_options.delete('source-format')
25
+ @proposed_options.delete('target-format')
26
+ end
29
27
  end
30
28
  self
31
29
  end
32
-
33
30
  end
34
-
35
31
  end
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
@@ -24,7 +22,5 @@ module Uricp::Strategy
24
22
  @proposed_options['from_uri'] = PIPE_URI
25
23
  self
26
24
  end
27
-
28
25
  end
29
-
30
26
  end
@@ -1,19 +1,15 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- if raw_target? && lz4_source? && to.scheme == 'file'
11
- if always_write_sparse?
12
- return proposal
13
- else
14
- debug "#{self.class.name}: using safe sparse expansion via stream"
15
- end
16
- end
8
+ if raw_target? && lz4_source? && to.scheme == 'file'
9
+ return proposal if always_write_sparse?
10
+
11
+ debug "#{self.class.name}: using safe sparse expansion via stream"
12
+ end
17
13
  end
18
14
  debug "#{self.class.name}: not appropriate"
19
15
  false
@@ -27,16 +23,14 @@ module Uricp::Strategy
27
23
  @proposed_options = options.dup
28
24
  @proposed_options.delete('source-format')
29
25
  if @proposed_options['target-format']
30
- @proposed_options['source-format'] = :raw
31
- if @proposed_options['source-format'] == @proposed_options['target-format']
32
- @proposed_options.delete('source-format')
33
- @proposed_options.delete('target-format')
34
- end
26
+ @proposed_options['source-format'] = :raw
27
+ if @proposed_options['source-format'] == @proposed_options['target-format']
28
+ @proposed_options.delete('source-format')
29
+ @proposed_options.delete('target-format')
30
+ end
35
31
  end
36
32
  @proposed_options['from_uri'] = @proposed_options['to_uri']
37
33
  self
38
34
  end
39
-
40
35
  end
41
-
42
36
  end
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalGet
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
@@ -22,8 +20,5 @@ module Uricp::Strategy
22
20
  @proposed_options['from_uri'] = PIPE_URI
23
21
  self
24
22
  end
25
-
26
23
  end
27
-
28
24
  end
29
-
@@ -1,14 +1,11 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalPut
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- case from.scheme
9
- when 'pipe'
10
- return proposal if to.scheme == 'file'
11
- end
6
+ return proposal if to.scheme == 'file' &&
7
+ from.scheme == 'pipe'
8
+
12
9
  debug "#{self.class.name}: not appropriate"
13
10
  false
14
11
  end
@@ -22,8 +19,5 @@ module Uricp::Strategy
22
19
  @proposed_options['from_uri'] = @proposed_options['to_uri']
23
20
  self
24
21
  end
25
-
26
22
  end
27
-
28
23
  end
29
-
@@ -0,0 +1,30 @@
1
+ require 'json'
2
+ module Uricp::Strategy
3
+ class PipedRbdGet
4
+ include Uricp::Strategy::Common
5
+ include Uricp::Strategy::CacheCommon
6
+ include Methadone::SH
7
+
8
+ def appropriate?
9
+ without_active_cache do
10
+ if from.scheme == 'rbd' &&
11
+ rbd_snapshot_spec?(from) &&
12
+ to.scheme != 'rbd'
13
+ return proposal unless sequence_complete?
14
+ end
15
+ end
16
+ debug "#{self.class.name}: not appropriate"
17
+ false
18
+ end
19
+
20
+ def command
21
+ "rbd export --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' - |"
22
+ end
23
+
24
+ def proposal
25
+ @proposed_options = options.dup
26
+ @proposed_options['from_uri'] = PIPE_URI
27
+ self
28
+ end
29
+ end
30
+ end
@@ -1,8 +1,6 @@
1
1
  require 'open-uri'
2
2
  module Uricp::Strategy
3
-
4
3
  class PipedRemoteGet
5
-
6
4
  include Uricp::Strategy::Common
7
5
 
8
6
  def appropriate?
@@ -10,33 +8,35 @@ module Uricp::Strategy
10
8
  when 'http', 'https'
11
9
  return proposal unless sequence_complete?
12
10
  else
13
- debug "#{self.class.name}: not appropriate"
14
- false
11
+ debug "#{self.class.name}: not appropriate"
12
+ false
15
13
  end
16
14
  end
17
15
 
18
- alias :command :curl_download_to_pipe
16
+ alias command curl_download_to_pipe
19
17
 
20
18
  def proposal
21
19
  @proposed_options = options.dup
22
20
  @proposed_options['from_uri'] = PIPE_URI
23
21
  if conversion_required?
24
22
  @proposed_options['source-format'] = format_peek
25
- if @proposed_options['source-format'] == @proposed_options['target-format']
26
- @proposed_options.delete('source-format')
27
- @proposed_options.delete('target-format')
28
- end
23
+ if @proposed_options['source-format'] == @proposed_options['target-format']
24
+ @proposed_options.delete('source-format')
25
+ @proposed_options.delete('target-format')
26
+ end
29
27
  end
30
28
  if options['max-cache'] &&
31
- size_peek.to_i > options['max-cache'].to_i
32
- @proposed_options.delete('cache')
33
- @proposed_options.delete('cache_name')
34
- @proposed_options.delete('max-cache')
29
+ size_peek.to_i > options['max-cache'].to_i
30
+ @proposed_options.delete('cache')
31
+ @proposed_options.delete('cache_name')
32
+ @proposed_options.delete('max-cache')
35
33
  end
36
34
  self
37
35
  end
38
36
 
39
37
  def format_peek
38
+ return options['target-format'] || 'raw' if dry_run?
39
+
40
40
  options['from_uri'].open(headers) do |u|
41
41
  encoding(u)
42
42
  end
@@ -48,14 +48,16 @@ module Uricp::Strategy
48
48
  raise
49
49
  end
50
50
  rescue SocketError => e
51
- raise SocketError, options['from_uri'].to_s+" inaccessible: "+e.message
51
+ raise SocketError, options['from_uri'].to_s + ' inaccessible: ' + e.message
52
52
  end
53
53
 
54
54
  def size_peek
55
- size_headers=headers
55
+ return options['max-cache'] if dry_run?
56
+
57
+ size_headers = headers
56
58
  size_headers['Range'] = 'bytes=0-0'
57
59
  options['from_uri'].open(headers) do |u|
58
- match = %r<bytes\s+(\d+)-(\d+)/(\d+|\*)>i.match(u.meta['content-range'])
60
+ match = %r{bytes\s+(\d+)-(\d+)/(\d+|\*)}i.match(u.meta['content-range'])
59
61
  match && match[3].to_i
60
62
  end
61
63
  rescue OpenURI::HTTPError => e
@@ -66,15 +68,13 @@ module Uricp::Strategy
66
68
  raise
67
69
  end
68
70
  rescue SocketError => e
69
- raise SocketError, options['from_uri'].to_s+" inaccessible: "+e.message
71
+ raise SocketError, options['from_uri'].to_s + ' inaccessible: ' + e.message
70
72
  end
71
73
 
72
74
  def headers
73
- headers={'Range' => 'bytes=0-7'}
75
+ headers = { 'Range' => 'bytes=0-7' }
74
76
  headers['X-Auth-Token'] = options['authenticator'].call if http_authentication?
75
77
  headers
76
78
  end
77
-
78
79
  end
79
-
80
80
  end