uricp 0.0.13 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -1
  3. data/.rubocop.yml +3 -0
  4. data/Gemfile.lock +20 -23
  5. data/Jenkinsfile +113 -0
  6. data/README.md +2 -2
  7. data/Rakefile +26 -31
  8. data/bionic/Dockerfile +20 -0
  9. data/centos7/Dockerfile +18 -0
  10. data/features/step_definitions/orbit_steps.rb +19 -13
  11. data/features/step_definitions/uricp_steps.rb +8 -4
  12. data/focal/Dockerfile +21 -0
  13. data/lib/segment_upload.rb +20 -22
  14. data/lib/uricp.rb +3 -29
  15. data/lib/uricp/curl_primitives.rb +31 -33
  16. data/lib/uricp/orbit_auth.rb +23 -27
  17. data/lib/uricp/segmenter.rb +12 -16
  18. data/lib/uricp/strategy/cache_common.rb +11 -11
  19. data/lib/uricp/strategy/cached_get.rb +14 -20
  20. data/lib/uricp/strategy/cleaner.rb +2 -8
  21. data/lib/uricp/strategy/common.rb +27 -19
  22. data/lib/uricp/strategy/local_convert.rb +10 -17
  23. data/lib/uricp/strategy/local_link.rb +10 -8
  24. data/lib/uricp/strategy/piped_cache.rb +7 -13
  25. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  26. data/lib/uricp/strategy/piped_compress.rb +3 -8
  27. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  28. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  29. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  30. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  31. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  32. data/lib/uricp/strategy/piped_rbd_get.rb +46 -0
  33. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  34. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  35. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  36. data/lib/uricp/strategy/remote_put.rb +11 -17
  37. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  38. data/lib/uricp/strategy/sweeper.rb +2 -8
  39. data/lib/uricp/uri_strategy.rb +14 -13
  40. data/lib/uricp/version.rb +4 -2
  41. data/uricp.gemspec +3 -4
  42. data/xenial/Dockerfile +20 -0
  43. metadata +39 -51
  44. data/Dockerfile_centos6.6 +0 -39
  45. data/Dockerfile_centos7 +0 -39
  46. data/Dockerfile_trusty-ruby193 +0 -32
  47. data/README.rdoc +0 -23
  48. data/spec/something_spec.rb +0 -5
@@ -1,13 +1,10 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class Cleaner
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- if options['clean'] && sequence_complete?
9
- return proposal
10
- end
6
+ return proposal if options['clean'] && sequence_complete?
7
+
11
8
  debug "#{self.class.name}: not appropriate"
12
9
  false
13
10
  end
@@ -21,8 +18,5 @@ module Uricp::Strategy
21
18
  @proposed_options.delete('clean')
22
19
  self
23
20
  end
24
-
25
21
  end
26
-
27
22
  end
28
-
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  module Common
4
-
5
3
  include Methadone::CLILogging
6
4
  include Uricp::CurlPrimitives
7
5
 
@@ -14,10 +12,10 @@ module Uricp::Strategy
14
12
 
15
13
  def unsupported_transfer
16
14
  raise Uricp::UnsupportedURLtype,
17
- "Unsupported transfer from #{from.to_s} to #{to.to_s}"
15
+ "Unsupported transfer from #{from} to #{to}"
18
16
  end
19
17
 
20
- alias :command :unsupported_transfer
18
+ alias command unsupported_transfer
21
19
 
22
20
  def all_local_files?
23
21
  !sequence_complete? && file_source? && to.scheme == 'file'
@@ -44,24 +42,23 @@ module Uricp::Strategy
44
42
  end
45
43
 
46
44
  def qcow2?(magic)
47
- magic.unpack('a3C') == ['QFI',0xfb]
45
+ magic.unpack('a3C') == ['QFI', 0xfb]
48
46
  end
49
47
 
50
48
  def encoding(io)
51
49
  magic = io.read(4).to_s
52
- case
53
- when lz4?(magic)
50
+ if lz4?(magic)
54
51
  :lz4
55
- when qcow2?(magic)
52
+ elsif qcow2?(magic)
56
53
  version = io.read(4)
57
- case version.unpack('N')
58
- when [2]
59
- :qcow2
60
- when [3]
61
- :qcow3
62
- else
63
- :qcow2un
64
- end
54
+ case version.unpack('N')
55
+ when [2]
56
+ :qcow2
57
+ when [3]
58
+ :qcow3
59
+ else
60
+ :qcow2un
61
+ end
65
62
  else
66
63
  :raw
67
64
  end
@@ -73,6 +70,10 @@ module Uricp::Strategy
73
70
  )
74
71
  end
75
72
 
73
+ def dry_run?
74
+ options['dry-run']
75
+ end
76
+
76
77
  def lz4_source?
77
78
  options['source-format'] == :lz4
78
79
  end
@@ -91,8 +92,9 @@ module Uricp::Strategy
91
92
 
92
93
  PIPE_URI = URI('pipe:/')
93
94
 
94
- def get_temp_filename(filename)
95
- Dir::Tmpname.make_tmpname(filename, nil)
95
+ def get_temp_filename(base_dir)
96
+ t = Time.now.strftime('%Y%m%d')
97
+ File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
96
98
  end
97
99
 
98
100
  def proposed_path
@@ -107,6 +109,12 @@ module Uricp::Strategy
107
109
  options['source-format'] && !lz4_source?
108
110
  end
109
111
 
110
- end
112
+ def rbd_target(uri)
113
+ uri.path[1..-1]
114
+ end
111
115
 
116
+ def rbd_id
117
+ 'libvirt'
118
+ end
119
+ end
112
120
  end
@@ -1,17 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalConvert
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if conversion_required? && file_source? && supported_source?
9
- if new_qemu? || supported_conversion?
10
- return proposal
11
- else
12
- raise Uricp::UnsupportedConversion,
13
- "qemu-img does not support required conversion"
14
- end
7
+ return proposal if new_qemu? || supported_conversion?
8
+
9
+ raise Uricp::UnsupportedConversion,
10
+ 'qemu-img does not support required conversion'
15
11
  end
16
12
  debug "#{self.class.name}: not appropriate"
17
13
  false
@@ -24,11 +20,11 @@ module Uricp::Strategy
24
20
  def proposal
25
21
  @proposed_options = options.dup
26
22
  if to.scheme == 'file'
27
- @proposed_options['from_uri'] = @proposed_options['to_uri']
23
+ @proposed_options['from_uri'] = @proposed_options['to_uri']
28
24
  else
29
25
  @proposed_options['from_uri'] = temp_uri
30
- @proposed_options['clean'] ||= []
31
- @proposed_options['clean'] << proposed_path
26
+ @proposed_options['clean'] ||= []
27
+ @proposed_options['clean'] << proposed_path
32
28
  end
33
29
  @proposed_options.delete('source-format')
34
30
  @proposed_options.delete('target-format')
@@ -36,11 +32,11 @@ module Uricp::Strategy
36
32
  end
37
33
 
38
34
  def new_qemu?
39
- @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /^compat/m)
35
+ @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
40
36
  end
41
37
 
42
38
  def supported_conversion?
43
- ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
39
+ ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
44
40
  end
45
41
 
46
42
  def qemu_img_command(target)
@@ -59,14 +55,11 @@ module Uricp::Strategy
59
55
  end
60
56
 
61
57
  def compat_option
62
- "-o compat=0.10" if new_qemu?
58
+ '-o compat=0.10' if new_qemu?
63
59
  end
64
60
 
65
61
  def compress_option
66
62
  options['compress'] && '-c'
67
63
  end
68
-
69
64
  end
70
-
71
65
  end
72
-
@@ -1,11 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalLink
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- return proposal if all_local_files? && !format_change? && linkable?
6
+ return proposal if !compression_required? &&
7
+ all_local_files? &&
8
+ !format_change? &&
9
+ linkable?
10
+
9
11
  debug "#{self.class.name}: not appropriate"
10
12
  false
11
13
  end
@@ -21,11 +23,11 @@ module Uricp::Strategy
21
23
  end
22
24
 
23
25
  def linkable?
24
- File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
25
- (!File.exist?(to.path) || File.file?(to.path))
26
+ from_path_dir = File.dirname(from.path)
27
+ to_path_dir = File.dirname(to.path)
28
+ File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
29
+ File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
30
+ (!File.exist?(to.path) || File.file?(to.path))
26
31
  end
27
-
28
32
  end
29
-
30
33
  end
31
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCache
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache!
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache!
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no cacheing requested"
17
+ debug "#{self.class.name}: no cacheing requested"
20
18
  end
21
19
  false
22
20
  end
@@ -32,9 +30,5 @@ module Uricp::Strategy
32
30
  @proposed_options.delete('cache_name')
33
31
  self
34
32
  end
35
-
36
33
  end
37
-
38
34
  end
39
-
40
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCacheConvert
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if conversion_required? && supported_source?
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache! if cache_root
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
9
+ if conversion_required? && supported_source?
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache! if cache_root
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no non-stream conversion detected"
17
+ debug "#{self.class.name}: no non-stream conversion detected"
20
18
  end
21
19
  false
22
20
  end
@@ -28,18 +26,16 @@ module Uricp::Strategy
28
26
  def proposal
29
27
  @proposed_options = options.dup
30
28
  if cache_root
31
- @proposed_options['from_uri'] = temp_cache_uri
32
- @proposed_options['sweep'] = [temp_cache_file, cache_file]
33
- @proposed_options.delete('cache')
34
- @proposed_options.delete('cache_name')
29
+ @proposed_options['from_uri'] = temp_cache_uri
30
+ @proposed_options['sweep'] = [temp_cache_file, cache_file]
31
+ @proposed_options.delete('cache')
32
+ @proposed_options.delete('cache_name')
35
33
  else
36
34
  @proposed_options['from_uri'] = temp_uri
37
- @proposed_options['clean'] ||= []
38
- @proposed_options['clean'] << proposed_path
35
+ @proposed_options['clean'] ||= []
36
+ @proposed_options['clean'] << proposed_path
39
37
  end
40
38
  self
41
39
  end
42
-
43
40
  end
44
-
45
41
  end
@@ -1,30 +1,25 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
-
8
5
  def appropriate?
9
6
  case from.scheme
10
7
  when 'pipe'
11
- return proposal if compression_required?
8
+ return proposal if compression_required?
12
9
  end
13
10
  debug "#{self.class.name}: not appropriate"
14
11
  false
15
12
  end
16
13
 
17
14
  def command
18
- "lz4 |"
15
+ 'lz4 |'
19
16
  end
20
17
 
21
18
  def proposal
22
19
  @proposed_options = options.dup
23
20
  @proposed_options.delete('compress')
24
- @proposed_options['encoding']='lz4'
21
+ @proposed_options['encoding'] = 'lz4'
25
22
  self
26
23
  end
27
-
28
24
  end
29
-
30
25
  end
@@ -1,35 +1,31 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- return proposal if lz4_source?
8
+ return proposal if lz4_source?
11
9
  end
12
10
  debug "#{self.class.name}: not appropriate"
13
11
  false
14
12
  end
15
13
 
16
14
  def command
17
- "lz4 -d |"
15
+ 'lz4 -d |'
18
16
  end
19
17
 
20
18
  def proposal
21
19
  @proposed_options = options.dup
22
20
  @proposed_options.delete('source-format')
23
21
  if @proposed_options['target-format']
24
- @proposed_options['source-format'] = :raw
25
- if @proposed_options['source-format'] == @proposed_options['target-format']
26
- @proposed_options.delete('source-format')
27
- @proposed_options.delete('target-format')
28
- end
22
+ @proposed_options['source-format'] = :raw
23
+ if @proposed_options['source-format'] == @proposed_options['target-format']
24
+ @proposed_options.delete('source-format')
25
+ @proposed_options.delete('target-format')
26
+ end
29
27
  end
30
28
  self
31
29
  end
32
-
33
30
  end
34
-
35
31
  end
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
@@ -24,7 +22,5 @@ module Uricp::Strategy
24
22
  @proposed_options['from_uri'] = PIPE_URI
25
23
  self
26
24
  end
27
-
28
25
  end
29
-
30
26
  end
@@ -1,19 +1,15 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- if raw_target? && lz4_source? && to.scheme == 'file'
11
- if always_write_sparse?
12
- return proposal
13
- else
14
- debug "#{self.class.name}: using safe sparse expansion via stream"
15
- end
16
- end
8
+ if raw_target? && lz4_source? && to.scheme == 'file'
9
+ return proposal if always_write_sparse?
10
+
11
+ debug "#{self.class.name}: using safe sparse expansion via stream"
12
+ end
17
13
  end
18
14
  debug "#{self.class.name}: not appropriate"
19
15
  false
@@ -27,16 +23,14 @@ module Uricp::Strategy
27
23
  @proposed_options = options.dup
28
24
  @proposed_options.delete('source-format')
29
25
  if @proposed_options['target-format']
30
- @proposed_options['source-format'] = :raw
31
- if @proposed_options['source-format'] == @proposed_options['target-format']
32
- @proposed_options.delete('source-format')
33
- @proposed_options.delete('target-format')
34
- end
26
+ @proposed_options['source-format'] = :raw
27
+ if @proposed_options['source-format'] == @proposed_options['target-format']
28
+ @proposed_options.delete('source-format')
29
+ @proposed_options.delete('target-format')
30
+ end
35
31
  end
36
32
  @proposed_options['from_uri'] = @proposed_options['to_uri']
37
33
  self
38
34
  end
39
-
40
35
  end
41
-
42
36
  end