uricp 0.0.11 → 0.0.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -1
  3. data/.rubocop.yml +3 -0
  4. data/Gemfile.lock +15 -19
  5. data/Jenkinsfile +113 -0
  6. data/README.md +2 -2
  7. data/Rakefile +26 -31
  8. data/bionic/Dockerfile +20 -0
  9. data/centos7/Dockerfile +20 -0
  10. data/features/auth_download.feature +17 -0
  11. data/features/step_definitions/orbit_steps.rb +19 -13
  12. data/lib/segment_upload.rb +20 -22
  13. data/lib/uricp.rb +3 -29
  14. data/lib/uricp/curl_primitives.rb +31 -33
  15. data/lib/uricp/orbit_auth.rb +23 -27
  16. data/lib/uricp/segmenter.rb +12 -16
  17. data/lib/uricp/strategy/cache_common.rb +11 -11
  18. data/lib/uricp/strategy/cached_get.rb +14 -20
  19. data/lib/uricp/strategy/cleaner.rb +2 -8
  20. data/lib/uricp/strategy/common.rb +27 -19
  21. data/lib/uricp/strategy/local_convert.rb +9 -16
  22. data/lib/uricp/strategy/local_link.rb +10 -8
  23. data/lib/uricp/strategy/piped_cache.rb +7 -13
  24. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  25. data/lib/uricp/strategy/piped_compress.rb +3 -8
  26. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  27. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  28. data/lib/uricp/strategy/piped_local_decompress.rb +11 -17
  29. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  30. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  31. data/lib/uricp/strategy/piped_rbd_get.rb +44 -0
  32. data/lib/uricp/strategy/piped_remote_get.rb +28 -19
  33. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  34. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  35. data/lib/uricp/strategy/remote_put.rb +11 -17
  36. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  37. data/lib/uricp/strategy/sweeper.rb +2 -8
  38. data/lib/uricp/uri_strategy.rb +14 -13
  39. data/lib/uricp/version.rb +2 -2
  40. data/trusty/Dockerfile +20 -0
  41. data/uricp.gemspec +3 -4
  42. data/xenial/Dockerfile +20 -0
  43. metadata +39 -51
  44. data/Dockerfile_centos6.6 +0 -39
  45. data/Dockerfile_centos7 +0 -39
  46. data/Dockerfile_trusty-ruby193 +0 -32
  47. data/README.rdoc +0 -23
  48. data/spec/something_spec.rb +0 -5
@@ -1,13 +1,10 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class Cleaner
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- if options['clean'] && sequence_complete?
9
- return proposal
10
- end
6
+ return proposal if options['clean'] && sequence_complete?
7
+
11
8
  debug "#{self.class.name}: not appropriate"
12
9
  false
13
10
  end
@@ -21,8 +18,5 @@ module Uricp::Strategy
21
18
  @proposed_options.delete('clean')
22
19
  self
23
20
  end
24
-
25
21
  end
26
-
27
22
  end
28
-
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  module Common
4
-
5
3
  include Methadone::CLILogging
6
4
  include Uricp::CurlPrimitives
7
5
 
@@ -14,10 +12,10 @@ module Uricp::Strategy
14
12
 
15
13
  def unsupported_transfer
16
14
  raise Uricp::UnsupportedURLtype,
17
- "Unsupported transfer from #{from.to_s} to #{to.to_s}"
15
+ "Unsupported transfer from #{from} to #{to}"
18
16
  end
19
17
 
20
- alias :command :unsupported_transfer
18
+ alias command unsupported_transfer
21
19
 
22
20
  def all_local_files?
23
21
  !sequence_complete? && file_source? && to.scheme == 'file'
@@ -44,24 +42,23 @@ module Uricp::Strategy
44
42
  end
45
43
 
46
44
  def qcow2?(magic)
47
- magic.unpack('a3C') == ['QFI',0xfb]
45
+ magic.unpack('a3C') == ['QFI', 0xfb]
48
46
  end
49
47
 
50
48
  def encoding(io)
51
49
  magic = io.read(4).to_s
52
- case
53
- when lz4?(magic)
50
+ if lz4?(magic)
54
51
  :lz4
55
- when qcow2?(magic)
52
+ elsif qcow2?(magic)
56
53
  version = io.read(4)
57
- case version.unpack('N')
58
- when [2]
59
- :qcow2
60
- when [3]
61
- :qcow3
62
- else
63
- :qcow2un
64
- end
54
+ case version.unpack('N')
55
+ when [2]
56
+ :qcow2
57
+ when [3]
58
+ :qcow3
59
+ else
60
+ :qcow2un
61
+ end
65
62
  else
66
63
  :raw
67
64
  end
@@ -73,6 +70,10 @@ module Uricp::Strategy
73
70
  )
74
71
  end
75
72
 
73
+ def dry_run?
74
+ options['dry-run']
75
+ end
76
+
76
77
  def lz4_source?
77
78
  options['source-format'] == :lz4
78
79
  end
@@ -91,8 +92,9 @@ module Uricp::Strategy
91
92
 
92
93
  PIPE_URI = URI('pipe:/')
93
94
 
94
- def get_temp_filename(filename)
95
- Dir::Tmpname.make_tmpname(filename, nil)
95
+ def get_temp_filename(base_dir)
96
+ t = Time.now.strftime('%Y%m%d')
97
+ File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
96
98
  end
97
99
 
98
100
  def proposed_path
@@ -107,6 +109,12 @@ module Uricp::Strategy
107
109
  options['source-format'] && !lz4_source?
108
110
  end
109
111
 
110
- end
112
+ def rbd_target(uri)
113
+ uri.path[1..-1]
114
+ end
111
115
 
116
+ def rbd_id
117
+ 'libvirt'
118
+ end
119
+ end
112
120
  end
@@ -1,17 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalConvert
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if conversion_required? && file_source? && supported_source?
9
- if new_qemu? || supported_conversion?
10
- return proposal
11
- else
12
- raise Uricp::UnsupportedConversion,
13
- "qemu-img does not support required conversion"
14
- end
7
+ return proposal if new_qemu? || supported_conversion?
8
+
9
+ raise Uricp::UnsupportedConversion,
10
+ 'qemu-img does not support required conversion'
15
11
  end
16
12
  debug "#{self.class.name}: not appropriate"
17
13
  false
@@ -24,11 +20,11 @@ module Uricp::Strategy
24
20
  def proposal
25
21
  @proposed_options = options.dup
26
22
  if to.scheme == 'file'
27
- @proposed_options['from_uri'] = @proposed_options['to_uri']
23
+ @proposed_options['from_uri'] = @proposed_options['to_uri']
28
24
  else
29
25
  @proposed_options['from_uri'] = temp_uri
30
- @proposed_options['clean'] ||= []
31
- @proposed_options['clean'] << proposed_path
26
+ @proposed_options['clean'] ||= []
27
+ @proposed_options['clean'] << proposed_path
32
28
  end
33
29
  @proposed_options.delete('source-format')
34
30
  @proposed_options.delete('target-format')
@@ -40,7 +36,7 @@ module Uricp::Strategy
40
36
  end
41
37
 
42
38
  def supported_conversion?
43
- ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty?
39
+ ([:qcow3, :qcow2v3] & [options['source-format'], options['target-format']]).empty? # rubocop:disable Style/SymbolArray
44
40
  end
45
41
 
46
42
  def qemu_img_command(target)
@@ -59,14 +55,11 @@ module Uricp::Strategy
59
55
  end
60
56
 
61
57
  def compat_option
62
- "-o compat=0.10" if new_qemu?
58
+ '-o compat=0.10' if new_qemu?
63
59
  end
64
60
 
65
61
  def compress_option
66
62
  options['compress'] && '-c'
67
63
  end
68
-
69
64
  end
70
-
71
65
  end
72
-
@@ -1,11 +1,13 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class LocalLink
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- return proposal if all_local_files? && !format_change? && linkable?
6
+ return proposal if !compression_required? &&
7
+ all_local_files? &&
8
+ !format_change? &&
9
+ linkable?
10
+
9
11
  debug "#{self.class.name}: not appropriate"
10
12
  false
11
13
  end
@@ -21,11 +23,11 @@ module Uricp::Strategy
21
23
  end
22
24
 
23
25
  def linkable?
24
- File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
25
- (!File.exist?(to.path) || File.file?(to.path))
26
+ from_path_dir = File.dirname(from.path)
27
+ to_path_dir = File.dirname(to.path)
28
+ File.directory?(from_path_dir) && File.directory?(to_path_dir) &&
29
+ File.stat(from_path_dir).dev == File.stat(to_path_dir).dev &&
30
+ (!File.exist?(to.path) || File.file?(to.path))
26
31
  end
27
-
28
32
  end
29
-
30
33
  end
31
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCache
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
9
  if cache_root
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache!
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache!
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no cacheing requested"
17
+ debug "#{self.class.name}: no cacheing requested"
20
18
  end
21
19
  false
22
20
  end
@@ -32,9 +30,5 @@ module Uricp::Strategy
32
30
  @proposed_options.delete('cache_name')
33
31
  self
34
32
  end
35
-
36
33
  end
37
-
38
34
  end
39
-
40
-
@@ -1,22 +1,20 @@
1
1
  require 'fileutils'
2
2
 
3
3
  module Uricp::Strategy
4
-
5
4
  class PipedCacheConvert
6
-
7
5
  include Uricp::Strategy::Common
8
6
  include Uricp::Strategy::CacheCommon
9
7
 
10
8
  def appropriate?
11
- if conversion_required? && supported_source?
12
- case from.scheme
13
- when 'pipe'
14
- validate_cache! if cache_root
15
- return proposal
16
- end
17
- debug "#{self.class.name}: not appropriate"
9
+ if conversion_required? && supported_source?
10
+ case from.scheme
11
+ when 'pipe'
12
+ validate_cache! if cache_root
13
+ return proposal
14
+ end
15
+ debug "#{self.class.name}: not appropriate"
18
16
  else
19
- debug "#{self.class.name}: no non-stream conversion detected"
17
+ debug "#{self.class.name}: no non-stream conversion detected"
20
18
  end
21
19
  false
22
20
  end
@@ -28,18 +26,16 @@ module Uricp::Strategy
28
26
  def proposal
29
27
  @proposed_options = options.dup
30
28
  if cache_root
31
- @proposed_options['from_uri'] = temp_cache_uri
32
- @proposed_options['sweep'] = [temp_cache_file, cache_file]
33
- @proposed_options.delete('cache')
34
- @proposed_options.delete('cache_name')
29
+ @proposed_options['from_uri'] = temp_cache_uri
30
+ @proposed_options['sweep'] = [temp_cache_file, cache_file]
31
+ @proposed_options.delete('cache')
32
+ @proposed_options.delete('cache_name')
35
33
  else
36
34
  @proposed_options['from_uri'] = temp_uri
37
- @proposed_options['clean'] ||= []
38
- @proposed_options['clean'] << proposed_path
35
+ @proposed_options['clean'] ||= []
36
+ @proposed_options['clean'] << proposed_path
39
37
  end
40
38
  self
41
39
  end
42
-
43
40
  end
44
-
45
41
  end
@@ -1,30 +1,25 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
-
8
5
  def appropriate?
9
6
  case from.scheme
10
7
  when 'pipe'
11
- return proposal if compression_required?
8
+ return proposal if compression_required?
12
9
  end
13
10
  debug "#{self.class.name}: not appropriate"
14
11
  false
15
12
  end
16
13
 
17
14
  def command
18
- "lz4 |"
15
+ 'lz4 |'
19
16
  end
20
17
 
21
18
  def proposal
22
19
  @proposed_options = options.dup
23
20
  @proposed_options.delete('compress')
24
- @proposed_options['encoding']='lz4'
21
+ @proposed_options['encoding'] = 'lz4'
25
22
  self
26
23
  end
27
-
28
24
  end
29
-
30
25
  end
@@ -1,35 +1,31 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- return proposal if lz4_source?
8
+ return proposal if lz4_source?
11
9
  end
12
10
  debug "#{self.class.name}: not appropriate"
13
11
  false
14
12
  end
15
13
 
16
14
  def command
17
- "lz4 -d |"
15
+ 'lz4 -d |'
18
16
  end
19
17
 
20
18
  def proposal
21
19
  @proposed_options = options.dup
22
20
  @proposed_options.delete('source-format')
23
21
  if @proposed_options['target-format']
24
- @proposed_options['source-format'] = :raw
25
- if @proposed_options['source-format'] == @proposed_options['target-format']
26
- @proposed_options.delete('source-format')
27
- @proposed_options.delete('target-format')
28
- end
22
+ @proposed_options['source-format'] = :raw
23
+ if @proposed_options['source-format'] == @proposed_options['target-format']
24
+ @proposed_options.delete('source-format')
25
+ @proposed_options.delete('target-format')
26
+ end
29
27
  end
30
28
  self
31
29
  end
32
-
33
30
  end
34
-
35
31
  end
@@ -1,7 +1,5 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalCompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
@@ -24,7 +22,5 @@ module Uricp::Strategy
24
22
  @proposed_options['from_uri'] = PIPE_URI
25
23
  self
26
24
  end
27
-
28
25
  end
29
-
30
26
  end
@@ -1,42 +1,36 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class PipedLocalDecompress
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  case from.scheme
9
7
  when 'pipe'
10
- if raw_target? && lz4_source? && to.scheme == 'file'
11
- if always_write_sparse?
12
- return proposal
13
- else
14
- debug "#{self.class.name}: using safe sparse expansion via stream"
15
- end
16
- end
8
+ if raw_target? && lz4_source? && to.scheme == 'file'
9
+ return proposal if always_write_sparse?
10
+
11
+ debug "#{self.class.name}: using safe sparse expansion via stream"
12
+ end
17
13
  end
18
14
  debug "#{self.class.name}: not appropriate"
19
15
  false
20
16
  end
21
17
 
22
18
  def command
23
- "lz4 -qd - #{to.path};"
19
+ "lz4 -qdf - #{to.path};"
24
20
  end
25
21
 
26
22
  def proposal
27
23
  @proposed_options = options.dup
28
24
  @proposed_options.delete('source-format')
29
25
  if @proposed_options['target-format']
30
- @proposed_options['source-format'] = :raw
31
- if @proposed_options['source-format'] == @proposed_options['target-format']
32
- @proposed_options.delete('source-format')
33
- @proposed_options.delete('target-format')
34
- end
26
+ @proposed_options['source-format'] = :raw
27
+ if @proposed_options['source-format'] == @proposed_options['target-format']
28
+ @proposed_options.delete('source-format')
29
+ @proposed_options.delete('target-format')
30
+ end
35
31
  end
36
32
  @proposed_options['from_uri'] = @proposed_options['to_uri']
37
33
  self
38
34
  end
39
-
40
35
  end
41
-
42
36
  end