uricp 0.0.15 → 0.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +3 -0
  3. data/Gemfile.lock +16 -21
  4. data/Jenkinsfile +90 -33
  5. data/Rakefile +24 -36
  6. data/almalinux8/Dockerfile +26 -0
  7. data/bin/uricp +25 -11
  8. data/bionic/Dockerfile +4 -3
  9. data/centos7/Dockerfile +17 -12
  10. data/cucumber.yml +1 -0
  11. data/features/check_uri_path.feature +14 -0
  12. data/features/rbd_access.feature +226 -0
  13. data/features/step_definitions/orbit_steps.rb +11 -4
  14. data/features/step_definitions/rbd_steps.rb +8 -0
  15. data/features/step_definitions/uricp_steps.rb +8 -4
  16. data/focal/Dockerfile +28 -0
  17. data/lib/segment_upload.rb +20 -22
  18. data/lib/uricp/curl_primitives.rb +31 -33
  19. data/lib/uricp/orbit_auth.rb +23 -27
  20. data/lib/uricp/segmenter.rb +12 -16
  21. data/lib/uricp/strategy/cache_common.rb +30 -12
  22. data/lib/uricp/strategy/cached_get.rb +19 -22
  23. data/lib/uricp/strategy/cleaner.rb +2 -8
  24. data/lib/uricp/strategy/common.rb +76 -18
  25. data/lib/uricp/strategy/local_convert.rb +10 -17
  26. data/lib/uricp/strategy/local_link.rb +10 -8
  27. data/lib/uricp/strategy/piped_cache.rb +11 -16
  28. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  29. data/lib/uricp/strategy/piped_compress.rb +3 -8
  30. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  31. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  32. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  33. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  34. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  35. data/lib/uricp/strategy/piped_rbd_get.rb +30 -0
  36. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  37. data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
  38. data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
  39. data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
  40. data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
  41. data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
  42. data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
  43. data/lib/uricp/strategy/rbd_flattener.rb +23 -0
  44. data/lib/uricp/strategy/rbd_put.rb +38 -0
  45. data/lib/uricp/strategy/rbd_snap.rb +56 -0
  46. data/lib/uricp/strategy/rbd_sweeper.rb +23 -0
  47. data/lib/uricp/strategy/remote_put.rb +11 -17
  48. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  49. data/lib/uricp/strategy/sweeper.rb +2 -8
  50. data/lib/uricp/uri_strategy.rb +22 -13
  51. data/lib/uricp/version.rb +4 -2
  52. data/lib/uricp.rb +27 -19
  53. data/uricp.gemspec +3 -3
  54. data/xenial/Dockerfile +4 -3
  55. metadata +41 -25
  56. data/spec/something_spec.rb +0 -5
  57. data/trusty/Dockerfile +0 -20
@@ -0,0 +1,27 @@
1
+ module Uricp::Strategy
2
+ class RbdCacheBaseSnap
3
+ include Uricp::Strategy::Common
4
+ include Methadone::SH
5
+
6
+ def appropriate?
7
+ if rbd_cache_name &&
8
+ rbd_image_spec(from) == rbd_cache_name
9
+ return proposal
10
+ end
11
+ debug "#{self.class.name}: not appropriate"
12
+ false
13
+ end
14
+
15
+ def command
16
+ "rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}' && "\
17
+ "rbd snap protect --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"
18
+ end
19
+
20
+ def proposal
21
+ @proposed_options = options.dup
22
+ @proposed_options['from_uri'] = rbd_uri(rbd_clone_snapshot(rbd_cache_name))
23
+ @proposed_options.delete('rbd_cache_name')
24
+ self
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,42 @@
1
+ require 'uri'
2
+
3
+ module Uricp::Strategy
4
+ class RbdCacheCheck
5
+ include Uricp::Strategy::Common
6
+ include Uricp::Strategy::CacheCommon
7
+ include Methadone::SH
8
+
9
+ def appropriate?
10
+ unless (from.scheme == 'rbd') != (to.scheme == 'rbd')
11
+ debug "#{self.class.name}: not appropriate"
12
+ return false
13
+ end
14
+
15
+ with_active_cache do
16
+ if from.scheme == 'rbd'
17
+ cache_check = in_rbd_cache(rbd_cache_image_spec(from))
18
+ return proposal(cache_check) if cache_check
19
+ debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
20
+ elsif to.scheme == 'rbd'
21
+ cache_check = in_rbd_cache(rbd_cache_image_spec(to))
22
+ return proposal(cache_check) if cache_check
23
+ debug "#{self.class.name}: no rbd cache entry for #{options['to_uri']}"
24
+ end
25
+ false
26
+ end
27
+ end
28
+
29
+ def command
30
+ ':;'
31
+ end
32
+
33
+ def proposal(cache_check)
34
+ @proposed_options = options.dup
35
+ @proposed_options['from_uri'] = rbd_uri(cache_check)
36
+ @proposed_options.delete('cache')
37
+ @proposed_options.delete('cache_name')
38
+ @proposed_options.delete('dry-cache')
39
+ self
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,40 @@
1
+ require 'uri'
2
+
3
+ module Uricp::Strategy
4
+ class RbdCacheClone
5
+ include Uricp::Strategy::Common
6
+ include Uricp::Strategy::CacheCommon
7
+ include Methadone::SH
8
+
9
+ def appropriate?
10
+ unless from.scheme == 'rbd'
11
+ debug "#{self.class.name}: not appropriate"
12
+ return false
13
+ end
14
+
15
+ with_active_cache do
16
+ options['cache_name'] = File.basename(options['to_uri'].path)
17
+ cache_target = rbd_cache_image_spec(from)
18
+ cache_check = in_rbd_cache(rbd_cache_image_spec(from))
19
+ if cache_check
20
+ debug "#{self.class.name}: Unexpected existing cache entry for #{options['to_uri']}"
21
+ unsupported_transfer
22
+ end
23
+ proposal(cache_target)
24
+ end
25
+ end
26
+
27
+ def command
28
+ ':;'
29
+ end
30
+
31
+ def proposal(cache_check)
32
+ @proposed_options = options.dup
33
+ @proposed_options['rbd_cache_target'] = rbd_uri(cache_check)
34
+ @proposed_options.delete('cache')
35
+ @proposed_options.delete('cache_name')
36
+ @proposed_options.delete('dry-cache')
37
+ self
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,40 @@
1
+ module Uricp::Strategy
2
+ class RbdCacheUpload
3
+ include Uricp::Strategy::Common
4
+ include Methadone::SH
5
+
6
+ def appropriate?
7
+ if compression_required? || conversion_required?
8
+ debug "#{self.class.name}: not ready to upload"
9
+ return false
10
+ end
11
+ if rbd_cache_name
12
+ case from.scheme
13
+ when 'pipe', 'file'
14
+ return proposal if to.scheme == 'rbd'
15
+ end
16
+ end
17
+ debug "#{self.class.name}: not appropriate"
18
+ false
19
+ end
20
+
21
+ def command
22
+ "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}' && "
23
+ end
24
+
25
+ def proposal
26
+ @proposed_options = options.dup
27
+ @proposed_options['from_uri'] = rbd_uri(rbd_cache_name)
28
+ self
29
+ end
30
+
31
+ def data_source
32
+ if from.scheme == 'pipe'
33
+ '-'
34
+ else
35
+ "'#{from.path}'"
36
+ end
37
+ end
38
+
39
+ end
40
+ end
@@ -0,0 +1,36 @@
1
+ require 'uri'
2
+
3
+ module Uricp::Strategy
4
+ class RbdCachedGet
5
+ include Uricp::Strategy::Common
6
+ include Uricp::Strategy::CacheCommon
7
+ include Methadone::SH
8
+
9
+ def appropriate?
10
+ if from.scheme != 'rbd' || rbd_snapshot_spec?(from)
11
+ debug "#{self.class.name}: not appropriate"
12
+ return false
13
+ end
14
+
15
+ with_active_cache do
16
+ cache_check = in_rbd_cache(rbd_image_spec(from))
17
+ return proposal(cache_check) if cache_check
18
+ debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
19
+ false
20
+ end
21
+ end
22
+
23
+ def command
24
+ ':;'
25
+ end
26
+
27
+ def proposal(cache_check)
28
+ @proposed_options = options.dup
29
+ @proposed_options['from_uri'] = rbd_uri(cache_check)
30
+ @proposed_options.delete('cache')
31
+ @proposed_options.delete('cache_name')
32
+ @proposed_options.delete('dry-cache')
33
+ self
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,33 @@
1
+ module Uricp::Strategy
2
+ class RbdCachedPut
3
+ include Uricp::Strategy::Common
4
+ include Uricp::Strategy::CacheCommon
5
+
6
+ def appropriate?
7
+ return proposal if to.scheme == 'rbd' &&
8
+ rbd_snapshot_spec?(from) &&
9
+ rbd_cache_name.nil? &&
10
+ !rbd_sequence_complete?
11
+
12
+ debug "#{self.class.name}: not appropriate"
13
+ false
14
+ end
15
+
16
+ def command
17
+ "rbd clone --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
18
+ end
19
+
20
+ def proposal
21
+ @proposed_options = options.dup
22
+ @proposed_options['from_uri'] = to
23
+ if options['rbd_cache_target']
24
+ @proposed_options['to_uri'] = options['rbd_cache_target']
25
+ @proposed_options['rbd_cache_name'] = rbd_image_spec(to)
26
+ @proposed_options['rbd_flatten'] ||= @proposed_options['rbd_cache_name']
27
+ @proposed_options.delete('rbd_cache_target')
28
+ end
29
+ self
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,23 @@
1
+ module Uricp::Strategy
2
+ class RbdFlattener
3
+ include Uricp::Strategy::Common
4
+
5
+ def appropriate?
6
+ return proposal if options['rbd_flatten'] &&
7
+ rbd_sequence_complete?
8
+
9
+ debug "#{self.class.name}: not appropriate"
10
+ false
11
+ end
12
+
13
+ def command
14
+ "rbd flatten --id #{rbd_id} --no-progress '#{options['rbd_flatten']}' && "
15
+ end
16
+
17
+ def proposal
18
+ @proposed_options = options.dup
19
+ @proposed_options.delete('rbd_flatten')
20
+ self
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,38 @@
1
+ module Uricp::Strategy
2
+ class RbdPut
3
+ include Uricp::Strategy::Common
4
+
5
+ def appropriate?
6
+ if compression_required? || conversion_required?
7
+ debug "#{self.class.name}: not ready to upload"
8
+ return false
9
+ end
10
+ unless rbd_cache_name
11
+ case from.scheme
12
+ when 'pipe', 'file'
13
+ return proposal if to.scheme == 'rbd'
14
+ end
15
+ end
16
+ debug "#{self.class.name}: not appropriate"
17
+ false
18
+ end
19
+
20
+ def command
21
+ "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_image_spec(to)}';"
22
+ end
23
+
24
+ def proposal
25
+ @proposed_options = options.dup
26
+ @proposed_options['from_uri'] = to
27
+ self
28
+ end
29
+
30
+ def data_source
31
+ if from.scheme == 'pipe'
32
+ '-'
33
+ else
34
+ "'#{from.path}'"
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,56 @@
1
+ require 'json'
2
+ module Uricp::Strategy
3
+ class RbdSnap
4
+ include Uricp::Strategy::Common
5
+ include Uricp::Strategy::CacheCommon
6
+ include Methadone::SH
7
+
8
+ def appropriate?
9
+ without_active_cache do
10
+ if from.scheme == 'rbd' &&
11
+ options['rbd_snapshot'].nil? &&
12
+ !rbd_snapshot_spec?(from) &&
13
+ (rbd_cache_name.nil? ||
14
+ !from.path.include?(rbd_cache_name))
15
+ if snap_in_progress?
16
+ debug "#{self.class.name}: detected snapshot in progress"
17
+ else
18
+ return proposal unless sequence_complete?
19
+ end
20
+ end
21
+ end
22
+ debug "#{self.class.name}: not appropriate"
23
+ false
24
+ end
25
+
26
+ def command
27
+ "rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
28
+ "rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
29
+ end
30
+
31
+ def proposal
32
+ @proposed_options = options.dup
33
+ @proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from)
34
+ @proposed_options['from_uri'] = rbd_uri(@proposed_options['rbd_snapshot'])
35
+ if options['rbd_cache_target']
36
+ @proposed_options['to_uri'] = options['rbd_cache_target']
37
+ @proposed_options['rbd_cache_target'] = to
38
+ end
39
+ self
40
+ end
41
+
42
+ def rbd_upload_snapshot(uri)
43
+ "#{rbd_image_spec(uri)}@#{rbd_snapshot_name}"
44
+ end
45
+
46
+ def snap_in_progress?
47
+ return false if dry_run?
48
+
49
+ result = false
50
+ sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)}" do |stdout|
51
+ result = JSON.parse(stdout).any? { |x| x['name'] == rbd_snapshot_name }
52
+ end
53
+ result
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,23 @@
1
+ module Uricp::Strategy
2
+ class RbdSweeper
3
+ include Uricp::Strategy::Common
4
+
5
+ def appropriate?
6
+ return proposal if options['rbd_snapshot'] && rbd_sequence_complete?
7
+
8
+ debug "#{self.class.name}: not appropriate"
9
+ false
10
+ end
11
+
12
+ def command
13
+ "rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}' && " \
14
+ "rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
15
+ end
16
+
17
+ def proposal
18
+ @proposed_options = options.dup
19
+ @proposed_options.delete('rbd_snapshot')
20
+ self
21
+ end
22
+ end
23
+ end
@@ -1,26 +1,22 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class RemotePut
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
6
  if compression_required? || conversion_required?
9
7
  debug "#{self.class.name}: not ready to upload"
10
- else
11
- case to.scheme
12
- when 'http', 'https'
13
- case from.scheme
14
- when 'file', 'pipe'
15
- if http_authentication?
16
- return proposal
17
- else
18
- unsupported_transfer
19
- end
20
- end
21
- end
22
- debug "#{self.class.name}: not appropriate"
8
+ return false
9
+ end
10
+ case to.scheme
11
+ when 'http', 'https'
12
+ case from.scheme
13
+ when 'file', 'pipe'
14
+ return proposal if http_authentication?
15
+
16
+ unsupported_transfer
17
+ end
23
18
  end
19
+ debug "#{self.class.name}: not appropriate"
24
20
  false
25
21
  end
26
22
 
@@ -41,7 +37,5 @@ module Uricp::Strategy
41
37
  from.path
42
38
  end
43
39
  end
44
-
45
40
  end
46
-
47
41
  end
@@ -1,35 +1,29 @@
1
1
  module Uricp::Strategy
2
-
3
2
  class SegmentedRemotePut
4
-
5
3
  include Uricp::Strategy::Common
6
4
 
7
5
  def appropriate?
8
- if segmented?
9
- if compression_required? || conversion_required?
10
- debug "#{self.class.name}: not ready to upload"
11
- else
12
- case to.scheme
13
- when 'http', 'https'
14
- case from.scheme
15
- when 'file', 'pipe'
16
- if http_authentication?
17
- return proposal
18
- else
19
- unsupported_transfer
20
- end
21
- end
22
- end
23
- debug "#{self.class.name}: not appropriate"
24
- end
25
- else
26
- debug "#{self.class.name}: no segmentation requested"
6
+ unless segmented?
7
+ debug "#{self.class.name}: not appropriate"
8
+ return false
9
+ end
10
+ if compression_required? || conversion_required?
11
+ debug "#{self.class.name}: not ready to upload"
12
+ return false
27
13
  end
14
+ case to.scheme
15
+ when 'http', 'https'
16
+ case from.scheme
17
+ when 'file', 'pipe'
18
+ return proposal if http_authentication?
19
+ end
20
+ end
21
+ unsupported_transfer
28
22
  false
29
23
  end
30
24
 
31
25
  def command
32
- "segment_upload --segment-size '#{options['segment-size']}' #{source_details} #{to.to_s};"
26
+ "segment_upload --segment-size '#{options['segment-size']}' #{source_details} #{to};"
33
27
  end
34
28
 
35
29
  def proposal
@@ -42,11 +36,5 @@ module Uricp::Strategy
42
36
  def source_details
43
37
  "--from #{from.path}" unless from.scheme == 'pipe'
44
38
  end
45
-
46
- def segmented?
47
- options['segment-size']
48
- end
49
-
50
39
  end
51
-
52
40
  end