uricp 0.0.15 → 0.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +16 -21
- data/Jenkinsfile +90 -33
- data/Rakefile +24 -36
- data/almalinux8/Dockerfile +26 -0
- data/bin/uricp +25 -11
- data/bionic/Dockerfile +4 -3
- data/centos7/Dockerfile +17 -12
- data/cucumber.yml +1 -0
- data/features/check_uri_path.feature +14 -0
- data/features/rbd_access.feature +226 -0
- data/features/step_definitions/orbit_steps.rb +11 -4
- data/features/step_definitions/rbd_steps.rb +8 -0
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +28 -0
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +30 -12
- data/lib/uricp/strategy/cached_get.rb +19 -22
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +76 -18
- data/lib/uricp/strategy/local_convert.rb +10 -17
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +11 -16
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +30 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
- data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
- data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
- data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
- data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
- data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
- data/lib/uricp/strategy/rbd_flattener.rb +23 -0
- data/lib/uricp/strategy/rbd_put.rb +38 -0
- data/lib/uricp/strategy/rbd_snap.rb +56 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +23 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +22 -13
- data/lib/uricp/version.rb +4 -2
- data/lib/uricp.rb +27 -19
- data/uricp.gemspec +3 -3
- data/xenial/Dockerfile +4 -3
- metadata +41 -25
- data/spec/something_spec.rb +0 -5
- data/trusty/Dockerfile +0 -20
@@ -0,0 +1,27 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdCacheBaseSnap
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
include Methadone::SH
|
5
|
+
|
6
|
+
def appropriate?
|
7
|
+
if rbd_cache_name &&
|
8
|
+
rbd_image_spec(from) == rbd_cache_name
|
9
|
+
return proposal
|
10
|
+
end
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
false
|
13
|
+
end
|
14
|
+
|
15
|
+
def command
|
16
|
+
"rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}' && "\
|
17
|
+
"rbd snap protect --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"
|
18
|
+
end
|
19
|
+
|
20
|
+
def proposal
|
21
|
+
@proposed_options = options.dup
|
22
|
+
@proposed_options['from_uri'] = rbd_uri(rbd_clone_snapshot(rbd_cache_name))
|
23
|
+
@proposed_options.delete('rbd_cache_name')
|
24
|
+
self
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Uricp::Strategy
|
4
|
+
class RbdCacheCheck
|
5
|
+
include Uricp::Strategy::Common
|
6
|
+
include Uricp::Strategy::CacheCommon
|
7
|
+
include Methadone::SH
|
8
|
+
|
9
|
+
def appropriate?
|
10
|
+
unless (from.scheme == 'rbd') != (to.scheme == 'rbd')
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
return false
|
13
|
+
end
|
14
|
+
|
15
|
+
with_active_cache do
|
16
|
+
if from.scheme == 'rbd'
|
17
|
+
cache_check = in_rbd_cache(rbd_cache_image_spec(from))
|
18
|
+
return proposal(cache_check) if cache_check
|
19
|
+
debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
|
20
|
+
elsif to.scheme == 'rbd'
|
21
|
+
cache_check = in_rbd_cache(rbd_cache_image_spec(to))
|
22
|
+
return proposal(cache_check) if cache_check
|
23
|
+
debug "#{self.class.name}: no rbd cache entry for #{options['to_uri']}"
|
24
|
+
end
|
25
|
+
false
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def command
|
30
|
+
':;'
|
31
|
+
end
|
32
|
+
|
33
|
+
def proposal(cache_check)
|
34
|
+
@proposed_options = options.dup
|
35
|
+
@proposed_options['from_uri'] = rbd_uri(cache_check)
|
36
|
+
@proposed_options.delete('cache')
|
37
|
+
@proposed_options.delete('cache_name')
|
38
|
+
@proposed_options.delete('dry-cache')
|
39
|
+
self
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Uricp::Strategy
|
4
|
+
class RbdCacheClone
|
5
|
+
include Uricp::Strategy::Common
|
6
|
+
include Uricp::Strategy::CacheCommon
|
7
|
+
include Methadone::SH
|
8
|
+
|
9
|
+
def appropriate?
|
10
|
+
unless from.scheme == 'rbd'
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
return false
|
13
|
+
end
|
14
|
+
|
15
|
+
with_active_cache do
|
16
|
+
options['cache_name'] = File.basename(options['to_uri'].path)
|
17
|
+
cache_target = rbd_cache_image_spec(from)
|
18
|
+
cache_check = in_rbd_cache(rbd_cache_image_spec(from))
|
19
|
+
if cache_check
|
20
|
+
debug "#{self.class.name}: Unexpected existing cache entry for #{options['to_uri']}"
|
21
|
+
unsupported_transfer
|
22
|
+
end
|
23
|
+
proposal(cache_target)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def command
|
28
|
+
':;'
|
29
|
+
end
|
30
|
+
|
31
|
+
def proposal(cache_check)
|
32
|
+
@proposed_options = options.dup
|
33
|
+
@proposed_options['rbd_cache_target'] = rbd_uri(cache_check)
|
34
|
+
@proposed_options.delete('cache')
|
35
|
+
@proposed_options.delete('cache_name')
|
36
|
+
@proposed_options.delete('dry-cache')
|
37
|
+
self
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdCacheUpload
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
include Methadone::SH
|
5
|
+
|
6
|
+
def appropriate?
|
7
|
+
if compression_required? || conversion_required?
|
8
|
+
debug "#{self.class.name}: not ready to upload"
|
9
|
+
return false
|
10
|
+
end
|
11
|
+
if rbd_cache_name
|
12
|
+
case from.scheme
|
13
|
+
when 'pipe', 'file'
|
14
|
+
return proposal if to.scheme == 'rbd'
|
15
|
+
end
|
16
|
+
end
|
17
|
+
debug "#{self.class.name}: not appropriate"
|
18
|
+
false
|
19
|
+
end
|
20
|
+
|
21
|
+
def command
|
22
|
+
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}' && "
|
23
|
+
end
|
24
|
+
|
25
|
+
def proposal
|
26
|
+
@proposed_options = options.dup
|
27
|
+
@proposed_options['from_uri'] = rbd_uri(rbd_cache_name)
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def data_source
|
32
|
+
if from.scheme == 'pipe'
|
33
|
+
'-'
|
34
|
+
else
|
35
|
+
"'#{from.path}'"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Uricp::Strategy
|
4
|
+
class RbdCachedGet
|
5
|
+
include Uricp::Strategy::Common
|
6
|
+
include Uricp::Strategy::CacheCommon
|
7
|
+
include Methadone::SH
|
8
|
+
|
9
|
+
def appropriate?
|
10
|
+
if from.scheme != 'rbd' || rbd_snapshot_spec?(from)
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
return false
|
13
|
+
end
|
14
|
+
|
15
|
+
with_active_cache do
|
16
|
+
cache_check = in_rbd_cache(rbd_image_spec(from))
|
17
|
+
return proposal(cache_check) if cache_check
|
18
|
+
debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
|
19
|
+
false
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def command
|
24
|
+
':;'
|
25
|
+
end
|
26
|
+
|
27
|
+
def proposal(cache_check)
|
28
|
+
@proposed_options = options.dup
|
29
|
+
@proposed_options['from_uri'] = rbd_uri(cache_check)
|
30
|
+
@proposed_options.delete('cache')
|
31
|
+
@proposed_options.delete('cache_name')
|
32
|
+
@proposed_options.delete('dry-cache')
|
33
|
+
self
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdCachedPut
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
include Uricp::Strategy::CacheCommon
|
5
|
+
|
6
|
+
def appropriate?
|
7
|
+
return proposal if to.scheme == 'rbd' &&
|
8
|
+
rbd_snapshot_spec?(from) &&
|
9
|
+
rbd_cache_name.nil? &&
|
10
|
+
!rbd_sequence_complete?
|
11
|
+
|
12
|
+
debug "#{self.class.name}: not appropriate"
|
13
|
+
false
|
14
|
+
end
|
15
|
+
|
16
|
+
def command
|
17
|
+
"rbd clone --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
|
18
|
+
end
|
19
|
+
|
20
|
+
def proposal
|
21
|
+
@proposed_options = options.dup
|
22
|
+
@proposed_options['from_uri'] = to
|
23
|
+
if options['rbd_cache_target']
|
24
|
+
@proposed_options['to_uri'] = options['rbd_cache_target']
|
25
|
+
@proposed_options['rbd_cache_name'] = rbd_image_spec(to)
|
26
|
+
@proposed_options['rbd_flatten'] ||= @proposed_options['rbd_cache_name']
|
27
|
+
@proposed_options.delete('rbd_cache_target')
|
28
|
+
end
|
29
|
+
self
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdFlattener
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
|
5
|
+
def appropriate?
|
6
|
+
return proposal if options['rbd_flatten'] &&
|
7
|
+
rbd_sequence_complete?
|
8
|
+
|
9
|
+
debug "#{self.class.name}: not appropriate"
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
13
|
+
def command
|
14
|
+
"rbd flatten --id #{rbd_id} --no-progress '#{options['rbd_flatten']}' && "
|
15
|
+
end
|
16
|
+
|
17
|
+
def proposal
|
18
|
+
@proposed_options = options.dup
|
19
|
+
@proposed_options.delete('rbd_flatten')
|
20
|
+
self
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdPut
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
|
5
|
+
def appropriate?
|
6
|
+
if compression_required? || conversion_required?
|
7
|
+
debug "#{self.class.name}: not ready to upload"
|
8
|
+
return false
|
9
|
+
end
|
10
|
+
unless rbd_cache_name
|
11
|
+
case from.scheme
|
12
|
+
when 'pipe', 'file'
|
13
|
+
return proposal if to.scheme == 'rbd'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
debug "#{self.class.name}: not appropriate"
|
17
|
+
false
|
18
|
+
end
|
19
|
+
|
20
|
+
def command
|
21
|
+
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_image_spec(to)}';"
|
22
|
+
end
|
23
|
+
|
24
|
+
def proposal
|
25
|
+
@proposed_options = options.dup
|
26
|
+
@proposed_options['from_uri'] = to
|
27
|
+
self
|
28
|
+
end
|
29
|
+
|
30
|
+
def data_source
|
31
|
+
if from.scheme == 'pipe'
|
32
|
+
'-'
|
33
|
+
else
|
34
|
+
"'#{from.path}'"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'json'
|
2
|
+
module Uricp::Strategy
|
3
|
+
class RbdSnap
|
4
|
+
include Uricp::Strategy::Common
|
5
|
+
include Uricp::Strategy::CacheCommon
|
6
|
+
include Methadone::SH
|
7
|
+
|
8
|
+
def appropriate?
|
9
|
+
without_active_cache do
|
10
|
+
if from.scheme == 'rbd' &&
|
11
|
+
options['rbd_snapshot'].nil? &&
|
12
|
+
!rbd_snapshot_spec?(from) &&
|
13
|
+
(rbd_cache_name.nil? ||
|
14
|
+
!from.path.include?(rbd_cache_name))
|
15
|
+
if snap_in_progress?
|
16
|
+
debug "#{self.class.name}: detected snapshot in progress"
|
17
|
+
else
|
18
|
+
return proposal unless sequence_complete?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
debug "#{self.class.name}: not appropriate"
|
23
|
+
false
|
24
|
+
end
|
25
|
+
|
26
|
+
def command
|
27
|
+
"rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
|
28
|
+
"rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
|
29
|
+
end
|
30
|
+
|
31
|
+
def proposal
|
32
|
+
@proposed_options = options.dup
|
33
|
+
@proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from)
|
34
|
+
@proposed_options['from_uri'] = rbd_uri(@proposed_options['rbd_snapshot'])
|
35
|
+
if options['rbd_cache_target']
|
36
|
+
@proposed_options['to_uri'] = options['rbd_cache_target']
|
37
|
+
@proposed_options['rbd_cache_target'] = to
|
38
|
+
end
|
39
|
+
self
|
40
|
+
end
|
41
|
+
|
42
|
+
def rbd_upload_snapshot(uri)
|
43
|
+
"#{rbd_image_spec(uri)}@#{rbd_snapshot_name}"
|
44
|
+
end
|
45
|
+
|
46
|
+
def snap_in_progress?
|
47
|
+
return false if dry_run?
|
48
|
+
|
49
|
+
result = false
|
50
|
+
sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)}" do |stdout|
|
51
|
+
result = JSON.parse(stdout).any? { |x| x['name'] == rbd_snapshot_name }
|
52
|
+
end
|
53
|
+
result
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdSweeper
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
|
5
|
+
def appropriate?
|
6
|
+
return proposal if options['rbd_snapshot'] && rbd_sequence_complete?
|
7
|
+
|
8
|
+
debug "#{self.class.name}: not appropriate"
|
9
|
+
false
|
10
|
+
end
|
11
|
+
|
12
|
+
def command
|
13
|
+
"rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}' && " \
|
14
|
+
"rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
|
15
|
+
end
|
16
|
+
|
17
|
+
def proposal
|
18
|
+
@proposed_options = options.dup
|
19
|
+
@proposed_options.delete('rbd_snapshot')
|
20
|
+
self
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,26 +1,22 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class RemotePut
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
6
|
if compression_required? || conversion_required?
|
9
7
|
debug "#{self.class.name}: not ready to upload"
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
end
|
21
|
-
end
|
22
|
-
debug "#{self.class.name}: not appropriate"
|
8
|
+
return false
|
9
|
+
end
|
10
|
+
case to.scheme
|
11
|
+
when 'http', 'https'
|
12
|
+
case from.scheme
|
13
|
+
when 'file', 'pipe'
|
14
|
+
return proposal if http_authentication?
|
15
|
+
|
16
|
+
unsupported_transfer
|
17
|
+
end
|
23
18
|
end
|
19
|
+
debug "#{self.class.name}: not appropriate"
|
24
20
|
false
|
25
21
|
end
|
26
22
|
|
@@ -41,7 +37,5 @@ module Uricp::Strategy
|
|
41
37
|
from.path
|
42
38
|
end
|
43
39
|
end
|
44
|
-
|
45
40
|
end
|
46
|
-
|
47
41
|
end
|
@@ -1,35 +1,29 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class SegmentedRemotePut
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
when 'file', 'pipe'
|
16
|
-
if http_authentication?
|
17
|
-
return proposal
|
18
|
-
else
|
19
|
-
unsupported_transfer
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
debug "#{self.class.name}: not appropriate"
|
24
|
-
end
|
25
|
-
else
|
26
|
-
debug "#{self.class.name}: no segmentation requested"
|
6
|
+
unless segmented?
|
7
|
+
debug "#{self.class.name}: not appropriate"
|
8
|
+
return false
|
9
|
+
end
|
10
|
+
if compression_required? || conversion_required?
|
11
|
+
debug "#{self.class.name}: not ready to upload"
|
12
|
+
return false
|
27
13
|
end
|
14
|
+
case to.scheme
|
15
|
+
when 'http', 'https'
|
16
|
+
case from.scheme
|
17
|
+
when 'file', 'pipe'
|
18
|
+
return proposal if http_authentication?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
unsupported_transfer
|
28
22
|
false
|
29
23
|
end
|
30
24
|
|
31
25
|
def command
|
32
|
-
"segment_upload --segment-size '#{options['segment-size']}' #{source_details} #{to
|
26
|
+
"segment_upload --segment-size '#{options['segment-size']}' #{source_details} #{to};"
|
33
27
|
end
|
34
28
|
|
35
29
|
def proposal
|
@@ -42,11 +36,5 @@ module Uricp::Strategy
|
|
42
36
|
def source_details
|
43
37
|
"--from #{from.path}" unless from.scheme == 'pipe'
|
44
38
|
end
|
45
|
-
|
46
|
-
def segmented?
|
47
|
-
options['segment-size']
|
48
|
-
end
|
49
|
-
|
50
39
|
end
|
51
|
-
|
52
40
|
end
|