uricp 0.0.16 → 0.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +16 -14
- data/Jenkinsfile +86 -56
- data/Rakefile +5 -0
- data/almalinux8/Dockerfile +26 -0
- data/bin/uricp +25 -11
- data/bionic/Dockerfile +2 -1
- data/centos7/Dockerfile +16 -11
- data/cucumber.yml +1 -0
- data/features/check_uri_path.feature +14 -0
- data/features/rbd_access.feature +226 -0
- data/features/step_definitions/rbd_steps.rb +8 -0
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +28 -0
- data/lib/uricp/strategy/cache_common.rb +19 -1
- data/lib/uricp/strategy/cached_get.rb +8 -5
- data/lib/uricp/strategy/common.rb +52 -1
- data/lib/uricp/strategy/local_convert.rb +1 -1
- data/lib/uricp/strategy/piped_cache.rb +5 -4
- data/lib/uricp/strategy/piped_rbd_get.rb +8 -22
- data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
- data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
- data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
- data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
- data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
- data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
- data/lib/uricp/strategy/rbd_flattener.rb +23 -0
- data/lib/uricp/strategy/{rbd_remote_put.rb → rbd_put.rb} +7 -5
- data/lib/uricp/strategy/rbd_snap.rb +56 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +2 -1
- data/lib/uricp/uri_strategy.rb +9 -1
- data/lib/uricp/version.rb +4 -2
- data/lib/uricp.rb +27 -19
- data/uricp.gemspec +3 -2
- data/xenial/Dockerfile +2 -1
- metadata +37 -8
- data/trusty/Dockerfile +0 -20
@@ -2,43 +2,29 @@ require 'json'
|
|
2
2
|
module Uricp::Strategy
|
3
3
|
class PipedRbdGet
|
4
4
|
include Uricp::Strategy::Common
|
5
|
+
include Uricp::Strategy::CacheCommon
|
5
6
|
include Methadone::SH
|
6
7
|
|
7
8
|
def appropriate?
|
8
|
-
|
9
|
-
|
10
|
-
|
9
|
+
without_active_cache do
|
10
|
+
if from.scheme == 'rbd' &&
|
11
|
+
rbd_snapshot_spec?(from) &&
|
12
|
+
to.scheme != 'rbd'
|
13
|
+
return proposal unless sequence_complete?
|
14
|
+
end
|
11
15
|
end
|
12
16
|
debug "#{self.class.name}: not appropriate"
|
13
17
|
false
|
14
18
|
end
|
15
19
|
|
16
20
|
def command
|
17
|
-
"rbd
|
18
|
-
"rbd export --no-progress --id #{rbd_id} '#{rbd_snap_target(from)}' - |"
|
21
|
+
"rbd export --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' - |"
|
19
22
|
end
|
20
23
|
|
21
24
|
def proposal
|
22
25
|
@proposed_options = options.dup
|
23
|
-
@proposed_options['rbd_snapshot'] = rbd_snap_target(from)
|
24
26
|
@proposed_options['from_uri'] = PIPE_URI
|
25
|
-
@proposed_options.delete('cache')
|
26
|
-
@proposed_options.delete('cache_name')
|
27
27
|
self
|
28
28
|
end
|
29
|
-
|
30
|
-
RBD_SNAPSHOT = 'uricp_snap'.freeze
|
31
|
-
|
32
|
-
def rbd_snap_target(uri)
|
33
|
-
"#{rbd_target(uri)}@#{RBD_SNAPSHOT}"
|
34
|
-
end
|
35
|
-
|
36
|
-
def snap_in_progress?
|
37
|
-
return false if dry_run?
|
38
|
-
|
39
|
-
sh "rbd snap ls --id #{rbd_id} --format json #{rbd_target(from)}" do |stdout|
|
40
|
-
JSON.parse(stdout).any? { |x| x['name'] == RBD_SNAPSHOT }
|
41
|
-
end
|
42
|
-
end
|
43
29
|
end
|
44
30
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdCacheBaseSnap
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
include Methadone::SH
|
5
|
+
|
6
|
+
def appropriate?
|
7
|
+
if rbd_cache_name &&
|
8
|
+
rbd_image_spec(from) == rbd_cache_name
|
9
|
+
return proposal
|
10
|
+
end
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
false
|
13
|
+
end
|
14
|
+
|
15
|
+
def command
|
16
|
+
"rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}' && "\
|
17
|
+
"rbd snap protect --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"
|
18
|
+
end
|
19
|
+
|
20
|
+
def proposal
|
21
|
+
@proposed_options = options.dup
|
22
|
+
@proposed_options['from_uri'] = rbd_uri(rbd_clone_snapshot(rbd_cache_name))
|
23
|
+
@proposed_options.delete('rbd_cache_name')
|
24
|
+
self
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Uricp::Strategy
|
4
|
+
class RbdCacheCheck
|
5
|
+
include Uricp::Strategy::Common
|
6
|
+
include Uricp::Strategy::CacheCommon
|
7
|
+
include Methadone::SH
|
8
|
+
|
9
|
+
def appropriate?
|
10
|
+
unless (from.scheme == 'rbd') != (to.scheme == 'rbd')
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
return false
|
13
|
+
end
|
14
|
+
|
15
|
+
with_active_cache do
|
16
|
+
if from.scheme == 'rbd'
|
17
|
+
cache_check = in_rbd_cache(rbd_cache_image_spec(from))
|
18
|
+
return proposal(cache_check) if cache_check
|
19
|
+
debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
|
20
|
+
elsif to.scheme == 'rbd'
|
21
|
+
cache_check = in_rbd_cache(rbd_cache_image_spec(to))
|
22
|
+
return proposal(cache_check) if cache_check
|
23
|
+
debug "#{self.class.name}: no rbd cache entry for #{options['to_uri']}"
|
24
|
+
end
|
25
|
+
false
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def command
|
30
|
+
':;'
|
31
|
+
end
|
32
|
+
|
33
|
+
def proposal(cache_check)
|
34
|
+
@proposed_options = options.dup
|
35
|
+
@proposed_options['from_uri'] = rbd_uri(cache_check)
|
36
|
+
@proposed_options.delete('cache')
|
37
|
+
@proposed_options.delete('cache_name')
|
38
|
+
@proposed_options.delete('dry-cache')
|
39
|
+
self
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Uricp::Strategy
|
4
|
+
class RbdCacheClone
|
5
|
+
include Uricp::Strategy::Common
|
6
|
+
include Uricp::Strategy::CacheCommon
|
7
|
+
include Methadone::SH
|
8
|
+
|
9
|
+
def appropriate?
|
10
|
+
unless from.scheme == 'rbd'
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
return false
|
13
|
+
end
|
14
|
+
|
15
|
+
with_active_cache do
|
16
|
+
options['cache_name'] = File.basename(options['to_uri'].path)
|
17
|
+
cache_target = rbd_cache_image_spec(from)
|
18
|
+
cache_check = in_rbd_cache(rbd_cache_image_spec(from))
|
19
|
+
if cache_check
|
20
|
+
debug "#{self.class.name}: Unexpected existing cache entry for #{options['to_uri']}"
|
21
|
+
unsupported_transfer
|
22
|
+
end
|
23
|
+
proposal(cache_target)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def command
|
28
|
+
':;'
|
29
|
+
end
|
30
|
+
|
31
|
+
def proposal(cache_check)
|
32
|
+
@proposed_options = options.dup
|
33
|
+
@proposed_options['rbd_cache_target'] = rbd_uri(cache_check)
|
34
|
+
@proposed_options.delete('cache')
|
35
|
+
@proposed_options.delete('cache_name')
|
36
|
+
@proposed_options.delete('dry-cache')
|
37
|
+
self
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdCacheUpload
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
include Methadone::SH
|
5
|
+
|
6
|
+
def appropriate?
|
7
|
+
if compression_required? || conversion_required?
|
8
|
+
debug "#{self.class.name}: not ready to upload"
|
9
|
+
return false
|
10
|
+
end
|
11
|
+
if rbd_cache_name
|
12
|
+
case from.scheme
|
13
|
+
when 'pipe', 'file'
|
14
|
+
return proposal if to.scheme == 'rbd'
|
15
|
+
end
|
16
|
+
end
|
17
|
+
debug "#{self.class.name}: not appropriate"
|
18
|
+
false
|
19
|
+
end
|
20
|
+
|
21
|
+
def command
|
22
|
+
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}' && "
|
23
|
+
end
|
24
|
+
|
25
|
+
def proposal
|
26
|
+
@proposed_options = options.dup
|
27
|
+
@proposed_options['from_uri'] = rbd_uri(rbd_cache_name)
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def data_source
|
32
|
+
if from.scheme == 'pipe'
|
33
|
+
'-'
|
34
|
+
else
|
35
|
+
"'#{from.path}'"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Uricp::Strategy
|
4
|
+
class RbdCachedGet
|
5
|
+
include Uricp::Strategy::Common
|
6
|
+
include Uricp::Strategy::CacheCommon
|
7
|
+
include Methadone::SH
|
8
|
+
|
9
|
+
def appropriate?
|
10
|
+
if from.scheme != 'rbd' || rbd_snapshot_spec?(from)
|
11
|
+
debug "#{self.class.name}: not appropriate"
|
12
|
+
return false
|
13
|
+
end
|
14
|
+
|
15
|
+
with_active_cache do
|
16
|
+
cache_check = in_rbd_cache(rbd_image_spec(from))
|
17
|
+
return proposal(cache_check) if cache_check
|
18
|
+
debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
|
19
|
+
false
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def command
|
24
|
+
':;'
|
25
|
+
end
|
26
|
+
|
27
|
+
def proposal(cache_check)
|
28
|
+
@proposed_options = options.dup
|
29
|
+
@proposed_options['from_uri'] = rbd_uri(cache_check)
|
30
|
+
@proposed_options.delete('cache')
|
31
|
+
@proposed_options.delete('cache_name')
|
32
|
+
@proposed_options.delete('dry-cache')
|
33
|
+
self
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdCachedPut
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
include Uricp::Strategy::CacheCommon
|
5
|
+
|
6
|
+
def appropriate?
|
7
|
+
return proposal if to.scheme == 'rbd' &&
|
8
|
+
rbd_snapshot_spec?(from) &&
|
9
|
+
rbd_cache_name.nil? &&
|
10
|
+
!rbd_sequence_complete?
|
11
|
+
|
12
|
+
debug "#{self.class.name}: not appropriate"
|
13
|
+
false
|
14
|
+
end
|
15
|
+
|
16
|
+
def command
|
17
|
+
"rbd clone --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
|
18
|
+
end
|
19
|
+
|
20
|
+
def proposal
|
21
|
+
@proposed_options = options.dup
|
22
|
+
@proposed_options['from_uri'] = to
|
23
|
+
if options['rbd_cache_target']
|
24
|
+
@proposed_options['to_uri'] = options['rbd_cache_target']
|
25
|
+
@proposed_options['rbd_cache_name'] = rbd_image_spec(to)
|
26
|
+
@proposed_options['rbd_flatten'] ||= @proposed_options['rbd_cache_name']
|
27
|
+
@proposed_options.delete('rbd_cache_target')
|
28
|
+
end
|
29
|
+
self
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Uricp::Strategy
|
2
|
+
class RbdFlattener
|
3
|
+
include Uricp::Strategy::Common
|
4
|
+
|
5
|
+
def appropriate?
|
6
|
+
return proposal if options['rbd_flatten'] &&
|
7
|
+
rbd_sequence_complete?
|
8
|
+
|
9
|
+
debug "#{self.class.name}: not appropriate"
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
13
|
+
def command
|
14
|
+
"rbd flatten --id #{rbd_id} --no-progress '#{options['rbd_flatten']}' && "
|
15
|
+
end
|
16
|
+
|
17
|
+
def proposal
|
18
|
+
@proposed_options = options.dup
|
19
|
+
@proposed_options.delete('rbd_flatten')
|
20
|
+
self
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
class
|
2
|
+
class RbdPut
|
3
3
|
include Uricp::Strategy::Common
|
4
4
|
|
5
5
|
def appropriate?
|
@@ -7,16 +7,18 @@ module Uricp::Strategy
|
|
7
7
|
debug "#{self.class.name}: not ready to upload"
|
8
8
|
return false
|
9
9
|
end
|
10
|
-
|
11
|
-
|
12
|
-
|
10
|
+
unless rbd_cache_name
|
11
|
+
case from.scheme
|
12
|
+
when 'pipe', 'file'
|
13
|
+
return proposal if to.scheme == 'rbd'
|
14
|
+
end
|
13
15
|
end
|
14
16
|
debug "#{self.class.name}: not appropriate"
|
15
17
|
false
|
16
18
|
end
|
17
19
|
|
18
20
|
def command
|
19
|
-
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{
|
21
|
+
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_image_spec(to)}';"
|
20
22
|
end
|
21
23
|
|
22
24
|
def proposal
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'json'
|
2
|
+
module Uricp::Strategy
|
3
|
+
class RbdSnap
|
4
|
+
include Uricp::Strategy::Common
|
5
|
+
include Uricp::Strategy::CacheCommon
|
6
|
+
include Methadone::SH
|
7
|
+
|
8
|
+
def appropriate?
|
9
|
+
without_active_cache do
|
10
|
+
if from.scheme == 'rbd' &&
|
11
|
+
options['rbd_snapshot'].nil? &&
|
12
|
+
!rbd_snapshot_spec?(from) &&
|
13
|
+
(rbd_cache_name.nil? ||
|
14
|
+
!from.path.include?(rbd_cache_name))
|
15
|
+
if snap_in_progress?
|
16
|
+
debug "#{self.class.name}: detected snapshot in progress"
|
17
|
+
else
|
18
|
+
return proposal unless sequence_complete?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
debug "#{self.class.name}: not appropriate"
|
23
|
+
false
|
24
|
+
end
|
25
|
+
|
26
|
+
def command
|
27
|
+
"rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
|
28
|
+
"rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
|
29
|
+
end
|
30
|
+
|
31
|
+
def proposal
|
32
|
+
@proposed_options = options.dup
|
33
|
+
@proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from)
|
34
|
+
@proposed_options['from_uri'] = rbd_uri(@proposed_options['rbd_snapshot'])
|
35
|
+
if options['rbd_cache_target']
|
36
|
+
@proposed_options['to_uri'] = options['rbd_cache_target']
|
37
|
+
@proposed_options['rbd_cache_target'] = to
|
38
|
+
end
|
39
|
+
self
|
40
|
+
end
|
41
|
+
|
42
|
+
def rbd_upload_snapshot(uri)
|
43
|
+
"#{rbd_image_spec(uri)}@#{rbd_snapshot_name}"
|
44
|
+
end
|
45
|
+
|
46
|
+
def snap_in_progress?
|
47
|
+
return false if dry_run?
|
48
|
+
|
49
|
+
result = false
|
50
|
+
sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)} 2>/dev/null" do |stdout|
|
51
|
+
result = JSON.parse(stdout).any? { |x| x['name'] == rbd_snapshot_name }
|
52
|
+
end
|
53
|
+
result
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -3,13 +3,14 @@ module Uricp::Strategy
|
|
3
3
|
include Uricp::Strategy::Common
|
4
4
|
|
5
5
|
def appropriate?
|
6
|
-
return proposal if options['rbd_snapshot'] &&
|
6
|
+
return proposal if options['rbd_snapshot'] && rbd_sequence_complete?
|
7
7
|
|
8
8
|
debug "#{self.class.name}: not appropriate"
|
9
9
|
false
|
10
10
|
end
|
11
11
|
|
12
12
|
def command
|
13
|
+
"rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}' && " \
|
13
14
|
"rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
|
14
15
|
end
|
15
16
|
|
data/lib/uricp/uri_strategy.rb
CHANGED
@@ -25,6 +25,10 @@ module Uricp
|
|
25
25
|
|
26
26
|
# This is an ordered list from the most specific to the most general
|
27
27
|
STRATEGIES = [
|
28
|
+
Strategy::RbdCachedGet,
|
29
|
+
Strategy::RbdCacheClone,
|
30
|
+
Strategy::RbdCacheCheck,
|
31
|
+
Strategy::RbdSnap,
|
28
32
|
Strategy::CachedGet,
|
29
33
|
Strategy::PipedRemoteGet,
|
30
34
|
Strategy::PipedRbdGet,
|
@@ -35,13 +39,17 @@ module Uricp
|
|
35
39
|
Strategy::LocalConvert,
|
36
40
|
Strategy::LocalLink,
|
37
41
|
Strategy::PipedCompress,
|
38
|
-
Strategy::
|
42
|
+
Strategy::RbdCacheUpload,
|
43
|
+
Strategy::RbdCachedPut,
|
44
|
+
Strategy::RbdPut,
|
45
|
+
Strategy::RbdCacheBaseSnap,
|
39
46
|
Strategy::SegmentedRemotePut,
|
40
47
|
Strategy::RemotePut,
|
41
48
|
Strategy::PipedLocalCompress,
|
42
49
|
Strategy::PipedLocalGet,
|
43
50
|
Strategy::PipedLocalPut,
|
44
51
|
Strategy::Cleaner,
|
52
|
+
Strategy::RbdFlattener,
|
45
53
|
Strategy::RbdSweeper,
|
46
54
|
Strategy::Sweeper
|
47
55
|
].freeze
|
data/lib/uricp/version.rb
CHANGED