uricp 0.0.16 → 0.0.21
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +16 -14
- data/Jenkinsfile +86 -56
- data/Rakefile +5 -0
- data/almalinux8/Dockerfile +26 -0
- data/bin/uricp +25 -11
- data/bionic/Dockerfile +2 -1
- data/centos7/Dockerfile +16 -11
- data/cucumber.yml +1 -0
- data/features/check_uri_path.feature +14 -0
- data/features/rbd_access.feature +226 -0
- data/features/step_definitions/rbd_steps.rb +8 -0
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +28 -0
- data/lib/uricp/strategy/cache_common.rb +19 -1
- data/lib/uricp/strategy/cached_get.rb +8 -5
- data/lib/uricp/strategy/common.rb +52 -1
- data/lib/uricp/strategy/local_convert.rb +1 -1
- data/lib/uricp/strategy/piped_cache.rb +5 -4
- data/lib/uricp/strategy/piped_rbd_get.rb +8 -22
- data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
- data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
- data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
- data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
- data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
- data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
- data/lib/uricp/strategy/rbd_flattener.rb +23 -0
- data/lib/uricp/strategy/{rbd_remote_put.rb → rbd_put.rb} +7 -5
- data/lib/uricp/strategy/rbd_snap.rb +56 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +2 -1
- data/lib/uricp/uri_strategy.rb +9 -1
- data/lib/uricp/version.rb +4 -2
- data/lib/uricp.rb +27 -19
- data/uricp.gemspec +3 -2
- data/xenial/Dockerfile +2 -1
- metadata +37 -8
- data/trusty/Dockerfile +0 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aeaa81822977930906959744feb0d8bdf9ca820afd10e6b97d32a726cbf67391
|
4
|
+
data.tar.gz: cd29b49f83ba53a275f4574c32cbb7f1b92c5218421cef668355a14ab5a36bec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 56c0e441da6e7e709580e09cdcc68c56b3ff731fc9fd028e56160dbfca427048ecab2f79c049f490bd15bcb027a67ec1b4aa01ddd2ce801cda89ad53f9ba6efa
|
7
|
+
data.tar.gz: 92122076440ca0cbd0345f29bf4386503c9f3830d6a4ffbfc92a4f55e40b9ea7790ff81cc1ebf1101ce7ac4cb552c312bf2be0cc5c3c49f0b727ab0247f46f90
|
data/Gemfile.lock
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
uricp (0.0.
|
4
|
+
uricp (0.0.21)
|
5
|
+
childprocess (~> 1.0)
|
5
6
|
filesize (= 0.0.2)
|
6
|
-
methadone (~> 2.0.
|
7
|
+
methadone (~> 2.0.2)
|
7
8
|
open4 (~> 1.3.0)
|
8
9
|
sendfile (~> 1.2.0)
|
9
10
|
|
@@ -14,32 +15,33 @@ GEM
|
|
14
15
|
childprocess (>= 0.3.6)
|
15
16
|
cucumber (>= 1.1.1)
|
16
17
|
rspec-expectations (>= 2.7.0)
|
17
|
-
builder (3.2.
|
18
|
-
childprocess (0.
|
19
|
-
|
18
|
+
builder (3.2.4)
|
19
|
+
childprocess (1.0.1)
|
20
|
+
rake (< 13.0)
|
20
21
|
cucumber (1.3.20)
|
21
22
|
builder (>= 2.1.2)
|
22
23
|
diff-lcs (>= 1.1.3)
|
23
24
|
gherkin (~> 2.12)
|
24
25
|
multi_json (>= 1.7.5, < 2.0)
|
25
26
|
multi_test (>= 0.1.2)
|
26
|
-
diff-lcs (1.
|
27
|
-
ffi (1.9.23)
|
27
|
+
diff-lcs (1.4.4)
|
28
28
|
filesize (0.0.2)
|
29
29
|
gherkin (2.12.2)
|
30
30
|
multi_json (~> 1.3)
|
31
31
|
inifile (1.1.0)
|
32
32
|
json (1.8.6)
|
33
|
-
methadone (2.0.
|
33
|
+
methadone (2.0.2)
|
34
34
|
bundler
|
35
|
-
multi_json (1.
|
35
|
+
multi_json (1.15.0)
|
36
36
|
multi_test (0.1.2)
|
37
37
|
open4 (1.3.4)
|
38
|
-
rake (
|
38
|
+
rake (12.3.3)
|
39
39
|
rdoc (4.2.2)
|
40
40
|
json (~> 1.4)
|
41
|
-
rspec-expectations (
|
42
|
-
diff-lcs (>= 1.
|
41
|
+
rspec-expectations (3.10.1)
|
42
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
43
|
+
rspec-support (~> 3.10.0)
|
44
|
+
rspec-support (3.10.2)
|
43
45
|
sendfile (1.2.2)
|
44
46
|
|
45
47
|
PLATFORMS
|
@@ -50,9 +52,9 @@ DEPENDENCIES
|
|
50
52
|
bundler (~> 1.7)
|
51
53
|
cucumber (~> 1.3)
|
52
54
|
inifile (~> 1.1)
|
53
|
-
rake (~>
|
55
|
+
rake (~> 12.3)
|
54
56
|
rdoc (~> 4.2.0)
|
55
57
|
uricp!
|
56
58
|
|
57
59
|
BUNDLED WITH
|
58
|
-
1.
|
60
|
+
1.17.3
|
data/Jenkinsfile
CHANGED
@@ -2,6 +2,8 @@ pipeline {
|
|
2
2
|
environment {
|
3
3
|
ORBIT = credentials('ced8ecd0-dbe4-4bf6-bdde-101a661565f5')
|
4
4
|
ORBIT_URL = credentials('c0c95c56-b4e9-45c2-85c5-bf292aef7301')
|
5
|
+
RBD = credentials('9bfb46f5-65eb-40c3-bfda-7ac5fe469ff4')
|
6
|
+
CEPH_MON = credentials('ceph-mon')
|
5
7
|
ORBIT_USER = "${ORBIT_USR}"
|
6
8
|
ORBIT_KEY = "${ORBIT_PSW}"
|
7
9
|
}
|
@@ -14,100 +16,128 @@ pipeline {
|
|
14
16
|
}
|
15
17
|
agent none
|
16
18
|
stages {
|
19
|
+
stage("AlmaLinux 8") {
|
20
|
+
agent {
|
21
|
+
dockerfile {
|
22
|
+
dir 'almalinux8'
|
23
|
+
label "docker"
|
24
|
+
additionalBuildArgs '--build-arg CEPH_MON --build-arg RBD_USR --build-arg RBD_PSW'
|
25
|
+
}
|
26
|
+
}
|
27
|
+
steps {
|
28
|
+
sh 'rbd --id "${RBD_USR}" ls servers'
|
29
|
+
sh 'bundle install --deployment'
|
30
|
+
sh 'bundle exec rake features:new_qemu'
|
31
|
+
}
|
32
|
+
post {
|
33
|
+
always {
|
34
|
+
sh 'bundle exec rake features:clean'
|
35
|
+
}
|
36
|
+
failure {
|
37
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
38
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
39
|
+
body: "${env.BUILD_URL}"
|
40
|
+
}
|
41
|
+
}
|
42
|
+
}
|
17
43
|
stage("CentOS 7") {
|
18
44
|
agent {
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
45
|
+
dockerfile {
|
46
|
+
dir 'centos7'
|
47
|
+
label "docker"
|
48
|
+
additionalBuildArgs '--build-arg CEPH_MON --build-arg RBD_USR --build-arg RBD_PSW'
|
49
|
+
}
|
23
50
|
}
|
24
51
|
steps {
|
25
|
-
|
26
|
-
|
52
|
+
sh 'rbd --id "${RBD_USR}" ls servers'
|
53
|
+
sh 'bundle install --deployment'
|
54
|
+
sh 'bundle exec rake features:new_qemu'
|
27
55
|
}
|
28
56
|
post {
|
29
57
|
always {
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
58
|
+
sh 'bundle exec rake features:clean'
|
59
|
+
}
|
60
|
+
failure {
|
61
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
62
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
63
|
+
body: "${env.BUILD_URL}"
|
64
|
+
}
|
37
65
|
}
|
38
66
|
}
|
39
|
-
stage("
|
67
|
+
stage("Focal, Ruby 2.7.0") {
|
40
68
|
agent {
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
69
|
+
dockerfile {
|
70
|
+
dir 'focal'
|
71
|
+
label "docker"
|
72
|
+
additionalBuildArgs '--build-arg CEPH_MON --build-arg RBD_USR --build-arg RBD_PSW'
|
73
|
+
}
|
45
74
|
}
|
46
75
|
steps {
|
47
|
-
|
48
|
-
|
76
|
+
sh 'rbd --id "${RBD_USR}" ls servers'
|
77
|
+
sh 'bundle install --deployment'
|
78
|
+
sh 'bundle exec rake features:new_qemu'
|
49
79
|
}
|
50
80
|
post {
|
51
81
|
always {
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
82
|
+
sh 'bundle exec rake features:clean'
|
83
|
+
}
|
84
|
+
failure {
|
85
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
86
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
87
|
+
body: "${env.BUILD_URL}"
|
88
|
+
}
|
59
89
|
}
|
60
90
|
}
|
61
91
|
stage("Xenial, Ruby 2.3.1") {
|
62
92
|
agent {
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
93
|
+
dockerfile {
|
94
|
+
dir 'xenial'
|
95
|
+
label "docker"
|
96
|
+
}
|
67
97
|
}
|
68
98
|
steps {
|
69
|
-
|
70
|
-
|
99
|
+
sh 'bundle install --deployment'
|
100
|
+
sh 'bundle exec rake features:new_qemu'
|
71
101
|
}
|
72
102
|
post {
|
73
103
|
always {
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
104
|
+
sh 'bundle exec rake features:clean'
|
105
|
+
}
|
106
|
+
failure {
|
107
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
108
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
109
|
+
body: "${env.BUILD_URL}"
|
110
|
+
}
|
81
111
|
}
|
82
112
|
}
|
83
113
|
stage("Bionic, Ruby 2.5.1") {
|
84
114
|
agent {
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
115
|
+
dockerfile {
|
116
|
+
dir 'bionic'
|
117
|
+
label "docker"
|
118
|
+
}
|
89
119
|
}
|
90
120
|
steps {
|
91
|
-
|
92
|
-
|
121
|
+
sh 'bundle install --deployment'
|
122
|
+
sh 'bundle exec rake features:new_qemu'
|
93
123
|
}
|
94
124
|
post {
|
95
125
|
always {
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
126
|
+
sh 'bundle exec rake features:clean'
|
127
|
+
}
|
128
|
+
failure {
|
129
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
130
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
131
|
+
body: "${env.BUILD_URL}"
|
132
|
+
}
|
103
133
|
}
|
104
134
|
}
|
105
135
|
}
|
106
136
|
post {
|
107
137
|
failure {
|
108
|
-
|
109
|
-
|
110
|
-
|
138
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
139
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
140
|
+
body: "${env.BUILD_URL}"
|
111
141
|
}
|
112
142
|
}
|
113
143
|
}
|
data/Rakefile
CHANGED
@@ -48,6 +48,11 @@ namespace(:features) do
|
|
48
48
|
t.fork = false
|
49
49
|
t.profile = 'old-qemu-image'
|
50
50
|
end
|
51
|
+
Cucumber::Rake::Task.new(:rbd) do |t|
|
52
|
+
t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
|
53
|
+
t.fork = false
|
54
|
+
t.profile = 'rbd'
|
55
|
+
end
|
51
56
|
|
52
57
|
task(:clean) do
|
53
58
|
if ENV['ORBIT_USER'] && ENV['ORBIT_KEY']
|
@@ -0,0 +1,26 @@
|
|
1
|
+
FROM almalinux:8
|
2
|
+
MAINTAINER support@brightbox.co.uk
|
3
|
+
|
4
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
5
|
+
RUN dnf -y install dnf-plugins-core
|
6
|
+
RUN dnf config-manager --add-repo https://repo.almalinux.org/almalinux/8.4/devel/almalinux-devel.repo
|
7
|
+
RUN dnf -y update
|
8
|
+
RUN dnf config-manager --set-enabled powertools,devel
|
9
|
+
RUN dnf -y group install "Development Tools"
|
10
|
+
RUN dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
|
11
|
+
RUN dnf -y install https://www.rdoproject.org/repos/rdo-release.el8.rpm
|
12
|
+
RUN dnf -y install \
|
13
|
+
git \
|
14
|
+
lz4 \
|
15
|
+
qemu-img \
|
16
|
+
ruby-devel \
|
17
|
+
rubygems \
|
18
|
+
rubygem-bundler \
|
19
|
+
python3-swiftclient \
|
20
|
+
ceph-common
|
21
|
+
RUN mkdir -p /etc/ceph
|
22
|
+
ARG RBD_USR=libvirt
|
23
|
+
ARG RBD_PSW
|
24
|
+
ARG CEPH_MON
|
25
|
+
RUN echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
|
26
|
+
RUN echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf
|
data/bin/uricp
CHANGED
@@ -4,7 +4,7 @@ require 'optparse'
|
|
4
4
|
require 'uri'
|
5
5
|
require 'methadone'
|
6
6
|
require 'filesize'
|
7
|
-
require 'uricp
|
7
|
+
require 'uricp'
|
8
8
|
|
9
9
|
class App
|
10
10
|
include Methadone::Main
|
@@ -52,20 +52,32 @@ class App
|
|
52
52
|
def self.build_command
|
53
53
|
command_sequence = Uricp::UriStrategy.choose_strategy(options)
|
54
54
|
debug "#{self.name} Command sequence is #{command_sequence.inspect}"
|
55
|
-
command_sequence.map
|
55
|
+
command_sequence.map do |c|
|
56
|
+
debug c.class.name
|
57
|
+
c.command
|
58
|
+
end.join('')
|
56
59
|
end
|
57
60
|
|
58
61
|
def self.validate_options
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
"'max-cache' requires 'cache' option"
|
67
|
-
end
|
62
|
+
case
|
63
|
+
when options['from_uri'].path.nil?
|
64
|
+
raise ::OptionParser::InvalidArgument,
|
65
|
+
"'from_uri' has missing path"
|
66
|
+
when options['to_uri'].path.nil?
|
67
|
+
raise ::OptionParser::InvalidArgument,
|
68
|
+
"'to_uri' has missing path"
|
68
69
|
end
|
70
|
+
return if options['cache']
|
71
|
+
|
72
|
+
case
|
73
|
+
when options['target-format']
|
74
|
+
raise ::OptionParser::MissingArgument,
|
75
|
+
"'target-format' requires 'cache' option"
|
76
|
+
when options['max-cache']
|
77
|
+
raise ::OptionParser::MissingArgument,
|
78
|
+
"'max-cache' requires 'cache' option"
|
79
|
+
end
|
80
|
+
|
69
81
|
end
|
70
82
|
|
71
83
|
def self.add_cache_name
|
@@ -109,6 +121,8 @@ class App
|
|
109
121
|
"Compress output")
|
110
122
|
on("--[no-]dry-run",
|
111
123
|
"Show what would be run,", "but don't run the command")
|
124
|
+
on("--[no-]dry-cache",
|
125
|
+
"On dry runs assume the cache is populated")
|
112
126
|
on("--[no-]force",
|
113
127
|
"Always write sparsely")
|
114
128
|
#Arguments
|
data/bionic/Dockerfile
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
FROM ubuntu:bionic
|
2
2
|
MAINTAINER support@brightbox.co.uk
|
3
3
|
|
4
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
5
|
+
|
4
6
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
|
5
7
|
|
6
8
|
#RUN apt-add-repository ppa:brightbox/ruby-ng
|
@@ -16,5 +18,4 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
|
16
18
|
python-swiftclient \
|
17
19
|
ruby-bundler
|
18
20
|
|
19
|
-
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
20
21
|
|
data/centos7/Dockerfile
CHANGED
@@ -1,20 +1,25 @@
|
|
1
1
|
FROM centos:7
|
2
2
|
MAINTAINER support@brightbox.co.uk
|
3
3
|
|
4
|
-
RUN
|
5
|
-
|
4
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
5
|
+
RUN yum update -y
|
6
|
+
RUN yum -y install \
|
7
|
+
centos-release-openstack-train \
|
8
|
+
epel-release
|
9
|
+
RUN yum -y install \
|
6
10
|
gcc \
|
7
11
|
make \
|
8
12
|
git \
|
9
13
|
qemu-img \
|
14
|
+
lz4 \
|
10
15
|
ruby-devel \
|
11
16
|
rubygems \
|
12
|
-
rubygem-bundler
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
RUN echo
|
20
|
-
|
17
|
+
rubygem-bundler \
|
18
|
+
ceph-common \
|
19
|
+
python-swiftclient
|
20
|
+
RUN mkdir -p /etc/ceph
|
21
|
+
ARG RBD_USR=libvirt
|
22
|
+
ARG RBD_PSW
|
23
|
+
ARG CEPH_MON
|
24
|
+
RUN echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
|
25
|
+
RUN echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf
|
data/cucumber.yml
CHANGED
@@ -0,0 +1,14 @@
|
|
1
|
+
Feature: Check URI paths
|
2
|
+
In order to ensure reliable uricp usage
|
3
|
+
As a command line user
|
4
|
+
I want to reject badly formed URIs
|
5
|
+
|
6
|
+
Scenario: source URI is incorrectly formatted
|
7
|
+
When I run `uricp --dry-run file:fred rbd:///servers/fred`
|
8
|
+
Then the exit status should not be 0
|
9
|
+
And the stderr should contain "invalid argument: 'from_uri' has missing path"
|
10
|
+
|
11
|
+
Scenario: target URI is incorrectly formatted
|
12
|
+
When I run `uricp --dry-run file:///fred rbd:servers/fred`
|
13
|
+
Then the exit status should not be 0
|
14
|
+
And the stderr should contain "invalid argument: 'to_uri' has missing path"
|