uricp 0.0.13 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +2 -1
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +20 -23
- data/Jenkinsfile +113 -0
- data/README.md +2 -2
- data/Rakefile +26 -31
- data/bionic/Dockerfile +20 -0
- data/centos7/Dockerfile +18 -0
- data/features/step_definitions/orbit_steps.rb +19 -13
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +21 -0
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp.rb +3 -29
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +11 -11
- data/lib/uricp/strategy/cached_get.rb +14 -20
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +27 -19
- data/lib/uricp/strategy/local_convert.rb +10 -17
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +7 -13
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +46 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +14 -13
- data/lib/uricp/version.rb +4 -2
- data/uricp.gemspec +3 -4
- data/xenial/Dockerfile +20 -0
- metadata +39 -51
- data/Dockerfile_centos6.6 +0 -39
- data/Dockerfile_centos7 +0 -39
- data/Dockerfile_trusty-ruby193 +0 -32
- data/README.rdoc +0 -23
- data/spec/something_spec.rb +0 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d9944582044883ff45f3847042ca62016416d06198a0af41751de93ab992c646
|
4
|
+
data.tar.gz: 19c14777647bb01fec3a4e6214e8183b29432843bf47b8b0c818d0401dafaea1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c157e985f79891ca962fb0b792dab2a4fab89bd651c15b940e75ba4baf0e86a42922b9dc8a0f96fa53b8de4151d3e9b6aeefac256ffe59faa2ed6ad5becfad47
|
7
|
+
data.tar.gz: 45444dfcbc92079f7221d0f239315db22f6af749843f6a89312e949c1f12068a2254f6ffd8830a4ab00833e3b66d16d259cfcccb9388f24dde9ff23fbad99ee2
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
data/Gemfile.lock
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
uricp (0.0.
|
4
|
+
uricp (0.0.18)
|
5
5
|
filesize (= 0.0.2)
|
6
|
-
methadone (~>
|
6
|
+
methadone (~> 2.0.0)
|
7
7
|
open4 (~> 1.3.0)
|
8
8
|
sendfile (~> 1.2.0)
|
9
9
|
|
@@ -14,38 +14,33 @@ GEM
|
|
14
14
|
childprocess (>= 0.3.6)
|
15
15
|
cucumber (>= 1.1.1)
|
16
16
|
rspec-expectations (>= 2.7.0)
|
17
|
-
builder (3.2.
|
18
|
-
childprocess (0.
|
19
|
-
|
17
|
+
builder (3.2.4)
|
18
|
+
childprocess (1.0.1)
|
19
|
+
rake (< 13.0)
|
20
20
|
cucumber (1.3.20)
|
21
21
|
builder (>= 2.1.2)
|
22
22
|
diff-lcs (>= 1.1.3)
|
23
23
|
gherkin (~> 2.12)
|
24
24
|
multi_json (>= 1.7.5, < 2.0)
|
25
25
|
multi_test (>= 0.1.2)
|
26
|
-
diff-lcs (1.
|
27
|
-
ffi (1.9.10)
|
26
|
+
diff-lcs (1.4.4)
|
28
27
|
filesize (0.0.2)
|
29
28
|
gherkin (2.12.2)
|
30
29
|
multi_json (~> 1.3)
|
31
30
|
inifile (1.1.0)
|
32
|
-
json (1.8.
|
33
|
-
methadone (
|
31
|
+
json (1.8.6)
|
32
|
+
methadone (2.0.2)
|
34
33
|
bundler
|
35
|
-
multi_json (1.
|
34
|
+
multi_json (1.15.0)
|
36
35
|
multi_test (0.1.2)
|
37
36
|
open4 (1.3.4)
|
38
|
-
rake (
|
39
|
-
rdoc (4.2.
|
37
|
+
rake (12.3.3)
|
38
|
+
rdoc (4.2.2)
|
40
39
|
json (~> 1.4)
|
41
|
-
rspec (
|
42
|
-
|
43
|
-
rspec-
|
44
|
-
|
45
|
-
rspec-core (2.99.2)
|
46
|
-
rspec-expectations (2.99.2)
|
47
|
-
diff-lcs (>= 1.1.3, < 2.0)
|
48
|
-
rspec-mocks (2.99.4)
|
40
|
+
rspec-expectations (3.10.1)
|
41
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
42
|
+
rspec-support (~> 3.10.0)
|
43
|
+
rspec-support (3.10.2)
|
49
44
|
sendfile (1.2.2)
|
50
45
|
|
51
46
|
PLATFORMS
|
@@ -56,7 +51,9 @@ DEPENDENCIES
|
|
56
51
|
bundler (~> 1.7)
|
57
52
|
cucumber (~> 1.3)
|
58
53
|
inifile (~> 1.1)
|
59
|
-
rake
|
60
|
-
rdoc
|
61
|
-
rspec (~> 2.99)
|
54
|
+
rake (~> 12.3)
|
55
|
+
rdoc (~> 4.2.0)
|
62
56
|
uricp!
|
57
|
+
|
58
|
+
BUNDLED WITH
|
59
|
+
1.17.3
|
data/Jenkinsfile
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
pipeline {
|
2
|
+
environment {
|
3
|
+
ORBIT = credentials('ced8ecd0-dbe4-4bf6-bdde-101a661565f5')
|
4
|
+
ORBIT_URL = credentials('c0c95c56-b4e9-45c2-85c5-bf292aef7301')
|
5
|
+
ORBIT_USER = "${ORBIT_USR}"
|
6
|
+
ORBIT_KEY = "${ORBIT_PSW}"
|
7
|
+
}
|
8
|
+
options {
|
9
|
+
disableConcurrentBuilds()
|
10
|
+
buildDiscarder(logRotator(numToKeepStr: '5'))
|
11
|
+
}
|
12
|
+
triggers {
|
13
|
+
cron('@weekly')
|
14
|
+
}
|
15
|
+
agent none
|
16
|
+
stages {
|
17
|
+
stage("CentOS 7") {
|
18
|
+
agent {
|
19
|
+
dockerfile {
|
20
|
+
dir 'centos7'
|
21
|
+
label "docker"
|
22
|
+
}
|
23
|
+
}
|
24
|
+
steps {
|
25
|
+
sh 'bundle install --deployment'
|
26
|
+
sh 'bundle exec rake features:new_qemu'
|
27
|
+
}
|
28
|
+
post {
|
29
|
+
always {
|
30
|
+
sh 'bundle exec rake features:clean'
|
31
|
+
}
|
32
|
+
failure {
|
33
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
34
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
35
|
+
body: "${env.BUILD_URL}"
|
36
|
+
}
|
37
|
+
}
|
38
|
+
}
|
39
|
+
stage("Xenial, Ruby 2.3.1") {
|
40
|
+
agent {
|
41
|
+
dockerfile {
|
42
|
+
dir 'xenial'
|
43
|
+
label "docker"
|
44
|
+
}
|
45
|
+
}
|
46
|
+
steps {
|
47
|
+
sh 'bundle install --deployment'
|
48
|
+
sh 'bundle exec rake features:new_qemu'
|
49
|
+
}
|
50
|
+
post {
|
51
|
+
always {
|
52
|
+
sh 'bundle exec rake features:clean'
|
53
|
+
}
|
54
|
+
failure {
|
55
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
56
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
57
|
+
body: "${env.BUILD_URL}"
|
58
|
+
}
|
59
|
+
}
|
60
|
+
}
|
61
|
+
stage("Bionic, Ruby 2.5.1") {
|
62
|
+
agent {
|
63
|
+
dockerfile {
|
64
|
+
dir 'bionic'
|
65
|
+
label "docker"
|
66
|
+
}
|
67
|
+
}
|
68
|
+
steps {
|
69
|
+
sh 'bundle install --deployment'
|
70
|
+
sh 'bundle exec rake features:new_qemu'
|
71
|
+
}
|
72
|
+
post {
|
73
|
+
always {
|
74
|
+
sh 'bundle exec rake features:clean'
|
75
|
+
}
|
76
|
+
failure {
|
77
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
78
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
79
|
+
body: "${env.BUILD_URL}"
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
stage("Focal, Ruby 2.7.0") {
|
84
|
+
agent {
|
85
|
+
dockerfile {
|
86
|
+
dir 'focal'
|
87
|
+
label "docker"
|
88
|
+
}
|
89
|
+
}
|
90
|
+
steps {
|
91
|
+
sh 'bundle install --deployment'
|
92
|
+
sh 'bundle exec rake features:new_qemu'
|
93
|
+
}
|
94
|
+
post {
|
95
|
+
always {
|
96
|
+
sh 'bundle exec rake features:clean'
|
97
|
+
}
|
98
|
+
failure {
|
99
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
100
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
101
|
+
body: "${env.BUILD_URL}"
|
102
|
+
}
|
103
|
+
}
|
104
|
+
}
|
105
|
+
}
|
106
|
+
post {
|
107
|
+
failure {
|
108
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
109
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
110
|
+
body: "${env.BUILD_URL}"
|
111
|
+
}
|
112
|
+
}
|
113
|
+
}
|
data/README.md
CHANGED
data/Rakefile
CHANGED
@@ -2,19 +2,19 @@ def dump_load_path
|
|
2
2
|
puts $LOAD_PATH.join("\n")
|
3
3
|
found = nil
|
4
4
|
$LOAD_PATH.each do |path|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
puts "!! no rake_task"
|
14
|
-
end
|
5
|
+
next unless File.exist?(File.join(path, 'rspec'))
|
6
|
+
|
7
|
+
puts "Found rspec in #{path}"
|
8
|
+
if File.exist?(File.join(path, 'rspec', 'core'))
|
9
|
+
puts 'Found core'
|
10
|
+
if File.exist?(File.join(path, 'rspec', 'core', 'rake_task'))
|
11
|
+
puts 'Found rake_task'
|
12
|
+
found = path
|
15
13
|
else
|
16
|
-
puts
|
14
|
+
puts '!! no rake_task'
|
17
15
|
end
|
16
|
+
else
|
17
|
+
puts '!!! no core'
|
18
18
|
end
|
19
19
|
end
|
20
20
|
if found.nil?
|
@@ -26,13 +26,6 @@ end
|
|
26
26
|
require 'bundler'
|
27
27
|
require 'rake/clean'
|
28
28
|
|
29
|
-
begin
|
30
|
-
require 'rspec/core/rake_task'
|
31
|
-
rescue LoadError
|
32
|
-
dump_load_path
|
33
|
-
raise
|
34
|
-
end
|
35
|
-
|
36
29
|
require 'cucumber'
|
37
30
|
require 'cucumber/rake/task'
|
38
31
|
gem 'rdoc' # we need the installed RDoc gem, not the system one
|
@@ -42,13 +35,7 @@ include Rake::DSL
|
|
42
35
|
|
43
36
|
Bundler::GemHelper.install_tasks
|
44
37
|
|
45
|
-
|
46
|
-
RSpec::Core::RakeTask.new do |t|
|
47
|
-
# Put spec opts in a file named .rspec in root
|
48
|
-
end
|
49
|
-
|
50
|
-
|
51
|
-
CUKE_RESULTS = 'results.html'
|
38
|
+
CUKE_RESULTS = 'results.html'.freeze
|
52
39
|
CLEAN << CUKE_RESULTS
|
53
40
|
namespace(:features) do
|
54
41
|
Cucumber::Rake::Task.new(:new_qemu) do |t|
|
@@ -61,14 +48,22 @@ namespace(:features) do
|
|
61
48
|
t.fork = false
|
62
49
|
t.profile = 'old-qemu-image'
|
63
50
|
end
|
51
|
+
|
52
|
+
task(:clean) do
|
53
|
+
if ENV['ORBIT_USER'] && ENV['ORBIT_KEY']
|
54
|
+
%w[temp_dlo test_upload].each do |test_file|
|
55
|
+
sh "swift --quiet --insecure -A #{ENV['ORBIT_URL'] || 'https://orbit.gb1.brightbox.com/v1'} -U #{ENV['ORBIT_USER']} -K #{ENV['ORBIT_KEY']} delete #{test_file}" do |ok, res|
|
56
|
+
puts "Delete failed: #{res.exitstatus}" unless ok
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
64
61
|
end
|
65
62
|
|
66
63
|
Rake::RDocTask.new do |rd|
|
67
|
-
|
68
|
-
rd.main = "README.rdoc"
|
69
|
-
|
70
|
-
rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
|
71
|
-
end
|
64
|
+
rd.main = 'README.rdoc'
|
72
65
|
|
73
|
-
|
66
|
+
rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb', 'bin/**/*')
|
67
|
+
end
|
74
68
|
|
69
|
+
task default: [:spec, 'features:new_qemu']
|
data/bionic/Dockerfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
FROM ubuntu:bionic
|
2
|
+
MAINTAINER support@brightbox.co.uk
|
3
|
+
|
4
|
+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
|
5
|
+
|
6
|
+
#RUN apt-add-repository ppa:brightbox/ruby-ng
|
7
|
+
|
8
|
+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
9
|
+
build-essential \
|
10
|
+
git \
|
11
|
+
qemu-utils \
|
12
|
+
liblz4-tool \
|
13
|
+
curl \
|
14
|
+
ruby \
|
15
|
+
ruby-dev \
|
16
|
+
python-swiftclient \
|
17
|
+
ruby-bundler
|
18
|
+
|
19
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
20
|
+
|
data/centos7/Dockerfile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
FROM centos:7
|
2
|
+
MAINTAINER support@brightbox.co.uk
|
3
|
+
|
4
|
+
RUN yum update -y && yum -y install \
|
5
|
+
epel-release \
|
6
|
+
gcc \
|
7
|
+
make \
|
8
|
+
git \
|
9
|
+
qemu-img \
|
10
|
+
ruby-devel \
|
11
|
+
rubygems \
|
12
|
+
rubygem-bundler
|
13
|
+
|
14
|
+
RUN yum -y install centos-release-openstack-train && \
|
15
|
+
yum -y install python-swiftclient
|
16
|
+
|
17
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
18
|
+
|
@@ -18,17 +18,24 @@ def fetch_orbit_token
|
|
18
18
|
@current_url_target = ENV['ORBIT_URL'] || 'https://orbit.brightbox.com/v1'
|
19
19
|
cmd = "curl -I #{@current_url_target} -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
|
20
20
|
run_simple(unescape(cmd))
|
21
|
-
stdout_from(cmd)
|
21
|
+
output = stdout_from(cmd)
|
22
|
+
output.each_line do |line|
|
22
23
|
key, value = line.strip.split(/\s*:\s*/,2)
|
23
|
-
|
24
|
-
|
24
|
+
case key.to_s.downcase
|
25
|
+
when 'x-auth-token'
|
26
|
+
@current_auth_token = value
|
27
|
+
when 'x-storage-url'
|
28
|
+
@current_storage_url = value
|
29
|
+
end
|
30
|
+
end
|
31
|
+
unless @current_auth_token && @current_storage_url
|
32
|
+
raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.\n#{output}")
|
25
33
|
end
|
26
|
-
raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.") unless @current_auth_token && @current_storage_url
|
27
34
|
end
|
28
35
|
|
29
36
|
def create_download_file(filetype, name, container)
|
30
37
|
upload_url = File.join(@current_storage_url, container, name)
|
31
|
-
tempfile = '
|
38
|
+
tempfile = 'fred55322'
|
32
39
|
FileUtils.rm_f(Dir.glob(tempfile+'*'))
|
33
40
|
format = 'raw'
|
34
41
|
size = '1G'
|
@@ -196,26 +203,25 @@ Then(/^a qcow2 entry should exist in container "(.*?)" called "(.*?)"$/) do |con
|
|
196
203
|
end
|
197
204
|
|
198
205
|
Before('@orbit') do
|
206
|
+
puts "Running Orbit Before"
|
199
207
|
fetch_orbit_token
|
200
|
-
|
208
|
+
$first_time ||= false
|
209
|
+
unless $first_time
|
210
|
+
puts("Clearing up temporary areas")
|
201
211
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test")
|
202
212
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete temp_dlo")
|
203
213
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test_upload")
|
204
|
-
|
205
|
-
end
|
206
|
-
|
207
|
-
Before('@orbitdownloads') do
|
208
|
-
$orbit_setup ||= false
|
209
|
-
unless $orbit_setup
|
214
|
+
puts("Setting up downloads")
|
210
215
|
create_container('test')
|
211
216
|
create_download_file('qcow2', 'img-qcow2', 'test')
|
212
217
|
create_download_file('lz4', 'img-lz4cy', 'test')
|
213
218
|
create_download_file('zero', 'img-zeroy', 'test')
|
214
|
-
$
|
219
|
+
$first_time = true
|
215
220
|
end
|
216
221
|
end
|
217
222
|
|
218
223
|
After('@orbit') do
|
224
|
+
puts "Running Orbit After"
|
219
225
|
if @current_auth_token && @current_container
|
220
226
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete #{@current_container} #{@current_name}")
|
221
227
|
@current_container = @current_name = nil
|
@@ -1,8 +1,10 @@
|
|
1
1
|
Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
|
2
2
|
case format
|
3
3
|
when 'lz4'
|
4
|
-
assert_exact_output(
|
5
|
-
|
4
|
+
assert_exact_output(
|
5
|
+
[0x184D2204].pack('V'),
|
6
|
+
File.open(filename, 'rb') { |f| f.read(4) } || String.new
|
7
|
+
)
|
6
8
|
when 'qcow2v3', 'qcow3'
|
7
9
|
steps %{
|
8
10
|
When I successfully run `qemu-img info #{filename}`
|
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
|
|
10
12
|
And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
|
11
13
|
}
|
12
14
|
else
|
13
|
-
assert_no_partial_output(
|
14
|
-
|
15
|
+
assert_no_partial_output(
|
16
|
+
[0x184D2204].pack('V'),
|
17
|
+
File.open(filename, 'rb') { |f| f.read(4) } || String.new
|
18
|
+
)
|
15
19
|
steps(%{
|
16
20
|
When I successfully run `qemu-img info #{filename}`
|
17
21
|
Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"
|