uricp 0.0.11 → 0.0.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +2 -1
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +15 -19
- data/Jenkinsfile +113 -0
- data/README.md +2 -2
- data/Rakefile +26 -31
- data/bionic/Dockerfile +20 -0
- data/centos7/Dockerfile +20 -0
- data/features/auth_download.feature +17 -0
- data/features/step_definitions/orbit_steps.rb +19 -13
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp.rb +3 -29
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +11 -11
- data/lib/uricp/strategy/cached_get.rb +14 -20
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +27 -19
- data/lib/uricp/strategy/local_convert.rb +9 -16
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +7 -13
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +11 -17
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +44 -0
- data/lib/uricp/strategy/piped_remote_get.rb +28 -19
- data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +14 -13
- data/lib/uricp/version.rb +2 -2
- data/trusty/Dockerfile +20 -0
- data/uricp.gemspec +3 -4
- data/xenial/Dockerfile +20 -0
- metadata +39 -51
- data/Dockerfile_centos6.6 +0 -39
- data/Dockerfile_centos7 +0 -39
- data/Dockerfile_trusty-ruby193 +0 -32
- data/README.rdoc +0 -23
- data/spec/something_spec.rb +0 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d7db104674249fa9443799b004c877667e22b142d26a7c7a2220ef668865dfbc
|
4
|
+
data.tar.gz: d333e085bb1b666e09883fbd9b9006334d6c15acd17ac781629697bcc0766e5a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7e16104719cca1e2def15b5d9b2d7cbdbcb84b68f7d69a566b8eddd64e699654ea72f066102bb8949e2eb583ffd54aad9549b5094746747382930cf18a1de5a8
|
7
|
+
data.tar.gz: b0f839f96f8975ce95121085fdba25ecb1451fc38dbc975de250a6277ffe584a5548f6033138cb46ae8f200a0e6c063e45b2eb25b26ea4e6cefad0d592b91294
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
data/Gemfile.lock
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
uricp (0.0.
|
4
|
+
uricp (0.0.16)
|
5
5
|
filesize (= 0.0.2)
|
6
|
-
methadone (~>
|
6
|
+
methadone (~> 2.0.0)
|
7
7
|
open4 (~> 1.3.0)
|
8
8
|
sendfile (~> 1.2.0)
|
9
9
|
|
@@ -14,8 +14,8 @@ GEM
|
|
14
14
|
childprocess (>= 0.3.6)
|
15
15
|
cucumber (>= 1.1.1)
|
16
16
|
rspec-expectations (>= 2.7.0)
|
17
|
-
builder (3.2.
|
18
|
-
childprocess (0.
|
17
|
+
builder (3.2.3)
|
18
|
+
childprocess (0.9.0)
|
19
19
|
ffi (~> 1.0, >= 1.0.11)
|
20
20
|
cucumber (1.3.20)
|
21
21
|
builder (>= 2.1.2)
|
@@ -23,29 +23,23 @@ GEM
|
|
23
23
|
gherkin (~> 2.12)
|
24
24
|
multi_json (>= 1.7.5, < 2.0)
|
25
25
|
multi_test (>= 0.1.2)
|
26
|
-
diff-lcs (1.
|
27
|
-
ffi (1.9.
|
26
|
+
diff-lcs (1.3)
|
27
|
+
ffi (1.9.23)
|
28
28
|
filesize (0.0.2)
|
29
29
|
gherkin (2.12.2)
|
30
30
|
multi_json (~> 1.3)
|
31
31
|
inifile (1.1.0)
|
32
|
-
json (1.8.
|
33
|
-
methadone (
|
32
|
+
json (1.8.6)
|
33
|
+
methadone (2.0.0)
|
34
34
|
bundler
|
35
|
-
multi_json (1.
|
35
|
+
multi_json (1.13.1)
|
36
36
|
multi_test (0.1.2)
|
37
37
|
open4 (1.3.4)
|
38
38
|
rake (10.4.2)
|
39
|
-
rdoc (4.2.
|
39
|
+
rdoc (4.2.2)
|
40
40
|
json (~> 1.4)
|
41
|
-
rspec (2.99.0)
|
42
|
-
rspec-core (~> 2.99.0)
|
43
|
-
rspec-expectations (~> 2.99.0)
|
44
|
-
rspec-mocks (~> 2.99.0)
|
45
|
-
rspec-core (2.99.2)
|
46
41
|
rspec-expectations (2.99.2)
|
47
42
|
diff-lcs (>= 1.1.3, < 2.0)
|
48
|
-
rspec-mocks (2.99.4)
|
49
43
|
sendfile (1.2.2)
|
50
44
|
|
51
45
|
PLATFORMS
|
@@ -56,7 +50,9 @@ DEPENDENCIES
|
|
56
50
|
bundler (~> 1.7)
|
57
51
|
cucumber (~> 1.3)
|
58
52
|
inifile (~> 1.1)
|
59
|
-
rake
|
60
|
-
rdoc
|
61
|
-
rspec (~> 2.99)
|
53
|
+
rake (~> 10.4.0)
|
54
|
+
rdoc (~> 4.2.0)
|
62
55
|
uricp!
|
56
|
+
|
57
|
+
BUNDLED WITH
|
58
|
+
1.16.1
|
data/Jenkinsfile
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
pipeline {
|
2
|
+
environment {
|
3
|
+
ORBIT = credentials('ced8ecd0-dbe4-4bf6-bdde-101a661565f5')
|
4
|
+
ORBIT_URL = credentials('c0c95c56-b4e9-45c2-85c5-bf292aef7301')
|
5
|
+
ORBIT_USER = "${ORBIT_USR}"
|
6
|
+
ORBIT_KEY = "${ORBIT_PSW}"
|
7
|
+
}
|
8
|
+
options {
|
9
|
+
disableConcurrentBuilds()
|
10
|
+
buildDiscarder(logRotator(numToKeepStr: '5'))
|
11
|
+
}
|
12
|
+
triggers {
|
13
|
+
cron('@weekly')
|
14
|
+
}
|
15
|
+
agent none
|
16
|
+
stages {
|
17
|
+
stage("CentOS 7") {
|
18
|
+
agent {
|
19
|
+
dockerfile {
|
20
|
+
dir 'centos7'
|
21
|
+
label "docker"
|
22
|
+
}
|
23
|
+
}
|
24
|
+
steps {
|
25
|
+
sh 'bundle install --deployment'
|
26
|
+
sh 'bundle exec rake features:new_qemu'
|
27
|
+
}
|
28
|
+
post {
|
29
|
+
always {
|
30
|
+
sh 'bundle exec rake features:clean'
|
31
|
+
}
|
32
|
+
failure {
|
33
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
34
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
35
|
+
body: "${env.BUILD_URL}"
|
36
|
+
}
|
37
|
+
}
|
38
|
+
}
|
39
|
+
stage("Trusty, Ruby 1.9.3") {
|
40
|
+
agent {
|
41
|
+
dockerfile {
|
42
|
+
dir 'trusty'
|
43
|
+
label "docker"
|
44
|
+
}
|
45
|
+
}
|
46
|
+
steps {
|
47
|
+
sh 'bundle install --deployment'
|
48
|
+
sh 'bundle exec rake features:new_qemu'
|
49
|
+
}
|
50
|
+
post {
|
51
|
+
always {
|
52
|
+
sh 'bundle exec rake features:clean'
|
53
|
+
}
|
54
|
+
failure {
|
55
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
56
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
57
|
+
body: "${env.BUILD_URL}"
|
58
|
+
}
|
59
|
+
}
|
60
|
+
}
|
61
|
+
stage("Xenial, Ruby 2.3.1") {
|
62
|
+
agent {
|
63
|
+
dockerfile {
|
64
|
+
dir 'xenial'
|
65
|
+
label "docker"
|
66
|
+
}
|
67
|
+
}
|
68
|
+
steps {
|
69
|
+
sh 'bundle install --deployment'
|
70
|
+
sh 'bundle exec rake features:new_qemu'
|
71
|
+
}
|
72
|
+
post {
|
73
|
+
always {
|
74
|
+
sh 'bundle exec rake features:clean'
|
75
|
+
}
|
76
|
+
failure {
|
77
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
78
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
79
|
+
body: "${env.BUILD_URL}"
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
stage("Bionic, Ruby 2.5.1") {
|
84
|
+
agent {
|
85
|
+
dockerfile {
|
86
|
+
dir 'bionic'
|
87
|
+
label "docker"
|
88
|
+
}
|
89
|
+
}
|
90
|
+
steps {
|
91
|
+
sh 'bundle install --deployment'
|
92
|
+
sh 'bundle exec rake features:new_qemu'
|
93
|
+
}
|
94
|
+
post {
|
95
|
+
always {
|
96
|
+
sh 'bundle exec rake features:clean'
|
97
|
+
}
|
98
|
+
failure {
|
99
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
100
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
101
|
+
body: "${env.BUILD_URL}"
|
102
|
+
}
|
103
|
+
}
|
104
|
+
}
|
105
|
+
}
|
106
|
+
post {
|
107
|
+
failure {
|
108
|
+
mail to: 'sysadmin@brightbox.co.uk',
|
109
|
+
subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
|
110
|
+
body: "${env.BUILD_URL}"
|
111
|
+
}
|
112
|
+
}
|
113
|
+
}
|
data/README.md
CHANGED
data/Rakefile
CHANGED
@@ -2,19 +2,19 @@ def dump_load_path
|
|
2
2
|
puts $LOAD_PATH.join("\n")
|
3
3
|
found = nil
|
4
4
|
$LOAD_PATH.each do |path|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
puts "!! no rake_task"
|
14
|
-
end
|
5
|
+
next unless File.exist?(File.join(path, 'rspec'))
|
6
|
+
|
7
|
+
puts "Found rspec in #{path}"
|
8
|
+
if File.exist?(File.join(path, 'rspec', 'core'))
|
9
|
+
puts 'Found core'
|
10
|
+
if File.exist?(File.join(path, 'rspec', 'core', 'rake_task'))
|
11
|
+
puts 'Found rake_task'
|
12
|
+
found = path
|
15
13
|
else
|
16
|
-
puts
|
14
|
+
puts '!! no rake_task'
|
17
15
|
end
|
16
|
+
else
|
17
|
+
puts '!!! no core'
|
18
18
|
end
|
19
19
|
end
|
20
20
|
if found.nil?
|
@@ -26,13 +26,6 @@ end
|
|
26
26
|
require 'bundler'
|
27
27
|
require 'rake/clean'
|
28
28
|
|
29
|
-
begin
|
30
|
-
require 'rspec/core/rake_task'
|
31
|
-
rescue LoadError
|
32
|
-
dump_load_path
|
33
|
-
raise
|
34
|
-
end
|
35
|
-
|
36
29
|
require 'cucumber'
|
37
30
|
require 'cucumber/rake/task'
|
38
31
|
gem 'rdoc' # we need the installed RDoc gem, not the system one
|
@@ -42,13 +35,7 @@ include Rake::DSL
|
|
42
35
|
|
43
36
|
Bundler::GemHelper.install_tasks
|
44
37
|
|
45
|
-
|
46
|
-
RSpec::Core::RakeTask.new do |t|
|
47
|
-
# Put spec opts in a file named .rspec in root
|
48
|
-
end
|
49
|
-
|
50
|
-
|
51
|
-
CUKE_RESULTS = 'results.html'
|
38
|
+
CUKE_RESULTS = 'results.html'.freeze
|
52
39
|
CLEAN << CUKE_RESULTS
|
53
40
|
namespace(:features) do
|
54
41
|
Cucumber::Rake::Task.new(:new_qemu) do |t|
|
@@ -61,14 +48,22 @@ namespace(:features) do
|
|
61
48
|
t.fork = false
|
62
49
|
t.profile = 'old-qemu-image'
|
63
50
|
end
|
51
|
+
|
52
|
+
task(:clean) do
|
53
|
+
if ENV['ORBIT_USER'] && ENV['ORBIT_KEY']
|
54
|
+
%w[temp_dlo test_upload].each do |test_file|
|
55
|
+
sh "swift --quiet --insecure -A #{ENV['ORBIT_URL'] || 'https://orbit.gb1.brightbox.com/v1'} -U #{ENV['ORBIT_USER']} -K #{ENV['ORBIT_KEY']} delete #{test_file}" do |ok, res|
|
56
|
+
puts "Delete failed: #{res.exitstatus}" unless ok
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
64
61
|
end
|
65
62
|
|
66
63
|
Rake::RDocTask.new do |rd|
|
67
|
-
|
68
|
-
rd.main = "README.rdoc"
|
69
|
-
|
70
|
-
rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
|
71
|
-
end
|
64
|
+
rd.main = 'README.rdoc'
|
72
65
|
|
73
|
-
|
66
|
+
rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb', 'bin/**/*')
|
67
|
+
end
|
74
68
|
|
69
|
+
task default: [:spec, 'features:new_qemu']
|
data/bionic/Dockerfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
FROM ubuntu:bionic
|
2
|
+
MAINTAINER support@brightbox.co.uk
|
3
|
+
|
4
|
+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
|
5
|
+
|
6
|
+
#RUN apt-add-repository ppa:brightbox/ruby-ng
|
7
|
+
|
8
|
+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
9
|
+
build-essential \
|
10
|
+
git \
|
11
|
+
qemu-utils \
|
12
|
+
liblz4-tool \
|
13
|
+
curl \
|
14
|
+
ruby \
|
15
|
+
ruby-dev \
|
16
|
+
python-swiftclient \
|
17
|
+
ruby-bundler
|
18
|
+
|
19
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
20
|
+
|
data/centos7/Dockerfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
FROM centos:7
|
2
|
+
MAINTAINER support@brightbox.co.uk
|
3
|
+
|
4
|
+
RUN yum update -y && yum -y install \
|
5
|
+
epel-release \
|
6
|
+
gcc \
|
7
|
+
make \
|
8
|
+
git \
|
9
|
+
qemu-img \
|
10
|
+
ruby-devel \
|
11
|
+
rubygems \
|
12
|
+
rubygem-bundler
|
13
|
+
|
14
|
+
RUN yum -y install python-setuptools && \
|
15
|
+
easy_install pip && \
|
16
|
+
pip install --upgrade setuptools && \
|
17
|
+
pip install python-swiftclient
|
18
|
+
|
19
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
20
|
+
|
@@ -37,6 +37,14 @@ Feature: Authenticated download of images from orbit
|
|
37
37
|
And a 0 byte file named "/tmp/uricp/cache/img-zeroy" should exist
|
38
38
|
And the file named "/tmp/uricp/cache/img-zeroy" should have a file format of "raw"
|
39
39
|
|
40
|
+
Scenario: zero download to raw, cache, size limited
|
41
|
+
Given a correctly initialised cache at "/tmp/uricp"
|
42
|
+
When I retrieve "img-zeroy" with options "--cache=/tmp/uricp --target-format=raw --max-cache=8MB" from container "test" into "file:////tmp/uricp/srv-testx"
|
43
|
+
Then a file named "/tmp/uricp/srv-testx" should exist
|
44
|
+
And the file named "/tmp/uricp/srv-testx" should have a file format of "raw"
|
45
|
+
And a 0 byte file named "/tmp/uricp/cache/img-zeroy" should exist
|
46
|
+
And the file named "/tmp/uricp/cache/img-zeroy" should have a file format of "raw"
|
47
|
+
|
40
48
|
Scenario: qcow download convert to raw, cache
|
41
49
|
Given a correctly initialised cache at "/tmp/uricp"
|
42
50
|
When I retrieve "img-qcow2" with options "--target-format=raw --cache=/tmp/uricp" from container "test" into "file:///tmp/uricp/srv-test4"
|
@@ -143,3 +151,12 @@ Feature: Authenticated download of images from orbit
|
|
143
151
|
And the file named "/tmp/uricp/srv-testa" should have a file format of "qcow2"
|
144
152
|
And a file named "/tmp/uricp/cache/img-lz4cy" should exist
|
145
153
|
And the file named "/tmp/uricp/cache/img-lz4cy" should have a file format of "lz4"
|
154
|
+
|
155
|
+
Scenario: lz4 download sparse convert to raw, cache, overwrite file
|
156
|
+
Given a correctly initialised cache at "/tmp/uricp"
|
157
|
+
And a 102400 byte file named "/tmp/uricp/srv-test9"
|
158
|
+
When I retrieve "img-lz4cy" with options "--force --target-format=raw --cache=/tmp/uricp" from container "test" into "file:///tmp/uricp/srv-test9"
|
159
|
+
Then a file named "/tmp/uricp/srv-test9" should exist
|
160
|
+
And the file named "/tmp/uricp/srv-test9" should have a file format of "raw"
|
161
|
+
And a file named "/tmp/uricp/cache/img-lz4cy" should exist
|
162
|
+
And the file named "/tmp/uricp/cache/img-lz4cy" should have a file format of "lz4"
|
@@ -18,17 +18,24 @@ def fetch_orbit_token
|
|
18
18
|
@current_url_target = ENV['ORBIT_URL'] || 'https://orbit.brightbox.com/v1'
|
19
19
|
cmd = "curl -I #{@current_url_target} -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
|
20
20
|
run_simple(unescape(cmd))
|
21
|
-
stdout_from(cmd)
|
21
|
+
output = stdout_from(cmd)
|
22
|
+
output.each_line do |line|
|
22
23
|
key, value = line.strip.split(/\s*:\s*/,2)
|
23
|
-
|
24
|
-
|
24
|
+
case key.to_s.downcase
|
25
|
+
when 'x-auth-token'
|
26
|
+
@current_auth_token = value
|
27
|
+
when 'x-storage-url'
|
28
|
+
@current_storage_url = value
|
29
|
+
end
|
30
|
+
end
|
31
|
+
unless @current_auth_token && @current_storage_url
|
32
|
+
raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.\n#{output}")
|
25
33
|
end
|
26
|
-
raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.") unless @current_auth_token && @current_storage_url
|
27
34
|
end
|
28
35
|
|
29
36
|
def create_download_file(filetype, name, container)
|
30
37
|
upload_url = File.join(@current_storage_url, container, name)
|
31
|
-
tempfile = '
|
38
|
+
tempfile = 'fred55322'
|
32
39
|
FileUtils.rm_f(Dir.glob(tempfile+'*'))
|
33
40
|
format = 'raw'
|
34
41
|
size = '1G'
|
@@ -196,26 +203,25 @@ Then(/^a qcow2 entry should exist in container "(.*?)" called "(.*?)"$/) do |con
|
|
196
203
|
end
|
197
204
|
|
198
205
|
Before('@orbit') do
|
206
|
+
puts "Running Orbit Before"
|
199
207
|
fetch_orbit_token
|
200
|
-
|
208
|
+
$first_time ||= false
|
209
|
+
unless $first_time
|
210
|
+
puts("Clearing up temporary areas")
|
201
211
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test")
|
202
212
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete temp_dlo")
|
203
213
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test_upload")
|
204
|
-
|
205
|
-
end
|
206
|
-
|
207
|
-
Before('@orbitdownloads') do
|
208
|
-
$orbit_setup ||= false
|
209
|
-
unless $orbit_setup
|
214
|
+
puts("Setting up downloads")
|
210
215
|
create_container('test')
|
211
216
|
create_download_file('qcow2', 'img-qcow2', 'test')
|
212
217
|
create_download_file('lz4', 'img-lz4cy', 'test')
|
213
218
|
create_download_file('zero', 'img-zeroy', 'test')
|
214
|
-
$
|
219
|
+
$first_time = true
|
215
220
|
end
|
216
221
|
end
|
217
222
|
|
218
223
|
After('@orbit') do
|
224
|
+
puts "Running Orbit After"
|
219
225
|
if @current_auth_token && @current_container
|
220
226
|
system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete #{@current_container} #{@current_name}")
|
221
227
|
@current_container = @current_name = nil
|