uricp 0.0.13 → 0.0.18

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -1
  3. data/.rubocop.yml +3 -0
  4. data/Gemfile.lock +20 -23
  5. data/Jenkinsfile +113 -0
  6. data/README.md +2 -2
  7. data/Rakefile +26 -31
  8. data/bionic/Dockerfile +20 -0
  9. data/centos7/Dockerfile +18 -0
  10. data/features/step_definitions/orbit_steps.rb +19 -13
  11. data/features/step_definitions/uricp_steps.rb +8 -4
  12. data/focal/Dockerfile +21 -0
  13. data/lib/segment_upload.rb +20 -22
  14. data/lib/uricp.rb +3 -29
  15. data/lib/uricp/curl_primitives.rb +31 -33
  16. data/lib/uricp/orbit_auth.rb +23 -27
  17. data/lib/uricp/segmenter.rb +12 -16
  18. data/lib/uricp/strategy/cache_common.rb +11 -11
  19. data/lib/uricp/strategy/cached_get.rb +14 -20
  20. data/lib/uricp/strategy/cleaner.rb +2 -8
  21. data/lib/uricp/strategy/common.rb +27 -19
  22. data/lib/uricp/strategy/local_convert.rb +10 -17
  23. data/lib/uricp/strategy/local_link.rb +10 -8
  24. data/lib/uricp/strategy/piped_cache.rb +7 -13
  25. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  26. data/lib/uricp/strategy/piped_compress.rb +3 -8
  27. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  28. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  29. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  30. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  31. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  32. data/lib/uricp/strategy/piped_rbd_get.rb +46 -0
  33. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  34. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  35. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  36. data/lib/uricp/strategy/remote_put.rb +11 -17
  37. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  38. data/lib/uricp/strategy/sweeper.rb +2 -8
  39. data/lib/uricp/uri_strategy.rb +14 -13
  40. data/lib/uricp/version.rb +4 -2
  41. data/uricp.gemspec +3 -4
  42. data/xenial/Dockerfile +20 -0
  43. metadata +39 -51
  44. data/Dockerfile_centos6.6 +0 -39
  45. data/Dockerfile_centos7 +0 -39
  46. data/Dockerfile_trusty-ruby193 +0 -32
  47. data/README.rdoc +0 -23
  48. data/spec/something_spec.rb +0 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 504ac718e28c260c29336ae9089d9a58d9948f7d
4
- data.tar.gz: ac2c9bc45430805b2a0845e80392e2c75ec2f471
2
+ SHA256:
3
+ metadata.gz: d9944582044883ff45f3847042ca62016416d06198a0af41751de93ab992c646
4
+ data.tar.gz: 19c14777647bb01fec3a4e6214e8183b29432843bf47b8b0c818d0401dafaea1
5
5
  SHA512:
6
- metadata.gz: b1518d8d9ae106c51d1eceb9ad5d50f0d93e3b7fbf9df43a8cca797c42d2dbdb6f44c96a7a3a4636562fd42523ca880a3859ff2b70374d70aebcf205e238a5d9
7
- data.tar.gz: 419cedbe0b8be9fcbef9ba4634906adb4ad6f346b6bcdbae06d627bea6efc4c21c6d71c646d2d0d1343081c3e0cfd34df3d273cedd4362db0aafbfb55112df93
6
+ metadata.gz: c157e985f79891ca962fb0b792dab2a4fab89bd651c15b940e75ba4baf0e86a42922b9dc8a0f96fa53b8de4151d3e9b6aeefac256ffe59faa2ed6ad5becfad47
7
+ data.tar.gz: 45444dfcbc92079f7221d0f239315db22f6af749843f6a89312e949c1f12068a2254f6ffd8830a4ab00833e3b66d16d259cfcccb9388f24dde9ff23fbad99ee2
data/.gitignore CHANGED
@@ -4,4 +4,5 @@ coverage
4
4
  pkg
5
5
  html
6
6
  *.swp
7
- Dockerfile
7
+ vendor
8
+ .bundle
data/.rubocop.yml ADDED
@@ -0,0 +1,3 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4
3
+
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- uricp (0.0.13)
4
+ uricp (0.0.18)
5
5
  filesize (= 0.0.2)
6
- methadone (~> 1.8.0)
6
+ methadone (~> 2.0.0)
7
7
  open4 (~> 1.3.0)
8
8
  sendfile (~> 1.2.0)
9
9
 
@@ -14,38 +14,33 @@ GEM
14
14
  childprocess (>= 0.3.6)
15
15
  cucumber (>= 1.1.1)
16
16
  rspec-expectations (>= 2.7.0)
17
- builder (3.2.2)
18
- childprocess (0.5.7)
19
- ffi (~> 1.0, >= 1.0.11)
17
+ builder (3.2.4)
18
+ childprocess (1.0.1)
19
+ rake (< 13.0)
20
20
  cucumber (1.3.20)
21
21
  builder (>= 2.1.2)
22
22
  diff-lcs (>= 1.1.3)
23
23
  gherkin (~> 2.12)
24
24
  multi_json (>= 1.7.5, < 2.0)
25
25
  multi_test (>= 0.1.2)
26
- diff-lcs (1.2.5)
27
- ffi (1.9.10)
26
+ diff-lcs (1.4.4)
28
27
  filesize (0.0.2)
29
28
  gherkin (2.12.2)
30
29
  multi_json (~> 1.3)
31
30
  inifile (1.1.0)
32
- json (1.8.3)
33
- methadone (1.8.0)
31
+ json (1.8.6)
32
+ methadone (2.0.2)
34
33
  bundler
35
- multi_json (1.11.2)
34
+ multi_json (1.15.0)
36
35
  multi_test (0.1.2)
37
36
  open4 (1.3.4)
38
- rake (10.4.2)
39
- rdoc (4.2.0)
37
+ rake (12.3.3)
38
+ rdoc (4.2.2)
40
39
  json (~> 1.4)
41
- rspec (2.99.0)
42
- rspec-core (~> 2.99.0)
43
- rspec-expectations (~> 2.99.0)
44
- rspec-mocks (~> 2.99.0)
45
- rspec-core (2.99.2)
46
- rspec-expectations (2.99.2)
47
- diff-lcs (>= 1.1.3, < 2.0)
48
- rspec-mocks (2.99.4)
40
+ rspec-expectations (3.10.1)
41
+ diff-lcs (>= 1.2.0, < 2.0)
42
+ rspec-support (~> 3.10.0)
43
+ rspec-support (3.10.2)
49
44
  sendfile (1.2.2)
50
45
 
51
46
  PLATFORMS
@@ -56,7 +51,9 @@ DEPENDENCIES
56
51
  bundler (~> 1.7)
57
52
  cucumber (~> 1.3)
58
53
  inifile (~> 1.1)
59
- rake
60
- rdoc
61
- rspec (~> 2.99)
54
+ rake (~> 12.3)
55
+ rdoc (~> 4.2.0)
62
56
  uricp!
57
+
58
+ BUNDLED WITH
59
+ 1.17.3
data/Jenkinsfile ADDED
@@ -0,0 +1,113 @@
1
+ pipeline {
2
+ environment {
3
+ ORBIT = credentials('ced8ecd0-dbe4-4bf6-bdde-101a661565f5')
4
+ ORBIT_URL = credentials('c0c95c56-b4e9-45c2-85c5-bf292aef7301')
5
+ ORBIT_USER = "${ORBIT_USR}"
6
+ ORBIT_KEY = "${ORBIT_PSW}"
7
+ }
8
+ options {
9
+ disableConcurrentBuilds()
10
+ buildDiscarder(logRotator(numToKeepStr: '5'))
11
+ }
12
+ triggers {
13
+ cron('@weekly')
14
+ }
15
+ agent none
16
+ stages {
17
+ stage("CentOS 7") {
18
+ agent {
19
+ dockerfile {
20
+ dir 'centos7'
21
+ label "docker"
22
+ }
23
+ }
24
+ steps {
25
+ sh 'bundle install --deployment'
26
+ sh 'bundle exec rake features:new_qemu'
27
+ }
28
+ post {
29
+ always {
30
+ sh 'bundle exec rake features:clean'
31
+ }
32
+ failure {
33
+ mail to: 'sysadmin@brightbox.co.uk',
34
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
35
+ body: "${env.BUILD_URL}"
36
+ }
37
+ }
38
+ }
39
+ stage("Xenial, Ruby 2.3.1") {
40
+ agent {
41
+ dockerfile {
42
+ dir 'xenial'
43
+ label "docker"
44
+ }
45
+ }
46
+ steps {
47
+ sh 'bundle install --deployment'
48
+ sh 'bundle exec rake features:new_qemu'
49
+ }
50
+ post {
51
+ always {
52
+ sh 'bundle exec rake features:clean'
53
+ }
54
+ failure {
55
+ mail to: 'sysadmin@brightbox.co.uk',
56
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
57
+ body: "${env.BUILD_URL}"
58
+ }
59
+ }
60
+ }
61
+ stage("Bionic, Ruby 2.5.1") {
62
+ agent {
63
+ dockerfile {
64
+ dir 'bionic'
65
+ label "docker"
66
+ }
67
+ }
68
+ steps {
69
+ sh 'bundle install --deployment'
70
+ sh 'bundle exec rake features:new_qemu'
71
+ }
72
+ post {
73
+ always {
74
+ sh 'bundle exec rake features:clean'
75
+ }
76
+ failure {
77
+ mail to: 'sysadmin@brightbox.co.uk',
78
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
79
+ body: "${env.BUILD_URL}"
80
+ }
81
+ }
82
+ }
83
+ stage("Focal, Ruby 2.7.0") {
84
+ agent {
85
+ dockerfile {
86
+ dir 'focal'
87
+ label "docker"
88
+ }
89
+ }
90
+ steps {
91
+ sh 'bundle install --deployment'
92
+ sh 'bundle exec rake features:new_qemu'
93
+ }
94
+ post {
95
+ always {
96
+ sh 'bundle exec rake features:clean'
97
+ }
98
+ failure {
99
+ mail to: 'sysadmin@brightbox.co.uk',
100
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
101
+ body: "${env.BUILD_URL}"
102
+ }
103
+ }
104
+ }
105
+ }
106
+ post {
107
+ failure {
108
+ mail to: 'sysadmin@brightbox.co.uk',
109
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
110
+ body: "${env.BUILD_URL}"
111
+ }
112
+ }
113
+ }
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Uricp
2
2
 
3
- TODO: Write a gem description
3
+ Copy one URI to another with optional cache
4
4
 
5
5
  ## Installation
6
6
 
@@ -20,7 +20,7 @@ Or install it yourself as:
20
20
 
21
21
  ## Usage
22
22
 
23
- TODO: Write usage instructions here
23
+
24
24
 
25
25
  ## Contributing
26
26
 
data/Rakefile CHANGED
@@ -2,19 +2,19 @@ def dump_load_path
2
2
  puts $LOAD_PATH.join("\n")
3
3
  found = nil
4
4
  $LOAD_PATH.each do |path|
5
- if File.exists?(File.join(path,"rspec"))
6
- puts "Found rspec in #{path}"
7
- if File.exists?(File.join(path,"rspec","core"))
8
- puts "Found core"
9
- if File.exists?(File.join(path,"rspec","core","rake_task"))
10
- puts "Found rake_task"
11
- found = path
12
- else
13
- puts "!! no rake_task"
14
- end
5
+ next unless File.exist?(File.join(path, 'rspec'))
6
+
7
+ puts "Found rspec in #{path}"
8
+ if File.exist?(File.join(path, 'rspec', 'core'))
9
+ puts 'Found core'
10
+ if File.exist?(File.join(path, 'rspec', 'core', 'rake_task'))
11
+ puts 'Found rake_task'
12
+ found = path
15
13
  else
16
- puts "!!! no core"
14
+ puts '!! no rake_task'
17
15
  end
16
+ else
17
+ puts '!!! no core'
18
18
  end
19
19
  end
20
20
  if found.nil?
@@ -26,13 +26,6 @@ end
26
26
  require 'bundler'
27
27
  require 'rake/clean'
28
28
 
29
- begin
30
- require 'rspec/core/rake_task'
31
- rescue LoadError
32
- dump_load_path
33
- raise
34
- end
35
-
36
29
  require 'cucumber'
37
30
  require 'cucumber/rake/task'
38
31
  gem 'rdoc' # we need the installed RDoc gem, not the system one
@@ -42,13 +35,7 @@ include Rake::DSL
42
35
 
43
36
  Bundler::GemHelper.install_tasks
44
37
 
45
-
46
- RSpec::Core::RakeTask.new do |t|
47
- # Put spec opts in a file named .rspec in root
48
- end
49
-
50
-
51
- CUKE_RESULTS = 'results.html'
38
+ CUKE_RESULTS = 'results.html'.freeze
52
39
  CLEAN << CUKE_RESULTS
53
40
  namespace(:features) do
54
41
  Cucumber::Rake::Task.new(:new_qemu) do |t|
@@ -61,14 +48,22 @@ namespace(:features) do
61
48
  t.fork = false
62
49
  t.profile = 'old-qemu-image'
63
50
  end
51
+
52
+ task(:clean) do
53
+ if ENV['ORBIT_USER'] && ENV['ORBIT_KEY']
54
+ %w[temp_dlo test_upload].each do |test_file|
55
+ sh "swift --quiet --insecure -A #{ENV['ORBIT_URL'] || 'https://orbit.gb1.brightbox.com/v1'} -U #{ENV['ORBIT_USER']} -K #{ENV['ORBIT_KEY']} delete #{test_file}" do |ok, res|
56
+ puts "Delete failed: #{res.exitstatus}" unless ok
57
+ end
58
+ end
59
+ end
60
+ end
64
61
  end
65
62
 
66
63
  Rake::RDocTask.new do |rd|
67
-
68
- rd.main = "README.rdoc"
69
-
70
- rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
71
- end
64
+ rd.main = 'README.rdoc'
72
65
 
73
- task :default => [:spec,"features:new_qemu"]
66
+ rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb', 'bin/**/*')
67
+ end
74
68
 
69
+ task default: [:spec, 'features:new_qemu']
data/bionic/Dockerfile ADDED
@@ -0,0 +1,20 @@
1
+ FROM ubuntu:bionic
2
+ MAINTAINER support@brightbox.co.uk
3
+
4
+ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
5
+
6
+ #RUN apt-add-repository ppa:brightbox/ruby-ng
7
+
8
+ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
9
+ build-essential \
10
+ git \
11
+ qemu-utils \
12
+ liblz4-tool \
13
+ curl \
14
+ ruby \
15
+ ruby-dev \
16
+ python-swiftclient \
17
+ ruby-bundler
18
+
19
+ RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
20
+
@@ -0,0 +1,18 @@
1
+ FROM centos:7
2
+ MAINTAINER support@brightbox.co.uk
3
+
4
+ RUN yum update -y && yum -y install \
5
+ epel-release \
6
+ gcc \
7
+ make \
8
+ git \
9
+ qemu-img \
10
+ ruby-devel \
11
+ rubygems \
12
+ rubygem-bundler
13
+
14
+ RUN yum -y install centos-release-openstack-train && \
15
+ yum -y install python-swiftclient
16
+
17
+ RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
18
+
@@ -18,17 +18,24 @@ def fetch_orbit_token
18
18
  @current_url_target = ENV['ORBIT_URL'] || 'https://orbit.brightbox.com/v1'
19
19
  cmd = "curl -I #{@current_url_target} -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
20
20
  run_simple(unescape(cmd))
21
- stdout_from(cmd).each_line do |line|
21
+ output = stdout_from(cmd)
22
+ output.each_line do |line|
22
23
  key, value = line.strip.split(/\s*:\s*/,2)
23
- @current_auth_token = value if key == 'X-Auth-Token'
24
- @current_storage_url = value if key == 'X-Storage-Url'
24
+ case key.to_s.downcase
25
+ when 'x-auth-token'
26
+ @current_auth_token = value
27
+ when 'x-storage-url'
28
+ @current_storage_url = value
29
+ end
30
+ end
31
+ unless @current_auth_token && @current_storage_url
32
+ raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.\n#{output}")
25
33
  end
26
- raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.") unless @current_auth_token && @current_storage_url
27
34
  end
28
35
 
29
36
  def create_download_file(filetype, name, container)
30
37
  upload_url = File.join(@current_storage_url, container, name)
31
- tempfile = '/tmp/fred55322'
38
+ tempfile = 'fred55322'
32
39
  FileUtils.rm_f(Dir.glob(tempfile+'*'))
33
40
  format = 'raw'
34
41
  size = '1G'
@@ -196,26 +203,25 @@ Then(/^a qcow2 entry should exist in container "(.*?)" called "(.*?)"$/) do |con
196
203
  end
197
204
 
198
205
  Before('@orbit') do
206
+ puts "Running Orbit Before"
199
207
  fetch_orbit_token
200
- unless $orbit_setup
208
+ $first_time ||= false
209
+ unless $first_time
210
+ puts("Clearing up temporary areas")
201
211
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test")
202
212
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete temp_dlo")
203
213
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test_upload")
204
- end
205
- end
206
-
207
- Before('@orbitdownloads') do
208
- $orbit_setup ||= false
209
- unless $orbit_setup
214
+ puts("Setting up downloads")
210
215
  create_container('test')
211
216
  create_download_file('qcow2', 'img-qcow2', 'test')
212
217
  create_download_file('lz4', 'img-lz4cy', 'test')
213
218
  create_download_file('zero', 'img-zeroy', 'test')
214
- $orbit_setup = true
219
+ $first_time = true
215
220
  end
216
221
  end
217
222
 
218
223
  After('@orbit') do
224
+ puts "Running Orbit After"
219
225
  if @current_auth_token && @current_container
220
226
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete #{@current_container} #{@current_name}")
221
227
  @current_container = @current_name = nil
@@ -1,8 +1,10 @@
1
1
  Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
2
2
  case format
3
3
  when 'lz4'
4
- assert_exact_output([0x184D2204].pack('V'),
5
- File.open(filename, 'rb') {|f| f.read(4) }.to_s)
4
+ assert_exact_output(
5
+ [0x184D2204].pack('V'),
6
+ File.open(filename, 'rb') { |f| f.read(4) } || String.new
7
+ )
6
8
  when 'qcow2v3', 'qcow3'
7
9
  steps %{
8
10
  When I successfully run `qemu-img info #{filename}`
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
10
12
  And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
11
13
  }
12
14
  else
13
- assert_no_partial_output([0x184D2204].pack('V'),
14
- File.open(filename, 'rb') {|f| f.read(4) }.to_s)
15
+ assert_no_partial_output(
16
+ [0x184D2204].pack('V'),
17
+ File.open(filename, 'rb') { |f| f.read(4) } || String.new
18
+ )
15
19
  steps(%{
16
20
  When I successfully run `qemu-img info #{filename}`
17
21
  Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"