uricp 0.0.13 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -1
  3. data/.rubocop.yml +3 -0
  4. data/Gemfile.lock +20 -23
  5. data/Jenkinsfile +113 -0
  6. data/README.md +2 -2
  7. data/Rakefile +26 -31
  8. data/bionic/Dockerfile +20 -0
  9. data/centos7/Dockerfile +18 -0
  10. data/features/step_definitions/orbit_steps.rb +19 -13
  11. data/features/step_definitions/uricp_steps.rb +8 -4
  12. data/focal/Dockerfile +21 -0
  13. data/lib/segment_upload.rb +20 -22
  14. data/lib/uricp.rb +3 -29
  15. data/lib/uricp/curl_primitives.rb +31 -33
  16. data/lib/uricp/orbit_auth.rb +23 -27
  17. data/lib/uricp/segmenter.rb +12 -16
  18. data/lib/uricp/strategy/cache_common.rb +11 -11
  19. data/lib/uricp/strategy/cached_get.rb +14 -20
  20. data/lib/uricp/strategy/cleaner.rb +2 -8
  21. data/lib/uricp/strategy/common.rb +27 -19
  22. data/lib/uricp/strategy/local_convert.rb +10 -17
  23. data/lib/uricp/strategy/local_link.rb +10 -8
  24. data/lib/uricp/strategy/piped_cache.rb +7 -13
  25. data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
  26. data/lib/uricp/strategy/piped_compress.rb +3 -8
  27. data/lib/uricp/strategy/piped_decompress.rb +7 -11
  28. data/lib/uricp/strategy/piped_local_compress.rb +0 -4
  29. data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
  30. data/lib/uricp/strategy/piped_local_get.rb +0 -5
  31. data/lib/uricp/strategy/piped_local_put.rb +3 -9
  32. data/lib/uricp/strategy/piped_rbd_get.rb +46 -0
  33. data/lib/uricp/strategy/piped_remote_get.rb +20 -20
  34. data/lib/uricp/strategy/rbd_remote_put.rb +36 -0
  35. data/lib/uricp/strategy/rbd_sweeper.rb +22 -0
  36. data/lib/uricp/strategy/remote_put.rb +11 -17
  37. data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
  38. data/lib/uricp/strategy/sweeper.rb +2 -8
  39. data/lib/uricp/uri_strategy.rb +14 -13
  40. data/lib/uricp/version.rb +4 -2
  41. data/uricp.gemspec +3 -4
  42. data/xenial/Dockerfile +20 -0
  43. metadata +39 -51
  44. data/Dockerfile_centos6.6 +0 -39
  45. data/Dockerfile_centos7 +0 -39
  46. data/Dockerfile_trusty-ruby193 +0 -32
  47. data/README.rdoc +0 -23
  48. data/spec/something_spec.rb +0 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 504ac718e28c260c29336ae9089d9a58d9948f7d
4
- data.tar.gz: ac2c9bc45430805b2a0845e80392e2c75ec2f471
2
+ SHA256:
3
+ metadata.gz: d9944582044883ff45f3847042ca62016416d06198a0af41751de93ab992c646
4
+ data.tar.gz: 19c14777647bb01fec3a4e6214e8183b29432843bf47b8b0c818d0401dafaea1
5
5
  SHA512:
6
- metadata.gz: b1518d8d9ae106c51d1eceb9ad5d50f0d93e3b7fbf9df43a8cca797c42d2dbdb6f44c96a7a3a4636562fd42523ca880a3859ff2b70374d70aebcf205e238a5d9
7
- data.tar.gz: 419cedbe0b8be9fcbef9ba4634906adb4ad6f346b6bcdbae06d627bea6efc4c21c6d71c646d2d0d1343081c3e0cfd34df3d273cedd4362db0aafbfb55112df93
6
+ metadata.gz: c157e985f79891ca962fb0b792dab2a4fab89bd651c15b940e75ba4baf0e86a42922b9dc8a0f96fa53b8de4151d3e9b6aeefac256ffe59faa2ed6ad5becfad47
7
+ data.tar.gz: 45444dfcbc92079f7221d0f239315db22f6af749843f6a89312e949c1f12068a2254f6ffd8830a4ab00833e3b66d16d259cfcccb9388f24dde9ff23fbad99ee2
data/.gitignore CHANGED
@@ -4,4 +4,5 @@ coverage
4
4
  pkg
5
5
  html
6
6
  *.swp
7
- Dockerfile
7
+ vendor
8
+ .bundle
data/.rubocop.yml ADDED
@@ -0,0 +1,3 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4
3
+
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- uricp (0.0.13)
4
+ uricp (0.0.18)
5
5
  filesize (= 0.0.2)
6
- methadone (~> 1.8.0)
6
+ methadone (~> 2.0.0)
7
7
  open4 (~> 1.3.0)
8
8
  sendfile (~> 1.2.0)
9
9
 
@@ -14,38 +14,33 @@ GEM
14
14
  childprocess (>= 0.3.6)
15
15
  cucumber (>= 1.1.1)
16
16
  rspec-expectations (>= 2.7.0)
17
- builder (3.2.2)
18
- childprocess (0.5.7)
19
- ffi (~> 1.0, >= 1.0.11)
17
+ builder (3.2.4)
18
+ childprocess (1.0.1)
19
+ rake (< 13.0)
20
20
  cucumber (1.3.20)
21
21
  builder (>= 2.1.2)
22
22
  diff-lcs (>= 1.1.3)
23
23
  gherkin (~> 2.12)
24
24
  multi_json (>= 1.7.5, < 2.0)
25
25
  multi_test (>= 0.1.2)
26
- diff-lcs (1.2.5)
27
- ffi (1.9.10)
26
+ diff-lcs (1.4.4)
28
27
  filesize (0.0.2)
29
28
  gherkin (2.12.2)
30
29
  multi_json (~> 1.3)
31
30
  inifile (1.1.0)
32
- json (1.8.3)
33
- methadone (1.8.0)
31
+ json (1.8.6)
32
+ methadone (2.0.2)
34
33
  bundler
35
- multi_json (1.11.2)
34
+ multi_json (1.15.0)
36
35
  multi_test (0.1.2)
37
36
  open4 (1.3.4)
38
- rake (10.4.2)
39
- rdoc (4.2.0)
37
+ rake (12.3.3)
38
+ rdoc (4.2.2)
40
39
  json (~> 1.4)
41
- rspec (2.99.0)
42
- rspec-core (~> 2.99.0)
43
- rspec-expectations (~> 2.99.0)
44
- rspec-mocks (~> 2.99.0)
45
- rspec-core (2.99.2)
46
- rspec-expectations (2.99.2)
47
- diff-lcs (>= 1.1.3, < 2.0)
48
- rspec-mocks (2.99.4)
40
+ rspec-expectations (3.10.1)
41
+ diff-lcs (>= 1.2.0, < 2.0)
42
+ rspec-support (~> 3.10.0)
43
+ rspec-support (3.10.2)
49
44
  sendfile (1.2.2)
50
45
 
51
46
  PLATFORMS
@@ -56,7 +51,9 @@ DEPENDENCIES
56
51
  bundler (~> 1.7)
57
52
  cucumber (~> 1.3)
58
53
  inifile (~> 1.1)
59
- rake
60
- rdoc
61
- rspec (~> 2.99)
54
+ rake (~> 12.3)
55
+ rdoc (~> 4.2.0)
62
56
  uricp!
57
+
58
+ BUNDLED WITH
59
+ 1.17.3
data/Jenkinsfile ADDED
@@ -0,0 +1,113 @@
1
+ pipeline {
2
+ environment {
3
+ ORBIT = credentials('ced8ecd0-dbe4-4bf6-bdde-101a661565f5')
4
+ ORBIT_URL = credentials('c0c95c56-b4e9-45c2-85c5-bf292aef7301')
5
+ ORBIT_USER = "${ORBIT_USR}"
6
+ ORBIT_KEY = "${ORBIT_PSW}"
7
+ }
8
+ options {
9
+ disableConcurrentBuilds()
10
+ buildDiscarder(logRotator(numToKeepStr: '5'))
11
+ }
12
+ triggers {
13
+ cron('@weekly')
14
+ }
15
+ agent none
16
+ stages {
17
+ stage("CentOS 7") {
18
+ agent {
19
+ dockerfile {
20
+ dir 'centos7'
21
+ label "docker"
22
+ }
23
+ }
24
+ steps {
25
+ sh 'bundle install --deployment'
26
+ sh 'bundle exec rake features:new_qemu'
27
+ }
28
+ post {
29
+ always {
30
+ sh 'bundle exec rake features:clean'
31
+ }
32
+ failure {
33
+ mail to: 'sysadmin@brightbox.co.uk',
34
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
35
+ body: "${env.BUILD_URL}"
36
+ }
37
+ }
38
+ }
39
+ stage("Xenial, Ruby 2.3.1") {
40
+ agent {
41
+ dockerfile {
42
+ dir 'xenial'
43
+ label "docker"
44
+ }
45
+ }
46
+ steps {
47
+ sh 'bundle install --deployment'
48
+ sh 'bundle exec rake features:new_qemu'
49
+ }
50
+ post {
51
+ always {
52
+ sh 'bundle exec rake features:clean'
53
+ }
54
+ failure {
55
+ mail to: 'sysadmin@brightbox.co.uk',
56
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
57
+ body: "${env.BUILD_URL}"
58
+ }
59
+ }
60
+ }
61
+ stage("Bionic, Ruby 2.5.1") {
62
+ agent {
63
+ dockerfile {
64
+ dir 'bionic'
65
+ label "docker"
66
+ }
67
+ }
68
+ steps {
69
+ sh 'bundle install --deployment'
70
+ sh 'bundle exec rake features:new_qemu'
71
+ }
72
+ post {
73
+ always {
74
+ sh 'bundle exec rake features:clean'
75
+ }
76
+ failure {
77
+ mail to: 'sysadmin@brightbox.co.uk',
78
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
79
+ body: "${env.BUILD_URL}"
80
+ }
81
+ }
82
+ }
83
+ stage("Focal, Ruby 2.7.0") {
84
+ agent {
85
+ dockerfile {
86
+ dir 'focal'
87
+ label "docker"
88
+ }
89
+ }
90
+ steps {
91
+ sh 'bundle install --deployment'
92
+ sh 'bundle exec rake features:new_qemu'
93
+ }
94
+ post {
95
+ always {
96
+ sh 'bundle exec rake features:clean'
97
+ }
98
+ failure {
99
+ mail to: 'sysadmin@brightbox.co.uk',
100
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
101
+ body: "${env.BUILD_URL}"
102
+ }
103
+ }
104
+ }
105
+ }
106
+ post {
107
+ failure {
108
+ mail to: 'sysadmin@brightbox.co.uk',
109
+ subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
110
+ body: "${env.BUILD_URL}"
111
+ }
112
+ }
113
+ }
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Uricp
2
2
 
3
- TODO: Write a gem description
3
+ Copy one URI to another with optional cache
4
4
 
5
5
  ## Installation
6
6
 
@@ -20,7 +20,7 @@ Or install it yourself as:
20
20
 
21
21
  ## Usage
22
22
 
23
- TODO: Write usage instructions here
23
+
24
24
 
25
25
  ## Contributing
26
26
 
data/Rakefile CHANGED
@@ -2,19 +2,19 @@ def dump_load_path
2
2
  puts $LOAD_PATH.join("\n")
3
3
  found = nil
4
4
  $LOAD_PATH.each do |path|
5
- if File.exists?(File.join(path,"rspec"))
6
- puts "Found rspec in #{path}"
7
- if File.exists?(File.join(path,"rspec","core"))
8
- puts "Found core"
9
- if File.exists?(File.join(path,"rspec","core","rake_task"))
10
- puts "Found rake_task"
11
- found = path
12
- else
13
- puts "!! no rake_task"
14
- end
5
+ next unless File.exist?(File.join(path, 'rspec'))
6
+
7
+ puts "Found rspec in #{path}"
8
+ if File.exist?(File.join(path, 'rspec', 'core'))
9
+ puts 'Found core'
10
+ if File.exist?(File.join(path, 'rspec', 'core', 'rake_task'))
11
+ puts 'Found rake_task'
12
+ found = path
15
13
  else
16
- puts "!!! no core"
14
+ puts '!! no rake_task'
17
15
  end
16
+ else
17
+ puts '!!! no core'
18
18
  end
19
19
  end
20
20
  if found.nil?
@@ -26,13 +26,6 @@ end
26
26
  require 'bundler'
27
27
  require 'rake/clean'
28
28
 
29
- begin
30
- require 'rspec/core/rake_task'
31
- rescue LoadError
32
- dump_load_path
33
- raise
34
- end
35
-
36
29
  require 'cucumber'
37
30
  require 'cucumber/rake/task'
38
31
  gem 'rdoc' # we need the installed RDoc gem, not the system one
@@ -42,13 +35,7 @@ include Rake::DSL
42
35
 
43
36
  Bundler::GemHelper.install_tasks
44
37
 
45
-
46
- RSpec::Core::RakeTask.new do |t|
47
- # Put spec opts in a file named .rspec in root
48
- end
49
-
50
-
51
- CUKE_RESULTS = 'results.html'
38
+ CUKE_RESULTS = 'results.html'.freeze
52
39
  CLEAN << CUKE_RESULTS
53
40
  namespace(:features) do
54
41
  Cucumber::Rake::Task.new(:new_qemu) do |t|
@@ -61,14 +48,22 @@ namespace(:features) do
61
48
  t.fork = false
62
49
  t.profile = 'old-qemu-image'
63
50
  end
51
+
52
+ task(:clean) do
53
+ if ENV['ORBIT_USER'] && ENV['ORBIT_KEY']
54
+ %w[temp_dlo test_upload].each do |test_file|
55
+ sh "swift --quiet --insecure -A #{ENV['ORBIT_URL'] || 'https://orbit.gb1.brightbox.com/v1'} -U #{ENV['ORBIT_USER']} -K #{ENV['ORBIT_KEY']} delete #{test_file}" do |ok, res|
56
+ puts "Delete failed: #{res.exitstatus}" unless ok
57
+ end
58
+ end
59
+ end
60
+ end
64
61
  end
65
62
 
66
63
  Rake::RDocTask.new do |rd|
67
-
68
- rd.main = "README.rdoc"
69
-
70
- rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
71
- end
64
+ rd.main = 'README.rdoc'
72
65
 
73
- task :default => [:spec,"features:new_qemu"]
66
+ rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb', 'bin/**/*')
67
+ end
74
68
 
69
+ task default: [:spec, 'features:new_qemu']
data/bionic/Dockerfile ADDED
@@ -0,0 +1,20 @@
1
+ FROM ubuntu:bionic
2
+ MAINTAINER support@brightbox.co.uk
3
+
4
+ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
5
+
6
+ #RUN apt-add-repository ppa:brightbox/ruby-ng
7
+
8
+ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
9
+ build-essential \
10
+ git \
11
+ qemu-utils \
12
+ liblz4-tool \
13
+ curl \
14
+ ruby \
15
+ ruby-dev \
16
+ python-swiftclient \
17
+ ruby-bundler
18
+
19
+ RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
20
+
@@ -0,0 +1,18 @@
1
+ FROM centos:7
2
+ MAINTAINER support@brightbox.co.uk
3
+
4
+ RUN yum update -y && yum -y install \
5
+ epel-release \
6
+ gcc \
7
+ make \
8
+ git \
9
+ qemu-img \
10
+ ruby-devel \
11
+ rubygems \
12
+ rubygem-bundler
13
+
14
+ RUN yum -y install centos-release-openstack-train && \
15
+ yum -y install python-swiftclient
16
+
17
+ RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
18
+
@@ -18,17 +18,24 @@ def fetch_orbit_token
18
18
  @current_url_target = ENV['ORBIT_URL'] || 'https://orbit.brightbox.com/v1'
19
19
  cmd = "curl -I #{@current_url_target} -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
20
20
  run_simple(unescape(cmd))
21
- stdout_from(cmd).each_line do |line|
21
+ output = stdout_from(cmd)
22
+ output.each_line do |line|
22
23
  key, value = line.strip.split(/\s*:\s*/,2)
23
- @current_auth_token = value if key == 'X-Auth-Token'
24
- @current_storage_url = value if key == 'X-Storage-Url'
24
+ case key.to_s.downcase
25
+ when 'x-auth-token'
26
+ @current_auth_token = value
27
+ when 'x-storage-url'
28
+ @current_storage_url = value
29
+ end
30
+ end
31
+ unless @current_auth_token && @current_storage_url
32
+ raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.\n#{output}")
25
33
  end
26
- raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.") unless @current_auth_token && @current_storage_url
27
34
  end
28
35
 
29
36
  def create_download_file(filetype, name, container)
30
37
  upload_url = File.join(@current_storage_url, container, name)
31
- tempfile = '/tmp/fred55322'
38
+ tempfile = 'fred55322'
32
39
  FileUtils.rm_f(Dir.glob(tempfile+'*'))
33
40
  format = 'raw'
34
41
  size = '1G'
@@ -196,26 +203,25 @@ Then(/^a qcow2 entry should exist in container "(.*?)" called "(.*?)"$/) do |con
196
203
  end
197
204
 
198
205
  Before('@orbit') do
206
+ puts "Running Orbit Before"
199
207
  fetch_orbit_token
200
- unless $orbit_setup
208
+ $first_time ||= false
209
+ unless $first_time
210
+ puts("Clearing up temporary areas")
201
211
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test")
202
212
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete temp_dlo")
203
213
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete test_upload")
204
- end
205
- end
206
-
207
- Before('@orbitdownloads') do
208
- $orbit_setup ||= false
209
- unless $orbit_setup
214
+ puts("Setting up downloads")
210
215
  create_container('test')
211
216
  create_download_file('qcow2', 'img-qcow2', 'test')
212
217
  create_download_file('lz4', 'img-lz4cy', 'test')
213
218
  create_download_file('zero', 'img-zeroy', 'test')
214
- $orbit_setup = true
219
+ $first_time = true
215
220
  end
216
221
  end
217
222
 
218
223
  After('@orbit') do
224
+ puts "Running Orbit After"
219
225
  if @current_auth_token && @current_container
220
226
  system("swift --quiet --insecure --os-storage-url=#{@current_storage_url} --os-auth-token=#{@current_auth_token} delete #{@current_container} #{@current_name}")
221
227
  @current_container = @current_name = nil
@@ -1,8 +1,10 @@
1
1
  Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
2
2
  case format
3
3
  when 'lz4'
4
- assert_exact_output([0x184D2204].pack('V'),
5
- File.open(filename, 'rb') {|f| f.read(4) }.to_s)
4
+ assert_exact_output(
5
+ [0x184D2204].pack('V'),
6
+ File.open(filename, 'rb') { |f| f.read(4) } || String.new
7
+ )
6
8
  when 'qcow2v3', 'qcow3'
7
9
  steps %{
8
10
  When I successfully run `qemu-img info #{filename}`
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
10
12
  And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
11
13
  }
12
14
  else
13
- assert_no_partial_output([0x184D2204].pack('V'),
14
- File.open(filename, 'rb') {|f| f.read(4) }.to_s)
15
+ assert_no_partial_output(
16
+ [0x184D2204].pack('V'),
17
+ File.open(filename, 'rb') { |f| f.read(4) } || String.new
18
+ )
15
19
  steps(%{
16
20
  When I successfully run `qemu-img info #{filename}`
17
21
  Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"