uricp 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34a83d1c4afd7b3c2b43eeec01b42e06ff8ea55edf8450110ca70363be693d64
-  data.tar.gz: 64d75d6bc450453f5e2204ab085759a912c265116934be3d7595861f1b5dafb8
+  metadata.gz: d7db104674249fa9443799b004c877667e22b142d26a7c7a2220ef668865dfbc
+  data.tar.gz: d333e085bb1b666e09883fbd9b9006334d6c15acd17ac781629697bcc0766e5a
 SHA512:
-  metadata.gz: f92262c06bf7b035a453fb51358d429e8617487c76df209520abf3d0d322644787e4cbec5d83fdd77fa8a749de41d29e30e1b00248963efd2a26bdfcaf8edc56
-  data.tar.gz: 1f6533f5334330c3653da3e952715428478f55f1b956d9fdb9b9b9290d7ff0c45391fce3363c132f8dcb85deeb47930ef4dda36cdc1baea9f52a12ae98c4d2f2
+  metadata.gz: 7e16104719cca1e2def15b5d9b2d7cbdbcb84b68f7d69a566b8eddd64e699654ea72f066102bb8949e2eb583ffd54aad9549b5094746747382930cf18a1de5a8
+  data.tar.gz: b0f839f96f8975ce95121085fdba25ecb1451fc38dbc975de250a6277ffe584a5548f6033138cb46ae8f200a0e6c063e45b2eb25b26ea4e6cefad0d592b91294

data/.rubocop.yml

@@ -0,0 +1,3 @@
+AllCops:
+  TargetRubyVersion: 2.4
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    uricp (0.0.15)
+    uricp (0.0.16)
       filesize (= 0.0.2)
       methadone (~> 2.0.0)
       open4 (~> 1.3.0)
@@ -38,14 +38,8 @@ GEM
     rake (10.4.2)
     rdoc (4.2.2)
       json (~> 1.4)
-    rspec (2.99.0)
-      rspec-core (~> 2.99.0)
-      rspec-expectations (~> 2.99.0)
-      rspec-mocks (~> 2.99.0)
-    rspec-core (2.99.2)
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.99.4)
     sendfile (1.2.2)
 
 PLATFORMS
@@ -58,7 +52,6 @@ DEPENDENCIES
   inifile (~> 1.1)
   rake (~> 10.4.0)
   rdoc (~> 4.2.0)
-  rspec (~> 2.99)
   uricp!
 
 BUNDLED WITH

data/Jenkinsfile

@@ -29,6 +29,11 @@ pipeline {
         always {
 	  sh 'bundle exec rake features:clean'
 	}
+	failure {
+	    mail to: 'sysadmin@brightbox.co.uk',
+		 subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
+		 body: "${env.BUILD_URL}"
+	}
       }
     }
     stage("Trusty, Ruby 1.9.3") {
@@ -46,6 +51,11 @@ pipeline {
         always {
 	  sh 'bundle exec rake features:clean'
 	}
+	failure {
+	    mail to: 'sysadmin@brightbox.co.uk',
+		 subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
+		 body: "${env.BUILD_URL}"
+	}
       }
     }
     stage("Xenial, Ruby 2.3.1") {
@@ -63,6 +73,11 @@ pipeline {
         always {
 	  sh 'bundle exec rake features:clean'
 	}
+	failure {
+	    mail to: 'sysadmin@brightbox.co.uk',
+		 subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
+		 body: "${env.BUILD_URL}"
+	}
       }
     }
     stage("Bionic, Ruby 2.5.1") {
@@ -80,7 +95,19 @@ pipeline {
         always {
 	  sh 'bundle exec rake features:clean'
 	}
+	failure {
+	    mail to: 'sysadmin@brightbox.co.uk',
+		 subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
+		 body: "${env.BUILD_URL}"
+	}
       }
     }
   }
+  post {
+    failure {
+	mail to: 'sysadmin@brightbox.co.uk',
+	     subject: "Uricp Tests Failed: ${currentBuild.fullDisplayName}",
+	     body: "${env.BUILD_URL}"
+    }
+  }
 }

data/Rakefile

@@ -2,19 +2,19 @@ def dump_load_path
   puts $LOAD_PATH.join("\n")
   found = nil
   $LOAD_PATH.each do |path|
-    if File.exists?(File.join(path,"rspec"))
-      puts "Found rspec in #{path}"
-      if File.exists?(File.join(path,"rspec","core"))
-        puts "Found core"
-        if File.exists?(File.join(path,"rspec","core","rake_task"))
-          puts "Found rake_task"
-          found = path
-        else
-          puts "!! no rake_task"
-        end
+    next unless File.exist?(File.join(path, 'rspec'))
+
+    puts "Found rspec in #{path}"
+    if File.exist?(File.join(path, 'rspec', 'core'))
+      puts 'Found core'
+      if File.exist?(File.join(path, 'rspec', 'core', 'rake_task'))
+        puts 'Found rake_task'
+        found = path
       else
-        puts "!!! no core"
+        puts '!! no rake_task'
       end
+    else
+      puts '!!! no core'
     end
   end
   if found.nil?
@@ -26,13 +26,6 @@ end
 require 'bundler'
 require 'rake/clean'
 
-begin
-require 'rspec/core/rake_task'
-rescue LoadError
-dump_load_path
-raise
-end
-
 require 'cucumber'
 require 'cucumber/rake/task'
 gem 'rdoc' # we need the installed RDoc gem, not the system one
@@ -42,13 +35,7 @@ include Rake::DSL
 
 Bundler::GemHelper.install_tasks
 
-
-RSpec::Core::RakeTask.new do |t|
-  # Put spec opts in a file named .rspec in root
-end
-
-
-CUKE_RESULTS = 'results.html'
+CUKE_RESULTS = 'results.html'.freeze
 CLEAN << CUKE_RESULTS
 namespace(:features) do
   Cucumber::Rake::Task.new(:new_qemu) do |t|
@@ -64,11 +51,9 @@ namespace(:features) do
 
   task(:clean) do
     if ENV['ORBIT_USER'] && ENV['ORBIT_KEY']
-      %w{temp_dlo test_upload}.each do |test_file|
-        sh "swift --quiet --insecure -A #{ENV['ORBIT_URL'] || "https://orbit.gb1.brightbox.com/v1"} -U #{ENV['ORBIT_USER']} -K #{ENV['ORBIT_KEY']} delete #{test_file}" do |ok, res|
-          if !ok
-            puts "Delete failed: #{res.exitstatus}"
-          end
+      %w[temp_dlo test_upload].each do |test_file|
+        sh "swift --quiet --insecure -A #{ENV['ORBIT_URL'] || 'https://orbit.gb1.brightbox.com/v1'} -U #{ENV['ORBIT_USER']} -K #{ENV['ORBIT_KEY']} delete #{test_file}" do |ok, res|
+          puts "Delete failed: #{res.exitstatus}" unless ok
         end
       end
     end
@@ -76,11 +61,9 @@ namespace(:features) do
 end
 
 Rake::RDocTask.new do |rd|
-  
-  rd.main = "README.rdoc"
-  
-  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
-end
+  rd.main = 'README.rdoc'
 
-task :default => [:spec,"features:new_qemu"]
+  rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb', 'bin/**/*')
+end
 
+task default: [:spec, 'features:new_qemu']

data/bionic/Dockerfile

@@ -13,8 +13,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
   curl \
   ruby \
   ruby-dev \
-  python-swiftclient
+  python-swiftclient \
+  ruby-bundler
 
 RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
-RUN gem install bundler
 

data/centos7/Dockerfile

@@ -8,7 +8,8 @@ RUN yum update -y && yum -y install \
     git \
     qemu-img \
     ruby-devel \
-    rubygems 
+    rubygems \
+    rubygem-bundler
 
 RUN yum -y install python-setuptools && \
     easy_install pip && \
@@ -16,5 +17,4 @@ RUN yum -y install python-setuptools && \
     pip install python-swiftclient
 
 RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
-RUN gem install bundler
 

data/features/step_definitions/orbit_steps.rb

@@ -18,12 +18,19 @@ def fetch_orbit_token
   @current_url_target = ENV['ORBIT_URL'] || 'https://orbit.brightbox.com/v1'
   cmd = "curl -I #{@current_url_target} -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
   run_simple(unescape(cmd))
-  stdout_from(cmd).each_line do |line|
+  output = stdout_from(cmd)
+  output.each_line do |line|
     key, value = line.strip.split(/\s*:\s*/,2)
-    @current_auth_token = value if key == 'X-Auth-Token'
-    @current_storage_url = value if key == 'X-Storage-Url'
+    case key.to_s.downcase
+    when 'x-auth-token'
+      @current_auth_token = value
+    when 'x-storage-url'
+      @current_storage_url = value
+    end
+  end
+  unless @current_auth_token && @current_storage_url
+    raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.\n#{output}")
   end
-  raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.") unless @current_auth_token && @current_storage_url 
 end
 
 def create_download_file(filetype, name, container)

data/lib/segment_upload.rb

@@ -1,36 +1,34 @@
-require "uricp/version"
-require "uricp/curl_primitives"
-require "uricp/orbit_auth"
-require "uricp/segmenter"
+require 'uricp/version'
+require 'uricp/curl_primitives'
+require 'uricp/orbit_auth'
+require 'uricp/segmenter'
 
 module Uricp
-
   UnsupportedURLtype = Class.new(ArgumentError)
-
 end
 
-#Monkey patch a copy_stream facility in using 'sendfile'
+# Monkey patch a copy_stream facility in using 'sendfile'
 unless IO.respond_to? :copy_stream
   require 'sendfile'
 
-  def IO.copy_stream(src, dst, copy_length=nil, offset=nil)
-    unless src.stat.pipe?
-      current_pos = src.pos
-      count = dst.sendfile(src, offset || current_pos, copy_length)
-      src.seek(count, IO::SEEK_CUR)
-      return count
-    else
+  def IO.copy_stream(src, dst, copy_length = nil, offset = nil)
+    if src.stat.pipe?
       amount = copy_length.to_i
       buf_size = [amount, 2**16].min
-      buffer=""
-      while amount > 0 do
-	src.read(buf_size, buffer)
-	amount_read = buffer.length
-	dst.write(buffer)
-	amount -= amount_read
-	break if src.eof?
+      buffer = ''
+      while amount > 0
+        src.read(buf_size, buffer)
+        amount_read = buffer.length
+        dst.write(buffer)
+        amount -= amount_read
+        break if src.eof?
       end
-      return copy_length.to_i - amount
+      copy_length.to_i - amount
+    else
+      current_pos = src.pos
+      count = dst.sendfile(src, offset || current_pos, copy_length)
+      src.seek(count, IO::SEEK_CUR)
+      count
     end
   rescue EOFError
     0

data/lib/uricp.rb

@@ -7,26 +7,26 @@ require 'uricp/strategy/local_link'
 require 'uricp/strategy/piped_cache'
 require 'uricp/strategy/piped_cache_convert'
 require 'uricp/strategy/piped_local_get'
+require 'uricp/strategy/piped_rbd_get'
 require 'uricp/strategy/piped_local_put'
 require 'uricp/strategy/piped_remote_get'
 require 'uricp/strategy/piped_compress'
 require 'uricp/strategy/piped_decompress'
 require 'uricp/strategy/piped_local_decompress'
 require 'uricp/strategy/piped_local_compress'
+require 'uricp/strategy/rbd_remote_put'
 require 'uricp/strategy/segmented_remote_put'
 require 'uricp/strategy/remote_put'
 require 'uricp/strategy/cached_get'
+require 'uricp/strategy/rbd_sweeper'
 require 'uricp/strategy/sweeper'
 require 'uricp/strategy/cleaner'
 require 'uricp/uri_strategy'
 require 'uricp/orbit_auth'
 
 module Uricp
-
   UnsupportedURLtype = Class.new(ArgumentError)
   MissingCache = Class.new(ArgumentError)
   UnsupportedConversion = Class.new(ArgumentError)
   ConversionRequired = Class.new(ArgumentError)
-
 end
-

data/lib/uricp/curl_primitives.rb

@@ -1,45 +1,43 @@
 module Uricp::CurlPrimitives
+  attr_reader :options
 
-    attr_reader :options
+  def from
+    options['from_uri']
+  end
 
-    def from
-      options['from_uri']
-    end
+  def from=(target)
+    options['from_uri'] = target
+  end
 
-    def from=(target)
-      options['from_uri'] = target
-    end
+  def to
+    options['to_uri']
+  end
 
-    def to
-      options['to_uri']
-    end
+  def to=(target)
+    options['to_uri'] = target
+  end
 
-    def to=(target)
-      options['to_uri'] = target
-    end
-  
-    def curl_command
-      "curl --fail --silent"
-    end
+  def curl_command
+    'curl --fail --silent'
+  end
 
-    def authentication
-      "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
-    end
+  def authentication
+    "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
+  end
 
-    def http_authentication?
-      options['authenticator']
-    end
+  def http_authentication?
+    options['authenticator']
+  end
 
-    def curl_upload_from(source, destination = to)
-      "#{curl_command} #{authentication} -T #{source} #{destination.to_s};"
-    end
+  def curl_upload_from(source, destination = to)
+    "#{curl_command} #{authentication} -T #{source} #{destination};"
+  end
 
-    def curl_download_to_pipe
-      "#{curl_command} #{authentication} #{from.to_s} |"
-    end
+  def curl_download_to_pipe
+    "#{curl_command} #{authentication} #{from} |"
+  end
 
-    def curl_manifest(object_manifest, destination = to)
-      "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination.to_s} --data-binary ''"
-    end
-  
+  def curl_manifest(object_manifest, destination = to)
+    "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination} --data-binary ''"
+  end
 end

data/lib/uricp/orbit_auth.rb

@@ -1,8 +1,6 @@
 require 'open-uri'
 module Uricp
-
   class OrbitAuth
-    
     AuthenticationFailure = Class.new(ArgumentError)
 
     def initialize(auth_uri, auth_id, auth_key)
@@ -14,9 +12,9 @@ module Uricp
     def storage_url
       @storage_url ||
         begin
-	  authenticate
-	  @storage_url
-	end
+          authenticate
+          @storage_url
+        end
     end
 
     def token
@@ -26,27 +24,27 @@ module Uricp
 
     def self.validate_options(options)
       if options['auth-token'] && (options['auth-key'] || options['auth-user'])
-	raise ::OptionParser::NeedlessArgument,
-	  "use either key based or token based authentication"
+        raise ::OptionParser::NeedlessArgument,
+              'use either key based or token based authentication'
       end
       if options['auth-key'].nil? ^ options['auth-user'].nil?
-	raise ::OptionParser::MissingArgument,
-	  "'auth-user' requires 'auth-key'"
+        raise ::OptionParser::MissingArgument,
+              "'auth-user' requires 'auth-key'"
       end
       if (options['auth-token'] || options['auth-user']) && options['auth_uri'].nil?
-	raise ::OptionParser::NeedlessArgument,
-	  "authentication is for http uris only"
+        raise ::OptionParser::NeedlessArgument,
+              'authentication is for http uris only'
       end
     end
 
     def self.add_auth_token(options)
       if options['auth-user']
-	orbit_credentials = self.new(options['auth_uri'],
-			      options['auth-user'], options['auth-key'])
-	options['authenticator'] = orbit_credentials.method(:token)
+        orbit_credentials = new(options['auth_uri'],
+                                options['auth-user'], options['auth-key'])
+        options['authenticator'] = orbit_credentials.method(:token)
       elsif options['auth-token']
         orbit_token = options['auth-token']
-        options['authenticator'] = lambda { orbit_token }
+        options['authenticator'] = -> { orbit_token }
       end
       options.delete('auth-key')
       options.delete('auth-user')
@@ -54,29 +52,27 @@ module Uricp
     end
 
     def self.add_auth_to_optionparser(app)
-      app.on("--auth-token AUTH_TOKEN",
-	"Use AUTH_TOKEN for non-local requests")
-      app.on("--auth-user AUTH_USER",
-	"Use AUTH_USER for authentication")
-      app.on("--auth-key AUTH_KEY",
-	"Use AUTH_KEY for authentication")
+      app.on('--auth-token AUTH_TOKEN',
+             'Use AUTH_TOKEN for non-local requests')
+      app.on('--auth-user AUTH_USER',
+             'Use AUTH_USER for authentication')
+      app.on('--auth-key AUTH_KEY',
+             'Use AUTH_KEY for authentication')
     end
 
-  private
+    private
 
     def authenticate
       @auth_uri.open(
         'X-Auth-User' => @auth_id,
-	'X-Auth-Key' => @auth_key,
-	'Range' => 'bytes=0-0'
+        'X-Auth-Key' => @auth_key,
+        'Range' => 'bytes=0-0'
       ) do |uri|
         @storage_url = uri.meta['x-storage-url']
-	@token = uri.meta['x-auth-token']
+        @token = uri.meta['x-auth-token']
       end
     rescue OpenURI::HTTPError => e
       raise AuthenticationFailure, "Cannot authenticate against #{@auth_uri}"
     end
-
   end
-
 end