uricp 0.0.13 → 0.0.18

data/focal/Dockerfile

@@ -0,0 +1,21 @@
+FROM ubuntu:focal
+MAINTAINER support@brightbox.co.uk
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
+
+#RUN apt-add-repository ppa:brightbox/ruby-ng
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
+  build-essential \
+  git \
+  qemu-utils \
+  liblz4-tool \
+  curl \
+  ruby \
+  ruby-dev \
+  python3-swiftclient \
+  ruby-bundler
+
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+
+RUN gem install bundler -v '~> 1.7'

data/lib/segment_upload.rb

@@ -1,36 +1,34 @@
-require "uricp/version"
-require "uricp/curl_primitives"
-require "uricp/orbit_auth"
-require "uricp/segmenter"
+require 'uricp/version'
+require 'uricp/curl_primitives'
+require 'uricp/orbit_auth'
+require 'uricp/segmenter'
 
 module Uricp
-
   UnsupportedURLtype = Class.new(ArgumentError)
-
 end
 
-#Monkey patch a copy_stream facility in using 'sendfile'
+# Monkey patch a copy_stream facility in using 'sendfile'
 unless IO.respond_to? :copy_stream
   require 'sendfile'
 
-  def IO.copy_stream(src, dst, copy_length=nil, offset=nil)
-    unless src.stat.pipe?
-      current_pos = src.pos
-      count = dst.sendfile(src, offset || current_pos, copy_length)
-      src.seek(count, IO::SEEK_CUR)
-      return count
-    else
+  def IO.copy_stream(src, dst, copy_length = nil, offset = nil)
+    if src.stat.pipe?
       amount = copy_length.to_i
       buf_size = [amount, 2**16].min
-      buffer=""
-      while amount > 0 do
-	src.read(buf_size, buffer)
-	amount_read = buffer.length
-	dst.write(buffer)
-	amount -= amount_read
-	break if src.eof?
+      buffer = ''
+      while amount > 0
+        src.read(buf_size, buffer)
+        amount_read = buffer.length
+        dst.write(buffer)
+        amount -= amount_read
+        break if src.eof?
       end
-      return copy_length.to_i - amount
+      copy_length.to_i - amount
+    else
+      current_pos = src.pos
+      count = dst.sendfile(src, offset || current_pos, copy_length)
+      src.seek(count, IO::SEEK_CUR)
+      count
     end
   rescue EOFError
     0

data/lib/uricp.rb

@@ -7,52 +7,26 @@ require 'uricp/strategy/local_link'
 require 'uricp/strategy/piped_cache'
 require 'uricp/strategy/piped_cache_convert'
 require 'uricp/strategy/piped_local_get'
+require 'uricp/strategy/piped_rbd_get'
 require 'uricp/strategy/piped_local_put'
 require 'uricp/strategy/piped_remote_get'
 require 'uricp/strategy/piped_compress'
 require 'uricp/strategy/piped_decompress'
 require 'uricp/strategy/piped_local_decompress'
 require 'uricp/strategy/piped_local_compress'
+require 'uricp/strategy/rbd_remote_put'
 require 'uricp/strategy/segmented_remote_put'
 require 'uricp/strategy/remote_put'
 require 'uricp/strategy/cached_get'
+require 'uricp/strategy/rbd_sweeper'
 require 'uricp/strategy/sweeper'
 require 'uricp/strategy/cleaner'
 require 'uricp/uri_strategy'
 require 'uricp/orbit_auth'
 
 module Uricp
-
   UnsupportedURLtype = Class.new(ArgumentError)
   MissingCache = Class.new(ArgumentError)
   UnsupportedConversion = Class.new(ArgumentError)
   ConversionRequired = Class.new(ArgumentError)
-
-end
-
-require 'tempfile'
-class Dir
-  module Tmpname
-    unless respond_to? :make_tmpname
-
-      module_function
-      def make_tmpname(prefix_suffix, n)
-	case prefix_suffix
-	when String
-	  prefix = prefix_suffix
-	  suffix = ""
-	when Array
-	  prefix = prefix_suffix[0]
-	  suffix = prefix_suffix[1]
-	else
-	  raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
-	end
-	t = Time.now.strftime("%Y%m%d")
-	path = "#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
-	path << "-#{n}" if n
-	path << suffix
-      end
-
-    end
-  end
 end

data/lib/uricp/curl_primitives.rb

@@ -1,45 +1,43 @@
 module Uricp::CurlPrimitives
+  attr_reader :options
 
-    attr_reader :options
+  def from
+    options['from_uri']
+  end
 
-    def from
-      options['from_uri']
-    end
+  def from=(target)
+    options['from_uri'] = target
+  end
 
-    def from=(target)
-      options['from_uri'] = target
-    end
+  def to
+    options['to_uri']
+  end
 
-    def to
-      options['to_uri']
-    end
+  def to=(target)
+    options['to_uri'] = target
+  end
 
-    def to=(target)
-      options['to_uri'] = target
-    end
-  
-    def curl_command
-      "curl --fail --silent"
-    end
+  def curl_command
+    'curl --fail --silent'
+  end
 
-    def authentication
-      "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
-    end
+  def authentication
+    "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
+  end
 
-    def http_authentication?
-      options['authenticator']
-    end
+  def http_authentication?
+    options['authenticator']
+  end
 
-    def curl_upload_from(source, destination = to)
-      "#{curl_command} #{authentication} -T #{source} #{destination.to_s};"
-    end
+  def curl_upload_from(source, destination = to)
+    "#{curl_command} #{authentication} -T #{source} #{destination};"
+  end
 
-    def curl_download_to_pipe
-      "#{curl_command} #{authentication} #{from.to_s} |"
-    end
+  def curl_download_to_pipe
+    "#{curl_command} #{authentication} #{from} |"
+  end
 
-    def curl_manifest(object_manifest, destination = to)
-      "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination.to_s} --data-binary ''"
-    end
-  
+  def curl_manifest(object_manifest, destination = to)
+    "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination} --data-binary ''"
+  end
 end

data/lib/uricp/orbit_auth.rb

@@ -1,8 +1,6 @@
 require 'open-uri'
 module Uricp
-
   class OrbitAuth
-    
     AuthenticationFailure = Class.new(ArgumentError)
 
     def initialize(auth_uri, auth_id, auth_key)
@@ -14,9 +12,9 @@ module Uricp
     def storage_url
       @storage_url ||
         begin
-	  authenticate
-	  @storage_url
-	end
+          authenticate
+          @storage_url
+        end
     end
 
     def token
@@ -26,27 +24,27 @@ module Uricp
 
     def self.validate_options(options)
       if options['auth-token'] && (options['auth-key'] || options['auth-user'])
-	raise ::OptionParser::NeedlessArgument,
-	  "use either key based or token based authentication"
+        raise ::OptionParser::NeedlessArgument,
+              'use either key based or token based authentication'
       end
       if options['auth-key'].nil? ^ options['auth-user'].nil?
-	raise ::OptionParser::MissingArgument,
-	  "'auth-user' requires 'auth-key'"
+        raise ::OptionParser::MissingArgument,
+              "'auth-user' requires 'auth-key'"
       end
       if (options['auth-token'] || options['auth-user']) && options['auth_uri'].nil?
-	raise ::OptionParser::NeedlessArgument,
-	  "authentication is for http uris only"
+        raise ::OptionParser::NeedlessArgument,
+              'authentication is for http uris only'
       end
     end
 
     def self.add_auth_token(options)
       if options['auth-user']
-	orbit_credentials = self.new(options['auth_uri'],
-			      options['auth-user'], options['auth-key'])
-	options['authenticator'] = orbit_credentials.method(:token)
+        orbit_credentials = new(options['auth_uri'],
+                                options['auth-user'], options['auth-key'])
+        options['authenticator'] = orbit_credentials.method(:token)
       elsif options['auth-token']
         orbit_token = options['auth-token']
-        options['authenticator'] = lambda { orbit_token }
+        options['authenticator'] = -> { orbit_token }
       end
       options.delete('auth-key')
       options.delete('auth-user')
@@ -54,29 +52,27 @@ module Uricp
     end
 
     def self.add_auth_to_optionparser(app)
-      app.on("--auth-token AUTH_TOKEN",
-	"Use AUTH_TOKEN for non-local requests")
-      app.on("--auth-user AUTH_USER",
-	"Use AUTH_USER for authentication")
-      app.on("--auth-key AUTH_KEY",
-	"Use AUTH_KEY for authentication")
+      app.on('--auth-token AUTH_TOKEN',
+             'Use AUTH_TOKEN for non-local requests')
+      app.on('--auth-user AUTH_USER',
+             'Use AUTH_USER for authentication')
+      app.on('--auth-key AUTH_KEY',
+             'Use AUTH_KEY for authentication')
     end
 
-  private
+    private
 
     def authenticate
       @auth_uri.open(
         'X-Auth-User' => @auth_id,
-	'X-Auth-Key' => @auth_key,
-	'Range' => 'bytes=0-0'
+        'X-Auth-Key' => @auth_key,
+        'Range' => 'bytes=0-0'
       ) do |uri|
         @storage_url = uri.meta['x-storage-url']
-	@token = uri.meta['x-auth-token']
+        @token = uri.meta['x-auth-token']
       end
     rescue OpenURI::HTTPError => e
       raise AuthenticationFailure, "Cannot authenticate against #{@auth_uri}"
     end
-
   end
-
 end

data/lib/uricp/segmenter.rb

@@ -1,9 +1,7 @@
 require 'pathname'
 
 module Uricp
-
   class Segmenter
-
     include Methadone::CLILogging
     include Methadone::SH
     include Uricp::CurlPrimitives
@@ -15,7 +13,7 @@ module Uricp
         @stream = File.pipe?(source) || File.chardev?(source)
       else
         @source = STDIN
-	@stream = true
+        @stream = true
       end
       split_path
     end
@@ -23,9 +21,9 @@ module Uricp
     def split_path
       elements = Pathname.new(to.path).enum_for(:each_filename).to_a
       elements.shift
-      @account=elements.shift
-      @container=elements.shift
-      @object_path=File.join(elements)
+      @account = elements.shift
+      @container = elements.shift
+      @object_path = File.join(elements)
     end
 
     def upload
@@ -33,7 +31,7 @@ module Uricp
         segmented_upload
       else
         sh! curl_upload_from(options['from']),
-	  :on_fail => "Upload to #{to} failed"
+            on_fail: "Upload to #{to} failed"
       end
     end
 
@@ -54,16 +52,16 @@ module Uricp
     end
 
     def object_manifest
-      @object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
+      @object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/'
     end
 
     def segmented_upload
       debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
       suffix = 0
       until @source.eof?
-	debug "#{self.class.name}: Uploading segment #{suffix}"
-	upload_segment(suffix)
-	suffix = suffix.next
+        debug "#{self.class.name}: Uploading segment #{suffix}"
+        upload_segment(suffix)
+        suffix = suffix.next
       end
       add_manifest
     end
@@ -71,18 +69,16 @@ module Uricp
     def upload_segment(segment_number)
       segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
       debug "Uploading with #{curl_upload_from('-', segment_name)}"
-      open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
+      open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
         copy_length = IO.copy_stream(@source, destination, segment_size)
-	debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
+        debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
       end
     end
 
     def add_manifest
       debug "Adding DLO object_manifest #{object_manifest}"
       sh! curl_manifest(object_manifest),
-	:on_fail => "Upload to #{to} failed"
+          on_fail: "Upload to #{to} failed"
     end
-
   end
-
 end

data/lib/uricp/strategy/cache_common.rb

@@ -1,16 +1,20 @@
 require 'fileutils'
 
 module Uricp::Strategy
-
   module CacheCommon
-    
     def validate_cache!
+      return if dry_run?
+
+      unless cache_exists?
+        raise Uricp::MissingCache,
+              "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory"
+      end
+      return if cache_name
+
       raise Uricp::MissingCache,
-	"no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory" unless cache_exists?
-      raise Uricp::MissingCache,
-	"no cache name found" unless cache_name
+            'no cache name found'
     end
-      
+
     def in_cache?
       File.readable? cache_file
     end
@@ -24,7 +28,7 @@ module Uricp::Strategy
     end
 
     def cache_exists?
-      cache_root && %w{temp cache}.all? do |d|
+      cache_root && %w[temp cache].all? do |d|
         File.directory?(File.join(cache_root, d))
       end
     end
@@ -40,9 +44,5 @@ module Uricp::Strategy
     def cache_file
       @cache_file ||= File.join(cache_root, 'cache', cache_name)
     end
-
   end
-
 end
-
-

data/lib/uricp/strategy/cached_get.rb

@@ -1,48 +1,42 @@
 require 'uri'
 
 module Uricp::Strategy
-  
   class CachedGet
-
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
 
     def appropriate?
       if cache_root
         validate_cache!
-	if in_cache? || file_source?
-	  return proposal
-	else
-	  debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
-	end
+        return proposal if in_cache? || file_source?
+
+        debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
       else
-	debug "#{self.class.name}: not appropriate"
+        debug "#{self.class.name}: not appropriate"
       end
       false
     end
 
     def command
-      ":;" 
+      ':;'
     end
 
     def proposal
       @proposed_options = options.dup
-      unless file_source?
-	@proposed_options['from_uri'] = URI.join('file:///', cache_file)
-      end
+      @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')
       if conversion_required?
-	@proposed_options['source-format'] =
-	  File.open(@proposed_options['from_uri'].path) {|f| encoding(f)}
-	if @proposed_options['source-format'] == @proposed_options['target-format']
-	  @proposed_options.delete('source-format')
-	  @proposed_options.delete('target-format')
-	end
+        unless dry_run?
+          @proposed_options['source-format'] =
+            File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
+        end
+        if @proposed_options['source-format'] == @proposed_options['target-format']
+          @proposed_options.delete('source-format')
+          @proposed_options.delete('target-format')
+        end
       end
       self
     end
-
   end
-
 end