uricp 0.0.15 → 0.0.20

data/lib/uricp/curl_primitives.rb

@@ -1,45 +1,43 @@
 module Uricp::CurlPrimitives
+  attr_reader :options
 
-    attr_reader :options
+  def from
+    options['from_uri']
+  end
 
-    def from
-      options['from_uri']
-    end
+  def from=(target)
+    options['from_uri'] = target
+  end
 
-    def from=(target)
-      options['from_uri'] = target
-    end
+  def to
+    options['to_uri']
+  end
 
-    def to
-      options['to_uri']
-    end
+  def to=(target)
+    options['to_uri'] = target
+  end
 
-    def to=(target)
-      options['to_uri'] = target
-    end
-  
-    def curl_command
-      "curl --fail --silent"
-    end
+  def curl_command
+    'curl --fail --silent'
+  end
 
-    def authentication
-      "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
-    end
+  def authentication
+    "-H X-Auth-Token:#{options['authenticator'].call}" if http_authentication?
+  end
 
-    def http_authentication?
-      options['authenticator']
-    end
+  def http_authentication?
+    options['authenticator']
+  end
 
-    def curl_upload_from(source, destination = to)
-      "#{curl_command} #{authentication} -T #{source} #{destination.to_s};"
-    end
+  def curl_upload_from(source, destination = to)
+    "#{curl_command} #{authentication} -T #{source} #{destination};"
+  end
 
-    def curl_download_to_pipe
-      "#{curl_command} #{authentication} #{from.to_s} |"
-    end
+  def curl_download_to_pipe
+    "#{curl_command} #{authentication} #{from} |"
+  end
 
-    def curl_manifest(object_manifest, destination = to)
-      "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination.to_s} --data-binary ''"
-    end
-  
+  def curl_manifest(object_manifest, destination = to)
+    "#{curl_command} #{authentication} -X PUT -H 'X-Object-Manifest: #{object_manifest}' #{destination} --data-binary ''"
+  end
 end

data/lib/uricp/orbit_auth.rb

@@ -1,8 +1,6 @@
 require 'open-uri'
 module Uricp
-
   class OrbitAuth
-    
     AuthenticationFailure = Class.new(ArgumentError)
 
     def initialize(auth_uri, auth_id, auth_key)
@@ -14,9 +12,9 @@ module Uricp
     def storage_url
       @storage_url ||
         begin
-	  authenticate
-	  @storage_url
-	end
+          authenticate
+          @storage_url
+        end
     end
 
     def token
@@ -26,27 +24,27 @@ module Uricp
 
     def self.validate_options(options)
       if options['auth-token'] && (options['auth-key'] || options['auth-user'])
-	raise ::OptionParser::NeedlessArgument,
-	  "use either key based or token based authentication"
+        raise ::OptionParser::NeedlessArgument,
+              'use either key based or token based authentication'
       end
       if options['auth-key'].nil? ^ options['auth-user'].nil?
-	raise ::OptionParser::MissingArgument,
-	  "'auth-user' requires 'auth-key'"
+        raise ::OptionParser::MissingArgument,
+              "'auth-user' requires 'auth-key'"
       end
       if (options['auth-token'] || options['auth-user']) && options['auth_uri'].nil?
-	raise ::OptionParser::NeedlessArgument,
-	  "authentication is for http uris only"
+        raise ::OptionParser::NeedlessArgument,
+              'authentication is for http uris only'
       end
     end
 
     def self.add_auth_token(options)
       if options['auth-user']
-	orbit_credentials = self.new(options['auth_uri'],
-			      options['auth-user'], options['auth-key'])
-	options['authenticator'] = orbit_credentials.method(:token)
+        orbit_credentials = new(options['auth_uri'],
+                                options['auth-user'], options['auth-key'])
+        options['authenticator'] = orbit_credentials.method(:token)
       elsif options['auth-token']
         orbit_token = options['auth-token']
-        options['authenticator'] = lambda { orbit_token }
+        options['authenticator'] = -> { orbit_token }
       end
       options.delete('auth-key')
       options.delete('auth-user')
@@ -54,29 +52,27 @@ module Uricp
     end
 
     def self.add_auth_to_optionparser(app)
-      app.on("--auth-token AUTH_TOKEN",
-	"Use AUTH_TOKEN for non-local requests")
-      app.on("--auth-user AUTH_USER",
-	"Use AUTH_USER for authentication")
-      app.on("--auth-key AUTH_KEY",
-	"Use AUTH_KEY for authentication")
+      app.on('--auth-token AUTH_TOKEN',
+             'Use AUTH_TOKEN for non-local requests')
+      app.on('--auth-user AUTH_USER',
+             'Use AUTH_USER for authentication')
+      app.on('--auth-key AUTH_KEY',
+             'Use AUTH_KEY for authentication')
     end
 
-  private
+    private
 
     def authenticate
       @auth_uri.open(
         'X-Auth-User' => @auth_id,
-	'X-Auth-Key' => @auth_key,
-	'Range' => 'bytes=0-0'
+        'X-Auth-Key' => @auth_key,
+        'Range' => 'bytes=0-0'
       ) do |uri|
         @storage_url = uri.meta['x-storage-url']
-	@token = uri.meta['x-auth-token']
+        @token = uri.meta['x-auth-token']
       end
     rescue OpenURI::HTTPError => e
       raise AuthenticationFailure, "Cannot authenticate against #{@auth_uri}"
     end
-
   end
-
 end

data/lib/uricp/segmenter.rb

@@ -1,9 +1,7 @@
 require 'pathname'
 
 module Uricp
-
   class Segmenter
-
     include Methadone::CLILogging
     include Methadone::SH
     include Uricp::CurlPrimitives
@@ -15,7 +13,7 @@ module Uricp
         @stream = File.pipe?(source) || File.chardev?(source)
       else
         @source = STDIN
-	@stream = true
+        @stream = true
       end
       split_path
     end
@@ -23,9 +21,9 @@ module Uricp
     def split_path
       elements = Pathname.new(to.path).enum_for(:each_filename).to_a
       elements.shift
-      @account=elements.shift
-      @container=elements.shift
-      @object_path=File.join(elements)
+      @account = elements.shift
+      @container = elements.shift
+      @object_path = File.join(elements)
     end
 
     def upload
@@ -33,7 +31,7 @@ module Uricp
         segmented_upload
       else
         sh! curl_upload_from(options['from']),
-	  :on_fail => "Upload to #{to} failed"
+            on_fail: "Upload to #{to} failed"
       end
     end
 
@@ -54,16 +52,16 @@ module Uricp
     end
 
     def object_manifest
-      @object_manifest ||= File.join(@container, @object_path, manifest_suffix)+'/'
+      @object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/'
     end
 
     def segmented_upload
       debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks."
       suffix = 0
       until @source.eof?
-	debug "#{self.class.name}: Uploading segment #{suffix}"
-	upload_segment(suffix)
-	suffix = suffix.next
+        debug "#{self.class.name}: Uploading segment #{suffix}"
+        upload_segment(suffix)
+        suffix = suffix.next
       end
       add_manifest
     end
@@ -71,18 +69,16 @@ module Uricp
     def upload_segment(segment_number)
       segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number)
       debug "Uploading with #{curl_upload_from('-', segment_name)}"
-      open('|'+curl_upload_from('-', segment_name), 'w') do |destination|
+      open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
         copy_length = IO.copy_stream(@source, destination, segment_size)
-	debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
+        debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}"
       end
     end
 
     def add_manifest
       debug "Adding DLO object_manifest #{object_manifest}"
       sh! curl_manifest(object_manifest),
-	:on_fail => "Upload to #{to} failed"
+          on_fail: "Upload to #{to} failed"
     end
-
   end
-
 end

data/lib/uricp/strategy/cache_common.rb

@@ -1,18 +1,40 @@
 require 'fileutils'
 
 module Uricp::Strategy
-
   module CacheCommon
-    
     def validate_cache!
+      return if dry_run?
+
+      unless cache_exists?
+        raise Uricp::MissingCache,
+              "no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory"
+      end
+      return if cache_name
+
       raise Uricp::MissingCache,
-	"no cache found at #{cache_root}. Expected a 'cache' and 'temp' directory" unless cache_exists?
-      raise Uricp::MissingCache,
-	"no cache name found" unless cache_name
+            'no cache name found'
+    end
+
+    def with_active_cache
+      if cache_root
+        yield
+      else
+        debug "#{self.class.name}: no cacheing requested"
+        false
+      end
     end
-      
+
+    def without_active_cache
+      unless cache_root
+        yield
+      else
+        debug "#{self.class.name}: cache active - not appropriate"
+        false
+      end
+    end
+
     def in_cache?
-      File.readable? cache_file
+      File.readable?(cache_file) || options['dry-cache']
     end
 
     def cache_root
@@ -24,7 +46,7 @@ module Uricp::Strategy
     end
 
     def cache_exists?
-      cache_root && %w{temp cache}.all? do |d|
+      cache_root && %w[temp cache].all? do |d|
         File.directory?(File.join(cache_root, d))
       end
     end
@@ -40,9 +62,5 @@ module Uricp::Strategy
     def cache_file
       @cache_file ||= File.join(cache_root, 'cache', cache_name)
     end
-
   end
-
 end
-
-

data/lib/uricp/strategy/cached_get.rb

@@ -1,48 +1,45 @@
 require 'uri'
 
 module Uricp::Strategy
-  
   class CachedGet
-
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
 
     def appropriate?
-      if cache_root
+      with_active_cache do
         validate_cache!
-	if in_cache? || file_source?
-	  return proposal
-	else
-	  debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
-	end
-      else
-	debug "#{self.class.name}: not appropriate"
+        return proposal if in_cache? || file_source?
+
+        debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
+        false
       end
-      false
     end
 
     def command
-      ":;" 
+      ':;'
     end
 
     def proposal
       @proposed_options = options.dup
-      unless file_source?
-	@proposed_options['from_uri'] = URI.join('file:///', cache_file)
+      @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
+      if to.scheme == 'rbd'
+        @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
       end
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')
       if conversion_required?
-	@proposed_options['source-format'] =
-	  File.open(@proposed_options['from_uri'].path) {|f| encoding(f)}
-	if @proposed_options['source-format'] == @proposed_options['target-format']
-	  @proposed_options.delete('source-format')
-	  @proposed_options.delete('target-format')
-	end
+        if dry_run?
+          @proposed_options['source-format'] = @proposed_options['target-format']
+        else
+          @proposed_options['source-format'] =
+            File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
+        end
+        if @proposed_options['source-format'] == @proposed_options['target-format']
+          @proposed_options.delete('source-format')
+          @proposed_options.delete('target-format')
+        end
       end
       self
     end
-
   end
-
 end

data/lib/uricp/strategy/cleaner.rb

@@ -1,13 +1,10 @@
 module Uricp::Strategy
-  
   class Cleaner
-
     include Uricp::Strategy::Common
 
     def appropriate?
-      if options['clean'] && sequence_complete?
-        return proposal
-      end
+      return proposal if options['clean'] && sequence_complete?
+
       debug "#{self.class.name}: not appropriate"
       false
     end
@@ -21,8 +18,5 @@ module Uricp::Strategy
       @proposed_options.delete('clean')
       self
     end
-
   end
-
 end
-

data/lib/uricp/strategy/common.rb

@@ -1,7 +1,5 @@
 module Uricp::Strategy
- 
   module Common
-    
     include Methadone::CLILogging
     include Uricp::CurlPrimitives
 
@@ -14,10 +12,10 @@ module Uricp::Strategy
 
     def unsupported_transfer
       raise Uricp::UnsupportedURLtype,
-        "Unsupported transfer from #{from.to_s} to #{to.to_s}"
+            "Unsupported transfer from #{from} to #{to}"
     end
 
-    alias :command :unsupported_transfer
+    alias command unsupported_transfer
 
     def all_local_files?
       !sequence_complete? && file_source? && to.scheme == 'file'
@@ -44,24 +42,23 @@ module Uricp::Strategy
     end
 
     def qcow2?(magic)
-      magic.unpack('a3C') == ['QFI',0xfb]
+      magic.unpack('a3C') == ['QFI', 0xfb]
     end
 
     def encoding(io)
       magic = io.read(4).to_s
-      case
-      when lz4?(magic)
+      if lz4?(magic)
         :lz4
-      when qcow2?(magic)
+      elsif qcow2?(magic)
         version = io.read(4)
-	case version.unpack('N')
-	when [2]
-	  :qcow2
-	when [3]
-	  :qcow3
-	else
-	  :qcow2un
-	end
+        case version.unpack('N')
+        when [2]
+          :qcow2
+        when [3]
+          :qcow3
+        else
+          :qcow2un
+        end
       else
         :raw
       end
@@ -73,6 +70,10 @@ module Uricp::Strategy
       )
     end
 
+    def dry_run?
+      options['dry-run']
+    end
+
     def lz4_source?
       options['source-format'] == :lz4
     end
@@ -90,9 +91,16 @@ module Uricp::Strategy
     end
 
     PIPE_URI = URI('pipe:/')
+    def rbd_base_name
+      'base'.freeze
+    end
+
+    def rbd_snapshot_name
+      'uricp_snap'.freeze
+    end
 
     def get_temp_filename(base_dir)
-      t = Time.now.strftime("%Y%m%d")
+      t = Time.now.strftime('%Y%m%d')
       File.join(base_dir, "uricp-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}")
     end
 
@@ -108,6 +116,56 @@ module Uricp::Strategy
       options['source-format'] && !lz4_source?
     end
 
-  end
+    def rbd_image_spec(uri)
+      uri.path[1..-1]
+    end
+
+    def rbd_cache_image_spec(uri)
+      File.join(File.dirname(uri.path)[1..-1], options['cache_name'])
+    end
+
+    def rbd_cache_name
+      options['rbd_cache_name']
+    end
+
+    def rbd_clone_snapshot(cache=rbd_cache_name)
+      "#{cache}@#{rbd_base_name}"
+    end
+
+    def rbd_uri(image_spec)
+      URI.join('rbd:///', image_spec)
+    end
 
+    def rbd_sequence_complete?
+      from.path.include?(to.path)
+    end
+
+    def rbd_id
+      'libvirt'
+    end
+
+    def in_rbd_cache?
+      options['in_rbd_cache']
+    end
+
+    def not_in_rbd_cache?
+      options['in_rbd_cache'] == false
+    end
+
+    def rbd_snapshot_spec?(uri)
+      uri.scheme == 'rbd' && uri.path.include?('@')
+    end
+
+    def in_rbd_cache(target)
+      result = false
+      if dry_run?
+        result = options['dry-cache'] && options['cache_name'] !~ /srv-...../
+      else
+        sh "rbd snap ls --id #{rbd_id} --format json #{target}" do |stdout|
+          result = JSON.parse(stdout).any? { |x| x['name'] == rbd_base_name }
+        end
+      end
+      result && rbd_clone_snapshot(target)
+    end
+  end
 end