uricp 0.0.1

data/lib/uricp/strategy/cleaner.rb

@@ -0,0 +1,28 @@
+module Uricp::Strategy
+  
+  class Cleaner
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      if options['clean'] && sequence_complete?
+        return proposal
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rm -f #{options['clean'].join(' ')};"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options.delete('clean')
+      self
+    end
+
+  end
+
+end
+

data/lib/uricp/strategy/common.rb

@@ -0,0 +1,108 @@
+module Uricp::Strategy
+ 
+  module Common
+    
+    include Methadone::CLILogging
+    include Uricp::CurlPrimitives
+
+    def initialize(options)
+      @options = options
+      debug "#{self.class.name}: options are #{options.inspect}"
+    end
+
+    attr_reader :proposed_options
+
+    def unsupported_transfer
+      raise Uricp::UnsupportedURLtype,
+        "Unsupported transfer from #{from.to_s} to #{to.to_s}"
+    end
+
+    alias :command :unsupported_transfer
+
+    def all_local_files?
+      !sequence_complete? && file_source? && to.scheme == 'file'
+    end
+
+    def compression_required?
+      options['compress']
+    end
+
+    def segmented?
+      options['segment-size']
+    end
+
+    def conversion_required?
+      options['target-format']
+    end
+
+    def lz4?(magic)
+      magic.unpack('V') == [0x184D2204]
+    end
+
+    def qcow2?(magic)
+      magic.unpack('a3C') == ['QFI',0xfb]
+    end
+
+    def encoding(io)
+      magic = io.read(4)
+      case
+      when lz4?(magic)
+        :lz4
+      when qcow2?(magic)
+        version = io.read(4)
+	case version.unpack('N')
+	when [2]
+	  :qcow2
+	when [3]
+	  :qcow3
+	else
+	  :qcow2un
+	end
+      else
+        :raw
+      end
+    end
+
+    def format_change?
+      conversion_required? && (
+        options['source-format'] != options['target-format']
+      )
+    end
+
+    def lz4_source?
+      options['source-format'] == :lz4
+    end
+
+    def raw_target?
+      options['target-format'] == :raw
+    end
+
+    def file_source?
+      from.scheme == 'file'
+    end
+
+    def sequence_complete?
+      from == to
+    end
+
+    PIPE_URI = URI('pipe:/')
+
+    def get_temp_filename(filename)
+      Dir::Tmpname.make_tmpname(filename, nil)
+    end
+
+    def proposed_path
+      proposed_options['from_uri'].path
+    end
+
+    def temp_uri
+      URI.join('file:///', get_temp_filename(options['temp']))
+    end
+
+    def supported_source?
+      options['source-format'] && !lz4_source?
+    end
+
+  end
+
+end

data/lib/uricp/strategy/local_convert.rb

@@ -0,0 +1,36 @@
+module Uricp::Strategy
+  
+  class LocalConvert
+
+    include Uricp::Strategy::LocalConvertCommon
+
+    def appropriate?
+      if conversion_required? && file_source? && supported_source?
+	return proposal
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      qemu_img_command(proposed_path)
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      if to.scheme == 'file'
+	@proposed_options['from_uri'] = @proposed_options['to_uri']
+      else
+        @proposed_options['from_uri'] = temp_uri
+	@proposed_options['clean'] ||= []
+	@proposed_options['clean'] << proposed_path
+      end
+      @proposed_options.delete('source-format')
+      @proposed_options.delete('target-format')
+      self
+    end
+
+  end
+
+end
+

data/lib/uricp/strategy/local_convert_common.rb

@@ -0,0 +1,29 @@
+module Uricp::Strategy
+  
+  module LocalConvertCommon
+
+    include Uricp::Strategy::Common
+
+    def qemu_img_command(target)
+      "qemu-img convert #{conversion_options} #{from.path} #{target};"
+    end
+      
+    def conversion_options
+      case options['target-format']
+      when :raw
+        '-O raw'
+      when :qcow2
+        "#{compress_option} -O qcow2 -o compat=0.10"
+      when :qcow3, :qcow2v3
+        "#{compress_option} -O qcow2 -o compat=1.1"
+      end
+    end
+
+    def compress_option
+      options['compress'] && '-c'
+    end
+
+  end
+
+end
+

data/lib/uricp/strategy/local_link.rb

@@ -0,0 +1,31 @@
+module Uricp::Strategy
+  
+  class LocalLink
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      return proposal if all_local_files? && !format_change? && linkable?
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "ln -fT #{from.path} #{to.path};"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = @proposed_options['to_uri']
+      self
+    end
+
+    def linkable?
+      File.stat(File.dirname(from.path)).dev == File.stat(File.dirname(to.path)).dev &&
+      (!File.exist?(to.path) || File.file?(to.path))
+    end
+
+  end
+
+end
+

data/lib/uricp/strategy/piped_cache.rb

@@ -0,0 +1,40 @@
+require 'fileutils'
+
+module Uricp::Strategy
+
+  class PipedCache
+    
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+
+    def appropriate?
+      if cache_root
+	case from.scheme
+	when 'pipe'
+	  validate_cache!
+	  return proposal
+	end
+	debug "#{self.class.name}: not appropriate"
+      else
+	debug "#{self.class.name}: no cacheing requested" 
+      end
+      false
+    end
+
+    def command
+      "tee #{temp_cache_file} |"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['sweep'] = [temp_cache_file, cache_file]
+      @proposed_options.delete('cache')
+      @proposed_options.delete('cache_name')
+      self
+    end
+
+  end
+
+end
+
+

data/lib/uricp/strategy/piped_cache_convert.rb

@@ -0,0 +1,45 @@
+require 'fileutils'
+
+module Uricp::Strategy
+
+  class PipedCacheConvert
+    
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+
+    def appropriate?
+      if conversion_required? && supported_source? 
+	case from.scheme
+	when 'pipe'
+	  validate_cache! if cache_root
+	  return proposal
+	end
+	debug "#{self.class.name}: not appropriate"
+      else
+	debug "#{self.class.name}: no non-stream conversion detected" 
+      end
+      false
+    end
+
+    def command
+      "cp --sparse=always /dev/stdin #{proposed_path};"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      if cache_root
+	@proposed_options['from_uri'] = temp_cache_uri
+	@proposed_options['sweep'] = [temp_cache_file, cache_file]
+	@proposed_options.delete('cache')
+	@proposed_options.delete('cache_name')
+      else
+        @proposed_options['from_uri'] = temp_uri
+	@proposed_options['clean'] ||= []
+	@proposed_options['clean'] << proposed_path
+      end
+      self
+    end
+
+  end
+
+end

data/lib/uricp/strategy/piped_compress.rb

@@ -0,0 +1,30 @@
+module Uricp::Strategy
+
+  class PipedCompress
+    
+    include Uricp::Strategy::Common
+
+
+    def appropriate?
+      case from.scheme
+      when 'pipe'
+	return proposal if compression_required?
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "lz4c |"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options.delete('compress')
+      @proposed_options['encoding']='lz4'
+      self
+    end
+
+  end
+
+end

data/lib/uricp/strategy/piped_decompress.rb

@@ -0,0 +1,35 @@
+module Uricp::Strategy
+
+  class PipedDecompress
+    
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      case from.scheme
+      when 'pipe'
+	return proposal if lz4_source? 
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "lz4c -d |"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options.delete('source-format')
+      if @proposed_options['target-format']
+	@proposed_options['source-format'] = :raw
+        if @proposed_options['source-format'] == @proposed_options['target-format'] 
+	  @proposed_options.delete('source-format')
+	  @proposed_options.delete('target-format')
+	end
+      end
+      self
+    end
+
+  end
+
+end

data/lib/uricp/strategy/piped_local_compress.rb

@@ -0,0 +1,30 @@
+module Uricp::Strategy
+
+  class PipedLocalCompress
+    
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      case from.scheme
+      when 'file'
+        return proposal if !sequence_complete? && compression_required?
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "lz4c <#{from.path} |"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options.delete('compress')
+      @proposed_options['encoding'] = 'lz4'
+      @proposed_options['from_uri'] = PIPE_URI
+      self
+    end
+
+  end
+
+end

data/lib/uricp/strategy/piped_local_get.rb

@@ -0,0 +1,29 @@
+module Uricp::Strategy
+  
+  class PipedLocalGet
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      case from.scheme
+      when 'file'
+        return proposal unless sequence_complete?
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "cat #{from.path} |"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = PIPE_URI
+      self
+    end
+
+  end
+
+end
+

data/lib/uricp/strategy/piped_local_put.rb

@@ -0,0 +1,29 @@
+module Uricp::Strategy
+  
+  class PipedLocalPut
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      case from.scheme
+      when 'pipe'
+        return proposal if to.scheme == 'file'
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "cp --sparse=always /dev/stdin #{to.path};"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = @proposed_options['to_uri']
+      self
+    end
+
+  end
+
+end
+

data/lib/uricp/strategy/piped_remote_get.rb

@@ -0,0 +1,38 @@
+require 'open-uri'
+module Uricp::Strategy
+  
+  class PipedRemoteGet
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      case from.scheme
+      when 'http', 'https'
+        return proposal unless sequence_complete?
+      else
+	debug "#{self.class.name}: not appropriate"
+	false
+      end
+    end
+
+    alias :command :curl_download_to_pipe
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = PIPE_URI
+      if options['target-format']
+	@proposed_options['source-format'] = 
+	  options['from_uri'].open(headers) { |u| encoding(u) }
+      end
+      self
+    end
+
+    def headers
+      headers={'Range' => 'bytes=0-7'}
+      headers['X-Auth-Token'] = options['auth-token'] if http_authentication?
+      headers
+    end
+
+  end
+
+end

data/lib/uricp/strategy/pruner.rb

@@ -0,0 +1,22 @@
+module Uricp::Strategy
+
+  class Pruner
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      if conversion_required? && options['source-format']
+        unsupported_conversion
+      end
+      debug "#{self.class.name}: no dead ends to prune"
+      false
+    end
+
+    def unsupported_conversion
+      raise Uricp::UnsupportedConversion,
+        "Unsupported source conversion to #{options['target-format'].to_s}"
+    end
+
+  end
+
+end

data/lib/uricp/strategy/remote_put.rb

@@ -0,0 +1,47 @@
+module Uricp::Strategy
+  
+  class RemotePut
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      if compression_required? || conversion_required?
+        debug "#{self.class.name}: not ready to upload"
+      else
+	case to.scheme
+	when 'http', 'https'
+	  case from.scheme
+	  when 'file', 'pipe'
+	    if http_authentication?
+	      return proposal 
+	    else
+	      unsupported_transfer
+	    end
+	  end
+	end
+	debug "#{self.class.name}: not appropriate"
+      end
+      false
+    end
+
+    def command
+      curl_upload_from(curl_source)
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = to
+      self
+    end
+
+    def curl_source
+      if from.scheme == 'pipe'
+        '-'
+      else
+        from.path
+      end
+    end
+
+  end
+
+end

data/lib/uricp/strategy/segmented_remote_put.rb

@@ -0,0 +1,52 @@
+module Uricp::Strategy
+  
+  class SegmentedRemotePut
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      if segmented?
+        if compression_required? || conversion_required?
+	  debug "#{self.class.name}: not ready to upload"
+	else
+	  case to.scheme
+	  when 'http', 'https'
+	    case from.scheme
+	    when 'file', 'pipe'
+	      if http_authentication?
+		return proposal
+	      else
+		unsupported_transfer
+	      end
+	    end
+	  end
+	  debug "#{self.class.name}: not appropriate"
+	end
+      else
+        debug "#{self.class.name}: no segmentation requested"
+      end
+      false
+    end
+
+    def command
+      "segment_upload --segment-size '#{options['segment-size']}' #{source_details} #{to.to_s};"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = to
+      @proposed_options['segment-size'] = nil
+      self
+    end
+
+    def source_details
+      "--from #{from.path}" unless from.scheme == 'pipe'
+    end
+
+    def segmented?
+      options['segment-size']
+    end
+
+  end
+
+end

data/lib/uricp/strategy/sweeper.rb

@@ -0,0 +1,28 @@
+module Uricp::Strategy
+  
+  class Sweeper
+
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      if options['sweep'] && sequence_complete?
+        return proposal
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "mv #{options['sweep'].first} #{options['sweep'].last};"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options.delete('sweep')
+      self
+    end
+
+  end
+
+end
+