uricp 0.0.17 → 0.0.22

data/features/step_definitions/rbd_steps.rb

@@ -0,0 +1,8 @@
+Then(/^a ceph entry named "(.*?)" should exist$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^a ceph snapshot named "(.*?)" should exist$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+

data/features/step_definitions/uricp_steps.rb

@@ -1,8 +1,10 @@
 Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
   case format
   when 'lz4'
-    assert_exact_output([0x184D2204].pack('V'),
-      File.open(filename, 'rb') {|f| f.read(4) }.to_s)
+    assert_exact_output(
+      [0x184D2204].pack('V'),
+      File.open(filename, 'rb') { |f| f.read(4) } || String.new
+    )
   when 'qcow2v3', 'qcow3'
     steps %{
       When I successfully run `qemu-img info #{filename}`
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
       And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
     }
   else
-    assert_no_partial_output([0x184D2204].pack('V'),
-      File.open(filename, 'rb') {|f| f.read(4) }.to_s)
+    assert_no_partial_output(
+      [0x184D2204].pack('V'),
+      File.open(filename, 'rb') { |f| f.read(4) } || String.new
+    )
     steps(%{
       When I successfully run `qemu-img info #{filename}`
       Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"

data/focal/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:focal
+MAINTAINER support@brightbox.co.uk
+
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
+
+#RUN apt-add-repository ppa:brightbox/ruby-ng
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
+  build-essential \
+  git \
+  qemu-utils \
+  liblz4-tool \
+  curl \
+  ruby \
+  ruby-dev \
+  python3-swiftclient \
+  ruby-bundler \
+  ceph-common
+
+RUN gem install bundler -v '~> 1.7'
+RUN mkdir -p /etc/ceph
+ARG RBD_USR=libvirt
+ARG RBD_PSW
+ARG CEPH_MON
+RUN /bin/echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
+RUN /bin/echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf

data/lib/uricp/strategy/cache_common.rb

@@ -15,8 +15,26 @@ module Uricp::Strategy
             'no cache name found'
     end
 
+    def with_active_cache
+      if cache_root
+        yield
+      else
+        debug "#{self.class.name}: no cacheing requested"
+        false
+      end
+    end
+
+    def without_active_cache
+      unless cache_root
+        yield
+      else
+        debug "#{self.class.name}: cache active - not appropriate"
+        false
+      end
+    end
+
     def in_cache?
-      File.readable? cache_file
+      File.readable?(cache_file) || options['dry-cache']
     end
 
     def cache_root

data/lib/uricp/strategy/cached_get.rb

@@ -6,15 +6,13 @@ module Uricp::Strategy
     include Uricp::Strategy::CacheCommon
 
     def appropriate?
-      if cache_root
+      with_active_cache do
         validate_cache!
         return proposal if in_cache? || file_source?
 
         debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
-      else
-        debug "#{self.class.name}: not appropriate"
+        false
       end
-      false
     end
 
     def command
@@ -24,10 +22,15 @@ module Uricp::Strategy
     def proposal
       @proposed_options = options.dup
       @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
+      if to.scheme == 'rbd'
+        @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
+      end
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')
       if conversion_required?
-        unless dry_run?
+        if dry_run?
+          @proposed_options['source-format'] = @proposed_options['target-format']
+        else
           @proposed_options['source-format'] =
             File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
         end

data/lib/uricp/strategy/common.rb

@@ -91,6 +91,15 @@ module Uricp::Strategy
     end
 
     PIPE_URI = URI('pipe:/')
+    DRY_SNAP = 'uricp_snap'.freeze
+
+    def rbd_base_name
+      'base'.freeze
+    end
+
+    def rbd_snapshot_name
+      @rbd_snapshot_name ||= dry_run? ? DRY_SNAP : SecureRandom.uuid
+    end
 
     def get_temp_filename(base_dir)
       t = Time.now.strftime('%Y%m%d')
@@ -109,12 +118,56 @@ module Uricp::Strategy
       options['source-format'] && !lz4_source?
     end
 
-    def rbd_target(uri)
+    def rbd_image_spec(uri)
       uri.path[1..-1]
     end
 
+    def rbd_cache_image_spec(uri)
+      File.join(File.dirname(uri.path)[1..-1], options['cache_name'])
+    end
+
+    def rbd_cache_name
+      options['rbd_cache_name']
+    end
+
+    def rbd_clone_snapshot(cache=rbd_cache_name)
+      "#{cache}@#{rbd_base_name}"
+    end
+
+    def rbd_uri(image_spec)
+      URI.join('rbd:///', image_spec)
+    end
+
+    def rbd_sequence_complete?
+      from.path.include?(to.path)
+    end
+
     def rbd_id
       'libvirt'
     end
+
+    def in_rbd_cache?
+      options['in_rbd_cache']
+    end
+
+    def not_in_rbd_cache?
+      options['in_rbd_cache'] == false
+    end
+
+    def rbd_snapshot_spec?(uri)
+      uri.scheme == 'rbd' && uri.path.include?('@')
+    end
+
+    def in_rbd_cache(target)
+      result = false
+      if dry_run?
+        result = options['dry-cache'] && options['cache_name'] !~ /srv-...../
+      else
+        sh "rbd snap ls --id #{rbd_id} --format json #{target} 2>/dev/null" do |stdout|
+          result = JSON.parse(stdout).any? { |x| x['name'] == rbd_base_name }
+        end
+      end
+      result && rbd_clone_snapshot(target)
+    end
   end
 end

data/lib/uricp/strategy/local_convert.rb

@@ -32,7 +32,7 @@ module Uricp::Strategy
     end
 
     def new_qemu?
-      @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /^compat/m)
+      @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
     end
 
     def supported_conversion?

data/lib/uricp/strategy/piped_cache.rb

@@ -6,17 +6,15 @@ module Uricp::Strategy
     include Uricp::Strategy::CacheCommon
 
     def appropriate?
-      if cache_root
+      with_active_cache do
         case from.scheme
         when 'pipe'
           validate_cache!
           return proposal
         end
         debug "#{self.class.name}: not appropriate"
-      else
-        debug "#{self.class.name}: no cacheing requested"
+        false
       end
-      false
     end
 
     def command
@@ -26,6 +24,9 @@ module Uricp::Strategy
     def proposal
       @proposed_options = options.dup
       @proposed_options['sweep'] = [temp_cache_file, cache_file]
+      if to.scheme == 'rbd'
+        @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
+      end
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')
       self

data/lib/uricp/strategy/piped_rbd_get.rb

@@ -2,45 +2,29 @@ require 'json'
 module Uricp::Strategy
   class PipedRbdGet
     include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
     include Methadone::SH
 
     def appropriate?
-      case from.scheme
-      when 'rbd'
-        return proposal unless sequence_complete? || snap_in_progress?
+      without_active_cache do
+        if from.scheme == 'rbd' &&
+          rbd_snapshot_spec?(from) &&
+          to.scheme != 'rbd'
+          return proposal unless sequence_complete?
+        end
       end
       debug "#{self.class.name}: not appropriate"
       false
     end
 
     def command
-      "rbd snap create --id #{rbd_id} '#{rbd_snap_target(from)}' && " \
-      "rbd export --no-progress --id #{rbd_id} '#{rbd_snap_target(from)}' - |"
+      "rbd export --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' - |"
     end
 
     def proposal
       @proposed_options = options.dup
-      @proposed_options['rbd_snapshot'] = rbd_snap_target(from)
       @proposed_options['from_uri'] = PIPE_URI
-      @proposed_options.delete('cache')
-      @proposed_options.delete('cache_name')
       self
     end
-
-    RBD_SNAPSHOT = 'uricp_snap'.freeze
-
-    def rbd_snap_target(uri)
-      "#{rbd_target(uri)}@#{RBD_SNAPSHOT}"
-    end
-
-    def snap_in_progress?
-      return false if dry_run?
-
-      result = false
-      sh "rbd snap ls --id #{rbd_id} --format json #{rbd_target(from)}" do |stdout|
-        result = JSON.parse(stdout).any? { |x| x['name'] == RBD_SNAPSHOT }
-      end
-      result
-    end
   end
 end

data/lib/uricp/strategy/rbd_cache_base_snap.rb

@@ -0,0 +1,27 @@
+module Uricp::Strategy
+  class RbdCacheBaseSnap
+    include Uricp::Strategy::Common
+    include Methadone::SH
+
+    def appropriate?
+      if rbd_cache_name &&
+        rbd_image_spec(from) == rbd_cache_name
+        return proposal
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"\
+      "rbd snap protect --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = rbd_uri(rbd_clone_snapshot(rbd_cache_name))
+      @proposed_options.delete('rbd_cache_name')
+      self
+    end
+  end
+end

data/lib/uricp/strategy/rbd_cache_check.rb

@@ -0,0 +1,42 @@
+require 'uri'
+
+module Uricp::Strategy
+  class RbdCacheCheck
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+    include Methadone::SH
+
+    def appropriate?
+      unless (from.scheme == 'rbd') != (to.scheme == 'rbd')
+        debug "#{self.class.name}: not appropriate"
+        return false
+      end
+
+      with_active_cache do
+        if from.scheme == 'rbd'
+          cache_check = in_rbd_cache(rbd_cache_image_spec(from))
+          return proposal(cache_check) if cache_check
+          debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
+        elsif to.scheme == 'rbd'
+          cache_check = in_rbd_cache(rbd_cache_image_spec(to))
+          return proposal(cache_check) if cache_check
+          debug "#{self.class.name}: no rbd cache entry for #{options['to_uri']}"
+        end
+        false
+      end
+    end
+
+    def command
+      ':;'
+    end
+
+    def proposal(cache_check)
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = rbd_uri(cache_check)
+      @proposed_options.delete('cache')
+      @proposed_options.delete('cache_name')
+      @proposed_options.delete('dry-cache')
+      self
+    end
+  end
+end

data/lib/uricp/strategy/rbd_cache_clone.rb

@@ -0,0 +1,40 @@
+require 'uri'
+
+module Uricp::Strategy
+  class RbdCacheClone
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+    include Methadone::SH
+
+    def appropriate?
+      unless from.scheme == 'rbd'
+        debug "#{self.class.name}: not appropriate"
+        return false
+      end
+
+      with_active_cache do
+        options['cache_name'] = File.basename(options['to_uri'].path)
+        cache_target = rbd_cache_image_spec(from)
+        cache_check = in_rbd_cache(rbd_cache_image_spec(from))
+        if cache_check 
+          debug "#{self.class.name}: Unexpected existing cache entry for #{options['to_uri']}"
+          unsupported_transfer
+        end
+        proposal(cache_target)
+      end
+    end
+
+    def command
+      ':;'
+    end
+
+    def proposal(cache_check)
+      @proposed_options = options.dup
+      @proposed_options['rbd_cache_target'] = rbd_uri(cache_check)
+      @proposed_options.delete('cache')
+      @proposed_options.delete('cache_name')
+      @proposed_options.delete('dry-cache')
+      self
+    end
+  end
+end

data/lib/uricp/strategy/rbd_cache_upload.rb

@@ -0,0 +1,40 @@
+module Uricp::Strategy
+  class RbdCacheUpload
+    include Uricp::Strategy::Common
+    include Methadone::SH
+
+    def appropriate?
+      if compression_required? || conversion_required?
+        debug "#{self.class.name}: not ready to upload"
+        return false
+      end
+      if rbd_cache_name
+        case from.scheme
+        when 'pipe', 'file'
+          return proposal if to.scheme == 'rbd'
+        end
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}';"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = rbd_uri(rbd_cache_name)
+      self
+    end
+
+    def data_source
+      if from.scheme == 'pipe'
+        '-'
+      else
+        "'#{from.path}'"
+      end
+    end
+
+  end
+end

data/lib/uricp/strategy/rbd_cached_get.rb

@@ -0,0 +1,36 @@
+require 'uri'
+
+module Uricp::Strategy
+  class RbdCachedGet
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+    include Methadone::SH
+
+    def appropriate?
+      if from.scheme != 'rbd' || rbd_snapshot_spec?(from)
+        debug "#{self.class.name}: not appropriate"
+        return false
+      end
+
+      with_active_cache do
+        cache_check = in_rbd_cache(rbd_image_spec(from))
+        return proposal(cache_check) if cache_check
+        debug "#{self.class.name}: no rbd cache entry for #{options['from_uri']}"
+        false
+      end
+    end
+
+    def command
+      ':;'
+    end
+
+    def proposal(cache_check)
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = rbd_uri(cache_check)
+      @proposed_options.delete('cache')
+      @proposed_options.delete('cache_name')
+      @proposed_options.delete('dry-cache')
+      self
+    end
+  end
+end

data/lib/uricp/strategy/rbd_cached_put.rb

@@ -0,0 +1,32 @@
+module Uricp::Strategy
+  class RbdCachedPut
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+
+    def appropriate?
+      return proposal if to.scheme == 'rbd' &&
+        rbd_snapshot_spec?(from) && 
+        rbd_cache_name.nil? &&
+        !rbd_sequence_complete?
+
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd clone --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = to
+      if options['rbd_cache_target']
+        @proposed_options['to_uri'] = options['rbd_cache_target']
+        @proposed_options['rbd_cache_name'] = rbd_image_spec(to)
+        @proposed_options.delete('rbd_cache_target')
+      end
+      self
+    end
+
+  end
+end

data/lib/uricp/strategy/{rbd_remote_put.rb → rbd_put.rb}

@@ -1,5 +1,5 @@
 module Uricp::Strategy
-  class RbdRemotePut
+  class RbdPut
     include Uricp::Strategy::Common
 
     def appropriate?
@@ -7,16 +7,18 @@ module Uricp::Strategy
         debug "#{self.class.name}: not ready to upload"
         return false
       end
-      case from.scheme
-      when 'pipe', 'file'
-        return proposal if to.scheme == 'rbd'
+      unless rbd_cache_name
+        case from.scheme
+        when 'pipe', 'file'
+          return proposal if to.scheme == 'rbd'
+        end
       end
       debug "#{self.class.name}: not appropriate"
       false
     end
 
     def command
-      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_target(to)}';"
+      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_image_spec(to)}';"
     end
 
     def proposal

data/lib/uricp/strategy/rbd_snap.rb

@@ -0,0 +1,57 @@
+require 'json'
+module Uricp::Strategy
+  class RbdSnap
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+    include Methadone::SH
+
+    def appropriate?
+      without_active_cache do
+        if from.scheme == 'rbd' &&
+          options['rbd_snapshot'].nil? &&
+          !rbd_snapshot_spec?(from) &&
+          (rbd_cache_name.nil? ||
+            !from.path.include?(rbd_cache_name))
+          if snap_in_progress?
+            debug "#{self.class.name}: detected snapshot in progress"
+          else
+            return proposal unless sequence_complete?
+          end
+        end
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}';" \
+      "rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}';" \
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = rbd_uri(rbd_upload_snapshot(from))
+      if options['rbd_cache_target']
+        @proposed_options['to_uri'] = options['rbd_cache_target']
+        @proposed_options['rbd_cache_target'] = to
+      else
+        @proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from) unless to.scheme == 'rbd'
+      end
+      self
+    end
+
+    def rbd_upload_snapshot(uri)
+      "#{rbd_image_spec(uri)}@#{rbd_snapshot_name}"
+    end
+
+    def snap_in_progress?
+      return false if dry_run?
+
+      result = false
+      sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)} 2>/dev/null" do |stdout|
+        result = JSON.parse(stdout).any? { |x| x['name'] == rbd_snapshot_name }
+      end
+      result
+    end
+  end
+end

data/lib/uricp/strategy/rbd_sweeper.rb

@@ -3,13 +3,14 @@ module Uricp::Strategy
     include Uricp::Strategy::Common
 
     def appropriate?
-      return proposal if options['rbd_snapshot'] && sequence_complete?
+      return proposal if options['rbd_snapshot'] && rbd_sequence_complete?
 
       debug "#{self.class.name}: not appropriate"
       false
     end
 
     def command
+      "rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}';" \
       "rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
     end
 

data/lib/uricp/uri_strategy.rb

@@ -25,6 +25,10 @@ module Uricp
 
     # This is an ordered list from the most specific to the most general
     STRATEGIES = [
+      Strategy::RbdCachedGet,
+      Strategy::RbdCacheClone,
+      Strategy::RbdCacheCheck,
+      Strategy::RbdSnap,
       Strategy::CachedGet,
       Strategy::PipedRemoteGet,
       Strategy::PipedRbdGet,
@@ -35,7 +39,10 @@ module Uricp
       Strategy::LocalConvert,
       Strategy::LocalLink,
       Strategy::PipedCompress,
-      Strategy::RbdRemotePut,
+      Strategy::RbdCacheUpload,
+      Strategy::RbdCachedPut,
+      Strategy::RbdPut,
+      Strategy::RbdCacheBaseSnap,
       Strategy::SegmentedRemotePut,
       Strategy::RemotePut,
       Strategy::PipedLocalCompress,

data/lib/uricp/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 module Uricp
-  VERSION = '0.0.17'.freeze
-  DEFAULT_SEGMENT_SIZE = '5 GiB'.freeze
+  VERSION = '0.0.22'
+  DEFAULT_SEGMENT_SIZE = '5 GiB'
 end