uricp 0.0.16 → 0.0.21

data/features/rbd_access.feature

@@ -0,0 +1,226 @@
+@rbd
+Feature: Manipulate file images in ceph
+  in order to manipulate file images in ceph
+  as a command line user
+  I want to retrieve the uri via an optional cache and copy correctly to target in the right format
+
+  background: 
+    given an empty directory named "/tmp/uricp"
+    and the default aruba timeout is 15 seconds
+
+  Scenario: HTTP URI qcow2 to rbd - no cache
+    When I successfully run `uricp --dry-run http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd import --no-progress --id libvirt - 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI qcow2 to raw via unpopulated rbd cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd import --no-progress --id libvirt - 'servers/freedos.qcow2'
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI from file cache via unpopulated rbd cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    And a 102400 byte file named "/tmp/uricp/cache/freedos.qcow2"
+    When I successfully run `uricp --dry-run --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd import --no-progress --id libvirt '/tmp/uricp/cache/freedos.qcow2' 'servers/freedos.qcow2'
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI to rbd via populated caches
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --dry-cache --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should not contain:
+    """
+    snap create
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    """
+
+  Scenario: rbd to rbd - no cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run rbd:///servers/img-testy rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/img-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/img-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/img-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/img-testy@uricp_snap';
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/img-testy@uricp_snap' 'servers/srv-testy'
+    """
+
+  Scenario: rbd to rbd - unpopulated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --cache=/tmp/uricp rbd:///servers/srv-testy rbd:///servers/img-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    """
+    And the output should not contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
+    """
+    And the output should not contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/img-testy@base' -
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd flatten --id libvirt --no-progress 'servers/img-testy'
+    """
+
+  Scenario: rbd to rbd - populated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --dry-cache --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
+    And the output should not contain:
+    """
+    snap create
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/img-testy@base' 'servers/srv-testy'
+    """
+
+  Scenario: RBD export.
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --compress rbd:///servers/srv-testy file:///tmp/img-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    """
+    And the output should contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
+    """
+
+  Scenario: RBD export with cacheing - populated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I run `uricp --dry-run --dry-cache --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
+    Then the exit status should be 70
+    And the output should contain:
+    """
+    Unsupported transfer
+    """
+
+  Scenario: RBD export with cacheing - empty cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    """
+    And the output should not contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
+    """
+    And the output should contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/img-testy@base' -
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd flatten --id libvirt --no-progress 'servers/img-testy'
+    """

data/features/step_definitions/rbd_steps.rb

@@ -0,0 +1,8 @@
+Then(/^a ceph entry named "(.*?)" should exist$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^a ceph snapshot named "(.*?)" should exist$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+

data/features/step_definitions/uricp_steps.rb

@@ -1,8 +1,10 @@
 Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
   case format
   when 'lz4'
-    assert_exact_output([0x184D2204].pack('V'),
-      File.open(filename, 'rb') {|f| f.read(4) }.to_s)
+    assert_exact_output(
+      [0x184D2204].pack('V'),
+      File.open(filename, 'rb') { |f| f.read(4) } || String.new
+    )
   when 'qcow2v3', 'qcow3'
     steps %{
       When I successfully run `qemu-img info #{filename}`
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
       And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
     }
   else
-    assert_no_partial_output([0x184D2204].pack('V'),
-      File.open(filename, 'rb') {|f| f.read(4) }.to_s)
+    assert_no_partial_output(
+      [0x184D2204].pack('V'),
+      File.open(filename, 'rb') { |f| f.read(4) } || String.new
+    )
     steps(%{
       When I successfully run `qemu-img info #{filename}`
       Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"

data/focal/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:focal
+MAINTAINER support@brightbox.co.uk
+
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
+
+#RUN apt-add-repository ppa:brightbox/ruby-ng
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
+  build-essential \
+  git \
+  qemu-utils \
+  liblz4-tool \
+  curl \
+  ruby \
+  ruby-dev \
+  python3-swiftclient \
+  ruby-bundler \
+  ceph-common
+
+RUN gem install bundler -v '~> 1.7'
+RUN mkdir -p /etc/ceph
+ARG RBD_USR=libvirt
+ARG RBD_PSW
+ARG CEPH_MON
+RUN /bin/echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
+RUN /bin/echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf

data/lib/uricp/strategy/cache_common.rb

@@ -15,8 +15,26 @@ module Uricp::Strategy
             'no cache name found'
     end
 
+    def with_active_cache
+      if cache_root
+        yield
+      else
+        debug "#{self.class.name}: no cacheing requested"
+        false
+      end
+    end
+
+    def without_active_cache
+      unless cache_root
+        yield
+      else
+        debug "#{self.class.name}: cache active - not appropriate"
+        false
+      end
+    end
+
     def in_cache?
-      File.readable? cache_file
+      File.readable?(cache_file) || options['dry-cache']
     end
 
     def cache_root

data/lib/uricp/strategy/cached_get.rb

@@ -6,15 +6,13 @@ module Uricp::Strategy
     include Uricp::Strategy::CacheCommon
 
     def appropriate?
-      if cache_root
+      with_active_cache do
         validate_cache!
         return proposal if in_cache? || file_source?
 
         debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
-      else
-        debug "#{self.class.name}: not appropriate"
+        false
       end
-      false
     end
 
     def command
@@ -24,10 +22,15 @@ module Uricp::Strategy
     def proposal
       @proposed_options = options.dup
       @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
+      if to.scheme == 'rbd'
+        @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
+      end
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')
       if conversion_required?
-        unless dry_run?
+        if dry_run?
+          @proposed_options['source-format'] = @proposed_options['target-format']
+        else
           @proposed_options['source-format'] =
             File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
         end

data/lib/uricp/strategy/common.rb

@@ -91,6 +91,13 @@ module Uricp::Strategy
     end
 
     PIPE_URI = URI('pipe:/')
+    def rbd_base_name
+      'base'.freeze
+    end
+
+    def rbd_snapshot_name
+      'uricp_snap'.freeze
+    end
 
     def get_temp_filename(base_dir)
       t = Time.now.strftime('%Y%m%d')
@@ -109,12 +116,56 @@ module Uricp::Strategy
       options['source-format'] && !lz4_source?
     end
 
-    def rbd_target(uri)
+    def rbd_image_spec(uri)
       uri.path[1..-1]
     end
 
+    def rbd_cache_image_spec(uri)
+      File.join(File.dirname(uri.path)[1..-1], options['cache_name'])
+    end
+
+    def rbd_cache_name
+      options['rbd_cache_name']
+    end
+
+    def rbd_clone_snapshot(cache=rbd_cache_name)
+      "#{cache}@#{rbd_base_name}"
+    end
+
+    def rbd_uri(image_spec)
+      URI.join('rbd:///', image_spec)
+    end
+
+    def rbd_sequence_complete?
+      from.path.include?(to.path)
+    end
+
     def rbd_id
       'libvirt'
     end
+
+    def in_rbd_cache?
+      options['in_rbd_cache']
+    end
+
+    def not_in_rbd_cache?
+      options['in_rbd_cache'] == false
+    end
+
+    def rbd_snapshot_spec?(uri)
+      uri.scheme == 'rbd' && uri.path.include?('@')
+    end
+
+    def in_rbd_cache(target)
+      result = false
+      if dry_run?
+        result = options['dry-cache'] && options['cache_name'] !~ /srv-...../
+      else
+        sh "rbd snap ls --id #{rbd_id} --format json #{target} 2>/dev/null" do |stdout|
+          result = JSON.parse(stdout).any? { |x| x['name'] == rbd_base_name }
+        end
+      end
+      result && rbd_clone_snapshot(target)
+    end
   end
 end

data/lib/uricp/strategy/local_convert.rb

@@ -32,7 +32,7 @@ module Uricp::Strategy
     end
 
     def new_qemu?
-      @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /^compat/m)
+      @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
     end
 
     def supported_conversion?

data/lib/uricp/strategy/piped_cache.rb

@@ -6,17 +6,15 @@ module Uricp::Strategy
     include Uricp::Strategy::CacheCommon
 
     def appropriate?
-      if cache_root
+      with_active_cache do
         case from.scheme
         when 'pipe'
           validate_cache!
           return proposal
         end
         debug "#{self.class.name}: not appropriate"
-      else
-        debug "#{self.class.name}: no cacheing requested"
+        false
       end
-      false
     end
 
     def command
@@ -26,6 +24,9 @@ module Uricp::Strategy
     def proposal
       @proposed_options = options.dup
       @proposed_options['sweep'] = [temp_cache_file, cache_file]
+      if to.scheme == 'rbd'
+        @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
+      end
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')
       self