uricp 0.0.15 → 0.0.20

data/bionic/Dockerfile

@@ -1,6 +1,8 @@
 FROM ubuntu:bionic
 MAINTAINER support@brightbox.co.uk
 
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
 
 #RUN apt-add-repository ppa:brightbox/ruby-ng
@@ -13,8 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
   curl \
   ruby \
   ruby-dev \
-  python-swiftclient
+  python-swiftclient \
+  ruby-bundler
 
-RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
-RUN gem install bundler
 

data/centos7/Dockerfile

@@ -1,20 +1,25 @@
 FROM centos:7
 MAINTAINER support@brightbox.co.uk
 
-RUN yum update -y && yum -y install \
-    epel-release \
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+RUN yum update -y
+RUN yum -y install \
+    centos-release-openstack-train \
+    epel-release 
+RUN yum -y install \
     gcc \
     make \
     git \
     qemu-img \
+    lz4 \
     ruby-devel \
-    rubygems 
-
-RUN yum -y install python-setuptools && \
-    easy_install pip && \
-    pip install --upgrade setuptools && \
-    pip install python-swiftclient
-
-RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
-RUN gem install bundler
-
+    rubygems \
+    rubygem-bundler \
+    ceph-common \
+    python-swiftclient
+RUN mkdir -p /etc/ceph
+ARG RBD_USR=libvirt
+ARG RBD_PSW
+ARG CEPH_MON
+RUN echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
+RUN echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf

data/cucumber.yml

@@ -1,3 +1,4 @@
 default: --profile new-qemu-image
 new-qemu-image: --tags ~@old-qemu-image
 old-qemu-image: --tags ~@new-qemu-image
+rbd: --tags @rbd

data/features/check_uri_path.feature

@@ -0,0 +1,14 @@
+Feature: Check URI paths
+  In order to ensure reliable uricp usage
+  As a command line user
+  I want to reject badly formed URIs
+
+  Scenario: source URI is incorrectly formatted
+        When I run `uricp --dry-run file:fred rbd:///servers/fred`
+    Then the exit status should not be 0
+    And the stderr should contain "invalid argument: 'from_uri' has missing path"
+
+  Scenario: target URI is incorrectly formatted
+        When I run `uricp --dry-run file:///fred rbd:servers/fred`
+    Then the exit status should not be 0
+    And the stderr should contain "invalid argument: 'to_uri' has missing path"

data/features/rbd_access.feature

@@ -0,0 +1,226 @@
+@rbd
+Feature: Manipulate file images in ceph
+  in order to manipulate file images in ceph
+  as a command line user
+  I want to retrieve the uri via an optional cache and copy correctly to target in the right format
+
+  background: 
+    given an empty directory named "/tmp/uricp"
+    and the default aruba timeout is 15 seconds
+
+  Scenario: HTTP URI qcow2 to rbd - no cache
+    When I successfully run `uricp --dry-run http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd import --no-progress --id libvirt - 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI qcow2 to raw via unpopulated rbd cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd import --no-progress --id libvirt - 'servers/freedos.qcow2'
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI from file cache via unpopulated rbd cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    And a 102400 byte file named "/tmp/uricp/cache/freedos.qcow2"
+    When I successfully run `uricp --dry-run --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd import --no-progress --id libvirt '/tmp/uricp/cache/freedos.qcow2' 'servers/freedos.qcow2'
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/freedos.qcow2@base'
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI to rbd via populated caches
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --dry-cache --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should not contain:
+    """
+    snap create
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    """
+
+  Scenario: rbd to rbd - no cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run rbd:///servers/img-testy rbd:///servers/srv-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/img-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/img-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/img-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/img-testy@uricp_snap';
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/img-testy@uricp_snap' 'servers/srv-testy'
+    """
+
+  Scenario: rbd to rbd - unpopulated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --cache=/tmp/uricp rbd:///servers/srv-testy rbd:///servers/img-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    """
+    And the output should not contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
+    """
+    And the output should not contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/img-testy@base' -
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd flatten --id libvirt --no-progress 'servers/img-testy'
+    """
+
+  Scenario: rbd to rbd - populated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --dry-cache --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
+    And the output should not contain:
+    """
+    snap create
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/img-testy@base' 'servers/srv-testy'
+    """
+
+  Scenario: RBD export.
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --compress rbd:///servers/srv-testy file:///tmp/img-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    """
+    And the output should contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
+    """
+
+  Scenario: RBD export with cacheing - populated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I run `uricp --dry-run --dry-cache --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
+    Then the exit status should be 70
+    And the output should contain:
+    """
+    Unsupported transfer
+    """
+
+  Scenario: RBD export with cacheing - empty cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
+    """
+    And the output should contain:
+    """
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    """
+    And the output should not contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
+    """
+    And the output should contain:
+    """
+    rbd clone --no-progress --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
+    """
+    And the output should contain:
+    """
+    rbd export --no-progress --id libvirt 'servers/img-testy@base' -
+    """
+    And the output should contain:
+    """
+    rbd snap create --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd snap protect --id libvirt 'servers/img-testy@base'
+    """
+    And the output should contain:
+    """
+    rbd flatten --id libvirt --no-progress 'servers/img-testy'
+    """

data/features/step_definitions/orbit_steps.rb

@@ -18,12 +18,19 @@ def fetch_orbit_token
   @current_url_target = ENV['ORBIT_URL'] || 'https://orbit.brightbox.com/v1'
   cmd = "curl -I #{@current_url_target} -H 'X-Auth-User: #{@clientid}' -H 'X-Auth-Key: #{@key}'"
   run_simple(unescape(cmd))
-  stdout_from(cmd).each_line do |line|
+  output = stdout_from(cmd)
+  output.each_line do |line|
     key, value = line.strip.split(/\s*:\s*/,2)
-    @current_auth_token = value if key == 'X-Auth-Token'
-    @current_storage_url = value if key == 'X-Storage-Url'
+    case key.to_s.downcase
+    when 'x-auth-token'
+      @current_auth_token = value
+    when 'x-storage-url'
+      @current_storage_url = value
+    end
+  end
+  unless @current_auth_token && @current_storage_url
+    raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.\n#{output}")
   end
-  raise(RuntimeError, "Credentials not accepted. Can't obtain auth token for tests.") unless @current_auth_token && @current_storage_url 
 end
 
 def create_download_file(filetype, name, container)

data/features/step_definitions/rbd_steps.rb

@@ -0,0 +1,8 @@
+Then(/^a ceph entry named "(.*?)" should exist$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^a ceph snapshot named "(.*?)" should exist$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+

data/features/step_definitions/uricp_steps.rb

@@ -1,8 +1,10 @@
 Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
   case format
   when 'lz4'
-    assert_exact_output([0x184D2204].pack('V'),
-      File.open(filename, 'rb') {|f| f.read(4) }.to_s)
+    assert_exact_output(
+      [0x184D2204].pack('V'),
+      File.open(filename, 'rb') { |f| f.read(4) } || String.new
+    )
   when 'qcow2v3', 'qcow3'
     steps %{
       When I successfully run `qemu-img info #{filename}`
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
       And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
     }
   else
-    assert_no_partial_output([0x184D2204].pack('V'),
-      File.open(filename, 'rb') {|f| f.read(4) }.to_s)
+    assert_no_partial_output(
+      [0x184D2204].pack('V'),
+      File.open(filename, 'rb') { |f| f.read(4) } || String.new
+    )
     steps(%{
       When I successfully run `qemu-img info #{filename}`
       Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"

data/focal/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:focal
+MAINTAINER support@brightbox.co.uk
+
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
+
+#RUN apt-add-repository ppa:brightbox/ruby-ng
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
+  build-essential \
+  git \
+  qemu-utils \
+  liblz4-tool \
+  curl \
+  ruby \
+  ruby-dev \
+  python3-swiftclient \
+  ruby-bundler \
+  ceph-common
+
+RUN gem install bundler -v '~> 1.7'
+RUN mkdir -p /etc/ceph
+ARG RBD_USR=libvirt
+ARG RBD_PSW
+ARG CEPH_MON
+RUN /bin/echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
+RUN /bin/echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf

data/lib/segment_upload.rb

@@ -1,36 +1,34 @@
-require "uricp/version"
-require "uricp/curl_primitives"
-require "uricp/orbit_auth"
-require "uricp/segmenter"
+require 'uricp/version'
+require 'uricp/curl_primitives'
+require 'uricp/orbit_auth'
+require 'uricp/segmenter'
 
 module Uricp
-
   UnsupportedURLtype = Class.new(ArgumentError)
-
 end
 
-#Monkey patch a copy_stream facility in using 'sendfile'
+# Monkey patch a copy_stream facility in using 'sendfile'
 unless IO.respond_to? :copy_stream
   require 'sendfile'
 
-  def IO.copy_stream(src, dst, copy_length=nil, offset=nil)
-    unless src.stat.pipe?
-      current_pos = src.pos
-      count = dst.sendfile(src, offset || current_pos, copy_length)
-      src.seek(count, IO::SEEK_CUR)
-      return count
-    else
+  def IO.copy_stream(src, dst, copy_length = nil, offset = nil)
+    if src.stat.pipe?
       amount = copy_length.to_i
       buf_size = [amount, 2**16].min
-      buffer=""
-      while amount > 0 do
-	src.read(buf_size, buffer)
-	amount_read = buffer.length
-	dst.write(buffer)
-	amount -= amount_read
-	break if src.eof?
+      buffer = ''
+      while amount > 0
+        src.read(buf_size, buffer)
+        amount_read = buffer.length
+        dst.write(buffer)
+        amount -= amount_read
+        break if src.eof?
       end
-      return copy_length.to_i - amount
+      copy_length.to_i - amount
+    else
+      current_pos = src.pos
+      count = dst.sendfile(src, offset || current_pos, copy_length)
+      src.seek(count, IO::SEEK_CUR)
+      count
     end
   rescue EOFError
     0