uricp 0.0.20 → 0.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e77694a18e045d2917d254afcb2db0a2c4cb7d81471d3ccabfe5c232bc340408
-  data.tar.gz: c063c5315485f25194e2f195c821470f0cc840cc242a0f88c174653a3a3bf167
+  metadata.gz: ff143aca9aad2f48ac2948322959e24a421b95cd04cb545a8921f18e601139f0
+  data.tar.gz: 719d814583ab1513f10bbc04f6c8487c0fb803a6ab8fb4a307de6c1bf6d7c39a
 SHA512:
-  metadata.gz: e8af0949b1de2c8c414d8c60f69390ebc5c722189ff9d86b3c0201c3a30aecded256c8c38be244516834ba97ca7e3ee58c325444304f766cef55b11053d018af
-  data.tar.gz: db89854277d707b488ef4c86bad899407445ff62bd9815aa60259f636b473e17905a83bd53b120f287eabe60012cb20681015a45ef752d3d98660ed6e4b01f71
+  metadata.gz: 78a4cab2edab9ef6ca65a483dc5fe5cad0b221bfd170f891812af0c5883ae11e9b64e60b49be535fc41829d91f21820016c5fcfb94850c83487b0dcb4ba9c36b
+  data.tar.gz: 740916783658e36e3346eda72ea799a7a35e58da3cbb40619516c2f788de343c72a976c95229d85beccd5e9a3b5afcc1ebc85b726ed5cc6e85c2a9723877efaf

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    uricp (0.0.20)
+    uricp (0.0.24)
       childprocess (~> 1.0)
       filesize (= 0.0.2)
       methadone (~> 2.0.2)

data/bin/uricp

@@ -4,6 +4,7 @@ require 'optparse'
 require 'uri'
 require 'methadone'
 require 'filesize'
+require 'securerandom'
 require 'uricp'
 
 class App
@@ -121,8 +122,8 @@ class App
     "Compress output")
   on("--[no-]dry-run",
     "Show what would be run,", "but don't run the command")
-  on("--[no-]dry-cache",
-    "On dry runs assume the cache is populated")
+  on("--dry-cache DRY_CACHE",[:rbd, :partial_rbd],
+    "On dry runs assume the cache is populated", "[rbd, partial_rbd]")
   on("--[no-]force",
     "Always write sparsely")
   #Arguments

data/features/rbd_access.feature

@@ -32,7 +32,7 @@ Feature: Manipulate file images in ceph
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
     """
 
   Scenario: HTTP URI from file cache via unpopulated rbd cache
@@ -53,19 +53,31 @@ Feature: Manipulate file images in ceph
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
     """
 
-  Scenario: HTTP URI to rbd via populated caches
+    Scenario: HTTP URI to rbd via partial rbd caches
     Given a correctly initialised cache at "/tmp/uricp"
-    When I successfully run `uricp --dry-run --dry-cache --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    When I successfully run `uricp --dry-run --dry-cache=partial_rbd --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
     And the output should not contain:
     """
     snap create
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
+    rbd import --no-progress --id libvirt - 'servers/srv-testy'
+    """
+
+  Scenario: HTTP URI to rbd via populated cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I successfully run `uricp --dry-run --dry-cache=rbd --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
+    And the output should not contain:
+    """
+    snap create
+    """
+    And the output should contain:
+    """
+    rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
     """
 
   Scenario: rbd to rbd - no cache
@@ -79,17 +91,17 @@ Feature: Manipulate file images in ceph
     """
     rbd snap protect --id libvirt 'servers/img-testy@uricp_snap'
     """
-    And the output should contain:
+    And the output should not contain:
     """
     rbd snap unprotect --id libvirt 'servers/img-testy@uricp_snap'
     """
-    And the output should contain:
+    And the output should not contain:
     """
-    rbd snap rm --id libvirt 'servers/img-testy@uricp_snap';
+    rbd snap rm --id libvirt 'servers/img-testy@uricp_snap'
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/img-testy@uricp_snap' 'servers/srv-testy'
+    rbd clone --id libvirt 'servers/img-testy@uricp_snap' 'servers/srv-testy'
     """
 
   Scenario: rbd to rbd - unpopulated cache
@@ -103,13 +115,13 @@ Feature: Manipulate file images in ceph
     """
     rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
     """
-    And the output should contain:
+    And the output should not contain:
     """
     rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
     """
-    And the output should contain:
+    And the output should not contain:
     """
-    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
     """
     And the output should not contain:
     """
@@ -117,7 +129,7 @@ Feature: Manipulate file images in ceph
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
+    rbd clone --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
     """
     And the output should not contain:
     """
@@ -131,21 +143,26 @@ Feature: Manipulate file images in ceph
     """
     rbd snap protect --id libvirt 'servers/img-testy@base'
     """
+
+  Scenario: rbd to rbd - partial cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I run `uricp --dry-run --dry-cache=partial_rbd --cache=/tmp/uricp rbd:///servers/srv-testy rbd:///servers/img-testy`
+    Then the exit status should be 70
     And the output should contain:
     """
-    rbd flatten --id libvirt --no-progress 'servers/img-testy'
+    Unsupported transfer
     """
 
   Scenario: rbd to rbd - populated cache
     Given a correctly initialised cache at "/tmp/uricp"
-    When I successfully run `uricp --dry-run --dry-cache --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
+    When I successfully run `uricp --dry-run --dry-cache=rbd --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
     And the output should not contain:
     """
     snap create
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/img-testy@base' 'servers/srv-testy'
+    rbd clone --id libvirt 'servers/img-testy@base' 'servers/srv-testy'
     """
 
   Scenario: RBD export.
@@ -165,7 +182,7 @@ Feature: Manipulate file images in ceph
     """
     And the output should contain:
     """
-    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
     """
     And the output should contain:
     """
@@ -174,7 +191,16 @@ Feature: Manipulate file images in ceph
 
   Scenario: RBD export with cacheing - populated cache
     Given a correctly initialised cache at "/tmp/uricp"
-    When I run `uricp --dry-run --dry-cache --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
+    When I run `uricp --dry-run --dry-cache=rbd --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
+    Then the exit status should be 70
+    And the output should contain:
+    """
+    Unsupported transfer
+    """
+
+  Scenario: RBD export with cacheing - partial cache
+    Given a correctly initialised cache at "/tmp/uricp"
+    When I run `uricp --dry-run --dry-cache=partial_rbd --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
     Then the exit status should be 70
     And the output should contain:
     """
@@ -192,13 +218,13 @@ Feature: Manipulate file images in ceph
     """
     rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
     """
-    And the output should contain:
+    And the output should not contain:
     """
     rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
     """
-    And the output should contain:
+    And the output should not contain:
     """
-    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
+    rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
     """
     And the output should not contain:
     """
@@ -206,7 +232,7 @@ Feature: Manipulate file images in ceph
     """
     And the output should contain:
     """
-    rbd clone --no-progress --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
+    rbd clone --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
     """
     And the output should contain:
     """
@@ -220,7 +246,3 @@ Feature: Manipulate file images in ceph
     """
     rbd snap protect --id libvirt 'servers/img-testy@base'
     """
-    And the output should contain:
-    """
-    rbd flatten --id libvirt --no-progress 'servers/img-testy'
-    """

data/lib/uricp/strategy/cache_common.rb

@@ -34,7 +34,7 @@ module Uricp::Strategy
     end
 
     def in_cache?
-      File.readable?(cache_file) || options['dry-cache']
+      File.readable?(cache_file) || options['dry-cache'] == :rbd
     end
 
     def cache_root

data/lib/uricp/strategy/common.rb

@@ -2,6 +2,7 @@ module Uricp::Strategy
   module Common
     include Methadone::CLILogging
     include Uricp::CurlPrimitives
+    include Methadone::SH
 
     def initialize(options)
       @options = options
@@ -91,12 +92,14 @@ module Uricp::Strategy
     end
 
     PIPE_URI = URI('pipe:/')
+    DRY_SNAP = 'uricp_snap'.freeze
+
     def rbd_base_name
       'base'.freeze
     end
 
     def rbd_snapshot_name
-      'uricp_snap'.freeze
+      @rbd_snapshot_name ||= dry_run? ? DRY_SNAP : SecureRandom.uuid
     end
 
     def get_temp_filename(base_dir)
@@ -156,12 +159,31 @@ module Uricp::Strategy
       uri.scheme == 'rbd' && uri.path.include?('@')
     end
 
+    def rbd_cache_upload_available?
+      rbd_cache_name && !rbd_cache_image_exists?(rbd_cache_name)
+    end
+
+    def rbd_cache_image_exists?(target)
+      command = "rbd status --id #{rbd_id} --format json #{target} 2>/dev/null"
+      if dry_run?
+        if options['dry-cache'] == :partial_rbd && options['cache_name'] !~ /srv-...../
+          command = "exit 0"
+        else
+          command = "exit 2"
+        end
+      end
+      sh!(command)
+      true
+    rescue Methadone::FailedCommandError
+      false
+    end
+
     def in_rbd_cache(target)
       result = false
       if dry_run?
-        result = options['dry-cache'] && options['cache_name'] !~ /srv-...../
+        result = options['dry-cache'] == :rbd && options['cache_name'] !~ /srv-...../
       else
-        sh "rbd snap ls --id #{rbd_id} --format json #{target}" do |stdout|
+        sh "rbd snap ls --id #{rbd_id} --format json #{target} 2>/dev/null" do |stdout|
           result = JSON.parse(stdout).any? { |x| x['name'] == rbd_base_name }
         end
       end

data/lib/uricp/strategy/piped_cache.rb

@@ -24,8 +24,9 @@ module Uricp::Strategy
     def proposal
       @proposed_options = options.dup
       @proposed_options['sweep'] = [temp_cache_file, cache_file]
+      image_spec_to_check = rbd_cache_image_spec(to)
       if to.scheme == 'rbd'
-        @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
+        @proposed_options['rbd_cache_name'] = image_spec_to_check
       end
       @proposed_options.delete('cache')
       @proposed_options.delete('cache_name')

data/lib/uricp/strategy/piped_rbd_get.rb

@@ -3,7 +3,6 @@ module Uricp::Strategy
   class PipedRbdGet
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
-    include Methadone::SH
 
     def appropriate?
       without_active_cache do

data/lib/uricp/strategy/rbd_cache_base_snap.rb

@@ -1,7 +1,6 @@
 module Uricp::Strategy
   class RbdCacheBaseSnap
     include Uricp::Strategy::Common
-    include Methadone::SH
 
     def appropriate?
       if rbd_cache_name &&
@@ -13,7 +12,7 @@ module Uricp::Strategy
     end
 
     def command
-      "rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}' && "\
+      "rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"\
       "rbd snap protect --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"
     end
 

data/lib/uricp/strategy/rbd_cache_check.rb

@@ -4,7 +4,6 @@ module Uricp::Strategy
   class RbdCacheCheck
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
-    include Methadone::SH
 
     def appropriate?
       unless (from.scheme == 'rbd') != (to.scheme == 'rbd')

data/lib/uricp/strategy/rbd_cache_clone.rb

@@ -4,7 +4,6 @@ module Uricp::Strategy
   class RbdCacheClone
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
-    include Methadone::SH
 
     def appropriate?
       unless from.scheme == 'rbd'
@@ -15,8 +14,7 @@ module Uricp::Strategy
       with_active_cache do
         options['cache_name'] = File.basename(options['to_uri'].path)
         cache_target = rbd_cache_image_spec(from)
-        cache_check = in_rbd_cache(rbd_cache_image_spec(from))
-        if cache_check 
+        if rbd_cache_image_exists?(cache_target) || in_rbd_cache(cache_target)
           debug "#{self.class.name}: Unexpected existing cache entry for #{options['to_uri']}"
           unsupported_transfer
         end

data/lib/uricp/strategy/rbd_cache_upload.rb

@@ -1,14 +1,13 @@
 module Uricp::Strategy
   class RbdCacheUpload
     include Uricp::Strategy::Common
-    include Methadone::SH
 
     def appropriate?
       if compression_required? || conversion_required?
         debug "#{self.class.name}: not ready to upload"
         return false
       end
-      if rbd_cache_name
+      if rbd_cache_upload_available?
         case from.scheme
         when 'pipe', 'file'
           return proposal if to.scheme == 'rbd'
@@ -19,7 +18,7 @@ module Uricp::Strategy
     end
 
     def command
-      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}' && "
+      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}';"
     end
 
     def proposal

data/lib/uricp/strategy/rbd_cached_get.rb

@@ -4,7 +4,6 @@ module Uricp::Strategy
   class RbdCachedGet
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
-    include Methadone::SH
 
     def appropriate?
       if from.scheme != 'rbd' || rbd_snapshot_spec?(from)

data/lib/uricp/strategy/rbd_cached_put.rb

@@ -14,7 +14,7 @@ module Uricp::Strategy
     end
 
     def command
-      "rbd clone --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
+      "rbd clone --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
     end
 
     def proposal
@@ -23,7 +23,6 @@ module Uricp::Strategy
       if options['rbd_cache_target']
         @proposed_options['to_uri'] = options['rbd_cache_target']
         @proposed_options['rbd_cache_name'] = rbd_image_spec(to)
-        @proposed_options['rbd_flatten'] ||= @proposed_options['rbd_cache_name']
         @proposed_options.delete('rbd_cache_target')
       end
       self

data/lib/uricp/strategy/rbd_put.rb

@@ -7,11 +7,13 @@ module Uricp::Strategy
         debug "#{self.class.name}: not ready to upload"
         return false
       end
-      unless rbd_cache_name
-        case from.scheme
-        when 'pipe', 'file'
-          return proposal if to.scheme == 'rbd'
-        end
+      if rbd_cache_upload_available?
+        debug "#{self.class.name}: cacheing in RBD instead"
+        return false
+      end
+      case from.scheme
+      when 'pipe', 'file'
+        return proposal if to.scheme == 'rbd'
       end
       debug "#{self.class.name}: not appropriate"
       false

data/lib/uricp/strategy/rbd_snap.rb

@@ -3,7 +3,6 @@ module Uricp::Strategy
   class RbdSnap
     include Uricp::Strategy::Common
     include Uricp::Strategy::CacheCommon
-    include Methadone::SH
 
     def appropriate?
       without_active_cache do
@@ -24,17 +23,18 @@ module Uricp::Strategy
     end
 
     def command
-      "rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
-      "rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
+      "rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}';" \
+      "rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}';" \
     end
 
     def proposal
       @proposed_options = options.dup
-      @proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from)
-      @proposed_options['from_uri'] = rbd_uri(@proposed_options['rbd_snapshot'])
+      @proposed_options['from_uri'] = rbd_uri(rbd_upload_snapshot(from))
       if options['rbd_cache_target']
         @proposed_options['to_uri'] = options['rbd_cache_target']
         @proposed_options['rbd_cache_target'] = to
+      else
+        @proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from) unless to.scheme == 'rbd'
       end
       self
     end
@@ -47,7 +47,7 @@ module Uricp::Strategy
       return false if dry_run?
 
       result = false
-      sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)}" do |stdout|
+      sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)} 2>/dev/null" do |stdout|
         result = JSON.parse(stdout).any? { |x| x['name'] == rbd_snapshot_name }
       end
       result

data/lib/uricp/strategy/rbd_sweeper.rb

@@ -10,7 +10,7 @@ module Uricp::Strategy
     end
 
     def command
-      "rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}' && " \
+      "rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}';" \
       "rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
     end
 

data/lib/uricp/uri_strategy.rb

@@ -49,7 +49,6 @@ module Uricp
       Strategy::PipedLocalGet,
       Strategy::PipedLocalPut,
       Strategy::Cleaner,
-      Strategy::RbdFlattener,
       Strategy::RbdSweeper,
       Strategy::Sweeper
     ].freeze

data/lib/uricp/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Uricp
-  VERSION = '0.0.20'
+  VERSION = '0.0.24'
   DEFAULT_SEGMENT_SIZE = '5 GiB'
 end

data/lib/uricp.rb

@@ -29,7 +29,6 @@ require 'uricp/strategy/rbd_cache_clone'
 require 'uricp/strategy/rbd_cached_get'
 require 'uricp/strategy/rbd_cached_put'
 require 'uricp/strategy/rbd_cache_upload'
-require 'uricp/strategy/rbd_flattener'
 require 'uricp/strategy/rbd_put'
 require 'uricp/strategy/rbd_snap'
 require 'uricp/strategy/rbd_sweeper'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: uricp
 version: !ruby/object:Gem::Version
-  version: 0.0.20
+  version: 0.0.24
 platform: ruby
 authors:
 - Neil Wilson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-14 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -229,7 +229,6 @@ files:
 - lib/uricp/strategy/rbd_cache_upload.rb
 - lib/uricp/strategy/rbd_cached_get.rb
 - lib/uricp/strategy/rbd_cached_put.rb
-- lib/uricp/strategy/rbd_flattener.rb
 - lib/uricp/strategy/rbd_put.rb
 - lib/uricp/strategy/rbd_snap.rb
 - lib/uricp/strategy/rbd_sweeper.rb

data/lib/uricp/strategy/rbd_flattener.rb

@@ -1,23 +0,0 @@
-module Uricp::Strategy
-  class RbdFlattener
-    include Uricp::Strategy::Common
-
-    def appropriate?
-      return proposal if options['rbd_flatten'] &&
-        rbd_sequence_complete?
-
-      debug "#{self.class.name}: not appropriate"
-      false
-    end
-
-    def command
-      "rbd flatten --id #{rbd_id} --no-progress '#{options['rbd_flatten']}' && "
-    end
-
-    def proposal
-      @proposed_options = options.dup
-      @proposed_options.delete('rbd_flatten')
-      self
-    end
-  end
-end