uricp 0.0.18 → 0.0.20

data/lib/uricp/strategy/rbd_cached_put.rb

@@ -0,0 +1,33 @@
+module Uricp::Strategy
+  class RbdCachedPut
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+
+    def appropriate?
+      return proposal if to.scheme == 'rbd' &&
+        rbd_snapshot_spec?(from) && 
+        rbd_cache_name.nil? &&
+        !rbd_sequence_complete?
+
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd clone --no-progress --id #{rbd_id} '#{rbd_image_spec(from)}' '#{rbd_image_spec(to)}';"
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['from_uri'] = to
+      if options['rbd_cache_target']
+        @proposed_options['to_uri'] = options['rbd_cache_target']
+        @proposed_options['rbd_cache_name'] = rbd_image_spec(to)
+        @proposed_options['rbd_flatten'] ||= @proposed_options['rbd_cache_name']
+        @proposed_options.delete('rbd_cache_target')
+      end
+      self
+    end
+
+  end
+end

data/lib/uricp/strategy/rbd_flattener.rb

@@ -0,0 +1,23 @@
+module Uricp::Strategy
+  class RbdFlattener
+    include Uricp::Strategy::Common
+
+    def appropriate?
+      return proposal if options['rbd_flatten'] &&
+        rbd_sequence_complete?
+
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd flatten --id #{rbd_id} --no-progress '#{options['rbd_flatten']}' && "
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options.delete('rbd_flatten')
+      self
+    end
+  end
+end

data/lib/uricp/strategy/{rbd_remote_put.rb → rbd_put.rb}

@@ -1,5 +1,5 @@
 module Uricp::Strategy
-  class RbdRemotePut
+  class RbdPut
     include Uricp::Strategy::Common
 
     def appropriate?
@@ -7,16 +7,18 @@ module Uricp::Strategy
         debug "#{self.class.name}: not ready to upload"
         return false
       end
-      case from.scheme
-      when 'pipe', 'file'
-        return proposal if to.scheme == 'rbd'
+      unless rbd_cache_name
+        case from.scheme
+        when 'pipe', 'file'
+          return proposal if to.scheme == 'rbd'
+        end
       end
       debug "#{self.class.name}: not appropriate"
       false
     end
 
     def command
-      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_target(to)}';"
+      "rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_image_spec(to)}';"
     end
 
     def proposal

data/lib/uricp/strategy/rbd_snap.rb

@@ -0,0 +1,56 @@
+require 'json'
+module Uricp::Strategy
+  class RbdSnap
+    include Uricp::Strategy::Common
+    include Uricp::Strategy::CacheCommon
+    include Methadone::SH
+
+    def appropriate?
+      without_active_cache do
+        if from.scheme == 'rbd' &&
+          options['rbd_snapshot'].nil? &&
+          !rbd_snapshot_spec?(from) &&
+          (rbd_cache_name.nil? ||
+            !from.path.include?(rbd_cache_name))
+          if snap_in_progress?
+            debug "#{self.class.name}: detected snapshot in progress"
+          else
+            return proposal unless sequence_complete?
+          end
+        end
+      end
+      debug "#{self.class.name}: not appropriate"
+      false
+    end
+
+    def command
+      "rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
+      "rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}' && " \
+    end
+
+    def proposal
+      @proposed_options = options.dup
+      @proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from)
+      @proposed_options['from_uri'] = rbd_uri(@proposed_options['rbd_snapshot'])
+      if options['rbd_cache_target']
+        @proposed_options['to_uri'] = options['rbd_cache_target']
+        @proposed_options['rbd_cache_target'] = to
+      end
+      self
+    end
+
+    def rbd_upload_snapshot(uri)
+      "#{rbd_image_spec(uri)}@#{rbd_snapshot_name}"
+    end
+
+    def snap_in_progress?
+      return false if dry_run?
+
+      result = false
+      sh "rbd snap ls --id #{rbd_id} --format json #{rbd_image_spec(from)}" do |stdout|
+        result = JSON.parse(stdout).any? { |x| x['name'] == rbd_snapshot_name }
+      end
+      result
+    end
+  end
+end

data/lib/uricp/strategy/rbd_sweeper.rb

@@ -3,13 +3,14 @@ module Uricp::Strategy
     include Uricp::Strategy::Common
 
     def appropriate?
-      return proposal if options['rbd_snapshot'] && sequence_complete?
+      return proposal if options['rbd_snapshot'] && rbd_sequence_complete?
 
       debug "#{self.class.name}: not appropriate"
       false
     end
 
     def command
+      "rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}' && " \
       "rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
     end
 

data/lib/uricp/uri_strategy.rb

@@ -25,6 +25,10 @@ module Uricp
 
     # This is an ordered list from the most specific to the most general
     STRATEGIES = [
+      Strategy::RbdCachedGet,
+      Strategy::RbdCacheClone,
+      Strategy::RbdCacheCheck,
+      Strategy::RbdSnap,
       Strategy::CachedGet,
       Strategy::PipedRemoteGet,
       Strategy::PipedRbdGet,
@@ -35,13 +39,17 @@ module Uricp
       Strategy::LocalConvert,
       Strategy::LocalLink,
       Strategy::PipedCompress,
-      Strategy::RbdRemotePut,
+      Strategy::RbdCacheUpload,
+      Strategy::RbdCachedPut,
+      Strategy::RbdPut,
+      Strategy::RbdCacheBaseSnap,
       Strategy::SegmentedRemotePut,
       Strategy::RemotePut,
       Strategy::PipedLocalCompress,
       Strategy::PipedLocalGet,
       Strategy::PipedLocalPut,
       Strategy::Cleaner,
+      Strategy::RbdFlattener,
       Strategy::RbdSweeper,
       Strategy::Sweeper
     ].freeze

data/lib/uricp/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Uricp
-  VERSION = '0.0.18'
+  VERSION = '0.0.20'
   DEFAULT_SEGMENT_SIZE = '5 GiB'
 end

data/lib/uricp.rb

@@ -1,32 +1,40 @@
-require 'uricp/version'
+module Uricp
+  UnsupportedURLtype = Class.new(ArgumentError)
+  MissingCache = Class.new(ArgumentError)
+  UnsupportedConversion = Class.new(ArgumentError)
+  ConversionRequired = Class.new(ArgumentError)
+end
+
 require 'uricp/curl_primitives'
+require 'uricp/orbit_auth'
 require 'uricp/strategy/common'
 require 'uricp/strategy/cache_common'
+require 'uricp/strategy/cached_get'
+require 'uricp/strategy/cleaner'
 require 'uricp/strategy/local_convert'
 require 'uricp/strategy/local_link'
 require 'uricp/strategy/piped_cache'
 require 'uricp/strategy/piped_cache_convert'
-require 'uricp/strategy/piped_local_get'
-require 'uricp/strategy/piped_rbd_get'
-require 'uricp/strategy/piped_local_put'
-require 'uricp/strategy/piped_remote_get'
 require 'uricp/strategy/piped_compress'
 require 'uricp/strategy/piped_decompress'
-require 'uricp/strategy/piped_local_decompress'
 require 'uricp/strategy/piped_local_compress'
-require 'uricp/strategy/rbd_remote_put'
-require 'uricp/strategy/segmented_remote_put'
-require 'uricp/strategy/remote_put'
-require 'uricp/strategy/cached_get'
+require 'uricp/strategy/piped_local_decompress'
+require 'uricp/strategy/piped_local_get'
+require 'uricp/strategy/piped_local_put'
+require 'uricp/strategy/piped_rbd_get'
+require 'uricp/strategy/piped_remote_get'
+require 'uricp/strategy/rbd_cache_base_snap'
+require 'uricp/strategy/rbd_cache_check'
+require 'uricp/strategy/rbd_cache_clone'
+require 'uricp/strategy/rbd_cached_get'
+require 'uricp/strategy/rbd_cached_put'
+require 'uricp/strategy/rbd_cache_upload'
+require 'uricp/strategy/rbd_flattener'
+require 'uricp/strategy/rbd_put'
+require 'uricp/strategy/rbd_snap'
 require 'uricp/strategy/rbd_sweeper'
+require 'uricp/strategy/remote_put'
+require 'uricp/strategy/segmented_remote_put'
 require 'uricp/strategy/sweeper'
-require 'uricp/strategy/cleaner'
 require 'uricp/uri_strategy'
-require 'uricp/orbit_auth'
-
-module Uricp
-  UnsupportedURLtype = Class.new(ArgumentError)
-  MissingCache = Class.new(ArgumentError)
-  UnsupportedConversion = Class.new(ArgumentError)
-  ConversionRequired = Class.new(ArgumentError)
-end
+require 'uricp/version'

data/uricp.gemspec

@@ -23,7 +23,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('aruba', '~> 0.6.0')
   spec.add_development_dependency('cucumber', '~> 1.3')
   spec.add_development_dependency('rake', '~> 12.3')
-  spec.add_dependency('methadone', '~> 2.0.0')
+  spec.add_dependency('childprocess', '~> 1.0')
+  spec.add_dependency('methadone', '~> 2.0.2')
   spec.add_dependency('open4', '~> 1.3.0')
   spec.add_dependency('filesize', '= 0.0.2')
   spec.add_dependency('sendfile', '~> 1.2.0')

data/xenial/Dockerfile

@@ -1,6 +1,8 @@
 FROM ubuntu:xenial
 MAINTAINER support@brightbox.co.uk
 
+RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
+
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
 
 #RUN apt-add-repository ppa:brightbox/ruby-ng
@@ -16,5 +18,4 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
   python-swiftclient \
   ruby-bundler
 
-RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: uricp
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.0.20
 platform: ruby
 authors:
 - Neil Wilson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-12 00:00:00.000000000 Z
+date: 2021-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,20 +80,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: childprocess
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: methadone
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: open4
   requirement: !ruby/object:Gem::Requirement
@@ -167,6 +181,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- almalinux8/Dockerfile
 - bin/segment_upload
 - bin/uricp
 - bionic/Dockerfile
@@ -174,12 +189,15 @@ files:
 - cucumber.yml
 - features/auth_download.feature
 - features/cacheable_from_uri.feature
+- features/check_uri_path.feature
 - features/documented_options.feature
 - features/qcow2_auth_download.feature
+- features/rbd_access.feature
 - features/remote_upload.feature
 - features/segmented_upload.feature
 - features/segmented_upload_options.feature
 - features/step_definitions/orbit_steps.rb
+- features/step_definitions/rbd_steps.rb
 - features/step_definitions/uricp_steps.rb
 - features/support/env.rb
 - features/userid_download.feature
@@ -205,7 +223,15 @@ files:
 - lib/uricp/strategy/piped_local_put.rb
 - lib/uricp/strategy/piped_rbd_get.rb
 - lib/uricp/strategy/piped_remote_get.rb
-- lib/uricp/strategy/rbd_remote_put.rb
+- lib/uricp/strategy/rbd_cache_base_snap.rb
+- lib/uricp/strategy/rbd_cache_check.rb
+- lib/uricp/strategy/rbd_cache_clone.rb
+- lib/uricp/strategy/rbd_cache_upload.rb
+- lib/uricp/strategy/rbd_cached_get.rb
+- lib/uricp/strategy/rbd_cached_put.rb
+- lib/uricp/strategy/rbd_flattener.rb
+- lib/uricp/strategy/rbd_put.rb
+- lib/uricp/strategy/rbd_snap.rb
 - lib/uricp/strategy/rbd_sweeper.rb
 - lib/uricp/strategy/remote_put.rb
 - lib/uricp/strategy/segmented_remote_put.rb
@@ -240,12 +266,15 @@ summary: Copy one URL to another with optional cacheing
 test_files:
 - features/auth_download.feature
 - features/cacheable_from_uri.feature
+- features/check_uri_path.feature
 - features/documented_options.feature
 - features/qcow2_auth_download.feature
+- features/rbd_access.feature
 - features/remote_upload.feature
 - features/segmented_upload.feature
 - features/segmented_upload_options.feature
 - features/step_definitions/orbit_steps.rb
+- features/step_definitions/rbd_steps.rb
 - features/step_definitions/uricp_steps.rb
 - features/support/env.rb
 - features/userid_download.feature