uricp 0.0.15 → 0.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/Gemfile.lock +16 -21
- data/Jenkinsfile +90 -33
- data/Rakefile +24 -36
- data/almalinux8/Dockerfile +26 -0
- data/bin/uricp +25 -11
- data/bionic/Dockerfile +4 -3
- data/centos7/Dockerfile +17 -12
- data/cucumber.yml +1 -0
- data/features/check_uri_path.feature +14 -0
- data/features/rbd_access.feature +226 -0
- data/features/step_definitions/orbit_steps.rb +11 -4
- data/features/step_definitions/rbd_steps.rb +8 -0
- data/features/step_definitions/uricp_steps.rb +8 -4
- data/focal/Dockerfile +28 -0
- data/lib/segment_upload.rb +20 -22
- data/lib/uricp/curl_primitives.rb +31 -33
- data/lib/uricp/orbit_auth.rb +23 -27
- data/lib/uricp/segmenter.rb +12 -16
- data/lib/uricp/strategy/cache_common.rb +30 -12
- data/lib/uricp/strategy/cached_get.rb +19 -22
- data/lib/uricp/strategy/cleaner.rb +2 -8
- data/lib/uricp/strategy/common.rb +76 -18
- data/lib/uricp/strategy/local_convert.rb +10 -17
- data/lib/uricp/strategy/local_link.rb +10 -8
- data/lib/uricp/strategy/piped_cache.rb +11 -16
- data/lib/uricp/strategy/piped_cache_convert.rb +14 -18
- data/lib/uricp/strategy/piped_compress.rb +3 -8
- data/lib/uricp/strategy/piped_decompress.rb +7 -11
- data/lib/uricp/strategy/piped_local_compress.rb +0 -4
- data/lib/uricp/strategy/piped_local_decompress.rb +10 -16
- data/lib/uricp/strategy/piped_local_get.rb +0 -5
- data/lib/uricp/strategy/piped_local_put.rb +3 -9
- data/lib/uricp/strategy/piped_rbd_get.rb +30 -0
- data/lib/uricp/strategy/piped_remote_get.rb +20 -20
- data/lib/uricp/strategy/rbd_cache_base_snap.rb +27 -0
- data/lib/uricp/strategy/rbd_cache_check.rb +42 -0
- data/lib/uricp/strategy/rbd_cache_clone.rb +40 -0
- data/lib/uricp/strategy/rbd_cache_upload.rb +40 -0
- data/lib/uricp/strategy/rbd_cached_get.rb +36 -0
- data/lib/uricp/strategy/rbd_cached_put.rb +33 -0
- data/lib/uricp/strategy/rbd_flattener.rb +23 -0
- data/lib/uricp/strategy/rbd_put.rb +38 -0
- data/lib/uricp/strategy/rbd_snap.rb +56 -0
- data/lib/uricp/strategy/rbd_sweeper.rb +23 -0
- data/lib/uricp/strategy/remote_put.rb +11 -17
- data/lib/uricp/strategy/segmented_remote_put.rb +16 -28
- data/lib/uricp/strategy/sweeper.rb +2 -8
- data/lib/uricp/uri_strategy.rb +22 -13
- data/lib/uricp/version.rb +4 -2
- data/lib/uricp.rb +27 -19
- data/uricp.gemspec +3 -3
- data/xenial/Dockerfile +4 -3
- metadata +41 -25
- data/spec/something_spec.rb +0 -5
- data/trusty/Dockerfile +0 -20
@@ -1,13 +1,10 @@
|
|
1
1
|
module Uricp::Strategy
|
2
|
-
|
3
2
|
class Sweeper
|
4
|
-
|
5
3
|
include Uricp::Strategy::Common
|
6
4
|
|
7
5
|
def appropriate?
|
8
|
-
if options['sweep'] && sequence_complete?
|
9
|
-
|
10
|
-
end
|
6
|
+
return proposal if options['sweep'] && sequence_complete?
|
7
|
+
|
11
8
|
debug "#{self.class.name}: not appropriate"
|
12
9
|
false
|
13
10
|
end
|
@@ -21,8 +18,5 @@ module Uricp::Strategy
|
|
21
18
|
@proposed_options.delete('sweep')
|
22
19
|
self
|
23
20
|
end
|
24
|
-
|
25
21
|
end
|
26
|
-
|
27
22
|
end
|
28
|
-
|
data/lib/uricp/uri_strategy.rb
CHANGED
@@ -1,20 +1,21 @@
|
|
1
1
|
require 'uri'
|
2
2
|
|
3
3
|
module Uricp
|
4
|
-
|
5
4
|
class UriStrategy
|
6
|
-
|
7
5
|
include Methadone::CLILogging
|
8
6
|
|
9
7
|
def self.choose_strategy(options)
|
10
|
-
current = options.reject {|k,
|
8
|
+
current = options.reject { |k, _v| k.is_a? Symbol }
|
11
9
|
strategy_list = []
|
12
|
-
while STRATEGIES.detect {|klass| @strategy = klass.new(current).appropriate? }
|
13
|
-
|
10
|
+
while STRATEGIES.detect { |klass| @strategy = klass.new(current).appropriate? }
|
11
|
+
debug "#{name}: Selected strategy #{@strategy.class.name}"
|
14
12
|
strategy_list << @strategy
|
15
|
-
|
13
|
+
current = @strategy.proposed_options
|
14
|
+
end
|
15
|
+
if incomplete_strategy?(strategy_list)
|
16
|
+
raise Uricp::UnsupportedURLtype, "Unsupported transfer from #{options['from_uri']} to #{options['to_uri']}"
|
16
17
|
end
|
17
|
-
|
18
|
+
|
18
19
|
strategy_list
|
19
20
|
end
|
20
21
|
|
@@ -22,12 +23,15 @@ module Uricp
|
|
22
23
|
list.empty?
|
23
24
|
end
|
24
25
|
|
25
|
-
|
26
|
-
|
27
|
-
#This is an ordered list from the most specific to the most general
|
26
|
+
# This is an ordered list from the most specific to the most general
|
28
27
|
STRATEGIES = [
|
28
|
+
Strategy::RbdCachedGet,
|
29
|
+
Strategy::RbdCacheClone,
|
30
|
+
Strategy::RbdCacheCheck,
|
31
|
+
Strategy::RbdSnap,
|
29
32
|
Strategy::CachedGet,
|
30
33
|
Strategy::PipedRemoteGet,
|
34
|
+
Strategy::PipedRbdGet,
|
31
35
|
Strategy::PipedCacheConvert,
|
32
36
|
Strategy::PipedCache,
|
33
37
|
Strategy::PipedLocalDecompress,
|
@@ -35,14 +39,19 @@ module Uricp
|
|
35
39
|
Strategy::LocalConvert,
|
36
40
|
Strategy::LocalLink,
|
37
41
|
Strategy::PipedCompress,
|
42
|
+
Strategy::RbdCacheUpload,
|
43
|
+
Strategy::RbdCachedPut,
|
44
|
+
Strategy::RbdPut,
|
45
|
+
Strategy::RbdCacheBaseSnap,
|
38
46
|
Strategy::SegmentedRemotePut,
|
39
47
|
Strategy::RemotePut,
|
40
48
|
Strategy::PipedLocalCompress,
|
41
49
|
Strategy::PipedLocalGet,
|
42
50
|
Strategy::PipedLocalPut,
|
43
51
|
Strategy::Cleaner,
|
44
|
-
Strategy::
|
45
|
-
|
46
|
-
|
52
|
+
Strategy::RbdFlattener,
|
53
|
+
Strategy::RbdSweeper,
|
54
|
+
Strategy::Sweeper
|
55
|
+
].freeze
|
47
56
|
end
|
48
57
|
end
|
data/lib/uricp/version.rb
CHANGED
data/lib/uricp.rb
CHANGED
@@ -1,32 +1,40 @@
|
|
1
|
-
|
1
|
+
module Uricp
|
2
|
+
UnsupportedURLtype = Class.new(ArgumentError)
|
3
|
+
MissingCache = Class.new(ArgumentError)
|
4
|
+
UnsupportedConversion = Class.new(ArgumentError)
|
5
|
+
ConversionRequired = Class.new(ArgumentError)
|
6
|
+
end
|
7
|
+
|
2
8
|
require 'uricp/curl_primitives'
|
9
|
+
require 'uricp/orbit_auth'
|
3
10
|
require 'uricp/strategy/common'
|
4
11
|
require 'uricp/strategy/cache_common'
|
12
|
+
require 'uricp/strategy/cached_get'
|
13
|
+
require 'uricp/strategy/cleaner'
|
5
14
|
require 'uricp/strategy/local_convert'
|
6
15
|
require 'uricp/strategy/local_link'
|
7
16
|
require 'uricp/strategy/piped_cache'
|
8
17
|
require 'uricp/strategy/piped_cache_convert'
|
9
|
-
require 'uricp/strategy/piped_local_get'
|
10
|
-
require 'uricp/strategy/piped_local_put'
|
11
|
-
require 'uricp/strategy/piped_remote_get'
|
12
18
|
require 'uricp/strategy/piped_compress'
|
13
19
|
require 'uricp/strategy/piped_decompress'
|
14
|
-
require 'uricp/strategy/piped_local_decompress'
|
15
20
|
require 'uricp/strategy/piped_local_compress'
|
16
|
-
require 'uricp/strategy/
|
21
|
+
require 'uricp/strategy/piped_local_decompress'
|
22
|
+
require 'uricp/strategy/piped_local_get'
|
23
|
+
require 'uricp/strategy/piped_local_put'
|
24
|
+
require 'uricp/strategy/piped_rbd_get'
|
25
|
+
require 'uricp/strategy/piped_remote_get'
|
26
|
+
require 'uricp/strategy/rbd_cache_base_snap'
|
27
|
+
require 'uricp/strategy/rbd_cache_check'
|
28
|
+
require 'uricp/strategy/rbd_cache_clone'
|
29
|
+
require 'uricp/strategy/rbd_cached_get'
|
30
|
+
require 'uricp/strategy/rbd_cached_put'
|
31
|
+
require 'uricp/strategy/rbd_cache_upload'
|
32
|
+
require 'uricp/strategy/rbd_flattener'
|
33
|
+
require 'uricp/strategy/rbd_put'
|
34
|
+
require 'uricp/strategy/rbd_snap'
|
35
|
+
require 'uricp/strategy/rbd_sweeper'
|
17
36
|
require 'uricp/strategy/remote_put'
|
18
|
-
require 'uricp/strategy/
|
37
|
+
require 'uricp/strategy/segmented_remote_put'
|
19
38
|
require 'uricp/strategy/sweeper'
|
20
|
-
require 'uricp/strategy/cleaner'
|
21
39
|
require 'uricp/uri_strategy'
|
22
|
-
require 'uricp/
|
23
|
-
|
24
|
-
module Uricp
|
25
|
-
|
26
|
-
UnsupportedURLtype = Class.new(ArgumentError)
|
27
|
-
MissingCache = Class.new(ArgumentError)
|
28
|
-
UnsupportedConversion = Class.new(ArgumentError)
|
29
|
-
ConversionRequired = Class.new(ArgumentError)
|
30
|
-
|
31
|
-
end
|
32
|
-
|
40
|
+
require 'uricp/version'
|
data/uricp.gemspec
CHANGED
@@ -22,11 +22,11 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_development_dependency('rdoc', '~> 4.2.0')
|
23
23
|
spec.add_development_dependency('aruba', '~> 0.6.0')
|
24
24
|
spec.add_development_dependency('cucumber', '~> 1.3')
|
25
|
-
spec.add_development_dependency('rake', '~>
|
26
|
-
spec.add_dependency('
|
25
|
+
spec.add_development_dependency('rake', '~> 12.3')
|
26
|
+
spec.add_dependency('childprocess', '~> 1.0')
|
27
|
+
spec.add_dependency('methadone', '~> 2.0.2')
|
27
28
|
spec.add_dependency('open4', '~> 1.3.0')
|
28
29
|
spec.add_dependency('filesize', '= 0.0.2')
|
29
30
|
spec.add_dependency('sendfile', '~> 1.2.0')
|
30
|
-
spec.add_development_dependency('rspec', '~> 2.99')
|
31
31
|
spec.add_development_dependency('inifile', '~> 1.1')
|
32
32
|
end
|
data/xenial/Dockerfile
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
FROM ubuntu:xenial
|
2
2
|
MAINTAINER support@brightbox.co.uk
|
3
3
|
|
4
|
+
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
5
|
+
|
4
6
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
|
5
7
|
|
6
8
|
#RUN apt-add-repository ppa:brightbox/ruby-ng
|
@@ -13,8 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
|
13
15
|
curl \
|
14
16
|
ruby \
|
15
17
|
ruby-dev \
|
16
|
-
python-swiftclient
|
18
|
+
python-swiftclient \
|
19
|
+
ruby-bundler
|
17
20
|
|
18
|
-
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
19
|
-
RUN gem install bundler
|
20
21
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: uricp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Neil Wilson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -72,28 +72,42 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: '12.3'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: '12.3'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: childprocess
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: methadone
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
86
100
|
requirements:
|
87
101
|
- - "~>"
|
88
102
|
- !ruby/object:Gem::Version
|
89
|
-
version: 2.0.
|
103
|
+
version: 2.0.2
|
90
104
|
type: :runtime
|
91
105
|
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
108
|
- - "~>"
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version: 2.0.
|
110
|
+
version: 2.0.2
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: open4
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,20 +150,6 @@ dependencies:
|
|
136
150
|
- - "~>"
|
137
151
|
- !ruby/object:Gem::Version
|
138
152
|
version: 1.2.0
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: rspec
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - "~>"
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: '2.99'
|
146
|
-
type: :development
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - "~>"
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: '2.99'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: inifile
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -174,12 +174,14 @@ extensions: []
|
|
174
174
|
extra_rdoc_files: []
|
175
175
|
files:
|
176
176
|
- ".gitignore"
|
177
|
+
- ".rubocop.yml"
|
177
178
|
- Gemfile
|
178
179
|
- Gemfile.lock
|
179
180
|
- Jenkinsfile
|
180
181
|
- LICENSE.txt
|
181
182
|
- README.md
|
182
183
|
- Rakefile
|
184
|
+
- almalinux8/Dockerfile
|
183
185
|
- bin/segment_upload
|
184
186
|
- bin/uricp
|
185
187
|
- bionic/Dockerfile
|
@@ -187,15 +189,19 @@ files:
|
|
187
189
|
- cucumber.yml
|
188
190
|
- features/auth_download.feature
|
189
191
|
- features/cacheable_from_uri.feature
|
192
|
+
- features/check_uri_path.feature
|
190
193
|
- features/documented_options.feature
|
191
194
|
- features/qcow2_auth_download.feature
|
195
|
+
- features/rbd_access.feature
|
192
196
|
- features/remote_upload.feature
|
193
197
|
- features/segmented_upload.feature
|
194
198
|
- features/segmented_upload_options.feature
|
195
199
|
- features/step_definitions/orbit_steps.rb
|
200
|
+
- features/step_definitions/rbd_steps.rb
|
196
201
|
- features/step_definitions/uricp_steps.rb
|
197
202
|
- features/support/env.rb
|
198
203
|
- features/userid_download.feature
|
204
|
+
- focal/Dockerfile
|
199
205
|
- lib/segment_upload.rb
|
200
206
|
- lib/uricp.rb
|
201
207
|
- lib/uricp/curl_primitives.rb
|
@@ -215,14 +221,23 @@ files:
|
|
215
221
|
- lib/uricp/strategy/piped_local_decompress.rb
|
216
222
|
- lib/uricp/strategy/piped_local_get.rb
|
217
223
|
- lib/uricp/strategy/piped_local_put.rb
|
224
|
+
- lib/uricp/strategy/piped_rbd_get.rb
|
218
225
|
- lib/uricp/strategy/piped_remote_get.rb
|
226
|
+
- lib/uricp/strategy/rbd_cache_base_snap.rb
|
227
|
+
- lib/uricp/strategy/rbd_cache_check.rb
|
228
|
+
- lib/uricp/strategy/rbd_cache_clone.rb
|
229
|
+
- lib/uricp/strategy/rbd_cache_upload.rb
|
230
|
+
- lib/uricp/strategy/rbd_cached_get.rb
|
231
|
+
- lib/uricp/strategy/rbd_cached_put.rb
|
232
|
+
- lib/uricp/strategy/rbd_flattener.rb
|
233
|
+
- lib/uricp/strategy/rbd_put.rb
|
234
|
+
- lib/uricp/strategy/rbd_snap.rb
|
235
|
+
- lib/uricp/strategy/rbd_sweeper.rb
|
219
236
|
- lib/uricp/strategy/remote_put.rb
|
220
237
|
- lib/uricp/strategy/segmented_remote_put.rb
|
221
238
|
- lib/uricp/strategy/sweeper.rb
|
222
239
|
- lib/uricp/uri_strategy.rb
|
223
240
|
- lib/uricp/version.rb
|
224
|
-
- spec/something_spec.rb
|
225
|
-
- trusty/Dockerfile
|
226
241
|
- uricp.gemspec
|
227
242
|
- xenial/Dockerfile
|
228
243
|
homepage: ''
|
@@ -244,21 +259,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
244
259
|
- !ruby/object:Gem::Version
|
245
260
|
version: '0'
|
246
261
|
requirements: []
|
247
|
-
|
248
|
-
rubygems_version: 2.7.6
|
262
|
+
rubygems_version: 3.1.2
|
249
263
|
signing_key:
|
250
264
|
specification_version: 4
|
251
265
|
summary: Copy one URL to another with optional cacheing
|
252
266
|
test_files:
|
253
267
|
- features/auth_download.feature
|
254
268
|
- features/cacheable_from_uri.feature
|
269
|
+
- features/check_uri_path.feature
|
255
270
|
- features/documented_options.feature
|
256
271
|
- features/qcow2_auth_download.feature
|
272
|
+
- features/rbd_access.feature
|
257
273
|
- features/remote_upload.feature
|
258
274
|
- features/segmented_upload.feature
|
259
275
|
- features/segmented_upload_options.feature
|
260
276
|
- features/step_definitions/orbit_steps.rb
|
277
|
+
- features/step_definitions/rbd_steps.rb
|
261
278
|
- features/step_definitions/uricp_steps.rb
|
262
279
|
- features/support/env.rb
|
263
280
|
- features/userid_download.feature
|
264
|
-
- spec/something_spec.rb
|
data/spec/something_spec.rb
DELETED
data/trusty/Dockerfile
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
FROM ubuntu:trusty
|
2
|
-
MAINTAINER support@brightbox.co.uk
|
3
|
-
|
4
|
-
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
|
5
|
-
|
6
|
-
RUN apt-add-repository ppa:brightbox/ruby-ng
|
7
|
-
|
8
|
-
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
9
|
-
build-essential \
|
10
|
-
git \
|
11
|
-
qemu-utils \
|
12
|
-
liblz4-tool \
|
13
|
-
curl \
|
14
|
-
ruby1.9.3 \
|
15
|
-
ruby-dev \
|
16
|
-
python-swiftclient
|
17
|
-
|
18
|
-
RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
|
19
|
-
RUN gem install bundler
|
20
|
-
|