uricp 0.0.21 → 0.0.25
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +5 -4
- data/bin/uricp +3 -2
- data/features/rbd_access.feature +41 -19
- data/lib/segment_upload.rb +2 -28
- data/lib/uricp/curl_primitives.rb +2 -0
- data/lib/uricp/orbit_auth.rb +2 -0
- data/lib/uricp/segmenter.rb +3 -1
- data/lib/uricp/strategy/cache_common.rb +6 -4
- data/lib/uricp/strategy/cached_get.rb +2 -0
- data/lib/uricp/strategy/cleaner.rb +2 -0
- data/lib/uricp/strategy/common.rb +29 -5
- data/lib/uricp/strategy/local_convert.rb +2 -0
- data/lib/uricp/strategy/local_link.rb +2 -0
- data/lib/uricp/strategy/piped_cache.rb +4 -1
- data/lib/uricp/strategy/piped_cache_convert.rb +2 -0
- data/lib/uricp/strategy/piped_compress.rb +2 -0
- data/lib/uricp/strategy/piped_decompress.rb +2 -0
- data/lib/uricp/strategy/piped_local_compress.rb +2 -0
- data/lib/uricp/strategy/piped_local_decompress.rb +2 -0
- data/lib/uricp/strategy/piped_local_get.rb +2 -0
- data/lib/uricp/strategy/piped_local_put.rb +2 -0
- data/lib/uricp/strategy/piped_rbd_get.rb +4 -3
- data/lib/uricp/strategy/piped_remote_get.rb +2 -0
- data/lib/uricp/strategy/rbd_cache_base_snap.rb +4 -3
- data/lib/uricp/strategy/rbd_cache_check.rb +2 -1
- data/lib/uricp/strategy/rbd_cache_clone.rb +3 -3
- data/lib/uricp/strategy/rbd_cache_upload.rb +4 -4
- data/lib/uricp/strategy/rbd_cached_get.rb +2 -1
- data/lib/uricp/strategy/rbd_cached_put.rb +5 -5
- data/lib/uricp/strategy/rbd_put.rb +9 -5
- data/lib/uricp/strategy/rbd_snap.rb +11 -9
- data/lib/uricp/strategy/rbd_sweeper.rb +3 -1
- data/lib/uricp/strategy/remote_put.rb +2 -0
- data/lib/uricp/strategy/segmented_remote_put.rb +2 -0
- data/lib/uricp/strategy/sweeper.rb +2 -0
- data/lib/uricp/uri_strategy.rb +2 -1
- data/lib/uricp/version.rb +1 -1
- data/lib/uricp.rb +0 -1
- data/uricp.gemspec +14 -14
- metadata +28 -43
- data/lib/uricp/strategy/rbd_flattener.rb +0 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: acf0f2c652c367c337a5df56eda0d3996799857614f6e91a3844b637b8d36fd6
|
4
|
+
data.tar.gz: 3778050f26b276236ca88ca1d6b6df2547adc1daf9eeb8b33a5d7e92cf424cbf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ddfdf97678c6c18e1582eb4ecb17d4328f7f713c31d59bc8512209a99cd47527acbcc7fd61953f1a4056ab623bed71c100c4c0658f96569cf3be2e7e0e94a66c
|
7
|
+
data.tar.gz: d2ce88f8d34a54c1ce394b87d94364afc38ee2b310c8dae05a3cd3bd4c1a130d8f758caa25f49053d7692a44880aa9a0094d2c31caa696a9d0c73f49d5366aa2
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,12 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
uricp (0.0.
|
4
|
+
uricp (0.0.25)
|
5
5
|
childprocess (~> 1.0)
|
6
6
|
filesize (= 0.0.2)
|
7
7
|
methadone (~> 2.0.2)
|
8
8
|
open4 (~> 1.3.0)
|
9
|
-
sendfile (~> 1.2.0)
|
10
9
|
|
11
10
|
GEM
|
12
11
|
remote: https://rubygems.org/
|
@@ -41,8 +40,7 @@ GEM
|
|
41
40
|
rspec-expectations (3.10.1)
|
42
41
|
diff-lcs (>= 1.2.0, < 2.0)
|
43
42
|
rspec-support (~> 3.10.0)
|
44
|
-
rspec-support (3.10.
|
45
|
-
sendfile (1.2.2)
|
43
|
+
rspec-support (3.10.3)
|
46
44
|
|
47
45
|
PLATFORMS
|
48
46
|
ruby
|
@@ -56,5 +54,8 @@ DEPENDENCIES
|
|
56
54
|
rdoc (~> 4.2.0)
|
57
55
|
uricp!
|
58
56
|
|
57
|
+
RUBY VERSION
|
58
|
+
ruby 2.7.0p0
|
59
|
+
|
59
60
|
BUNDLED WITH
|
60
61
|
1.17.3
|
data/bin/uricp
CHANGED
@@ -4,6 +4,7 @@ require 'optparse'
|
|
4
4
|
require 'uri'
|
5
5
|
require 'methadone'
|
6
6
|
require 'filesize'
|
7
|
+
require 'securerandom'
|
7
8
|
require 'uricp'
|
8
9
|
|
9
10
|
class App
|
@@ -121,8 +122,8 @@ class App
|
|
121
122
|
"Compress output")
|
122
123
|
on("--[no-]dry-run",
|
123
124
|
"Show what would be run,", "but don't run the command")
|
124
|
-
on("--
|
125
|
-
"On dry runs assume the cache is populated")
|
125
|
+
on("--dry-cache DRY_CACHE",[:rbd, :partial_rbd],
|
126
|
+
"On dry runs assume the cache is populated", "[rbd, partial_rbd]")
|
126
127
|
on("--[no-]force",
|
127
128
|
"Always write sparsely")
|
128
129
|
#Arguments
|
data/features/rbd_access.feature
CHANGED
@@ -56,9 +56,21 @@ Feature: Manipulate file images in ceph
|
|
56
56
|
rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
|
57
57
|
"""
|
58
58
|
|
59
|
-
|
59
|
+
Scenario: HTTP URI to rbd via partial rbd caches
|
60
60
|
Given a correctly initialised cache at "/tmp/uricp"
|
61
|
-
When I successfully run `uricp --dry-run --dry-cache --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
|
61
|
+
When I successfully run `uricp --dry-run --dry-cache=partial_rbd --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
|
62
|
+
And the output should not contain:
|
63
|
+
"""
|
64
|
+
snap create
|
65
|
+
"""
|
66
|
+
And the output should contain:
|
67
|
+
"""
|
68
|
+
rbd import --no-progress --id libvirt - 'servers/srv-testy'
|
69
|
+
"""
|
70
|
+
|
71
|
+
Scenario: HTTP URI to rbd via populated cache
|
72
|
+
Given a correctly initialised cache at "/tmp/uricp"
|
73
|
+
When I successfully run `uricp --dry-run --dry-cache=rbd --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
|
62
74
|
And the output should not contain:
|
63
75
|
"""
|
64
76
|
snap create
|
@@ -79,13 +91,13 @@ Feature: Manipulate file images in ceph
|
|
79
91
|
"""
|
80
92
|
rbd snap protect --id libvirt 'servers/img-testy@uricp_snap'
|
81
93
|
"""
|
82
|
-
And the output should contain:
|
94
|
+
And the output should not contain:
|
83
95
|
"""
|
84
96
|
rbd snap unprotect --id libvirt 'servers/img-testy@uricp_snap'
|
85
97
|
"""
|
86
|
-
And the output should contain:
|
98
|
+
And the output should not contain:
|
87
99
|
"""
|
88
|
-
rbd snap rm --id libvirt 'servers/img-testy@uricp_snap'
|
100
|
+
rbd snap rm --id libvirt 'servers/img-testy@uricp_snap'
|
89
101
|
"""
|
90
102
|
And the output should contain:
|
91
103
|
"""
|
@@ -103,13 +115,13 @@ Feature: Manipulate file images in ceph
|
|
103
115
|
"""
|
104
116
|
rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
|
105
117
|
"""
|
106
|
-
And the output should contain:
|
118
|
+
And the output should not contain:
|
107
119
|
"""
|
108
120
|
rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
|
109
121
|
"""
|
110
|
-
And the output should contain:
|
122
|
+
And the output should not contain:
|
111
123
|
"""
|
112
|
-
rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
|
124
|
+
rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
|
113
125
|
"""
|
114
126
|
And the output should not contain:
|
115
127
|
"""
|
@@ -131,14 +143,19 @@ Feature: Manipulate file images in ceph
|
|
131
143
|
"""
|
132
144
|
rbd snap protect --id libvirt 'servers/img-testy@base'
|
133
145
|
"""
|
146
|
+
|
147
|
+
Scenario: rbd to rbd - partial cache
|
148
|
+
Given a correctly initialised cache at "/tmp/uricp"
|
149
|
+
When I run `uricp --dry-run --dry-cache=partial_rbd --cache=/tmp/uricp rbd:///servers/srv-testy rbd:///servers/img-testy`
|
150
|
+
Then the exit status should be 70
|
134
151
|
And the output should contain:
|
135
152
|
"""
|
136
|
-
|
153
|
+
Unsupported transfer
|
137
154
|
"""
|
138
155
|
|
139
156
|
Scenario: rbd to rbd - populated cache
|
140
157
|
Given a correctly initialised cache at "/tmp/uricp"
|
141
|
-
When I successfully run `uricp --dry-run --dry-cache --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
|
158
|
+
When I successfully run `uricp --dry-run --dry-cache=rbd --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
|
142
159
|
And the output should not contain:
|
143
160
|
"""
|
144
161
|
snap create
|
@@ -165,7 +182,7 @@ Feature: Manipulate file images in ceph
|
|
165
182
|
"""
|
166
183
|
And the output should contain:
|
167
184
|
"""
|
168
|
-
rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
|
185
|
+
rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
|
169
186
|
"""
|
170
187
|
And the output should contain:
|
171
188
|
"""
|
@@ -174,7 +191,16 @@ Feature: Manipulate file images in ceph
|
|
174
191
|
|
175
192
|
Scenario: RBD export with cacheing - populated cache
|
176
193
|
Given a correctly initialised cache at "/tmp/uricp"
|
177
|
-
When I run `uricp --dry-run --dry-cache --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
|
194
|
+
When I run `uricp --dry-run --dry-cache=rbd --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
|
195
|
+
Then the exit status should be 70
|
196
|
+
And the output should contain:
|
197
|
+
"""
|
198
|
+
Unsupported transfer
|
199
|
+
"""
|
200
|
+
|
201
|
+
Scenario: RBD export with cacheing - partial cache
|
202
|
+
Given a correctly initialised cache at "/tmp/uricp"
|
203
|
+
When I run `uricp --dry-run --dry-cache=partial_rbd --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
|
178
204
|
Then the exit status should be 70
|
179
205
|
And the output should contain:
|
180
206
|
"""
|
@@ -192,13 +218,13 @@ Feature: Manipulate file images in ceph
|
|
192
218
|
"""
|
193
219
|
rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
|
194
220
|
"""
|
195
|
-
And the output should contain:
|
221
|
+
And the output should not contain:
|
196
222
|
"""
|
197
223
|
rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
|
198
224
|
"""
|
199
|
-
And the output should contain:
|
225
|
+
And the output should not contain:
|
200
226
|
"""
|
201
|
-
rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
|
227
|
+
rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap'
|
202
228
|
"""
|
203
229
|
And the output should not contain:
|
204
230
|
"""
|
@@ -220,7 +246,3 @@ Feature: Manipulate file images in ceph
|
|
220
246
|
"""
|
221
247
|
rbd snap protect --id libvirt 'servers/img-testy@base'
|
222
248
|
"""
|
223
|
-
And the output should contain:
|
224
|
-
"""
|
225
|
-
rbd flatten --id libvirt --no-progress 'servers/img-testy'
|
226
|
-
"""
|
data/lib/segment_upload.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uricp/version'
|
2
4
|
require 'uricp/curl_primitives'
|
3
5
|
require 'uricp/orbit_auth'
|
@@ -6,31 +8,3 @@ require 'uricp/segmenter'
|
|
6
8
|
module Uricp
|
7
9
|
UnsupportedURLtype = Class.new(ArgumentError)
|
8
10
|
end
|
9
|
-
|
10
|
-
# Monkey patch a copy_stream facility in using 'sendfile'
|
11
|
-
unless IO.respond_to? :copy_stream
|
12
|
-
require 'sendfile'
|
13
|
-
|
14
|
-
def IO.copy_stream(src, dst, copy_length = nil, offset = nil)
|
15
|
-
if src.stat.pipe?
|
16
|
-
amount = copy_length.to_i
|
17
|
-
buf_size = [amount, 2**16].min
|
18
|
-
buffer = ''
|
19
|
-
while amount > 0
|
20
|
-
src.read(buf_size, buffer)
|
21
|
-
amount_read = buffer.length
|
22
|
-
dst.write(buffer)
|
23
|
-
amount -= amount_read
|
24
|
-
break if src.eof?
|
25
|
-
end
|
26
|
-
copy_length.to_i - amount
|
27
|
-
else
|
28
|
-
current_pos = src.pos
|
29
|
-
count = dst.sendfile(src, offset || current_pos, copy_length)
|
30
|
-
src.seek(count, IO::SEEK_CUR)
|
31
|
-
count
|
32
|
-
end
|
33
|
-
rescue EOFError
|
34
|
-
0
|
35
|
-
end
|
36
|
-
end
|
data/lib/uricp/orbit_auth.rb
CHANGED
data/lib/uricp/segmenter.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'pathname'
|
2
4
|
|
3
5
|
module Uricp
|
@@ -67,7 +69,7 @@ module Uricp
|
|
67
69
|
end
|
68
70
|
|
69
71
|
def upload_segment(segment_number)
|
70
|
-
segment_name = File.join(to.to_s, manifest_suffix, '%08d'
|
72
|
+
segment_name = File.join(to.to_s, manifest_suffix, format('%08d', segment_number))
|
71
73
|
debug "Uploading with #{curl_upload_from('-', segment_name)}"
|
72
74
|
open('|' + curl_upload_from('-', segment_name), 'w') do |destination|
|
73
75
|
copy_length = IO.copy_stream(@source, destination, segment_size)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'fileutils'
|
2
4
|
|
3
5
|
module Uricp::Strategy
|
@@ -25,16 +27,16 @@ module Uricp::Strategy
|
|
25
27
|
end
|
26
28
|
|
27
29
|
def without_active_cache
|
28
|
-
|
29
|
-
yield
|
30
|
-
else
|
30
|
+
if cache_root
|
31
31
|
debug "#{self.class.name}: cache active - not appropriate"
|
32
32
|
false
|
33
|
+
else
|
34
|
+
yield
|
33
35
|
end
|
34
36
|
end
|
35
37
|
|
36
38
|
def in_cache?
|
37
|
-
File.readable?(cache_file) || options['dry-cache']
|
39
|
+
File.readable?(cache_file) || options['dry-cache'] == :rbd
|
38
40
|
end
|
39
41
|
|
40
42
|
def cache_root
|
@@ -1,7 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Uricp::Strategy
|
2
4
|
module Common
|
3
5
|
include Methadone::CLILogging
|
4
6
|
include Uricp::CurlPrimitives
|
7
|
+
include Methadone::SH
|
5
8
|
|
6
9
|
def initialize(options)
|
7
10
|
@options = options
|
@@ -91,17 +94,19 @@ module Uricp::Strategy
|
|
91
94
|
end
|
92
95
|
|
93
96
|
PIPE_URI = URI('pipe:/')
|
97
|
+
DRY_SNAP = 'uricp_snap'
|
98
|
+
|
94
99
|
def rbd_base_name
|
95
|
-
'base'
|
100
|
+
'base'
|
96
101
|
end
|
97
102
|
|
98
103
|
def rbd_snapshot_name
|
99
|
-
|
104
|
+
@rbd_snapshot_name ||= dry_run? ? DRY_SNAP : SecureRandom.uuid
|
100
105
|
end
|
101
106
|
|
102
107
|
def get_temp_filename(base_dir)
|
103
108
|
t = Time.now.strftime('%Y%m%d')
|
104
|
-
File.join(base_dir, "uricp-#{t}-#{
|
109
|
+
File.join(base_dir, "uricp-#{t}-#{$PROCESS_ID}-#{rand(0x100000000).to_s(36)}")
|
105
110
|
end
|
106
111
|
|
107
112
|
def proposed_path
|
@@ -128,7 +133,7 @@ module Uricp::Strategy
|
|
128
133
|
options['rbd_cache_name']
|
129
134
|
end
|
130
135
|
|
131
|
-
def rbd_clone_snapshot(cache=rbd_cache_name)
|
136
|
+
def rbd_clone_snapshot(cache = rbd_cache_name)
|
132
137
|
"#{cache}@#{rbd_base_name}"
|
133
138
|
end
|
134
139
|
|
@@ -156,10 +161,29 @@ module Uricp::Strategy
|
|
156
161
|
uri.scheme == 'rbd' && uri.path.include?('@')
|
157
162
|
end
|
158
163
|
|
164
|
+
def rbd_cache_upload_available?
|
165
|
+
rbd_cache_name && !rbd_cache_image_exists?(rbd_cache_name)
|
166
|
+
end
|
167
|
+
|
168
|
+
def rbd_cache_image_exists?(target)
|
169
|
+
command = "rbd status --id #{rbd_id} --format json #{target} 2>/dev/null"
|
170
|
+
if dry_run?
|
171
|
+
if options['dry-cache'] == :partial_rbd && options['cache_name'] !~ /srv-...../
|
172
|
+
command = 'exit 0'
|
173
|
+
else
|
174
|
+
command = 'exit 2'
|
175
|
+
end
|
176
|
+
end
|
177
|
+
sh!(command)
|
178
|
+
true
|
179
|
+
rescue Methadone::FailedCommandError
|
180
|
+
false
|
181
|
+
end
|
182
|
+
|
159
183
|
def in_rbd_cache(target)
|
160
184
|
result = false
|
161
185
|
if dry_run?
|
162
|
-
result = options['dry-cache'] && options['cache_name'] !~ /srv-...../
|
186
|
+
result = options['dry-cache'] == :rbd && options['cache_name'] !~ /srv-...../
|
163
187
|
else
|
164
188
|
sh "rbd snap ls --id #{rbd_id} --format json #{target} 2>/dev/null" do |stdout|
|
165
189
|
result = JSON.parse(stdout).any? { |x| x['name'] == rbd_base_name }
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'fileutils'
|
2
4
|
|
3
5
|
module Uricp::Strategy
|
@@ -24,8 +26,9 @@ module Uricp::Strategy
|
|
24
26
|
def proposal
|
25
27
|
@proposed_options = options.dup
|
26
28
|
@proposed_options['sweep'] = [temp_cache_file, cache_file]
|
29
|
+
image_spec_to_check = rbd_cache_image_spec(to)
|
27
30
|
if to.scheme == 'rbd'
|
28
|
-
@proposed_options['rbd_cache_name'] =
|
31
|
+
@proposed_options['rbd_cache_name'] = image_spec_to_check
|
29
32
|
end
|
30
33
|
@proposed_options.delete('cache')
|
31
34
|
@proposed_options.delete('cache_name')
|
@@ -1,15 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'json'
|
2
4
|
module Uricp::Strategy
|
3
5
|
class PipedRbdGet
|
4
6
|
include Uricp::Strategy::Common
|
5
7
|
include Uricp::Strategy::CacheCommon
|
6
|
-
include Methadone::SH
|
7
8
|
|
8
9
|
def appropriate?
|
9
10
|
without_active_cache do
|
10
11
|
if from.scheme == 'rbd' &&
|
11
|
-
|
12
|
-
|
12
|
+
rbd_snapshot_spec?(from) &&
|
13
|
+
to.scheme != 'rbd'
|
13
14
|
return proposal unless sequence_complete?
|
14
15
|
end
|
15
16
|
end
|
@@ -1,11 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Uricp::Strategy
|
2
4
|
class RbdCacheBaseSnap
|
3
5
|
include Uricp::Strategy::Common
|
4
|
-
include Methadone::SH
|
5
6
|
|
6
7
|
def appropriate?
|
7
8
|
if rbd_cache_name &&
|
8
|
-
|
9
|
+
rbd_image_spec(from) == rbd_cache_name
|
9
10
|
return proposal
|
10
11
|
end
|
11
12
|
debug "#{self.class.name}: not appropriate"
|
@@ -13,7 +14,7 @@ module Uricp::Strategy
|
|
13
14
|
end
|
14
15
|
|
15
16
|
def command
|
16
|
-
"rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}'
|
17
|
+
"rbd snap create --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"\
|
17
18
|
"rbd snap protect --id #{rbd_id} '#{rbd_clone_snapshot(rbd_cache_name)}';"
|
18
19
|
end
|
19
20
|
|
@@ -1,10 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
4
|
|
3
5
|
module Uricp::Strategy
|
4
6
|
class RbdCacheCheck
|
5
7
|
include Uricp::Strategy::Common
|
6
8
|
include Uricp::Strategy::CacheCommon
|
7
|
-
include Methadone::SH
|
8
9
|
|
9
10
|
def appropriate?
|
10
11
|
unless (from.scheme == 'rbd') != (to.scheme == 'rbd')
|
@@ -1,10 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
4
|
|
3
5
|
module Uricp::Strategy
|
4
6
|
class RbdCacheClone
|
5
7
|
include Uricp::Strategy::Common
|
6
8
|
include Uricp::Strategy::CacheCommon
|
7
|
-
include Methadone::SH
|
8
9
|
|
9
10
|
def appropriate?
|
10
11
|
unless from.scheme == 'rbd'
|
@@ -15,8 +16,7 @@ module Uricp::Strategy
|
|
15
16
|
with_active_cache do
|
16
17
|
options['cache_name'] = File.basename(options['to_uri'].path)
|
17
18
|
cache_target = rbd_cache_image_spec(from)
|
18
|
-
|
19
|
-
if cache_check
|
19
|
+
if rbd_cache_image_exists?(cache_target) || in_rbd_cache(cache_target)
|
20
20
|
debug "#{self.class.name}: Unexpected existing cache entry for #{options['to_uri']}"
|
21
21
|
unsupported_transfer
|
22
22
|
end
|
@@ -1,14 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Uricp::Strategy
|
2
4
|
class RbdCacheUpload
|
3
5
|
include Uricp::Strategy::Common
|
4
|
-
include Methadone::SH
|
5
6
|
|
6
7
|
def appropriate?
|
7
8
|
if compression_required? || conversion_required?
|
8
9
|
debug "#{self.class.name}: not ready to upload"
|
9
10
|
return false
|
10
11
|
end
|
11
|
-
if
|
12
|
+
if rbd_cache_upload_available?
|
12
13
|
case from.scheme
|
13
14
|
when 'pipe', 'file'
|
14
15
|
return proposal if to.scheme == 'rbd'
|
@@ -19,7 +20,7 @@ module Uricp::Strategy
|
|
19
20
|
end
|
20
21
|
|
21
22
|
def command
|
22
|
-
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}'
|
23
|
+
"rbd import --no-progress --id #{rbd_id} #{data_source} '#{rbd_cache_name}';"
|
23
24
|
end
|
24
25
|
|
25
26
|
def proposal
|
@@ -35,6 +36,5 @@ module Uricp::Strategy
|
|
35
36
|
"'#{from.path}'"
|
36
37
|
end
|
37
38
|
end
|
38
|
-
|
39
39
|
end
|
40
40
|
end
|
@@ -1,10 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
4
|
|
3
5
|
module Uricp::Strategy
|
4
6
|
class RbdCachedGet
|
5
7
|
include Uricp::Strategy::Common
|
6
8
|
include Uricp::Strategy::CacheCommon
|
7
|
-
include Methadone::SH
|
8
9
|
|
9
10
|
def appropriate?
|
10
11
|
if from.scheme != 'rbd' || rbd_snapshot_spec?(from)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Uricp::Strategy
|
2
4
|
class RbdCachedPut
|
3
5
|
include Uricp::Strategy::Common
|
@@ -5,9 +7,9 @@ module Uricp::Strategy
|
|
5
7
|
|
6
8
|
def appropriate?
|
7
9
|
return proposal if to.scheme == 'rbd' &&
|
8
|
-
|
9
|
-
|
10
|
-
|
10
|
+
rbd_snapshot_spec?(from) &&
|
11
|
+
rbd_cache_name.nil? &&
|
12
|
+
!rbd_sequence_complete?
|
11
13
|
|
12
14
|
debug "#{self.class.name}: not appropriate"
|
13
15
|
false
|
@@ -23,11 +25,9 @@ module Uricp::Strategy
|
|
23
25
|
if options['rbd_cache_target']
|
24
26
|
@proposed_options['to_uri'] = options['rbd_cache_target']
|
25
27
|
@proposed_options['rbd_cache_name'] = rbd_image_spec(to)
|
26
|
-
@proposed_options['rbd_flatten'] ||= @proposed_options['rbd_cache_name']
|
27
28
|
@proposed_options.delete('rbd_cache_target')
|
28
29
|
end
|
29
30
|
self
|
30
31
|
end
|
31
|
-
|
32
32
|
end
|
33
33
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Uricp::Strategy
|
2
4
|
class RbdPut
|
3
5
|
include Uricp::Strategy::Common
|
@@ -7,11 +9,13 @@ module Uricp::Strategy
|
|
7
9
|
debug "#{self.class.name}: not ready to upload"
|
8
10
|
return false
|
9
11
|
end
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
if rbd_cache_upload_available?
|
13
|
+
debug "#{self.class.name}: cacheing in RBD instead"
|
14
|
+
return false
|
15
|
+
end
|
16
|
+
case from.scheme
|
17
|
+
when 'pipe', 'file'
|
18
|
+
return proposal if to.scheme == 'rbd'
|
15
19
|
end
|
16
20
|
debug "#{self.class.name}: not appropriate"
|
17
21
|
false
|
@@ -1,17 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'json'
|
2
4
|
module Uricp::Strategy
|
3
5
|
class RbdSnap
|
4
6
|
include Uricp::Strategy::Common
|
5
7
|
include Uricp::Strategy::CacheCommon
|
6
|
-
include Methadone::SH
|
7
8
|
|
8
9
|
def appropriate?
|
9
10
|
without_active_cache do
|
10
11
|
if from.scheme == 'rbd' &&
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
options['rbd_snapshot'].nil? &&
|
13
|
+
!rbd_snapshot_spec?(from) &&
|
14
|
+
(rbd_cache_name.nil? ||
|
15
|
+
!from.path.include?(rbd_cache_name))
|
15
16
|
if snap_in_progress?
|
16
17
|
debug "#{self.class.name}: detected snapshot in progress"
|
17
18
|
else
|
@@ -24,17 +25,18 @@ module Uricp::Strategy
|
|
24
25
|
end
|
25
26
|
|
26
27
|
def command
|
27
|
-
"rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}'
|
28
|
-
"rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}'
|
28
|
+
"rbd snap create --id #{rbd_id} '#{rbd_upload_snapshot(from)}';" \
|
29
|
+
"rbd snap protect --id #{rbd_id} '#{rbd_upload_snapshot(from)}';" \
|
29
30
|
end
|
30
31
|
|
31
32
|
def proposal
|
32
33
|
@proposed_options = options.dup
|
33
|
-
@proposed_options['
|
34
|
-
@proposed_options['from_uri'] = rbd_uri(@proposed_options['rbd_snapshot'])
|
34
|
+
@proposed_options['from_uri'] = rbd_uri(rbd_upload_snapshot(from))
|
35
35
|
if options['rbd_cache_target']
|
36
36
|
@proposed_options['to_uri'] = options['rbd_cache_target']
|
37
37
|
@proposed_options['rbd_cache_target'] = to
|
38
|
+
else
|
39
|
+
@proposed_options['rbd_snapshot'] = rbd_upload_snapshot(from) unless to.scheme == 'rbd'
|
38
40
|
end
|
39
41
|
self
|
40
42
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Uricp::Strategy
|
2
4
|
class RbdSweeper
|
3
5
|
include Uricp::Strategy::Common
|
@@ -10,7 +12,7 @@ module Uricp::Strategy
|
|
10
12
|
end
|
11
13
|
|
12
14
|
def command
|
13
|
-
"rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}'
|
15
|
+
"rbd snap unprotect --id #{rbd_id} '#{options['rbd_snapshot']}';" \
|
14
16
|
"rbd snap rm --id #{rbd_id} '#{options['rbd_snapshot']}';"
|
15
17
|
end
|
16
18
|
|
data/lib/uricp/uri_strategy.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
4
|
|
3
5
|
module Uricp
|
@@ -49,7 +51,6 @@ module Uricp
|
|
49
51
|
Strategy::PipedLocalGet,
|
50
52
|
Strategy::PipedLocalPut,
|
51
53
|
Strategy::Cleaner,
|
52
|
-
Strategy::RbdFlattener,
|
53
54
|
Strategy::RbdSweeper,
|
54
55
|
Strategy::Sweeper
|
55
56
|
].freeze
|
data/lib/uricp/version.rb
CHANGED
data/lib/uricp.rb
CHANGED
@@ -29,7 +29,6 @@ require 'uricp/strategy/rbd_cache_clone'
|
|
29
29
|
require 'uricp/strategy/rbd_cached_get'
|
30
30
|
require 'uricp/strategy/rbd_cached_put'
|
31
31
|
require 'uricp/strategy/rbd_cache_upload'
|
32
|
-
require 'uricp/strategy/rbd_flattener'
|
33
32
|
require 'uricp/strategy/rbd_put'
|
34
33
|
require 'uricp/strategy/rbd_snap'
|
35
34
|
require 'uricp/strategy/rbd_sweeper'
|
data/uricp.gemspec
CHANGED
@@ -1,32 +1,32 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
lib = File.expand_path('../lib', __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'uricp/version'
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
8
|
+
spec.name = 'uricp'
|
8
9
|
spec.version = Uricp::VERSION
|
9
|
-
spec.authors = [
|
10
|
-
spec.email = [
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
14
|
-
spec.license =
|
10
|
+
spec.authors = ['Neil Wilson']
|
11
|
+
spec.email = ['neil@aldur.co.uk']
|
12
|
+
spec.summary = 'Copy one URL to another with optional cacheing'
|
13
|
+
spec.description = 'Copy one URL to another with optional cacheing'
|
14
|
+
spec.homepage = ''
|
15
|
+
spec.license = 'gplv3'
|
15
16
|
|
16
17
|
spec.files = `git ls-files -z`.split("\x0")
|
17
18
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
-
spec.require_paths = [
|
20
|
+
spec.require_paths = ['lib']
|
20
21
|
|
21
|
-
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
-
spec.add_development_dependency('rdoc', '~> 4.2.0')
|
23
22
|
spec.add_development_dependency('aruba', '~> 0.6.0')
|
23
|
+
spec.add_development_dependency 'bundler', '~> 1.7'
|
24
24
|
spec.add_development_dependency('cucumber', '~> 1.3')
|
25
|
+
spec.add_development_dependency('inifile', '~> 1.1')
|
25
26
|
spec.add_development_dependency('rake', '~> 12.3')
|
27
|
+
spec.add_development_dependency('rdoc', '~> 4.2.0')
|
26
28
|
spec.add_dependency('childprocess', '~> 1.0')
|
29
|
+
spec.add_dependency('filesize', '= 0.0.2')
|
27
30
|
spec.add_dependency('methadone', '~> 2.0.2')
|
28
31
|
spec.add_dependency('open4', '~> 1.3.0')
|
29
|
-
spec.add_dependency('filesize', '= 0.0.2')
|
30
|
-
spec.add_dependency('sendfile', '~> 1.2.0')
|
31
|
-
spec.add_development_dependency('inifile', '~> 1.1')
|
32
32
|
end
|
metadata
CHANGED
@@ -1,71 +1,71 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: uricp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.25
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Neil Wilson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: aruba
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 0.6.0
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 0.6.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: '1.7'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: '1.7'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: cucumber
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '1.3'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '1.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: inifile
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '1.
|
61
|
+
version: '1.1'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '1.
|
68
|
+
version: '1.1'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rake
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,47 +81,33 @@ dependencies:
|
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '12.3'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - "~>"
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '1.0'
|
90
|
-
type: :runtime
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - "~>"
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '1.0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: methadone
|
84
|
+
name: rdoc
|
99
85
|
requirement: !ruby/object:Gem::Requirement
|
100
86
|
requirements:
|
101
87
|
- - "~>"
|
102
88
|
- !ruby/object:Gem::Version
|
103
|
-
version: 2.0
|
104
|
-
type: :
|
89
|
+
version: 4.2.0
|
90
|
+
type: :development
|
105
91
|
prerelease: false
|
106
92
|
version_requirements: !ruby/object:Gem::Requirement
|
107
93
|
requirements:
|
108
94
|
- - "~>"
|
109
95
|
- !ruby/object:Gem::Version
|
110
|
-
version: 2.0
|
96
|
+
version: 4.2.0
|
111
97
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
98
|
+
name: childprocess
|
113
99
|
requirement: !ruby/object:Gem::Requirement
|
114
100
|
requirements:
|
115
101
|
- - "~>"
|
116
102
|
- !ruby/object:Gem::Version
|
117
|
-
version: 1.
|
103
|
+
version: '1.0'
|
118
104
|
type: :runtime
|
119
105
|
prerelease: false
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
121
107
|
requirements:
|
122
108
|
- - "~>"
|
123
109
|
- !ruby/object:Gem::Version
|
124
|
-
version: 1.
|
110
|
+
version: '1.0'
|
125
111
|
- !ruby/object:Gem::Dependency
|
126
112
|
name: filesize
|
127
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -137,33 +123,33 @@ dependencies:
|
|
137
123
|
- !ruby/object:Gem::Version
|
138
124
|
version: 0.0.2
|
139
125
|
- !ruby/object:Gem::Dependency
|
140
|
-
name:
|
126
|
+
name: methadone
|
141
127
|
requirement: !ruby/object:Gem::Requirement
|
142
128
|
requirements:
|
143
129
|
- - "~>"
|
144
130
|
- !ruby/object:Gem::Version
|
145
|
-
version:
|
131
|
+
version: 2.0.2
|
146
132
|
type: :runtime
|
147
133
|
prerelease: false
|
148
134
|
version_requirements: !ruby/object:Gem::Requirement
|
149
135
|
requirements:
|
150
136
|
- - "~>"
|
151
137
|
- !ruby/object:Gem::Version
|
152
|
-
version:
|
138
|
+
version: 2.0.2
|
153
139
|
- !ruby/object:Gem::Dependency
|
154
|
-
name:
|
140
|
+
name: open4
|
155
141
|
requirement: !ruby/object:Gem::Requirement
|
156
142
|
requirements:
|
157
143
|
- - "~>"
|
158
144
|
- !ruby/object:Gem::Version
|
159
|
-
version:
|
160
|
-
type: :
|
145
|
+
version: 1.3.0
|
146
|
+
type: :runtime
|
161
147
|
prerelease: false
|
162
148
|
version_requirements: !ruby/object:Gem::Requirement
|
163
149
|
requirements:
|
164
150
|
- - "~>"
|
165
151
|
- !ruby/object:Gem::Version
|
166
|
-
version:
|
152
|
+
version: 1.3.0
|
167
153
|
description: Copy one URL to another with optional cacheing
|
168
154
|
email:
|
169
155
|
- neil@aldur.co.uk
|
@@ -229,7 +215,6 @@ files:
|
|
229
215
|
- lib/uricp/strategy/rbd_cache_upload.rb
|
230
216
|
- lib/uricp/strategy/rbd_cached_get.rb
|
231
217
|
- lib/uricp/strategy/rbd_cached_put.rb
|
232
|
-
- lib/uricp/strategy/rbd_flattener.rb
|
233
218
|
- lib/uricp/strategy/rbd_put.rb
|
234
219
|
- lib/uricp/strategy/rbd_snap.rb
|
235
220
|
- lib/uricp/strategy/rbd_sweeper.rb
|
@@ -1,23 +0,0 @@
|
|
1
|
-
module Uricp::Strategy
|
2
|
-
class RbdFlattener
|
3
|
-
include Uricp::Strategy::Common
|
4
|
-
|
5
|
-
def appropriate?
|
6
|
-
return proposal if options['rbd_flatten'] &&
|
7
|
-
rbd_sequence_complete?
|
8
|
-
|
9
|
-
debug "#{self.class.name}: not appropriate"
|
10
|
-
false
|
11
|
-
end
|
12
|
-
|
13
|
-
def command
|
14
|
-
"rbd flatten --id #{rbd_id} --no-progress '#{options['rbd_flatten']}' && "
|
15
|
-
end
|
16
|
-
|
17
|
-
def proposal
|
18
|
-
@proposed_options = options.dup
|
19
|
-
@proposed_options.delete('rbd_flatten')
|
20
|
-
self
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|