uricp 0.0.16 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,226 @@
1
+ @rbd
2
+ Feature: Manipulate file images in ceph
3
+ in order to manipulate file images in ceph
4
+ as a command line user
5
+ I want to retrieve the uri via an optional cache and copy correctly to target in the right format
6
+
7
+ background:
8
+ given an empty directory named "/tmp/uricp"
9
+ and the default aruba timeout is 15 seconds
10
+
11
+ Scenario: HTTP URI qcow2 to rbd - no cache
12
+ When I successfully run `uricp --dry-run http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
13
+ And the output should contain:
14
+ """
15
+ rbd import --no-progress --id libvirt - 'servers/srv-testy'
16
+ """
17
+
18
+ Scenario: HTTP URI qcow2 to raw via unpopulated rbd cache
19
+ Given a correctly initialised cache at "/tmp/uricp"
20
+ When I successfully run `uricp --dry-run --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
21
+ And the output should contain:
22
+ """
23
+ rbd import --no-progress --id libvirt - 'servers/freedos.qcow2'
24
+ """
25
+ And the output should contain:
26
+ """
27
+ rbd snap create --id libvirt 'servers/freedos.qcow2@base'
28
+ """
29
+ And the output should contain:
30
+ """
31
+ rbd snap protect --id libvirt 'servers/freedos.qcow2@base'
32
+ """
33
+ And the output should contain:
34
+ """
35
+ rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
36
+ """
37
+
38
+ Scenario: HTTP URI from file cache via unpopulated rbd cache
39
+ Given a correctly initialised cache at "/tmp/uricp"
40
+ And a 102400 byte file named "/tmp/uricp/cache/freedos.qcow2"
41
+ When I successfully run `uricp --dry-run --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
42
+ And the output should contain:
43
+ """
44
+ rbd import --no-progress --id libvirt '/tmp/uricp/cache/freedos.qcow2' 'servers/freedos.qcow2'
45
+ """
46
+ And the output should contain:
47
+ """
48
+ rbd snap create --id libvirt 'servers/freedos.qcow2@base'
49
+ """
50
+ And the output should contain:
51
+ """
52
+ rbd snap protect --id libvirt 'servers/freedos.qcow2@base'
53
+ """
54
+ And the output should contain:
55
+ """
56
+ rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
57
+ """
58
+
59
+ Scenario: HTTP URI to rbd via populated caches
60
+ Given a correctly initialised cache at "/tmp/uricp"
61
+ When I successfully run `uricp --dry-run --dry-cache --target-format=raw --cache=/tmp/uricp http://orbit.brightbox.com/v1/acc-tqs4c/downloads/freedos.qcow2 rbd:///servers/srv-testy`
62
+ And the output should not contain:
63
+ """
64
+ snap create
65
+ """
66
+ And the output should contain:
67
+ """
68
+ rbd clone --id libvirt 'servers/freedos.qcow2@base' 'servers/srv-testy'
69
+ """
70
+
71
+ Scenario: rbd to rbd - no cache
72
+ Given a correctly initialised cache at "/tmp/uricp"
73
+ When I successfully run `uricp --dry-run rbd:///servers/img-testy rbd:///servers/srv-testy`
74
+ And the output should contain:
75
+ """
76
+ rbd snap create --id libvirt 'servers/img-testy@uricp_snap'
77
+ """
78
+ And the output should contain:
79
+ """
80
+ rbd snap protect --id libvirt 'servers/img-testy@uricp_snap'
81
+ """
82
+ And the output should contain:
83
+ """
84
+ rbd snap unprotect --id libvirt 'servers/img-testy@uricp_snap'
85
+ """
86
+ And the output should contain:
87
+ """
88
+ rbd snap rm --id libvirt 'servers/img-testy@uricp_snap';
89
+ """
90
+ And the output should contain:
91
+ """
92
+ rbd clone --id libvirt 'servers/img-testy@uricp_snap' 'servers/srv-testy'
93
+ """
94
+
95
+ Scenario: rbd to rbd - unpopulated cache
96
+ Given a correctly initialised cache at "/tmp/uricp"
97
+ When I successfully run `uricp --dry-run --cache=/tmp/uricp rbd:///servers/srv-testy rbd:///servers/img-testy`
98
+ And the output should contain:
99
+ """
100
+ rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
101
+ """
102
+ And the output should contain:
103
+ """
104
+ rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
105
+ """
106
+ And the output should contain:
107
+ """
108
+ rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
109
+ """
110
+ And the output should contain:
111
+ """
112
+ rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
113
+ """
114
+ And the output should not contain:
115
+ """
116
+ rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
117
+ """
118
+ And the output should contain:
119
+ """
120
+ rbd clone --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
121
+ """
122
+ And the output should not contain:
123
+ """
124
+ rbd export --no-progress --id libvirt 'servers/img-testy@base' -
125
+ """
126
+ And the output should contain:
127
+ """
128
+ rbd snap create --id libvirt 'servers/img-testy@base'
129
+ """
130
+ And the output should contain:
131
+ """
132
+ rbd snap protect --id libvirt 'servers/img-testy@base'
133
+ """
134
+ And the output should contain:
135
+ """
136
+ rbd flatten --id libvirt --no-progress 'servers/img-testy'
137
+ """
138
+
139
+ Scenario: rbd to rbd - populated cache
140
+ Given a correctly initialised cache at "/tmp/uricp"
141
+ When I successfully run `uricp --dry-run --dry-cache --cache=/tmp/uricp rbd:///servers/img-testy rbd:///servers/srv-testy`
142
+ And the output should not contain:
143
+ """
144
+ snap create
145
+ """
146
+ And the output should contain:
147
+ """
148
+ rbd clone --id libvirt 'servers/img-testy@base' 'servers/srv-testy'
149
+ """
150
+
151
+ Scenario: RBD export.
152
+ Given a correctly initialised cache at "/tmp/uricp"
153
+ When I successfully run `uricp --dry-run --compress rbd:///servers/srv-testy file:///tmp/img-testy`
154
+ And the output should contain:
155
+ """
156
+ rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
157
+ """
158
+ And the output should contain:
159
+ """
160
+ rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
161
+ """
162
+ And the output should contain:
163
+ """
164
+ rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
165
+ """
166
+ And the output should contain:
167
+ """
168
+ rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
169
+ """
170
+ And the output should contain:
171
+ """
172
+ rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
173
+ """
174
+
175
+ Scenario: RBD export with cacheing - populated cache
176
+ Given a correctly initialised cache at "/tmp/uricp"
177
+ When I run `uricp --dry-run --dry-cache --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
178
+ Then the exit status should be 70
179
+ And the output should contain:
180
+ """
181
+ Unsupported transfer
182
+ """
183
+
184
+ Scenario: RBD export with cacheing - empty cache
185
+ Given a correctly initialised cache at "/tmp/uricp"
186
+ When I successfully run `uricp --dry-run --compress --cache=/tmp/uricp rbd:///servers/srv-testy file:///tmp/img-testy`
187
+ And the output should contain:
188
+ """
189
+ rbd snap create --id libvirt 'servers/srv-testy@uricp_snap'
190
+ """
191
+ And the output should contain:
192
+ """
193
+ rbd snap protect --id libvirt 'servers/srv-testy@uricp_snap'
194
+ """
195
+ And the output should contain:
196
+ """
197
+ rbd snap unprotect --id libvirt 'servers/srv-testy@uricp_snap'
198
+ """
199
+ And the output should contain:
200
+ """
201
+ rbd snap rm --id libvirt 'servers/srv-testy@uricp_snap';
202
+ """
203
+ And the output should not contain:
204
+ """
205
+ rbd export --no-progress --id libvirt 'servers/srv-testy@uricp_snap' -
206
+ """
207
+ And the output should contain:
208
+ """
209
+ rbd clone --id libvirt 'servers/srv-testy@uricp_snap' 'servers/img-testy'
210
+ """
211
+ And the output should contain:
212
+ """
213
+ rbd export --no-progress --id libvirt 'servers/img-testy@base' -
214
+ """
215
+ And the output should contain:
216
+ """
217
+ rbd snap create --id libvirt 'servers/img-testy@base'
218
+ """
219
+ And the output should contain:
220
+ """
221
+ rbd snap protect --id libvirt 'servers/img-testy@base'
222
+ """
223
+ And the output should contain:
224
+ """
225
+ rbd flatten --id libvirt --no-progress 'servers/img-testy'
226
+ """
@@ -0,0 +1,8 @@
1
+ Then(/^a ceph entry named "(.*?)" should exist$/) do |arg1|
2
+ pending # express the regexp above with the code you wish you had
3
+ end
4
+
5
+ Then(/^a ceph snapshot named "(.*?)" should exist$/) do |arg1|
6
+ pending # express the regexp above with the code you wish you had
7
+ end
8
+
@@ -1,8 +1,10 @@
1
1
  Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filename, format|
2
2
  case format
3
3
  when 'lz4'
4
- assert_exact_output([0x184D2204].pack('V'),
5
- File.open(filename, 'rb') {|f| f.read(4) }.to_s)
4
+ assert_exact_output(
5
+ [0x184D2204].pack('V'),
6
+ File.open(filename, 'rb') { |f| f.read(4) } || String.new
7
+ )
6
8
  when 'qcow2v3', 'qcow3'
7
9
  steps %{
8
10
  When I successfully run `qemu-img info #{filename}`
@@ -10,8 +12,10 @@ Then(/^the file named "(.*?)" should have a file format of "(.*?)"$/) do |filena
10
12
  And the output from "qemu-img info #{filename}" should contain "compat: 1.1"
11
13
  }
12
14
  else
13
- assert_no_partial_output([0x184D2204].pack('V'),
14
- File.open(filename, 'rb') {|f| f.read(4) }.to_s)
15
+ assert_no_partial_output(
16
+ [0x184D2204].pack('V'),
17
+ File.open(filename, 'rb') { |f| f.read(4) } || String.new
18
+ )
15
19
  steps(%{
16
20
  When I successfully run `qemu-img info #{filename}`
17
21
  Then the output from "qemu-img info #{filename}" should contain "file format: #{format}"
data/focal/Dockerfile ADDED
@@ -0,0 +1,28 @@
1
+ FROM ubuntu:focal
2
+ MAINTAINER support@brightbox.co.uk
3
+
4
+ RUN echo "gem: --no-ri --no-rdoc" >> "$HOME/.gemrc"
5
+
6
+ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq software-properties-common
7
+
8
+ #RUN apt-add-repository ppa:brightbox/ruby-ng
9
+
10
+ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
11
+ build-essential \
12
+ git \
13
+ qemu-utils \
14
+ liblz4-tool \
15
+ curl \
16
+ ruby \
17
+ ruby-dev \
18
+ python3-swiftclient \
19
+ ruby-bundler \
20
+ ceph-common
21
+
22
+ RUN gem install bundler -v '~> 1.7'
23
+ RUN mkdir -p /etc/ceph
24
+ ARG RBD_USR=libvirt
25
+ ARG RBD_PSW
26
+ ARG CEPH_MON
27
+ RUN /bin/echo -e "[client.$RBD_USR]\nkey = $RBD_PSW\n" > /etc/ceph/ceph.client.$RBD_USR.keyring
28
+ RUN /bin/echo -e "[global]\nmon_host = $CEPH_MON\n" > /etc/ceph/ceph.conf
@@ -15,8 +15,26 @@ module Uricp::Strategy
15
15
  'no cache name found'
16
16
  end
17
17
 
18
+ def with_active_cache
19
+ if cache_root
20
+ yield
21
+ else
22
+ debug "#{self.class.name}: no cacheing requested"
23
+ false
24
+ end
25
+ end
26
+
27
+ def without_active_cache
28
+ unless cache_root
29
+ yield
30
+ else
31
+ debug "#{self.class.name}: cache active - not appropriate"
32
+ false
33
+ end
34
+ end
35
+
18
36
  def in_cache?
19
- File.readable? cache_file
37
+ File.readable?(cache_file) || options['dry-cache']
20
38
  end
21
39
 
22
40
  def cache_root
@@ -6,15 +6,13 @@ module Uricp::Strategy
6
6
  include Uricp::Strategy::CacheCommon
7
7
 
8
8
  def appropriate?
9
- if cache_root
9
+ with_active_cache do
10
10
  validate_cache!
11
11
  return proposal if in_cache? || file_source?
12
12
 
13
13
  debug "#{self.class.name}: no cache entry for #{options['from_uri']}"
14
- else
15
- debug "#{self.class.name}: not appropriate"
14
+ false
16
15
  end
17
- false
18
16
  end
19
17
 
20
18
  def command
@@ -24,10 +22,15 @@ module Uricp::Strategy
24
22
  def proposal
25
23
  @proposed_options = options.dup
26
24
  @proposed_options['from_uri'] = URI.join('file:///', cache_file) unless file_source?
25
+ if to.scheme == 'rbd'
26
+ @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
27
+ end
27
28
  @proposed_options.delete('cache')
28
29
  @proposed_options.delete('cache_name')
29
30
  if conversion_required?
30
- unless dry_run?
31
+ if dry_run?
32
+ @proposed_options['source-format'] = @proposed_options['target-format']
33
+ else
31
34
  @proposed_options['source-format'] =
32
35
  File.open(@proposed_options['from_uri'].path) { |f| encoding(f) }
33
36
  end
@@ -91,6 +91,13 @@ module Uricp::Strategy
91
91
  end
92
92
 
93
93
  PIPE_URI = URI('pipe:/')
94
+ def rbd_base_name
95
+ 'base'.freeze
96
+ end
97
+
98
+ def rbd_snapshot_name
99
+ 'uricp_snap'.freeze
100
+ end
94
101
 
95
102
  def get_temp_filename(base_dir)
96
103
  t = Time.now.strftime('%Y%m%d')
@@ -109,12 +116,56 @@ module Uricp::Strategy
109
116
  options['source-format'] && !lz4_source?
110
117
  end
111
118
 
112
- def rbd_target(uri)
119
+ def rbd_image_spec(uri)
113
120
  uri.path[1..-1]
114
121
  end
115
122
 
123
+ def rbd_cache_image_spec(uri)
124
+ File.join(File.dirname(uri.path)[1..-1], options['cache_name'])
125
+ end
126
+
127
+ def rbd_cache_name
128
+ options['rbd_cache_name']
129
+ end
130
+
131
+ def rbd_clone_snapshot(cache=rbd_cache_name)
132
+ "#{cache}@#{rbd_base_name}"
133
+ end
134
+
135
+ def rbd_uri(image_spec)
136
+ URI.join('rbd:///', image_spec)
137
+ end
138
+
139
+ def rbd_sequence_complete?
140
+ from.path.include?(to.path)
141
+ end
142
+
116
143
  def rbd_id
117
144
  'libvirt'
118
145
  end
146
+
147
+ def in_rbd_cache?
148
+ options['in_rbd_cache']
149
+ end
150
+
151
+ def not_in_rbd_cache?
152
+ options['in_rbd_cache'] == false
153
+ end
154
+
155
+ def rbd_snapshot_spec?(uri)
156
+ uri.scheme == 'rbd' && uri.path.include?('@')
157
+ end
158
+
159
+ def in_rbd_cache(target)
160
+ result = false
161
+ if dry_run?
162
+ result = options['dry-cache'] && options['cache_name'] !~ /srv-...../
163
+ else
164
+ sh "rbd snap ls --id #{rbd_id} --format json #{target} 2>/dev/null" do |stdout|
165
+ result = JSON.parse(stdout).any? { |x| x['name'] == rbd_base_name }
166
+ end
167
+ end
168
+ result && rbd_clone_snapshot(target)
169
+ end
119
170
  end
120
171
  end
@@ -32,7 +32,7 @@ module Uricp::Strategy
32
32
  end
33
33
 
34
34
  def new_qemu?
35
- @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /^compat/m)
35
+ @new_qemu ||= !!(`qemu-img convert -O qcow2 -o ? a b` =~ /\bcompat\b/m)
36
36
  end
37
37
 
38
38
  def supported_conversion?
@@ -6,17 +6,15 @@ module Uricp::Strategy
6
6
  include Uricp::Strategy::CacheCommon
7
7
 
8
8
  def appropriate?
9
- if cache_root
9
+ with_active_cache do
10
10
  case from.scheme
11
11
  when 'pipe'
12
12
  validate_cache!
13
13
  return proposal
14
14
  end
15
15
  debug "#{self.class.name}: not appropriate"
16
- else
17
- debug "#{self.class.name}: no cacheing requested"
16
+ false
18
17
  end
19
- false
20
18
  end
21
19
 
22
20
  def command
@@ -26,6 +24,9 @@ module Uricp::Strategy
26
24
  def proposal
27
25
  @proposed_options = options.dup
28
26
  @proposed_options['sweep'] = [temp_cache_file, cache_file]
27
+ if to.scheme == 'rbd'
28
+ @proposed_options['rbd_cache_name'] = rbd_cache_image_spec(to)
29
+ end
29
30
  @proposed_options.delete('cache')
30
31
  @proposed_options.delete('cache_name')
31
32
  self