bosh_cli 0.16 → 0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +2 -1
- data/lib/cli.rb +2 -0
- data/lib/cli/blob_manager.rb +377 -0
- data/lib/cli/commands/base.rb +4 -74
- data/lib/cli/commands/blob_management.rb +47 -0
- data/lib/cli/commands/release.rb +37 -16
- data/lib/cli/runner.rb +14 -10
- data/lib/cli/templates/help_message.erb +3 -2
- data/lib/cli/version.rb +1 -1
- data/spec/unit/blob_manager_spec.rb +291 -0
- data/spec/unit/cli_commands_spec.rb +37 -170
- data/spec/unit/runner_spec.rb +5 -0
- metadata +157 -89
- data/lib/cli/commands/blob.rb +0 -125
data/Rakefile
CHANGED
data/lib/cli.rb
CHANGED
@@ -0,0 +1,377 @@
|
|
1
|
+
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
+
|
3
|
+
module Bosh::Cli
|
4
|
+
# In order to avoid storing large objects in git repo,
|
5
|
+
# release might save them in the blobstore instead.
|
6
|
+
# BlobManager encapsulates most of the blob operations.
|
7
|
+
class BlobManager
|
8
|
+
DEFAULT_INDEX_NAME = "blobs.yml"
|
9
|
+
|
10
|
+
attr_reader :new_blobs, :updated_blobs
|
11
|
+
|
12
|
+
# @param [Bosh::Cli::Release] release BOSH Release object
|
13
|
+
def initialize(release)
|
14
|
+
@release = release
|
15
|
+
@index_file = File.join(@release.dir, "config", DEFAULT_INDEX_NAME)
|
16
|
+
|
17
|
+
legacy_index_file = File.join(@release.dir, "blob_index.yml")
|
18
|
+
|
19
|
+
if File.exists?(legacy_index_file)
|
20
|
+
if File.exists?(@index_file)
|
21
|
+
err("Found both new and legacy blob index, please fix it")
|
22
|
+
end
|
23
|
+
FileUtils.mv(legacy_index_file, @index_file)
|
24
|
+
end
|
25
|
+
|
26
|
+
if File.file?(@index_file)
|
27
|
+
@index = load_yaml_file(@index_file)
|
28
|
+
else
|
29
|
+
@index = {}
|
30
|
+
end
|
31
|
+
|
32
|
+
@src_dir = File.join(@release.dir, "src")
|
33
|
+
unless File.directory?(@src_dir)
|
34
|
+
err("`src' directory is missing")
|
35
|
+
end
|
36
|
+
|
37
|
+
@storage_dir = File.join(@release.dir, ".blobs")
|
38
|
+
unless File.directory?(@storage_dir)
|
39
|
+
FileUtils.mkdir(@storage_dir)
|
40
|
+
end
|
41
|
+
|
42
|
+
@blobs_dir = File.join(@release.dir, "blobs")
|
43
|
+
unless File.directory?(@blobs_dir)
|
44
|
+
FileUtils.mkdir(@blobs_dir)
|
45
|
+
end
|
46
|
+
|
47
|
+
@blobstore = @release.blobstore
|
48
|
+
if @blobstore.nil?
|
49
|
+
err("Blobstore is not configured")
|
50
|
+
end
|
51
|
+
|
52
|
+
@new_blobs = []
|
53
|
+
@updated_blobs = []
|
54
|
+
end
|
55
|
+
|
56
|
+
# Returns a list of blobs that need to be uploaded
|
57
|
+
# @return [Array]
|
58
|
+
def blobs_to_upload
|
59
|
+
@new_blobs + @updated_blobs
|
60
|
+
end
|
61
|
+
|
62
|
+
# Returns whether blobs directory is dirty
|
63
|
+
# @return Boolean
|
64
|
+
def dirty?
|
65
|
+
@new_blobs.size > 0 || @updated_blobs.size > 0
|
66
|
+
end
|
67
|
+
|
68
|
+
# Prints out blobs status
|
69
|
+
# @return [void]
|
70
|
+
def print_status
|
71
|
+
total_file_size = @index.inject(0) do |total, (_, entry)|
|
72
|
+
total += entry["size"].to_i
|
73
|
+
total
|
74
|
+
end
|
75
|
+
|
76
|
+
say("Total: #{@index.size}, #{pretty_size(total_file_size)}")
|
77
|
+
process_blobs_directory
|
78
|
+
|
79
|
+
unless dirty?
|
80
|
+
say("No blobs to upload".green)
|
81
|
+
return
|
82
|
+
end
|
83
|
+
|
84
|
+
nl
|
85
|
+
say("You have some blobs that need to be uploaded:")
|
86
|
+
@new_blobs.each do |blob|
|
87
|
+
size = File.size(File.join(@blobs_dir, blob))
|
88
|
+
say("%s\t%s\t%s" % ["new".green, blob, pretty_size(size)])
|
89
|
+
end
|
90
|
+
|
91
|
+
@updated_blobs.each do |blob|
|
92
|
+
size = File.size(File.join(@blobs_dir, blob))
|
93
|
+
say("%s\t%s\t%s" % ["new version".yellow, blob, pretty_size(size)])
|
94
|
+
end
|
95
|
+
|
96
|
+
nl
|
97
|
+
say("When ready please run `#{"bosh upload blobs".green}'")
|
98
|
+
end
|
99
|
+
|
100
|
+
# Registers a file as BOSH blob
|
101
|
+
# @param [String] local_path Local file path
|
102
|
+
# @param [String] blob_path Blob path relative to blobs directory
|
103
|
+
# @return [void]
|
104
|
+
def add_blob(local_path, blob_path)
|
105
|
+
unless File.exists?(local_path)
|
106
|
+
err("File `#{local_path}' not found")
|
107
|
+
end
|
108
|
+
|
109
|
+
if File.directory?(local_path)
|
110
|
+
err("`#{local_path}' is a directory")
|
111
|
+
end
|
112
|
+
|
113
|
+
if blob_path[0..0] == "/"
|
114
|
+
err("Blob path should be a relative path")
|
115
|
+
end
|
116
|
+
|
117
|
+
if blob_path[0..5] == "blobs/"
|
118
|
+
err("Blob path should not start with `blobs/'")
|
119
|
+
end
|
120
|
+
|
121
|
+
blob_dst = File.join(@blobs_dir, blob_path)
|
122
|
+
|
123
|
+
if File.directory?(blob_dst)
|
124
|
+
err("`#{blob_dst}' is a directory, please pick a different path")
|
125
|
+
end
|
126
|
+
|
127
|
+
update = false
|
128
|
+
if File.exists?(blob_dst)
|
129
|
+
if file_checksum(blob_dst) == file_checksum(local_path)
|
130
|
+
err("Already tracking the same version of `#{blob_path}'")
|
131
|
+
end
|
132
|
+
update = true
|
133
|
+
FileUtils.rm(blob_dst)
|
134
|
+
end
|
135
|
+
|
136
|
+
FileUtils.mkdir_p(File.dirname(blob_dst))
|
137
|
+
FileUtils.cp(local_path, blob_dst, :preserve => true)
|
138
|
+
FileUtils.chmod(0644, blob_dst)
|
139
|
+
if update
|
140
|
+
say("Updated #{blob_path.yellow}")
|
141
|
+
else
|
142
|
+
say("Added #{blob_path.yellow}")
|
143
|
+
end
|
144
|
+
|
145
|
+
say("When you are done testing the new blob, please run\n" +
|
146
|
+
"`#{"bosh upload blobs".green}' and commit changes.")
|
147
|
+
end
|
148
|
+
|
149
|
+
# Synchronizes the contents of blobs directory with blobs index.
|
150
|
+
# @return [void]
|
151
|
+
def sync
|
152
|
+
say("Syncing blobs...")
|
153
|
+
remove_symlinks
|
154
|
+
process_blobs_directory
|
155
|
+
process_index
|
156
|
+
end
|
157
|
+
|
158
|
+
# Processes all files in blobs directory and only leaves non-symlinks.
|
159
|
+
# Marks blobs as dirty if there are any non-symlink files.
|
160
|
+
# @return [void]
|
161
|
+
def process_blobs_directory
|
162
|
+
@updated_blobs = []
|
163
|
+
@new_blobs = []
|
164
|
+
|
165
|
+
Dir[File.join(@blobs_dir, "**", "*")].each do |file|
|
166
|
+
next if File.directory?(file) || File.symlink?(file)
|
167
|
+
# We don't care about symlinks because they represent blobs
|
168
|
+
# that are already tracked.
|
169
|
+
# Regular files are more interesting: it's either a new version
|
170
|
+
# of an existing blob or a completely new blob.
|
171
|
+
path = strip_blobs_dir(file)
|
172
|
+
|
173
|
+
if File.exists?(File.join(@src_dir, path))
|
174
|
+
err("File `#{path}' is in both `blobs' and `src' directory.\n" +
|
175
|
+
"Please fix release repo before proceeding")
|
176
|
+
end
|
177
|
+
|
178
|
+
if @index.has_key?(path)
|
179
|
+
if file_checksum(file) == @index[path]["sha"]
|
180
|
+
# Already have exactly the same file in the index,
|
181
|
+
# no need to keep it around. Also handles the migration
|
182
|
+
# scenario for people with old blobs checked out.
|
183
|
+
local_path = File.join(@storage_dir, @index[path]["sha"])
|
184
|
+
if File.exists?(local_path)
|
185
|
+
FileUtils.rm_rf(file)
|
186
|
+
else
|
187
|
+
FileUtils.mv(file, local_path)
|
188
|
+
end
|
189
|
+
install_blob(local_path, path, @index[path]["sha"])
|
190
|
+
else
|
191
|
+
@updated_blobs << path
|
192
|
+
end
|
193
|
+
else
|
194
|
+
@new_blobs << path
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
# Removes all symlinks from blobs directory
|
200
|
+
# @return [void]
|
201
|
+
def remove_symlinks
|
202
|
+
Dir[File.join(@blobs_dir, "**", "*")].each do |file|
|
203
|
+
FileUtils.rm_rf(file) if File.symlink?(file)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
# Processes blobs index, fetches any missing or mismatched blobs,
|
208
|
+
# establishes symlinks in blobs directory to any files present in index.
|
209
|
+
# @return [void]
|
210
|
+
def process_index
|
211
|
+
@index.each_pair do |path, entry|
|
212
|
+
if File.exists?(File.join(@src_dir, path))
|
213
|
+
err("File `#{path}' is in both blob index and src directory.\n" +
|
214
|
+
"Please fix release repo before proceeding")
|
215
|
+
end
|
216
|
+
|
217
|
+
local_path = File.join(@storage_dir, entry["sha"])
|
218
|
+
need_download = true
|
219
|
+
|
220
|
+
if File.exists?(local_path)
|
221
|
+
checksum = file_checksum(local_path)
|
222
|
+
if checksum == entry["sha"]
|
223
|
+
need_download = false
|
224
|
+
else
|
225
|
+
progress(path, "checksum mismatch, re-downloading...\n".red)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
if need_download
|
230
|
+
local_path = download_blob(path)
|
231
|
+
end
|
232
|
+
|
233
|
+
install_blob(local_path, path, entry["sha"])
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
# Uploads blob to a blobstore, updates blobs index.
|
238
|
+
# @param [String] path Blob path relative to blobs dir
|
239
|
+
def upload_blob(path)
|
240
|
+
blob_path = File.join(@blobs_dir, path)
|
241
|
+
|
242
|
+
unless File.exists?(blob_path)
|
243
|
+
err("Cannot upload blob, local file `#{blob_path}' doesn't exist")
|
244
|
+
end
|
245
|
+
|
246
|
+
if File.symlink?(blob_path)
|
247
|
+
err("`#{blob_path}' is a symlink")
|
248
|
+
end
|
249
|
+
|
250
|
+
object_id = rand(36**9).to_s(36)
|
251
|
+
checksum = file_checksum(blob_path)
|
252
|
+
|
253
|
+
progress(path, "uploading...")
|
254
|
+
object_id = @blobstore.create(File.open(blob_path, "r"))
|
255
|
+
progress(path, "uploaded\n".green)
|
256
|
+
|
257
|
+
@index[path] = {
|
258
|
+
"object_id" => object_id,
|
259
|
+
"sha" => checksum,
|
260
|
+
"size" => File.size(blob_path)
|
261
|
+
}
|
262
|
+
|
263
|
+
update_index
|
264
|
+
install_blob(blob_path, path, checksum)
|
265
|
+
object_id
|
266
|
+
end
|
267
|
+
|
268
|
+
# Downloads blob from a blobstore
|
269
|
+
# @param [String] path Downloaded blob file path
|
270
|
+
def download_blob(path)
|
271
|
+
unless @index.has_key?(path)
|
272
|
+
err("Unknown blob path `#{path}'")
|
273
|
+
end
|
274
|
+
|
275
|
+
blob = @index[path]
|
276
|
+
size = blob["size"].to_i
|
277
|
+
tmp_file = Tempfile.new("bosh-blob")
|
278
|
+
|
279
|
+
download_label = "downloading"
|
280
|
+
if size > 0
|
281
|
+
download_label += " " + pretty_size(size)
|
282
|
+
end
|
283
|
+
|
284
|
+
progress_bar = Thread.new do
|
285
|
+
loop do
|
286
|
+
break unless size > 0
|
287
|
+
if File.exists?(tmp_file.path)
|
288
|
+
pct = 100 * File.size(tmp_file.path).to_f / size
|
289
|
+
progress(path, "#{download_label} (#{pct.to_i}%)...")
|
290
|
+
end
|
291
|
+
sleep(0.2)
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
progress(path, "#{download_label}...")
|
296
|
+
@blobstore.get(blob["object_id"], tmp_file)
|
297
|
+
tmp_file.close
|
298
|
+
progress_bar.kill
|
299
|
+
progress(path, "downloaded\n".green)
|
300
|
+
|
301
|
+
if file_checksum(tmp_file.path) != blob["sha"]
|
302
|
+
err("Checksum mismatch for downloaded blob `#{path}'")
|
303
|
+
end
|
304
|
+
|
305
|
+
tmp_file.path
|
306
|
+
end
|
307
|
+
|
308
|
+
private
|
309
|
+
|
310
|
+
# Renders blob operation progress
|
311
|
+
# @param [String] path Blob path relative to blobs dir
|
312
|
+
# @param [String] label Operation happening to a blob
|
313
|
+
def progress(path, label)
|
314
|
+
say("\r", " " * 80)
|
315
|
+
say("\r#{path.truncate(40).yellow} #{label}", "")
|
316
|
+
Bosh::Cli::Config.output.flush # Ruby 1.8 compatibility
|
317
|
+
end
|
318
|
+
|
319
|
+
# @param [String] src Path to a file containing the blob
|
320
|
+
# @param [String] dst Resulting blob path relative to blobs dir
|
321
|
+
# @param [String] checksum Blob checksum
|
322
|
+
def install_blob(src, dst, checksum)
|
323
|
+
store_path = File.join(@storage_dir, checksum)
|
324
|
+
symlink_path = File.join(@blobs_dir, dst)
|
325
|
+
|
326
|
+
FileUtils.chmod(0644, src)
|
327
|
+
|
328
|
+
unless File.exists?(store_path) && realpath(src) == realpath(store_path)
|
329
|
+
# Move blob to a storage dir if it's not there yet
|
330
|
+
FileUtils.mv(src, store_path)
|
331
|
+
end
|
332
|
+
|
333
|
+
unless File.exists?(symlink_path) && !File.symlink?(symlink_path)
|
334
|
+
FileUtils.mkdir_p(File.dirname(symlink_path))
|
335
|
+
FileUtils.rm_rf(symlink_path)
|
336
|
+
FileUtils.ln_s(store_path, symlink_path)
|
337
|
+
end
|
338
|
+
end
|
339
|
+
|
340
|
+
# Returns blob path relative to blobs dir, fails if blob is not in blobs
|
341
|
+
# dir.
|
342
|
+
# @param [String] path Absolute or relative blob path
|
343
|
+
def strip_blobs_dir(path)
|
344
|
+
blob_path = realpath(path)
|
345
|
+
blobs_dir = realpath(@blobs_dir)
|
346
|
+
|
347
|
+
if blob_path[0..blobs_dir.size] == blobs_dir + "/"
|
348
|
+
blob_path[blobs_dir.size+1..-1]
|
349
|
+
else
|
350
|
+
err("File `#{blob_path}' is not under `blobs' directory")
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
# Updates blobs index
|
355
|
+
def update_index
|
356
|
+
yaml = YAML.dump(@index).gsub(/\s*$/, "")
|
357
|
+
|
358
|
+
index_file = Tempfile.new("blob_index")
|
359
|
+
index_file.puts(yaml)
|
360
|
+
index_file.close
|
361
|
+
|
362
|
+
FileUtils.mv(index_file.path, @index_file)
|
363
|
+
end
|
364
|
+
|
365
|
+
# Returns file SHA1 checksum
|
366
|
+
# @param [String] path File path
|
367
|
+
def file_checksum(path)
|
368
|
+
Digest::SHA1.file(path).hexdigest
|
369
|
+
end
|
370
|
+
|
371
|
+
# Returns real file path (resolves symlinks)
|
372
|
+
# @param [String] path File path
|
373
|
+
def realpath(path)
|
374
|
+
Pathname.new(path).realpath.to_s
|
375
|
+
end
|
376
|
+
end
|
377
|
+
end
|
data/lib/cli/commands/base.rb
CHANGED
@@ -37,6 +37,10 @@ module Bosh::Cli
|
|
37
37
|
@release = Bosh::Cli::Release.new(@work_dir)
|
38
38
|
end
|
39
39
|
|
40
|
+
def blob_manager
|
41
|
+
@blob_manager ||= Bosh::Cli::BlobManager.new(release)
|
42
|
+
end
|
43
|
+
|
40
44
|
def blobstore
|
41
45
|
release.blobstore
|
42
46
|
end
|
@@ -166,80 +170,6 @@ module Bosh::Cli
|
|
166
170
|
URI.parse(url).to_s
|
167
171
|
end
|
168
172
|
|
169
|
-
def check_if_blobs_supported
|
170
|
-
check_if_release_dir
|
171
|
-
unless File.directory?(BLOBS_DIR)
|
172
|
-
err("Can't find blob directory '#{BLOBS_DIR}'.")
|
173
|
-
end
|
174
|
-
|
175
|
-
unless File.file?(BLOBS_INDEX_FILE)
|
176
|
-
err("Can't find '#{BLOBS_INDEX_FILE}'")
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
180
|
-
def check_dirty_blobs
|
181
|
-
if File.file?(File.join(work_dir, BLOBS_INDEX_FILE)) && blob_status != 0
|
182
|
-
err("Your 'blobs' directory is not in sync. " +
|
183
|
-
"Resolve using 'bosh sync blobs' command")
|
184
|
-
end
|
185
|
-
end
|
186
|
-
|
187
|
-
def get_blobs_index
|
188
|
-
load_yaml_file(File.join(work_dir, BLOBS_INDEX_FILE))
|
189
|
-
end
|
190
|
-
|
191
|
-
def blob_status(verbose = false)
|
192
|
-
check_if_blobs_supported
|
193
|
-
untracked = []
|
194
|
-
modified = []
|
195
|
-
tracked= []
|
196
|
-
unsynced = []
|
197
|
-
|
198
|
-
local_blobs = {}
|
199
|
-
Dir.chdir(BLOBS_DIR) do
|
200
|
-
Dir.glob("**/*").select { |entry| File.file?(entry) }.each do |file|
|
201
|
-
local_blobs[file] = Digest::SHA1.file(file).hexdigest
|
202
|
-
end
|
203
|
-
end
|
204
|
-
remote_blobs = get_blobs_index
|
205
|
-
|
206
|
-
local_blobs.each do |blob_name, blob_sha|
|
207
|
-
if remote_blobs[blob_name].nil?
|
208
|
-
untracked << blob_name
|
209
|
-
elsif blob_sha != remote_blobs[blob_name]["sha"]
|
210
|
-
modified << blob_name
|
211
|
-
else
|
212
|
-
tracked << blob_name
|
213
|
-
end
|
214
|
-
end
|
215
|
-
|
216
|
-
remote_blobs.each_key do |blob_name|
|
217
|
-
unsynced << blob_name if local_blobs[blob_name].nil?
|
218
|
-
end
|
219
|
-
|
220
|
-
changes = modified.size + untracked.size + unsynced.size
|
221
|
-
return changes unless verbose
|
222
|
-
|
223
|
-
if modified.size > 0
|
224
|
-
say("\nModified blobs ('bosh upload blob' to update): ".green)
|
225
|
-
modified.each { |blob| say(blob) }
|
226
|
-
end
|
227
|
-
|
228
|
-
if untracked.size > 0
|
229
|
-
say("\nNew blobs ('bosh upload blob' to add): ".green)
|
230
|
-
untracked.each { |blob| say(blob) }
|
231
|
-
end
|
232
|
-
|
233
|
-
if unsynced.size > 0
|
234
|
-
say("\nMissing blobs ('bosh sync blobs' to fetch) : ".green)
|
235
|
-
unsynced.each { |blob| say(blob) }
|
236
|
-
end
|
237
|
-
|
238
|
-
if changes == 0
|
239
|
-
say("\nRelease blobs are up to date".green)
|
240
|
-
end
|
241
|
-
changes
|
242
|
-
end
|
243
173
|
end
|
244
174
|
end
|
245
175
|
end
|