bosh_cli 0.16 → 0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -50,6 +50,7 @@ task "build" do
50
50
  end
51
51
 
52
52
  desc "Build and install BOSH CLI into system gems"
53
- task "install_head" do
53
+ task "install" do
54
54
  gem_helper.install_gem
55
55
  end
56
+ task :install_head => :install
data/lib/cli.rb CHANGED
@@ -64,6 +64,8 @@ require "cli/release_builder"
64
64
  require "cli/release_compiler"
65
65
  require "cli/release_tarball"
66
66
 
67
+ require "cli/blob_manager"
68
+
67
69
  require "cli/command_definition"
68
70
  require "cli/runner"
69
71
 
@@ -0,0 +1,377 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh::Cli
4
+ # In order to avoid storing large objects in git repo,
5
+ # release might save them in the blobstore instead.
6
+ # BlobManager encapsulates most of the blob operations.
7
+ class BlobManager
8
+ DEFAULT_INDEX_NAME = "blobs.yml"
9
+
10
+ attr_reader :new_blobs, :updated_blobs
11
+
12
+ # @param [Bosh::Cli::Release] release BOSH Release object
13
+ def initialize(release)
14
+ @release = release
15
+ @index_file = File.join(@release.dir, "config", DEFAULT_INDEX_NAME)
16
+
17
+ legacy_index_file = File.join(@release.dir, "blob_index.yml")
18
+
19
+ if File.exists?(legacy_index_file)
20
+ if File.exists?(@index_file)
21
+ err("Found both new and legacy blob index, please fix it")
22
+ end
23
+ FileUtils.mv(legacy_index_file, @index_file)
24
+ end
25
+
26
+ if File.file?(@index_file)
27
+ @index = load_yaml_file(@index_file)
28
+ else
29
+ @index = {}
30
+ end
31
+
32
+ @src_dir = File.join(@release.dir, "src")
33
+ unless File.directory?(@src_dir)
34
+ err("`src' directory is missing")
35
+ end
36
+
37
+ @storage_dir = File.join(@release.dir, ".blobs")
38
+ unless File.directory?(@storage_dir)
39
+ FileUtils.mkdir(@storage_dir)
40
+ end
41
+
42
+ @blobs_dir = File.join(@release.dir, "blobs")
43
+ unless File.directory?(@blobs_dir)
44
+ FileUtils.mkdir(@blobs_dir)
45
+ end
46
+
47
+ @blobstore = @release.blobstore
48
+ if @blobstore.nil?
49
+ err("Blobstore is not configured")
50
+ end
51
+
52
+ @new_blobs = []
53
+ @updated_blobs = []
54
+ end
55
+
56
+ # Returns a list of blobs that need to be uploaded
57
+ # @return [Array]
58
+ def blobs_to_upload
59
+ @new_blobs + @updated_blobs
60
+ end
61
+
62
+ # Returns whether blobs directory is dirty
63
+ # @return Boolean
64
+ def dirty?
65
+ @new_blobs.size > 0 || @updated_blobs.size > 0
66
+ end
67
+
68
+ # Prints out blobs status
69
+ # @return [void]
70
+ def print_status
71
+ total_file_size = @index.inject(0) do |total, (_, entry)|
72
+ total += entry["size"].to_i
73
+ total
74
+ end
75
+
76
+ say("Total: #{@index.size}, #{pretty_size(total_file_size)}")
77
+ process_blobs_directory
78
+
79
+ unless dirty?
80
+ say("No blobs to upload".green)
81
+ return
82
+ end
83
+
84
+ nl
85
+ say("You have some blobs that need to be uploaded:")
86
+ @new_blobs.each do |blob|
87
+ size = File.size(File.join(@blobs_dir, blob))
88
+ say("%s\t%s\t%s" % ["new".green, blob, pretty_size(size)])
89
+ end
90
+
91
+ @updated_blobs.each do |blob|
92
+ size = File.size(File.join(@blobs_dir, blob))
93
+ say("%s\t%s\t%s" % ["new version".yellow, blob, pretty_size(size)])
94
+ end
95
+
96
+ nl
97
+ say("When ready please run `#{"bosh upload blobs".green}'")
98
+ end
99
+
100
+ # Registers a file as BOSH blob
101
+ # @param [String] local_path Local file path
102
+ # @param [String] blob_path Blob path relative to blobs directory
103
+ # @return [void]
104
+ def add_blob(local_path, blob_path)
105
+ unless File.exists?(local_path)
106
+ err("File `#{local_path}' not found")
107
+ end
108
+
109
+ if File.directory?(local_path)
110
+ err("`#{local_path}' is a directory")
111
+ end
112
+
113
+ if blob_path[0..0] == "/"
114
+ err("Blob path should be a relative path")
115
+ end
116
+
117
+ if blob_path[0..5] == "blobs/"
118
+ err("Blob path should not start with `blobs/'")
119
+ end
120
+
121
+ blob_dst = File.join(@blobs_dir, blob_path)
122
+
123
+ if File.directory?(blob_dst)
124
+ err("`#{blob_dst}' is a directory, please pick a different path")
125
+ end
126
+
127
+ update = false
128
+ if File.exists?(blob_dst)
129
+ if file_checksum(blob_dst) == file_checksum(local_path)
130
+ err("Already tracking the same version of `#{blob_path}'")
131
+ end
132
+ update = true
133
+ FileUtils.rm(blob_dst)
134
+ end
135
+
136
+ FileUtils.mkdir_p(File.dirname(blob_dst))
137
+ FileUtils.cp(local_path, blob_dst, :preserve => true)
138
+ FileUtils.chmod(0644, blob_dst)
139
+ if update
140
+ say("Updated #{blob_path.yellow}")
141
+ else
142
+ say("Added #{blob_path.yellow}")
143
+ end
144
+
145
+ say("When you are done testing the new blob, please run\n" +
146
+ "`#{"bosh upload blobs".green}' and commit changes.")
147
+ end
148
+
149
+ # Synchronizes the contents of blobs directory with blobs index.
150
+ # @return [void]
151
+ def sync
152
+ say("Syncing blobs...")
153
+ remove_symlinks
154
+ process_blobs_directory
155
+ process_index
156
+ end
157
+
158
+ # Processes all files in blobs directory and only leaves non-symlinks.
159
+ # Marks blobs as dirty if there are any non-symlink files.
160
+ # @return [void]
161
+ def process_blobs_directory
162
+ @updated_blobs = []
163
+ @new_blobs = []
164
+
165
+ Dir[File.join(@blobs_dir, "**", "*")].each do |file|
166
+ next if File.directory?(file) || File.symlink?(file)
167
+ # We don't care about symlinks because they represent blobs
168
+ # that are already tracked.
169
+ # Regular files are more interesting: it's either a new version
170
+ # of an existing blob or a completely new blob.
171
+ path = strip_blobs_dir(file)
172
+
173
+ if File.exists?(File.join(@src_dir, path))
174
+ err("File `#{path}' is in both `blobs' and `src' directory.\n" +
175
+ "Please fix release repo before proceeding")
176
+ end
177
+
178
+ if @index.has_key?(path)
179
+ if file_checksum(file) == @index[path]["sha"]
180
+ # Already have exactly the same file in the index,
181
+ # no need to keep it around. Also handles the migration
182
+ # scenario for people with old blobs checked out.
183
+ local_path = File.join(@storage_dir, @index[path]["sha"])
184
+ if File.exists?(local_path)
185
+ FileUtils.rm_rf(file)
186
+ else
187
+ FileUtils.mv(file, local_path)
188
+ end
189
+ install_blob(local_path, path, @index[path]["sha"])
190
+ else
191
+ @updated_blobs << path
192
+ end
193
+ else
194
+ @new_blobs << path
195
+ end
196
+ end
197
+ end
198
+
199
+ # Removes all symlinks from blobs directory
200
+ # @return [void]
201
+ def remove_symlinks
202
+ Dir[File.join(@blobs_dir, "**", "*")].each do |file|
203
+ FileUtils.rm_rf(file) if File.symlink?(file)
204
+ end
205
+ end
206
+
207
+ # Processes blobs index, fetches any missing or mismatched blobs,
208
+ # establishes symlinks in blobs directory to any files present in index.
209
+ # @return [void]
210
+ def process_index
211
+ @index.each_pair do |path, entry|
212
+ if File.exists?(File.join(@src_dir, path))
213
+ err("File `#{path}' is in both blob index and src directory.\n" +
214
+ "Please fix release repo before proceeding")
215
+ end
216
+
217
+ local_path = File.join(@storage_dir, entry["sha"])
218
+ need_download = true
219
+
220
+ if File.exists?(local_path)
221
+ checksum = file_checksum(local_path)
222
+ if checksum == entry["sha"]
223
+ need_download = false
224
+ else
225
+ progress(path, "checksum mismatch, re-downloading...\n".red)
226
+ end
227
+ end
228
+
229
+ if need_download
230
+ local_path = download_blob(path)
231
+ end
232
+
233
+ install_blob(local_path, path, entry["sha"])
234
+ end
235
+ end
236
+
237
+ # Uploads blob to a blobstore, updates blobs index.
238
+ # @param [String] path Blob path relative to blobs dir
239
+ def upload_blob(path)
240
+ blob_path = File.join(@blobs_dir, path)
241
+
242
+ unless File.exists?(blob_path)
243
+ err("Cannot upload blob, local file `#{blob_path}' doesn't exist")
244
+ end
245
+
246
+ if File.symlink?(blob_path)
247
+ err("`#{blob_path}' is a symlink")
248
+ end
249
+
250
+ object_id = rand(36**9).to_s(36)
251
+ checksum = file_checksum(blob_path)
252
+
253
+ progress(path, "uploading...")
254
+ object_id = @blobstore.create(File.open(blob_path, "r"))
255
+ progress(path, "uploaded\n".green)
256
+
257
+ @index[path] = {
258
+ "object_id" => object_id,
259
+ "sha" => checksum,
260
+ "size" => File.size(blob_path)
261
+ }
262
+
263
+ update_index
264
+ install_blob(blob_path, path, checksum)
265
+ object_id
266
+ end
267
+
268
+ # Downloads blob from a blobstore
269
+ # @param [String] path Downloaded blob file path
270
+ def download_blob(path)
271
+ unless @index.has_key?(path)
272
+ err("Unknown blob path `#{path}'")
273
+ end
274
+
275
+ blob = @index[path]
276
+ size = blob["size"].to_i
277
+ tmp_file = Tempfile.new("bosh-blob")
278
+
279
+ download_label = "downloading"
280
+ if size > 0
281
+ download_label += " " + pretty_size(size)
282
+ end
283
+
284
+ progress_bar = Thread.new do
285
+ loop do
286
+ break unless size > 0
287
+ if File.exists?(tmp_file.path)
288
+ pct = 100 * File.size(tmp_file.path).to_f / size
289
+ progress(path, "#{download_label} (#{pct.to_i}%)...")
290
+ end
291
+ sleep(0.2)
292
+ end
293
+ end
294
+
295
+ progress(path, "#{download_label}...")
296
+ @blobstore.get(blob["object_id"], tmp_file)
297
+ tmp_file.close
298
+ progress_bar.kill
299
+ progress(path, "downloaded\n".green)
300
+
301
+ if file_checksum(tmp_file.path) != blob["sha"]
302
+ err("Checksum mismatch for downloaded blob `#{path}'")
303
+ end
304
+
305
+ tmp_file.path
306
+ end
307
+
308
+ private
309
+
310
+ # Renders blob operation progress
311
+ # @param [String] path Blob path relative to blobs dir
312
+ # @param [String] label Operation happening to a blob
313
+ def progress(path, label)
314
+ say("\r", " " * 80)
315
+ say("\r#{path.truncate(40).yellow} #{label}", "")
316
+ Bosh::Cli::Config.output.flush # Ruby 1.8 compatibility
317
+ end
318
+
319
+ # @param [String] src Path to a file containing the blob
320
+ # @param [String] dst Resulting blob path relative to blobs dir
321
+ # @param [String] checksum Blob checksum
322
+ def install_blob(src, dst, checksum)
323
+ store_path = File.join(@storage_dir, checksum)
324
+ symlink_path = File.join(@blobs_dir, dst)
325
+
326
+ FileUtils.chmod(0644, src)
327
+
328
+ unless File.exists?(store_path) && realpath(src) == realpath(store_path)
329
+ # Move blob to a storage dir if it's not there yet
330
+ FileUtils.mv(src, store_path)
331
+ end
332
+
333
+ unless File.exists?(symlink_path) && !File.symlink?(symlink_path)
334
+ FileUtils.mkdir_p(File.dirname(symlink_path))
335
+ FileUtils.rm_rf(symlink_path)
336
+ FileUtils.ln_s(store_path, symlink_path)
337
+ end
338
+ end
339
+
340
+ # Returns blob path relative to blobs dir, fails if blob is not in blobs
341
+ # dir.
342
+ # @param [String] path Absolute or relative blob path
343
+ def strip_blobs_dir(path)
344
+ blob_path = realpath(path)
345
+ blobs_dir = realpath(@blobs_dir)
346
+
347
+ if blob_path[0..blobs_dir.size] == blobs_dir + "/"
348
+ blob_path[blobs_dir.size+1..-1]
349
+ else
350
+ err("File `#{blob_path}' is not under `blobs' directory")
351
+ end
352
+ end
353
+
354
+ # Updates blobs index
355
+ def update_index
356
+ yaml = YAML.dump(@index).gsub(/\s*$/, "")
357
+
358
+ index_file = Tempfile.new("blob_index")
359
+ index_file.puts(yaml)
360
+ index_file.close
361
+
362
+ FileUtils.mv(index_file.path, @index_file)
363
+ end
364
+
365
+ # Returns file SHA1 checksum
366
+ # @param [String] path File path
367
+ def file_checksum(path)
368
+ Digest::SHA1.file(path).hexdigest
369
+ end
370
+
371
+ # Returns real file path (resolves symlinks)
372
+ # @param [String] path File path
373
+ def realpath(path)
374
+ Pathname.new(path).realpath.to_s
375
+ end
376
+ end
377
+ end
@@ -37,6 +37,10 @@ module Bosh::Cli
37
37
  @release = Bosh::Cli::Release.new(@work_dir)
38
38
  end
39
39
 
40
+ def blob_manager
41
+ @blob_manager ||= Bosh::Cli::BlobManager.new(release)
42
+ end
43
+
40
44
  def blobstore
41
45
  release.blobstore
42
46
  end
@@ -166,80 +170,6 @@ module Bosh::Cli
166
170
  URI.parse(url).to_s
167
171
  end
168
172
 
169
- def check_if_blobs_supported
170
- check_if_release_dir
171
- unless File.directory?(BLOBS_DIR)
172
- err("Can't find blob directory '#{BLOBS_DIR}'.")
173
- end
174
-
175
- unless File.file?(BLOBS_INDEX_FILE)
176
- err("Can't find '#{BLOBS_INDEX_FILE}'")
177
- end
178
- end
179
-
180
- def check_dirty_blobs
181
- if File.file?(File.join(work_dir, BLOBS_INDEX_FILE)) && blob_status != 0
182
- err("Your 'blobs' directory is not in sync. " +
183
- "Resolve using 'bosh sync blobs' command")
184
- end
185
- end
186
-
187
- def get_blobs_index
188
- load_yaml_file(File.join(work_dir, BLOBS_INDEX_FILE))
189
- end
190
-
191
- def blob_status(verbose = false)
192
- check_if_blobs_supported
193
- untracked = []
194
- modified = []
195
- tracked= []
196
- unsynced = []
197
-
198
- local_blobs = {}
199
- Dir.chdir(BLOBS_DIR) do
200
- Dir.glob("**/*").select { |entry| File.file?(entry) }.each do |file|
201
- local_blobs[file] = Digest::SHA1.file(file).hexdigest
202
- end
203
- end
204
- remote_blobs = get_blobs_index
205
-
206
- local_blobs.each do |blob_name, blob_sha|
207
- if remote_blobs[blob_name].nil?
208
- untracked << blob_name
209
- elsif blob_sha != remote_blobs[blob_name]["sha"]
210
- modified << blob_name
211
- else
212
- tracked << blob_name
213
- end
214
- end
215
-
216
- remote_blobs.each_key do |blob_name|
217
- unsynced << blob_name if local_blobs[blob_name].nil?
218
- end
219
-
220
- changes = modified.size + untracked.size + unsynced.size
221
- return changes unless verbose
222
-
223
- if modified.size > 0
224
- say("\nModified blobs ('bosh upload blob' to update): ".green)
225
- modified.each { |blob| say(blob) }
226
- end
227
-
228
- if untracked.size > 0
229
- say("\nNew blobs ('bosh upload blob' to add): ".green)
230
- untracked.each { |blob| say(blob) }
231
- end
232
-
233
- if unsynced.size > 0
234
- say("\nMissing blobs ('bosh sync blobs' to fetch) : ".green)
235
- unsynced.each { |blob| say(blob) }
236
- end
237
-
238
- if changes == 0
239
- say("\nRelease blobs are up to date".green)
240
- end
241
- changes
242
- end
243
173
  end
244
174
  end
245
175
  end