bosh_cli 0.16 → 0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,47 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh::Cli::Command
4
+ class BlobManagement < Base
5
+
6
+ # Prints out blobs status
7
+ def status
8
+ blob_manager.print_status
9
+ end
10
+
11
+ # Adds blob to managed blobs
12
+ # @param [String] local_path Local file path
13
+ # @param [optional, String] blob_dir Directory to store blob in, relative
14
+ # to blobs dir
15
+ def add(local_path, blob_dir = nil)
16
+ blob_path = File.basename(local_path)
17
+ if blob_dir
18
+ # We don't need about blobs prefix,
19
+ # but it might be handy for people who rely on auto-completion
20
+ if blob_dir[0..5] == "blobs/"
21
+ blob_dir = blob_dir[6..-1]
22
+ end
23
+ blob_path = File.join(blob_dir, blob_path)
24
+ end
25
+ blob_manager.add_blob(local_path, blob_path)
26
+ end
27
+
28
+ # Uploads all blobs that need to be uploaded
29
+ def upload
30
+ blob_manager.print_status
31
+
32
+ blob_manager.blobs_to_upload.each do |blob|
33
+ nl
34
+ if confirmed?("Upload blob #{blob.yellow}?")
35
+ blob_manager.upload_blob(blob)
36
+ end
37
+ end
38
+ end
39
+
40
+ # Syncs blobs with blobstore
41
+ def sync
42
+ blob_manager.sync
43
+ blob_manager.print_status
44
+ end
45
+
46
+ end
47
+ end
@@ -8,8 +8,12 @@ module Bosh::Cli::Command
8
8
  include Bosh::Cli::VersionCalc
9
9
 
10
10
  def init(base=nil, *options)
11
- flags = options.inject({}) { |h, option| h[option] = true; h }
12
- git = flags.delete("--git")
11
+ if base[0..0] == "-"
12
+ # TODO: need to add some option parsing helpers to avoid that
13
+ options.unshift(base)
14
+ base = nil
15
+ end
16
+ git = options.include?("--git")
13
17
 
14
18
  if base
15
19
  FileUtils.mkdir_p(base) unless Dir.exist?(base)
@@ -17,17 +21,15 @@ module Bosh::Cli::Command
17
21
  end
18
22
 
19
23
  err("Release already initialized") if in_release_dir?
20
-
21
24
  git_init if git
22
25
 
23
- %w[jobs packages src blobs].each do |dir|
26
+ %w[config jobs packages src blobs].each do |dir|
24
27
  FileUtils.mkdir(dir)
25
28
  end
26
29
 
27
- # initialize an empty blob_index file
28
- blobs = {}
29
- File.open("blob_index.yml", "w") do |f|
30
- YAML.dump(blobs, f)
30
+ # Initialize an empty blobs index
31
+ File.open(File.join("config", "blobs.yml"), "w") do |f|
32
+ YAML.dump({}, f)
31
33
  end
32
34
 
33
35
  say("Release directory initialized".green)
@@ -39,11 +41,23 @@ module Bosh::Cli::Command
39
41
  say("error running 'git init':\n#{out}")
40
42
  else
41
43
  File.open(".gitignore", "w") do |f|
42
- f.puts("config/dev.yml\nconfig/private.yml")
43
- f.puts("blobs\nreleases/*.tgz")
44
- f.puts("dev_releases\n.dev_builds")
45
- f.puts(".final_builds/jobs/**/*.tgz")
46
- f.puts(".final_builds/packages/**/*.tgz")
44
+ f << <<-EOS.gsub(/^\s{10}/, '')
45
+ config/dev.yml
46
+ config/private.yml
47
+ releases/*.tgz
48
+ dev_releases
49
+ .blobs
50
+ blobs
51
+ .dev_builds
52
+ .idea
53
+ .DS_Store
54
+ .final_builds/jobs/**/*.tgz
55
+ .final_builds/packages/**/*.tgz
56
+ *.swp
57
+ *~
58
+ *#
59
+ #*
60
+ EOS
47
61
  end
48
62
  end
49
63
  rescue Errno::ENOENT
@@ -202,11 +216,18 @@ module Bosh::Cli::Command
202
216
  exit(1)
203
217
  end
204
218
 
205
- if !force
206
- check_dirty_blobs
207
- check_if_dirty_state
219
+ blob_manager.sync
220
+ if blob_manager.dirty?
221
+ blob_manager.print_status
222
+ if force
223
+ say("Proceeding with dirty blobs as '--force' is given".red)
224
+ else
225
+ err("Please use '--force' or upload new blobs")
226
+ end
208
227
  end
209
228
 
229
+ check_if_dirty_state unless force
230
+
210
231
  confirmation = "Are you sure you want to " +
211
232
  "generate #{'final'.red} version? "
212
233
 
data/lib/cli/runner.rb CHANGED
@@ -537,24 +537,28 @@ module Bosh::Cli
537
537
  route :cloud_check, :perform
538
538
  end
539
539
 
540
- command :upload_blob do
541
- usage "upload blob <blobs>"
542
- desc "Upload given blob to the blobstore"
543
- option "--force", "bypass duplicate checking"
544
- route :blob, :upload_blob
540
+ command :add_blob do
541
+ usage "add blob <local_path> [<blob_dir>]"
542
+ desc "Add a local file as BOSH blob"
543
+ route :blob_management, :add
544
+ end
545
+
546
+ command :upload_blobs do
547
+ usage "upload blobs"
548
+ desc "Upload new and updated blobs to the blobstore"
549
+ route :blob_management, :upload
545
550
  end
546
551
 
547
552
  command :sync_blobs do
548
553
  usage "sync blobs"
549
554
  desc "Sync blob with the blobstore"
550
- option "--force", "overwrite all local copies with the remote blob"
551
- route :blob, :sync_blobs
555
+ route :blob_management, :sync
552
556
  end
553
557
 
554
- command :blobs do
558
+ command :blobs_status do
555
559
  usage "blobs"
556
- desc "Print blob status"
557
- route :blob, :blobs_info
560
+ desc "Print current blobs status"
561
+ route :blob_management, :status
558
562
  end
559
563
 
560
564
  def define_plugin_commands
@@ -70,6 +70,7 @@ Remote access
70
70
  <%= command_usage(:ssh_cleanup) %>
71
71
 
72
72
  Blob
73
- <%= command_usage(:upload_blob) %>
73
+ <%= command_usage(:add_blob) %>
74
+ <%= command_usage(:upload_blobs) %>
74
75
  <%= command_usage(:sync_blobs) %>
75
- <%= command_usage(:blobs) %>
76
+ <%= command_usage(:blobs_status) %>
data/lib/cli/version.rb CHANGED
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Bosh
4
4
  module Cli
5
- VERSION = "0.16"
5
+ VERSION = "0.17"
6
6
  end
7
7
  end
@@ -0,0 +1,291 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ require "spec_helper"
4
+
5
+ describe Bosh::Cli::BlobManager do
6
+
7
+ def make_manager(release)
8
+ Bosh::Cli::BlobManager.new(release)
9
+ end
10
+
11
+ before(:each) do
12
+ @blobstore = mock("blobstore")
13
+ @dir = Dir.mktmpdir
14
+ @src_dir = FileUtils.mkdir(File.join(@dir, "src"))
15
+ @config_dir = File.join(@dir, "config")
16
+ FileUtils.mkdir(@config_dir)
17
+ @blobs_dir = File.join(@dir, "blobs")
18
+ @release = mock("release", :dir => @dir, :blobstore => @blobstore)
19
+ end
20
+
21
+ describe "initialization" do
22
+ it "fails if 'src' directory is missing" do
23
+ FileUtils.rm_rf(@src_dir)
24
+ expect {
25
+ make_manager(@release)
26
+ }.to raise_error("`src' directory is missing")
27
+ end
28
+
29
+ it "fails if blobstore is not configured" do
30
+ @release.stub!(:blobstore).and_return(nil)
31
+ expect {
32
+ make_manager(@release)
33
+ }.to raise_error("Blobstore is not configured")
34
+ end
35
+
36
+ it "creates necessary directories in release dir" do
37
+ make_manager(@release)
38
+ File.directory?(File.join(@dir, "blobs")).should be_true
39
+ File.directory?(File.join(@dir, ".blobs")).should be_true
40
+ end
41
+
42
+ it "has dirty flag cleared and upload list empty" do
43
+ manager = make_manager(@release)
44
+ manager.dirty?.should be_false
45
+ manager.blobs_to_upload.should == []
46
+ end
47
+
48
+ it "doesn't like bad index file'" do
49
+ File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
50
+ YAML.dump("string", f)
51
+ end
52
+ expect {
53
+ make_manager(@release)
54
+ }.to raise_error(/Incorrect file format/)
55
+ end
56
+
57
+ it "migrates legacy index file" do
58
+ legacy_file = File.join(@release.dir, "blob_index.yml")
59
+ test_hash = { "foo" => "bar" }
60
+
61
+ File.open(legacy_file, "w") do |f|
62
+ YAML.dump({ "foo" => "bar" }, f)
63
+ end
64
+
65
+ make_manager(@release)
66
+ File.exists?(legacy_file).should be_false
67
+ YAML.load_file(File.join(@config_dir, "blobs.yml")).should == test_hash
68
+ end
69
+ end
70
+
71
+ describe "adding a blob" do
72
+ before(:each) do
73
+ @manager = make_manager(@release)
74
+ @blob = Tempfile.new("blob")
75
+ @blob.write("blob contents")
76
+ @blob.close
77
+ end
78
+
79
+ it "cannot add non-existing file" do
80
+ expect {
81
+ @manager.add_blob("tmp/foobar.tgz", "test")
82
+ }.to raise_error("File `tmp/foobar.tgz' not found")
83
+ end
84
+
85
+ it "cannot add directory" do
86
+ tmp_dir = Dir.mktmpdir
87
+ expect {
88
+ @manager.add_blob(tmp_dir, "test")
89
+ }.to raise_error("`#{tmp_dir}' is a directory")
90
+ end
91
+
92
+ it "cannot use absolute path as blob destination" do
93
+ expect {
94
+ @manager.add_blob(@blob.path, "/test")
95
+ }.to raise_error("Blob path should be a relative path")
96
+ end
97
+
98
+ it "cannot use 'blobs' prefix for blob destination" do
99
+ expect {
100
+ @manager.add_blob(@blob.path, "blobs/foo/bar")
101
+ }.to raise_error("Blob path should not start with `blobs/'")
102
+ end
103
+
104
+ it "cannot use directory as blob destination" do
105
+ foo_dir = File.join(@blobs_dir, "foo")
106
+ FileUtils.mkdir(foo_dir)
107
+ expect {
108
+ @manager.add_blob(@blob.path, "foo")
109
+ }.to raise_error("`#{foo_dir}' is a directory, " +
110
+ "please pick a different path")
111
+ end
112
+
113
+ it "adds blob to a blobs directory" do
114
+ blob_dst = File.join(@blobs_dir, "foo", "blob")
115
+ @manager.add_blob(@blob.path, "foo/blob")
116
+ File.exists?(blob_dst).should be_true
117
+ File.read(blob_dst).should == "blob contents"
118
+ File.symlink?(blob_dst).should be_false
119
+ File.stat(blob_dst).mode.to_s(8)[-4..-1].should == "0644"
120
+ File.exists?(@blob.path).should be_true # original still exists
121
+
122
+ @manager.process_blobs_directory
123
+ @manager.dirty?.should be_true
124
+ @manager.new_blobs.should == %w(foo/blob)
125
+ @manager.updated_blobs.should == []
126
+ end
127
+
128
+ it "prevents double adds of the same file" do
129
+ @manager.add_blob(@blob.path, "foo/blob")
130
+ expect {
131
+ @manager.add_blob(@blob.path, "foo/blob")
132
+ }.to raise_error(/Already tracking/)
133
+ end
134
+
135
+ it "updates blob" do
136
+ new_blob = Tempfile.new("new-blob")
137
+ new_blob.write("foobar")
138
+ new_blob.close
139
+ blob_dst = File.join(@blobs_dir, "foo", "blob")
140
+ @manager.add_blob(@blob.path, "foo/blob")
141
+ File.read(blob_dst).should == "blob contents"
142
+ @manager.add_blob(new_blob.path, "foo/blob")
143
+ File.read(blob_dst).should == "foobar"
144
+ end
145
+ end
146
+
147
+ describe "downloading a blob" do
148
+ it "cannot download blob if path is not in index" do
149
+ @manager = make_manager(@release)
150
+
151
+ expect {
152
+ @manager.download_blob("foo")
153
+ }.to raise_error(/Unknown blob path/)
154
+ end
155
+
156
+ it "downloads blob from blobstore" do
157
+ index = {
158
+ "foo" => {
159
+ "size" => "1000",
160
+ "object_id" => "deadbeef",
161
+ "sha" => Digest::SHA1.hexdigest("blob contents")
162
+ }
163
+ }
164
+
165
+ tmp_file = Tempfile.new("mock-blob")
166
+ Tempfile.stub!(:new).with("bosh-blob").and_return(tmp_file)
167
+
168
+ File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
169
+ YAML.dump(index, f)
170
+ end
171
+
172
+ @manager = make_manager(@release)
173
+ @blobstore.should_receive(:get).with("deadbeef",
174
+ an_instance_of(Tempfile)).
175
+ and_return { tmp_file.write("blob contents"); tmp_file.close }
176
+
177
+ path = @manager.download_blob("foo")
178
+ File.read(path).should == "blob contents"
179
+ end
180
+ end
181
+
182
+ describe "uploading a blob" do
183
+ before(:each) do
184
+ @manager = make_manager(@release)
185
+ end
186
+
187
+ it "needs blob path to exist" do
188
+ expect {
189
+ @manager.upload_blob("foo")
190
+ }.to raise_error(/doesn't exist/)
191
+ end
192
+
193
+ it "doesn't follow symlinks" do
194
+ FileUtils.touch(File.join(@dir, "blob"))
195
+ FileUtils.ln_s(File.join(@dir, "blob"), File.join(@blobs_dir, "foo"))
196
+ expect {
197
+ @manager.upload_blob("foo")
198
+ }.to raise_error(/is a symlink/)
199
+ end
200
+
201
+ it "uploads file to a blobstore, updates index and symlinks blob" do
202
+ new_blob = File.join(@dir, "blob")
203
+ File.open(new_blob, "w") { |f| f.write("test blob") }
204
+ @manager.add_blob(new_blob, "foo")
205
+
206
+ @blobstore.should_receive(:create).and_return("deadbeef")
207
+ @manager.upload_blob("foo").should == "deadbeef"
208
+
209
+ blob_dst = File.join(@blobs_dir, "foo")
210
+ checksum = Digest::SHA1.hexdigest("test blob")
211
+
212
+ File.symlink?(blob_dst).should be_true
213
+ File.readlink(blob_dst).should == File.join(@dir, ".blobs", checksum)
214
+ File.read(blob_dst).should == "test blob"
215
+ end
216
+ end
217
+
218
+ describe "syncing blobs" do
219
+ it "includes several steps" do
220
+ @manager = make_manager(@release)
221
+ @manager.should_receive(:remove_symlinks).ordered
222
+ @manager.should_receive(:process_blobs_directory).ordered
223
+ @manager.should_receive(:process_index).ordered
224
+ @manager.sync
225
+ end
226
+
227
+ it "processes blobs directory" do
228
+ @manager = make_manager(@release)
229
+ @blobstore.stub!(:create).and_return("new-object-id")
230
+
231
+ new_blob = Tempfile.new("new-blob")
232
+ new_blob.write("test")
233
+ new_blob.close
234
+
235
+ @manager.add_blob(new_blob.path, "foo")
236
+ @manager.process_blobs_directory
237
+ @manager.new_blobs.should == %w(foo)
238
+
239
+ @manager.add_blob(new_blob.path, "bar")
240
+ @manager.process_blobs_directory
241
+ @manager.new_blobs.sort.should == %w(bar foo)
242
+
243
+ @manager.upload_blob("bar")
244
+
245
+ new_blob.open
246
+ new_blob.write("stuff")
247
+ new_blob.close
248
+
249
+ @manager.add_blob(new_blob.path, "bar")
250
+ @manager.process_blobs_directory
251
+ @manager.new_blobs.sort.should == %w(foo)
252
+ @manager.updated_blobs.sort.should == %w(bar)
253
+ end
254
+
255
+ it "downloads missing blobs" do
256
+ index = {
257
+ "foo" => {
258
+ "size" => 1000,
259
+ "sha" => Digest::SHA1.hexdigest("foo"),
260
+ "object_id" => "da-foo"
261
+ },
262
+ "bar" => {
263
+ "size" => 500,
264
+ "sha" => Digest::SHA1.hexdigest("bar"),
265
+ "object_id" => "da-bar"
266
+ }
267
+ }
268
+
269
+ File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
270
+ YAML.dump(index, f)
271
+ end
272
+
273
+ foo = Tempfile.new("foo")
274
+ foo.write("foo")
275
+ foo.close
276
+
277
+ bar = Tempfile.new("bar")
278
+ bar.write("bar")
279
+ bar.close
280
+
281
+ @manager = make_manager(@release)
282
+ @manager.should_receive(:download_blob).with("foo").and_return(foo.path)
283
+ @manager.should_receive(:download_blob).with("bar").and_return(bar.path)
284
+
285
+ @manager.process_index
286
+
287
+ File.read(File.join(@blobs_dir, "foo")).should == "foo"
288
+ File.read(File.join(@blobs_dir, "bar")).should == "bar"
289
+ end
290
+ end
291
+ end