bosh_cli 0.16 → 0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +2 -1
- data/lib/cli.rb +2 -0
- data/lib/cli/blob_manager.rb +377 -0
- data/lib/cli/commands/base.rb +4 -74
- data/lib/cli/commands/blob_management.rb +47 -0
- data/lib/cli/commands/release.rb +37 -16
- data/lib/cli/runner.rb +14 -10
- data/lib/cli/templates/help_message.erb +3 -2
- data/lib/cli/version.rb +1 -1
- data/spec/unit/blob_manager_spec.rb +291 -0
- data/spec/unit/cli_commands_spec.rb +37 -170
- data/spec/unit/runner_spec.rb +5 -0
- metadata +157 -89
- data/lib/cli/commands/blob.rb +0 -125
@@ -0,0 +1,47 @@
|
|
1
|
+
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
+
|
3
|
+
module Bosh::Cli::Command
|
4
|
+
class BlobManagement < Base
|
5
|
+
|
6
|
+
# Prints out blobs status
|
7
|
+
def status
|
8
|
+
blob_manager.print_status
|
9
|
+
end
|
10
|
+
|
11
|
+
# Adds blob to managed blobs
|
12
|
+
# @param [String] local_path Local file path
|
13
|
+
# @param [optional, String] blob_dir Directory to store blob in, relative
|
14
|
+
# to blobs dir
|
15
|
+
def add(local_path, blob_dir = nil)
|
16
|
+
blob_path = File.basename(local_path)
|
17
|
+
if blob_dir
|
18
|
+
# We don't need about blobs prefix,
|
19
|
+
# but it might be handy for people who rely on auto-completion
|
20
|
+
if blob_dir[0..5] == "blobs/"
|
21
|
+
blob_dir = blob_dir[6..-1]
|
22
|
+
end
|
23
|
+
blob_path = File.join(blob_dir, blob_path)
|
24
|
+
end
|
25
|
+
blob_manager.add_blob(local_path, blob_path)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Uploads all blobs that need to be uploaded
|
29
|
+
def upload
|
30
|
+
blob_manager.print_status
|
31
|
+
|
32
|
+
blob_manager.blobs_to_upload.each do |blob|
|
33
|
+
nl
|
34
|
+
if confirmed?("Upload blob #{blob.yellow}?")
|
35
|
+
blob_manager.upload_blob(blob)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# Syncs blobs with blobstore
|
41
|
+
def sync
|
42
|
+
blob_manager.sync
|
43
|
+
blob_manager.print_status
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
data/lib/cli/commands/release.rb
CHANGED
@@ -8,8 +8,12 @@ module Bosh::Cli::Command
|
|
8
8
|
include Bosh::Cli::VersionCalc
|
9
9
|
|
10
10
|
def init(base=nil, *options)
|
11
|
-
|
12
|
-
|
11
|
+
if base[0..0] == "-"
|
12
|
+
# TODO: need to add some option parsing helpers to avoid that
|
13
|
+
options.unshift(base)
|
14
|
+
base = nil
|
15
|
+
end
|
16
|
+
git = options.include?("--git")
|
13
17
|
|
14
18
|
if base
|
15
19
|
FileUtils.mkdir_p(base) unless Dir.exist?(base)
|
@@ -17,17 +21,15 @@ module Bosh::Cli::Command
|
|
17
21
|
end
|
18
22
|
|
19
23
|
err("Release already initialized") if in_release_dir?
|
20
|
-
|
21
24
|
git_init if git
|
22
25
|
|
23
|
-
%w[jobs packages src blobs].each do |dir|
|
26
|
+
%w[config jobs packages src blobs].each do |dir|
|
24
27
|
FileUtils.mkdir(dir)
|
25
28
|
end
|
26
29
|
|
27
|
-
#
|
28
|
-
blobs
|
29
|
-
|
30
|
-
YAML.dump(blobs, f)
|
30
|
+
# Initialize an empty blobs index
|
31
|
+
File.open(File.join("config", "blobs.yml"), "w") do |f|
|
32
|
+
YAML.dump({}, f)
|
31
33
|
end
|
32
34
|
|
33
35
|
say("Release directory initialized".green)
|
@@ -39,11 +41,23 @@ module Bosh::Cli::Command
|
|
39
41
|
say("error running 'git init':\n#{out}")
|
40
42
|
else
|
41
43
|
File.open(".gitignore", "w") do |f|
|
42
|
-
f.
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
44
|
+
f << <<-EOS.gsub(/^\s{10}/, '')
|
45
|
+
config/dev.yml
|
46
|
+
config/private.yml
|
47
|
+
releases/*.tgz
|
48
|
+
dev_releases
|
49
|
+
.blobs
|
50
|
+
blobs
|
51
|
+
.dev_builds
|
52
|
+
.idea
|
53
|
+
.DS_Store
|
54
|
+
.final_builds/jobs/**/*.tgz
|
55
|
+
.final_builds/packages/**/*.tgz
|
56
|
+
*.swp
|
57
|
+
*~
|
58
|
+
*#
|
59
|
+
#*
|
60
|
+
EOS
|
47
61
|
end
|
48
62
|
end
|
49
63
|
rescue Errno::ENOENT
|
@@ -202,11 +216,18 @@ module Bosh::Cli::Command
|
|
202
216
|
exit(1)
|
203
217
|
end
|
204
218
|
|
205
|
-
|
206
|
-
|
207
|
-
|
219
|
+
blob_manager.sync
|
220
|
+
if blob_manager.dirty?
|
221
|
+
blob_manager.print_status
|
222
|
+
if force
|
223
|
+
say("Proceeding with dirty blobs as '--force' is given".red)
|
224
|
+
else
|
225
|
+
err("Please use '--force' or upload new blobs")
|
226
|
+
end
|
208
227
|
end
|
209
228
|
|
229
|
+
check_if_dirty_state unless force
|
230
|
+
|
210
231
|
confirmation = "Are you sure you want to " +
|
211
232
|
"generate #{'final'.red} version? "
|
212
233
|
|
data/lib/cli/runner.rb
CHANGED
@@ -537,24 +537,28 @@ module Bosh::Cli
|
|
537
537
|
route :cloud_check, :perform
|
538
538
|
end
|
539
539
|
|
540
|
-
command :
|
541
|
-
usage "
|
542
|
-
desc "
|
543
|
-
|
544
|
-
|
540
|
+
command :add_blob do
|
541
|
+
usage "add blob <local_path> [<blob_dir>]"
|
542
|
+
desc "Add a local file as BOSH blob"
|
543
|
+
route :blob_management, :add
|
544
|
+
end
|
545
|
+
|
546
|
+
command :upload_blobs do
|
547
|
+
usage "upload blobs"
|
548
|
+
desc "Upload new and updated blobs to the blobstore"
|
549
|
+
route :blob_management, :upload
|
545
550
|
end
|
546
551
|
|
547
552
|
command :sync_blobs do
|
548
553
|
usage "sync blobs"
|
549
554
|
desc "Sync blob with the blobstore"
|
550
|
-
|
551
|
-
route :blob, :sync_blobs
|
555
|
+
route :blob_management, :sync
|
552
556
|
end
|
553
557
|
|
554
|
-
command :
|
558
|
+
command :blobs_status do
|
555
559
|
usage "blobs"
|
556
|
-
desc "Print
|
557
|
-
route :
|
560
|
+
desc "Print current blobs status"
|
561
|
+
route :blob_management, :status
|
558
562
|
end
|
559
563
|
|
560
564
|
def define_plugin_commands
|
@@ -70,6 +70,7 @@ Remote access
|
|
70
70
|
<%= command_usage(:ssh_cleanup) %>
|
71
71
|
|
72
72
|
Blob
|
73
|
-
<%= command_usage(:
|
73
|
+
<%= command_usage(:add_blob) %>
|
74
|
+
<%= command_usage(:upload_blobs) %>
|
74
75
|
<%= command_usage(:sync_blobs) %>
|
75
|
-
<%= command_usage(:
|
76
|
+
<%= command_usage(:blobs_status) %>
|
data/lib/cli/version.rb
CHANGED
@@ -0,0 +1,291 @@
|
|
1
|
+
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
+
|
3
|
+
require "spec_helper"
|
4
|
+
|
5
|
+
describe Bosh::Cli::BlobManager do
|
6
|
+
|
7
|
+
def make_manager(release)
|
8
|
+
Bosh::Cli::BlobManager.new(release)
|
9
|
+
end
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
@blobstore = mock("blobstore")
|
13
|
+
@dir = Dir.mktmpdir
|
14
|
+
@src_dir = FileUtils.mkdir(File.join(@dir, "src"))
|
15
|
+
@config_dir = File.join(@dir, "config")
|
16
|
+
FileUtils.mkdir(@config_dir)
|
17
|
+
@blobs_dir = File.join(@dir, "blobs")
|
18
|
+
@release = mock("release", :dir => @dir, :blobstore => @blobstore)
|
19
|
+
end
|
20
|
+
|
21
|
+
describe "initialization" do
|
22
|
+
it "fails if 'src' directory is missing" do
|
23
|
+
FileUtils.rm_rf(@src_dir)
|
24
|
+
expect {
|
25
|
+
make_manager(@release)
|
26
|
+
}.to raise_error("`src' directory is missing")
|
27
|
+
end
|
28
|
+
|
29
|
+
it "fails if blobstore is not configured" do
|
30
|
+
@release.stub!(:blobstore).and_return(nil)
|
31
|
+
expect {
|
32
|
+
make_manager(@release)
|
33
|
+
}.to raise_error("Blobstore is not configured")
|
34
|
+
end
|
35
|
+
|
36
|
+
it "creates necessary directories in release dir" do
|
37
|
+
make_manager(@release)
|
38
|
+
File.directory?(File.join(@dir, "blobs")).should be_true
|
39
|
+
File.directory?(File.join(@dir, ".blobs")).should be_true
|
40
|
+
end
|
41
|
+
|
42
|
+
it "has dirty flag cleared and upload list empty" do
|
43
|
+
manager = make_manager(@release)
|
44
|
+
manager.dirty?.should be_false
|
45
|
+
manager.blobs_to_upload.should == []
|
46
|
+
end
|
47
|
+
|
48
|
+
it "doesn't like bad index file'" do
|
49
|
+
File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
|
50
|
+
YAML.dump("string", f)
|
51
|
+
end
|
52
|
+
expect {
|
53
|
+
make_manager(@release)
|
54
|
+
}.to raise_error(/Incorrect file format/)
|
55
|
+
end
|
56
|
+
|
57
|
+
it "migrates legacy index file" do
|
58
|
+
legacy_file = File.join(@release.dir, "blob_index.yml")
|
59
|
+
test_hash = { "foo" => "bar" }
|
60
|
+
|
61
|
+
File.open(legacy_file, "w") do |f|
|
62
|
+
YAML.dump({ "foo" => "bar" }, f)
|
63
|
+
end
|
64
|
+
|
65
|
+
make_manager(@release)
|
66
|
+
File.exists?(legacy_file).should be_false
|
67
|
+
YAML.load_file(File.join(@config_dir, "blobs.yml")).should == test_hash
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe "adding a blob" do
|
72
|
+
before(:each) do
|
73
|
+
@manager = make_manager(@release)
|
74
|
+
@blob = Tempfile.new("blob")
|
75
|
+
@blob.write("blob contents")
|
76
|
+
@blob.close
|
77
|
+
end
|
78
|
+
|
79
|
+
it "cannot add non-existing file" do
|
80
|
+
expect {
|
81
|
+
@manager.add_blob("tmp/foobar.tgz", "test")
|
82
|
+
}.to raise_error("File `tmp/foobar.tgz' not found")
|
83
|
+
end
|
84
|
+
|
85
|
+
it "cannot add directory" do
|
86
|
+
tmp_dir = Dir.mktmpdir
|
87
|
+
expect {
|
88
|
+
@manager.add_blob(tmp_dir, "test")
|
89
|
+
}.to raise_error("`#{tmp_dir}' is a directory")
|
90
|
+
end
|
91
|
+
|
92
|
+
it "cannot use absolute path as blob destination" do
|
93
|
+
expect {
|
94
|
+
@manager.add_blob(@blob.path, "/test")
|
95
|
+
}.to raise_error("Blob path should be a relative path")
|
96
|
+
end
|
97
|
+
|
98
|
+
it "cannot use 'blobs' prefix for blob destination" do
|
99
|
+
expect {
|
100
|
+
@manager.add_blob(@blob.path, "blobs/foo/bar")
|
101
|
+
}.to raise_error("Blob path should not start with `blobs/'")
|
102
|
+
end
|
103
|
+
|
104
|
+
it "cannot use directory as blob destination" do
|
105
|
+
foo_dir = File.join(@blobs_dir, "foo")
|
106
|
+
FileUtils.mkdir(foo_dir)
|
107
|
+
expect {
|
108
|
+
@manager.add_blob(@blob.path, "foo")
|
109
|
+
}.to raise_error("`#{foo_dir}' is a directory, " +
|
110
|
+
"please pick a different path")
|
111
|
+
end
|
112
|
+
|
113
|
+
it "adds blob to a blobs directory" do
|
114
|
+
blob_dst = File.join(@blobs_dir, "foo", "blob")
|
115
|
+
@manager.add_blob(@blob.path, "foo/blob")
|
116
|
+
File.exists?(blob_dst).should be_true
|
117
|
+
File.read(blob_dst).should == "blob contents"
|
118
|
+
File.symlink?(blob_dst).should be_false
|
119
|
+
File.stat(blob_dst).mode.to_s(8)[-4..-1].should == "0644"
|
120
|
+
File.exists?(@blob.path).should be_true # original still exists
|
121
|
+
|
122
|
+
@manager.process_blobs_directory
|
123
|
+
@manager.dirty?.should be_true
|
124
|
+
@manager.new_blobs.should == %w(foo/blob)
|
125
|
+
@manager.updated_blobs.should == []
|
126
|
+
end
|
127
|
+
|
128
|
+
it "prevents double adds of the same file" do
|
129
|
+
@manager.add_blob(@blob.path, "foo/blob")
|
130
|
+
expect {
|
131
|
+
@manager.add_blob(@blob.path, "foo/blob")
|
132
|
+
}.to raise_error(/Already tracking/)
|
133
|
+
end
|
134
|
+
|
135
|
+
it "updates blob" do
|
136
|
+
new_blob = Tempfile.new("new-blob")
|
137
|
+
new_blob.write("foobar")
|
138
|
+
new_blob.close
|
139
|
+
blob_dst = File.join(@blobs_dir, "foo", "blob")
|
140
|
+
@manager.add_blob(@blob.path, "foo/blob")
|
141
|
+
File.read(blob_dst).should == "blob contents"
|
142
|
+
@manager.add_blob(new_blob.path, "foo/blob")
|
143
|
+
File.read(blob_dst).should == "foobar"
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
describe "downloading a blob" do
|
148
|
+
it "cannot download blob if path is not in index" do
|
149
|
+
@manager = make_manager(@release)
|
150
|
+
|
151
|
+
expect {
|
152
|
+
@manager.download_blob("foo")
|
153
|
+
}.to raise_error(/Unknown blob path/)
|
154
|
+
end
|
155
|
+
|
156
|
+
it "downloads blob from blobstore" do
|
157
|
+
index = {
|
158
|
+
"foo" => {
|
159
|
+
"size" => "1000",
|
160
|
+
"object_id" => "deadbeef",
|
161
|
+
"sha" => Digest::SHA1.hexdigest("blob contents")
|
162
|
+
}
|
163
|
+
}
|
164
|
+
|
165
|
+
tmp_file = Tempfile.new("mock-blob")
|
166
|
+
Tempfile.stub!(:new).with("bosh-blob").and_return(tmp_file)
|
167
|
+
|
168
|
+
File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
|
169
|
+
YAML.dump(index, f)
|
170
|
+
end
|
171
|
+
|
172
|
+
@manager = make_manager(@release)
|
173
|
+
@blobstore.should_receive(:get).with("deadbeef",
|
174
|
+
an_instance_of(Tempfile)).
|
175
|
+
and_return { tmp_file.write("blob contents"); tmp_file.close }
|
176
|
+
|
177
|
+
path = @manager.download_blob("foo")
|
178
|
+
File.read(path).should == "blob contents"
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
describe "uploading a blob" do
|
183
|
+
before(:each) do
|
184
|
+
@manager = make_manager(@release)
|
185
|
+
end
|
186
|
+
|
187
|
+
it "needs blob path to exist" do
|
188
|
+
expect {
|
189
|
+
@manager.upload_blob("foo")
|
190
|
+
}.to raise_error(/doesn't exist/)
|
191
|
+
end
|
192
|
+
|
193
|
+
it "doesn't follow symlinks" do
|
194
|
+
FileUtils.touch(File.join(@dir, "blob"))
|
195
|
+
FileUtils.ln_s(File.join(@dir, "blob"), File.join(@blobs_dir, "foo"))
|
196
|
+
expect {
|
197
|
+
@manager.upload_blob("foo")
|
198
|
+
}.to raise_error(/is a symlink/)
|
199
|
+
end
|
200
|
+
|
201
|
+
it "uploads file to a blobstore, updates index and symlinks blob" do
|
202
|
+
new_blob = File.join(@dir, "blob")
|
203
|
+
File.open(new_blob, "w") { |f| f.write("test blob") }
|
204
|
+
@manager.add_blob(new_blob, "foo")
|
205
|
+
|
206
|
+
@blobstore.should_receive(:create).and_return("deadbeef")
|
207
|
+
@manager.upload_blob("foo").should == "deadbeef"
|
208
|
+
|
209
|
+
blob_dst = File.join(@blobs_dir, "foo")
|
210
|
+
checksum = Digest::SHA1.hexdigest("test blob")
|
211
|
+
|
212
|
+
File.symlink?(blob_dst).should be_true
|
213
|
+
File.readlink(blob_dst).should == File.join(@dir, ".blobs", checksum)
|
214
|
+
File.read(blob_dst).should == "test blob"
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
describe "syncing blobs" do
|
219
|
+
it "includes several steps" do
|
220
|
+
@manager = make_manager(@release)
|
221
|
+
@manager.should_receive(:remove_symlinks).ordered
|
222
|
+
@manager.should_receive(:process_blobs_directory).ordered
|
223
|
+
@manager.should_receive(:process_index).ordered
|
224
|
+
@manager.sync
|
225
|
+
end
|
226
|
+
|
227
|
+
it "processes blobs directory" do
|
228
|
+
@manager = make_manager(@release)
|
229
|
+
@blobstore.stub!(:create).and_return("new-object-id")
|
230
|
+
|
231
|
+
new_blob = Tempfile.new("new-blob")
|
232
|
+
new_blob.write("test")
|
233
|
+
new_blob.close
|
234
|
+
|
235
|
+
@manager.add_blob(new_blob.path, "foo")
|
236
|
+
@manager.process_blobs_directory
|
237
|
+
@manager.new_blobs.should == %w(foo)
|
238
|
+
|
239
|
+
@manager.add_blob(new_blob.path, "bar")
|
240
|
+
@manager.process_blobs_directory
|
241
|
+
@manager.new_blobs.sort.should == %w(bar foo)
|
242
|
+
|
243
|
+
@manager.upload_blob("bar")
|
244
|
+
|
245
|
+
new_blob.open
|
246
|
+
new_blob.write("stuff")
|
247
|
+
new_blob.close
|
248
|
+
|
249
|
+
@manager.add_blob(new_blob.path, "bar")
|
250
|
+
@manager.process_blobs_directory
|
251
|
+
@manager.new_blobs.sort.should == %w(foo)
|
252
|
+
@manager.updated_blobs.sort.should == %w(bar)
|
253
|
+
end
|
254
|
+
|
255
|
+
it "downloads missing blobs" do
|
256
|
+
index = {
|
257
|
+
"foo" => {
|
258
|
+
"size" => 1000,
|
259
|
+
"sha" => Digest::SHA1.hexdigest("foo"),
|
260
|
+
"object_id" => "da-foo"
|
261
|
+
},
|
262
|
+
"bar" => {
|
263
|
+
"size" => 500,
|
264
|
+
"sha" => Digest::SHA1.hexdigest("bar"),
|
265
|
+
"object_id" => "da-bar"
|
266
|
+
}
|
267
|
+
}
|
268
|
+
|
269
|
+
File.open(File.join(@config_dir, "blobs.yml"), "w") do |f|
|
270
|
+
YAML.dump(index, f)
|
271
|
+
end
|
272
|
+
|
273
|
+
foo = Tempfile.new("foo")
|
274
|
+
foo.write("foo")
|
275
|
+
foo.close
|
276
|
+
|
277
|
+
bar = Tempfile.new("bar")
|
278
|
+
bar.write("bar")
|
279
|
+
bar.close
|
280
|
+
|
281
|
+
@manager = make_manager(@release)
|
282
|
+
@manager.should_receive(:download_blob).with("foo").and_return(foo.path)
|
283
|
+
@manager.should_receive(:download_blob).with("bar").and_return(bar.path)
|
284
|
+
|
285
|
+
@manager.process_index
|
286
|
+
|
287
|
+
File.read(File.join(@blobs_dir, "foo")).should == "foo"
|
288
|
+
File.read(File.join(@blobs_dir, "bar")).should == "bar"
|
289
|
+
end
|
290
|
+
end
|
291
|
+
end
|