bosh_cli 0.19.6 → 1.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. data/bin/bosh +3 -0
  2. data/lib/cli.rb +15 -5
  3. data/lib/cli/{commands/base.rb → base_command.rb} +38 -44
  4. data/lib/cli/command_discovery.rb +40 -0
  5. data/lib/cli/command_handler.rb +135 -0
  6. data/lib/cli/commands/biff.rb +16 -12
  7. data/lib/cli/commands/blob_management.rb +10 -3
  8. data/lib/cli/commands/cloudcheck.rb +13 -11
  9. data/lib/cli/commands/complete.rb +29 -0
  10. data/lib/cli/commands/deployment.rb +137 -28
  11. data/lib/cli/commands/help.rb +96 -0
  12. data/lib/cli/commands/job.rb +4 -1
  13. data/lib/cli/commands/job_management.rb +36 -23
  14. data/lib/cli/commands/job_rename.rb +11 -12
  15. data/lib/cli/commands/log_management.rb +28 -32
  16. data/lib/cli/commands/maintenance.rb +6 -1
  17. data/lib/cli/commands/misc.rb +129 -87
  18. data/lib/cli/commands/package.rb +6 -65
  19. data/lib/cli/commands/property_management.rb +20 -8
  20. data/lib/cli/commands/release.rb +211 -206
  21. data/lib/cli/commands/ssh.rb +178 -188
  22. data/lib/cli/commands/stemcell.rb +114 -51
  23. data/lib/cli/commands/task.rb +74 -56
  24. data/lib/cli/commands/user.rb +6 -3
  25. data/lib/cli/commands/vms.rb +17 -15
  26. data/lib/cli/config.rb +27 -1
  27. data/lib/cli/core_ext.rb +27 -1
  28. data/lib/cli/deployment_helper.rb +47 -0
  29. data/lib/cli/director.rb +18 -9
  30. data/lib/cli/errors.rb +6 -0
  31. data/lib/cli/job_builder.rb +75 -23
  32. data/lib/cli/job_property_collection.rb +87 -0
  33. data/lib/cli/job_property_validator.rb +130 -0
  34. data/lib/cli/package_builder.rb +32 -5
  35. data/lib/cli/release.rb +2 -0
  36. data/lib/cli/release_builder.rb +9 -13
  37. data/lib/cli/release_compiler.rb +5 -34
  38. data/lib/cli/release_tarball.rb +4 -19
  39. data/lib/cli/runner.rb +118 -694
  40. data/lib/cli/version.rb +1 -1
  41. data/spec/assets/config/swift-hp/config/final.yml +6 -0
  42. data/spec/assets/config/swift-hp/config/private.yml +7 -0
  43. data/spec/assets/config/swift-rackspace/config/final.yml +6 -0
  44. data/spec/assets/config/swift-rackspace/config/private.yml +6 -0
  45. data/spec/spec_helper.rb +0 -5
  46. data/spec/unit/base_command_spec.rb +32 -37
  47. data/spec/unit/biff_spec.rb +11 -10
  48. data/spec/unit/cli_commands_spec.rb +96 -88
  49. data/spec/unit/core_ext_spec.rb +1 -1
  50. data/spec/unit/deployment_manifest_spec.rb +36 -0
  51. data/spec/unit/director_spec.rb +17 -3
  52. data/spec/unit/job_builder_spec.rb +2 -2
  53. data/spec/unit/job_property_collection_spec.rb +111 -0
  54. data/spec/unit/job_property_validator_spec.rb +7 -0
  55. data/spec/unit/job_rename_spec.rb +7 -6
  56. data/spec/unit/package_builder_spec.rb +2 -2
  57. data/spec/unit/release_builder_spec.rb +33 -0
  58. data/spec/unit/release_spec.rb +54 -0
  59. data/spec/unit/release_tarball_spec.rb +2 -7
  60. data/spec/unit/runner_spec.rb +1 -151
  61. data/spec/unit/ssh_spec.rb +15 -9
  62. metadata +41 -12
  63. data/lib/cli/command_definition.rb +0 -52
  64. data/lib/cli/templates/help_message.erb +0 -80
data/lib/cli/errors.rb CHANGED
@@ -16,6 +16,8 @@ module Bosh::Cli
16
16
  def self.exit_code(code = nil)
17
17
  define_method(:exit_code) { code }
18
18
  end
19
+
20
+ error_code(42)
19
21
  end
20
22
 
21
23
  class UnknownCommand < CliError; error_code(100); end
@@ -45,4 +47,8 @@ module Bosh::Cli
45
47
  class UndefinedProperty < CliError; error_code(509); end
46
48
  class MalformedManifest < CliError; error_code(511); end
47
49
  class MissingTarget < CliError; error_code(512); end
50
+ class InvalidProperty < CliError; error_code(513); end
51
+ class InvalidManifest < CliError; error_code(514); end
52
+ class PropertyMismatch < CliError; error_code(515); end
53
+ class InvalidPropertyMapping < CliError; error_code(516); end
48
54
  end
@@ -7,6 +7,9 @@ module Bosh::Cli
7
7
  attr_reader :name, :version, :packages, :templates,
8
8
  :release_dir, :built_packages, :tarball_path
9
9
 
10
+ # @return [Hash] Properties defined in this job
11
+ attr_reader :properties
12
+
10
13
  def self.run_prepare_script(script_path)
11
14
  unless File.exists?(script_path)
12
15
  raise InvalidJob, "Prepare script at `#{script_path}' doesn't exist"
@@ -27,42 +30,88 @@ module Bosh::Cli
27
30
  # with CLI itself
28
31
  %w{ BUNDLE_GEMFILE RUBYOPT }.each { |key| ENV.delete(key) }
29
32
 
33
+ output = nil
30
34
  Dir.chdir(script_dir) do
31
35
  cmd = "./#{script_name} 2>&1"
32
- say("Running #{cmd}...")
33
- script_output = `#{cmd}`
34
- script_output.split("\n").each do |line|
35
- say("> #{line}")
36
- end
36
+ output = `#{cmd}`
37
37
  end
38
38
 
39
39
  unless $?.exitstatus == 0
40
40
  raise InvalidJob, "`#{script_path}' script failed"
41
41
  end
42
+
43
+ output
42
44
  ensure
43
45
  ENV.each_pair { |k, v| ENV[k] = old_env[k] }
44
46
  end
45
47
  end
46
48
 
49
+ # @param [String] directory Release directory
50
+ # @param [Hash] options Build options
51
+ def self.discover(directory, options = {})
52
+ builders = []
53
+
54
+ Dir[File.join(directory, "jobs", "*")].each do |job_dir|
55
+ next unless File.directory?(job_dir)
56
+ job_dirname = File.basename(job_dir)
57
+
58
+ prepare_script = File.join(job_dir, "prepare")
59
+ if File.exists?(prepare_script)
60
+ run_prepare_script(prepare_script)
61
+ end
62
+
63
+ job_spec = load_yaml_file(File.join(job_dir, "spec"))
64
+ if job_spec["name"] != job_dirname
65
+ raise InvalidJob,
66
+ "Found `#{job_spec["name"]}' job in " +
67
+ "`#{job_dirname}' directory, please fix it"
68
+ end
69
+
70
+ final = options[:final]
71
+ dry_run = options[:dry_run]
72
+ blobstore = options[:blobstore]
73
+ package_names = options[:package_names]
74
+
75
+ builder = new(job_spec, directory, final, blobstore, package_names)
76
+ builder.dry_run = true if dry_run
77
+ builders << builder
78
+ end
79
+
80
+ builders
81
+ end
82
+
47
83
  def initialize(spec, release_dir, final, blobstore, built_packages = [])
48
84
  spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
49
85
 
50
- @name = spec["name"]
51
- @packages = spec["packages"].to_a
86
+ @name = spec["name"]
87
+ @version = nil
88
+ @tarball_path = nil
89
+ @packages = spec["packages"].to_a
52
90
  @built_packages = built_packages.to_a
53
- @release_dir = release_dir
54
- @templates_dir = File.join(job_dir, "templates")
55
- @tarballs_dir = File.join(release_dir, "tmp", "jobs")
56
- @final = final
57
- @blobstore = blobstore
58
- @artefact_type = "job"
91
+ @release_dir = release_dir
92
+ @templates_dir = File.join(job_dir, "templates")
93
+ @tarballs_dir = File.join(release_dir, "tmp", "jobs")
94
+ @final = final
95
+ @blobstore = blobstore
96
+ @artefact_type = "job"
59
97
 
60
98
  case spec["templates"]
61
99
  when Hash
62
100
  @templates = spec["templates"].keys
63
101
  else
64
102
  raise InvalidJob, "Incorrect templates section in `#{@name}' " +
65
- "job spec (should resolve to a hash)"
103
+ "job spec (Hash expected, #{spec["properties"].class} given)"
104
+ end
105
+
106
+ if spec.has_key?("properties")
107
+ if spec["properties"].is_a?(Hash)
108
+ @properties = spec["properties"]
109
+ else
110
+ raise InvalidJob, "Incorrect properties section in `#{@name}' " +
111
+ "job spec (Hash expected, #{spec["properties"].class} given)"
112
+ end
113
+ else
114
+ @properties = {}
66
115
  end
67
116
 
68
117
  if @name.blank?
@@ -109,8 +158,6 @@ module Bosh::Cli
109
158
  FileUtils.mkdir_p(@dev_builds_dir)
110
159
  FileUtils.mkdir_p(@final_builds_dir)
111
160
 
112
- at_exit { FileUtils.rm_rf(build_dir) }
113
-
114
161
  init_indices
115
162
  end
116
163
 
@@ -139,7 +186,7 @@ module Bosh::Cli
139
186
  end
140
187
 
141
188
  def prepare_files
142
- preparation_script = File.join(job_dir, "prepare")
189
+ File.join(job_dir, "prepare")
143
190
  end
144
191
 
145
192
  def build_dir
@@ -176,17 +223,22 @@ module Bosh::Cli
176
223
  self
177
224
  end
178
225
 
226
+ # @return [Array<String>] Returns full paths of all templates in the job
227
+ # (regular job templates and monit)
228
+ def all_templates
229
+ regular_templates = @templates.map do |template|
230
+ File.join(@templates_dir, template)
231
+ end
232
+
233
+ regular_templates.sort + monit_files
234
+ end
235
+
179
236
  private
180
237
 
181
238
  def make_fingerprint
182
239
  contents = ""
183
240
 
184
- # templates, monit, spec
185
- files = templates.map do |template|
186
- File.join(@templates_dir, template)
187
- end.sort
188
-
189
- files += monit_files
241
+ files = all_templates
190
242
  files << File.join(job_dir, "spec")
191
243
 
192
244
  files.each do |filename|
@@ -0,0 +1,87 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh::Cli
4
+ class JobPropertyCollection
5
+ include Enumerable
6
+ include Bosh::Common::PropertyHelper
7
+
8
+ # @param [JobBuilder] job_builder Which job this property collection is for
9
+ # @param [Hash] global_properties Globally defined properties
10
+ # @param [Hash] job_properties Properties defined for this job only
11
+ # @param [Hash] mappings Property mappings for this job
12
+ def initialize(job_builder, global_properties, job_properties, mappings)
13
+ @job_builder = job_builder
14
+
15
+ @job_properties = deep_copy(job_properties || {})
16
+ merge(@job_properties, deep_copy(global_properties))
17
+
18
+ @mappings = mappings || {}
19
+ @properties = []
20
+
21
+ resolve_mappings
22
+ filter_properties
23
+ end
24
+
25
+ def each
26
+ @properties.each { |property| yield property }
27
+ end
28
+
29
+ # @return [Hash] Property hash (keys are property name components)
30
+ def to_hash
31
+ @properties
32
+ end
33
+
34
+ private
35
+
36
+ def resolve_mappings
37
+ @mappings.each_pair do |to, from|
38
+ resolved = lookup_property(@job_properties, from)
39
+
40
+ if resolved.nil?
41
+ raise InvalidPropertyMapping,
42
+ "Cannot satisfy property mapping `#{to}: #{from}', " +
43
+ "as `#{from}' is not in deployment properties"
44
+ end
45
+
46
+ @job_properties[to] = resolved
47
+ end
48
+ end
49
+
50
+ # @return [void] Modifies @properties
51
+ def filter_properties
52
+ if @job_builder.properties.empty?
53
+ # If at least one template doesn't have properties defined, we
54
+ # need all properties to be available to job (backward-compatibility)
55
+ @properties = @job_properties
56
+ return
57
+ end
58
+
59
+ @properties = {}
60
+
61
+ @job_builder.properties.each_pair do |name, definition|
62
+ copy_property(
63
+ @properties, @job_properties, name, definition["default"])
64
+ end
65
+ end
66
+
67
+ # @param [Object] object Serializable object
68
+ # @return [Object] Deep copy of the object
69
+ def deep_copy(object)
70
+ Marshal.load(Marshal.dump(object))
71
+ end
72
+
73
+ # @param [Hash] base
74
+ # @param [Hash] extras
75
+ # @return [void] Modifies base
76
+ def merge(base, extras)
77
+ base.merge!(extras) do |_, old_value, new_value|
78
+ if old_value.is_a?(Hash) && new_value.is_a?(Hash)
79
+ merge(old_value, new_value)
80
+ end
81
+
82
+ old_value
83
+ end
84
+ end
85
+
86
+ end
87
+ end
@@ -0,0 +1,130 @@
1
+ # Copyright (c) 2009-2012 VMware, Inc.
2
+
3
+ module Bosh::Cli
4
+ class JobPropertyValidator
5
+ # TODO: tests
6
+
7
+ attr_reader :template_errors
8
+ attr_reader :jobs_without_properties
9
+
10
+ # @param [Array<JobBuilder>] built_jobs Built job templates
11
+ # @param [Hash] manifest Deployment manifest
12
+ def initialize(built_jobs, manifest)
13
+ @built_jobs = {}
14
+ @manifest = manifest
15
+
16
+ @jobs_without_properties = []
17
+
18
+ built_jobs.each do |job|
19
+ @built_jobs[job.name] = job
20
+ if job.properties.empty?
21
+ @jobs_without_properties << job
22
+ end
23
+ end
24
+
25
+ unless @manifest["properties"].is_a?(Hash)
26
+ bad_manifest("Invalid properties format in deployment " +
27
+ "manifest, Hash expected, #{@manifest["properties"].class} given")
28
+ end
29
+
30
+ unless @manifest["jobs"].is_a?(Array)
31
+ bad_manifest("Invalid jobs format in deployment " +
32
+ "manifest, Array expected, #{@manifest["jobs"].class} given")
33
+ end
34
+
35
+ @manifest["jobs"].each do |job|
36
+ unless job.is_a?(Hash)
37
+ bad_manifest("Invalid job spec in the manifest " +
38
+ "Hash expected, #{job.class} given")
39
+ end
40
+
41
+ job_name = job["name"]
42
+ if job_name.nil?
43
+ bad_manifest("Manifest contains at least one job without name")
44
+ end
45
+
46
+ if job["template"].nil?
47
+ bad_manifest("Job `#{job_name}' doesn't have a template")
48
+ end
49
+ end
50
+
51
+ @template_errors = []
52
+ # TODO: track missing props and show the list to user (super helpful!)
53
+ end
54
+
55
+ def validate
56
+ @manifest["jobs"].each do |job_spec|
57
+ validate_templates(job_spec)
58
+ end
59
+ end
60
+
61
+ # Tries to fill in each job template with job properties, collects errors
62
+ # @param [Hash] job_spec Job spec from the manifest
63
+ def validate_templates(job_spec)
64
+ built_job = @built_jobs[job_spec["template"]]
65
+
66
+ if built_job.nil?
67
+ raise CliError, "Job `#{job_spec["template"]}' has not been built"
68
+ end
69
+
70
+ collection = JobPropertyCollection.new(
71
+ built_job, @manifest["properties"],
72
+ job_spec["properties"], job_spec["property_mappings"])
73
+
74
+ # Spec is usually more than that but jobs rarely use anything but
75
+ # networks and properties.
76
+ # TODO: provide all keys in the spec?
77
+ spec = {
78
+ "job" => {
79
+ "name" => job_spec["name"]
80
+ },
81
+ "networks" => {
82
+ "default" => {"ip" => "10.0.0.1"}
83
+ },
84
+ "properties" => collection.to_hash,
85
+ "index" => 0
86
+ }
87
+
88
+ built_job.all_templates.each do |template_path|
89
+ # TODO: add progress bar?
90
+ evaluate_template(built_job, template_path, spec)
91
+ end
92
+ end
93
+
94
+ private
95
+
96
+ # @param [JobBuilder] job Job builder
97
+ # @param [String] template_path Template path
98
+ # @param [Hash] spec Fake instance spec
99
+ def evaluate_template(job, template_path, spec)
100
+ erb = ERB.new(File.read(template_path))
101
+ context = Bosh::Common::TemplateEvaluationContext.new(spec)
102
+ begin
103
+ erb.result(context.get_binding)
104
+ rescue Exception => e
105
+ @template_errors << TemplateError.new(job, template_path, e)
106
+ end
107
+ end
108
+
109
+ def bad_manifest(message)
110
+ raise InvalidManifest, message
111
+ end
112
+
113
+ class TemplateError
114
+ attr_reader :job
115
+ attr_reader :template_path
116
+ attr_reader :exception
117
+ attr_reader :line
118
+
119
+ # @param [JobBuilder] job
120
+ # @param [String] template_path
121
+ # @param [Exception] exception
122
+ def initialize(job, template_path, exception)
123
+ @job = job
124
+ @template_path = template_path
125
+ @exception = exception
126
+ @line = exception.backtrace.first.split(":")[1]
127
+ end
128
+ end
129
+ end
130
+ end
@@ -43,6 +43,35 @@ module Bosh::Cli
43
43
  # final build tarballs should be ignored as well
44
44
  # final builds metadata should be checked in
45
45
 
46
+ # @param [String] directory Release directory
47
+ # @param [Hash] options Package build options
48
+ def self.discover(directory, options = {})
49
+ builders = []
50
+
51
+ Dir[File.join(directory, "packages", "*")].each do |package_dir|
52
+ next unless File.directory?(package_dir)
53
+ package_dirname = File.basename(package_dir)
54
+ package_spec = load_yaml_file(File.join(package_dir, "spec"))
55
+
56
+ if package_spec["name"] != package_dirname
57
+ raise InvalidPackage,
58
+ "Found `#{package_spec["name"]}' package in " +
59
+ "`#{package_dirname}' directory, please fix it"
60
+ end
61
+
62
+ is_final = options[:final]
63
+ blobstore = options[:blobstore]
64
+ dry_run = options[:dry_run]
65
+
66
+ builder = new(package_spec, directory, is_final, blobstore)
67
+ builder.dry_run = true if dry_run
68
+
69
+ builders << builder
70
+ end
71
+
72
+ builders
73
+ end
74
+
46
75
  def initialize(spec, release_dir, final, blobstore,
47
76
  sources_dir = nil, blobs_dir = nil, alt_src_dir = nil)
48
77
  spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
@@ -87,8 +116,6 @@ module Bosh::Cli
87
116
  FileUtils.mkdir_p(@dev_builds_dir)
88
117
  FileUtils.mkdir_p(@final_builds_dir)
89
118
 
90
- at_exit { FileUtils.rm_rf(build_dir) }
91
-
92
119
  init_indices
93
120
  end
94
121
 
@@ -166,10 +193,10 @@ module Bosh::Cli
166
193
  ENV["RELEASE_DIR"] = @release_dir
167
194
  in_build_dir do
168
195
  pre_packaging_out = `bash -x pre_packaging 2>&1`
169
- pre_packaging_out.split("\n").each do |line|
170
- say("> #{line}")
171
- end
172
196
  unless $?.exitstatus == 0
197
+ pre_packaging_out.split("\n").each do |line|
198
+ say("> #{line}")
199
+ end
173
200
  raise InvalidPackage, "`#{name}' pre-packaging failed"
174
201
  end
175
202
  end