the-maestro 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/.document +5 -0
  2. data/.gitignore +25 -0
  3. data/LICENSE +23 -0
  4. data/README.rdoc +378 -0
  5. data/Rakefile +116 -0
  6. data/VERSION +1 -0
  7. data/lib/maestro.rb +354 -0
  8. data/lib/maestro/cloud.rb +384 -0
  9. data/lib/maestro/cloud/aws.rb +1231 -0
  10. data/lib/maestro/dsl_property.rb +15 -0
  11. data/lib/maestro/log4r/console_formatter.rb +18 -0
  12. data/lib/maestro/log4r/file_formatter.rb +24 -0
  13. data/lib/maestro/node.rb +123 -0
  14. data/lib/maestro/operating_system.rb +53 -0
  15. data/lib/maestro/operating_system/cent_os.rb +23 -0
  16. data/lib/maestro/operating_system/debian.rb +40 -0
  17. data/lib/maestro/operating_system/fedora.rb +23 -0
  18. data/lib/maestro/operating_system/ubuntu.rb +100 -0
  19. data/lib/maestro/role.rb +36 -0
  20. data/lib/maestro/tasks.rb +52 -0
  21. data/lib/maestro/validator.rb +32 -0
  22. data/rails/init.rb +1 -0
  23. data/test/integration/base_aws.rb +156 -0
  24. data/test/integration/fixtures/config/maestro/cookbooks/emacs/metadata.json +41 -0
  25. data/test/integration/fixtures/config/maestro/cookbooks/emacs/metadata.rb +3 -0
  26. data/test/integration/fixtures/config/maestro/cookbooks/emacs/recipes/default.rb +21 -0
  27. data/test/integration/fixtures/config/maestro/roles/default.json +9 -0
  28. data/test/integration/fixtures/config/maestro/roles/web.json +9 -0
  29. data/test/integration/helper.rb +8 -0
  30. data/test/integration/test_aws_cloud.rb +805 -0
  31. data/test/integration/test_cent_os.rb +104 -0
  32. data/test/integration/test_debian.rb +119 -0
  33. data/test/integration/test_fedora.rb +104 -0
  34. data/test/integration/test_ubuntu.rb +149 -0
  35. data/test/unit/fixtures/invalid-clouds-not-a-directory/config/maestro/clouds +1 -0
  36. data/test/unit/fixtures/invalid-cookbooks-not-a-directory/config/maestro/cookbooks +0 -0
  37. data/test/unit/fixtures/invalid-maestro-not-a-directory/config/maestro +0 -0
  38. data/test/unit/fixtures/invalid-missing-cookbooks/config/maestro/clouds/valid.yml +21 -0
  39. data/test/unit/fixtures/invalid-missing-roles/config/maestro/clouds/valid.yml +21 -0
  40. data/test/unit/fixtures/invalid-roles-not-a-directory/config/maestro/roles +1 -0
  41. data/test/unit/fixtures/ssh/id_rsa-maestro-test-keypair +27 -0
  42. data/test/unit/helper.rb +6 -0
  43. data/test/unit/test_aws_cloud.rb +133 -0
  44. data/test/unit/test_aws_ec2_node.rb +76 -0
  45. data/test/unit/test_aws_elb_node.rb +221 -0
  46. data/test/unit/test_aws_rds_node.rb +380 -0
  47. data/test/unit/test_cent_os.rb +28 -0
  48. data/test/unit/test_cloud.rb +142 -0
  49. data/test/unit/test_debian.rb +62 -0
  50. data/test/unit/test_fedora.rb +28 -0
  51. data/test/unit/test_invalid_mode.rb +11 -0
  52. data/test/unit/test_maestro.rb +140 -0
  53. data/test/unit/test_node.rb +50 -0
  54. data/test/unit/test_operating_system.rb +19 -0
  55. data/test/unit/test_rails_mode.rb +77 -0
  56. data/test/unit/test_role.rb +59 -0
  57. data/test/unit/test_standalone_mode.rb +75 -0
  58. data/test/unit/test_ubuntu.rb +95 -0
  59. data/the-maestro.gemspec +150 -0
  60. metadata +228 -0
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.0
data/lib/maestro.rb ADDED
@@ -0,0 +1,354 @@
1
+ require "maestro/dsl_property"
2
+ require "find"
3
+ require "ftools"
4
+ require "maestro/cloud"
5
+ require "maestro/cloud/aws"
6
+ require "maestro/operating_system"
7
+ require "log4r"
8
+ require "log4r/configurator"
9
+ require "maestro/log4r/console_formatter"
10
+
11
+
12
+ def aws_cloud(name, &block)
13
+ Maestro::Cloud::Aws.new(name, &block)
14
+ end
15
+
16
+
17
+ module Maestro
18
+
19
+ # ENV key used to point to a Maestro configuration directory
20
+ MAESTRO_DIR_ENV_VAR = 'MAESTRO_DIR'
21
+
22
+ # Directory underneath RAILS_ROOT where Maestro expects to find config files
23
+ MAESTRO_RAILS_CONFIG_DIRECTORY = '/config/maestro'
24
+
25
+ # Directory underneath RAILS_ROOT where Maestro log files will be written
26
+ MAESTRO_RAILS_LOG_DIRECTORY = '/log/maestro'
27
+
28
+ # Name of the Maestro Chef assets archive
29
+ MAESTRO_CHEF_ARCHIVE = 'maestro_chef_assets.tar.gz'
30
+
31
+ # add a PROGRESS level
32
+ Log4r::Configurator.custom_levels(:DEBUG, :INFO, :PROGRESS, :WARN, :ERROR, :FATAL)
33
+ @logger = Log4r::Logger.new("maestro")
34
+ outputter = Log4r::Outputter.stdout
35
+ outputter.formatter = ConsoleFormatter.new
36
+ @logger.add(outputter)
37
+
38
+
39
+ # Creates the Maestro config directory structure. If the directories already exist, no action is taken.
40
+ #
41
+ # In a Rails environment:
42
+ #
43
+ # RAILS_ROOT/config/maestro
44
+ # RAILS_ROOT/config/maestro/clouds
45
+ # RAILS_ROOT/config/maestro/cookbooks
46
+ # RAILS_ROOT/config/maestro/roles
47
+ #
48
+ # In a standalone environment, the following directories will be created under
49
+ # the directory specified by the ENV['MAESTRO_DIR'] environment variable:
50
+ #
51
+ # MAESTRO_DIR/config/maestro
52
+ # MAESTRO_DIR/config/maestro/clouds
53
+ # MAESTRO_DIR/config/maestro/cookbooks
54
+ # MAESTRO_DIR/config/maestro/roles
55
+ def self.create_config_dirs
56
+ if defined? RAILS_ROOT
57
+ create_configs(rails_config_dir)
58
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
59
+ create_configs(standalone_config_dir)
60
+ else
61
+ raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
62
+ end
63
+ end
64
+
65
+ # Creates the Maestro log directories. If the directories already exist, no action is taken.
66
+ #
67
+ # In a Rails environment:
68
+ #
69
+ # RAILS_ROOT/log/maestro
70
+ # RAILS_ROOT/log/maestro/clouds
71
+ #
72
+ # In a standalone environment, the following directories will be created under
73
+ # the directory specified by the ENV['MAESTRO_DIR'] environment variable:
74
+ #
75
+ # MAESTRO_DIR/log/maestro
76
+ # MAESTRO_DIR/log/maestro/clouds
77
+ def self.create_log_dirs
78
+ if defined? RAILS_ROOT
79
+ create_logs(rails_log_dir)
80
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
81
+ create_logs(standalone_log_dir)
82
+ else
83
+ raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
84
+ end
85
+ end
86
+
87
+ # Validates your maestro configs. This method returns an Array with two elements:
88
+ # * element[0] boolean indicating whether your maestro configs are valid
89
+ # * element[1] Array of Strings containing a report of the validation
90
+ def self.validate_configs
91
+ if defined? RAILS_ROOT
92
+ validate_rails_config
93
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
94
+ validate_standalone_config
95
+ else
96
+ return [false, ["Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"]]
97
+ end
98
+ end
99
+
100
+ # Returns a Hash of Clouds defined in the Maestro clouds configuration directory
101
+ def self.clouds
102
+ if defined? RAILS_ROOT
103
+ get_clouds(clouds_config_dir(rails_maestro_config_dir))
104
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
105
+ get_clouds(clouds_config_dir(standalone_maestro_config_dir))
106
+ else
107
+ raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
108
+ end
109
+ end
110
+
111
+ # Returns the top level log directory
112
+ def self.log_directory
113
+ if defined? RAILS_ROOT
114
+ rails_log_dir
115
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
116
+ standalone_log_dir
117
+ else
118
+ raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
119
+ end
120
+ end
121
+
122
+ # Returns the maestro log directory
123
+ def self.maestro_log_directory
124
+ if defined? RAILS_ROOT
125
+ rails_log_dir + "/maestro"
126
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
127
+ standalone_log_dir + "/maestro"
128
+ else
129
+ raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
130
+ end
131
+ end
132
+
133
+ # Creates a .tar.gz file containing the Chef cookbooks/ and roles/ directories within the maestro config directory, and returns the path to the file
134
+ def self.chef_archive
135
+ require 'tempfile'
136
+ require 'zlib'
137
+ require 'archive/tar/minitar'
138
+
139
+ dir = nil
140
+ if defined? RAILS_ROOT
141
+ dir = rails_maestro_config_dir
142
+ elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
143
+ dir = standalone_maestro_config_dir
144
+ else
145
+ raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
146
+ end
147
+ temp_file = Dir.tmpdir + "/" + MAESTRO_CHEF_ARCHIVE
148
+ File.delete(temp_file) if File.exist?(temp_file)
149
+
150
+ pwd = Dir.pwd
151
+ open temp_file, 'wb' do |io|
152
+ Zlib::GzipWriter.wrap io do |gzip|
153
+ begin
154
+ out = Archive::Tar::Minitar::Output.new(gzip)
155
+ Dir.chdir(dir) # don't store full paths in archive
156
+ Dir.glob("cookbooks/**/**").each do |file|
157
+ Archive::Tar::Minitar.pack_file(file, out) if File.file?(file) || File.directory?(file)
158
+ end
159
+ Dir.glob("roles/**/**").each do |file|
160
+ Archive::Tar::Minitar.pack_file(file, out) if File.file?(file) || File.directory?(file)
161
+ end
162
+ ensure
163
+ gzip.finish
164
+ end
165
+ end
166
+ end
167
+ Dir.chdir(pwd)
168
+ temp_file
169
+ end
170
+
171
+
172
+ private
173
+
174
+ # Creates the Maestro config directory structure in the given directory
175
+ #
176
+ # - dir: The base directory in which to create the maestro config directory structure
177
+ def self.create_configs(dir)
178
+ raise "Cannot create Maestro config directory structure: #{dir} doesn't exist." if !File.exist?(dir)
179
+ raise "Cannot create Maestro config directory structure: #{dir} is not a directory." if !File.directory?(dir)
180
+ raise "Cannot create Maestro config directory structure: #{dir} is not writable." if !File.writable?(dir)
181
+ base_dir = dir
182
+ base_dir.chop! if base_dir =~ /\/$/
183
+ if !File.exist?("#{base_dir}/maestro")
184
+ Dir.mkdir("#{base_dir}/maestro")
185
+ @logger.progress "Created #{base_dir}/maestro"
186
+ end
187
+ if !File.exist?("#{base_dir}/maestro/clouds")
188
+ Dir.mkdir("#{base_dir}/maestro/clouds")
189
+ @logger.info "Created #{base_dir}/maestro/clouds"
190
+ end
191
+ if !File.exist?("#{base_dir}/maestro/cookbooks")
192
+ Dir.mkdir("#{base_dir}/maestro/cookbooks")
193
+ @logger.info "Created #{base_dir}/maestro/cookbooks"
194
+ end
195
+ if !File.exist?("#{base_dir}/maestro/roles")
196
+ Dir.mkdir("#{base_dir}/maestro/roles")
197
+ @logger.info "Created #{base_dir}/maestro/roles"
198
+ end
199
+ end
200
+
201
+ # Creates the Maestro config directory structure in the given directory
202
+ #
203
+ # - dir: The base directory in which to create the maestro log directory structure
204
+ def self.create_logs(dir)
205
+ raise "Cannot create Maestro log directory: #{dir} doesn't exist." if !File.exist?(dir)
206
+ raise "Cannot create Maestro log directory: #{dir} is not a directory." if !File.directory?(dir)
207
+ raise "Cannot create Maestro log directory: #{dir} is not writable." if !File.writable?(dir)
208
+ begin
209
+ base_dir = dir
210
+ base_dir.chop! if base_dir =~ /\/$/
211
+ if !File.exist?("#{base_dir}/maestro")
212
+ Dir.mkdir("#{base_dir}/maestro")
213
+ @logger.info "Created #{base_dir}/maestro"
214
+ end
215
+ if !File.exist?("#{base_dir}/maestro/clouds")
216
+ Dir.mkdir("#{base_dir}/maestro/clouds")
217
+ @logger.info "Created #{base_dir}/maestro/clouds"
218
+ end
219
+ rescue SystemCallError => syserr
220
+ @logger.error "Error creating cloud directory"
221
+ @logger.error syserr
222
+ rescue StandardError => serr
223
+ @logger.error "Unexpected Error"
224
+ @logger.error serr
225
+ end
226
+ end
227
+
228
+ # Validates the maestro configuration found at RAILS_ROOT/config/maestro
229
+ def self.validate_rails_config
230
+ validate_config rails_maestro_config_dir
231
+ end
232
+
233
+ # Validates the maestro configuration found at ENV['MAESTRO_DIR']
234
+ def self.validate_standalone_config
235
+ validate_config standalone_maestro_config_dir
236
+ end
237
+
238
+ # Validates the maestro configuration found at the given maestro_directory
239
+ def self.validate_config(maestro_directory)
240
+ valid = true
241
+ error_messages = Array.new
242
+ if !File.exist?(maestro_directory)
243
+ valid = false
244
+ error_messages << "Maestro config directory does not exist: #{maestro_directory}"
245
+ end
246
+ if !File.directory?(maestro_directory)
247
+ valid = false
248
+ error_messages << "Maestro config directory is not a directory: #{maestro_directory}"
249
+ end
250
+ clouds_directory = clouds_config_dir maestro_directory
251
+ if !File.exist?(clouds_directory)
252
+ valid = false
253
+ error_messages << "Maestro clouds config directory does not exist: #{clouds_directory}"
254
+ end
255
+ if !File.directory?(clouds_directory)
256
+ valid = false
257
+ error_messages << "Maestro clouds config directory is not a directory: #{clouds_directory}"
258
+ end
259
+ cookbooks_directory = cookbooks_dir maestro_directory
260
+ if !File.exist?(cookbooks_directory)
261
+ valid = false
262
+ error_messages << "Chef cookbooks directory does not exist: #{cookbooks_directory}"
263
+ else
264
+ if !File.directory?(cookbooks_directory)
265
+ valid = false
266
+ error_messages << "Chef cookbooks directory is not a directory: #{cookbooks_directory}"
267
+ end
268
+ end
269
+ roles_directory = roles_dir maestro_directory
270
+ if !File.exist?(roles_directory)
271
+ valid = false
272
+ error_messages << "Chef roles directory does not exist: #{roles_directory}"
273
+ else
274
+ if !File.directory?(roles_directory)
275
+ valid = false
276
+ error_messages << "Chef roles directory is not a directory: #{roles_directory}"
277
+ end
278
+ end
279
+ if valid
280
+ clouds = get_clouds(clouds_directory)
281
+ clouds.each do |name, cloud|
282
+ cloud.validate
283
+ if !cloud.valid?
284
+ valid = false
285
+ error_messages << "INVALID: #{cloud.config_file}"
286
+ cloud.validation_errors.each {|error| error_messages << " #{error}"}
287
+ else
288
+ error_messages << "VALID: #{cloud.config_file}"
289
+ end
290
+ end
291
+ end
292
+ return [valid, error_messages]
293
+ end
294
+
295
+ def self.rails_config_dir
296
+ "#{RAILS_ROOT}/config"
297
+ end
298
+
299
+ def self.rails_log_dir
300
+ "#{RAILS_ROOT}/log"
301
+ end
302
+
303
+ def self.rails_maestro_config_dir
304
+ "#{RAILS_ROOT}#{MAESTRO_RAILS_CONFIG_DIRECTORY}"
305
+ end
306
+
307
+ def self.standalone_maestro_config_dir
308
+ "#{ENV[MAESTRO_DIR_ENV_VAR]}#{MAESTRO_RAILS_CONFIG_DIRECTORY}"
309
+ end
310
+
311
+ def self.standalone_config_dir
312
+ "#{ENV[MAESTRO_DIR_ENV_VAR]}/config"
313
+ end
314
+
315
+ def self.standalone_log_dir
316
+ "#{ENV[MAESTRO_DIR_ENV_VAR]}/log"
317
+ end
318
+
319
+ def self.clouds_config_dir(maestro_directory)
320
+ "#{maestro_directory}/clouds"
321
+ end
322
+
323
+ def self.cookbooks_dir(maestro_directory)
324
+ "#{maestro_directory}/cookbooks"
325
+ end
326
+
327
+ def self.roles_dir(maestro_directory)
328
+ "#{maestro_directory}/roles"
329
+ end
330
+
331
+ # Gets the Hash of Clouds found in clouds_directory
332
+ def self.get_clouds(clouds_directory)
333
+ clouds_directory << "/" unless clouds_directory =~ /\/$/
334
+ clouds = {}
335
+ config_files = get_cloud_config_files(clouds_directory)
336
+ config_files.each do |config_file|
337
+ cloud_name = config_file[clouds_directory.length, (config_file.length-(clouds_directory.length+File.extname(config_file).length))]
338
+ cloud = Cloud::Base.create_from_file(config_file)
339
+ clouds[cloud_name] = cloud
340
+ end
341
+ clouds
342
+ end
343
+
344
+ # Returns and array of all Cloud files found in clouds_directory
345
+ def self.get_cloud_config_files(clouds_directory)
346
+ config_files = []
347
+ Find.find(clouds_directory) do |path|
348
+ if FileTest.file?(path) && (File.extname(path).eql?(".rb"))
349
+ config_files << path
350
+ end
351
+ end
352
+ config_files
353
+ end
354
+ end
@@ -0,0 +1,384 @@
1
+ require "ftools"
2
+ require "maestro/role"
3
+ require "maestro/node"
4
+ require 'maestro/validator'
5
+ require "net/ssh/multi"
6
+ require "log4r"
7
+ require "maestro/log4r/console_formatter"
8
+ require "maestro/log4r/file_formatter"
9
+
10
+
11
+ module Maestro
12
+ # A named cloud (i.e. production, staging, test, dev, etc)
13
+ module Cloud
14
+ class Base
15
+ include Validator
16
+
17
+ # the name of this cloud
18
+ attr_reader :name
19
+ # the config file for this cloud
20
+ attr_accessor :config_file
21
+ # the Hash of Configurable Nodes in this Cloud
22
+ attr_reader :configurable_nodes
23
+ # String containing the full path to this Cloud's log directory
24
+ attr_reader :log_directory
25
+ dsl_property :keypair_name, :keypair_file
26
+
27
+ # Creates a new Cloud object.
28
+ # * name: the name of the Cloud
29
+ # * cfg_file: Pointer to the file containing the Cloud configuration (optional)
30
+ # * block: contents of the Cloud
31
+ def initialize(name, cfg_file=nil, &block)
32
+ super()
33
+ raise StandardError, "Cloud name cannot contain spaces: #{name}" if name.is_a?(String) && !name.index(/\s/).nil?
34
+ @name = name
35
+ @config_file = cfg_file
36
+ @roles = Hash.new
37
+ @nodes = Hash.new
38
+ @configurable_nodes = Hash.new
39
+ @valid = true
40
+ @logger = Log4r::Logger.new(Regexp::quote(@name.to_s))
41
+ outputter = Log4r::StdoutOutputter.new("#{@name.to_s}-stdout")
42
+ outputter.formatter = ConsoleFormatter.new
43
+ @logger.add(outputter)
44
+ init_logs
45
+ instance_eval(&block) if block_given?
46
+ end
47
+
48
+ # Creates a Cloud from the contents of the given file
49
+ def self.create_from_file(config_file)
50
+ cloud = eval(File.read(config_file))
51
+ cloud.config_file = config_file
52
+ return cloud
53
+ end
54
+
55
+ # creates the Roles for this Cloud if a block is given. Otherwise, returns the roles Hash
56
+ def roles(&block)
57
+ if block_given?
58
+ instance_eval(&block)
59
+ else
60
+ @roles
61
+ end
62
+ end
63
+
64
+ # sets the roles Hash
65
+ def roles=(roles)
66
+ @roles = roles
67
+ end
68
+
69
+ # creates a Role
70
+ def role(name, &block)
71
+ if @roles.has_key?(name)
72
+ invalidate "Duplicate role definition: #{name}"
73
+ else
74
+ @roles[name] = Role.new(name, self, &block)
75
+ end
76
+ end
77
+
78
+ # creates the Nodes for this Cloud if a block is given, otherwise returns the Nodes in this Cloud
79
+ def nodes(&block)
80
+ if block_given?
81
+ instance_eval(&block)
82
+ else
83
+ @nodes
84
+ end
85
+ end
86
+
87
+ # sets the nodes Hash
88
+ def nodes=(nodes)
89
+ @nodes = nodes
90
+ end
91
+
92
+ def method_missing(name, *params) #:nodoc:
93
+ @valid = false
94
+ @validation_errors << "Unexpected attribute: #{name}"
95
+ end
96
+
97
+ # Reports the current status of this Cloud
98
+ def status
99
+ @logger.info "#{@name} Cloud status:"
100
+ node_statuses
101
+ end
102
+
103
+ # Starts this Cloud. Takes no action if the Cloud is already running as currently configured
104
+ def start
105
+ @logger.info "Starting #{@name} Cloud. This may take a few minutes..."
106
+ end
107
+
108
+ # Configures the Nodes in this Cloud
109
+ def configure
110
+ @logger.info "Configuring #{@name} Cloud"
111
+ if !@configurable_nodes.empty?
112
+ session = open_ssh_session
113
+ result = chef_solo_installed?(session)
114
+ if !result[0]
115
+ names = result[1].collect {|n| n.name}
116
+ @logger.progress "Installing chef-solo on Nodes #{names.inspect}. This may take a few minutes..."
117
+ session.close
118
+ session = open_ssh_session(result[1])
119
+ install_chef_solo(session)
120
+ configure_chef_solo(session)
121
+ session.close
122
+ @logger.progress "\n"
123
+ else
124
+ @logger.info "chef-solo already installed on Nodes #{@configurable_nodes.keys.inspect}"
125
+ end
126
+ @logger.info "Running chef-solo on Nodes #{@configurable_nodes.keys.inspect}..."
127
+ session = open_ssh_session
128
+ run_chef_solo(session)
129
+ session.close
130
+ @logger.info "Configuration of #{@name} Cloud complete"
131
+ end
132
+ end
133
+
134
+ # Checks if chef-solo is installed on each of the Configurable Nodes in this Cloud.
135
+ # This method returns an Array with two elements:
136
+ # * element[0] boolean indicating whether chef-solo is installed on all Configurable Nodes
137
+ # * element[1] Array of Nodes which need chef-solo installed
138
+ def chef_solo_installed?(session=nil)
139
+ close_session = false
140
+ if session.nil?
141
+ session = open_ssh_session
142
+ close_session = true
143
+ end
144
+ @logger.info "Checking for installation of chef-solo..."
145
+ valid = true
146
+ needs_chef = Array.new
147
+ session.open_channel do |channel|
148
+ # Find the node for this channel's host
149
+ the_node = nil
150
+ @configurable_nodes.each_pair {|name, node| the_node = node if channel[:host].eql? node.hostname}
151
+ if the_node.nil?
152
+ @logger.error "Could not find node matching hostname #{channel[:host]}. This should not happen."
153
+ else
154
+ channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
155
+ channel.exec("chef-solo --version") do |ch, success|
156
+ ch.on_data do |ch, data|
157
+ if !data.include?("Chef: 0.8")
158
+ valid = false
159
+ needs_chef << the_node
160
+ end
161
+ end
162
+ end
163
+ end
164
+ end
165
+ session.loop(60)
166
+ session.close if close_session
167
+ return [valid, needs_chef]
168
+ end
169
+
170
+ # installs chef-solo on the Configurable Nodes the given session is set up with
171
+ def install_chef_solo(session=nil)
172
+ close_session = false
173
+ if session.nil?
174
+ session = open_ssh_session
175
+ close_session = true
176
+ end
177
+ etc_issue = nil
178
+ session.open_channel do |channel|
179
+ channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
180
+ channel.exec("cat /etc/issue") do |ch, success|
181
+ ch.on_data {|ch, data| etc_issue = data}
182
+ end
183
+ end
184
+ session.loop(60)
185
+
186
+ # shut off the stdout outputter and only log to the nodes' log files
187
+ @configurable_nodes.each_pair {|name, node| node.disable_stdout}
188
+ os = Maestro::OperatingSystem.create_from_etc_issue(etc_issue)
189
+ os.chef_install_script.each do |cmd|
190
+ session.open_channel do |channel|
191
+ # Find the node for this channel's host
192
+ the_node = nil
193
+ @configurable_nodes.each_pair {|name, node| the_node = node if channel[:host].eql? node.hostname}
194
+ if the_node.nil?
195
+ @logger.error "Could not find node matching hostname #{channel[:host]}. This should not happen."
196
+ else
197
+ the_node.logger.info "Installing chef-solo"
198
+ channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
199
+ channel.exec(cmd) do |ch, success|
200
+ @logger.progress "."
201
+ ch.on_data {|ch, data| the_node.logger.info data}
202
+ ch.on_extended_data {|ch, data| the_node.logger.error }
203
+ end
204
+ end
205
+ end
206
+ session.loop(60)
207
+ end
208
+ # turn the stdout outputter back on
209
+ @configurable_nodes.each_pair {|name, node| node.enable_stdout}
210
+ session.close if close_session
211
+ end
212
+
213
+ # configures chef-solo
214
+ def configure_chef_solo(session=nil)
215
+ close_session = false
216
+ if session.nil?
217
+ session = open_ssh_session
218
+ close_session = true
219
+ end
220
+ # write the chef-solo config file
221
+ chef_solo_config =
222
+ ["sudo rm /tmp/chef-solo.rb",
223
+ "sudo mkdir -p /tmp/chef-solo",
224
+ "sudo mkdir -p /tmp/chef-solo/cookbooks",
225
+ "sudo mkdir -p /tmp/chef-solo/roles",
226
+ "sudo sh -c 'echo file_cache_path \\\"/tmp/chef-solo\\\" >> /tmp/chef-solo.rb'",
227
+ "sudo sh -c 'echo cookbook_path \\\"/tmp/chef-solo/cookbooks\\\" >> /tmp/chef-solo.rb'",
228
+ "sudo sh -c 'echo role_path \\\"/tmp/chef-solo/roles\\\" >> /tmp/chef-solo.rb'"]
229
+ chef_solo_config.each do |str|
230
+ session.open_channel do |channel|
231
+ channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
232
+ channel.exec(str)
233
+ end
234
+ session.loop(60)
235
+ end
236
+ session.close if close_session
237
+ end
238
+
239
+ # runs chef-solo
240
+ def run_chef_solo(session=nil)
241
+ close_session = false
242
+ if session.nil?
243
+ session = open_ssh_session
244
+ close_session = true
245
+ end
246
+ commands =
247
+ ["sudo chef-solo -c /tmp/chef-solo.rb -r '#{chef_assets_url()}'"]
248
+ # shut off the stdout outputter and only log to the nodes' log files
249
+ @configurable_nodes.each_pair {|name, node| node.disable_stdout}
250
+ commands.each do |cmd|
251
+ session.open_channel do |channel|
252
+ channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
253
+ # Find the node for this channel's host
254
+ the_node = nil
255
+ @configurable_nodes.each_pair {|name, node| the_node = node if channel[:host].eql? node.hostname}
256
+ if the_node.nil?
257
+ @logger.error "Could not find node matching hostname #{channel[:host]}. This should not happen."
258
+ else
259
+ node_cmd = cmd + " -j '#{node_json_url(the_node)}'"
260
+ channel.exec(node_cmd) do |ch, success|
261
+ ch.on_data {|ch2, data2| the_node.logger.info data2}
262
+ ch.on_extended_data {|ch2, data2| the_node.logger.error data2}
263
+ end
264
+ end
265
+ end
266
+ session.loop(60)
267
+ end
268
+ # turn the stdout outputter back on
269
+ @configurable_nodes.each_pair {|name, node| node.enable_stdout}
270
+ session.close if close_session
271
+ end
272
+
273
+ # Shuts down this Cloud. Takes no action if the Cloud is not running
274
+ def shutdown
275
+ @logger.info "Shutting down #{@name} Cloud"
276
+ end
277
+
278
+
279
+ protected
280
+
281
+ # creates log directory and files for this cloud. If the log directory or files exist, no action is taken.
282
+ def init_logs
283
+ begin
284
+ if !File.exists?(Maestro.maestro_log_directory)
285
+ Maestro.create_log_dirs
286
+ @logger.info "Created #{Maestro.maestro_log_directory}"
287
+ end
288
+ clouds_dir = Maestro.maestro_log_directory + "/clouds"
289
+ if !File.exists?(clouds_dir)
290
+ Dir.mkdir(clouds_dir)
291
+ @logger.info "Created #{clouds_dir}"
292
+ end
293
+ cloud_dir = clouds_dir + "/#{@name}"
294
+ if !File.exists?(cloud_dir)
295
+ Dir.mkdir(cloud_dir)
296
+ @logger.info "Created #{cloud_dir}"
297
+ end
298
+ @log_directory = cloud_dir
299
+ cloud_log_file = cloud_dir + "/#{@name}.log"
300
+ if !File.exists?(cloud_log_file)
301
+ File.new(cloud_log_file, "a+")
302
+ @logger.info "Created #{cloud_log_file}"
303
+ end
304
+ outputter = Log4r::FileOutputter.new("#{@name}-file", :formatter => FileFormatter.new, :filename => cloud_log_file, :truncate => false)
305
+ @logger.add(outputter)
306
+ rescue RuntimeError => rerr
307
+ if !rerr.message.eql?("Maestro not configured correctly. Either RAILS_ROOT or ENV['MAESTRO_DIR'] must be defined")
308
+ @logger.error "Unexpected Error"
309
+ @logger.error rerr
310
+ end
311
+ rescue SystemCallError => syserr
312
+ @logger.error "Error creating cloud directory"
313
+ @logger.error syserr
314
+ rescue StandardError => serr
315
+ @logger.error "Unexpected Error"
316
+ @logger.error serr
317
+ end
318
+ end
319
+
320
+ # opens a multi ssh session. If the cnodes argument is nil, then a session
321
+ # is opened up to each Configurable Node in this Cloud. Otherwise, a session
322
+ # is opened to each Configurable Node in the cnodes array.
323
+ def open_ssh_session(cnodes=[])
324
+ handler = Proc.new do |server|
325
+ server[:connection_attempts] ||= 0
326
+ if server[:connection_attempts] < 50
327
+ server[:connection_attempts] += 1
328
+ sleep 2
329
+ throw :go, :retry
330
+ else
331
+ throw :go, :raise
332
+ end
333
+ end
334
+
335
+ session = Net::SSH::Multi.start(:concurrent_connections => 10, :on_error => handler)
336
+ if cnodes.empty?
337
+ @configurable_nodes.each_pair {|node_name, node| session.use node.hostname, :user => node.ssh_user, :keys => [keypair_file]}
338
+ else
339
+ cnodes.each {|node| session.use node.hostname, :user => node.ssh_user, :keys => [keypair_file]}
340
+ end
341
+ return session
342
+ end
343
+
344
+
345
+ private
346
+
347
+ # validates this Cloud
348
+ def validate_internal
349
+ invalidate "Missing keypair_name" if @keypair_name.nil?
350
+ invalidate "Missing keypair_file" if @keypair_file.nil?
351
+ validate_roles
352
+ validate_nodes
353
+ end
354
+
355
+ # validates the roles in the cloud config
356
+ def validate_roles
357
+ if @roles.nil? || @roles.empty?
358
+ invalidate "Missing roles"
359
+ else
360
+ @roles.each do |name, role|
361
+ role.validate
362
+ if !role.valid?
363
+ role.validation_errors.each {|error_str| invalidate error_str}
364
+ end
365
+ end
366
+ end
367
+ end
368
+
369
+ # validates the nodes in the cloud config
370
+ def validate_nodes
371
+ if @nodes.nil? || @nodes.empty?
372
+ invalidate "Missing nodes"
373
+ else
374
+ @nodes.each do |name, node|
375
+ node.validate
376
+ if !node.valid?
377
+ node.validation_errors.each {|error_str| invalidate error_str}
378
+ end
379
+ end
380
+ end
381
+ end
382
+ end
383
+ end
384
+ end