the-maestro 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +25 -0
- data/LICENSE +23 -0
- data/README.rdoc +378 -0
- data/Rakefile +116 -0
- data/VERSION +1 -0
- data/lib/maestro.rb +354 -0
- data/lib/maestro/cloud.rb +384 -0
- data/lib/maestro/cloud/aws.rb +1231 -0
- data/lib/maestro/dsl_property.rb +15 -0
- data/lib/maestro/log4r/console_formatter.rb +18 -0
- data/lib/maestro/log4r/file_formatter.rb +24 -0
- data/lib/maestro/node.rb +123 -0
- data/lib/maestro/operating_system.rb +53 -0
- data/lib/maestro/operating_system/cent_os.rb +23 -0
- data/lib/maestro/operating_system/debian.rb +40 -0
- data/lib/maestro/operating_system/fedora.rb +23 -0
- data/lib/maestro/operating_system/ubuntu.rb +100 -0
- data/lib/maestro/role.rb +36 -0
- data/lib/maestro/tasks.rb +52 -0
- data/lib/maestro/validator.rb +32 -0
- data/rails/init.rb +1 -0
- data/test/integration/base_aws.rb +156 -0
- data/test/integration/fixtures/config/maestro/cookbooks/emacs/metadata.json +41 -0
- data/test/integration/fixtures/config/maestro/cookbooks/emacs/metadata.rb +3 -0
- data/test/integration/fixtures/config/maestro/cookbooks/emacs/recipes/default.rb +21 -0
- data/test/integration/fixtures/config/maestro/roles/default.json +9 -0
- data/test/integration/fixtures/config/maestro/roles/web.json +9 -0
- data/test/integration/helper.rb +8 -0
- data/test/integration/test_aws_cloud.rb +805 -0
- data/test/integration/test_cent_os.rb +104 -0
- data/test/integration/test_debian.rb +119 -0
- data/test/integration/test_fedora.rb +104 -0
- data/test/integration/test_ubuntu.rb +149 -0
- data/test/unit/fixtures/invalid-clouds-not-a-directory/config/maestro/clouds +1 -0
- data/test/unit/fixtures/invalid-cookbooks-not-a-directory/config/maestro/cookbooks +0 -0
- data/test/unit/fixtures/invalid-maestro-not-a-directory/config/maestro +0 -0
- data/test/unit/fixtures/invalid-missing-cookbooks/config/maestro/clouds/valid.yml +21 -0
- data/test/unit/fixtures/invalid-missing-roles/config/maestro/clouds/valid.yml +21 -0
- data/test/unit/fixtures/invalid-roles-not-a-directory/config/maestro/roles +1 -0
- data/test/unit/fixtures/ssh/id_rsa-maestro-test-keypair +27 -0
- data/test/unit/helper.rb +6 -0
- data/test/unit/test_aws_cloud.rb +133 -0
- data/test/unit/test_aws_ec2_node.rb +76 -0
- data/test/unit/test_aws_elb_node.rb +221 -0
- data/test/unit/test_aws_rds_node.rb +380 -0
- data/test/unit/test_cent_os.rb +28 -0
- data/test/unit/test_cloud.rb +142 -0
- data/test/unit/test_debian.rb +62 -0
- data/test/unit/test_fedora.rb +28 -0
- data/test/unit/test_invalid_mode.rb +11 -0
- data/test/unit/test_maestro.rb +140 -0
- data/test/unit/test_node.rb +50 -0
- data/test/unit/test_operating_system.rb +19 -0
- data/test/unit/test_rails_mode.rb +77 -0
- data/test/unit/test_role.rb +59 -0
- data/test/unit/test_standalone_mode.rb +75 -0
- data/test/unit/test_ubuntu.rb +95 -0
- data/the-maestro.gemspec +150 -0
- metadata +228 -0
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.2.0
|
data/lib/maestro.rb
ADDED
@@ -0,0 +1,354 @@
|
|
1
|
+
require "maestro/dsl_property"
|
2
|
+
require "find"
|
3
|
+
require "ftools"
|
4
|
+
require "maestro/cloud"
|
5
|
+
require "maestro/cloud/aws"
|
6
|
+
require "maestro/operating_system"
|
7
|
+
require "log4r"
|
8
|
+
require "log4r/configurator"
|
9
|
+
require "maestro/log4r/console_formatter"
|
10
|
+
|
11
|
+
|
12
|
+
def aws_cloud(name, &block)
|
13
|
+
Maestro::Cloud::Aws.new(name, &block)
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
module Maestro
|
18
|
+
|
19
|
+
# ENV key used to point to a Maestro configuration directory
|
20
|
+
MAESTRO_DIR_ENV_VAR = 'MAESTRO_DIR'
|
21
|
+
|
22
|
+
# Directory underneath RAILS_ROOT where Maestro expects to find config files
|
23
|
+
MAESTRO_RAILS_CONFIG_DIRECTORY = '/config/maestro'
|
24
|
+
|
25
|
+
# Directory underneath RAILS_ROOT where Maestro log files will be written
|
26
|
+
MAESTRO_RAILS_LOG_DIRECTORY = '/log/maestro'
|
27
|
+
|
28
|
+
# Name of the Maestro Chef assets archive
|
29
|
+
MAESTRO_CHEF_ARCHIVE = 'maestro_chef_assets.tar.gz'
|
30
|
+
|
31
|
+
# add a PROGRESS level
|
32
|
+
Log4r::Configurator.custom_levels(:DEBUG, :INFO, :PROGRESS, :WARN, :ERROR, :FATAL)
|
33
|
+
@logger = Log4r::Logger.new("maestro")
|
34
|
+
outputter = Log4r::Outputter.stdout
|
35
|
+
outputter.formatter = ConsoleFormatter.new
|
36
|
+
@logger.add(outputter)
|
37
|
+
|
38
|
+
|
39
|
+
# Creates the Maestro config directory structure. If the directories already exist, no action is taken.
|
40
|
+
#
|
41
|
+
# In a Rails environment:
|
42
|
+
#
|
43
|
+
# RAILS_ROOT/config/maestro
|
44
|
+
# RAILS_ROOT/config/maestro/clouds
|
45
|
+
# RAILS_ROOT/config/maestro/cookbooks
|
46
|
+
# RAILS_ROOT/config/maestro/roles
|
47
|
+
#
|
48
|
+
# In a standalone environment, the following directories will be created under
|
49
|
+
# the directory specified by the ENV['MAESTRO_DIR'] environment variable:
|
50
|
+
#
|
51
|
+
# MAESTRO_DIR/config/maestro
|
52
|
+
# MAESTRO_DIR/config/maestro/clouds
|
53
|
+
# MAESTRO_DIR/config/maestro/cookbooks
|
54
|
+
# MAESTRO_DIR/config/maestro/roles
|
55
|
+
def self.create_config_dirs
|
56
|
+
if defined? RAILS_ROOT
|
57
|
+
create_configs(rails_config_dir)
|
58
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
59
|
+
create_configs(standalone_config_dir)
|
60
|
+
else
|
61
|
+
raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
# Creates the Maestro log directories. If the directories already exist, no action is taken.
|
66
|
+
#
|
67
|
+
# In a Rails environment:
|
68
|
+
#
|
69
|
+
# RAILS_ROOT/log/maestro
|
70
|
+
# RAILS_ROOT/log/maestro/clouds
|
71
|
+
#
|
72
|
+
# In a standalone environment, the following directories will be created under
|
73
|
+
# the directory specified by the ENV['MAESTRO_DIR'] environment variable:
|
74
|
+
#
|
75
|
+
# MAESTRO_DIR/log/maestro
|
76
|
+
# MAESTRO_DIR/log/maestro/clouds
|
77
|
+
def self.create_log_dirs
|
78
|
+
if defined? RAILS_ROOT
|
79
|
+
create_logs(rails_log_dir)
|
80
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
81
|
+
create_logs(standalone_log_dir)
|
82
|
+
else
|
83
|
+
raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# Validates your maestro configs. This method returns an Array with two elements:
|
88
|
+
# * element[0] boolean indicating whether your maestro configs are valid
|
89
|
+
# * element[1] Array of Strings containing a report of the validation
|
90
|
+
def self.validate_configs
|
91
|
+
if defined? RAILS_ROOT
|
92
|
+
validate_rails_config
|
93
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
94
|
+
validate_standalone_config
|
95
|
+
else
|
96
|
+
return [false, ["Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"]]
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# Returns a Hash of Clouds defined in the Maestro clouds configuration directory
|
101
|
+
def self.clouds
|
102
|
+
if defined? RAILS_ROOT
|
103
|
+
get_clouds(clouds_config_dir(rails_maestro_config_dir))
|
104
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
105
|
+
get_clouds(clouds_config_dir(standalone_maestro_config_dir))
|
106
|
+
else
|
107
|
+
raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# Returns the top level log directory
|
112
|
+
def self.log_directory
|
113
|
+
if defined? RAILS_ROOT
|
114
|
+
rails_log_dir
|
115
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
116
|
+
standalone_log_dir
|
117
|
+
else
|
118
|
+
raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
# Returns the maestro log directory
|
123
|
+
def self.maestro_log_directory
|
124
|
+
if defined? RAILS_ROOT
|
125
|
+
rails_log_dir + "/maestro"
|
126
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
127
|
+
standalone_log_dir + "/maestro"
|
128
|
+
else
|
129
|
+
raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# Creates a .tar.gz file containing the Chef cookbooks/ and roles/ directories within the maestro config directory, and returns the path to the file
|
134
|
+
def self.chef_archive
|
135
|
+
require 'tempfile'
|
136
|
+
require 'zlib'
|
137
|
+
require 'archive/tar/minitar'
|
138
|
+
|
139
|
+
dir = nil
|
140
|
+
if defined? RAILS_ROOT
|
141
|
+
dir = rails_maestro_config_dir
|
142
|
+
elsif ENV.has_key? MAESTRO_DIR_ENV_VAR
|
143
|
+
dir = standalone_maestro_config_dir
|
144
|
+
else
|
145
|
+
raise "Maestro not configured correctly. Either RAILS_ROOT or ENV['#{MAESTRO_DIR_ENV_VAR}'] must be defined"
|
146
|
+
end
|
147
|
+
temp_file = Dir.tmpdir + "/" + MAESTRO_CHEF_ARCHIVE
|
148
|
+
File.delete(temp_file) if File.exist?(temp_file)
|
149
|
+
|
150
|
+
pwd = Dir.pwd
|
151
|
+
open temp_file, 'wb' do |io|
|
152
|
+
Zlib::GzipWriter.wrap io do |gzip|
|
153
|
+
begin
|
154
|
+
out = Archive::Tar::Minitar::Output.new(gzip)
|
155
|
+
Dir.chdir(dir) # don't store full paths in archive
|
156
|
+
Dir.glob("cookbooks/**/**").each do |file|
|
157
|
+
Archive::Tar::Minitar.pack_file(file, out) if File.file?(file) || File.directory?(file)
|
158
|
+
end
|
159
|
+
Dir.glob("roles/**/**").each do |file|
|
160
|
+
Archive::Tar::Minitar.pack_file(file, out) if File.file?(file) || File.directory?(file)
|
161
|
+
end
|
162
|
+
ensure
|
163
|
+
gzip.finish
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
Dir.chdir(pwd)
|
168
|
+
temp_file
|
169
|
+
end
|
170
|
+
|
171
|
+
|
172
|
+
private
|
173
|
+
|
174
|
+
# Creates the Maestro config directory structure in the given directory
|
175
|
+
#
|
176
|
+
# - dir: The base directory in which to create the maestro config directory structure
|
177
|
+
def self.create_configs(dir)
|
178
|
+
raise "Cannot create Maestro config directory structure: #{dir} doesn't exist." if !File.exist?(dir)
|
179
|
+
raise "Cannot create Maestro config directory structure: #{dir} is not a directory." if !File.directory?(dir)
|
180
|
+
raise "Cannot create Maestro config directory structure: #{dir} is not writable." if !File.writable?(dir)
|
181
|
+
base_dir = dir
|
182
|
+
base_dir.chop! if base_dir =~ /\/$/
|
183
|
+
if !File.exist?("#{base_dir}/maestro")
|
184
|
+
Dir.mkdir("#{base_dir}/maestro")
|
185
|
+
@logger.progress "Created #{base_dir}/maestro"
|
186
|
+
end
|
187
|
+
if !File.exist?("#{base_dir}/maestro/clouds")
|
188
|
+
Dir.mkdir("#{base_dir}/maestro/clouds")
|
189
|
+
@logger.info "Created #{base_dir}/maestro/clouds"
|
190
|
+
end
|
191
|
+
if !File.exist?("#{base_dir}/maestro/cookbooks")
|
192
|
+
Dir.mkdir("#{base_dir}/maestro/cookbooks")
|
193
|
+
@logger.info "Created #{base_dir}/maestro/cookbooks"
|
194
|
+
end
|
195
|
+
if !File.exist?("#{base_dir}/maestro/roles")
|
196
|
+
Dir.mkdir("#{base_dir}/maestro/roles")
|
197
|
+
@logger.info "Created #{base_dir}/maestro/roles"
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
# Creates the Maestro config directory structure in the given directory
|
202
|
+
#
|
203
|
+
# - dir: The base directory in which to create the maestro log directory structure
|
204
|
+
def self.create_logs(dir)
|
205
|
+
raise "Cannot create Maestro log directory: #{dir} doesn't exist." if !File.exist?(dir)
|
206
|
+
raise "Cannot create Maestro log directory: #{dir} is not a directory." if !File.directory?(dir)
|
207
|
+
raise "Cannot create Maestro log directory: #{dir} is not writable." if !File.writable?(dir)
|
208
|
+
begin
|
209
|
+
base_dir = dir
|
210
|
+
base_dir.chop! if base_dir =~ /\/$/
|
211
|
+
if !File.exist?("#{base_dir}/maestro")
|
212
|
+
Dir.mkdir("#{base_dir}/maestro")
|
213
|
+
@logger.info "Created #{base_dir}/maestro"
|
214
|
+
end
|
215
|
+
if !File.exist?("#{base_dir}/maestro/clouds")
|
216
|
+
Dir.mkdir("#{base_dir}/maestro/clouds")
|
217
|
+
@logger.info "Created #{base_dir}/maestro/clouds"
|
218
|
+
end
|
219
|
+
rescue SystemCallError => syserr
|
220
|
+
@logger.error "Error creating cloud directory"
|
221
|
+
@logger.error syserr
|
222
|
+
rescue StandardError => serr
|
223
|
+
@logger.error "Unexpected Error"
|
224
|
+
@logger.error serr
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
# Validates the maestro configuration found at RAILS_ROOT/config/maestro
|
229
|
+
def self.validate_rails_config
|
230
|
+
validate_config rails_maestro_config_dir
|
231
|
+
end
|
232
|
+
|
233
|
+
# Validates the maestro configuration found at ENV['MAESTRO_DIR']
|
234
|
+
def self.validate_standalone_config
|
235
|
+
validate_config standalone_maestro_config_dir
|
236
|
+
end
|
237
|
+
|
238
|
+
# Validates the maestro configuration found at the given maestro_directory
|
239
|
+
def self.validate_config(maestro_directory)
|
240
|
+
valid = true
|
241
|
+
error_messages = Array.new
|
242
|
+
if !File.exist?(maestro_directory)
|
243
|
+
valid = false
|
244
|
+
error_messages << "Maestro config directory does not exist: #{maestro_directory}"
|
245
|
+
end
|
246
|
+
if !File.directory?(maestro_directory)
|
247
|
+
valid = false
|
248
|
+
error_messages << "Maestro config directory is not a directory: #{maestro_directory}"
|
249
|
+
end
|
250
|
+
clouds_directory = clouds_config_dir maestro_directory
|
251
|
+
if !File.exist?(clouds_directory)
|
252
|
+
valid = false
|
253
|
+
error_messages << "Maestro clouds config directory does not exist: #{clouds_directory}"
|
254
|
+
end
|
255
|
+
if !File.directory?(clouds_directory)
|
256
|
+
valid = false
|
257
|
+
error_messages << "Maestro clouds config directory is not a directory: #{clouds_directory}"
|
258
|
+
end
|
259
|
+
cookbooks_directory = cookbooks_dir maestro_directory
|
260
|
+
if !File.exist?(cookbooks_directory)
|
261
|
+
valid = false
|
262
|
+
error_messages << "Chef cookbooks directory does not exist: #{cookbooks_directory}"
|
263
|
+
else
|
264
|
+
if !File.directory?(cookbooks_directory)
|
265
|
+
valid = false
|
266
|
+
error_messages << "Chef cookbooks directory is not a directory: #{cookbooks_directory}"
|
267
|
+
end
|
268
|
+
end
|
269
|
+
roles_directory = roles_dir maestro_directory
|
270
|
+
if !File.exist?(roles_directory)
|
271
|
+
valid = false
|
272
|
+
error_messages << "Chef roles directory does not exist: #{roles_directory}"
|
273
|
+
else
|
274
|
+
if !File.directory?(roles_directory)
|
275
|
+
valid = false
|
276
|
+
error_messages << "Chef roles directory is not a directory: #{roles_directory}"
|
277
|
+
end
|
278
|
+
end
|
279
|
+
if valid
|
280
|
+
clouds = get_clouds(clouds_directory)
|
281
|
+
clouds.each do |name, cloud|
|
282
|
+
cloud.validate
|
283
|
+
if !cloud.valid?
|
284
|
+
valid = false
|
285
|
+
error_messages << "INVALID: #{cloud.config_file}"
|
286
|
+
cloud.validation_errors.each {|error| error_messages << " #{error}"}
|
287
|
+
else
|
288
|
+
error_messages << "VALID: #{cloud.config_file}"
|
289
|
+
end
|
290
|
+
end
|
291
|
+
end
|
292
|
+
return [valid, error_messages]
|
293
|
+
end
|
294
|
+
|
295
|
+
def self.rails_config_dir
|
296
|
+
"#{RAILS_ROOT}/config"
|
297
|
+
end
|
298
|
+
|
299
|
+
def self.rails_log_dir
|
300
|
+
"#{RAILS_ROOT}/log"
|
301
|
+
end
|
302
|
+
|
303
|
+
def self.rails_maestro_config_dir
|
304
|
+
"#{RAILS_ROOT}#{MAESTRO_RAILS_CONFIG_DIRECTORY}"
|
305
|
+
end
|
306
|
+
|
307
|
+
def self.standalone_maestro_config_dir
|
308
|
+
"#{ENV[MAESTRO_DIR_ENV_VAR]}#{MAESTRO_RAILS_CONFIG_DIRECTORY}"
|
309
|
+
end
|
310
|
+
|
311
|
+
def self.standalone_config_dir
|
312
|
+
"#{ENV[MAESTRO_DIR_ENV_VAR]}/config"
|
313
|
+
end
|
314
|
+
|
315
|
+
def self.standalone_log_dir
|
316
|
+
"#{ENV[MAESTRO_DIR_ENV_VAR]}/log"
|
317
|
+
end
|
318
|
+
|
319
|
+
def self.clouds_config_dir(maestro_directory)
|
320
|
+
"#{maestro_directory}/clouds"
|
321
|
+
end
|
322
|
+
|
323
|
+
def self.cookbooks_dir(maestro_directory)
|
324
|
+
"#{maestro_directory}/cookbooks"
|
325
|
+
end
|
326
|
+
|
327
|
+
def self.roles_dir(maestro_directory)
|
328
|
+
"#{maestro_directory}/roles"
|
329
|
+
end
|
330
|
+
|
331
|
+
# Gets the Hash of Clouds found in clouds_directory
|
332
|
+
def self.get_clouds(clouds_directory)
|
333
|
+
clouds_directory << "/" unless clouds_directory =~ /\/$/
|
334
|
+
clouds = {}
|
335
|
+
config_files = get_cloud_config_files(clouds_directory)
|
336
|
+
config_files.each do |config_file|
|
337
|
+
cloud_name = config_file[clouds_directory.length, (config_file.length-(clouds_directory.length+File.extname(config_file).length))]
|
338
|
+
cloud = Cloud::Base.create_from_file(config_file)
|
339
|
+
clouds[cloud_name] = cloud
|
340
|
+
end
|
341
|
+
clouds
|
342
|
+
end
|
343
|
+
|
344
|
+
# Returns and array of all Cloud files found in clouds_directory
|
345
|
+
def self.get_cloud_config_files(clouds_directory)
|
346
|
+
config_files = []
|
347
|
+
Find.find(clouds_directory) do |path|
|
348
|
+
if FileTest.file?(path) && (File.extname(path).eql?(".rb"))
|
349
|
+
config_files << path
|
350
|
+
end
|
351
|
+
end
|
352
|
+
config_files
|
353
|
+
end
|
354
|
+
end
|
@@ -0,0 +1,384 @@
|
|
1
|
+
require "ftools"
|
2
|
+
require "maestro/role"
|
3
|
+
require "maestro/node"
|
4
|
+
require 'maestro/validator'
|
5
|
+
require "net/ssh/multi"
|
6
|
+
require "log4r"
|
7
|
+
require "maestro/log4r/console_formatter"
|
8
|
+
require "maestro/log4r/file_formatter"
|
9
|
+
|
10
|
+
|
11
|
+
module Maestro
|
12
|
+
# A named cloud (i.e. production, staging, test, dev, etc)
|
13
|
+
module Cloud
|
14
|
+
class Base
|
15
|
+
include Validator
|
16
|
+
|
17
|
+
# the name of this cloud
|
18
|
+
attr_reader :name
|
19
|
+
# the config file for this cloud
|
20
|
+
attr_accessor :config_file
|
21
|
+
# the Hash of Configurable Nodes in this Cloud
|
22
|
+
attr_reader :configurable_nodes
|
23
|
+
# String containing the full path to this Cloud's log directory
|
24
|
+
attr_reader :log_directory
|
25
|
+
dsl_property :keypair_name, :keypair_file
|
26
|
+
|
27
|
+
# Creates a new Cloud object.
|
28
|
+
# * name: the name of the Cloud
|
29
|
+
# * cfg_file: Pointer to the file containing the Cloud configuration (optional)
|
30
|
+
# * block: contents of the Cloud
|
31
|
+
def initialize(name, cfg_file=nil, &block)
|
32
|
+
super()
|
33
|
+
raise StandardError, "Cloud name cannot contain spaces: #{name}" if name.is_a?(String) && !name.index(/\s/).nil?
|
34
|
+
@name = name
|
35
|
+
@config_file = cfg_file
|
36
|
+
@roles = Hash.new
|
37
|
+
@nodes = Hash.new
|
38
|
+
@configurable_nodes = Hash.new
|
39
|
+
@valid = true
|
40
|
+
@logger = Log4r::Logger.new(Regexp::quote(@name.to_s))
|
41
|
+
outputter = Log4r::StdoutOutputter.new("#{@name.to_s}-stdout")
|
42
|
+
outputter.formatter = ConsoleFormatter.new
|
43
|
+
@logger.add(outputter)
|
44
|
+
init_logs
|
45
|
+
instance_eval(&block) if block_given?
|
46
|
+
end
|
47
|
+
|
48
|
+
# Creates a Cloud from the contents of the given file
|
49
|
+
def self.create_from_file(config_file)
|
50
|
+
cloud = eval(File.read(config_file))
|
51
|
+
cloud.config_file = config_file
|
52
|
+
return cloud
|
53
|
+
end
|
54
|
+
|
55
|
+
# creates the Roles for this Cloud if a block is given. Otherwise, returns the roles Hash
|
56
|
+
def roles(&block)
|
57
|
+
if block_given?
|
58
|
+
instance_eval(&block)
|
59
|
+
else
|
60
|
+
@roles
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# sets the roles Hash
|
65
|
+
def roles=(roles)
|
66
|
+
@roles = roles
|
67
|
+
end
|
68
|
+
|
69
|
+
# creates a Role
|
70
|
+
def role(name, &block)
|
71
|
+
if @roles.has_key?(name)
|
72
|
+
invalidate "Duplicate role definition: #{name}"
|
73
|
+
else
|
74
|
+
@roles[name] = Role.new(name, self, &block)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# creates the Nodes for this Cloud if a block is given, otherwise returns the Nodes in this Cloud
|
79
|
+
def nodes(&block)
|
80
|
+
if block_given?
|
81
|
+
instance_eval(&block)
|
82
|
+
else
|
83
|
+
@nodes
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# sets the nodes Hash
|
88
|
+
def nodes=(nodes)
|
89
|
+
@nodes = nodes
|
90
|
+
end
|
91
|
+
|
92
|
+
def method_missing(name, *params) #:nodoc:
|
93
|
+
@valid = false
|
94
|
+
@validation_errors << "Unexpected attribute: #{name}"
|
95
|
+
end
|
96
|
+
|
97
|
+
# Reports the current status of this Cloud
|
98
|
+
def status
|
99
|
+
@logger.info "#{@name} Cloud status:"
|
100
|
+
node_statuses
|
101
|
+
end
|
102
|
+
|
103
|
+
# Starts this Cloud. Takes no action if the Cloud is already running as currently configured
|
104
|
+
def start
|
105
|
+
@logger.info "Starting #{@name} Cloud. This may take a few minutes..."
|
106
|
+
end
|
107
|
+
|
108
|
+
# Configures the Nodes in this Cloud
|
109
|
+
def configure
|
110
|
+
@logger.info "Configuring #{@name} Cloud"
|
111
|
+
if !@configurable_nodes.empty?
|
112
|
+
session = open_ssh_session
|
113
|
+
result = chef_solo_installed?(session)
|
114
|
+
if !result[0]
|
115
|
+
names = result[1].collect {|n| n.name}
|
116
|
+
@logger.progress "Installing chef-solo on Nodes #{names.inspect}. This may take a few minutes..."
|
117
|
+
session.close
|
118
|
+
session = open_ssh_session(result[1])
|
119
|
+
install_chef_solo(session)
|
120
|
+
configure_chef_solo(session)
|
121
|
+
session.close
|
122
|
+
@logger.progress "\n"
|
123
|
+
else
|
124
|
+
@logger.info "chef-solo already installed on Nodes #{@configurable_nodes.keys.inspect}"
|
125
|
+
end
|
126
|
+
@logger.info "Running chef-solo on Nodes #{@configurable_nodes.keys.inspect}..."
|
127
|
+
session = open_ssh_session
|
128
|
+
run_chef_solo(session)
|
129
|
+
session.close
|
130
|
+
@logger.info "Configuration of #{@name} Cloud complete"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
# Checks if chef-solo is installed on each of the Configurable Nodes in this Cloud.
|
135
|
+
# This method returns an Array with two elements:
|
136
|
+
# * element[0] boolean indicating whether chef-solo is installed on all Configurable Nodes
|
137
|
+
# * element[1] Array of Nodes which need chef-solo installed
|
138
|
+
def chef_solo_installed?(session=nil)
|
139
|
+
close_session = false
|
140
|
+
if session.nil?
|
141
|
+
session = open_ssh_session
|
142
|
+
close_session = true
|
143
|
+
end
|
144
|
+
@logger.info "Checking for installation of chef-solo..."
|
145
|
+
valid = true
|
146
|
+
needs_chef = Array.new
|
147
|
+
session.open_channel do |channel|
|
148
|
+
# Find the node for this channel's host
|
149
|
+
the_node = nil
|
150
|
+
@configurable_nodes.each_pair {|name, node| the_node = node if channel[:host].eql? node.hostname}
|
151
|
+
if the_node.nil?
|
152
|
+
@logger.error "Could not find node matching hostname #{channel[:host]}. This should not happen."
|
153
|
+
else
|
154
|
+
channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
|
155
|
+
channel.exec("chef-solo --version") do |ch, success|
|
156
|
+
ch.on_data do |ch, data|
|
157
|
+
if !data.include?("Chef: 0.8")
|
158
|
+
valid = false
|
159
|
+
needs_chef << the_node
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
session.loop(60)
|
166
|
+
session.close if close_session
|
167
|
+
return [valid, needs_chef]
|
168
|
+
end
|
169
|
+
|
170
|
+
# installs chef-solo on the Configurable Nodes the given session is set up with
|
171
|
+
def install_chef_solo(session=nil)
|
172
|
+
close_session = false
|
173
|
+
if session.nil?
|
174
|
+
session = open_ssh_session
|
175
|
+
close_session = true
|
176
|
+
end
|
177
|
+
etc_issue = nil
|
178
|
+
session.open_channel do |channel|
|
179
|
+
channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
|
180
|
+
channel.exec("cat /etc/issue") do |ch, success|
|
181
|
+
ch.on_data {|ch, data| etc_issue = data}
|
182
|
+
end
|
183
|
+
end
|
184
|
+
session.loop(60)
|
185
|
+
|
186
|
+
# shut off the stdout outputter and only log to the nodes' log files
|
187
|
+
@configurable_nodes.each_pair {|name, node| node.disable_stdout}
|
188
|
+
os = Maestro::OperatingSystem.create_from_etc_issue(etc_issue)
|
189
|
+
os.chef_install_script.each do |cmd|
|
190
|
+
session.open_channel do |channel|
|
191
|
+
# Find the node for this channel's host
|
192
|
+
the_node = nil
|
193
|
+
@configurable_nodes.each_pair {|name, node| the_node = node if channel[:host].eql? node.hostname}
|
194
|
+
if the_node.nil?
|
195
|
+
@logger.error "Could not find node matching hostname #{channel[:host]}. This should not happen."
|
196
|
+
else
|
197
|
+
the_node.logger.info "Installing chef-solo"
|
198
|
+
channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
|
199
|
+
channel.exec(cmd) do |ch, success|
|
200
|
+
@logger.progress "."
|
201
|
+
ch.on_data {|ch, data| the_node.logger.info data}
|
202
|
+
ch.on_extended_data {|ch, data| the_node.logger.error }
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
session.loop(60)
|
207
|
+
end
|
208
|
+
# turn the stdout outputter back on
|
209
|
+
@configurable_nodes.each_pair {|name, node| node.enable_stdout}
|
210
|
+
session.close if close_session
|
211
|
+
end
|
212
|
+
|
213
|
+
# configures chef-solo
|
214
|
+
def configure_chef_solo(session=nil)
|
215
|
+
close_session = false
|
216
|
+
if session.nil?
|
217
|
+
session = open_ssh_session
|
218
|
+
close_session = true
|
219
|
+
end
|
220
|
+
# write the chef-solo config file
|
221
|
+
chef_solo_config =
|
222
|
+
["sudo rm /tmp/chef-solo.rb",
|
223
|
+
"sudo mkdir -p /tmp/chef-solo",
|
224
|
+
"sudo mkdir -p /tmp/chef-solo/cookbooks",
|
225
|
+
"sudo mkdir -p /tmp/chef-solo/roles",
|
226
|
+
"sudo sh -c 'echo file_cache_path \\\"/tmp/chef-solo\\\" >> /tmp/chef-solo.rb'",
|
227
|
+
"sudo sh -c 'echo cookbook_path \\\"/tmp/chef-solo/cookbooks\\\" >> /tmp/chef-solo.rb'",
|
228
|
+
"sudo sh -c 'echo role_path \\\"/tmp/chef-solo/roles\\\" >> /tmp/chef-solo.rb'"]
|
229
|
+
chef_solo_config.each do |str|
|
230
|
+
session.open_channel do |channel|
|
231
|
+
channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
|
232
|
+
channel.exec(str)
|
233
|
+
end
|
234
|
+
session.loop(60)
|
235
|
+
end
|
236
|
+
session.close if close_session
|
237
|
+
end
|
238
|
+
|
239
|
+
# runs chef-solo
|
240
|
+
def run_chef_solo(session=nil)
|
241
|
+
close_session = false
|
242
|
+
if session.nil?
|
243
|
+
session = open_ssh_session
|
244
|
+
close_session = true
|
245
|
+
end
|
246
|
+
commands =
|
247
|
+
["sudo chef-solo -c /tmp/chef-solo.rb -r '#{chef_assets_url()}'"]
|
248
|
+
# shut off the stdout outputter and only log to the nodes' log files
|
249
|
+
@configurable_nodes.each_pair {|name, node| node.disable_stdout}
|
250
|
+
commands.each do |cmd|
|
251
|
+
session.open_channel do |channel|
|
252
|
+
channel.request_pty {|ch, success| abort "could not obtain pty" if !success}
|
253
|
+
# Find the node for this channel's host
|
254
|
+
the_node = nil
|
255
|
+
@configurable_nodes.each_pair {|name, node| the_node = node if channel[:host].eql? node.hostname}
|
256
|
+
if the_node.nil?
|
257
|
+
@logger.error "Could not find node matching hostname #{channel[:host]}. This should not happen."
|
258
|
+
else
|
259
|
+
node_cmd = cmd + " -j '#{node_json_url(the_node)}'"
|
260
|
+
channel.exec(node_cmd) do |ch, success|
|
261
|
+
ch.on_data {|ch2, data2| the_node.logger.info data2}
|
262
|
+
ch.on_extended_data {|ch2, data2| the_node.logger.error data2}
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
session.loop(60)
|
267
|
+
end
|
268
|
+
# turn the stdout outputter back on
|
269
|
+
@configurable_nodes.each_pair {|name, node| node.enable_stdout}
|
270
|
+
session.close if close_session
|
271
|
+
end
|
272
|
+
|
273
|
+
# Shuts down this Cloud. Takes no action if the Cloud is not running
|
274
|
+
def shutdown
|
275
|
+
@logger.info "Shutting down #{@name} Cloud"
|
276
|
+
end
|
277
|
+
|
278
|
+
|
279
|
+
protected
|
280
|
+
|
281
|
+
# creates log directory and files for this cloud. If the log directory or files exist, no action is taken.
|
282
|
+
def init_logs
|
283
|
+
begin
|
284
|
+
if !File.exists?(Maestro.maestro_log_directory)
|
285
|
+
Maestro.create_log_dirs
|
286
|
+
@logger.info "Created #{Maestro.maestro_log_directory}"
|
287
|
+
end
|
288
|
+
clouds_dir = Maestro.maestro_log_directory + "/clouds"
|
289
|
+
if !File.exists?(clouds_dir)
|
290
|
+
Dir.mkdir(clouds_dir)
|
291
|
+
@logger.info "Created #{clouds_dir}"
|
292
|
+
end
|
293
|
+
cloud_dir = clouds_dir + "/#{@name}"
|
294
|
+
if !File.exists?(cloud_dir)
|
295
|
+
Dir.mkdir(cloud_dir)
|
296
|
+
@logger.info "Created #{cloud_dir}"
|
297
|
+
end
|
298
|
+
@log_directory = cloud_dir
|
299
|
+
cloud_log_file = cloud_dir + "/#{@name}.log"
|
300
|
+
if !File.exists?(cloud_log_file)
|
301
|
+
File.new(cloud_log_file, "a+")
|
302
|
+
@logger.info "Created #{cloud_log_file}"
|
303
|
+
end
|
304
|
+
outputter = Log4r::FileOutputter.new("#{@name}-file", :formatter => FileFormatter.new, :filename => cloud_log_file, :truncate => false)
|
305
|
+
@logger.add(outputter)
|
306
|
+
rescue RuntimeError => rerr
|
307
|
+
if !rerr.message.eql?("Maestro not configured correctly. Either RAILS_ROOT or ENV['MAESTRO_DIR'] must be defined")
|
308
|
+
@logger.error "Unexpected Error"
|
309
|
+
@logger.error rerr
|
310
|
+
end
|
311
|
+
rescue SystemCallError => syserr
|
312
|
+
@logger.error "Error creating cloud directory"
|
313
|
+
@logger.error syserr
|
314
|
+
rescue StandardError => serr
|
315
|
+
@logger.error "Unexpected Error"
|
316
|
+
@logger.error serr
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
# opens a multi ssh session. If the cnodes argument is nil, then a session
|
321
|
+
# is opened up to each Configurable Node in this Cloud. Otherwise, a session
|
322
|
+
# is opened to each Configurable Node in the cnodes array.
|
323
|
+
def open_ssh_session(cnodes=[])
|
324
|
+
handler = Proc.new do |server|
|
325
|
+
server[:connection_attempts] ||= 0
|
326
|
+
if server[:connection_attempts] < 50
|
327
|
+
server[:connection_attempts] += 1
|
328
|
+
sleep 2
|
329
|
+
throw :go, :retry
|
330
|
+
else
|
331
|
+
throw :go, :raise
|
332
|
+
end
|
333
|
+
end
|
334
|
+
|
335
|
+
session = Net::SSH::Multi.start(:concurrent_connections => 10, :on_error => handler)
|
336
|
+
if cnodes.empty?
|
337
|
+
@configurable_nodes.each_pair {|node_name, node| session.use node.hostname, :user => node.ssh_user, :keys => [keypair_file]}
|
338
|
+
else
|
339
|
+
cnodes.each {|node| session.use node.hostname, :user => node.ssh_user, :keys => [keypair_file]}
|
340
|
+
end
|
341
|
+
return session
|
342
|
+
end
|
343
|
+
|
344
|
+
|
345
|
+
private
|
346
|
+
|
347
|
+
# validates this Cloud
|
348
|
+
def validate_internal
|
349
|
+
invalidate "Missing keypair_name" if @keypair_name.nil?
|
350
|
+
invalidate "Missing keypair_file" if @keypair_file.nil?
|
351
|
+
validate_roles
|
352
|
+
validate_nodes
|
353
|
+
end
|
354
|
+
|
355
|
+
# validates the roles in the cloud config
|
356
|
+
def validate_roles
|
357
|
+
if @roles.nil? || @roles.empty?
|
358
|
+
invalidate "Missing roles"
|
359
|
+
else
|
360
|
+
@roles.each do |name, role|
|
361
|
+
role.validate
|
362
|
+
if !role.valid?
|
363
|
+
role.validation_errors.each {|error_str| invalidate error_str}
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
end
|
368
|
+
|
369
|
+
# validates the nodes in the cloud config
|
370
|
+
def validate_nodes
|
371
|
+
if @nodes.nil? || @nodes.empty?
|
372
|
+
invalidate "Missing nodes"
|
373
|
+
else
|
374
|
+
@nodes.each do |name, node|
|
375
|
+
node.validate
|
376
|
+
if !node.valid?
|
377
|
+
node.validation_errors.each {|error_str| invalidate error_str}
|
378
|
+
end
|
379
|
+
end
|
380
|
+
end
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|