ec2launcher 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,154 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require 'rubygems'
4
+
5
+ require 'optparse'
6
+ require 'ostruct'
7
+
8
+ require 'json'
9
+
10
+ SETUP_SCRIPT = "setup_instance.rb"
11
+ SETUP_SCRIPT_URL = "https://s3.amazonaws.com/startup-scripts/#{SETUP_SCRIPT}"
12
+
13
+ class InitOptions
14
+ def initialize
15
+ @opts = OptionParser.new do |opts|
16
+ opts.banner = "Usage: #{__FILE__} [SETUP.JSON] [options]"
17
+ opts.separator ""
18
+
19
+ opts.on("-e", "--environment ENV", "The environment for the server.") do |env|
20
+ @options.environ = env
21
+ end
22
+
23
+ opts.on("-a", "--application NAME", "The name of the application class for the new server.") do |app_name|
24
+ @options.application = app_name
25
+ end
26
+
27
+ opts.on("-h", "--hostname NAME", "The name for the new server.") do |hostname|
28
+ @options.hostname = hostname
29
+ end
30
+
31
+ opts.separator ""
32
+ opts.separator "Additional launch options:"
33
+
34
+ opts.on("-c", "--clone HOST", "Clone the latest snapshots from a specific host.") do |clone_host|
35
+ @options.clone_host = clone_host
36
+ end
37
+
38
+ opts.separator ""
39
+ opts.separator "Common options:"
40
+
41
+ # No argument, shows at tail. This will print an options summary.
42
+ # Try it and see!
43
+ opts.on_tail("-?", "--help", "Show this message") do
44
+ puts opts
45
+ exit
46
+ end
47
+ end
48
+ end
49
+
50
+ def parse(args)
51
+ @options = OpenStruct.new
52
+
53
+ @options.environ = nil
54
+ @options.application = nil
55
+ @options.hostname = nil
56
+
57
+ @options.clone_host = nil
58
+
59
+ @opts.parse!(args)
60
+ @options
61
+ end
62
+
63
+ def help
64
+ puts @opts
65
+ end
66
+ end
67
+
68
+ # Runs a command and displays the output line by line
69
+ def run_command(cmd)
70
+ IO.popen(cmd) do |f|
71
+ while ! f.eof
72
+ puts f.gets
73
+ end
74
+ end
75
+ end
76
+
77
+ option_parser = InitOptions.new
78
+ options = option_parser.parse(ARGV)
79
+
80
+ setup_json_filename = ARGV[0]
81
+
82
+ # Read the setup JSON file
83
+ instance_data = JSON.parse(File.read(setup_json_filename))
84
+
85
+ # Pre-install gems
86
+ unless instance_data["gems"].nil?
87
+ instance_data["gems"].each {|gem_name| puts `/usr/bin/gem install --no-rdoc --no-ri #{gem_name}` }
88
+ end
89
+
90
+ # Pre-install packages
91
+ unless instance_data["packages"].nil?
92
+ instance_data["packages"].each {|pkg_name| puts `yum install -y #{pkg_name}` }
93
+ end
94
+
95
+
96
+ # Load the AWS access keys
97
+ properties = {}
98
+ File.open(instance_data['aws_keyfile'], 'r') do |file|
99
+ file.read.each_line do |line|
100
+ line.strip!
101
+ if (line[0] != ?# and line[0] != ?=)
102
+ i = line.index('=')
103
+ if (i)
104
+ properties[line[0..i - 1].strip] = line[i + 1..-1].strip
105
+ else
106
+ properties[line] = ''
107
+ end
108
+ end
109
+ end
110
+ end
111
+ AWS_ACCESS_KEY = properties["AWS_ACCESS_KEY"].gsub('"', '')
112
+ AWS_SECRET_ACCESS_KEY = properties["AWS_SECRET_ACCESS_KEY"].gsub('"', '')
113
+
114
+ # Create s3curl auth file
115
+ s3curl_auth_data = <<EOF
116
+ %awsSecretAccessKeys = (
117
+ # personal account
118
+ startup => {
119
+ id => '#{AWS_ACCESS_KEY}',
120
+ key => '#{AWS_SECRET_ACCESS_KEY}'
121
+ }
122
+ );
123
+ EOF
124
+
125
+ home_folder = `echo $HOME`.strip
126
+ File.open("#{home_folder}/.s3curl", "w") do |f|
127
+ f.puts s3curl_auth_data
128
+ end
129
+ `chmod 600 #{home_folder}/.s3curl`
130
+
131
+ # Retrieve validation.pem
132
+ puts "Retrieving Chef validation.pem ..."
133
+ puts `s3curl.pl --id startup #{instance_data['chef_validation_pem_url']} > /etc/chef/validation.pem`
134
+
135
+ # Setting hostname
136
+ puts "Setting hostname ... #{options.hostname}"
137
+ `hostname #{options.hostname}`
138
+ `sed -i 's/^HOSTNAME=.*$/HOSTNAME=#{options.hostname}/' /etc/sysconfig/network`
139
+
140
+ # Set Chef node name
141
+ File.open("/etc/chef/client.rb", 'a') { |f| f.write("node_name \"#{options.hostname}\"") }
142
+
143
+ # Setup Chef client
144
+ puts "Connecting to Chef ..."
145
+ `rm -f /etc/chef/client.pem`
146
+ puts `chef-client`
147
+
148
+ # Retrieve secondary setup script and run it
149
+ puts "Getting role setup script ..."
150
+ puts `s3curl.pl --id startup #{SETUP_SCRIPT_URL} > /tmp/#{SETUP_SCRIPT} && chmod +x /tmp/#{SETUP_SCRIPT}`
151
+ command = "/tmp/#{SETUP_SCRIPT} -a #{options.application} -e #{options.environ} -h #{options.hostname} #{setup_json_filename}"
152
+ command += " -c #{options.clone_host}" unless options.clone_host.nil?
153
+ command += " > /var/log/cloud-init.log"
154
+ run_command(command)
@@ -0,0 +1,340 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require 'rubygems'
4
+
5
+ require 'optparse'
6
+ require 'ostruct'
7
+
8
+ require 'json'
9
+
10
+ require 'aws-sdk'
11
+
12
+ AWS_KEYS = "/etc/aws/startup_runner_keys"
13
+
14
+ class InitOptions
15
+ def initialize
16
+ @opts = OptionParser.new do |opts|
17
+ opts.banner = "Usage: #{__FILE__} [SETUP.JSON] [options]"
18
+ opts.separator ""
19
+
20
+ opts.on("-e", "--environment ENV", "The environment for the server.") do |env|
21
+ @options.environ = env
22
+ end
23
+
24
+ opts.on("-a", "--application NAME", "The name of the application class for the new server.") do |app_name|
25
+ @options.application = app_name
26
+ end
27
+
28
+ opts.on("-h", "--hostname NAME", "The name for the new server.") do |hostname|
29
+ @options.hostname = hostname
30
+ end
31
+
32
+ opts.separator ""
33
+ opts.separator "Additional launch options:"
34
+
35
+ opts.on("-c", "--clone HOST", "Clone the latest snapshots from a specific host.") do |clone_host|
36
+ @options.clone_host = clone_host
37
+ end
38
+
39
+ opts.separator ""
40
+ opts.separator "Common options:"
41
+
42
+ # No argument, shows at tail. This will print an options summary.
43
+ # Try it and see!
44
+ opts.on_tail("-?", "--help", "Show this message") do
45
+ puts opts
46
+ exit
47
+ end
48
+ end
49
+ end
50
+
51
+ def parse(args)
52
+ @options = OpenStruct.new
53
+
54
+ @options.environ = nil
55
+ @options.application = nil
56
+ @options.hostname = nil
57
+
58
+ @options.clone_host = nil
59
+
60
+ @opts.parse!(args)
61
+ @options
62
+ end
63
+
64
+ def help
65
+ puts @opts
66
+ end
67
+ end
68
+
69
+ ##############################
70
+ # Wrapper that retries failed calls to AWS
71
+ # with an exponential back-off rate.
72
+ def retry_aws_with_backoff(&block)
73
+ timeout = 1
74
+ result = nil
75
+ while timeout < 33 && result.nil?
76
+ begin
77
+ result = yield
78
+ rescue AWS::Errors::ServerError
79
+ puts "Error contacting Amazon. Sleeping #{timeout} seconds."
80
+ sleep timeout
81
+ timeout *= 2
82
+ result = nil
83
+ end
84
+ end
85
+ result
86
+ end
87
+
88
+
89
+ # Runs a command and displays the output line by line
90
+ def run_command(cmd)
91
+ IO.popen(cmd) do |f|
92
+ while ! f.eof
93
+ puts f.gets
94
+ end
95
+ end
96
+ end
97
+
98
+ option_parser = InitOptions.new
99
+ options = option_parser.parse(ARGV)
100
+
101
+ setup_json_filename = ARGV[0]
102
+
103
+ # Load the AWS access keys
104
+ properties = {}
105
+ File.open(AWS_KEYS, 'r') do |file|
106
+ file.read.each_line do |line|
107
+ line.strip!
108
+ if (line[0] != ?# and line[0] != ?=)
109
+ i = line.index('=')
110
+ if (i)
111
+ properties[line[0..i - 1].strip] = line[i + 1..-1].strip
112
+ else
113
+ properties[line] = ''
114
+ end
115
+ end
116
+ end
117
+ end
118
+ AWS_ACCESS_KEY = properties["AWS_ACCESS_KEY"].gsub('"', '')
119
+ AWS_SECRET_ACCESS_KEY = properties["AWS_SECRET_ACCESS_KEY"].gsub('"', '')
120
+
121
+ ##############################
122
+ # Find current instance data
123
+ EC2_INSTANCE_TYPE = `wget -T 5 -q -O - http://169.254.169.254/latest/meta-data/instance-type`
124
+
125
+ # Read the setup JSON file
126
+ instance_data = JSON.parse(File.read(setup_json_filename))
127
+
128
+ ##############################
129
+ # Block devices
130
+ ##############################
131
+
132
+ # Creates filesystem on a device
133
+ # XFS on 64-bit
134
+ # ext4 on 32-bit
135
+ def format_filesystem(system_arch, device)
136
+ fs_type = system_arch == "x86_64" ? "XFS" : "ext4"
137
+ puts "Formatting #{fs_type} filesystem on #{device} ..."
138
+
139
+ command = case system_arch
140
+ when "x86_64" then "/sbin/mkfs.xfs -f #{device}"
141
+ else "/sbin/mkfs.ext4 -F #{device}"
142
+ end
143
+ IO.popen(command) do |f|
144
+ while ! f.eof
145
+ puts f.gets
146
+ end
147
+ end
148
+ end
149
+
150
+ # Creates and formats a RAID array, given a
151
+ # list of partitioned devices
152
+ def initialize_raid_array(system_arch, device_list, raid_device = '/dev/md0', raid_type = 0)
153
+ partitions = device_list.collect {|device| "#{device}1" }
154
+
155
+ puts "Creating RAID-#{raid_type.to_s} array #{raid_device} ..."
156
+ command = "/sbin/mdadm --create #{raid_device} --level #{raid_type.to_s} --raid-devices #{partitions.length} #{partitions.join(' ')}"
157
+ puts command
158
+ puts `#{command}`
159
+
160
+ format_filesystem(system_arch, raid_device)
161
+ end
162
+
163
+ # Creates a mount point, mounts the device and adds it to fstab
164
+ def mount_device(device, mount_point, owner, group, fs_type)
165
+ puts `echo #{device} #{mount_point} #{fs_type} noatime 0 0|tee -a /etc/fstab`
166
+ puts `mkdir -p #{mount_point}`
167
+ puts `mount #{mount_point}`
168
+ puts `chown #{owner}:#{owner} #{mount_point}`
169
+ end
170
+
171
+ # Partitions a list of mounted EBS volumes
172
+ def partition_devices(device_list)
173
+ puts "Partioning devices ..."
174
+ device_list.each do |device|
175
+ puts " * #{device}"
176
+ `echo 0|sfdisk #{device}`
177
+ end
178
+
179
+ puts "Sleeping 10 seconds to reload partition tables ..."
180
+ sleep 10
181
+ end
182
+
183
+ ##############################
184
+ # Assembles a set of existing partitions into a RAID array.
185
+ def assemble_raid_array(partition_list, raid_device = '/dev/md0', raid_type = 0)
186
+ mylog.info "Assembling cloned RAID-#{raid_type.to_s} array #{raid_device} ..."
187
+ command = "/sbin/mdadm --assemble #{raid_device} #{partition_list.join(' ')}"
188
+ puts command
189
+ puts `#{command}`
190
+ end
191
+
192
+ # Initializes a raid array with existing EBS volumes that are already attached to the instace.
193
+ # Partitions & formats new volumes.
194
+ # Returns the RAID device name.
195
+ def setup_attached_raid_array(system_arch, devices, raid_device = '/dev/md0', raid_type = 0, clone = false)
196
+ partitions = devices.collect {|device| "#{device}1" }
197
+
198
+ unless clone
199
+ partition_devices(devices)
200
+ initialize_raid_array(system_arch, devices, raid_device, raid_type)
201
+ else
202
+ assemble_raid_array(partitions, raid_device, raid_type)
203
+ end
204
+ `echo DEVICE #{partitions.join(' ')} |tee -a /etc/mdadm.conf`
205
+
206
+ # RAID device name can be a symlink on occasion, so we
207
+ # want to de-reference the symlink to keep everything clear.
208
+ raid_info = `/sbin/mdadm --detail --scan`.split("\n")[-1].split()
209
+ Pathname.new(raid_info[1]).realpath.to_s
210
+ end
211
+
212
+ def build_block_devices(count, device = "xvdj", &block)
213
+ device_name = device
214
+ 0.upto(count - 1).each do |index|
215
+ yield device_name, index
216
+ device_name.next!
217
+ end
218
+ end
219
+
220
+ system_arch = `uname -p`.strip
221
+ default_fs_type = system_arch == "x86_64" ? "xfs" : "ext4"
222
+
223
+ # Process ephemeral devices first
224
+ ephemeral_drive_count = case EC2_INSTANCE_TYPE
225
+ when "m1.small" then 1
226
+ when "m1.medium" then 1
227
+ when "m2.xlarge" then 1
228
+ when "m2.2xlarge" then 1
229
+ when "c1.medium" then 1
230
+ when "m1.large" then 2
231
+ when "m2.4xlarge" then 2
232
+ when "cc1.4xlarge" then 2
233
+ when "cg1.4xlarge" then 2
234
+ when "m1.xlarge" then 4
235
+ when "c1.xlarge" then 4
236
+ when "cc2.8xlarge" then 4
237
+ else 0
238
+ end
239
+
240
+ # Partition the ephemeral drives
241
+ partition_list = []
242
+ build_block_devices(ephemeral_drive_count, "xvdf") do |device_name, index|
243
+ partition_list << "/dev/#{device_name}"
244
+ end
245
+ partition_devices(partition_list)
246
+
247
+ # Format and mount the ephemeral drives
248
+ build_block_devices(ephemeral_drive_count, "xvdf") do |device_name, index|
249
+ format_filesystem(system_arch, "/dev/#{device_name}1")
250
+
251
+ mount_point = case index
252
+ when 1 then "/mnt"
253
+ else "/mnt/extra#{index - 1}"
254
+ end
255
+ mount_device("/dev/#{device_name}1", mount_point, "root", "root", default_fs_type)
256
+ end
257
+
258
+ # Process EBS volumes
259
+ unless instance_data["block_devices"].nil?
260
+ next_device_name = "xvdj"
261
+ instance_data["block_devices"].each do |block_device_json|
262
+ if block_device_json["raid_level"].nil?
263
+ # If we're not cloning an existing snapshot, then we need to partition and format the drive.
264
+ if options.clone_host.nil?
265
+ partition_devices([ "/dev/#{next_device_name}" ])
266
+ format_filesystem(system_arch, "/dev/#{next_device_name}1")
267
+ end
268
+ mount_device("#{next_device_name}1", block_device_json["mount_point"], block_device_json["owner"], block_device_json["group"], default_fs_type)
269
+ next_device_name.next!
270
+ else
271
+ raid_devices = []
272
+ build_block_devices(block_device_json["count"], next_device_name) do |device_name, index|
273
+ raid_devices << "/dev/#{device_name}"
274
+ next_device_name = device_name
275
+ end
276
+ raid_device_name = setup_attached_raid_array(system_arch, raid_devices, "/dev/md0", block_device_json["raid_level"].to_i, ! options.clone_host.nil?)
277
+ mount_device(raid_device_name, block_device_json["mount_point"], block_device_json["owner"], block_device_json["group"], default_fs_type)
278
+ end
279
+ end
280
+ end
281
+
282
+ ##############################
283
+ # CHEF SETUP
284
+ ##############################
285
+
286
+ ##############################
287
+ # Create knife configuration
288
+ knife_config = <<EOF
289
+ log_level :info
290
+ log_location STDOUT
291
+ node_name '#{options.hostname}'
292
+ client_key '/etc/chef/client.pem'
293
+ validation_client_name 'chef-validator'
294
+ validation_key '/etc/chef/validation.pem'
295
+ chef_server_url '#{instance_data["chef_server_url"]}'
296
+ cache_type 'BasicFile'
297
+ cache_options( :path => '/etc/chef/checksums' )
298
+ EOF
299
+ home_folder = `echo $HOME`.strip
300
+ `mkdir -p #{home_folder}/.chef && chown 700 #{home_folder}/.chef`
301
+ File.open("#{home_folder}/.chef/knife.rb", "w") do |f|
302
+ f.puts knife_config
303
+ end
304
+ `chmod 600 #{home_folder}/.chef/knife.rb`
305
+
306
+ ##############################
307
+ # Add roles
308
+ instance_data["roles"].each do |role|
309
+ cmd = "knife node run_list add #{options.hostname} \"role[#{role}]\""
310
+ puts cmd
311
+ puts `#{cmd}`
312
+ end
313
+
314
+ ##############################
315
+ # Launch Chef
316
+ IO.popen("chef-client") do |f|
317
+ while ! f.eof
318
+ puts f.gets
319
+ end
320
+ end
321
+
322
+ ##############################
323
+ # EMAIL NOTIFICATION
324
+ ##############################
325
+
326
+ unless instance_data["email_notification"].nil?
327
+ # Email notification through SES
328
+ AWS.config({
329
+ :access_key_id => instance_data["email_notification"]["ses_access_key"],
330
+ :secret_access_key => instance_data["email_notification"]["ses_secret_key"]
331
+ })
332
+ ses = AWS::SimpleEmailService.new
333
+ ses.send_email(
334
+ :from => instance_data["email_notification"]["from"],
335
+ :to => instance_data["email_notification"]["to"],
336
+ :subject => "Server setup complete: #{hostname}",
337
+ :body_text => "Server setup is complete for Host: #{hostname}, Environment: #{options.environ}, Application: #{options.application}",
338
+ :body_html => "<div>Server setup is complete for:</div><div><strong>Host:</strong> #{hostname}</div><div><strong>Environment:</strong> #{options.environ}</div><div><strong>Application:</strong> #{options.application}</div>"
339
+ )
340
+ end
metadata ADDED
@@ -0,0 +1,81 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ec2launcher
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Sean Laurent
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-07-17 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: aws-sdk
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: 1.5.0
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.5.0
30
+ description: Tool to manage application configurations and launch new EC2 instances
31
+ based on the configurations.
32
+ email:
33
+ executables:
34
+ - ec2launcher
35
+ extensions: []
36
+ extra_rdoc_files: []
37
+ files:
38
+ - .gitignore
39
+ - Gemfile
40
+ - LICENSE
41
+ - README.md
42
+ - Rakefile
43
+ - bin/ec2launcher
44
+ - ec2launcher.gemspec
45
+ - lib/ec2launcher.rb
46
+ - lib/ec2launcher/application.rb
47
+ - lib/ec2launcher/block_device.rb
48
+ - lib/ec2launcher/block_device_builder.rb
49
+ - lib/ec2launcher/config.rb
50
+ - lib/ec2launcher/defaults.rb
51
+ - lib/ec2launcher/email_notification.rb
52
+ - lib/ec2launcher/environment.rb
53
+ - lib/ec2launcher/init_options.rb
54
+ - lib/ec2launcher/version.rb
55
+ - startup-scripts/setup.rb
56
+ - startup-scripts/setup_instance.rb
57
+ homepage: https://github.com/StudyBlue/ec2launcher
58
+ licenses: []
59
+ post_install_message:
60
+ rdoc_options: []
61
+ require_paths:
62
+ - lib
63
+ required_ruby_version: !ruby/object:Gem::Requirement
64
+ none: false
65
+ requirements:
66
+ - - ! '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ required_rubygems_version: !ruby/object:Gem::Requirement
70
+ none: false
71
+ requirements:
72
+ - - ! '>='
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ requirements: []
76
+ rubyforge_project:
77
+ rubygems_version: 1.8.24
78
+ signing_key:
79
+ specification_version: 3
80
+ summary: Tool to launch EC2 instances.
81
+ test_files: []