stacco 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ *.gem
@@ -0,0 +1,9 @@
1
+ task :default do
2
+ system "gem build stacco.gemspec"
3
+ system "sudo gem install *.gem"
4
+ end
5
+
6
+ task :clean do
7
+ system "sudo gem uninstall -x stacco"
8
+ system "rm -f *.gem"
9
+ end
@@ -0,0 +1,192 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'stacco'
4
+ require 'pathname'
5
+
6
+ class String
7
+ def colored(color)
8
+ color_number = (color.kind_of?(Integer) ? color : COLORS[color]) + 30
9
+ "\x1b[#{color_number}m#{self}\x1b[0m"
10
+ end
11
+
12
+ def guess_color
13
+ return :black if self.nil?
14
+ return :cyan unless self.kind_of? String
15
+
16
+ case self
17
+ when /COMPLETE$/
18
+ :green
19
+ when /IN_PROGRESS$/
20
+ :yellow
21
+ when /WORKING$/
22
+ :yellow
23
+ when /started$/
24
+ :blue
25
+ when /FAILED$/
26
+ :red
27
+ when /^CREATE/
28
+ :green
29
+ when /^DELETE/
30
+ :blue
31
+ when /^ROLLBACK/
32
+ :red
33
+ else
34
+ :white
35
+ end
36
+ end
37
+
38
+ COLORS = {
39
+ black: 0,
40
+ red: 1,
41
+ green: 2,
42
+ yellow: 3,
43
+ blue: 4,
44
+ magenta: 5,
45
+ cyan: 6,
46
+ white: 7
47
+ }
48
+ end
49
+
50
+
51
+ unless ARGV.length >= 1
52
+ $stderr.puts "usage: #{File.basename($0)} <subcommand ...>"
53
+ Kernel.exit 1
54
+ end
55
+
56
+ subcommand = ARGV.shift.intern
57
+ orch_config_path = Pathname.new 'orchestrator.yml'
58
+
59
+ if subcommand == :init
60
+ orch_config_path.open('w') do |f|
61
+ f.write $stdin.read
62
+ end
63
+
64
+ orch = Stacco::Orchestrator.from_config(orch_config_path)
65
+ puts "orchestrator initialized"
66
+ Kernel.exit 0
67
+ end
68
+
69
+ unless orch_config_path.file?
70
+ $stderr.puts "orchestrator not initialized!"
71
+ $stderr.puts "Did you run 'stacco init'?"
72
+ Kernel.exit 1
73
+ end
74
+
75
+ orch = Stacco::Orchestrator.from_config(orch_config_path)
76
+
77
+ if subcommand == :define
78
+ stack_defn = $stdin.read
79
+ stack = orch.define_stack stack_defn
80
+ puts "stack '#{stack.name}' defined"
81
+ Kernel.exit 0
82
+ end
83
+
84
+ stack = orch.stacks.first
85
+
86
+ case subcommand
87
+ when :up
88
+ stack.up!
89
+
90
+ when :down
91
+ stack.down!
92
+
93
+ when :"bake-template"
94
+ puts stack.cloudformation_template
95
+
96
+ when :repl
97
+ Stack = stack
98
+ require 'irb'
99
+ IRB.start
100
+
101
+ when :"cloudfront:init"
102
+ stack.initialize_distributions!
103
+
104
+ when :connect
105
+ conns = stack.connections
106
+
107
+ unless host_logicalname = ARGV.shift
108
+ conns.each do |resource_name, eip|
109
+ puts "#{resource_name}: #{eip.ip_address}"
110
+ end
111
+
112
+ Kernel.exit 0
113
+ end
114
+
115
+ unless conns.has_key? host_logicalname
116
+ $stderr.puts "#{host_logicalname}: not available"
117
+ Kernel.exit 1
118
+ end
119
+ use_eip = conns[host_logicalname]
120
+
121
+ private_key_path = Pathname.new("id_rsa")
122
+ private_key_path.open('wb'){ |f| f.write(stack.iam_private_key_material) }
123
+ private_key_path.chmod(0600)
124
+
125
+ Kernel.exec 'ssh', '-t', '-v',
126
+ '-i', private_key_path.to_s,
127
+ '-o', 'UserKnownHostsFile=/dev/null',
128
+ '-o', 'StrictHostKeyChecking=no',
129
+ "ubuntu@#{use_eip.ip_address}",
130
+ 'sudo', '/bin/bash', '-il'
131
+
132
+ when :status
133
+ stack.must_be_up!
134
+ now = Time.now
135
+
136
+ puts "#{stack.description.colored(stack.aws_status.guess_color)} (#{stack.aws_status})"
137
+ puts
138
+
139
+ summaries = stack.resource_summaries.find_all{ |res| res[:resource_status] != "DELETE_COMPLETE" }
140
+ summaries.each{ |res| res[:op], res[:status] = res[:resource_status].split('_', 2) }
141
+ ops = summaries.group_by{ |res| res[:op] }
142
+
143
+ ops.each do |op, ress|
144
+ colored_ress = ress.sort_by{ |res| res[:logical_resource_id] }.map{ |res| res[:logical_resource_id].colored(res[:status].guess_color) }
145
+ errored_ress = ress.find_all{ |res| (res[:resource_status_reason] and res[:resource_status_reason] !~ / Initiated$/) }
146
+
147
+ puts "#{op}: #{colored_ress.join(', ')}"
148
+ errored_ress.each do |res|
149
+ resource_part = res[:logical_resource_id].colored(res[:resource_status].guess_color)
150
+ time_ago = "%.1fs" % (now - res[:last_updated_timestamp])
151
+ puts " #{resource_part}: #{res[:resource_status_reason]} (#{time_ago} ago)"
152
+ end
153
+ end
154
+
155
+ when :events
156
+ stack.must_be_up!
157
+
158
+ terminal_height, terminal_width = `stty size`.chomp.split(" ").map{ |d| d.to_i }
159
+
160
+ operation_start_time = stack.up_since
161
+
162
+ trap('INT') do
163
+ $stderr.puts
164
+ Kernel.exit 0
165
+ end
166
+
167
+ stack.stream_events.each do |ev|
168
+ status = ev.status
169
+ color = status.guess_color
170
+
171
+ if ev.resource_type == "AWS::CloudFormation::Stack"
172
+ operation_start_time = ev.timestamp
173
+
174
+ status_part = "#{ev.operation} #{ev.logical_resource_id} #{status} at #{ev.timestamp}"
175
+ puts
176
+ puts "=== #{status_part.colored(color)} ".ljust(terminal_width, '=')
177
+ puts
178
+ else
179
+ relative_timestamp = ev.timestamp - operation_start_time
180
+
181
+ timestamp_part = "[#{("+%.1f" % relative_timestamp).rjust(12, ' ')}]"
182
+ status_part = "[#{status}]".rjust(10, ' ')
183
+ action_part = "#{status_part} #{ev.logical_resource_id}"
184
+ puts "#{timestamp_part} #{action_part.colored(color)} #{ev.error}"
185
+ end
186
+ end
187
+
188
+ $stderr.puts
189
+ else
190
+ $stderr.puts "unknown subcommand"
191
+ Kernel.exit 1
192
+ end
@@ -0,0 +1,414 @@
1
+ require 'set'
2
+ require 'ostruct'
3
+ require 'base64'
4
+ require 'pathname'
5
+ require 'shellwords'
6
+
7
+ require 'json'
8
+ require 'yaml'
9
+ require 'erb'
10
+ require 'inifile'
11
+
12
+ require 'aws'
13
+
14
+ class AWS::CloudFormation::StackEvent
15
+ def operation
16
+ self.resource_status.split('_', 2).first
17
+ end
18
+
19
+ def status
20
+ stat = self.resource_status.split('_', 2).last
21
+ return "started" if (self.resource_type == "AWS::CloudFormation::Stack" and stat == "IN_PROGRESS")
22
+ return "WORKING" if stat == "IN_PROGRESS"
23
+ stat
24
+ end
25
+
26
+ def error
27
+ self.resource_status_reason if (self.resource_status_reason and self.resource_status_reason !~ / Initiated$/)
28
+ end
29
+ end
30
+
31
+ class AWS::CloudFormation::Stack
32
+ attr_accessor :service_registry
33
+
34
+ def resources_of_type(type_name)
35
+ self.resources.find_all{ |r| r.resource_type == "AWS::#{type_name}" && r.resource_status =~ /_COMPLETE/ && r.resource_status != "DELETE_COMPLETE" }
36
+ end
37
+
38
+ def distributions
39
+ self.resources_of_type("CloudFront::Distribution").map{ |res| @service_registry[:cloudfront].distributions[res.physical_resource_id] }
40
+ end
41
+
42
+ def buckets
43
+ self.resources_of_type("S3::Bucket").map{ |res| @service_registry[:s3].buckets[res.physical_resource_id] }
44
+ end
45
+
46
+ def elastic_ips
47
+ self.resources_of_type("EC2::EIP").map{ |res| @service_registry[:ec2].elastic_ips[res.physical_resource_id] }
48
+ end
49
+ end
50
+
51
+ class AWS::CloudFront
52
+ def distributions
53
+ self.client.list_distributions[:items].map{ |dist| Distribution.new(self.client, self.client.get_distribution(id: dist[:id])) }
54
+ end
55
+ end
56
+
57
+ class AWS::CloudFront::Distribution
58
+ def initialize(client, data)
59
+ @client = client
60
+ @data = data
61
+ end
62
+
63
+ def aliases
64
+ @data[:distribution_config][:aliases][:items]
65
+ end
66
+
67
+ def id
68
+ @data[:id]
69
+ end
70
+
71
+ def config
72
+ self.make_exportable(@data[:distribution_config])
73
+ end
74
+
75
+ def make_exportable(o)
76
+ case o
77
+ when nil
78
+ ""
79
+ when Hash
80
+ h = {}
81
+ o.each{ |k, v| h[k] = make_exportable(v) }
82
+ h
83
+ when Array
84
+ o.map{ |e| make_exportable(e) }
85
+ else
86
+ o
87
+ end
88
+ end
89
+
90
+ def update
91
+ begin
92
+ yield
93
+
94
+ @client.update_distribution(
95
+ id: self.id,
96
+ distribution_config: self.config,
97
+ if_match: @data[:etag]
98
+ )
99
+ end
100
+ end
101
+
102
+ def price_class
103
+ @data[:distribution_config][:price_class].split('_').last.downcase.intern
104
+ end
105
+
106
+ def price_class=(new_class)
107
+ @data[:distribution_config][:price_class] = "PriceClass_#{new_class.to_s.capitalize}"
108
+ end
109
+
110
+ def certificate
111
+ @data[:distribution_config][:viewer_certificate][:iam_certificate_id]
112
+ end
113
+
114
+ def certificate=(cert_id)
115
+ @data[:distribution_config][:viewer_certificate] = {
116
+ iam_certificate_id: cert_id,
117
+ ssl_support_method: "sni-only"
118
+ }
119
+ end
120
+ end
121
+
122
+ module Stacco
123
+ module Resources
124
+ RootDir = Pathname.new(File.expand_path("../../priv", __FILE__))
125
+
126
+ templates_dir = Stacco::Resources::RootDir + "templates"
127
+ Templates = {
128
+ cloudformation: (templates_dir + "cloudformation.json.erb").read
129
+ }
130
+
131
+ module CloudInit
132
+ cloud_init_scripts_dir = Stacco::Resources::RootDir + "cloud-init"
133
+
134
+ CommonScripts = {
135
+ before: (cloud_init_scripts_dir + "common.before.sh").readlines,
136
+ after: (cloud_init_scripts_dir + "common.after.sh").readlines
137
+ }
138
+
139
+ RoleScripts = {}
140
+ (cloud_init_scripts_dir + "roles").children.each{ |path| RoleScripts[path.basename('.sh').to_s.intern] = path.readlines }
141
+ end
142
+ end
143
+ end
144
+
145
+ class Stacco::Orchestrator
146
+ def self.from_config(config_body)
147
+ config_body = config_body.read if config_body.respond_to?(:read)
148
+ self.new YAML.load(config_body)
149
+ end
150
+
151
+ def initialize(config)
152
+ @config = config
153
+
154
+ storage_bucket_name = @config['storage_bucket']
155
+
156
+ s3 = AWS::S3.new(
157
+ access_key_id: @config['aws']['access_key_id'],
158
+ secret_access_key: @config['aws']['secret_access_key'],
159
+ region: @config['aws']['region']
160
+ )
161
+
162
+ @bucket = s3.buckets[storage_bucket_name]
163
+ s3.buckets.create(storage_bucket_name) unless @bucket.exists?
164
+ end
165
+
166
+ def stacks
167
+ @bucket.objects.with_prefix('stack/').to_a[1..-1].map{ |obj| Stacco::Stack.new(obj) }
168
+ end
169
+
170
+ def define_stack(config_body)
171
+ config_body = config_body.read if config_body.respond_to?(:read)
172
+
173
+ config = YAML.load(config_body)
174
+
175
+ stack_name = config['name']
176
+
177
+ stack_object = @bucket.objects["stack/#{stack_name}"]
178
+ stack_object.write(config.to_yaml)
179
+
180
+ Stacco::Stack.new(stack_object)
181
+ end
182
+ end
183
+
184
+ class Stacco::Stack
185
+ def initialize(config_object)
186
+ @config_object = config_object
187
+
188
+ aws_creds = self.aws_credentials
189
+
190
+ @services = {
191
+ ec2: AWS::EC2.new(aws_creds),
192
+ s3: AWS::S3.new(aws_creds),
193
+ cloudformation: AWS::CloudFormation.new(aws_creds),
194
+ cloudfront: AWS::CloudFront.new(aws_creds)
195
+ }
196
+
197
+ @aws_stack = @services[:cloudformation].stacks[self.name]
198
+ @aws_stack.service_registry = @services
199
+ end
200
+
201
+ def connections
202
+ connections = {}
203
+ @aws_stack.elastic_ips.each{ |eip| connections[eip.instance.tags["aws:cloudformation:logical-id"]] = eip }
204
+ connections
205
+ end
206
+
207
+ def resource_summaries
208
+ @aws_stack.resource_summaries
209
+ end
210
+
211
+ def must_be_up!
212
+ unless self.up?
213
+ $stderr.puts "stack #{self.name} is down"
214
+ Kernel.exit 1
215
+ end
216
+ end
217
+
218
+ def aws_status
219
+ @aws_stack.status
220
+ end
221
+
222
+ def config
223
+ YAML.load(@config_object.read)
224
+ end
225
+
226
+ def config=(new_config)
227
+ @config_object.write(new_config.to_yaml)
228
+ end
229
+
230
+ def update_config
231
+ # TODO
232
+ end
233
+
234
+ def aws_credentials
235
+ Hash[ *(self.config['aws'].map{ |k, v| [k.intern, v] }.flatten) ]
236
+ end
237
+
238
+ def description
239
+ self.config['description']
240
+ end
241
+
242
+ def name
243
+ self.config['name']
244
+ end
245
+
246
+ def name=(new_name)
247
+ update_config{ |config| config.merge("name" => new_name) }
248
+ end
249
+
250
+ def up?
251
+ @aws_stack.exists?
252
+ end
253
+
254
+ def up!
255
+ if @aws_stack.exists?
256
+ @aws_stack.update(template: self.cloudformation_template)
257
+ else
258
+ @services[:cloudformation].stacks.create(self.name, self.cloudformation_template)
259
+ end
260
+ end
261
+
262
+ def up_since
263
+ @aws_stack.creation_time if @aws_stack.exists?
264
+ end
265
+
266
+ def initialize_distributions!
267
+ cloudfront_certs = self.config['cloudfront']['certificates']
268
+ @services[:cloudfront].distributions.each do |dist|
269
+ dist.update do
270
+ dist.price_class = :"100"
271
+ dist.certificate = cloudfront_certs[dist.aliases.first]
272
+ end
273
+ end
274
+ end
275
+
276
+ def down!
277
+ return false unless self.up?
278
+
279
+ @aws_stack.buckets.each{ |bucket| bucket.delete! }
280
+ @aws_stack.delete
281
+
282
+ Kernel.sleep(2) while self.up?
283
+
284
+ true
285
+ end
286
+
287
+ def cloudformation_template
288
+ tpl = Stacco::Template.new(self, Stacco::Resources::Templates[:cloudformation])
289
+ tpl.result(self.config)
290
+ end
291
+
292
+ def iam_private_key
293
+ @config_object.bucket.objects.with_prefix("sshkey/#{self.name}-").to_a.sort_by{ |obj| obj.key.split('/').last.split('-').last.to_i }.last
294
+ end
295
+
296
+ def iam_keypair_name
297
+ "stacco-" + self.iam_private_key.key.split('/').last
298
+ end
299
+
300
+ def iam_private_key_material
301
+ self.iam_private_key.read
302
+ end
303
+
304
+ def stream_events
305
+ Enumerator.new do |out|
306
+ known_events = Set.new
307
+ ticks_without_add = 0
308
+
309
+ while self.up?
310
+ added = 0
311
+
312
+ @aws_stack.events.sort_by{ |ev| ev.timestamp }.each do |event|
313
+ next if known_events.include? event.event_id
314
+ out.yield event
315
+
316
+ known_events.add event.event_id
317
+ added += 1
318
+ ticks_without_add = 0
319
+
320
+ end
321
+
322
+ ticks_without_add += 1 if added == 0
323
+
324
+ if ticks_without_add >= 8 and (Math.log2(ticks_without_add) % 1) == 0.0
325
+ jobs = @aws_stack.resource_summaries
326
+ active_jobs = jobs.find_all{ |job| job[:resource_status] =~ /IN_PROGRESS$/ }.map{ |job| job[:logical_resource_id] }.sort
327
+ unless active_jobs.empty?
328
+ out.yield OpenStruct.new(
329
+ logical_resource_id: "Scheduler",
330
+ status: "WAIT",
331
+ operation: "WAIT",
332
+ timestamp: Time.now,
333
+ error: "waiting on #{active_jobs.join(', ')}"
334
+ )
335
+ end
336
+ end
337
+
338
+ Kernel.sleep 2
339
+ end
340
+ end
341
+ end
342
+ end
343
+
344
+ class Stacco::Template
345
+ class RenderContext < OpenStruct
346
+ def initialize(stack, config, vars)
347
+ @stack = stack
348
+ @config = config
349
+ super(vars)
350
+ end
351
+
352
+ def j(str)
353
+ str.to_json
354
+ end
355
+
356
+ def ja(indent_n, lns)
357
+ indent = " " * indent_n
358
+ newline = "\n".to_json
359
+ lns.map do |ln|
360
+ if ln.chomp.empty?
361
+ "%s \n" % [indent]
362
+ else
363
+ "%s%s, %s,\n" % [indent, ln.chomp.to_json, newline]
364
+ end
365
+ end.join[indent_n .. -3]
366
+ end
367
+ end
368
+
369
+ def udscript(roles, vars)
370
+ roles = roles.map{ |role| role.intern }
371
+
372
+ unless roles.include? :backend
373
+ vars = vars.dup
374
+ vars.delete 'wallet_data'
375
+ end
376
+
377
+ lns = []
378
+ lns.concat(vars.map do |k, v|
379
+ var_val = v.include?("\0") ? Base64.encode64(v) : v
380
+ "export #{k.to_s.upcase}=#{Shellwords.escape(var_val)}\n"
381
+ end)
382
+ lns.concat Stacco::Resources::CloudInit::CommonScripts[:before]
383
+ roles.each{ |role| lns.concat Stacco::Resources::CloudInit::RoleScripts[role] }
384
+ lns.concat Stacco::Resources::CloudInit::CommonScripts[:after]
385
+ lns
386
+ end
387
+
388
+ def initialize(stack, template_body)
389
+ @stack = stack
390
+ template_body = template_body.read if template_body.respond_to?(:read)
391
+ @template_body = template_body
392
+ end
393
+
394
+ def result(vars)
395
+ cloudformation_vars = {}
396
+ cloudformation_vars.update vars
397
+ cloudformation_vars.update vars['cloudformation']
398
+
399
+ cloudinit_vars = {}
400
+ cloudinit_vars.update vars
401
+ cloudinit_vars.update vars['cloud-init']
402
+ cloudinit_vars.each do |k, v|
403
+ cloudinit_vars.delete(k) if v.kind_of?(Hash) or v.kind_of?(Array)
404
+ end
405
+
406
+ cloudformation_vars['userdata'] = {}
407
+ cloudformation_vars['roles'].each do |host_name, host_roles|
408
+ cloudformation_vars['userdata'][host_name] = udscript(host_roles, cloudinit_vars)
409
+ end
410
+
411
+ b = RenderContext.new(@stack, vars, cloudformation_vars).instance_eval{ binding }
412
+ ERB.new(@template_body).result(b)
413
+ end
414
+ end