stacco 0.1.0

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/Rakefile

@@ -0,0 +1,9 @@
+task :default do
+  system "gem build stacco.gemspec"
+  system "sudo gem install *.gem"
+end
+
+task :clean do
+  system "sudo gem uninstall -x stacco"
+  system "rm -f *.gem"
+end

data/bin/stacco

@@ -0,0 +1,192 @@
+#!/usr/bin/env ruby
+
+require 'stacco'
+require 'pathname'
+
+class String
+  def colored(color)
+    color_number = (color.kind_of?(Integer) ? color : COLORS[color]) + 30
+    "\x1b[#{color_number}m#{self}\x1b[0m"
+  end
+
+  def guess_color
+    return :black if self.nil?
+    return :cyan unless self.kind_of? String
+
+    case self
+    when /COMPLETE$/
+      :green
+    when /IN_PROGRESS$/
+      :yellow
+    when /WORKING$/
+      :yellow
+    when /started$/
+      :blue
+    when /FAILED$/
+      :red
+    when /^CREATE/
+      :green
+    when /^DELETE/
+      :blue
+    when /^ROLLBACK/
+      :red
+    else
+      :white
+    end
+  end
+
+  COLORS = {
+    black: 0,
+    red: 1,
+    green: 2,
+    yellow: 3,
+    blue: 4,
+    magenta: 5,
+    cyan: 6,
+    white: 7
+  }
+end
+
+
+unless ARGV.length >= 1
+  $stderr.puts "usage: #{File.basename($0)} <subcommand ...>"
+  Kernel.exit 1
+end
+
+subcommand = ARGV.shift.intern
+orch_config_path = Pathname.new 'orchestrator.yml'
+
+if subcommand == :init
+  orch_config_path.open('w') do |f|
+    f.write $stdin.read
+  end
+
+  orch = Stacco::Orchestrator.from_config(orch_config_path)
+  puts "orchestrator initialized"
+  Kernel.exit 0
+end
+
+unless orch_config_path.file?
+  $stderr.puts "orchestrator not initialized!"
+  $stderr.puts "Did you run 'stacco init'?"
+  Kernel.exit 1
+end
+
+orch = Stacco::Orchestrator.from_config(orch_config_path)
+
+if subcommand == :define
+  stack_defn = $stdin.read
+  stack = orch.define_stack stack_defn
+  puts "stack '#{stack.name}' defined"
+  Kernel.exit 0
+end
+
+stack = orch.stacks.first
+
+case subcommand
+when :up
+  stack.up!
+
+when :down
+  stack.down!
+
+when :"bake-template"
+  puts stack.cloudformation_template
+
+when :repl
+  Stack = stack
+  require 'irb'
+  IRB.start
+
+when :"cloudfront:init"
+  stack.initialize_distributions!
+
+when :connect
+  conns = stack.connections
+
+  unless host_logicalname = ARGV.shift
+    conns.each do |resource_name, eip|
+      puts "#{resource_name}: #{eip.ip_address}"
+    end
+
+    Kernel.exit 0
+  end
+
+  unless conns.has_key? host_logicalname
+    $stderr.puts "#{host_logicalname}: not available"
+    Kernel.exit 1
+  end
+  use_eip = conns[host_logicalname]
+
+  private_key_path = Pathname.new("id_rsa")
+  private_key_path.open('wb'){ |f| f.write(stack.iam_private_key_material) }
+  private_key_path.chmod(0600)
+
+  Kernel.exec 'ssh', '-t', '-v',
+    '-i', private_key_path.to_s,
+    '-o', 'UserKnownHostsFile=/dev/null',
+    '-o', 'StrictHostKeyChecking=no',
+    "ubuntu@#{use_eip.ip_address}",
+    'sudo', '/bin/bash', '-il'
+
+when :status
+  stack.must_be_up!
+  now = Time.now
+
+  puts "#{stack.description.colored(stack.aws_status.guess_color)} (#{stack.aws_status})"
+  puts
+
+  summaries = stack.resource_summaries.find_all{ |res| res[:resource_status] != "DELETE_COMPLETE" }
+  summaries.each{ |res| res[:op], res[:status] = res[:resource_status].split('_', 2) }
+  ops = summaries.group_by{ |res| res[:op] }
+
+  ops.each do |op, ress|
+    colored_ress = ress.sort_by{ |res| res[:logical_resource_id] }.map{ |res| res[:logical_resource_id].colored(res[:status].guess_color) }
+    errored_ress = ress.find_all{ |res| (res[:resource_status_reason] and res[:resource_status_reason] !~ / Initiated$/) }
+
+    puts "#{op}: #{colored_ress.join(', ')}"
+    errored_ress.each do |res|
+      resource_part = res[:logical_resource_id].colored(res[:resource_status].guess_color)
+      time_ago = "%.1fs" % (now - res[:last_updated_timestamp])
+      puts "  #{resource_part}: #{res[:resource_status_reason]} (#{time_ago} ago)"
+    end
+  end
+
+when :events
+  stack.must_be_up!
+
+  terminal_height, terminal_width = `stty size`.chomp.split(" ").map{ |d| d.to_i }
+
+  operation_start_time = stack.up_since
+
+  trap('INT') do
+    $stderr.puts
+    Kernel.exit 0
+  end
+
+  stack.stream_events.each do |ev|
+    status = ev.status
+    color = status.guess_color
+
+    if ev.resource_type == "AWS::CloudFormation::Stack"
+      operation_start_time = ev.timestamp
+
+      status_part = "#{ev.operation} #{ev.logical_resource_id} #{status} at #{ev.timestamp}"
+      puts
+      puts "=== #{status_part.colored(color)} ".ljust(terminal_width, '=')
+      puts
+    else
+      relative_timestamp = ev.timestamp - operation_start_time
+
+      timestamp_part = "[#{("+%.1f" % relative_timestamp).rjust(12, ' ')}]"
+      status_part = "[#{status}]".rjust(10, ' ')
+      action_part = "#{status_part} #{ev.logical_resource_id}"
+      puts "#{timestamp_part} #{action_part.colored(color)} #{ev.error}"
+    end
+  end
+
+  $stderr.puts
+else
+  $stderr.puts "unknown subcommand"
+  Kernel.exit 1
+end

data/lib/stacco.rb

@@ -0,0 +1,414 @@
+require 'set'
+require 'ostruct'
+require 'base64'
+require 'pathname'
+require 'shellwords'
+
+require 'json'
+require 'yaml'
+require 'erb'
+require 'inifile'
+
+require 'aws'
+
+class AWS::CloudFormation::StackEvent
+  def operation
+    self.resource_status.split('_', 2).first
+  end
+
+  def status
+    stat = self.resource_status.split('_', 2).last
+    return "started" if (self.resource_type == "AWS::CloudFormation::Stack" and stat == "IN_PROGRESS")
+    return "WORKING" if stat == "IN_PROGRESS"
+    stat
+  end
+
+  def error
+    self.resource_status_reason if (self.resource_status_reason and self.resource_status_reason !~ / Initiated$/)
+  end
+end
+
+class AWS::CloudFormation::Stack
+  attr_accessor :service_registry
+
+  def resources_of_type(type_name)
+    self.resources.find_all{ |r| r.resource_type == "AWS::#{type_name}" && r.resource_status =~ /_COMPLETE/ && r.resource_status != "DELETE_COMPLETE" }
+  end
+
+  def distributions
+    self.resources_of_type("CloudFront::Distribution").map{ |res| @service_registry[:cloudfront].distributions[res.physical_resource_id] }
+  end
+
+  def buckets
+    self.resources_of_type("S3::Bucket").map{ |res| @service_registry[:s3].buckets[res.physical_resource_id] }
+  end
+
+  def elastic_ips
+    self.resources_of_type("EC2::EIP").map{ |res| @service_registry[:ec2].elastic_ips[res.physical_resource_id] }
+  end
+end
+
+class AWS::CloudFront
+  def distributions
+    self.client.list_distributions[:items].map{ |dist| Distribution.new(self.client, self.client.get_distribution(id: dist[:id])) }
+  end
+end
+
+class AWS::CloudFront::Distribution
+  def initialize(client, data)
+    @client = client
+    @data = data
+  end
+
+  def aliases
+    @data[:distribution_config][:aliases][:items]
+  end
+
+  def id
+    @data[:id]
+  end
+
+  def config
+    self.make_exportable(@data[:distribution_config])
+  end
+
+  def make_exportable(o)
+    case o
+    when nil
+      ""
+    when Hash
+      h = {}
+      o.each{ |k, v| h[k] = make_exportable(v) }
+      h
+    when Array
+      o.map{ |e| make_exportable(e) }
+    else
+      o
+    end
+  end
+
+  def update
+    begin
+      yield
+
+      @client.update_distribution(
+        id: self.id,
+        distribution_config: self.config,
+        if_match: @data[:etag]
+      )
+    end
+  end
+
+  def price_class
+    @data[:distribution_config][:price_class].split('_').last.downcase.intern
+  end
+
+  def price_class=(new_class)
+    @data[:distribution_config][:price_class] = "PriceClass_#{new_class.to_s.capitalize}"
+  end
+
+  def certificate
+    @data[:distribution_config][:viewer_certificate][:iam_certificate_id]
+  end
+
+  def certificate=(cert_id)
+    @data[:distribution_config][:viewer_certificate] = {
+      iam_certificate_id: cert_id,
+      ssl_support_method: "sni-only"
+    }
+  end
+end
+
+module Stacco
+  module Resources
+    RootDir = Pathname.new(File.expand_path("../../priv", __FILE__))
+
+    templates_dir = Stacco::Resources::RootDir + "templates"
+    Templates = {
+      cloudformation: (templates_dir + "cloudformation.json.erb").read
+    }
+
+    module CloudInit
+      cloud_init_scripts_dir = Stacco::Resources::RootDir + "cloud-init"
+
+      CommonScripts = {
+        before: (cloud_init_scripts_dir + "common.before.sh").readlines,
+        after: (cloud_init_scripts_dir + "common.after.sh").readlines
+      }
+
+      RoleScripts = {}
+      (cloud_init_scripts_dir + "roles").children.each{ |path| RoleScripts[path.basename('.sh').to_s.intern] = path.readlines }
+    end
+  end
+end
+
+class Stacco::Orchestrator
+  def self.from_config(config_body)
+    config_body = config_body.read if config_body.respond_to?(:read)
+    self.new YAML.load(config_body)
+  end
+
+  def initialize(config)
+    @config = config
+
+    storage_bucket_name = @config['storage_bucket']
+
+    s3 = AWS::S3.new(
+      access_key_id: @config['aws']['access_key_id'],
+      secret_access_key: @config['aws']['secret_access_key'],
+      region: @config['aws']['region']
+    )
+
+    @bucket = s3.buckets[storage_bucket_name]
+    s3.buckets.create(storage_bucket_name) unless @bucket.exists?
+  end
+
+  def stacks
+    @bucket.objects.with_prefix('stack/').to_a[1..-1].map{ |obj| Stacco::Stack.new(obj) }
+  end
+
+  def define_stack(config_body)
+    config_body = config_body.read if config_body.respond_to?(:read)
+
+    config = YAML.load(config_body)
+
+    stack_name = config['name']
+
+    stack_object = @bucket.objects["stack/#{stack_name}"]
+    stack_object.write(config.to_yaml)
+
+    Stacco::Stack.new(stack_object)
+  end
+end
+
+class Stacco::Stack
+  def initialize(config_object)
+    @config_object = config_object
+
+    aws_creds = self.aws_credentials
+
+    @services = {
+      ec2: AWS::EC2.new(aws_creds),
+      s3: AWS::S3.new(aws_creds),
+      cloudformation: AWS::CloudFormation.new(aws_creds),
+      cloudfront: AWS::CloudFront.new(aws_creds)
+    }
+
+    @aws_stack = @services[:cloudformation].stacks[self.name]
+    @aws_stack.service_registry = @services
+  end
+
+  def connections
+    connections = {}
+    @aws_stack.elastic_ips.each{ |eip| connections[eip.instance.tags["aws:cloudformation:logical-id"]] = eip }
+    connections
+  end
+
+  def resource_summaries
+    @aws_stack.resource_summaries
+  end
+
+  def must_be_up!
+    unless self.up?
+      $stderr.puts "stack #{self.name} is down"
+      Kernel.exit 1
+    end
+  end
+
+  def aws_status
+    @aws_stack.status
+  end
+
+  def config
+    YAML.load(@config_object.read)
+  end
+
+  def config=(new_config)
+    @config_object.write(new_config.to_yaml)
+  end
+
+  def update_config
+    # TODO
+  end
+
+  def aws_credentials
+    Hash[ *(self.config['aws'].map{ |k, v| [k.intern, v] }.flatten) ]
+  end
+
+  def description
+    self.config['description']
+  end
+
+  def name
+    self.config['name']
+  end
+
+  def name=(new_name)
+    update_config{ |config| config.merge("name" => new_name) }
+  end
+
+  def up?
+    @aws_stack.exists?
+  end
+
+  def up!
+    if @aws_stack.exists?
+      @aws_stack.update(template: self.cloudformation_template)
+    else
+      @services[:cloudformation].stacks.create(self.name, self.cloudformation_template)
+    end
+  end
+
+  def up_since
+    @aws_stack.creation_time if @aws_stack.exists?
+  end
+
+  def initialize_distributions!
+    cloudfront_certs = self.config['cloudfront']['certificates']
+    @services[:cloudfront].distributions.each do |dist|
+      dist.update do
+        dist.price_class = :"100"
+        dist.certificate = cloudfront_certs[dist.aliases.first]
+      end
+    end
+  end
+
+  def down!
+    return false unless self.up?
+
+    @aws_stack.buckets.each{ |bucket| bucket.delete! }
+    @aws_stack.delete
+
+    Kernel.sleep(2) while self.up?
+
+    true
+  end
+
+  def cloudformation_template
+    tpl = Stacco::Template.new(self, Stacco::Resources::Templates[:cloudformation])
+    tpl.result(self.config)
+  end
+
+  def iam_private_key
+    @config_object.bucket.objects.with_prefix("sshkey/#{self.name}-").to_a.sort_by{ |obj| obj.key.split('/').last.split('-').last.to_i }.last
+  end
+
+  def iam_keypair_name
+    "stacco-" + self.iam_private_key.key.split('/').last
+  end
+
+  def iam_private_key_material
+    self.iam_private_key.read
+  end
+
+  def stream_events
+    Enumerator.new do |out|
+      known_events = Set.new
+      ticks_without_add = 0
+
+      while self.up?
+        added = 0
+
+        @aws_stack.events.sort_by{ |ev| ev.timestamp }.each do |event|
+          next if known_events.include? event.event_id
+          out.yield event
+
+          known_events.add event.event_id
+          added += 1
+          ticks_without_add = 0
+
+        end
+
+        ticks_without_add += 1 if added == 0
+
+        if ticks_without_add >= 8 and (Math.log2(ticks_without_add) % 1) == 0.0
+          jobs = @aws_stack.resource_summaries
+          active_jobs = jobs.find_all{ |job| job[:resource_status] =~ /IN_PROGRESS$/ }.map{ |job| job[:logical_resource_id] }.sort
+          unless active_jobs.empty?
+            out.yield OpenStruct.new(
+              logical_resource_id: "Scheduler",
+              status: "WAIT",
+              operation: "WAIT",
+              timestamp: Time.now,
+              error: "waiting on #{active_jobs.join(', ')}"
+            )
+          end
+        end
+
+        Kernel.sleep 2
+      end
+    end
+  end
+end
+
+class Stacco::Template
+  class RenderContext < OpenStruct
+    def initialize(stack, config, vars)
+      @stack = stack
+      @config = config
+      super(vars)
+    end
+
+    def j(str)
+      str.to_json
+    end
+
+    def ja(indent_n, lns)
+      indent = " " * indent_n
+      newline = "\n".to_json
+      lns.map do |ln|
+        if ln.chomp.empty?
+          "%s \n" % [indent]
+        else
+          "%s%s, %s,\n" % [indent, ln.chomp.to_json, newline]
+        end
+      end.join[indent_n .. -3]
+    end
+  end
+
+  def udscript(roles, vars)
+    roles = roles.map{ |role| role.intern }
+
+    unless roles.include? :backend
+      vars = vars.dup
+      vars.delete 'wallet_data'
+    end
+
+    lns = []
+    lns.concat(vars.map do |k, v|
+      var_val = v.include?("\0") ? Base64.encode64(v) : v
+      "export #{k.to_s.upcase}=#{Shellwords.escape(var_val)}\n"
+    end)
+    lns.concat Stacco::Resources::CloudInit::CommonScripts[:before]
+    roles.each{ |role| lns.concat Stacco::Resources::CloudInit::RoleScripts[role] }
+    lns.concat Stacco::Resources::CloudInit::CommonScripts[:after]
+    lns
+  end
+
+  def initialize(stack, template_body)
+    @stack = stack
+    template_body = template_body.read if template_body.respond_to?(:read)
+    @template_body = template_body
+  end
+
+  def result(vars)
+    cloudformation_vars = {}
+    cloudformation_vars.update vars
+    cloudformation_vars.update vars['cloudformation']
+
+    cloudinit_vars = {}
+    cloudinit_vars.update vars
+    cloudinit_vars.update vars['cloud-init']
+    cloudinit_vars.each do |k, v|
+      cloudinit_vars.delete(k) if v.kind_of?(Hash) or v.kind_of?(Array)
+    end
+
+    cloudformation_vars['userdata'] = {}
+    cloudformation_vars['roles'].each do |host_name, host_roles|
+      cloudformation_vars['userdata'][host_name] = udscript(host_roles, cloudinit_vars)
+    end
+
+    b = RenderContext.new(@stack, vars, cloudformation_vars).instance_eval{ binding }
+    ERB.new(@template_body).result(b)
+  end
+end