stacco 0.1.10 → 0.1.17

data/.gitignore

@@ -1 +1,4 @@
+*.swp
 *.gem
+.bundle/
+sites/

data/Dockerfile

@@ -0,0 +1,10 @@
+FROM tsutsu/ubuntu:latest
+
+RUN /lib/container/do-with-clean <<EOF \
+  apt-get update \
+  apt-get install -fqy openssh-client ruby2.0 bundler \
+  gem install -y --no-rdoc --no-ri stacco \
+EOF
+
+WORKDIR /var/lib/stacco
+ENTRYPOINT ["/usr/local/bin/stacco"]

data/README.md

@@ -0,0 +1,5 @@
+
+
+$  aws cloudformation create-stack --stack-name bexbugbounty --template-body "$(./bake-template sites/bexbugbounty.com.yml)"
+
+$  ./follow-stack bexbugbounty --profile='bbb'

data/bin/stacco

@@ -1,7 +1,11 @@
 #!/usr/bin/env ruby
 
+$:.unshift File.expand_path("../../lib", __FILE__)
+
 require 'stacco'
 require 'pathname'
+require 'highline'
+require 'inifile'
 
 class String
   def colored(color)
@@ -55,26 +59,81 @@ unless ARGV.length >= 1
   Kernel.exit 1
 end
 
+console = HighLine.new
+
 subcommand = ARGV.shift.intern
-orch_config_path = Pathname.new 'orchestrator.yml'
+
+orch_config = {
+  local: Pathname.new('orchestrator.yml'),
+  user: Pathname.new(ENV['HOME']) + '.config' + 'stacco' + 'orchestrator.yml'
+}
+orch_config = orch_config[:local].file? ? orch_config[:local] : orch_config[:user]
+orch_config.parent.mkpath
 
 if subcommand == :init
-  orch_config_path.open('w') do |f|
-    f.write $stdin.read
+  if $stdin.tty?
+    new_orch_config = {aws: {}}
+
+    aws_config = Pathname.new(ENV['HOME']) + '.aws' + 'config'
+    if aws_config.file?
+      aws_config = IniFile.new(filename: aws_config.to_s)
+      aws_profiles = aws_config.sections - ["preview"]
+
+      if aws_profiles.length > 1
+        aws_profile = console.choose(aws_profiles) do |menu|
+          menu.prompt = "Local AWS profile to source: "
+        end
+      else
+        console.say "Sourcing AWS config from local AWS profile."
+        aws_profile = aws_profiles.first
+      end
+
+      new_orch_config[:aws][:access_key_id] = aws_config[aws_profile]['aws_access_key_id']
+      new_orch_config[:aws][:secret_access_key] = aws_config[aws_profile]['aws_secret_access_key']
+      new_orch_config[:aws][:region] = aws_config[aws_profile]['region']
+    end
+
+    new_orch_config[:aws][:access_key_id] = console.ask("AWS account ID: ") do |question|
+      question.default = new_orch_config[:aws][:access_key_id]
+      question.validate = /^[0-9A-Z]{20}$/
+    end
+
+    new_orch_config[:aws][:secret_access_key] = console.ask("AWS account secret: ") do |question|
+      question.default = new_orch_config[:aws][:secret_access_key]
+      question.validate = /^[=+\/0-9A-Za-z]{32,64}$/
+    end
+
+    new_orch_config[:aws][:region] = console.choose(*(AWS.regions.map{ |r| r.name }.sort)) do |menu|
+      menu.prompt = "Orchestrator storage-bucket region: "
+      menu.default = new_orch_config[:aws][:region]
+      menu.layout = :one_line
+      menu.select_by = :name_or_index
+    end
+
+    new_orch_config[:storage_bucket] = console.ask("Orchestrator storage-bucket name: ") do |question|
+      question.validate = /^[A-Za-z][-.0-9A-Za-z]+[A-Za-z0-9]$/
+    end
+
+    # convert symbol-keys to string-keys
+    new_orch_config = JSON.parse(new_orch_config.to_json)
+  else
+    new_orch_config = YAML.load($stdin.read)
   end
 
-  orch = Stacco::Orchestrator.from_config(orch_config_path)
+  orch_config.open('w'){ |f| f.write(new_orch_config.to_yaml) }
+
+  orch = Stacco::Orchestrator.from_config(orch_config)
   puts "orchestrator initialized"
   Kernel.exit 0
 end
 
-unless orch_config_path.file?
+unless orch_config.file?
   $stderr.puts "orchestrator not initialized!"
   $stderr.puts "Did you run 'stacco init'?"
   Kernel.exit 1
 end
 
-orch = Stacco::Orchestrator.from_config(orch_config_path)
+orch = Stacco::Orchestrator.from_config(orch_config)
 stacks = orch.stacks
 
 case subcommand
@@ -132,9 +191,29 @@ when :edit
   stack.config = YAML.load(tmp_stackdef_path.read)
 
 
-when :"bake-template"
+when :"template:bake"
   puts stack.cloudformation_template
 
+when :"template:validate"
+  success, err, pos = stack.validate
+  if success
+    puts "template is valid"
+  else
+    $stderr.puts err.colored(:red)
+    if pos
+      $stderr.puts
+      line, column = pos
+      line_before, line_after = line[0...column], line[column..-1]
+      if line_after
+        token_at, rest_of_line = line_after.split(' ', 2)
+        $stderr.puts line_before + (token_at || '').colored(:red) + (rest_of_line || '')
+      else
+        $stderr.puts line_before + ' #'.colored(:blue) + ' <-'.colored(:red)
+      end
+      $stderr.puts
+    end
+  end
+
 when :repl
   Stack = stack
   require 'irb'
@@ -146,9 +225,12 @@ when :"cloudfront:init"
 when :connect
   conns = stack.connections
 
+  bastion_host = conns.find{|k,v| k =~ /bastion/i and v.dns_name }[1]
+
   unless host_logicalname = ARGV.shift
-    conns.each do |resource_name, eip|
-      puts "#{resource_name}: #{eip.ip_address}"
+    conns.each do |resource_name, inst|
+      connection_path = inst.dns_name ? inst.dns_name : ("%s -> %s" % [bastion_host.dns_name, inst.private_dns_name])
+      puts "#{resource_name}: #{connection_path}"
     end
 
     Kernel.exit 0
@@ -158,18 +240,45 @@ when :connect
     $stderr.puts "#{host_logicalname}: not available"
     Kernel.exit 1
   end
-  use_eip = conns[host_logicalname]
 
-  private_key_path = Pathname.new("id_rsa")
+  connect_to_host = conns[host_logicalname]
+
+  private_key_path = Pathname.new("id_rsa").expand_path
   private_key_path.open('wb'){ |f| f.write(stack.iam_private_key_material) }
   private_key_path.chmod(0600)
 
-  Kernel.exec 'ssh', '-t', '-v',
-    '-i', private_key_path.to_s,
-    '-o', 'UserKnownHostsFile=/dev/null',
-    '-o', 'StrictHostKeyChecking=no',
-    "ubuntu@#{use_eip.ip_address}",
-    'sudo', 'su', '-'
+
+  begin
+    Kernel.system 'ssh-add', private_key_path.to_s
+    if connect_to_host.dns_name
+      Kernel.system 'ssh', '-A', '-tt', '-v',
+        '-i', private_key_path.to_s,
+        '-o', 'StrictHostKeyChecking=no',
+        '-o', 'UserKnownHostsFile=/dev/null',
+        "ubuntu@#{connect_to_host.dns_name}",
+        'sudo', 'su', '-'
+    else
+      Kernel.system 'ssh', '-A', '-tt', '-v',
+        '-o', 'StrictHostKeyChecking=no',
+        '-o', 'UserKnownHostsFile=/dev/null',
+        "ubuntu@#{bastion_host.dns_name}",
+        'ssh', '-A', '-tt', '-v',
+        '-o', 'StrictHostKeyChecking=no',
+        '-o', 'UserKnownHostsFile=/dev/null',
+        "ubuntu@#{connect_to_host.private_dns_name}",
+        'sudo', 'su', '-'
+    end
+  ensure
+    Kernel.system 'ssh-add', '-d', private_key_path.to_s
+  end
+
+when :"status:wait"
+  desired_status = ARGV.shift
+  if desired_status == 'down'
+    Kernel.sleep(2) while stack.up?
+  else
+    Kernel.sleep(2) until stack.up? and stack.aws_status == desired_status
+  end
 
 when :status
   stack.must_be_up!

data/lib/aws/with-stacco-patches.rb

@@ -0,0 +1,149 @@
+require 'aws'
+
+class AWS::CloudFormation::StackEvent
+  def operation
+    self.resource_status.split('_', 2).first
+  end
+
+  def status
+    stat = self.resource_status.split('_', 2).last
+    return "started" if (self.resource_type == "AWS::CloudFormation::Stack" and stat == "IN_PROGRESS")
+    return "WORKING" if stat == "IN_PROGRESS"
+    stat
+  end
+
+  def error
+    self.resource_status_reason if (self.resource_status_reason and self.resource_status_reason !~ / Initiated$/)
+  end
+end
+
+class AWS::CloudFormation::Stack
+  attr_accessor :service_registry
+
+  def resources_of_type(type_name)
+    self.resources.find_all{ |r| r.resource_type == "AWS::#{type_name}" && r.resource_status =~ /_COMPLETE/ && r.resource_status != "DELETE_COMPLETE" }
+  end
+
+  def distributions
+    self.resources_of_type("CloudFront::Distribution").map{ |res| @service_registry[:cloudfront].distributions[res.physical_resource_id] }
+  end
+
+  def buckets
+    self.resources_of_type("S3::Bucket").map{ |res| @service_registry[:s3].buckets[res.physical_resource_id] }.find_all{ |r| r.exists? }
+  end
+
+  def instances
+    self.resources_of_type("EC2::Instance").map{ |res| @service_registry[:ec2].instances[res.physical_resource_id] }
+  end
+
+  def elastic_ips
+    self.resources_of_type("EC2::EIP").map{ |res| @service_registry[:ec2].elastic_ips[res.physical_resource_id] }
+  end
+
+  def server_certificates(opts = {})
+    certs = @service_registry[:iam].server_certificates.find_all{ |cert| cert.path == "/cloudfront/#{self.name}/" }
+
+    if cnames = opts.delete(:domain)
+      certs = Hash[ *(certs.map{ |cert| [cert.name, cert] }.flatten) ]
+
+      cnames = [cnames] unless cnames.kind_of?(Array)
+      cnames = cnames.flatten
+
+      possible_cert_names = cnames.map{ |cname| parts = cname.split('.').reverse; (1..parts.length).map{ |len| parts[0, len].reverse.join('.') }.reverse }.flatten
+
+      possible_cert_names.map{ |cname| certs[cname] }.compact
+    else
+      certs
+    end
+  end
+end
+
+class AWS::CloudFront
+  def distributions
+    DistributionCollection.new(self)
+  end
+
+  class DistributionCollection
+    def initialize(svc)
+      @svc = svc
+    end
+
+    def [](k)
+      Distribution.new(@svc, @svc.client.get_distribution(id: k))
+    end
+
+    def each
+      dists = @svc.client.list_distributions[:items]
+      dists.each do |dist|
+        yield Distribution.new(@svc, @svc.client.get_distribution(id: dist[:id]))
+      end
+    end
+
+    include Enumerable
+  end
+end
+
+class AWS::CloudFront::Distribution
+  def initialize(svc, data)
+    @svc = svc
+    @data = data
+  end
+
+  def aliases
+    @data[:distribution_config][:aliases][:items]
+  end
+
+  def id
+    @data[:id]
+  end
+
+  def config
+    self.make_exportable(@data[:distribution_config])
+  end
+
+  def make_exportable(o)
+    case o
+    when nil
+      ""
+    when Hash
+      h = {}
+      o.each{ |k, v| h[k] = make_exportable(v) }
+      h
+    when Array
+      o.map{ |e| make_exportable(e) }
+    else
+      o
+    end
+  end
+
+  def update
+    begin
+      yield
+
+      @svc.client.update_distribution(
+        id: self.id,
+        distribution_config: self.config,
+        if_match: @data[:etag]
+      )
+    end
+  end
+
+  def price_class
+    @data[:distribution_config][:price_class].split('_').last.downcase.intern
+  end
+
+  def price_class=(new_class)
+    @data[:distribution_config][:price_class] = "PriceClass_#{new_class.to_s.capitalize}"
+  end
+
+  def certificate
+    @data[:distribution_config][:viewer_certificate][:iam_certificate_id]
+  end
+
+  def certificate=(cert_id)
+    @data[:distribution_config][:viewer_certificate] = {
+      iam_certificate_id: cert_id,
+      ssl_support_method: "sni-only"
+    }
+  end
+end

data/lib/stacco.rb

@@ -1,418 +1,4 @@
-require 'set'
-require 'ostruct'
-require 'base64'
-require 'pathname'
-require 'shellwords'
-
-require 'json'
-require 'yaml'
-require 'erb'
-require 'inifile'
-
-require 'aws'
-
-class AWS::CloudFormation::StackEvent
-  def operation
-    self.resource_status.split('_', 2).first
-  end
-
-  def status
-    stat = self.resource_status.split('_', 2).last
-    return "started" if (self.resource_type == "AWS::CloudFormation::Stack" and stat == "IN_PROGRESS")
-    return "WORKING" if stat == "IN_PROGRESS"
-    stat
-  end
-
-  def error
-    self.resource_status_reason if (self.resource_status_reason and self.resource_status_reason !~ / Initiated$/)
-  end
-end
-
-class AWS::CloudFormation::Stack
-  attr_accessor :service_registry
-
-  def resources_of_type(type_name)
-    self.resources.find_all{ |r| r.resource_type == "AWS::#{type_name}" && r.resource_status =~ /_COMPLETE/ && r.resource_status != "DELETE_COMPLETE" }
-  end
-
-  def distributions
-    self.resources_of_type("CloudFront::Distribution").map{ |res| @service_registry[:cloudfront].distributions[res.physical_resource_id] }
-  end
-
-  def buckets
-    self.resources_of_type("S3::Bucket").map{ |res| @service_registry[:s3].buckets[res.physical_resource_id] }.find_all{ |r| r.exists? }
-  end
-
-  def elastic_ips
-    self.resources_of_type("EC2::EIP").map{ |res| @service_registry[:ec2].elastic_ips[res.physical_resource_id] }
-  end
-end
-
-class AWS::CloudFront
-  def distributions
-    self.client.list_distributions[:items].map{ |dist| Distribution.new(self.client, self.client.get_distribution(id: dist[:id])) }
-  end
-end
-
-class AWS::CloudFront::Distribution
-  def initialize(client, data)
-    @client = client
-    @data = data
-  end
-
-  def aliases
-    @data[:distribution_config][:aliases][:items]
-  end
-
-  def id
-    @data[:id]
-  end
-
-  def config
-    self.make_exportable(@data[:distribution_config])
-  end
-
-  def make_exportable(o)
-    case o
-    when nil
-      ""
-    when Hash
-      h = {}
-      o.each{ |k, v| h[k] = make_exportable(v) }
-      h
-    when Array
-      o.map{ |e| make_exportable(e) }
-    else
-      o
-    end
-  end
-
-  def update
-    begin
-      yield
-
-      @client.update_distribution(
-        id: self.id,
-        distribution_config: self.config,
-        if_match: @data[:etag]
-      )
-    end
-  end
-
-  def price_class
-    @data[:distribution_config][:price_class].split('_').last.downcase.intern
-  end
-
-  def price_class=(new_class)
-    @data[:distribution_config][:price_class] = "PriceClass_#{new_class.to_s.capitalize}"
-  end
-
-  def certificate
-    @data[:distribution_config][:viewer_certificate][:iam_certificate_id]
-  end
-
-  def certificate=(cert_id)
-    @data[:distribution_config][:viewer_certificate] = {
-      iam_certificate_id: cert_id,
-      ssl_support_method: "sni-only"
-    }
-  end
-end
-
-module Stacco
-  module Resources
-    RootDir = Pathname.new(File.expand_path("../../priv", __FILE__))
-
-    templates_dir = Stacco::Resources::RootDir + "templates"
-    Templates = {
-      cloudformation: (templates_dir + "cloudformation.json.erb").read
-    }
-
-    module CloudInit
-      cloud_init_scripts_dir = Stacco::Resources::RootDir + "cloud-init"
-
-      CommonScripts = {
-        before: (cloud_init_scripts_dir + "common.before.sh").readlines,
-        after: (cloud_init_scripts_dir + "common.after.sh").readlines
-      }
-
-      RoleScripts = {}
-      (cloud_init_scripts_dir + "roles").children.each{ |path| RoleScripts[path.basename('.sh').to_s.intern] = path.readlines }
-    end
-  end
-end
-
-class Stacco::Orchestrator
-  def self.from_config(config_body)
-    config_body = config_body.read if config_body.respond_to?(:read)
-    self.new YAML.load(config_body)
-  end
-
-  def initialize(config)
-    @config = config
-
-    storage_bucket_name = @config['storage_bucket']
-
-    s3 = AWS::S3.new(
-      access_key_id: @config['aws']['access_key_id'],
-      secret_access_key: @config['aws']['secret_access_key'],
-      region: @config['aws']['region']
-    )
-
-    @bucket = s3.buckets[storage_bucket_name]
-    s3.buckets.create(storage_bucket_name) unless @bucket.exists?
-  end
-
-  def stacks
-    @bucket.objects.with_prefix('stack/').to_a[1..-1].map{ |obj| Stacco::Stack.new(obj) }
-  end
-
-  def define_stack(config_body)
-    config_body = config_body.read if config_body.respond_to?(:read)
-
-    config = YAML.load(config_body)
-
-    stack_name = config['name']
-
-    stack_object = @bucket.objects["stack/#{stack_name}"]
-    stack_object.write(config.to_yaml)
-
-    Stacco::Stack.new(stack_object)
-  end
-end
-
-class Stacco::Stack
-  def initialize(config_object)
-    @config_object = config_object
-
-    aws_creds = self.aws_credentials
-
-    @services = {
-      ec2: AWS::EC2.new(aws_creds),
-      s3: AWS::S3.new(aws_creds),
-      cloudformation: AWS::CloudFormation.new(aws_creds),
-      cloudfront: AWS::CloudFront.new(aws_creds)
-    }
-
-    @aws_stack = @services[:cloudformation].stacks[self.name]
-    @aws_stack.service_registry = @services
-  end
-
-  def connections
-    connections = {}
-    @aws_stack.elastic_ips.each{ |eip| connections[eip.instance.tags["aws:cloudformation:logical-id"]] = eip }
-    connections
-  end
-
-  def resource_summaries
-    @aws_stack.resource_summaries
-  end
-
-  def must_be_up!
-    unless self.up?
-      $stderr.puts "stack #{self.name} is down"
-      Kernel.exit 1
-    end
-  end
-
-  def aws_status
-    @aws_stack.status
-  end
-
-  def status
-    self.up? ? self.aws_status : "DOWN"
-  end
-
-  def config
-    YAML.load(@config_object.read)
-  end
-
-  def config=(new_config)
-    @config_object.write(new_config.to_yaml)
-  end
-
-  def update_config
-    # TODO
-  end
-
-  def aws_credentials
-    Hash[ *(self.config['aws'].map{ |k, v| [k.intern, v] }.flatten) ]
-  end
-
-  def description
-    self.config['description']
-  end
-
-  def name
-    self.config['name']
-  end
-
-  def name=(new_name)
-    update_config{ |config| config.merge("name" => new_name) }
-  end
-
-  def up?
-    @aws_stack.exists?
-  end
-
-  def up!
-    if @aws_stack.exists?
-      @aws_stack.update(template: self.cloudformation_template)
-    else
-      @services[:cloudformation].stacks.create(self.name, self.cloudformation_template)
-    end
-  end
-
-  def up_since
-    @aws_stack.creation_time if @aws_stack.exists?
-  end
-
-  def initialize_distributions!
-    cloudfront_certs = self.config['cloudfront']['certificates']
-    @services[:cloudfront].distributions.each do |dist|
-      dist.update do
-        dist.price_class = :"100"
-        dist.certificate = cloudfront_certs[dist.aliases.first]
-      end
-    end
-  end
-
-  def down!
-    return false unless self.up?
-
-    @aws_stack.buckets.each{ |bucket| bucket.delete! }
-    @aws_stack.delete
-
-    Kernel.sleep(2) while self.up?
-
-    true
-  end
-
-  def cloudformation_template
-    tpl = Stacco::Template.new(self, Stacco::Resources::Templates[:cloudformation])
-    tpl.result(self.config)
-  end
-
-  def iam_private_key
-    @config_object.bucket.objects.with_prefix("sshkey/#{self.name}-").to_a.sort_by{ |obj| obj.key.split('/').last.split('-').last.to_i }.last
-  end
-
-  def iam_keypair_name
-    "stacco-" + self.iam_private_key.key.split('/').last
-  end
-
-  def iam_private_key_material
-    self.iam_private_key.read
-  end
-
-  def stream_events
-    Enumerator.new do |out|
-      known_events = Set.new
-      ticks_without_add = 0
-
-      while self.up?
-        added = 0
-
-        @aws_stack.events.sort_by{ |ev| ev.timestamp }.each do |event|
-          next if known_events.include? event.event_id
-          out.yield event
-
-          known_events.add event.event_id
-          added += 1
-          ticks_without_add = 0
-
-        end
-
-        ticks_without_add += 1 if added == 0
-
-        if ticks_without_add >= 8 and (Math.log2(ticks_without_add) % 1) == 0.0
-          jobs = @aws_stack.resource_summaries
-          active_jobs = jobs.find_all{ |job| job[:resource_status] =~ /IN_PROGRESS$/ }.map{ |job| job[:logical_resource_id] }.sort
-          unless active_jobs.empty?
-            out.yield OpenStruct.new(
-              logical_resource_id: "Scheduler",
-              status: "WAIT",
-              operation: "WAIT",
-              timestamp: Time.now,
-              error: "waiting on #{active_jobs.join(', ')}"
-            )
-          end
-        end
-
-        Kernel.sleep 2
-      end
-    end
-  end
-end
-
-class Stacco::Template
-  class RenderContext < OpenStruct
-    def initialize(stack, config, vars)
-      @stack = stack
-      @config = config
-      super(vars)
-    end
-
-    def j(str)
-      str.to_json
-    end
-
-    def ja(indent_n, lns)
-      indent = " " * indent_n
-      newline = "\n".to_json
-      lns.map do |ln|
-        if ln.chomp.empty?
-          "%s \n" % [indent]
-        else
-          "%s%s, %s,\n" % [indent, ln.chomp.to_json, newline]
-        end
-      end.join[indent_n .. -3]
-    end
-  end
-
-  def udscript(roles, vars)
-    roles = roles.map{ |role| role.intern }
-
-    unless roles.include? :backend
-      vars = vars.dup
-      vars.delete 'wallet_data'
-    end
-
-    lns = []
-    lns.concat(vars.map do |k, v|
-      var_val = v.include?("\0") ? Base64.encode64(v) : v
-      "export #{k.to_s.upcase}=#{Shellwords.escape(var_val)}\n"
-    end)
-    lns.concat Stacco::Resources::CloudInit::CommonScripts[:before]
-    roles.each{ |role| lns.concat Stacco::Resources::CloudInit::RoleScripts[role] }
-    lns.concat Stacco::Resources::CloudInit::CommonScripts[:after]
-    lns
-  end
-
-  def initialize(stack, template_body)
-    @stack = stack
-    template_body = template_body.read if template_body.respond_to?(:read)
-    @template_body = template_body
-  end
-
-  def result(vars)
-    cloudformation_vars = {}
-    cloudformation_vars.update vars
-    cloudformation_vars.update vars['cloudformation']
-
-    cloudinit_vars = {}
-    cloudinit_vars.update vars
-    cloudinit_vars.update vars['cloud-init']
-    cloudinit_vars.each do |k, v|
-      cloudinit_vars.delete(k) if v.kind_of?(Hash) or v.kind_of?(Array)
-    end
-
-    cloudformation_vars['userdata'] = {}
-    cloudformation_vars['roles'].each do |host_name, host_roles|
-      cloudformation_vars['userdata'][host_name] = udscript(host_roles, cloudinit_vars)
-    end
-
-    b = RenderContext.new(@stack, vars, cloudformation_vars).instance_eval{ binding }
-    ERB.new(@template_body).result(b)
-  end
-end
+require 'stacco/resources'
+require 'stacco/stack'
+require 'stacco/orchestrator'
+require 'stacco/template'