stacco 0.1.10 → 0.1.17

data/lib/stacco/base.rb

@@ -0,0 +1,2 @@
+module Stacco
+end

data/lib/stacco/orchestrator.rb

@@ -0,0 +1,43 @@
+require 'aws/with-stacco-patches'
+require 'yaml'
+
+require 'stacco/base'
+
+class Stacco::Orchestrator
+  def self.from_config(config_body)
+    config_body = config_body.read if config_body.respond_to?(:read)
+    self.new YAML.load(config_body)
+  end
+
+  def initialize(config)
+    @config = config
+
+    storage_bucket_name = @config['storage_bucket']
+
+    s3 = AWS::S3.new(
+      access_key_id: @config['aws']['access_key_id'],
+      secret_access_key: @config['aws']['secret_access_key'],
+      region: @config['aws']['region']
+    )
+
+    @bucket = s3.buckets[storage_bucket_name]
+    s3.buckets.create(storage_bucket_name) unless @bucket.exists?
+  end
+
+  def stacks
+    @bucket.objects.with_prefix('stack/').to_a[1..-1].map{ |obj| Stacco::Stack.new(obj) }
+  end
+
+  def define_stack(config_body)
+    config_body = config_body.read if config_body.respond_to?(:read)
+
+    config = YAML.load(config_body)
+
+    stack_name = config['name']
+
+    stack_object = @bucket.objects["stack/#{stack_name}"]
+    stack_object.write(config.to_yaml)
+
+    Stacco::Stack.new(stack_object)
+  end
+end

data/lib/stacco/resources.rb

@@ -0,0 +1,25 @@
+require 'pathname'
+
+require 'stacco/base'
+
+module Stacco::Resources
+  RootDir = Pathname.new(File.expand_path("../../../priv", __FILE__))
+
+  templates_dir = Stacco::Resources::RootDir + "templates"
+  Templates = {
+    cloudformation: (templates_dir + "cloudformation.json.erb").read
+    #cloudformation: (templates_dir + "bexng.rb").read
+  }
+
+  module CloudInit
+    cloud_init_scripts_dir = Stacco::Resources::RootDir + "cloud-init"
+
+    CommonScripts = {
+      before: (cloud_init_scripts_dir + "common.before.sh").readlines,
+      after: (cloud_init_scripts_dir + "common.after.sh").readlines
+    }
+
+    RoleScripts = {}
+    (cloud_init_scripts_dir + "roles").children.each{ |path| RoleScripts[path.basename('.sh').to_s.intern] = path.readlines }
+  end
+end

data/lib/stacco/stack.rb

@@ -0,0 +1,195 @@
+require 'set'
+require 'ostruct'
+require 'yaml'
+
+require 'aws/with-stacco-patches'
+
+require 'stacco/base'
+
+require 'stacco/template/old'
+
+class Stacco::Stack
+  def initialize(config_object)
+    @config_object = config_object
+
+    aws_creds = self.aws_credentials
+
+    @services = {
+      ec2: AWS::EC2.new(aws_creds),
+      s3: AWS::S3.new(aws_creds),
+      cloudformation: AWS::CloudFormation.new(aws_creds),
+      cloudfront: AWS::CloudFront.new(aws_creds),
+      iam: AWS::IAM.new(aws_creds)
+    }
+
+    @aws_stack = @services[:cloudformation].stacks[self.name]
+    @aws_stack.service_registry = @services
+  end
+
+  def connections
+    connections = {}
+    @aws_stack.instances.each{ |i| connections[i.tags["aws:cloudformation:logical-id"]] = i }
+    connections
+  end
+
+  def resource_summaries
+    @aws_stack.resource_summaries
+  end
+
+  def must_be_up!
+    unless self.up?
+      $stderr.puts "stack #{self.name} is down"
+      Kernel.exit 1
+    end
+  end
+
+  def aws_status
+    @aws_stack.status
+  end
+
+  def status
+    self.up? ? self.aws_status : "DOWN"
+  end
+
+  def config
+    YAML.load(@config_object.read)
+  end
+
+  def config=(new_config)
+    @config_object.write(new_config.to_yaml)
+  end
+
+  def update_config
+    # TODO
+  end
+
+  def aws_credentials
+    Hash[ *(self.config['aws'].map{ |k, v| [k.intern, v] }.flatten) ]
+  end
+
+  def description
+    self.config['description']
+  end
+
+  def name
+    self.config['name']
+  end
+
+  def name=(new_name)
+    update_config{ |config| config.merge("name" => new_name) }
+  end
+
+  def up?
+    @aws_stack.exists?
+  end
+
+  def up!
+    if @aws_stack.exists?
+      @aws_stack.update(template: self.cloudformation_template)
+    else
+      @services[:cloudformation].stacks.create(self.name, self.cloudformation_template)
+    end
+  end
+
+  def up_since
+    @aws_stack.creation_time if @aws_stack.exists?
+  end
+
+  def initialize_distributions!
+    @services[:cloudfront].distributions.each do |dist|
+      dist.update do
+        dist.price_class = :"100"
+        dist.certificate = @aws_stack.server_certificates(domain: dist.aliases).first.id
+      end
+    end
+  end
+
+  def down!
+    return false unless self.up?
+
+    @aws_stack.buckets.each{ |bucket| bucket.delete! }
+    @aws_stack.delete
+
+    true
+  end
+
+  def cloudformation_template
+    #Kernel.eval(Stacco::Resources::Templates[:cloudformation])
+    tpl = Stacco::Template.const_get(self.config['template']).new
+    tpl.to_json(stack: self)
+  end
+
+  def cloudformation_template_body
+    Stacco::Resources::Templates[:cloudformation]
+  end
+
+  def validate
+    baked_template = self.cloudformation_template
+    test_template = baked_template.gsub(/"[cm][123]\.(\dx)?(small|medium|large)"/, '"m1.small"')
+
+    begin
+      @services[:cloudformation].estimate_template_cost test_template
+      [true]
+    rescue AWS::CloudFormation::Errors::ValidationError => e
+      msg = e.message
+      match = msg.scan(/^Template format error: JSON not well-formed. \(line (\d+), column (\d+)\)$/)
+      if match.length.nonzero?
+        line, column = match.to_a.flatten.map{ |el| el.to_i }
+        [false, msg, [baked_template.split("\n")[line.to_i], column]]
+      else
+        [false, msg]
+      end
+    end
+  end
+
+  def iam_private_key
+    @config_object.bucket.objects.with_prefix("sshkey/#{self.name}-").to_a.sort_by{ |obj| obj.key.split('/').last.split('-').last.to_i }.last
+  end
+
+  def iam_keypair_name
+    "stacco-" + self.iam_private_key.key.split('/').last
+  end
+
+  def iam_private_key_material
+    self.iam_private_key.read
+  end
+
+  def stream_events
+    Enumerator.new do |out|
+      known_events = Set.new
+      ticks_without_add = 0
+
+      while self.up?
+        added = 0
+
+        @aws_stack.events.sort_by{ |ev| ev.timestamp }.each do |event|
+          next if known_events.include? event.event_id
+          out.yield event
+
+          known_events.add event.event_id
+          added += 1
+          ticks_without_add = 0
+
+        end
+
+        ticks_without_add += 1 if added == 0
+
+        if ticks_without_add >= 8 and (Math.log2(ticks_without_add) % 1) == 0.0
+          jobs = @aws_stack.resource_summaries
+          active_jobs = jobs.find_all{ |job| job[:resource_status] =~ /IN_PROGRESS$/ }.map{ |job| job[:logical_resource_id] }.sort
+          unless active_jobs.empty?
+            out.yield OpenStruct.new(
+              logical_resource_id: "Scheduler",
+              status: "WAIT",
+              operation: "WAIT",
+              timestamp: Time.now,
+              error: "waiting on #{active_jobs.join(', ')}"
+            )
+          end
+        end
+
+        Kernel.sleep 2
+      end
+    end
+  end
+end

data/lib/stacco/template.rb

@@ -0,0 +1,209 @@
+require 'set'
+require 'ostruct'
+require 'base64'
+require 'shellwords'
+require 'nokogiri'
+require 'json'
+require 'erb'
+require 'open-uri'
+
+require 'stacco/base'
+
+class String
+  UpcasedInitialisms = Set['ec2', 'iam', 'elb', 'vpc', 'dhcp', 'aws', 'dns']
+
+  def camelize
+    return self if (self != self.downcase)
+    self.split('_').map{ |part| UpcasedInitialisms.include?(part) ? part.upcase : part.capitalize }.join
+  end
+end
+
+class Symbol
+  def camelize
+    self.to_s.camelize.intern
+  end
+end
+
+class Stacco::Template
+  def initialize
+    @parameters = []
+    @resources = []
+  end
+
+  def udscript(roles, vars)
+    roles = roles.map{ |role| role.intern }
+
+    unless roles.include? :backend
+      vars = vars.dup
+      vars.delete 'wallet_data'
+    end
+
+    lns = []
+    lns.concat(vars.map do |k, v|
+      var_val = v.include?("\0") ? Base64.encode64(v) : v
+      "export #{k.to_s.upcase}=#{Shellwords.escape(var_val)}\n"
+    end)
+    lns.concat Stacco::Resources::CloudInit::CommonScripts[:before]
+    roles.each{ |role| lns.concat Stacco::Resources::CloudInit::RoleScripts[role] }
+    lns.concat Stacco::Resources::CloudInit::CommonScripts[:after]
+    lns
+  end
+
+  class Path < Array
+    def elements_for_json
+      self
+    end
+  end
+
+  class ColonPath < Path
+    def to_s
+      self.elements_for_json.map{ |sym| sym.to_s.camelize }.join('::')
+    end
+  end
+
+  class ConcatenatedPath < Path
+    def to_s
+      self.elements_for_json.map{ |sym| sym.to_s.camelize }.join
+    end
+  end
+
+  class Var < ConcatenatedPath
+    def elements_for_json
+      [:stacco] + self
+    end
+  end
+
+  class StackVar < Var
+    def elements_for_json
+      [:stacco, :stack] + self
+    end
+  end
+
+  class Type < ColonPath
+    def elements_for_json
+      [:aws] + self
+    end
+
+    def resource(name = nil)
+      Resource[*(self + [name].compact)]
+    end
+  end
+
+  class Resource < ConcatenatedPath
+    def elements_for_json
+      [:stacco, :resource] + self
+    end
+  end
+
+  class Mapping < ConcatenatedPath
+    def elements_for_json
+      [:stacco, :mapping] + self
+    end
+  end
+
+  AttachmentTypes = {
+    Set[Type[:ec2, :vpc], Type[:ec2, :internet_gateway]] => Type[:ec2, :vpc_gateway_attachment],
+  }
+
+
+  def parameter(name, type, opts = {})
+    @parameters.push [name, type, opts]
+  end
+
+  def resource(type, opts = {})
+    name = type.resource(opts.delete(:name))
+    @resources.push [name, type, opts]
+    name
+  end
+
+  def mapping(name, hsh)
+    @mappings.push [name, hsh]
+  end
+
+  def attach(resource_a, resource_b, opts = {})
+  end
+
+  def render
+    tpl = {AWSTemplateFormatVersion: "2010-09-09"}
+    tpl[:description] = "Stacco stack"
+
+    tpl[:Mappings] = {}
+
+    @mappings.each do |(name, hsh)|
+      name = [name] if name.kind_of?(String) or name.kind_of?(Symbol)
+      tpl[:Mappings][Mapping[*name]] = hsh
+    end
+
+    tpl[:Parameters] = {}
+
+    @parameters.each do |(name, type, args)|
+      name = [name] if name.kind_of?(String) or name.kind_of?(Symbol)
+      constraints = {}
+      args.merge(type: type.camelize).each do |k,v|
+        constraints[k.camelize] = v
+      end
+
+      tpl[:Parameters][name] = constraints
+    end
+
+    tpl[:Resources] = {}
+
+    @resources.each do |(name, type, args)|
+      name = [name] if name.kind_of?(String) or name.kind_of?(Symbol)
+      properties = {}
+      args.each do |k,v|
+        properties[k.camelize] = v
+      end
+
+      tpl[:Resources][name] = {"Type" => type, "DependsOn" => [], "Properties" => properties}
+    end
+
+    tpl
+  end
+
+  def export_for_json(o, is_key = false)
+    case o
+    when Numeric
+      o.to_s
+    when TrueClass, FalseClass
+      o.to_s
+    when NilClass
+      raise ArgumentError, o
+    when Var
+      if is_key
+        o.to_s
+      else
+        {"Ref" => o.to_s}
+      end
+    when Symbol
+      ConcatenatedPath[o].to_s
+    when Path
+      o.to_s
+    when Array
+      o.map{ |el| export_for_json(el) }
+    when Hash
+      nh = {}
+      o.each{ |k, v| nh[ export_for_json(k, true) ] = export_for_json(v) }
+      nh
+    when Regexp
+      o.source
+    else
+      o
+    end
+  end
+
+  def to_json(opts = {})
+    self.export_for_json(self.render).to_json
+  end
+end
+
+if __FILE__ == $0
+  tpl = Stacco::Template.new
+
+  if ARGV.delete('-d')
+    require 'pp'
+    pp JSON.parse(tpl.to_json)
+  else
+    puts tpl.to_json
+  end
+end