awsdsl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ae154e947361276d4545c34fffffbc0eb07caf3e
+  data.tar.gz: b0a01325a610b74f8d8a9b38bf314a79e9f85324
+SHA512:
+  metadata.gz: 5537d5bfd37391d089dec9988d5af873d04c4b0a9e5a4ee748684a061a51d4b5e6535b06523d686b0f24a595b5efc85e235b96cded6251e8c1738119a4a0f929
+  data.tar.gz: eedb01fb25690bd8c417f9dadfc3cd6f7f65ea1088ace302a50bc8538e88d6920724b929151d316a6bd113f73a0b608b8606b877aa9bfe8619f479024873d851

data/.gitignore

@@ -0,0 +1,28 @@
+*.gem
+*.rbc
+.bundle
+.config
+.rvmrc
+.yardoc
+/Gemfile.lock
+_yardoc
+/coverage
+/doc/
+/pkg
+/spec/reports
+/Berksfile*
+tmp
+*~
+*.tar*
+\#*
+.DS_Store
+/spec/knife.rb
+/spec/*.pem
+/features/config.yml
+*.sw[op]
+\.\#*
+rerun.txt
+.rspec
+.kitchen
+vendor/ruby
+.ruby-version

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'rake'
+gem 'rspec'
+gem 'pry'

data/Gemfile.lock

@@ -0,0 +1,167 @@
+GIT
+  remote: https://github.com/josephglanville/gersberms
+  revision: b979728441156433c8a57d845a058c1b51e5e755
+  specs:
+    gersberms (0.0.1)
+      aws-sdk (~> 1.0)
+      berkshelf (~> 3.2.1)
+      net-scp (~> 1.2.1)
+      net-ssh (~> 2.9.1)
+      rake
+      thor (~> 0.19)
+
+PATH
+  remote: .
+  specs:
+    awsdsl (0.0.1)
+      activesupport (~> 4)
+      aws-sdk (~> 1.0)
+      cfndsl
+      clamp (~> 0.6)
+      gersberms
+      netaddr
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.3.7)
+    aws-sdk (1.63.0)
+      aws-sdk-v1 (= 1.63.0)
+    aws-sdk-v1 (1.63.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    berkshelf (3.2.3)
+      addressable (~> 2.3.4)
+      berkshelf-api-client (~> 1.2)
+      buff-config (~> 1.0)
+      buff-extensions (~> 1.0)
+      buff-shell_out (~> 0.1)
+      celluloid (~> 0.16.0)
+      celluloid-io (~> 0.16.1)
+      cleanroom (~> 1.0)
+      faraday (~> 0.9.0)
+      minitar (~> 0.5.4)
+      octokit (~> 3.0)
+      retryable (~> 2.0)
+      ridley (~> 4.0)
+      solve (~> 1.1)
+      thor (~> 0.19)
+    berkshelf-api-client (1.2.1)
+      faraday (~> 0.9.0)
+    buff-config (1.0.1)
+      buff-extensions (~> 1.0)
+      varia_model (~> 0.4)
+    buff-extensions (1.0.0)
+    buff-ignore (1.1.1)
+    buff-ruby_engine (0.1.0)
+    buff-shell_out (0.2.0)
+      buff-ruby_engine (~> 0.1.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    celluloid-io (0.16.2)
+      celluloid (>= 0.16.0)
+      nio4r (>= 1.1.0)
+    cfndsl (0.1.14)
+    clamp (0.6.4)
+    cleanroom (1.0.0)
+    coderay (1.1.0)
+    dep-selector-libgecode (1.0.2)
+    dep_selector (1.0.3)
+      dep-selector-libgecode (~> 1.0)
+      ffi (~> 1.9)
+    diff-lcs (1.2.5)
+    erubis (2.7.0)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.8)
+    hashie (2.1.2)
+    hitimes (1.2.2)
+    i18n (0.7.0)
+    json (1.8.2)
+    method_source (0.8.2)
+    mini_portile (0.6.2)
+    minitar (0.5.4)
+    minitest (5.6.0)
+    mixlib-authentication (1.3.0)
+      mixlib-log
+    mixlib-log (1.6.0)
+    multipart-post (2.0.0)
+    net-http-persistent (2.9.4)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    netaddr (1.5.0)
+    nio4r (1.1.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    octokit (3.8.0)
+      sawyer (~> 0.6.0, >= 0.5.3)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.4.2)
+    retryable (2.0.1)
+    ridley (4.1.2)
+      addressable
+      buff-config (~> 1.0)
+      buff-extensions (~> 1.0)
+      buff-ignore (~> 1.1)
+      buff-shell_out (~> 0.1)
+      celluloid (~> 0.16.0)
+      celluloid-io (~> 0.16.1)
+      erubis
+      faraday (~> 0.9.0)
+      hashie (>= 2.0.2, < 3.0.0)
+      json (>= 1.7.7)
+      mixlib-authentication (>= 1.3.0)
+      net-http-persistent (>= 2.8)
+      retryable (>= 2.0.0)
+      semverse (~> 1.1)
+      varia_model (~> 0.4)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.2)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    sawyer (0.6.0)
+      addressable (~> 2.3.5)
+      faraday (~> 0.8, < 0.10)
+    semverse (1.2.1)
+    slop (3.6.0)
+    solve (1.2.1)
+      dep_selector (~> 1.0)
+      semverse (~> 1.1)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    timers (4.0.1)
+      hitimes
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    varia_model (0.4.0)
+      buff-extensions (~> 1.0)
+      hashie (>= 2.0.2, < 3.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awsdsl!
+  gersberms!
+  pry
+  rake
+  rspec

data/README.md

@@ -0,0 +1,212 @@
+AWS DSL
+======
+
+This project is an opinionated take on running applications on AWS.
+It leverages [CloudFormation](http://aws.amazon.com/cloudformation/) and [Gersberms](https://github.com/josephglanville/gersberms) to build your application into an Amazon Machine Image and deploy it on bare EC2 servers.
+
+In a nutshell you declare your infrastructure in a Stackfile, which is a Ruby DSL that describes CloudFormation resources and AMI build instructions.
+
+It is in some ways analagous to OpsWorks however it's less intrusive and focuses on building immutable AMIs and replacing machines during updates rather than updating already running machines by re-running Chef.
+
+That said it you are already using OpsWorks it would be easy to get started using AWS DSL.
+
+Install
+-------
+
+For now I recommending using Bundler to install and manage AWS DSL.
+Simply add this to your Gemfile
+
+```ruby
+gem 'awsdsl', git: 'https://github.com/josephglanville/awsdsl'
+```
+
+I will publish the gem to RubyGems when I feel it's stabilized.
+
+Getting Started
+---------------
+
+To get started with AWS DSL you need a few things.
+
+* Your application deployable using Chef.
+* Your Chef cookbooks managed with Berkshelf.
+* Basic understanding of EC2, ELB and any other resources you might need.
+
+Because AWS DSL abstracts away the vast majority of CloudFormation you shouldn't need an indepth understanding of CloudFormation but it doesn't hurt.
+
+Simple Stackfile
+----------------
+
+```ruby
+stack 'static' do
+  description 'static files example'
+
+  vpc 'static' do
+    region 'ap-southeast-2'
+    subnet 'public' do
+      az 'a', 'b'
+    end
+  end
+
+  role 'nginx' do
+    vpc 'static'
+    subnet 'public'
+    load_balancer 'static' do
+      listener port: 80
+      health_check target: 'HTTP:80/'
+    end
+    update_policy min_inservice: 0
+    instance_type 't2.micro'
+    key_pair 'joseph@reinteractive.net'
+    chef_provisioner runlist: 'nginx'
+    vars nginx: {
+      init_style: 'upstart'
+    }
+  end
+end
+```
+
+This simple stack declares a simple role along with provisioning a full VPC.
+Said VPC includes 2 subnets across 2 AZs and an Internet Gateway to allow public addressing to work.
+It runs the nginx::default recipe when preparing the AMI, providing the contents of vars as the node attributes.
+
+Complex Stackfile
+-----------------
+
+```ruby
+stack 'logs' do
+  description 'logstash cluster'
+  ssl_cert_arn = 'arn:aws:iam::account_id::certificate'
+  zone_arn = 'arn:aws:route53:::hostedzone/zone_id'
+  snapshot_bucket_arn = 'arn:aws:s3:::snapshot_bucket'
+  cloudtrail_queue_arn = 'arn:aws:sqs:ap-southeast-2:account_id:queue'
+
+  role_profile 'es_comms' do
+    security_group 'sg-id'
+    subnets 'subnet-id'
+    vpc 'vpc-id'
+  end
+
+  role_profile 'es_bucket' do
+    policy_statement effect: 'Allow', action: 's3:ListBucket', resource: snapshot_bucket_arn
+    policy_statement effect: 'Allow',
+                     action: %w(s3:GetObject s3:PutObjecs3:DeleteObject s3:DeleteObject),
+                     resource: "#{snapshot_bucket_arn}/*"
+  end
+
+  role_profile 'ec2_discovery' do
+    policy_statement effect: 'Allow', action: 'ec2:DescribeInstances', resource: '*'
+  end
+
+  role 'logstash' do
+    include_profile 'ec2_discovery', 'es_comms'
+    load_balancer 'logstash' do
+      listener port: 80
+      listener port: 443, proto: 'HTTPS', cert: ssl_cert_arn
+      listener port: 9000, proto: 'TCP'
+      dns_record name: 'logstash.zone.com', zone: 'zone-id'
+      health_check target: 'HTTP:80/health'
+    end
+    policy_statement effect: 'Allow', action: 'sqs:*', resource: cloudtrail_queue_arn
+    min_size 2
+    max_size 4
+    tgt_size 2
+    update_policy pause_time: '5M'
+    instance_type 't2.micro'
+    chef_provisioner runlist: 'logstash'
+  end
+
+  role 'elasticsearch' do
+    include_profile 'ec2_discovery', 'es_bucket', 'es_comms'
+    load_balancer 'elasticsearch' do
+      listener port: 9200
+      health_check target: 'HTTP:9200/'
+      dns_record name: 'elasticsearch.zone.com', zone: 'zone-id'
+      security_group 'sg-id'
+      internal true
+    end
+    min_size 3
+    max_size 5
+    tgt_size 5
+    update_policy pause_time: '10M', min_inservice: 3
+    instance_type 't2.micro'
+    block_device name: '/dev/sda1', size: 20
+    chef_provisioner runlist: 'elasticsearch'
+    allow role: 'logstash', ports: 9200
+    allow role: 'utility', ports: 9200
+    allow role: 'elasticsearch', ports: 9200
+  end
+
+  role 'utility' do
+    include_profile 'es_bucket', 'es_comms'
+    policy_statement effect: 'Allow',
+                     action: [
+                       'route53:ChangeResourceRecordSets',
+                       'route53:GetHostedZone',
+                       'route53:ListResourceRecordSets'
+                     ],
+                     resource: zone_arn
+    policy_statement effect: 'Allow', action: 'route53:ListHostedZones', resource: '*'
+    min_size 0
+    max_size 1
+    tgt_size 1
+    update_policy min_inservice: 0
+    instance_type 't2.micro'
+    chef_provisioner runlist: 'utility'
+  end
+
+  elasticache 'redis' do
+    vpc 'vpc-id'
+    subnet 'subnet-id'
+    engine 'redis'
+    node_type 't2.micro'
+    allow role: 'logstash'
+  end
+end
+```
+
+This much more complex example deploys a full Elasticsearch Logstash Kibana cluster. Along with standing up an Elasticache cluster running Redis.
+It uses advanced features like Role Profiles (which are effectively Role mixins) and the powerful "allow" syntax which makes configuring security groups a breeze.
+
+As you can see AWS DSL doesn't get in your way if you need to declare additional policy documents or do advanced things like setup SSL listeners or configure DNS records for your ELBs.
+
+Command Line
+------------
+
+Once you have your Stackfile you will need to build the AMIs.
+
+```
+bundle exec awsdsl build
+```
+
+Then create your stack
+
+```
+bundle exec awsdsl create
+```
+
+When you build new AMIs or update settings in your Stackfile you can push updates like so
+
+```
+bundle exec awsdsl update
+```
+
+Philosophy
+----------
+
+AWS DSL was written to enable the versioning of infrastructure alongside code.
+All infrastructure concerns are declared in the AWS DSL Stackfile including how to configure the application runtime environment which will be built into an AMI.
+This is advantagous as updates to code that require infrastructure support can be done in unison.
+
+AWS DSL thinks about your application in terms of Roles. A role is a singular purposed entity in your application and represents a build target and a scaling primitive.
+You specify how to package your application into an AMI, tell it how many instances you want to run and any other considerations like security groups and away it goes.
+
+To DRY up this process AWS DSL has Role Profiles. Role Profiles are analagous to mixins (or multiple inheritance if you must), anything you can put in a Role can be put in Role Profile and then you can mixin multiple Role Profiles into a Role with the include_profile keyword.
+
+
+
+TODO
+----
+
+* cloud-init/cfn-init integration and environment variable system
+* build AMIs seperately
+* cfndsl section to allow arbitrary resource creation

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: [:spec]
+
+task :integration do
+  ENV['INTEGRATION'] = 'true'
+  Rake::Task['spec'].execute
+end

data/awsdsl.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'awsdsl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'awsdsl'
+  spec.version       = AWSDSL::VERSION
+  spec.authors       = ['Joseph Glanville']
+  spec.email         = ['jpg@jpg.id.au']
+  spec.summary       = 'A simple DSL for deploying and running apps on AWS'
+  spec.description   = 'A simple DSL for deploying and running apps on AWS'
+  spec.homepage      = 'https://github.com/josephglanville/awsdsl'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'aws-sdk', '~> 1.0'
+  spec.add_dependency 'activesupport', '~> 4'
+  spec.add_dependency 'clamp', '~> 0.6'
+  spec.add_dependency 'cfndsl', '~> 0.1'
+  spec.add_dependency 'gersberms', '~> 1.0'
+  spec.add_dependency 'netaddr', '~> 1.5'
+end

data/bin/awsdsl

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'awsdsl/command_line'
+
+AWSDSL::CommandLine.run

data/lib/awsdsl/ami_builder.rb

@@ -0,0 +1,98 @@
+require 'gersberms'
+require 'awsdsl/base_ami'
+
+module AWSDSL
+  class AMIBuilder
+    def initialize(stack)
+      @stack = stack
+    end
+
+    def build
+      @stack.roles.each do |role|
+        build_ami(role)
+      end
+    end
+
+    def latest_amis
+      @stack.roles.each do |role|
+        role.ami = latest_ami(role).id
+      end
+      @stack
+    end
+
+    def self.build(stack)
+      AMIBuilder.new(stack).build
+    end
+
+    def self.latest_amis(stack)
+      AMIBuilder.new(stack).latest_amis
+    end
+
+    def build_ami(role)
+      output_ami = ami_name(role)
+      # TODO(jpg): This needs to be better, also deep_merge
+      json = (@stack.vars || {}).merge(role.vars || {})
+      @builder = Gersberms::Gersberms.new base_ami: base_ami(role),
+                                          ami_name: output_ami,
+                                          json: json
+      begin
+        start_builder
+        role.file_provisioners.each do |provisioner|
+          @builder.options[:files] = provisioner
+          @builder.upload_files
+        end
+        role.chef_provisioners.each do |provisioner|
+          runlist = provisioner[:runlist]
+          runlist = [runlist] unless runlist.is_a? Array
+          @builder.options[:runlist] = runlist
+          @builder.run_chef
+        end
+        shutdown_builder
+        role.ami = @builder.image.id
+      rescue => e
+        @builder.destroy_instance
+        @builder.destroy_keypair
+        raise "Failed to build AMI for #{role.name}:\nError: #{e.message}\nBacktrace: #{e.backtrace.join("\n")}"
+      end
+    end
+
+    def start_builder
+      @builder.preflight
+      @builder.create_keypair
+      @builder.create_instance
+      @builder.install_chef
+      @builder.upload_cookbooks
+    end
+
+    def shutdown_builder
+      @builder.stop_instance
+      @builder.create_ami
+      @builder.destroy_instance
+      @builder.destroy_keypair
+    end
+
+    def base_ami(role)
+      base = role.base_ami || 'ubuntu'
+      if BaseAMI::DISTROS.include?(base)
+        base = BaseAMI.find(base)
+      end
+      base
+    end
+
+    def ami_name(role)
+      last = latest_ami(role)
+      num = last.name.split('-').last.to_i + 1 if last
+      num ||= 1
+      "#{@stack.name}-#{role.name}-#{num}"
+    end
+
+    def latest_ami(role)
+      ec2 = AWS::EC2.new
+      amis = ec2.images.with_owner('self').select do |i|
+        i.name.start_with?("#{@stack.name}-#{role.name}")
+      end
+      latest_num = amis.map { |i| i.name.split('-').last.to_i }.sort.last
+      amis.select { |i| i.name == "#{@stack.name}-#{role.name}-#{latest_num}" }.first
+    end
+  end
+end

data/lib/awsdsl/base_ami.rb

@@ -0,0 +1,17 @@
+module AWSDSL
+  module BaseAMI
+    DISTROS = %w(ubuntu)
+
+    def self.find(distro)
+      send(distro)
+    end
+
+    def self.ubuntu
+      ec2 = AWS::EC2.new
+      ami = ec2.images.with_owner('099720109477')
+            .filter('name', 'ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server*')
+            .sort_by(&:name).last
+      ami.id
+    end
+  end
+end