rubycfn 0.3.2 → 0.3.3

data/templates/compiler.rb.erb

@@ -0,0 +1,61 @@
+require "git-revision"
+
+def update_references(contents, environment, artifact_bucket)
+  contents["Resources"].map do |resource|
+    resource_name = resource.shift
+    resource_values = resource.shift
+    if resource_values["Type"] == "AWS::CloudFormation::Stack"
+      template_hash = @stack_hashes[resource_name.to_sym]
+      s3_url = "https://s3.amazonaws.com/#{artifact_bucket}/#{environment}-" \
+               "#{resource_name.downcase}-#{template_hash}.json"
+      resource_values["Properties"]["TemplateURL"] = s3_url
+    end
+  end
+  JSON.pretty_generate(JSON.parse(contents.to_json))
+end
+
+def inject_dummy_resource(stack)
+  hack_stack = JSON.parse(stack)
+  hack_stack["Resources"] = {} if hack_stack["Resources"].nil?
+  hack_stack["Resources"]["CloudFormationDummyResource"] = {
+    "Type": "AWS::CloudFormation::WaitConditionHandle",
+    "Metadata": {
+      "Comment": "Resource to update stack even if there are no changes",
+      "GitCommitHash": Git::Revision.commit
+    }
+  }
+  hack_stack.to_json
+end
+
+def compile_stacks(skip_creation = false)
+  stacks = {}
+  FileUtils.mkdir_p "build" unless skip_creation
+  Module.constants.select do |mod|
+    if mod =~ /Stack$/
+      send("include", Object.const_get("SharedConcerns"))
+      stacks[mod.to_sym] = send("include", Object.const_get(mod)).render_template("AWS")
+    end
+  end
+
+  stacks.each do |stack_name, stack|
+    stack = inject_dummy_resource(stack)
+    next if JSON.parse(stack)["Resources"].nil?
+    stack_to_md5(stack_name, stack)
+    unless skip_creation
+      puts "- Saved #{stack_name} to build/#{ENV["ENVIRONMENT"]}-#{stack_name.downcase}.json"
+      File.open("build/#{ENV["ENVIRONMENT"]}-#{stack_name.downcase}.json", "w") { |f| f.write(JSON.pretty_generate(JSON.parse(stack))) }
+    end
+  end
+
+  stacks.each do |stack_name, stack|
+    stack = inject_dummy_resource(stack)
+    next if JSON.parse(stack)["Resources"].nil?
+    stack = update_references(JSON.parse(stack), ENV["ENVIRONMENT"], ENV["ARTIFACT_BUCKET"])
+    stacks[stack_name] = stack
+    stack_to_md5(stack_name, stack)
+    unless skip_creation
+      File.open("build/#{ENV["ENVIRONMENT"]}-#{stack_name.downcase}.json", "w") { |f| f.write(JSON.pretty_generate(JSON.parse(stack))) }
+    end
+  end
+  stacks
+end

data/templates/core_compile.rb.erb

@@ -0,0 +1,6 @@
+require "digest/md5"
+require "fileutils"
+
+require_relative "../aws_helper/main"
+
+compile_stacks

data/templates/core_deploy.rb.erb

@@ -0,0 +1,115 @@
+require "colorize"
+require "digest/md5"
+require "dotenv"
+require "aws-sdk-s3"
+require "aws-sdk"
+
+require_relative "../aws_helper/main"
+
+env_vars = load_env_vars
+
+set_aws_credentials(
+  env_vars[:aws_region],
+  env_vars[:aws_access_key_id],
+  env_vars[:aws_secret_access_key]
+)
+
+s3_filename = get_parent_stack_s3_location(
+  env_vars[:artifact_bucket],
+  env_vars[:environment]
+)
+
+client = Aws::CloudFormation::Client.new
+
+# Store previous CloudFormation events in an array, so that we don't output
+# events from previous deploys.
+
+stack_exists = false
+previous_statuses = []
+80.times do
+  previous_events = get_prior_events(client, env_vars[:stack_name])
+  previous_statuses = previous_events.map(&:event_id)
+  stack_exists = previous_events.size.to_i.positive? ? true : false
+  break unless stack_exists
+
+  events_last_deploy = get_events_last_deploy(previous_events)
+  last_event = events_last_deploy.shift
+  break if last_event \
+    && (last_event.logical_resource_id == env_vars[:stack_name]) \
+    && (last_event.stack_name == env_vars[:stack_name]) \
+    && (DEPLOYABLE_STATES.include? last_event.resource_status)
+  puts "Stack is currently in #{last_event.resource_status} mode. Waiting for it to finish..." if last_event
+  sleep 15
+end
+
+if stack_exists
+  client.update_stack(
+    stack_name: env_vars[:stack_name],
+    template_url: s3_filename,
+    capabilities: %w(CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND),
+    # stack_policy_body: "StackPolicyBody",
+    # stack_policy_url: "StackPolicyURL",
+    tags: [
+      {
+        key: "team",
+        value: "<%= name.downcase %>"
+      }
+    ],
+    # client_request_token: "ClientRequestToken",
+  )
+else
+  client.create_stack(
+    stack_name: env_vars[:stack_name],
+    template_url: s3_filename,
+    timeout_in_minutes: 60,
+    capabilities: %w(CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND),
+    on_failure: "ROLLBACK",
+    # stack_policy_body: "StackPolicyBody",
+    # stack_policy_url: "StackPolicyURL",
+    tags: [
+      {
+        key: "team",
+        value: "<%= name.downcase %>"
+      }
+    ],
+    # client_request_token: "ClientRequestToken",
+    enable_termination_protection: false
+  )
+end
+
+## Rudimentary feedback for CI/CD
+shown_log_lines = {}
+
+360.times do
+  resp = client.describe_stack_events(
+    stack_name: env_vars[:stack_name]
+  )
+  resp.stack_events.to_a.reverse.each_with_index do |event, index|
+    next if previous_statuses.include? event.event_id
+    @completed = false
+    @stack_name = event.stack_name
+    @logical_resource_id = event.logical_resource_id
+    @resource_type = event.resource_type
+    @timestamp = event.timestamp
+    @resource_status = event.resource_status
+    @resource_status_reason = event.resource_status_reason
+    log_line = "[#{@logical_resource_id.green}] #{@timestamp.to_s.blue}: " \
+               "#{@resource_status.white} #{@resource_status_reason.to_s.red}"
+    puts log_line unless shown_log_lines[log_line]
+    shown_log_lines[log_line] = true
+    if (@stack_name == env_vars[:stack_name]) \
+      && (@logical_resource_id == env_vars[:stack_name]) \
+      && (END_STATES.include? @resource_status) \
+      && (index + 1 == resp.stack_events.to_a.size)
+      @completed = true
+    end
+  end
+  if @completed
+    puts "==================================="
+    puts " STATUS: #{@resource_status} #{@resource_status_reason}"
+    puts "==================================="
+    raise "#{@resource_status} #{@resource_status_reason}" if FAILURE_STATES.include? @resource_status
+    break
+  end
+  sleep(10)
+end

data/templates/core_diff.rb.erb

@@ -0,0 +1,59 @@
+require "diffy"
+require_relative "../aws_helper/main"
+
+env_vars = load_env_vars
+
+set_aws_credentials(
+  env_vars[:aws_region],
+  env_vars[:aws_access_key_id],
+  env_vars[:aws_secret_access_key]
+)
+
+client = Aws::CloudFormation::Client.new
+template = client.get_template(
+  stack_name: "#{ENV["ENVIRONMENT"]}-#{ENV["PROJECT_NAME"]}"
+)
+
+s3 = Aws::S3::Resource.new(region: env_vars[:aws_region])
+orig_template = {}
+
+template = JSON.parse(template.template_body)
+template["Resources"].each do |resource_name, content|
+  if content["Type"] == "AWS::CloudFormation::Stack"
+    stack_name = "#{ENV["PROJECT_NAME"].capitalize}Stack"
+    orig_template[stack_name] = JSON.pretty_generate(
+      JSON.parse(
+        template.to_json
+      )
+    )
+  end
+  next unless content["Type"] == "AWS::CloudFormation::Stack"
+  s3_filename = content["Properties"]["TemplateURL"].split("/").last
+  orig_template[resource_name] = JSON.pretty_generate(
+    JSON.parse(
+      s3.client.get_object(
+        bucket: env_vars[:artifact_bucket],
+        key: s3_filename
+      ).body.read
+    )
+  )
+end
+
+stacks = compile_stacks(true)
+@stack_hashes.each do |stack_name, _hash|
+  new_template = JSON.pretty_generate(
+    JSON.parse(
+      stacks[stack_name]
+    )
+  )
+  diff = Diffy::Diff.new(
+    orig_template[stack_name.to_s], new_template
+  ).to_s(:color)
+
+  if diff.strip.empty?
+    puts "No difference between local #{stack_name} and remote #{stack_name}"
+  else
+    puts "Orig #{stack_name} vs #{stack_name}:"
+    puts diff
+  end
+end

data/templates/core_upload.rb.erb

@@ -0,0 +1,3 @@
+require_relative "../aws_helper/main"
+
+upload_stacks

data/templates/dependencies.rb.erb

@@ -0,0 +1,23 @@
+def load_env_vars
+  Dotenv.load(".env")
+  Dotenv.load(".env.#{ENV["ENVIRONMENT"]}")
+  Dotenv.load(".env.private")
+  check_dependencies
+  {
+    aws_region: ENV["AWS_REGION"],
+    aws_access_key_id: ENV["AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"],
+    artifact_bucket: ENV["ARTIFACT_BUCKET"],
+    environment: ENV["ENVIRONMENT"],
+    stack_name: ENV["STACK_NAME"]
+  }
+end
+
+def check_dependencies
+  ENV["AWS_REGION"] ||= ENV["AWS_DEFAULT_REGION"]
+  raise "AWS_REGION not set" unless ENV["AWS_REGION"]
+  raise "ARTIFACT_BUCKET not set" unless ENV["ARTIFACT_BUCKET"]
+  raise "ENVIRONMENT not set" unless ENV["ENVIRONMENT"]
+  raise "STACK_NAME not set" unless ENV["STACK_NAME"]
+  raise "AWS CREDENTIALS NOT SET" unless ENV["AWS_ACCESS_KEY_ID"] && ENV["AWS_SECRET_ACCESS_KEY"]
+end

data/templates/deploy.rb.erb

@@ -0,0 +1,53 @@
+WAITING_STATES = %w(
+  CREATE_IN_PROGRESS DELETE_IN_PROGRESS ROLLBACK_IN_PROGRESS
+  UPDATE_COMPLETE_CLEANUP_IN_PROGRESS UPDATE_IN_PROGRESS
+  UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS UPDATE_ROLLBACK_IN_PROGRESS
+).freeze
+SUCCESS_STATES = %w(CREATE_COMPLETE UPDATE_COMPLETE).freeze
+FAILURE_STATES = %w(
+  CREATE_FAILED DELETE_FAILED UPDATE_ROLLBACK_FAILED
+  ROLLBACK_FAILED ROLLBACK_COMPLETE ROLLBACK_FAILED
+  UPDATE_ROLLBACK_COMPLETE UPDATE_ROLLBACK_FAILED
+).freeze
+END_STATES = SUCCESS_STATES + FAILURE_STATES
+DEPLOYABLE_STATES = %w(
+  CREATE_COMPLETE UPDATE_COMPLETE ROLLBACK_COMPLETE
+  UPDATE_ROLLBACK_COMPLETE
+).freeze
+
+# Method to retrieve paste CF events.
+# Arguments: Aws::Client and stack name
+def get_prior_events(client, stack_name)
+  client.describe_stack_events(
+    stack_name: stack_name
+  ).stack_events.to_a
+rescue => exception
+  raise exception unless exception.class == Aws::CloudFormation::Errors::ValidationError
+  []
+end
+
+def get_events_last_deploy(previous_events)
+  initiated = false
+  previous_events.map do |event|
+    initiated = true if event.resource_status_reason == "User Initiated"
+    initiated ? nil : event
+  end.to_a.compact
+end
+
+# Method to predict location of the s3 CloudFormation artifact
+def get_parent_stack_s3_location(bucket, environment)
+  stacks = compile_stacks(true)
+  parent_stack = nil
+
+  stacks.each do |stack_name, stack|
+    next if JSON.parse(stack)["Resources"].nil?
+    JSON.parse(stack)["Resources"].each do |_resource, payload|
+      if payload["Type"] == "AWS::CloudFormation::Stack"
+        parent_stack = stack_name
+        break
+      end
+    end
+  end
+
+  "https://s3.amazonaws.com/#{bucket}/#{environment}-#{parent_stack.downcase}-#{@stack_hashes[parent_stack.to_sym]}.json"
+end

data/templates/ecs_stack.rb.erb

@@ -0,0 +1,12 @@
+module EcsStack
+  extend ActiveSupport::Concern
+  include Rubycfn
+
+  included do
+    include Concerns::GlobalVariables
+    include Concerns::SharedMethods
+    include EcsStack::EcsCluster
+
+    description generate_stack_description("EcsStack")
+  end
+end

data/templates/ecs_stack_concern.rb.erb

@@ -0,0 +1,20 @@
+module EcsStack
+  module EcsCluster
+    extend ActiveSupport::Concern
+
+    included do
+      transform
+      parameter :vpc,
+                description: "VPC ID"
+
+      # Create an empty ECS Cluster to launch fargate bastions (or other things) in
+      resource :<%= name.downcase %>_ecs_cluster,
+               type: "AWS::ECS::Cluster"
+
+      output :<%= name.downcase %>_ecs_cluster,
+             value: "<%= name %>EcsCluster".ref
+      output :<%= name.downcase %>_ecs_cluster_arn,
+             value: "<%= name.downcase %>EcsCluster".ref("Arn")
+    end
+  end
+end

data/templates/global_variables.rb.erb

@@ -7,6 +7,10 @@ module Concerns
                default: "test",
                global: true,
                value: ENV["ENVIRONMENT"]
+
+      variable :stack_name,
+               default: "#{environment}-<%= name %>",
+               value: ENV["STACK_NAME"]
     end
   end
 end

data/templates/helper_methods.rb.erb

@@ -0,0 +1,3 @@
+def file_to_inline(filename)
+  File.open(filename).read.split("\n")
+end

data/templates/helpers.rb.erb

@@ -0,0 +1,7 @@
+def stack_to_md5(stack_name, stack)
+  @stack_hashes[stack_name] = Digest::MD5.hexdigest(JSON.pretty_generate(JSON.parse(stack.to_s)))
+end
+
+def generate_s3_filename(filename, hash)
+  "#{File.basename(filename, ".json")}-#{hash}.json"
+end

data/templates/main.rb.erb

@@ -6,7 +6,7 @@ Dotenv.load(".env.private")
 Dotenv.load(".env.#{ENV["ENVIRONMENT"]}")
 
 # Include all group concerns
-Dir[File.expand_path('../shared_concerns/', __FILE__) + '/*.rb'].sort.each do |file|
+Dir[File.expand_path("../shared_concerns/", __FILE__) + "/**/*.rb"].sort.each do |file|
   require file
 end
 
@@ -21,9 +21,9 @@ module SharedConcerns
 end
 
 # Include all stack concerns
-Dir[File.expand_path('../stacks/', __FILE__) + '/*.rb'].sort.each do |file|
-  subdir = File.basename(file, ".rb") 
-  Dir[File.expand_path('../stacks/', __FILE__) + "/#{subdir}/*.rb"].sort.each do |subfile|
+Dir[File.expand_path("../stacks/", __FILE__) + "/**/*.rb"].sort.each do |file|
+  subdir = File.basename(file, ".rb")
+  Dir[File.expand_path("../stacks/", __dir__) + "/#{subdir}/**/*.rb"].sort.each do |subfile|
     require subfile
   end
   require file

data/templates/main_aws_helper.rb.erb

@@ -0,0 +1,16 @@
+require "aws-sdk-s3"
+require "aws-sdk"
+require "colorize"
+require "digest/md5"
+require "dotenv"
+require "fileutils"
+require "json"
+
+require_relative "aws_sdk"
+require_relative "compiler"
+require_relative "dependencies"
+require_relative "deploy"
+require_relative "helpers"
+require_relative "upload_stack"
+
+@stack_hashes = {}

data/templates/parent_stack_spec.rb.erb

@@ -0,0 +1,38 @@
+require "spec_helper"
+
+require "rubycfn"
+require "active_support/concern"
+require_relative "../../lib/main.rb"
+
+module ParentSpec
+  extend ActiveSupport::Concern
+  include Rubycfn
+
+  included do
+    description "<%= name %> RSpec"
+    include Concerns::GlobalVariables
+    include Concerns::SharedMethods
+    include <%= name %>Stack::Parent
+  end
+end
+
+ParentSpecCfn = include ParentSpec
+
+describe ParentSpec do
+  RspecParentSpec = ParentSpecCfn.render_template
+  let(:template) { JSON.parse(RspecParentSpec) }
+
+  context "Renders template" do
+    subject { template }
+
+    it { should have_key "Resources" }
+
+    context "Has Required Resources" do
+      let(:resources) { template["Resources"] }
+      subject { resources }
+
+      it { should have_key "VpcStack" }
+      it { should have_key "EcsStack" }
+    end
+  end
+end

data/templates/project_concern.rb.erb

@@ -1,59 +1,25 @@
+require_relative "../vpc_stack/subnets"
+
 module <%= name %>Stack
-  module Main
+  module Parent
     extend ActiveSupport::Concern
 
     included do
-      # Example stack parameter
-      #parameter :example_stack_parameter_name,
-      #          default: "example_value"
-
-      # Example variables
-      variable :example_variable,
-               required: false,
-               value: ENV["SOME_ENV_VAR"],
-               default: "default_value"
-
-      variable :cidr_block,
-               default: "10.0.0.0/16"
-      variable :enable_dns_support,
-               default: true
-      variable :enable_dns_hostnames,
-               default: true
-
-      # Example resource, custom name
-      resource :my_example_s3_bucket,
-               type: "AWS::S3::Bucket" do |r|
-        #r.depends_on "SomeLogicalResourceName"
-        r.property(:bucket_name) { example_variable }
+      resource :vpc_stack,
+               type: "AWS::CloudFormation::Stack" do |r|
+        r.property(:template_u_r_l) { "vpcstack" }
+        r.property(:timeout_in_minutes) { "5" }
       end
 
-      # Example VPC, creates VPC, IGW, Route, and VPCGatewayAttachment
-      # Settable variables:
-      # - cidr_block (defaults to 10.0.0.0/16)
-      # - enable_dns_support (boolean, default true)
-      # - enable_dns_hostnames (boolean, default true)
-      # - instance_tenancy (`default` or `dedicated`) 
-      resource :my_first,
-               type: RubyCfn::VPC do |r|
-        r.set(:cidr_block) { "10.1.0.0/16" }
-      end
-
-      # Example resource, dynamically generated name
-      resource :my_example_s3_bucket,
-               type: "AWS::S3::Bucket"
-
-      # Multiple resources of the same type
-      resource :my_sqs_queue,
-               amount: 3,
-               type: "AWS::SQS::Queue"
-
-      queue_names = %w(MyFirstQueue MySecondQueue)
-
-      # Multiple resources with index, starts at 0
-      resource :sqs_with_index,
-               amount: 2,
-               type: "AWS::SQS::Queue" do |r, index|
-         r.property(:queue_name) { queue_names[index] }
+      resource :ecs_stack,
+               type: "AWS::CloudFormation::Stack" do |r|
+        r.depends_on %w(VpcStack)
+        r.property(:template_u_r_l) { "ecsstack" }
+        r.property(:parameters) do
+          {
+            "Vpc": "VpcStack".ref("Outputs.<%= name %>Vpc")
+          }
+        end
       end
     end
   end

data/templates/project_stack.rb.erb

@@ -3,7 +3,10 @@ module <%= name %>Stack
   include Rubycfn
 
   included do
-    include <%= name %>Stack::Main
-    include <%= name %>Stack::CICD
+    include Concerns::GlobalVariables
+    include Concerns::SharedMethods
+    include <%= name %>Stack::Parent
+
+    description generate_stack_description("ParentStack")
   end
 end

data/templates/shared_methods.rb.erb

@@ -0,0 +1,38 @@
+require "git-revision"
+
+module Git
+  class Revision
+    class << self
+      def dirty?
+        !`git diff --numstat | wc -l`.strip.to_i.zero?
+      end
+    end
+  end
+end
+
+module Concerns
+  module SharedMethods
+    extend ActiveSupport::Concern
+    included do
+      # Convert array
+      def recipients_to_array(val)
+        val.split(",").map do |email|
+          {
+            "Endpoint": email,
+            "Protocol": "email-json"
+          }
+        end
+      end
+
+      # Returns 1 if enviroments[] match the environment, otherwise 0
+      def get_resources_amount(environments = %w(production rspec))
+        ((environments.include? ENV["ENVIRONMENT"]) && 1) || 0
+      end
+
+      def generate_stack_description(stack_name)
+        "#{stack_name}-#{Git::Revision.commit}" \
+        "#{Git::Revision.dirty? ? "-dirty" : ""}"
+      end
+    end
+  end
+end

data/templates/spec_helper.rb.erb

@@ -9,6 +9,8 @@ require "rspec"
 require "rspec/its"
 require "rspec/given"
 
+ENV["ENVIRONMENT"] = "rspec"
+
 RSpec.configure do |config|
   config.color = true
   config.formatter = "documentation"
@@ -23,4 +25,4 @@ RSpec.configure do |config|
   config.run_all_when_everything_filtered = true
   config.filter_run_excluding :slow unless ENV["SLOW_SPECS"]
   config.filter_run_excluding :debug unless ENV["DEBUG_SPECS"]
-end
+end

data/templates/subnets.rb.erb

@@ -0,0 +1,18 @@
+def vpc_subnets
+  [
+    {
+      "ec2_public": {
+        "owner": "<%= name.downcase %>",
+        "public": true,
+        "offset": 2
+      }
+    },
+    {
+      "ec2_private": {
+        "owner": "<%= name.downcase %>",
+        "public": false,
+        "offset": 3
+      }
+    }
+  ]
+end

data/templates/upload_stack.rb.erb

@@ -0,0 +1,27 @@
+def upload_stacks
+  env_vars = load_env_vars
+
+  set_aws_credentials(
+    env_vars[:aws_region],
+    env_vars[:aws_access_key_id],
+    env_vars[:aws_secret_access_key]
+  )
+
+  s3 = create_bucket_if_not_exists(
+    env_vars[:aws_region],
+    env_vars[:artifact_bucket]
+  )
+
+  stacks = compile_stacks(true)
+  stacks.each do |stack_name, stack|
+    next if JSON.parse(stack)["Resources"].nil?
+    hash = @stack_hashes[stack_name.to_sym]
+    local_file = "#{env_vars[:environment]}-#{stack_name.downcase}.json"
+    s3_filename = "#{env_vars[:environment]}-#{stack_name.downcase}-#{hash}.json"
+    # content = JSON.parse(stack).to_json
+    obj = s3.bucket(env_vars[:artifact_bucket]).object(s3_filename)
+    content = File.open("build/#{local_file}").read
+    obj.put(body: content)
+    puts "Uploaded #{stack_name} to s3://#{env_vars[:artifact_bucket]}/#{s3_filename}"
+  end
+end