geoengineer 0.1.0

data/bin/geo

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/geoengineer'
+require_relative '../lib/geoengineer/cli/geo_cli'
+
+geo = GeoCLI.instance
+geo.run

data/lib/geoengineer.rb

@@ -0,0 +1,29 @@
+########################################################################
+# GeoEngineer main Module
+########################################################################
+module GeoEngineer
+end
+
+########################################################################
+# GeoEngineer::Resources Collection of Resources
+########################################################################
+module GeoEngineer::Resources
+end
+
+########################################################################
+# GeoEngineer::Templates Collection of Templates
+########################################################################
+module GeoEngineer::Templates
+end
+
+require 'aws-sdk'
+require 'json'
+require 'ostruct'
+require 'uri'
+require 'securerandom'
+
+Dir["#{File.dirname(__FILE__)}/geoengineer/utils/**/*.rb"].each { |f| require f }
+
+Dir["#{File.dirname(__FILE__)}/geoengineer/*.rb"].each { |f| require f }
+
+Dir["#{File.dirname(__FILE__)}/geoengineer/resources/**/*.rb"].each { |f| require f }

data/lib/geoengineer/cli/geo_cli.rb

@@ -0,0 +1,208 @@
+require_relative '../../geoengineer'
+require 'open3'
+require 'commander'
+require 'colorize'
+require 'terminal-table'
+require 'fileutils'
+require 'json'
+require 'singleton'
+
+# Create GeoCLI for the command requires
+class GeoCLI
+end
+
+require_relative './status_command'
+require_relative './terraform_commands'
+
+def environment(name, &block)
+  GeoCLI.instance.create_environment(name, &block)
+end
+
+def env
+  GeoCLI.instance.environment
+end
+
+def project(org, name, &block)
+  GeoCLI.instance.environment.project(org, name, &block)
+end
+
+# GeoCLI context
+class GeoCLI
+  include Commander::Methods
+  include Singleton
+  include StatusCommand
+  include TerraformCommands
+
+  attr_accessor :environment, :env_name
+
+  # CLI FLAGS AND OPTIONS
+  attr_accessor :verbose, :no_color
+
+  def init_tmp_dir(name)
+    @tmpdir = "#{Dir.pwd}/tmp/#{name}"
+    FileUtils.mkdir_p @tmpdir
+  end
+
+  def init_terraform_files
+    @terraform_file       = "terraform.tf.json"
+    @terraform_state_file = "terraform.tfstate"
+    @plan_file            = "plan.terraform"
+
+    files = [
+      "#{@tmpdir}/#{@terraform_state_file}.backup",
+      "#{@tmpdir}/#{@terraform_file}",
+      "#{@tmpdir}/#{@terraform_state_file}",
+      "#{@tmpdir}/#{@plan_file}"
+    ]
+
+    files.each do |file|
+      File.delete(file) if File.exist?(file)
+    end
+  end
+
+  def create_environment(name, &block)
+    return @environment if @environment
+    if name != @env_name
+      puts "Not loading environment #{name} as env_name is #{@env_name}" if @verbose
+      return NullObject.new
+    end
+
+    @environment = GeoEngineer::Environment.new(name, &block)
+    init_tmp_dir(name)
+    init_terraform_files()
+    @environment
+  end
+
+  def require_from_pwd(file)
+    require "#{Dir.pwd}/#{file}"
+  end
+
+  def require_environment(options)
+    @env_name = options.environment || ENV['GEO_ENV'] || 'staging'
+    puts "Using environment '#{@env_name}'\n" if @verbose
+    begin
+      require_from_pwd "environments/#{@env_name}"
+    rescue LoadError
+      puts "unable to load 'environments/#{@env_name}'" if @verbose
+    end
+  end
+
+  def require_all_projects
+    Dir["#{Dir.pwd}/projects/**/*.rb"].each do |project_file|
+      puts "LOADING #{project_file}" if @verbose
+      require project_file
+    end
+  end
+
+  def require_project_file(project_file)
+    if !File.exist?(project_file) && !File.exist?("#{project_file}.rb")
+      throw "The file \"#{project_file}\" does not exist"
+    end
+    require_from_pwd project_file
+  end
+
+  def require_geo_files(args)
+    return require_all_projects if args.empty?
+    args.each { |project_file| require_project_file(project_file) }
+  end
+
+  def print_validation_errors(errs)
+    puts errs.map { |s| "ERROR: #{s}".colorize(:red) }
+    puts "Total Errors #{errs.length}"
+  end
+
+  def shell_exec(cmd, verbose = @verbose)
+    stdin, stdout_and_stderr, wait_thr = Open3.popen2e({}, *cmd)
+
+    puts(">> #{cmd}\n") if verbose
+    stdout_and_stderr.each do |line|
+      puts(line) if verbose
+    end
+    puts("<< Exited with status: #{wait_thr.value.exitstatus}\n\n") if verbose
+
+    stdin.close
+    stdout_and_stderr.close
+
+    wait_thr.value
+  end
+
+  # This defines the typical action in geo engineer
+  # - require the environment
+  # - require the geo files
+  # - ensure everything is valid
+  # - execute the action
+  # - execute the after hook
+  def init_action(action_name)
+    lambda do |args, options|
+      require_environment(options)
+      require_geo_files(args)
+      throw "Environment not set" unless @environment
+
+      @environment.execute_lifecycle(:before, action_name.to_sym)
+
+      errs = @environment.errors.flatten.sort
+      return print_validation_errors(errs) unless errs.empty?
+
+      yield args, options
+      @environment.execute_lifecycle(:after, action_name.to_sym)
+    end
+  end
+
+  def yes?(question)
+    answer = ask question
+    answer.strip.upcase.start_with? "YES"
+  end
+
+  def graph_cmd
+    command :graph do |c|
+      c.syntax = 'geo graph [<geo_files>]'
+      c.description = 'Generate and graph of the environment resources to GraphViz'
+      action = lambda do |args, options|
+        puts env.to_dot
+      end
+      c.action init_action(:graph, &action)
+    end
+  end
+
+  def gloabl_options
+    global_option('-e', '--environment <name>', "Environment to use")
+
+    @verbose = true
+    global_option('--quiet', 'reduce the noisy outputs (default they are on)') {
+      @verbose = false
+    }
+
+    @no_color = ''
+    global_option('--no-color', 'removes color from the terraform output') {
+      String.disable_colorization = true
+      @no_color = ' -no-color'
+    }
+  end
+
+  def terraform_installed?
+    terraform_version = shell_exec('which terraform')
+    terraform_version.exitstatus.zero?
+  end
+
+  def run
+    program :name, 'GeoEngineer'
+    program :version, '0.0.1'
+    program :description, 'GeoEngineer will help you Terraform your resources'
+    always_trace!
+
+    # check terraform installed
+    return puts "Please install terraform" unless terraform_installed?
+
+    # gloabl_options
+    gloabl_options
+
+    # Add commands
+    plan_cmd
+    apply_cmd
+    graph_cmd
+    status_cmd
+
+    # Execute the CLI
+    run!
+  end
+end

data/lib/geoengineer/cli/status_command.rb

@@ -0,0 +1,101 @@
+# Status Command for Geo
+module GeoCLI::StatusCommand
+  def calculate_type_status(codified, uncodified)
+    total = codified.count + uncodified.count
+    {
+      codified: codified.count,
+      uncodified: uncodified.count,
+      total: total,
+      percent: (100.0 * codified.count) / total
+    }
+  end
+
+  def status_resource_rows(reses)
+    rows = []
+    rows << :separator
+    rows << ['TerraformID', 'GeoID']
+    rows << :separator
+    reses.each do |sg|
+      g_id = sg._geo_id
+      g_id = "<<" if sg._terraform_id == sg._geo_id
+      rows << [sg._terraform_id, g_id]
+    end
+    rows << :separator
+    rows
+  end
+
+  def status_type_rows(type, codified, uncodified, stats)
+    rows = []
+
+    # Codified resources
+    rows << [{ value: "### CODIFIED #{type} ###".colorize(:green), colspan: 2, alignment: :left }]
+    rows.concat status_resource_rows(codified)
+
+    # Uncodified resources
+    rows << [{ value: "### UNCODIFIED #{type} ###".colorize(:red), colspan: 2, alignment: :left }]
+    rows.concat status_resource_rows(uncodified)
+
+    rows.concat status_rows(stats)
+    puts Terminal::Table.new({ rows: rows })
+  end
+
+  def default_status_types
+    [
+      "aws_security_group",
+      "aws_elb",
+      "aws_db_instance",
+      "aws_elasticache_cluster",
+      "aws_s3_bucket",
+      "aws_sqs_queue"
+    ]
+  end
+
+  def calculate_status(type_stats)
+    totals = {
+      codified: 0,
+      uncodified: 0,
+      total: 0
+    }
+    type_stats.each do |type, stats|
+      totals[:codified] += stats[:codified]
+      totals[:uncodified] += stats[:uncodified]
+      totals[:total] += stats[:total]
+    end
+    totals[:percent] = (100.0 * totals[:codified]) / totals[:total]
+    totals
+  end
+
+  def status_rows(stats)
+    rows = []
+    rows << ['CODIFIED'.colorize(:green), stats[:codified]]
+    rows << ['UNCODIFIED'.colorize(:red), stats[:uncodified]]
+    rows << ['TOTAL'.colorize(:blue), stats[:total]]
+    rows << ['PERCENT CODIFIED'.colorize({ mode: :bold }), format('%.2f%', stats[:percent])]
+    rows
+  end
+
+  def status_action
+    lambda do |args, options|
+      type_stats = {}
+      default_status_types.each do |type|
+        codified = @environment.codified_resources(type)
+        uncodified = @environment.uncodified_resources(type)
+        type_stats[type] = calculate_type_status(codified, uncodified)
+        status_type_rows(type, codified, uncodified, type_stats[type]) if @verbose
+      end
+
+      status = calculate_status(type_stats)
+      puts Terminal::Table.new({ rows: status_rows(status) }) if @verbose
+      puts JSON.pretty_generate(status)
+    end
+  end
+
+  def status_cmd
+    command :status do |c|
+      c.syntax = 'geo status [<geo_files>]'
+      c.description = 'Displays the the new, managed and unmanaged resources'
+      action = status_action
+      c.action init_action(:status, &action)
+    end
+  end
+end

data/lib/geoengineer/cli/terraform_commands.rb

@@ -0,0 +1,59 @@
+########################################################################
+# TerraformCommands provides command line terraform commands
+# +plan+ and +apply+ for GeoEngineer
+########################################################################
+module GeoCLI::TerraformCommands
+  def create_terraform_files
+    # create terraform file
+    File.open("#{@tmpdir}/#{@terraform_file}", 'w') { |file|
+      file.write(JSON.pretty_generate(@environment.to_terraform_json()))
+    }
+
+    # create terrafrom state
+    File.open("#{@tmpdir}/#{@terraform_state_file}", 'w') { |file|
+      file.write(JSON.pretty_generate(@environment.to_terraform_state()))
+    }
+  end
+
+  def terraform_plan
+    plan_commands = [
+      "cd #{@tmpdir}",
+      "terraform plan -state=#{@terraform_state_file} -out=#{@plan_file} #{@no_color}"
+    ]
+    shell_exec(plan_commands.join(" && "), true)
+  end
+
+  def terraform_apply
+    apply_commands = [
+      "cd #{@tmpdir}",
+      "terraform apply -state=#{@terraform_state_file} #{@plan_file} #{@no_color}"
+    ]
+    shell_exec(apply_commands.join(" && "), true)
+  end
+
+  def plan_cmd
+    command :plan do |c|
+      c.syntax = 'geo plan [<geo_files>]'
+      c.description = 'Generate and show an execution plan'
+      action = lambda do |args, options|
+        create_terraform_files
+        terraform_plan
+      end
+      c.action init_action(:plan, &action)
+    end
+  end
+
+  def apply_cmd
+    command :apply do |c|
+      c.syntax = 'geo apply [<geo_files>]'
+      c.description = 'Apply an execution plan'
+      action = lambda do |args, options|
+        create_terraform_files
+        return puts "Plan Broken" if terraform_plan.exitstatus.nonzero?
+        return puts "Rejecting Plan" unless yes?("Apply the above plan? [YES/NO]")
+        terraform_apply
+      end
+      c.action init_action(:apply, &action)
+    end
+  end
+end

data/lib/geoengineer/environment.rb

@@ -0,0 +1,186 @@
+########################################################################
+# An Environment is a group of projects, resources and attributes,
+# build to create a terraform file.
+# The goal of GeoEngineer is to build an environment that can be created.
+#
+# An environment has resources, has arbitrary attributes, validations and lifecycle hooks
+########################################################################
+class GeoEngineer::Environment
+  include HasAttributes
+  include HasResources
+  include HasValidations
+  include HasLifecycle
+
+  attr_reader :name, :projects
+
+  validate -> { validate_required_attributes([:region, :account_id]) }
+
+  # Validate resources have unique attributes
+  validate -> {
+    resources_grouped_by(&:terraform_name)
+      .select { |k, v| v.length > 1 }
+      .map { |k, v| "Non-unique type.id #{v.first.for_resource}" }
+  }
+
+  validate -> {
+    resources_grouped_by(&:_terraform_id)
+      .select { |k, v| v.length > 1 && !v.first._terraform_id.nil? }
+      .map { |k, v| "Non-unique _terraform_id #{v.first._terraform_id} #{v.first.for_resource}" }
+  }
+
+  validate -> {
+    resources_grouped_by(&:_geo_id)
+      .select { |k, v| v.length > 1 }
+      .map { |k, v| "Non-unique _geo_id #{v.first._geo_id} #{v.first.for_resource}" }
+  }
+
+  # Validate all resources
+  validate -> { resources.map(&:errors).flatten }
+
+  # Validate all projects (which validate resources)
+  validate -> { projects.map(&:errors).flatten }
+
+  before :validation, -> { self.region = self.region || ENV['AWS_REGION'] }
+
+  def initialize(name, &block)
+    @name = name
+    @projects = []
+    @outputs = []
+    self.send("#{name}?=", true) # e.g. staging?
+    instance_exec(self, &block) if block_given?
+  end
+
+  def resource(type, id, &block)
+    return find_resource(type, id) unless block_given?
+    resource = create_resource(type, id, &block)
+    resource.environment = self
+    resource
+  end
+
+  def output(id, value, &block)
+    output = GeoEngineer::Output.new(id, value, &block)
+    @outputs << output
+    output
+  end
+
+  def all_resources
+    reses = resources
+    @projects.each { |project| reses += project.all_resources }
+    reses
+  end
+
+  # Factory for creating projects inside an environment
+  def project(org, name, &block)
+    # do not add the project a second time
+    exists = @projects.select { |p| p.org == org && p.name == name }.first
+    return exists if exists
+
+    project = GeoEngineer::Project.new(org, name, self, &block)
+
+    supported_environments = [project.environments].flatten
+    # do not add the project if the project is not supported by this environment
+    return NullObject.new unless supported_environments.include? @name
+
+    @projects << project
+    project
+  end
+
+  # DOT Methods
+  # Given an attribute it tries to identify a dependency and return it
+  def extract_dependencies(x)
+    if x.is_a? Array
+      x.map { |y| extract_dependencies(y) }.flatten
+    elsif x.is_a?(String)
+      res = self.find_resource_by_ref(x)
+      return [res] if res
+    elsif x.is_a?(GeoEngineer::Resource)
+      return [x]
+    end
+    []
+  end
+
+  def depends_on(res)
+    all_attributes = []
+    all_attributes.concat res.attributes.values
+    all_attributes.concat res.subresources.map { |sr| sr.attributes.values }.flatten
+    dependencies = Set.new(all_attributes.map { |x| extract_dependencies(x) }.flatten)
+    dependencies.delete(nil)
+    dependencies
+  end
+
+  def to_dot
+    str = ["digraph {"]
+    str.concat(projects.map(&:to_dot))
+    all_resources.each do |res|
+      str << depends_on(res).map { |r| "  #{res.to_ref.inspect} -> #{r.to_ref.inspect}" }
+    end
+    str << " }"
+    str.join("\n")
+  end
+
+  # Terraform Methods
+  def to_terraform
+    # Force preventing the destruction of any resource unless explicitly set
+    # Hopefully this will stop accidentally the environment
+    unless self.allow_destroy
+      all_resources.each { |r|
+        r.lifecycle { prevent_destroy true }
+      }
+    end
+
+    tf_resources = all_resources.map(&:to_terraform)
+    tf_resources += @outputs.compact.map(&:to_terraform)
+    tf_resources.join("\n\n")
+  end
+
+  def to_terraform_json
+    unless self.allow_destroy
+      all_resources.each { |r|
+        r.lifecycle { prevent_destroy true }
+      }
+    end
+
+    h = { resource: json_resources }
+    h[:output] = @outputs.map(&:to_terraform_json) unless @outputs.empty?
+    h
+  end
+
+  def json_resources
+    all_resources.each_with_object({}) do |r, c|
+      c[r.type] ||= {}
+      c[r.type][r.id] = r.to_terraform_json
+      c
+    end
+  end
+
+  def to_terraform_state
+    reses = all_resources.select(&:_terraform_id) # _terraform_id must not be nil
+    reses = reses.map { |r| { "#{r.type}.#{r.id}" => r.to_terraform_state() } }.reduce({}, :merge)
+
+    {
+      version: 1,
+      serial: 1,
+      modules: [
+        {
+          path: [:root],
+          outputs: {},
+          resources: reses
+        }
+      ]
+    }
+  end
+
+  # This method looks into AWS for resources that are not yet codified
+  def codified_resources(type)
+    # managed resources have a remote resource
+    res = self.resources_of_type(type).select { |r| !r.remote_resource.nil? }
+    res.sort_by(&:terraform_name)
+  end
+
+  def uncodified_resources(type)
+    # unmanaged resources have a remote resource without local_resource
+    clazz = self.class.get_resource_class_from_type(type)
+    res = clazz.fetch_remote_resources.select { |r| r.local_resource.nil? }
+    res.sort_by(&:terraform_name)
+  end
+end