demeter-cli 0.0.4

data/lib/demeter/commands/apply.rb

@@ -0,0 +1,15 @@
+require 'demeter/aws/manage_security_groups'
+require 'terminal-table'
+require 'logger'
+require_relative 'base'
+
+module Demeter
+  module Commands
+    class Apply < Base
+      def start
+        sgs_manager = Demeter::Aws::ManageSecurityGroups.new(ec2:@ec2, options:@options)
+        diff = sgs_manager.apply
+      end
+    end
+  end
+end

data/lib/demeter/commands/base.rb

@@ -0,0 +1,21 @@
+require 'logger'
+
+module Demeter
+  module Commands
+    class Base
+      def initialize(options)
+        check_path
+        ENV['DEMETER_ENV'] = options['environment']
+        Dotenv.load(".env.#{options['environment']}")
+        ::Aws.config.update(:logger => Logger.new(STDOUT))
+        @ec2 = ::Aws::EC2::Client.new()
+        @options = options
+      end
+
+      def check_path
+        fail "configs directory not found!" if !File.directory?('./configs')        
+        fail "variables directory not found!" if !File.directory?('./variables')        
+      end
+    end
+  end
+end

data/lib/demeter/commands/generate.rb

@@ -0,0 +1,109 @@
+require 'colorize'
+require 'demeter/aws/manage_security_groups'
+require_relative 'base'
+
+module Demeter
+  module Commands
+    class Generate < Base
+      def initialize(options)
+        super
+        @ids = options[:ids]
+      end
+      
+      def project_key name
+        name
+        .gsub('::', '_')
+        .gsub('/', '_')
+        .gsub('-', '_')
+        .gsub(' ', '_')
+        .downcase
+      end
+
+      def start
+        # collect vars
+        res = @ec2.describe_security_groups
+        res[:security_groups].each do |object|
+          name_tag = object['tags'].detect{|tag| tag['key'].downcase == 'name'}
+          sg_key = name_tag ? project_key(name_tag['value']) : project_key(object.group_name)  
+          Demeter::set_var("security_group.#{sg_key}.id", object.group_id)
+          Demeter::set_var(object.group_id, "<% security_group.#{sg_key}.id %>")
+        end
+
+        resp = @ec2.describe_security_groups({group_ids: @ids})
+        
+        template = {
+          'environments' => [@options['environment']],
+          'security_groups' => []
+        }
+
+        resp[:security_groups].each do |_sg|
+          name_tag = _sg['tags'].detect{|tag| tag['key'].downcase == 'name'}
+          sg_key = name_tag ? project_key(name_tag['value']) : project_key(_sg.group_name)  
+          
+          sg_template = {
+            'name' => (name_tag ? name_tag['value'] : _sg.group_name),
+            'vpc_id' => '<% env.vpc_id %>',
+            'ingress' => [],
+            'egress' => []
+          }
+         
+          # Ingress 
+          _sg['ip_permissions'].each do |_rule|
+            rule = {
+              'protocol' => _rule.ip_protocol,
+              'from_port' => _rule.from_port.to_i,
+              'to_port' => _rule.to_port.to_i,
+            }
+            
+            if !_rule['user_id_group_pairs'].empty?
+              rule['source_security_groups'] = []
+              _rule['user_id_group_pairs'].each do |_group|
+                group_key = Demeter::vars[_group['group_id']] ? Demeter::vars[_group['group_id']] : _group['group_id']
+                rule['source_security_groups'] << group_key 
+              end
+            end
+ 
+            if !_rule['ip_ranges'].empty?
+              rule['cidr_blocks'] = []
+              _rule['ip_ranges'].each do |_range|
+                rule['cidr_blocks'] << _range['cidr_ip']
+              end
+            end
+
+            sg_template['ingress'] << rule
+          end
+
+          # Egress
+          _sg['ip_permissions_egress'].each do |_rule|
+            rule = {
+              'protocol' => _rule.ip_protocol,
+              'from_port' => _rule.from_port.to_i,
+              'to_port' => _rule.to_port.to_i,
+            }
+            
+            if !_rule['user_id_group_pairs'].empty?
+              rule['source_security_groups'] = []
+              _rule['user_id_group_pairs'].each do |_group|
+                group_key = Demeter::vars[_group['group_id']] ? Demeter::vars[_group['group_id']] : _group['group_id']
+                rule['source_security_groups'] << group_key 
+              end
+            end
+ 
+            if !_rule['ip_ranges'].empty?
+              rule['cidr_blocks'] = []
+              _rule['ip_ranges'].each do |_range|
+                rule['cidr_blocks'] << _range['cidr_ip']
+              end
+            end
+
+            sg_template['egress'] << rule
+          end
+
+          template['security_groups'] << sg_template
+        end
+
+        puts template.to_yaml.gsub('"', '')
+      end
+    end
+  end
+end

data/lib/demeter/commands/plan.rb

@@ -0,0 +1,40 @@
+require 'demeter/aws/manage_security_groups'
+require 'terminal-table'
+require_relative 'base'
+
+module Demeter
+  module Commands
+    class Plan < Base
+      def start
+        sgs_manager = Demeter::Aws::ManageSecurityGroups.new(ec2: @ec2, options: @options)
+        diff = sgs_manager.diff_all
+        rows = []
+        i = 0
+        diff.each do |sg, diffs|
+          rows << :separator if i > 0
+          rows << [{:value => sg, :colspan => 3, :alignment => :left}]
+          rows << :separator
+ 
+          diffs.sort_by! { |d| d[0] }
+          diffs.each do |_diff|
+            if _diff[2].is_a? Array
+              _diff[2].each do |__diff|
+                rows << [_diff[0] == '+' ? _diff[0].colorize(:green) : _diff[0].colorize(:red), _diff[1], __diff.to_s]             
+              end
+            else
+              rows << [_diff[0] == '+' ? _diff[0].colorize(:green) : _diff[0].colorize(:red), _diff[1], _diff[2].to_s] 
+            end
+          end
+          i += 1
+        end
+
+        if rows.empty?
+          puts "All #{Demeter::env} security groups are in sync"
+        else
+          table = Terminal::Table.new :rows => rows
+          puts table
+        end
+      end
+    end
+  end
+end

data/lib/demeter/commands/status.rb

@@ -0,0 +1,42 @@
+require 'colorize'
+require 'demeter/aws/manage_security_groups'
+require 'terminal-table'
+require_relative 'base'
+
+module Demeter
+  module Commands
+    class Status < Base
+      def start
+        sgs_manager = Demeter::Aws::ManageSecurityGroups.new(ec2:@ec2, options:@options)
+        status = sgs_manager.status
+        rows = []
+ 
+        rows << [{:value => "### MANAGED SECURITY GROUPS ###".colorize(:green), :colspan => 3, :alignment => :left}]
+        rows << :separator
+        rows << ['Name', 'Group Name', 'Group ID']
+        rows << :separator
+        
+        status[:managed].each do |sg|
+          rows << [sg[:name], sg[:group_name], sg[:group_id]]
+        end 
+        
+        rows << :separator
+        rows << [{:value => "### UNMANAGED SECURITY GROUPS ###".colorize(:red), :colspan => 3, :alignment => :left}]
+        rows << :separator
+        rows << ['Name', 'Group Name', 'Group ID']
+        rows << :separator
+        
+        status[:unmanaged].each do |sg|
+          rows << [sg[:name], sg[:group_name], sg[:group_id]]
+        end 
+      
+        puts Terminal::Table.new :rows => rows
+ 
+        puts ""
+        puts "#{'MANAGED'.colorize(:green)}: #{status[:managed].count}" 
+        puts "#{'UNMANAGED'.colorize(:red)}: #{status[:unmanaged].count}"
+        puts ""
+      end
+    end
+  end
+end

data/lib/demeter/version.rb

@@ -0,0 +1,6 @@
+module Demeter
+  # Defines the version
+  #
+  # @since 0.0.1
+  VERSION = '0.0.4'.freeze
+end

data/spec/ec2_stub.rb

@@ -0,0 +1,77 @@
+class Ec2Stub
+
+  def self.describe_with_security_groups
+    Aws::EC2::Client.new(stub_responses: {
+      describe_security_groups: { 
+        security_groups: [{ 
+          group_name: "ec2::infra/apollo::web", 
+          group_id: "sg-11111", 
+          description: "Managed by Demeter", 
+          ip_permissions: [
+            { ip_protocol: "tcp", 
+              from_port: 22, 
+              to_port: 22, 
+              user_id_group_pairs: [], 
+              ip_ranges: [
+                {cidr_ip: "10.0.0.0/8"}, 
+                {cidr_ip: "10.10.0.0/24"}
+              ], 
+              prefix_list_ids: [] 
+            }, 
+            { ip_protocol: "tcp", 
+              from_port: 22, to_port: 22, 
+              user_id_group_pairs: [{ user_id: "2222", group_name: nil, group_id: "sg-11111" }], 
+              ip_ranges: [], 
+              prefix_list_ids: []
+            }
+          ], 
+          ip_permissions_egress: [
+            { ip_protocol: "-1", from_port: nil, to_port: nil, user_id_group_pairs: [], ip_ranges: [{cidr_ip: "0.0.0.0/0"}], prefix_list_ids: [] }
+          ],
+          vpc_id: "vpc-12345", 
+          tags: [
+            { :key => "Name", :value => "ec2::infra/apollo::web" }
+          ]
+          }] 
+        }
+      })
+  end
+
+  def self.describe_with_no_groups
+    Aws::EC2::Client.new(stub_responses: {
+      describe_security_groups: { 
+        security_groups: [{ 
+          group_name: "does-not-exist", 
+          group_id: "sg-111111111", 
+          description: "Some things", 
+          ip_permissions: [
+            { ip_protocol: "tcp", 
+              from_port: 22, 
+              to_port: 22, 
+              user_id_group_pairs: [], 
+              ip_ranges: [
+                {cidr_ip: "10.0.0.0/8"}, 
+                {cidr_ip: "10.10.0.0/24"}
+              ], 
+              prefix_list_ids: [] 
+            }, 
+            { ip_protocol: "tcp", 
+              from_port: 22, to_port: 22, 
+              user_id_group_pairs: [{ user_id: "564673040929", group_name: nil, group_id: "sg-111111111" }], 
+              ip_ranges: [], 
+              prefix_list_ids: []
+            }
+          ], 
+          ip_permissions_egress: [
+            { ip_protocol: "-1", from_port: nil, to_port: nil, user_id_group_pairs: [], ip_ranges: [{cidr_ip: "0.0.0.0/0"}], prefix_list_ids: [] }
+          ],
+          vpc_id: "vpc-12345", 
+          tags: [
+            { :key => "Name", :value => "does-not-exist" }
+          ]
+          }] 
+        }
+      })
+  end
+
+end

data/spec/manage_security_groups_spec.rb

@@ -0,0 +1,72 @@
+require_relative './spec_helper'
+require_relative 'ec2_stub'
+require 'demeter/aws/manage_security_groups'
+require 'demeter/aws/security_group'
+require 'demeter'
+
+describe Demeter::Aws::ManageSecurityGroups do
+  describe '::describe' do
+
+    context 'with an existing aws SG and a local file' do
+      let(:ec2) do 
+        Ec2Stub.describe_with_security_groups
+      end 
+
+      let(:load_path) { File.expand_path(File.join(__dir__, "projects/simple/*.yml")) }
+      let(:vars_path) { File.expand_path(__dir__) }
+      let(:smanage) { Demeter::Aws::ManageSecurityGroups.new(ec2:ec2, project_path:load_path) }
+
+      it 'loads both aws and local file' do
+        Demeter.root(vars_path)
+        result = smanage.describe
+        expect(result).to_not eq(nil)
+      end
+
+      it 'shows the diff with both aws and local file' do
+        all_diffs = smanage.diff_all
+      end
+
+    end
+
+    context 'with non-existing aws SG and a local file' do
+      let(:ec2) do 
+        Ec2Stub.describe_with_no_groups
+      end
+      let(:load_path) { File.expand_path(File.join(__dir__, "projects/simple/**/*.yml")) }
+      let(:smanage) { Demeter::Aws::ManageSecurityGroups.new(ec2:ec2, project_path:load_path) }
+
+      it 'creates the new security group' do
+        allow(ec2).to receive(:create_security_group).and_return(double('ec2_response', group_id: "sg-12345"))
+        smanage.create_all
+        expect(ec2).to have_received(:create_security_group)
+      end
+      
+      it 'has no diffs' do
+        the_diff = smanage.diff_all                                                                                                                           
+        pluses = the_diff.select { |s| s[0] == "+" }                                                                                                          
+        minuses = the_diff.select { |s| s[0] == "-" }                                                                                                         
+        expect(minuses.size).to eq(0)                                                                                                                         
+        expect(pluses.size).to eq(0)                                                                                                                          
+      end
+
+    end
+
+    context 'with different config in the local file than exists in aws' do
+      let(:ec2) do 
+        Ec2Stub.describe_with_security_groups
+      end
+      let(:load_path) { File.expand_path(File.join(__dir__, "projects/simple/ec2_apollo.yml")) }
+      let(:smanage) { Demeter::Aws::ManageSecurityGroups.new(ec2:ec2, project_path:load_path) }
+
+      it 'removes source security group and adds cidr block' do
+        the_diff = smanage.diff_all                                                                                                                           
+        pluses = the_diff[the_diff.keys.first].select { |s| s[0] == "+" }
+        minuses = the_diff[the_diff.keys.first].select { |s| s[0] == "-" }
+        expect(minuses.size).to eq(1)
+        expect(pluses.size).to eq(1)
+      end
+    end
+
+  end
+
+end

data/spec/projects/simple/ec2_apollo.yml

@@ -0,0 +1,17 @@
+security_groups:
+  - name: "ec2::infra/apollo::web"
+    vpc_id: "vpc-11111"
+    ingress:
+      - from_port: 22
+        to_port: 22
+        protocol: "tcp"
+        cidr_blocks:
+          - "10.0.0.0/8"
+          - "10.20.0.0/8"
+          - "10.10.0.0/24"
+    egress:
+      - from_port: 0
+        to_port: 0
+        protocol: "-1"
+        cidr_blocks:
+          - "0.0.0.0/0"

data/spec/projects/with_vars/ec2_apollo.yml

@@ -0,0 +1,17 @@
+security_groups:
+  - name: ec2::infra/apollo::web
+    vpc_id: <% env.vpc_id %>
+    ingress:
+      - from_port: 22
+        to_port: 22
+        protocol: tcp
+        cidr_blocks:
+          - 10.0.0.0/8
+          - 10.20.0.0/8
+          - 10.10.0.0/24
+          - <% global.var_cidr_blocks %>
+        source_security_groups: 
+          - sg-12345
+    egress:
+      - from_port: -1
+        to_port: -1  

data/spec/projects/with_vars/ec2_bastion.yml

@@ -0,0 +1,12 @@
+security_groups:
+  - name: <% ec2::infra/bastion %>
+    vpc_id: <% var.vpc_id %>
+    ingress:
+      - from_port: 22
+        to_port: 22
+        protocol: tcp
+        cidr_blocks:
+          - 10.0.0.0/8
+    egress:
+      - from_port: -1
+        to_port: -1  

data/spec/security_group_spec.rb

@@ -0,0 +1,73 @@
+require_relative './spec_helper'
+require 'demeter/aws/security_group'
+
+describe Demeter::Aws::SecurityGroup do
+    describe '::initialize()' do
+    context 'with aws security group' do
+      let(:ec2) { 
+        Ec2Stub.describe_with_security_groups
+      }
+      let(:security_group_hash) { {name: 'ec2::infra::project', description: 'Demeter'} }
+      let(:security_group) { Demeter::Aws::SecurityGroup.new(ec2) }
+      let(:load_path) { File.expand_path(File.join(__dir__, "projects/with_vars/ec2_apollo.yml")) }
+      let(:vars_path) { File.expand_path(__dir__) }
+
+      it 'loads security group from aws result' do
+        sg = ec2.describe_security_groups(filters: [
+          {name: "group-name", values: ["ec2::infra/apollo::web"]}
+        ]).security_groups.first
+
+        result = security_group.load_aws(sg)
+        expect(result).to eq(true)
+        expect(security_group.group_name).to eq("ec2::infra/apollo::web")
+      end   
+      
+      it 'load_aws with the vars set' do
+        sg = ec2.describe_security_groups(filters: [
+          {name: "group-name", values: ["ec2::infra/apollo::web"]}
+        ]).security_groups.first
+        security_group.load_aws(sg)
+        my_project = security_group.project_key
+        expect(Demeter::vars).to include("security_group.#{my_project}.id")
+      end
+
+      it 'replaces configs with vars' do
+        sg = ec2.describe_security_groups(filters: [
+          {name: "group-name", values: ["ec2::infra/apollo::web"]}
+        ]).security_groups.first
+        security_group.load_aws(sg)
+        expect(security_group.diff).to eq([
+          ["-", "egress", [
+            {:protocol=>"-1", :from_port=>0, :to_port=>0, :cidr_block=>"0.0.0.0/0"}
+          ]],
+          ["-", "ingress", [
+            {:protocol=>"tcp", :from_port=>22, :to_port=>22, :cidr_block=>"10.0.0.0/8"},
+            {:protocol=>"tcp", :from_port=>22, :to_port=>22, :cidr_block=>"10.10.0.0/24"},
+            {:protocol=>"tcp", :from_port=>22, :to_port=>22, :source_security_group=>"sg-11111"}
+          ]],
+        ])
+      end
+
+      it 'replaces array config vars' do
+        Demeter::root(vars_path)
+        project_config = YAML::load_file(load_path)
+        security_group.load_local(project_config['security_groups'].first)
+        diffs = security_group.diff
+        vpcone = diffs.detect { |d| d[1] == "vpc_id" }
+        cidrs = diffs.detect { |s| s[2].is_a?(Array) && !s[2].empty? && s[2].first.has_key?(:cidr_block) }
+        expect(cidrs[2].size).to eq(7)
+      end
+
+      it 'replaces array config vars with update_vars' do
+        Demeter::root(vars_path)
+        project_config = YAML::load_file(load_path)
+        security_group.load_local(project_config['security_groups'].first)
+        updated_hash = security_group.update_vars
+        expect(updated_hash[:ingress].size).to eq(7)
+      end
+    end
+  end 
+end
+
+
+