environments-list-builder 0.1.8 → 0.1.9

data/lib/cicd/builder/environments-list/mixlib/lib/mixins/no_commands.rb

@@ -0,0 +1,309 @@
+require 'thor'
+require 'awesome_print'
+require 'inifile'
+
+module Amplify
+  module AWS
+    LOG_LEVELS = [:trace, :debug, :info, :step, :warn, :error, :fatal, :todo]
+
+    class CredentialsHash < Hash
+      def initialize
+        super()
+      end
+    end
+    
+    class IniFileCredentials < CredentialsHash
+      def initialize(inifile)
+        super()
+        self.[]=(:inifile, inifile)
+      end
+      def get_options
+        {
+            :inifile => self.[](:inifile),
+            :account => File.basename(self.[](:inifile)),
+        }
+      end
+    end
+
+    class ProfileCredentials < CredentialsHash
+      def initialize(profile)
+        super()
+        self.[]=(:profile, profile)
+      end
+      def get_options
+        {
+            :profile => self.[](:profile),
+            :account => self.[](:profile),
+        }
+      end
+    end
+
+    module MixIns
+      module NoCommands
+        require 'dldinternet/mixlib/logging'
+        include DLDInternet::Mixlib::Logging
+
+        def validate_options
+          @config = @options.dup
+          if @config[:log_level]
+            log_level = @config[:log_level].to_sym
+            raise "Invalid log-level: #{log_level}" unless LOG_LEVELS.include?(log_level)
+            @config[:log_level] = log_level
+          end
+          @config[:log_level] ||= :info
+        end
+
+        def parse_options
+          validate_options
+
+          colormap = {
+              :trace => :blue,
+              :debug => :cyan,
+              :info  => :green,
+              :step  => :green,
+              :warn  => :yellow,
+              :error => :red,
+              :fatal => :red,
+              :todo  => :purple,
+          }
+          unless options[:color]
+            LOG_LEVELS.each do |l,_|
+              colormap[l] = :clear
+            end
+          end
+          lcs = ::Logging::ColorScheme.new( 'compiler', :levels => colormap)
+          scheme = lcs.scheme
+          if options[:color]
+            scheme['trace'] = "\e[38;5;33m"
+            scheme['fatal'] = "\e[38;5;89m"
+            scheme['todo']  = "\e[38;5;55m"
+          end
+          lcs.scheme scheme
+          @config[:log_opts] = lambda{|mlll| {
+              :pattern      => "%#{mlll}l: %m %g\n",
+              :date_pattern => '%Y-%m-%d %H:%M:%S',
+              :color_scheme => 'compiler',
+              :trace        => (@config[:trace].nil? ? false : @config[:trace]),
+              # [2014-06-30 Christo] DO NOT do this ... it needs to be a FixNum!!!!
+              # If you want to do ::Logging.init first then fine ... go ahead :)
+              # :level        => @config[:log_level],
+          }
+          }
+          @logger = getLogger(@config)
+
+          if @options[:inifile]
+            load_credentials_inifile(@options[:inifile], 'global')
+          elsif @options[:profile]
+            unless File.directory?(File.expand_path('~/.aws'))
+              msg = 'Cannot load profile. ~/.aws does not exist!?'
+              @logger.error msg
+              raise msg
+            end
+            unless File.exists?(File.expand_path('~/.aws/config'))
+              msg = 'Cannot load profile. ~/.aws/config does not exist!?'
+              @logger.error msg
+              raise msg
+            end
+            unless File.exists?(File.expand_path('~/.aws/credentials'))
+              msg = 'Cannot load profile. ~/.aws/credentials does not exist!?'
+              @logger.error msg
+              raise msg
+            end
+            @logger.debug "Profile: #{@options[:profile]}"
+            load_credentials_profile(@options[:profile],  {
+                                                            'output'                => 'AWS_DEFAULT_OUTPUT',
+                                                            'region'                => 'AWS_DEFAULT_REGION',
+                                                            'aws_access_key_id'     => 'AWS_ACCESS_KEY_ID',
+                                                            'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY',
+                                                          })
+          end
+
+          if options[:debug]
+            @logger.info "Options:\n#{options.ai}"
+          end
+
+        end
+
+        def expand_options()
+          def _expand(opts,k,v,regex,rerun)
+            matches = v.match(regex)
+            if matches
+              var = matches[1]
+              if opts[var]
+                opts[k]=v.gsub(/\%\(#{var}\)/,opts[var]).gsub(/\%#{var}/,opts[var])
+              else
+                rerun[var] = 1
+              end
+            end
+          end
+
+          pending = nil
+          rerun   = {}
+          begin
+            pending = rerun
+            rerun   = {}
+            @options.to_hash.each{|k,v|
+              if v.to_s.match(/\%/)
+                _expand(@options,k,v,%r'[^\\]\%\((\w+)\)', rerun)
+                _expand(@options,k,v,%r'[^\\]\%(\w+)',     rerun)
+              end
+            }
+            # Should break out the first time that we make no progress!
+          end while pending != rerun
+        end
+
+        def load_credentials_profile(profile, map=nil)
+          begin
+            if @saved_env and @saved_env.size > 0
+              @saved_env.each do |k,v|
+                ENV[k] = v
+              end
+              ENV.to_hash.each do |k,v|
+                unless @saved_env.has_key?(k)
+                  ENV.delete(k)
+                end
+              end
+            else
+              @saved_env = ENV.to_hash.dup
+            end
+            ini = load_inifile('~/.aws/config')
+            ini['default'].each{ |key,value|
+              k = if map[key.to_s]
+                    map[key.to_s]
+                  else
+                    key.to_s
+                  end
+              ENV[k]=value
+            }
+            ini["profile #{profile}"].each{ |key,value|
+              k = if map[key.to_s]
+                    map[key.to_s]
+                  else
+                    key.to_s
+                  end
+              ENV[k]=value
+            }
+             ini = load_inifile('~/.aws/credentials')
+            unless ini.sections.include?(profile)
+              logger.error "No credentials found for profile #{profile}"
+              exit 11
+            end
+            ini[profile].each{ |key,value|
+              k = if map[key.to_s]
+                    map[key.to_s]
+                  else
+                    key.to_s
+                  end
+              ENV[k]=value
+            }
+            expand_options()
+          rescue ::IniFile::Error => e
+            # noop
+          rescue SystemExit => e
+            raise e
+          rescue ::Exception => e
+            @logger.error "#{e.class.name} #{e.message}"
+            raise e
+          end
+        end
+
+        def load_inifile(inifile)
+          inifile = File.expand_path(inifile)
+          unless File.exist?(inifile)
+            raise "#{inifile} not found!"
+          end
+          begin
+            ini = ::IniFile.load(inifile)
+          rescue ::Exception => e
+            @logger.error "#{e.class.name} #{e.message}"
+            raise e
+          end
+        end
+
+        def load_credentials_inifile(inifile, section='global', map=nil)
+          begin
+            ini = load_inifile(inifile)
+            ini[section].each{ |key,value|
+              #@options[key.to_s]=value
+              @logger.debug "#{key} '#{value}'"
+              ENV[key.to_s]=value.to_s
+            }
+            expand_options()
+          rescue ::IniFile::Error => e
+            # noop
+          rescue ::Exception => e
+            @logger.error "#{e.class.name} #{e.message}"
+            raise e
+          end
+        end
+
+        def check_missing_options(method,optionset)
+          parse_options
+          missing = []
+          optionset.each do |req|
+            case req.class.name
+              when /String|Symbol/
+                missing << req unless @options[req]
+              when /Array/
+                flag = false
+                req.each do |o|
+                  if @options[o]
+                    flag = true
+                    break
+                  end
+                end
+                missing << req.join('|') unless flag
+              else
+                logger.fatal "Internal: #{method}: Unsupported missing option type: #{req.class}"
+                exit 99
+            end
+          end
+
+          if missing.size > 0
+            logger.error "#{method}: Missing required options: #{missing.join(', ')}"
+            exit 1
+          end
+        end
+
+        def print_error(err)
+
+          logger.error err.to_s
+
+        end
+
+        def prepare_accounts
+          @accounts = []
+          if @options[:iniglob]
+            iniglob = File.expand_path(@options[:iniglob])
+            Dir.glob( iniglob) {| filename |
+              @accounts << IniFileCredentials.new(filename)
+            }
+          elsif @options[:inifile]
+            @accounts << IniFileCredentials.new(@options[:inifile]) unless (@options[:inifile].nil? or @options[:inifile].empty?)
+          end
+
+          if @options[:profile]
+            if @options[:profiles]
+              @options[:profiles] << @options[:profile]
+            else
+              @options[:profiles] = [@options[:profile]]
+            end
+          end
+          if @options[:profiles]
+            @options[:profiles].each {| profile |
+              @accounts << ProfileCredentials.new(profile)
+            }
+          end
+          unless @accounts.size > 0
+            parse_options
+            logger.fatal 'No options allow for account identification'
+          end
+          # Make the options mutable
+          unless @options.is_a?(Hash) or @options.is_a?(Hashie::Mash)
+            @options = Hashie::Mash.new(@options.to_hash)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cicd/builder/environments-list/mixlib/lib/update_bucket_policy.rb

@@ -0,0 +1,452 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'awesome_print'
+require 'inifile'
+require 'rubygems'
+require 'aws-sdk-core'
+require 'json'
+
+# =====================================================================================================================
+# A little include path tom-foolery ...
+path = File.dirname(__FILE__)
+# Borrowing from "whiches" gem ...
+cmd  = File.basename(__FILE__, '.rb')
+exes = []
+exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+ENV['PATH'].split(File::PATH_SEPARATOR).each do |pth|
+  exts.each { |ext|
+    exe = File.join(pth, "#{cmd}#{ext}")
+    exes << exe if File.executable? exe
+  }
+end
+if exes.size > 0
+  path = File.dirname(exes[0])
+end
+
+# add_path = File.expand_path(File.join(path, '../../lib'))
+# $:.unshift(add_path)
+# add_path = File.expand_path(File.join(path, 'lib'))
+add_path = path
+$:.unshift(add_path)
+
+# =====================================================================================================================
+class UpdateBucketPolicy < Thor
+  KNOWN = %w(38.117.159.162/32 70.151.98.131/32 )
+  class_option :verbose,      :type => :boolean
+  class_option :debug,        :type => :boolean
+  class_option :log_level,    :type => :string, :banner => 'Log level ([:trace, :debug, :info, :step, :warn, :error, :fatal, :todo])', :default => :step
+  class_option :bucket,       :type => :string, :default => 'wgen-sto-artifacts'
+  class_option :bucketini,    :type => :string
+  class_option :bucketprofile,:type => :string
+  class_option :basepolicy,   :type => :string
+  class_option :policy,       :type => :string
+  class_option :iniglob,      :type => :string
+  class_option :inifile,      :type => :string
+  class_option :profiles,     :type => :array
+  class_option :color,        :type => :boolean, :default => false
+  class_option :yes,          :type => :boolean
+  attr_reader   :accounts, :environments
+  attr_accessor :logger
+
+  no_commands do
+
+    require 'mixins/no_commands'
+    include Amplify::AWS::MixIns::NoCommands
+
+    def get_environments
+      logger.step 'Get environments ...'
+
+      @environments = {}
+      @nats         = {}
+      @accounts.each do |acc|
+        @options.merge!(acc.get_options())
+        parse_options
+        get_environments_for_account
+      end
+      logger.info "Environments:\n"+@environments.ai
+      logger.info "NATs:\n"+@nats.ai
+
+    end
+
+    def get_environments_for_account
+      logger.step "Get environments for #{@options[:account]} ..."
+
+      config = @options
+
+      # noinspection RubyArgCount
+      cfn = Aws::CloudFormation::Client.new(retry_limit: 10)
+
+      stacks = []
+      resp = cfn.list_stacks()
+
+      stacks << resp[:stack_summaries]
+      while resp[:next_token]
+        resp = cfn.list_stacks(next_token: resp[:next_token])
+        logger.debug resp.size
+        stacks << resp[:stack_summaries]
+      end
+      stacks.flatten!
+      stacks = stacks.select{ |stack| stack[:stack_status].match(%r'^(UPDATE|CREATE).*?_COMPLETE$')}
+      logger.debug stacks.ai
+
+      stacks.each do |stack|
+        env = nil
+        resp = cfn.describe_stacks(stack_name: stack[:stack_id])
+        stck = resp[:stacks].shift
+        tags = stck[:tags]
+        tags.each do |tag|
+          if tag[:key] == 'EnvironmentName'
+            env = tag[:value]
+            break
+          end
+        end
+        unless env
+          env = stack[:stack_name]
+        end
+        @environments[env] ||= []
+        @environments[env] << stack[:stack_name]
+
+        resources = []
+        resp = cfn.describe_stack_resources(stack_name: stack[:stack_id], logical_resource_id: 'NATIPAddress')
+        resources << resp[:stack_resources]
+        resp = cfn.describe_stack_resources(stack_name: stack[:stack_id], logical_resource_id: 'BastionIPAddress')
+        resources << resp[:stack_resources]
+        resources.flatten!
+        logger.debug resources.ai
+
+        _nats = resources.map{ |r|
+          r.to_h[:physical_resource_id]
+        }
+
+        @nats[env] = @nats[env] ? [ @nats[env], _nats ].flatten : _nats
+      end
+
+    end
+
+    def get_eips_for_account
+      logger.step "Get EIPs for #{@options[:inifile]} ..."
+
+      # noinspection RubyArgCount
+      ec2 = Aws::EC2::Client.new(retry_limit: 10)
+
+      resp = ec2.describe_addresses()
+
+      @eips << resp[:addresses].map{ |a| a[:public_ip] }
+      @eips.flatten!
+      logger.debug @eips.ai
+
+    end
+
+    def get_eips
+      logger.step 'Get EIPs ...'
+
+      @eips         = []
+      @accounts.each do |acc|
+        @options.merge!(acc.get_options())
+        parse_options
+        get_eips_for_account
+      end
+      @eips.uniq!
+      logger.info "EIPs:\n"+@eips.sort_by { |ip| ip.split(%r'[\./]').map{ |octet| octet.to_i} }.ai
+
+    end
+
+    def read_bucket_policy
+
+      parse_bucket_options(__method__, [ :policy, [:bucketini, :bucketprofile], :bucket ])
+
+      logger.step "Read the current policy for #{@options[:bucket]} ..."
+
+      # noinspection RubyArgCount
+      s3 = Aws::S3::Client.new()
+
+      resp = s3.get_bucket_policy(bucket: options[:bucket])
+      logger.debug resp.size
+      @policies = []
+      resp.each do |item|
+        json = item.policy.read
+        @policies << JSON.load(json)
+
+        logger.debug JSON.pretty_generate(@policies[-1], { indent: "\t", space: ' '})
+      end
+
+      logger.debug @policies.ai
+
+      json = ''
+      @policies.each do |pol|
+        json += JSON.pretty_generate(pol, { indent: "\t", space: ' '})
+      end
+      IO.write("#{options[:policy]}", json)
+
+    end
+
+    def inspect_bucket_policy
+
+      parse_bucket_options(__method__, [ :policy, [:bucketini, :bucketprofile], :bucket ])
+
+      logger.step "Inspect the policy for #{@options[:bucket]} ..."
+
+      config = @options.dup
+      tempfile =
+      @options[:policy] = Tempfile.new('policy').path+'.json'
+      read_bucket_policy()
+      @options = config
+
+      get_eips()
+
+      get_environments()
+
+      ips = [ @nats.values, @eips ].flatten.map{ |ip| "#{ip}/32"}
+      ips = [ ips, KNOWN ].flatten.uniq.select{ |ip| not ip.nil? }
+      ips = ips.sort_by { |ip| ip.split(%r'[\./]').map{ |octet| octet.to_i} }.uniq
+      logger.info "Valid IPs: \n"+ips.ai
+
+      json = ''
+      @policies.each do |pol|
+        unknown = 0
+        missing = 0
+        unadded = 0
+        if pol['Statement']
+          pol['Statement'].each do |smt|
+            logger.debug smt.ai
+            if smt['Condition'] and smt['Condition']['IpAddress'] and smt['Condition']['IpAddress']['aws:SourceIp']
+              smt['Condition']['IpAddress']['aws:SourceIp'].sort_by { |ip| ip.split(%r'[\./]').map{ |octet| octet.to_i} }.each do |entry|
+                unless ips.include?(entry)
+                  logger.warn "#{entry} unknown"
+                  unknown += 1
+                end
+              end
+              ips.each do |entry|
+                unless smt['Condition']['IpAddress']['aws:SourceIp'].include?(entry)
+                  logger.info "#{entry} missing"
+                  missing += 1
+                  smt['Condition']['IpAddress']['aws:SourceIp'] << entry
+                end
+              end
+              if missing > 0
+                ips.each do |entry|
+                  unless smt['Condition']['IpAddress']['aws:SourceIp'].include?(entry)
+                    logger.warn "#{entry} missing"
+                    unadded += 1
+                  end
+                end
+              end
+              smt['Condition']['IpAddress']['aws:SourceIp'] = smt['Condition']['IpAddress']['aws:SourceIp'].sort_by { |ip| ip.split(%r'[\./]').map{ |octet| octet.to_i} }
+            end
+          end
+        else
+          logger.warn "Policy has no Statement: #{pol.ai}"
+        end
+        if missing+unknown > 0
+          logger.warn "Policy needs to be updated for #{unknown} unknown IPs, #{missing} missing and #{unadded} IPs which failed to add"
+        end
+        # require 'date_time'
+        logger.debug JSON.pretty_generate(pol, { indent: "\t", space: ' '})
+        pol['Id'] = "Policy-#{DateTime.now.strftime('%Y%m%dT%H%M%S')}"
+        json += JSON.pretty_generate(pol, { indent: "\t", space: ' '})
+      end
+      IO.write(options[:policy], json)
+
+    end
+
+    def update_bucket_policy
+
+      parse_bucket_options(__method__, [ :policy, [:bucketini, :bucketprofile], :bucket ])
+
+      logger.step "Update the current policy for #{@options[:bucket]} ..."
+
+      inspect_bucket_policy
+
+      write_bucket_policy
+
+    end
+
+    def write_bucket_policy
+
+      parse_bucket_options(__method__, [ :policy, [:bucketini, :bucketprofile], :bucket ])
+
+      logger.step "Write the new policy for #{@options[:bucket]} ..."
+
+      @options[:inifile] = @options[:bucketini]
+      parse_options
+      @logger.info options.ai
+
+      json = IO.read(options[:policy])
+
+      # noinspection RubyArgCount
+      s3 = Aws::S3::Client.new()
+
+      resp = s3.put_bucket_policy(
+                                    bucket: options[:bucket],
+                                    policy: json
+                                  )
+      logger.debug resp.ai
+      if resp.error
+        logger.error resp.error
+      end
+    end
+
+    def parse_bucket_options(method,optionset)
+      check_missing_options(method, optionset)
+
+      if @options[:bucketini]
+        @options[:inifile] = @options[:bucketini]
+      else
+        @options[:profile] = @options[:bucketprofile]
+      end
+      parse_options
+      @logger.info options.ai
+    end
+  end
+
+  def initialize(args = [], local_options = {}, config = {})
+    super(args,local_options,config)
+  end
+
+  desc 'read', 'read the policy for BUCKET'
+  def read()
+    prepare_accounts
+
+    read_bucket_policy
+
+    n = 1
+    @policies.each do |pol|
+      logger.step "#{n+=1}\n"+JSON.pretty_generate(pol, { indent: "\t", space: ' '})
+    end
+
+    0
+  end
+
+  desc 'inspect', 'inspect the policy for BUCKET'
+  def inspect()
+    prepare_accounts()
+
+    inspect_bucket_policy()
+
+    @policies.each do |pol|
+      logger.debug JSON.pretty_generate(pol, { indent: "\t", space: ' '})
+    end
+    0
+  end
+
+  desc 'update', 'update the policy for BUCKET'
+  def update()
+    prepare_accounts()
+
+    update_bucket_policy()
+
+    0
+  end
+
+  desc 'write', 'write the new POLICY to BUCKET'
+  def write()
+    prepare_accounts
+    @options[:inifile] = @options[:bucketini]
+    parse_options
+    @logger.info options.ai
+
+    write_bucket_policy
+
+    0
+  end
+
+end
+
+# =====================================================================================================================
+# UpdateBucketPolicy.start(ARGV)
+
+
+__END__
+					"aws:SourceIp": [
+						"54.84.38.255/32",
+						"54.86.79.230/32",
+						"54.84.206.23/32",
+						"54.172.63.6/32",
+						"54.86.33.222/32",
+						"38.117.159.162/32",
+						"70.151.98.131/32",
+						"54.208.252.192/32",
+						"54.86.6.21/32",
+						"54.86.156.100/32",
+						"54.209.155.115/32",
+						"107.23.90.79/32",
+						"54.172.109.59/32",
+						"54.209.93.210/32",
+						"54.88.205.140/32",
+						"54.209.196.100/32",
+						"54.84.6.106/32",
+						"54.165.122.109/32",
+						"107.21.36.141/32",
+						"54.86.150.196/32",
+						"54.209.17.121/32",
+						"54.84.31.228/32",
+						"54.88.2.227/32",
+						"54.85.80.166/32",
+						"54.86.31.209/32",
+						"50.19.114.108/32",
+						"54.85.159.92/32",
+						"54.85.129.109/32",
+						"54.165.86.129/32",
+						"54.165.114.103/32",
+						"54.172.8.125/32",
+						"54.86.144.175/32",
+						"107.23.57.237/32",
+						"54.165.231.248/32",
+						"54.165.228.106/32"
+					]
+
+{
+	"Version": "2008-10-17",
+	"Id": "Policy13947375644752",
+	"Statement": [
+		{
+			"Sid": "Stmt13947375624982",
+			"Effect": "Allow",
+			"Principal": {
+				"AWS": "*"
+			},
+			"Action": "s3:*",
+			"Resource": "arn:aws:s3:::wgen-sto-artifacts/*",
+			"Condition": {
+				"IpAddress": {
+					"aws:SourceIp": [
+						"54.86.79.230/32",
+						"54.84.206.23/32",
+						"54.86.33.222/32",
+						"38.117.159.162/32",
+						"70.151.98.131/32",
+						"54.209.155.115/32",
+						"107.23.90.79/32",
+						"54.209.93.210/32",
+						"54.88.205.140/32",
+						"54.84.6.106/32",
+						"107.21.36.141/32",
+						"54.86.150.196/32",
+						"54.209.17.121/32",
+						"54.84.31.228/32",
+						"54.86.31.209/32",
+						"50.19.114.108/32",
+						"54.85.129.109/32",
+						"54.165.86.129/32",
+						"54.165.114.103/32",
+						"54.172.8.125/32",
+						"54.86.144.175/32",
+						"107.23.57.237/32",
+						"54.165.231.248/32",
+						"54.165.228.106/32"
+					]
+				}
+			}
+		},
+		{
+			"Sid": "2",
+			"Effect": "Allow",
+			"Principal": {
+				"AWS": "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity EXU2IEU8NKUSP"
+			},
+			"Action": "s3:GetObject",
+			"Resource": "arn:aws:s3:::wgen-sto-artifacts/*"
+		}
+	]
+}