aws_pocketknife 0.1.7

data/lib/aws_pocketknife/cli/elb.rb

@@ -0,0 +1,20 @@
+require "thor"
+require "aws_pocketknife"
+
+module AwsPocketknife
+  module Cli
+    class Elb < Thor
+
+      desc "desc ELB_NAME", "describe elastic load balancer"
+      def desc(elb_name)
+        elb = AwsPocketknife::Elb.describe_elb_by_name(name: elb_name)
+        if elb.nil?
+          puts "ELB #{elb_name} not found"
+        else
+          AwsPocketknife::Ec2.nice_print(object: elb.to_h)
+        end
+      end
+
+    end
+  end
+end

data/lib/aws_pocketknife/cli/iam.rb

@@ -0,0 +1,31 @@
+require "thor"
+require "aws_pocketknife"
+
+module AwsPocketknife
+  module Cli
+    class Iam < Thor
+
+      desc "list_ssl_certs", "list ssl certs"
+      def list_ssl_certs
+        certs = AwsPocketknife::Iam.list_ssl_certificates
+        AwsPocketknife::Iam.nice_print(object: certs.to_h)
+      end
+
+      desc "create_user USERNAME", "create user"
+      def create_user(username)
+        AwsPocketknife::Iam.create_iam_user username
+      end
+
+      desc "create_group GROUP_NAME", "create group"
+      def create_group(group_name)
+        AwsPocketknife::Iam.create_group group_name
+      end
+
+      desc "add_user_to_group USERNAME GROUP_NAME", "add user to group"
+      def add_user_to_group(username, group_name)
+        AwsPocketknife::Iam.add_user_to_group username, group_name
+      end
+
+    end
+  end
+end

data/lib/aws_pocketknife/cli/main.rb

@@ -0,0 +1,34 @@
+require "thor"
+require "aws_pocketknife"
+
+module AwsPocketknife
+  module Cli
+    class Main < Thor
+
+      desc "ec2 SUBCOMMAND ...ARGS", "ec2 command lines"
+      subcommand "ec2", AwsPocketknife::Cli::Ec2
+
+      desc "ami SUBCOMMAND ...ARGS", "ami command lines"
+      subcommand "ami", AwsPocketknife::Cli::Ami
+
+      desc "eb SUBCOMMAND ...ARGS", "elastic beanstalk command lines"
+      subcommand "eb", AwsPocketknife::Cli::Eb
+
+      desc "route53 SUBCOMMAND ...ARGS", "route53 command lines"
+      subcommand "route53", AwsPocketknife::Cli::Route53
+
+      desc "iam SUBCOMMAND ...ARGS", "iam command lines"
+      subcommand "iam", AwsPocketknife::Cli::Iam
+
+      desc "rds SUBCOMMAND ...ARGS", "rds command lines"
+      subcommand "rds", AwsPocketknife::Cli::Rds
+
+      desc "asg SUBCOMMAND ...ARGS", "asg command lines"
+      subcommand "asg", AwsPocketknife::Cli::Asg
+
+      desc "elb SUBCOMMAND ...ARGS", "elb command lines"
+      subcommand "elb", AwsPocketknife::Cli::Elb
+
+    end
+  end
+end

data/lib/aws_pocketknife/cli/rds.rb

@@ -0,0 +1,13 @@
+require "thor"
+require "aws_pocketknife"
+
+module AwsPocketknife
+  module Cli
+    class Rds < Thor
+
+      desc "snapshot SUBCOMMAND ...ARGS", "snapshot command lines"
+      subcommand "snapshot", AwsPocketknife::Cli::RdsSnapshot
+
+    end
+  end
+end

data/lib/aws_pocketknife/cli/rds_snapshot.rb

@@ -0,0 +1,44 @@
+require "thor"
+require "aws_pocketknife"
+
+module AwsPocketknife
+  module Cli
+    class RdsSnapshot < Thor
+
+      desc "list_snapshots DB_NAME", "list snapshots"
+      def list(db_name)
+        snapshots = AwsPocketknife::Rds.describe_snapshots(db_name: db_name)
+        headers = [ 'Name', 'Creation Time', 'Snapshot Type', 'Status','Port', 'Engine', 'Version', 'Storage (Gb)', 'IOPS']
+        data = []
+        snapshots.each do |h|
+          data << [h.db_snapshot_identifier,
+                   h.snapshot_create_time,
+                   h.snapshot_type,
+                   h.status,
+                   h.port,
+                   h.engine,
+                   h.engine_version,
+                   h.allocated_storage,
+                   h.iops
+          ]
+        end
+        AwsPocketknife::Rds.pretty_table(headers: headers, data: data)
+      end
+
+      desc "clean DB_NAME DAYS --dry_run", "Remove manual snapshots with creation time lower than DAYS for database_name."
+      option :dry_run, :type => :boolean, :default => true, :desc => 'just show images that would be deleted'
+      def clean(db_name, days)
+        dry_run = options.fetch("dry_run", true)
+        AwsPocketknife::Rds.clean_snapshots db_name: db_name,
+                                            days: days,
+                                            dry_run: dry_run
+      end
+
+      desc "create DB_NAME", "Creates a snapshot for database_name."
+      def create(db_name)
+        AwsPocketknife::Rds.create_snapshot db_name: db_name
+      end
+
+    end
+  end
+end

data/lib/aws_pocketknife/cli/route53.rb

@@ -0,0 +1,56 @@
+require "thor"
+require "aws_pocketknife"
+
+module AwsPocketknife
+  module Cli
+    class Route53 < Thor
+
+      desc "describe_hosted_zone HOSTED_ZONE", "describe hosted zone"
+      def describe_hosted_zone(hosted_zone)
+        hosted_zone = AwsPocketknife::Route53.describe_hosted_zone(hosted_zone: hosted_zone)
+        unless hosted_zone.nil?
+          AwsPocketknife::Route53.nice_print(object: hosted_zone.to_h)
+        else
+          puts "#{hosted_zone} not found"
+        end
+      end
+
+      desc "list", "list hosted zones"
+      def list
+        hosted_zones = AwsPocketknife::Route53.list_hosted_zones
+        headers = [ 'Name', 'Zone ID', 'Comment']
+        data = []
+        hosted_zones.each do |h|
+          data << [h.name,
+                   AwsPocketknife::Route53.get_hosted_zone_id(hosted_zone: h.id),
+                   h.config.comment]
+        end
+        AwsPocketknife::Route53.pretty_table(headers: headers, data: data)
+      end
+
+      desc "list_records HOSTED_ZONE", "list records for hosted zone"
+      def list_records(hosted_zone)
+        records = AwsPocketknife::Route53.list_records_for_zone_name(hosted_zone_name: hosted_zone)
+        headers = ["Name", "Type", "DNS Name"]
+        data = []
+        if records.length > 0
+          records.each do |record|
+            if record.type == 'CNAME'
+              data << [record.name, record.type, record.resource_records[0].value]
+            else
+              if record.alias_target.nil?
+                data << [record.name, record.type, "N/A"]
+              else
+                data << [record.name, record.type, record.alias_target.dns_name]
+              end
+            end
+          end
+          AwsPocketknife::Route53.pretty_table(headers: headers, data: data)
+        else
+          puts "No records found hosted zone #{hosted_zone}"
+        end
+      end
+
+    end
+  end
+end

data/lib/aws_pocketknife/cloudwatch_logs.rb

@@ -0,0 +1,25 @@
+require 'aws_pocketknife'
+require 'base64'
+require 'openssl'
+require 'recursive-open-struct'
+
+module AwsPocketknife
+  module CloudwatchLogs
+
+    class << self
+      include AwsPocketknife::Common::Utils
+
+      def create_log_group(log_group_name: "")
+
+        unless log_group_name.empty?
+          cloudwatch_logs_client.create_log_group({
+               log_group_name: log_group_name, # required
+           })
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/aws_pocketknife/common/logging.rb

@@ -0,0 +1,31 @@
+require 'log4r'
+
+module AwsPocketknife
+  module Common
+    module Logging
+
+      include Log4r
+      Logger = Log4r::Logger
+      Log4r::Logger.root.level = Log4r::INFO
+
+      class << self
+
+        def logger
+          @log ||= initialize_log
+        end
+
+        def initialize_log(name: "aws_pocketknife", pattern: "[%l] %d %m")
+          log = Logger.new(name)
+
+          log_format = Log4r::PatternFormatter.new(:pattern => pattern)
+          log_output = Log4r::StdoutOutputter.new 'console'
+          log_output.formatter = log_format
+          log.add(log_output)
+
+          return log
+        end
+
+      end
+    end
+  end
+end

data/lib/aws_pocketknife/common/utils.rb

@@ -0,0 +1,63 @@
+require "pretty_table"
+require "awesome_print"
+require_relative "logging"
+
+module AwsPocketknife
+  module Common
+    module Utils
+      #include AwsPocketknife::Common::Logging
+
+      def ec2_client
+        @ec2_client ||= AwsPocketknife.ec2_client
+      end
+
+      def iam_client
+        @iam_client ||= AwsPocketknife.iam_client
+      end
+
+      def route53_client
+        @route53_client ||= AwsPocketknife.route53_client
+      end
+
+      def rds_client
+        @rds_client ||= AwsPocketknife.rds_client
+      end
+
+      def elb_client
+        @elb_client ||= AwsPocketknife.elb_client
+      end
+
+      def asg_client
+        @asg_client ||= AwsPocketknife.asg_client
+      end
+
+      def cloudwatch_logs_client
+        @cloudwatch_logs_client ||= AwsPocketknife.cloudwatch_logs_client
+      end
+
+      def elastic_beanstalk_client
+        @elastic_beanstalk_client ||= AwsPocketknife.elastic_beanstalk_client
+      end
+
+      def pretty_table(headers: [], data: [])
+        puts PrettyTable.new(data, headers).to_s
+      end
+
+      # https://github.com/michaeldv/awesome_print
+      def nice_print(object: nil)
+        ap object
+      end
+
+      def get_tag_value(tags: [], tag_key: "")
+        unless tags.empty? or tag_key.length == 0
+          tag =  tags.select { |tag| tag.key == tag_key }
+          return tag[0].value if tag.length == 1
+          return "" if tag.length == 0
+        else
+          return ""
+        end
+      end
+
+    end
+  end
+end

data/lib/aws_pocketknife/ec2.rb

@@ -0,0 +1,308 @@
+require 'aws_pocketknife'
+require 'base64'
+require 'openssl'
+require 'retryable'
+require 'recursive-open-struct'
+
+module AwsPocketknife
+  module Ec2
+
+    MAX_ATTEMPTS = 15
+    DELAY_SECONDS = 10
+
+    STATE_PENDING = 'pending'
+    STATE_AVAILABLE = 'available'
+    STATE_DEREGISTERED = 'deregistered'
+    STATE_INVALID = 'invalid'
+    STATE_FAILED = 'failed'
+    STATE_ERROR = 'error'
+
+    class << self
+      include AwsPocketknife::Common::Utils
+      #include AwsPocketknife::Common::Logging
+
+      Logging = Common::Logging.logger
+
+      def find_ami_by_name(name: '')
+        ec2_client.describe_images({dry_run: false,
+                                    filters: [
+                                        {
+                                            name: "tag:Name",
+                                            values: [name]
+                                        }
+                                    ]}).images
+      end
+
+      def find_ami_by_id(id: '')
+        ec2_client.describe_images({dry_run: false,
+                                    image_ids: [id]}).images.first
+      end
+
+      def delete_ami_by_id(id: '')
+        image = find_ami_by_id(id: id)
+        snapshot_ids = snapshot_ids(image)
+        ec2_client.deregister_image(image_id: id)
+
+        Retryable.retryable(:tries => 20, :sleep => lambda { |n| 2**n }, :on => StandardError) do |retries, exception|
+          image = find_ami_by_id(id: id)
+          message =  "retry #{retries} - Deleting image #{id}"
+          message << " State: #{image.state}" if image
+          Logging.info message
+          raise StandardError unless image.nil?
+        end
+
+        delete_snapshots(snapshot_ids: snapshot_ids)
+      end
+
+      def delete_snapshots(snapshot_ids: [])
+        snapshot_ids.each do |snapshot_id|
+          Logging.info "Deleting Snapshot: #{snapshot_id}"
+          ec2_client.delete_snapshot(snapshot_id: snapshot_id)
+        end
+      end
+
+      def snapshot_ids(image)
+        snapshot_ids = []
+        image.block_device_mappings.each do |device_mapping|
+          ebs = device_mapping.ebs
+          snapshot_ids << ebs.snapshot_id if ebs && !ebs.snapshot_id.to_s.empty?
+        end
+        snapshot_ids
+      end
+
+      def clean_ami(options)
+        Logging.info "options: #{options}"
+
+        dry_run = options.fetch(:dry_run, true)
+        image_ids = find_ami_by_creation_time(options)
+        images_to_delete = find_unused_ami(image_ids: image_ids)
+
+        Logging.info "images (#{image_ids.length}): #{image_ids}"
+        Logging.info "images to delete (#{images_to_delete.length}): #{images_to_delete}"
+
+        unless dry_run
+          images_to_delete.each do |image_id|
+            Logging.info "deleting image #{image_id}"
+            delete_ami_by_id(id: image_id)
+          end
+        end
+
+      end
+
+      def find_unused_ami(image_ids: [])
+        images_to_delete = []
+        image_ids.each do |image_id|
+          # check if there is any instance using the image id
+          instances = describe_instances_by_image_id(image_id_list: [image_id])
+          if instances.empty?
+            images_to_delete << image_id
+          else
+            Logging.info "#{image_id} is used by instance #{instances.map { |instance| instance.instance_id }}"
+          end
+          Kernel.sleep 2
+        end
+        return images_to_delete
+      end
+
+      def find_ami_by_creation_time(options)
+
+        days = options.fetch(:days, '30').to_i * 24 * 3600
+        creation_time = Time.now-days
+        Logging.info "Cleaning up images older than #{days} days, i.e, with creation_time < #{creation_time})"
+
+        image_ids = []
+        images = find_ami_by_name(name: options.fetch(:ami_name_pattern, ''))
+        images.each do |image|
+          image_creation_time = Time.parse(image.creation_date)
+          msg = "image #{image.image_id} (#{creation_time}) < (image_creation_time: #{image_creation_time})? "
+          if creation_time <= image_creation_time
+            image_ids << image.image_id
+            msg << "YES, marking to be deleted"
+          else
+            msg << "NO"
+          end
+          Logging.info msg
+        end
+        return image_ids
+      end
+
+      def share_ami(image_id: '', user_id: '', options: {})
+        begin
+          options = {}
+          options[:image_id] = image_id
+          options[:launch_permission] = create_launch_permission(user_id)
+          Logging.info "Sharing Image #{image_id} with #{user_id} with options #{options}"
+          response = @ec2_client.modify_image_attribute(options=options)
+          return response
+        rescue Exception => e
+          Logging.error "## Got an error when sharing the image... #{e.cause} -> #{e.message}"
+          raise
+        end
+      end
+
+
+      def create_image(instance_id: "", name: "", description: "Created at #{Time.now}",
+                       timeout: 1800, publish_to_account: "",
+                       volume_type: "gp2",
+                       iops: 3,
+                       encrypted: false,
+                       volume_size: 60
+      )
+
+        begin
+          Logging.info "creating image"
+          instance = find_by_id(instance_id: instance_id)
+          instance = ec2.instances[instance_id]
+          image = instance.create_image(name, :description => description)
+          sleep 2 until image.exists?
+          Logging.info "image #{image.id} state: #{image.state}"
+          sleep 10 until image.state != :pending
+          if image.state == :failed
+            raise "Create image failed"
+          end
+          Logging.info "image created"
+        rescue => e
+          Logging.error "Creating AMI failed #{e.message}"
+          Logging.error e.backtrace.join("\n")
+          raise e
+        end
+        if publish_to_account.length != 0
+          Logging.info "add permissions for #{publish_to_account}"
+          image.permissions.add(publish_to_account.gsub(/-/, ''))
+        end
+        image.id.tap do |image_id|
+          Logging.info "Image #{@name}[#{image_id}] created"
+          return image_id
+        end
+      end
+
+      def stop_instance_by_id(instance_ids)
+        instance_id_list = get_instance_id_list(instance_ids: instance_ids)
+        Logging.info "Stoping instance id: #{instance_id_list}"
+        resp = ec2_client.stop_instances({ instance_ids: instance_id_list })
+        wait_till_instance_is_stopped(instance_id_list, max_attempts: MAX_ATTEMPTS, delay_seconds: DELAY_SECONDS)
+        Logging.info "Stopped ec2 instance #{instance_id_list}"
+      end
+
+      def start_instance_by_id(instance_ids)
+        instance_id_list = get_instance_id_list(instance_ids: instance_ids)
+        Logging.info "Start instance id: #{instance_id_list}"
+        ec2_client.start_instances({ instance_ids: instance_id_list })
+      end
+
+      # http://serverfault.com/questions/560337/search-ec2-instance-by-its-name-from-aws-command-line-tool
+      def find_by_name(name: "")
+        instances = []
+        resp = ec2_client.describe_instances({dry_run: false,
+                                              filters: [
+                                                  {
+                                                      name: "tag:Name",
+                                                      values: [name]
+                                                  }
+                                              ]})
+        resp.reservations.each do |reservation|
+          reservation.instances.each do |instance|
+            instances << instance
+          end
+        end
+        instances
+      end
+
+      def describe_instances_by_image_id(image_id_list: [])
+        instances = []
+        resp = ec2_client.describe_instances({dry_run: false,
+                                               filters: [
+                                                   {
+                                                       name: "image-id",
+                                                       values: image_id_list
+                                                   }
+                                               ]})
+        resp.reservations.each do |reservation|
+          reservation.instances.each do |instance|
+            instances << instance
+          end
+        end
+        instances
+      end
+
+      def find_by_id(instance_id: "")
+        resp = ec2_client.describe_instances({dry_run: false, instance_ids: [instance_id.to_s]})
+        if resp.nil? or resp.reservations.length == 0 or resp.reservations[0].instances.length == 0
+          return nil
+        else
+          return resp.reservations.first.instances.first
+        end
+      end
+
+      def get_windows_password(instance_id: "")
+
+        private_keyfile_dir = ENV["AWS_POCKETKNIFE_KEYFILE_DIR"] || ""
+        raise "Environment variable AWS_POCKETKNIFE_KEYFILE_DIR is not defined" if private_keyfile_dir.length == 0
+
+        instance = find_by_id(instance_id: instance_id)
+        key_name = instance.key_name
+        private_keyfile = File.join(private_keyfile_dir, "#{key_name}.pem")
+        raise "File #{private_keyfile} not found" unless File.exist?(private_keyfile)
+
+        resp = ec2_client.get_password_data({dry_run: false,
+                                              instance_id: instance_id})
+        encrypted_password = resp.password_data
+        decrypted_password = decrypt_windows_password(encrypted_password, private_keyfile)
+
+        RecursiveOpenStruct.new({password: decrypted_password,
+                                instance_id: instance.instance_id,
+                                 private_ip_address: instance.private_ip_address,
+                                 public_ip_address: instance.public_ip_address}, recurse_over_arrays: true)
+      end
+
+      # def ec2
+      #   @ec2 ||= Aws::EC2.new(:ec2_endpoint => "ec2.#{AwsPocketknife::AWS_REGION}.amazonaws.com")
+      # end
+
+      private
+
+      def create_launch_permission(user_id)
+        {
+            add: [
+                {
+                    user_id: user_id
+                },
+            ]
+        }
+      end
+
+      # Decrypts an encrypted password using a provided RSA
+      # private key file (PEM-format).
+      def decrypt_windows_password(encrypted_password, private_keyfile)
+        encrypted_password_bytes = Base64.decode64(encrypted_password)
+        private_keydata = File.open(private_keyfile, "r").read
+        private_key = OpenSSL::PKey::RSA.new(private_keydata)
+        private_key.private_decrypt(encrypted_password_bytes)
+      end
+
+      def get_instance_id_list(instance_ids: "")
+        instance_ids.strip.split(";")
+      end
+
+      def wait_till_instance_is_stopped(instance_ids, max_attempts: 12, delay_seconds: 10)
+        total_wait_seconds = max_attempts * delay_seconds;
+        Logging.info "Waiting up to #{total_wait_seconds} seconds with #{delay_seconds} seconds delay for ec2 instance #{instance_ids} to be stopped"
+        ec2_client.wait_until(:instance_stopped, { instance_ids: instance_ids }) do |w|
+          w.max_attempts = max_attempts
+          w.delay = delay_seconds
+        end
+      end
+
+      def wait_till_instance_is_terminated(instance_ids, max_attempts: 12, delay_seconds: 10)
+        total_wait_seconds = max_attempts * delay_seconds;
+        Logging.info "Waiting up to #{total_wait_seconds} seconds with #{delay_seconds} seconds delay for ec2 instance #{instance_ids} to be terminated"
+        ec2_client.wait_until(:instance_terminated, { instance_ids: instance_ids }) do |w|
+          w.max_attempts = max_attempts
+          w.delay = delay_seconds
+        end
+      end
+
+    end
+
+  end
+end