awsutils 2.0.3 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f36e779b5b931a45339568e6ca14938acf904f7
-  data.tar.gz: 2569db86afaf403fee076dc126f421c6fef32a46
+  metadata.gz: 5cd2972b98d1b03ecb9704dc58b10836697d9d9e
+  data.tar.gz: 1072bcfb550967e616027ecc9ebad509693c88de
 SHA512:
-  metadata.gz: 9bfdc90b264970ab5023277cc8483a38655e7d717a1f08dbf8cff787b0cd3dd4b9383a6ed6365352835929740f8458aa26271ea3f1cd098531fb678235d5e98f
-  data.tar.gz: 15ff775c7f3efae54eae92353a36b2d2a4c28d04486d8ef7be1c816b87ce7d659e935c698217d1697226f06355ccdd3c02a66e2b872e96a2b79f0eb537885a7f
+  metadata.gz: b58da8539855b7a01148655cab7fefad6d4d751bc066682aa85564267e82507f330c086c5033665c2a5ba888c968c117b1ebc9f28a85eb786b029cede28644ce
+  data.tar.gz: c2f60425d9c4f43c888850857604402b799e144d2e7eb163943b6a83b413e664c1498ca31ce713a60f74ac35db5b184142e2cbcfd4aeb29a070f0815fe50cece

data/README.md

@@ -18,6 +18,10 @@ Or install it yourself as:
 
 ## Usage
 
+awslogs
+-------
+Dumps logs from CloudWatch Logs for easy grepping (see `awslogs --help` for details).
+
 ec2listmachines
 ---------------
 Show a list of all EC2 instances in your account.

data/awsutils.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'facets', '~> 2.9'
   spec.add_dependency 'rainbow', '~> 2.0'
-  spec.add_dependency 'fog-aws', '~> 0.9.1'
+  spec.add_dependency 'fog-aws', '~> 0.10.0'
   spec.add_dependency 'trollop'
+  spec.add_dependency 'aws-sdk'
 end

data/bin/awslogs

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Eric.
+#
+# The application 'awslogs' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+# require 'rubygems'
+require 'awsutils/version'
+require 'awsutils/awslogs'
+
+awslogs = AwsUtils::AwsLogs.new
+awslogs.print_events

data/lib/awsutils/awslogs.rb

@@ -0,0 +1,109 @@
+require 'trollop'
+require 'aws-sdk'
+require 'time'
+
+module AwsUtils
+  class AwsLogs
+    LOG_LEVELS = %w(TRACE DEBUG INFO NOTICE WARNING ERROR FATAL).freeze
+
+    def opts
+      @opts ||= Trollop.options do
+        opt :group,
+            'Log group (e.g. /aws/lambda/sf-updater)',
+            required: true,
+            type: :string,
+            short: 'g'
+        opt :filter_pattern,
+            'See: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/FilterAndPatternSyntax.html',
+            short: 'f'
+        opt :streams_prefix,
+            'E.g. 2016/08',
+            default: Time.now.strftime('%Y/%m/%d'),
+            short: 's'
+        opt :timestamp,
+            'Include the timestamp from the event metadata in the output',
+            short: 't',
+            default: false
+        opt :show_request_id,
+            'Show the request ID in every log message',
+            short: 'r',
+            default: false
+        opt :log_level,
+            'Lowest log level to show',
+            default: 'INFO',
+            short: 'l'
+        opt :show_stream_name,
+            'Include the name of the log stream on the output line',
+            default: false
+      end
+    end
+
+    def chunk_events(streams_chunk, token = nil)
+      parameters = {
+        log_group_name: opts[:group],
+        log_stream_names: streams_chunk,
+        next_token: token
+      }
+      parameters[:filter_pattern] = opts[:filter_pattern] if opts[:filter_pattern]
+      response = cloudwatchlogs.filter_log_events parameters
+      collector = response.events
+      collector += chunk_events(streams_chunk, response.next_token) if response.next_token
+      collector
+    end
+
+    def log_events
+      # puts "Filtering from #{streams.count} streams in #{streams.count / 50 + 1} chunks"
+      collector = []
+      streams.each_slice(50) { |streams_chunk| collector += chunk_events(streams_chunk) }
+
+      puts 'No events found' if collector.empty?
+
+      collector.sort_by(&:timestamp)
+    end
+
+    def print_events
+      log_events.each do |ev|
+        if ev.message !~ /^\[(INFO|DEBUG|WARNING|ERROR|NOTICE)\]/ # Check if the message is in the standard format
+          print "#{ev.log_stream_name}: " if opts[:show_stream_name]
+          print Time.at(ev.timestamp / 1e3).iso8601(3) + ' ' if opts[:timestamp]
+          print ev.message
+          next
+        end
+
+        msg_parts = ev.message.split("\t")
+        level      = msg_parts[0].scan(/\[(\w*)\]/)[-1][0]
+        timestamp  = msg_parts[1]
+        request_id = msg_parts[2]
+        message    = msg_parts[3..-1].join("\t")
+
+        next unless show_logentry? level
+
+        print "#{ev.log_stream_name}: " if opts[:show_stream_name]
+        print Time.at(ev.timestamp / 1e3).iso8601(3) if opts[:timestamp]
+        printf('%-24s %-10s', timestamp, "[#{level}]")
+        printf('%-37s', request_id) if opts[:show_request_id]
+        print(message)
+      end
+    end
+
+    def show_logentry?(level)
+      return true unless LOG_LEVELS.include? level
+      LOG_LEVELS.index(level.upcase) >= LOG_LEVELS.index(opts[:log_level].upcase)
+    end
+
+    def streams(token = nil)
+      response = cloudwatchlogs.describe_log_streams(
+        log_group_name: opts[:group],
+        log_stream_name_prefix: opts[:streams_prefix],
+        next_token: token
+      )
+      collector = response.log_streams.map(&:log_stream_name)
+      collector += streams(response.next_token) if response.next_token
+      collector
+    end
+
+    def cloudwatchlogs
+      @cloudwatchlogs ||= Aws::CloudWatchLogs::Client.new
+    end
+  end
+end

data/lib/awsutils/ec2lsgrp.rb

@@ -40,17 +40,32 @@ module AwsUtils
     end
 
     def run
-      fail ArgumentError, 'Please specify a security group' unless search
+      raise ArgumentError, 'Please specify a security group' unless search
       unless group_o = group(search) # rubocop:disable Lint/AssignmentInCondition
-        fail GroupDoesNotExist
+        raise GroupDoesNotExist
       end
       return group_details(group_o) unless opts[:list_refs]
       refs = references(group_o.group_id)
       if refs.empty?
         puts 'No references'
       else
-        puts "References: #{refs.keys.join(', ')}"
-        puts refs.to_yaml if opts[:verbose]
+        print_refs refs
+      end
+    end
+
+    def print_refs(refs)
+      refs.each do |grp, data|
+        print "#{grp} (#{data['groupId']}): "
+        perm_strings = data['references'].map do |perm|
+          if perm['ipProtocol'] == '-1'
+            'all'
+          else
+            "#{perm['ipProtocol']}/#{perm['fromPort']}" +
+              (perm['fromPort'] != perm['toPort'] ? "-#{perm['toPort']}" : '')
+          end
+        end
+        print perm_strings.join ', '
+        puts ''
       end
     end
 
@@ -64,7 +79,7 @@ module AwsUtils
     def group_perm_string(group_perm)
       group_perm.map do |g|
         if g['userId'] == owner_id
-          "#{g['groupId']} (#{group(g['groupId']).name})"
+          "#{g['groupId']} (#{group(g['groupId']).group_name})"
         else
           "#{g['groupId']} (#{g['groupName']}, owner: #{g['userId']})"
         end

data/lib/awsutils/ec2sg.rb

@@ -1,46 +1,42 @@
-require 'rubygems'
 require 'fog/aws'
 
 module AwsUtils
   class Ec2SecurityGroup
     def connection
-      @connection ||= begin
-        options = {}
+      @connection ||= Fog::Compute::AWS.new
+    end
 
-        if ENV['AWS_ACCESS_KEY']
-          options = {
-            aws_access_key_id: ENV['AWS_ACCESS_KEY'],
-            aws_secret_access_key: ENV['AWS_SECRET_KEY']
+    def references(search_string)
+      search =
+        if search_string =~ /^sg-/
+          {
+            id: search_string,
+            name: groups.find { |gr| gr.group_id == search_string }.name
+          }
+        else
+          {
+            id: groups.find { |gr| gr.name == search_string }.group_id,
+            name: search_string
           }
         end
 
-        Fog::Compute::AWS.new options
-      end
-    end
-
-    def references(search)
-      if search =~ /^sg-/
-        search_id = search
-      else
-        search_id = groups.find { |g| g.name == search }.group_id
-      end
-
       groups.each_with_object({}) do |grp, m|
-        assoc_p = grp.ip_permissions.select do |ip_perm|
-          !ip_perm['groups'].select { |src_grp|
-            src_grp['groupName'] == search ||
-              src_grp['groupId'] == search_id
-          }.empty?
+        permission_references = grp.ip_permissions.select do |ip_perm|
+          ip_perm['groups'].find do |pair|
+            pair['groupId'] == search[:id] ||
+              pair['groupName'] == search[:name]
+          end
         end
-        if assoc_p.empty?
-          next
-        else
-          m[grp.name] = {
+
+        next if permission_references.empty?
+
+        m[grp.name] = { 'groupId' => grp.group_id }
+        m[grp.name]['references'] = permission_references.map do |pr|
+          {
             'groupId' => grp.group_id,
-            'ipPermissions' => assoc_p.map do |ap|
-              ap.delete('ipRanges')
-              ap
-            end
+            'ipProtocol' => pr['ipProtocol'],
+            'fromPort' => pr['fromPort'],
+            'toPort' => pr['toPort']
           }
         end
       end
@@ -57,7 +53,7 @@ module AwsUtils
         server.tags['Name'] ? server.tags['Name'] : server.id
       end.compact
 
-      return false unless servers_using_group.length > 0
+      return false unless servers_using_group.empty?
       print 'The following servers are still using this group: '
       puts servers_using_group.join(',')
 

data/lib/awsutils/version.rb

@@ -1,3 +1,3 @@
 module AwsUtils
-  VERSION = '2.0.3'
+  VERSION = '2.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: awsutils
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.1.0
 platform: ruby
 authors:
 - Eric Herot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-25 00:00:00.000000000 Z
+date: 2016-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.1
+        version: 0.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.1
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: trollop
   requirement: !ruby/object:Gem::Requirement
@@ -164,10 +164,25 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A set of tools for interacting with AWS
 email:
 - eric.rubygems@herot.com
 executables:
+- awslogs
 - ec2addsg
 - ec2delsg
 - ec2info
@@ -185,6 +200,7 @@ files:
 - README.md
 - Rakefile
 - awsutils.gemspec
+- bin/awslogs
 - bin/ec2addsg
 - bin/ec2delsg
 - bin/ec2info
@@ -194,6 +210,7 @@ files:
 - bin/elbls
 - bin/r53ls
 - lib/awsutils.rb
+- lib/awsutils/awslogs.rb
 - lib/awsutils/ec2addsg.rb
 - lib/awsutils/ec2delsg.rb
 - lib/awsutils/ec2info.rb