aws_log_cleaner 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01b45eb13e994162c2b137d83a39fb5aea229324
-  data.tar.gz: 8611794db35d12cd38b166613e5ec1a3f022f192
+  metadata.gz: fd1ee81183128c4954349661dfceb8ec7c1e1cf4
+  data.tar.gz: 3b689c2c7a60e27d589fe0042f66fc8c7796d491
 SHA512:
-  metadata.gz: 94ba3d272f3611bd54d9255204c10f56a879f9adb693a06c0b48bbd4e0cebabed1d6f623f9897dfeaa0474de48efe0d78cb3ebfc27d69021a354630bc96247e6
-  data.tar.gz: be81383cc1e304308ed6915e749698dbf655bf771e5374e54d9a68c2eeab6c533bae379060e0471482659d25ab9e3dabab8df25a298c7dc5706507b0e9ed2e63
+  metadata.gz: cd34d68fef24670b8cb6a6cd2b336fa3b4b38fd7e0cdbcd1fbe13c3c38086f6617855fbc27bb0ff1d4fa24b09a2ab30a629081778b1db7e02c83f1e5b1629de1
+  data.tar.gz: d80fa1bf6e02a4df64b84f2402896da4ddfb4df1c1b19b2d469902b2a355190df50051cfd023f619d16244ed8b8107c347cc92fa9d0c4afa853c972be8e8b576

data/.appveyor.yml

@@ -0,0 +1,14 @@
+install:
+  - set PATH=C:\Ruby21-x64\bin;%PATH%
+  - bundle install
+  - gem install bundler -v 1.14.6
+
+build: off
+
+before_test:
+  - ruby -v
+  - gem -v
+  - bundle -v
+
+test_script:
+  - bundle exec rake

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/daniebker/aws_log_cleaner.svg?branch=master)](https://travis-ci.org/daniebker/aws_log_cleaner)
+[![Build Status](https://travis-ci.org/daniebker/aws_log_cleaner.svg?branch=master)](https://travis-ci.org/daniebker/aws_log_cleaner) [![Build status](https://ci.appveyor.com/api/projects/status/0rn4dqa7wyr3cqh9?svg=true)](https://ci.appveyor.com/project/daniebker/aws-log-cleaner) [![Gem Version](https://badge.fury.io/rb/aws_log_cleaner.svg)](https://badge.fury.io/rb/aws_log_cleaner) 
 
 # AwsLogCleaner
 
@@ -26,33 +26,39 @@ Or install it yourself as:
 
 * `AWS_SECRET_KEY` & `AWS_ACCESS_KEY` env variables must be set.  
 OR
-* A credentials file must be present in  `~\.aws`
+* A credentials file must be present in  `~\.aws`   
 
 The application defaults to `eu-west-1` but this can be overridden using `AWS_REGION` environment variable, or by passing -r [--region] on the commandline.
 
 ### From the command line
 
-* Get help by using:
+Get help by using:
 
-`aws_log_cleaner -h`
+    $ aws_log_cleaner -h
 
-* Run the plan command in eu-west-1 where prefix like some_text.
+Run the plan command in eu-west-1 where prefix like some_text.
 
-`aws_log_cleaner -p -l 'some_text' -r 'eu-west-1' -k [ACCESS_KEY] -s [SECRET]` 
+    $ aws_log_cleaner -p -l 'some_text' -r 'eu-west-1' -k [ACCESS_KEY] -s [SECRET]
 
 ## Code 
 
 Initialise a new LogCleaner 
 
-`log_cleaner = AwsLogCleaner::AwsLogCleaner.new`
+```ruby
+log_cleaner = AwsLogCleaner::AwsLogCleaner.new
+```
 
 Run a plan
 
-`log_cleaner.plan('some_text')`
+```ruby
+log_cleaner.plan('some_text')
+```
 
 Or run a delete 
 
-`log_cleaner.delete('some_text')`
+```ruby
+log_cleaner.delete('some_text')
+```
 
 ## Development
 

data/bin/aws_log_cleaner

@@ -5,7 +5,7 @@ require 'colorize'
 
 USAGE = <<ENDUSAGE.freeze
   Usage:
-    aws_log_cleaner [-h] ([-p] or [-a]) [-r region] [-l like]
+    aws_log_cleaner [-h] ([-p] or [-a]) [-o] [-r region] [-l like]
 ENDUSAGE
 
 HELP = <<ENDHELP.freeze
@@ -17,6 +17,8 @@ HELP = <<ENDHELP.freeze
    -k,  --access_key The access key to use for AWS.
    -s,  --secret     The secret to use for AWS.
    -P,  --profile    The profile to use for AWS Credentials.
+   -o,  --orphan     Clean orphan logs.
+   -v,  --version    Print version.
 ENDHELP
 
 args = { region: nil, access_key: nil, secret: nil }
@@ -27,6 +29,8 @@ ARGV.each do |arg|
   when '-h', '--help'       then args[:help]      = true
   when '-p', '--plan'       then args[:plan]      = true
   when '-a', '--apply'      then args[:delete]    = true
+  when '-o', '--orphan'     then args[:orphan]    = true
+  when '-v', '--version'    then args[:version]   = true
   when '-l', '--like'       then next_arg         = :like
   when '-r', '--region'     then next_arg         = :region
   when '-k', '--access_key' then next_arg         = :access_key
@@ -41,15 +45,18 @@ ARGV.each do |arg|
   end
 end
 
+if args[:version]
+  puts AwsLogCleaner::VERSION
+  exit
+end
+
 if args[:help] || (!args[:plan] && !args[:delete])
   puts USAGE
   puts HELP
   exit
 end
 
-like = args[:like].nil? ? '' : args[:like].downcase
-
-credentials_args = 
+credentials_args =
   if args[:access_key] && args[:secret]
     {
       access_key: args[:access_key],
@@ -59,26 +66,20 @@ credentials_args =
     {
       profile: args[:profile]
     }
-  else 
-    nil
   end
 
-credentials_args[:region] = args[:region] unless args[:region].nil? 
+credentials_args[:region] = args[:region] unless args[:region].nil?
 
 log_cleaner = \
-  AwsLogCleaner::AwsLogCleaner.new(credentials_args)
+  AwsLogCleaner::AwsLogCleaner.new(credentials_args, args[:like], args[:orphan])
 
 if args[:plan]
   puts "== PLAN MODE ==\n\n"
-  
-  result = log_cleaner.plan(like)
-
+  result = log_cleaner.plan
   puts "Found #{result.count} item(s) to remove\n\n"
 elsif args[:delete] && !args[:plan]
   puts "== DELETE MODE ==\n\n"
-
-  result = log_cleaner.delete(like)
-
+  result = log_cleaner.delete
   puts "Deleted #{result.count} item(s)\n\n"
 end
 

data/lib/aws_log_cleaner.rb

@@ -1,18 +1,14 @@
 require_relative 'aws_log_cleaner/cloud_watch_logs'
-require_relative 'aws_log_cleaner/log_group_cleaner_commands'
+require_relative 'aws_log_cleaner/log_group_cleaner'
 require_relative 'aws_log_cleaner/api_gateway'
 require_relative 'aws_log_cleaner/api_gateway_retriever'
 require_relative 'aws_log_cleaner/log_group_filterer'
 require_relative 'aws_log_cleaner/aws_credentials'
 require_relative 'aws_log_cleaner/version'
 
-
 module AwsLogCleaner
-
-  # Application wrapper
   class AwsLogCleaner
-
-    def initialize(credentials_args)
+    def initialize(credentials_args, like, clean_orphans)
       credentials = AwsCredentials.new(credentials_args)
       cloud_watch_logs = CloudWatchLogs.new(credentials)
       api_gateway = ApiGateway.new(credentials)
@@ -20,20 +16,21 @@ module AwsLogCleaner
       api_retriever = ApiGatewayRetriever.new(api_gateway)
       log_group_filterer = LogGroupFilterer.new(cloud_watch_logs)
 
-      @log_cleaner = LogGroupCleanerCommands.new(
+      @log_cleaner = LogGroupCleaner.new(
         cloud_watch_logs,
         api_retriever,
-        log_group_filterer
+        log_group_filterer,
+        like,
+        clean_orphans
       )
     end
 
-    def plan(like)
-      @log_cleaner.plan(like)
+    def plan
+      @log_cleaner.plan
     end
 
-    def delete(like)
-      @log_cleaner.delete(like)
+    def delete
+      @log_cleaner.delete
     end
-
   end
-end
+end

data/lib/aws_log_cleaner/api_gateway.rb

@@ -3,11 +3,8 @@ require 'aws-sdk'
 Aws.use_bundled_cert!
 
 module AwsLogCleaner
-
-  # Class responsible for interacting with the Aws
-  # ApiGatewayClient
+  # Class responsible for interacting with the AWS ApiGatewayClient
   class ApiGateway
-
     def initialize(credentials)
       @api_client = Aws::APIGateway::Client.new(
         region: credentials.region,
@@ -16,6 +13,13 @@ module AwsLogCleaner
     end
 
     def list_all_apis
+      @rest_apis = get_rest_apis if @rest_apis.nil?
+      @rest_apis
+    end
+
+    private
+
+    def get_rest_apis
       Enumerator.new do |enum|
         request = { limit: 25 }
         loop do
@@ -28,6 +32,5 @@ module AwsLogCleaner
         end
       end
     end
-
   end
 end

data/lib/aws_log_cleaner/api_gateway_retriever.rb

@@ -1,17 +1,17 @@
-
 module AwsLogCleaner
-
   # Responsible for retrieving API gateway instances filtered by a given text
   class ApiGatewayRetriever
-
     def initialize(api_gateway)
       @api_gateway = api_gateway
     end
 
     def retrieve(text)
       apis = @api_gateway.list_all_apis
-      apis.select{ |item| item.name.to_s.downcase.include?(text) }
+      apis.select { |item| item.name.to_s.downcase.include?(text) }
     end
 
+    def retrieve_all
+      @api_gateway.list_all_apis
+    end
   end
 end

data/lib/aws_log_cleaner/cloud_watch_logs.rb

@@ -1,10 +1,8 @@
 require 'aws-sdk'
 
 module AwsLogCleaner
-
   # Class responsible for interacting with AWS Cloudwatch
   class CloudWatchLogs
-
     # Required for Windows users.
     Aws.use_bundled_cert!
 
@@ -15,18 +13,9 @@ module AwsLogCleaner
       )
     end
 
-    def list_all_log_groups(token = nil)
-      log_groups = []
-      resp = describe_log_groups(token)
-      log_groups.concat(resp.log_groups)
-
-      if resp.next_token.to_s.empty?
-        log_groups
-      else
-        log_groups.concat(
-            list_all_log_groups(resp.next_token)
-        )
-      end
+    def list_all_log_groups
+      @log_groups = describe_log_groups if @log_groups.nil?
+      @log_groups
     end
 
     def delete_log_groups(log_group_names)
@@ -39,15 +28,15 @@ module AwsLogCleaner
 
     private
 
-    def describe_log_groups(token)
-      if token.nil?
-        @cloud_watch_logs.describe_log_groups
-      else
-        @cloud_watch_logs.describe_log_groups(
-          next_token: token
-        )
+    def describe_log_groups
+      log_groups = []
+      token = nil
+      loop do
+        resp = @cloud_watch_logs.describe_log_groups(next_token: token)
+        log_groups.concat(resp.log_groups)
+        token = resp.next_token
+        return log_groups if token.to_s.empty?
       end
     end
-
   end
 end

data/lib/aws_log_cleaner/log_group_cleaner.rb

@@ -0,0 +1,80 @@
+require_relative 'cloud_watch_logs'
+
+module AwsLogCleaner
+  class LogGroupCleaner
+    API_GATEWAY_LOG_PREFIX = 'API-Gateway-Execution-Logs_'.freeze
+
+    def initialize(
+      cloud_watch_logs,
+      api_retriever,
+      log_group_filterer,
+      like,
+      clean_orphans
+    )
+      @cloud_watch_logs = cloud_watch_logs
+      @api_retriever = api_retriever
+      @log_group_filterer = log_group_filterer
+      @like = like.downcase unless like.nil?
+      @clean_orphans = clean_orphans
+    end
+
+    def plan
+      result(log_groups_to_delete)
+    end
+
+    def delete
+      log_groups = log_groups_to_delete
+      @cloud_watch_logs.delete_log_groups(log_groups)
+      result(log_groups)
+    end
+
+    private
+
+    def log_groups_to_delete
+      log_groups = []
+      log_groups.concat(log_group_names_containing(@like)) if @like
+      log_groups.concat(log_group_names_orphan) if @clean_orphans
+      log_groups.uniq
+    end
+
+    def log_group_names_containing(text)
+      log_groups = @log_group_filterer.filter_by_name_includes(text)
+
+      apis = @api_retriever.retrieve(text)
+      apis.each do |api|
+        log_groups.concat(
+          @log_group_filterer.filter_by_name_includes(
+            "#{API_GATEWAY_LOG_PREFIX}#{api.id}"
+          )
+        )
+      end
+
+      log_groups.map(&:log_group_name)
+    end
+
+    def log_group_names_orphan
+      groups = @cloud_watch_logs.list_all_log_groups
+      groups = groups.select { |log_group| orphan?(log_group) }
+      groups.map(&:log_group_name)
+    end
+
+    def orphan?(log_group)
+      apis = @api_retriever.retrieve_all
+      api_id = extract_api_id(log_group.log_group_name)
+      api_id.nil? ? false : apis.none? { |api| api.id == api_id }
+    end
+
+    def extract_api_id(log_group_name)
+      match = log_group_name.match(%r{#{API_GATEWAY_LOG_PREFIX}(\w+)/\w+})
+      match.captures[0] unless match.nil?
+    end
+
+    def result(log_groups)
+      to_delete = []
+      log_groups.each do |name|
+        to_delete.push("(-) #{name}")
+      end
+      to_delete
+    end
+  end
+end

data/lib/aws_log_cleaner/log_group_filterer.rb

@@ -1,17 +1,14 @@
-
 module AwsLogCleaner
-
   class LogGroupFilterer
-
     def initialize(cloud_watch_logs)
       @cloud_watch_logs = cloud_watch_logs
     end
 
     def filter_by_name_includes(text)
       log_groups = @cloud_watch_logs.list_all_log_groups
-      log_groups.select{ |item| item.log_group_name.to_s.downcase.include?(text) }
+      log_groups.select do |item|
+        item.log_group_name.to_s.downcase.include?(text)
+      end
     end
-
   end
-
 end

data/lib/aws_log_cleaner/version.rb

@@ -1,3 +1,3 @@
 module AwsLogCleaner
-  VERSION = '0.1.7'
+  VERSION = '0.1.8'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws_log_cleaner
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - dbaker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-08 00:00:00.000000000 Z
+date: 2017-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,6 +90,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".appveyor.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -106,7 +107,7 @@ files:
 - lib/aws_log_cleaner/api_gateway_retriever.rb
 - lib/aws_log_cleaner/aws_credentials.rb
 - lib/aws_log_cleaner/cloud_watch_logs.rb
-- lib/aws_log_cleaner/log_group_cleaner_commands.rb
+- lib/aws_log_cleaner/log_group_cleaner.rb
 - lib/aws_log_cleaner/log_group_filterer.rb
 - lib/aws_log_cleaner/version.rb
 homepage: https://github.com/daniebker/aws_log_cleaner

data/lib/aws_log_cleaner/log_group_cleaner_commands.rb

@@ -1,56 +0,0 @@
-require_relative 'cloud_watch_logs'
-
-module AwsLogCleaner
-
-  # Responsible for cleaning log groups in AWS.
-  class LogGroupCleanerCommands
-
-    def initialize(cloud_watch_logs, api_retriever, log_group_filterer)
-      @cloud_watch_logs = cloud_watch_logs
-      @api_retriever = api_retriever
-      @log_group_filterer = log_group_filterer
-    end
-
-    def plan(text)
-      result(text)
-    end
-
-    def delete(text)
-      @cloud_watch_logs.delete_log_groups(log_group_names_containing(text))
-      result(text)
-    end
-
-    private
-
-    def log_group_names_containing(text)
-      if @log_group_names.nil?
-        @log_group_names = []
-        log_groups =
-          @log_group_filterer.filter_by_name_includes(text)
-
-        apis = @api_retriever.retrieve(text)
-        apis.each do |api|
-          log_groups.concat(
-            @log_group_filterer.filter_by_name_includes(api.id.to_s)
-          )
-        end
-
-        log_groups.each do |log_group|
-          @log_group_names.push(log_group.log_group_name)
-        end
-      end
-
-      @log_group_names
-    end
-
-    def result(text)
-      to_delete = []
-      log_group_names_containing(text).each do |name|
-        to_delete.push("(-) #{name}")
-      end
-      to_delete
-    end
-
-  end
-
-end