aws_log_cleaner 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +126 -0
- data/.rspec +1 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +137 -0
- data/LICENSE +13 -0
- data/README.md +59 -0
- data/Rakefile +13 -0
- data/aws_log_cleaner.gemspec +31 -0
- data/bin/aws_log_cleaner +69 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/aws_log_cleaner.rb +39 -0
- data/lib/aws_log_cleaner/api_gateway.rb +33 -0
- data/lib/aws_log_cleaner/api_gateway_retriever.rb +17 -0
- data/lib/aws_log_cleaner/aws_credentials.rb +25 -0
- data/lib/aws_log_cleaner/cloud_watch_logs.rb +53 -0
- data/lib/aws_log_cleaner/log_group_cleaner_commands.rb +56 -0
- data/lib/aws_log_cleaner/log_group_filterer.rb +17 -0
- data/lib/aws_log_cleaner/version.rb +3 -0
- metadata +137 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e0253331366894b4b77ece9c5d7bc1fd3d9d652f
|
4
|
+
data.tar.gz: 00cd865249540f4268a55303f748e4a15e6a6127
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3534b15480fb89f62168e5800c791303c26fa5fb54f8dd8a0a347637d91d39fe85d4798a2cdad8651c157253a90b2f048048c3fceed6b363cfc28b816d5dba52
|
7
|
+
data.tar.gz: 9fc84b6ade388df9e2449ad368990b4ee14254d26df212c5906e5085977aacf5bbcb99998818ec09fe763f2ee33c3cc7fb77a1fd820d3857211d266e99ab6fac
|
data/.gitignore
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
|
2
|
+
# Created by https://www.gitignore.io/api/ruby,visualstudiocode,rubymine
|
3
|
+
|
4
|
+
### Ruby ###
|
5
|
+
*.gem
|
6
|
+
*.rbc
|
7
|
+
/.config
|
8
|
+
/coverage/
|
9
|
+
/InstalledFiles
|
10
|
+
/pkg/
|
11
|
+
/spec/reports/
|
12
|
+
/spec/examples.txt
|
13
|
+
/test/tmp/
|
14
|
+
/test/version_tmp/
|
15
|
+
/tmp/
|
16
|
+
|
17
|
+
# Used by dotenv library to load environment variables.
|
18
|
+
# .env
|
19
|
+
|
20
|
+
## Specific to RubyMotion:
|
21
|
+
.dat*
|
22
|
+
.repl_history
|
23
|
+
build/
|
24
|
+
*.bridgesupport
|
25
|
+
build-iPhoneOS/
|
26
|
+
build-iPhoneSimulator/
|
27
|
+
|
28
|
+
## Specific to RubyMotion (use of CocoaPods):
|
29
|
+
#
|
30
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
31
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
32
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
33
|
+
#
|
34
|
+
# vendor/Pods/
|
35
|
+
|
36
|
+
## Documentation cache and generated files:
|
37
|
+
/.yardoc/
|
38
|
+
/_yardoc/
|
39
|
+
/doc/
|
40
|
+
/rdoc/
|
41
|
+
|
42
|
+
## Environment normalization:
|
43
|
+
/.bundle/
|
44
|
+
/vendor/bundle
|
45
|
+
/lib/bundler/man/
|
46
|
+
|
47
|
+
# for a library or gem, you might want to ignore these files since the code is
|
48
|
+
# intended to run in multiple environments; otherwise, check them in:
|
49
|
+
# Gemfile.lock
|
50
|
+
# .ruby-version
|
51
|
+
# .ruby-gemset
|
52
|
+
|
53
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
54
|
+
.rvmrc
|
55
|
+
|
56
|
+
### RubyMine ###
|
57
|
+
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
|
58
|
+
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
59
|
+
|
60
|
+
# User-specific stuff:
|
61
|
+
.idea/**/workspace.xml
|
62
|
+
.idea/**/tasks.xml
|
63
|
+
.idea/dictionaries
|
64
|
+
|
65
|
+
# Sensitive or high-churn files:
|
66
|
+
.idea/**/dataSources/
|
67
|
+
.idea/**/dataSources.ids
|
68
|
+
.idea/**/dataSources.xml
|
69
|
+
.idea/**/dataSources.local.xml
|
70
|
+
.idea/**/sqlDataSources.xml
|
71
|
+
.idea/**/dynamic.xml
|
72
|
+
.idea/**/uiDesigner.xml
|
73
|
+
|
74
|
+
# Gradle:
|
75
|
+
.idea/**/gradle.xml
|
76
|
+
.idea/**/libraries
|
77
|
+
|
78
|
+
# CMake
|
79
|
+
cmake-build-debug/
|
80
|
+
|
81
|
+
# Mongo Explorer plugin:
|
82
|
+
.idea/**/mongoSettings.xml
|
83
|
+
|
84
|
+
## File-based project format:
|
85
|
+
*.iws
|
86
|
+
|
87
|
+
## Plugin-specific files:
|
88
|
+
|
89
|
+
# IntelliJ
|
90
|
+
/out/
|
91
|
+
|
92
|
+
# mpeltonen/sbt-idea plugin
|
93
|
+
.idea_modules/
|
94
|
+
|
95
|
+
# JIRA plugin
|
96
|
+
atlassian-ide-plugin.xml
|
97
|
+
|
98
|
+
# Cursive Clojure plugin
|
99
|
+
.idea/replstate.xml
|
100
|
+
|
101
|
+
# Crashlytics plugin (for Android Studio and IntelliJ)
|
102
|
+
com_crashlytics_export_strings.xml
|
103
|
+
crashlytics.properties
|
104
|
+
crashlytics-build.properties
|
105
|
+
fabric.properties
|
106
|
+
|
107
|
+
### RubyMine Patch ###
|
108
|
+
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
|
109
|
+
|
110
|
+
.idea/
|
111
|
+
*.iml
|
112
|
+
modules.xml
|
113
|
+
.idea/misc.xml
|
114
|
+
*.ipr
|
115
|
+
|
116
|
+
# Sonarlint plugin
|
117
|
+
.idea/sonarlint
|
118
|
+
|
119
|
+
### VisualStudioCode ###
|
120
|
+
.vscode/*
|
121
|
+
!.vscode/settings.json
|
122
|
+
!.vscode/tasks.json
|
123
|
+
!.vscode/launch.json
|
124
|
+
!.vscode/extensions.json
|
125
|
+
|
126
|
+
# End of https://www.gitignore.io/api/ruby,visualstudiocode,rubymine
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
remote: http://vbuartifactory.vistaprint.svc/artifactory/api/gems/gems-release-local/
|
4
|
+
specs:
|
5
|
+
artifactory (2.2.1)
|
6
|
+
artifactory-client-mcp (0.0.9)
|
7
|
+
artifactory (~> 2.2.0)
|
8
|
+
vp_rake_core (~> 0.9.44)
|
9
|
+
asciidoctor (0.1.4)
|
10
|
+
aws-sdk (2.9.25)
|
11
|
+
aws-sdk-resources (= 2.9.25)
|
12
|
+
aws-sdk-core (2.9.25)
|
13
|
+
aws-sigv4 (~> 1.0)
|
14
|
+
jmespath (~> 1.0)
|
15
|
+
aws-sdk-resources (2.9.25)
|
16
|
+
aws-sdk-core (= 2.9.25)
|
17
|
+
aws-sigv4 (1.0.0)
|
18
|
+
colorize (0.8.1)
|
19
|
+
diff-lcs (1.3)
|
20
|
+
facter (2.4.6-x86-mingw32)
|
21
|
+
ffi (~> 1.9.5)
|
22
|
+
win32-dir (~> 0.4.8)
|
23
|
+
win32-security (~> 0.2.5)
|
24
|
+
win32console (~> 1.3.2)
|
25
|
+
faraday (0.9.2)
|
26
|
+
multipart-post (>= 1.2, < 3)
|
27
|
+
faraday_middleware (0.10.1)
|
28
|
+
faraday (>= 0.7.4, < 1.0)
|
29
|
+
fast_gettext (1.1.0)
|
30
|
+
ffi (1.9.18-x86-mingw32)
|
31
|
+
ffi-win32-extensions (1.0.3)
|
32
|
+
ffi
|
33
|
+
gettext (3.2.2)
|
34
|
+
locale (>= 2.0.5)
|
35
|
+
text (>= 1.3.0)
|
36
|
+
gettext-setup (0.25)
|
37
|
+
fast_gettext (~> 1.1.0)
|
38
|
+
gettext (>= 3.0.2)
|
39
|
+
locale
|
40
|
+
hiera (1.3.4)
|
41
|
+
json_pure
|
42
|
+
jmespath (1.3.1)
|
43
|
+
json_pure (1.8.6)
|
44
|
+
librarian-puppet (2.2.3)
|
45
|
+
librarianp (>= 0.6.3)
|
46
|
+
puppet_forge (~> 2.1)
|
47
|
+
rsync
|
48
|
+
librarianp (0.6.3)
|
49
|
+
thor (~> 0.15)
|
50
|
+
locale (2.1.2)
|
51
|
+
minitar (0.5.4)
|
52
|
+
multi_json (1.12.1)
|
53
|
+
multipart-post (2.0.0)
|
54
|
+
puppet (3.7.5-x86-mingw32)
|
55
|
+
facter (> 1.6, < 3)
|
56
|
+
ffi (~> 1.9.5)
|
57
|
+
hiera (~> 1.0)
|
58
|
+
json_pure
|
59
|
+
minitar (~> 0.5.4)
|
60
|
+
win32-dir (~> 0.4.9)
|
61
|
+
win32-eventlog (~> 0.6.1)
|
62
|
+
win32-process (~> 0.7.4)
|
63
|
+
win32-security (~> 0.2.5)
|
64
|
+
win32-service (~> 0.8.6)
|
65
|
+
win32console (= 1.3.2)
|
66
|
+
puppet_forge (2.2.4)
|
67
|
+
faraday (~> 0.9.0)
|
68
|
+
faraday_middleware (>= 0.9.0, < 0.11.0)
|
69
|
+
gettext-setup (~> 0.11)
|
70
|
+
minitar
|
71
|
+
semantic_puppet (~> 0.1.0)
|
72
|
+
rake (12.0.0)
|
73
|
+
rspec (3.6.0)
|
74
|
+
rspec-core (~> 3.6.0)
|
75
|
+
rspec-expectations (~> 3.6.0)
|
76
|
+
rspec-mocks (~> 3.6.0)
|
77
|
+
rspec-core (3.6.0)
|
78
|
+
rspec-support (~> 3.6.0)
|
79
|
+
rspec-expectations (3.6.0)
|
80
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
81
|
+
rspec-support (~> 3.6.0)
|
82
|
+
rspec-mocks (3.6.0)
|
83
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
84
|
+
rspec-support (~> 3.6.0)
|
85
|
+
rspec-support (3.6.0)
|
86
|
+
rsync (1.0.9)
|
87
|
+
rubyzip (1.1.7)
|
88
|
+
semantic_puppet (0.1.4)
|
89
|
+
gettext-setup (>= 0.3)
|
90
|
+
simplecov (0.7.1)
|
91
|
+
multi_json (~> 1.0)
|
92
|
+
simplecov-html (~> 0.7.1)
|
93
|
+
simplecov-html (0.7.1)
|
94
|
+
text (1.3.1)
|
95
|
+
thor (0.19.4)
|
96
|
+
vp_devkit_dotnet (1.0.45)
|
97
|
+
artifactory-client-mcp (~> 0.0.1)
|
98
|
+
asciidoctor (~> 0.1.4)
|
99
|
+
json_pure (~> 1.7)
|
100
|
+
vp_ivy (~> 1.0.1)
|
101
|
+
vp_puppet (~> 0.9.25)
|
102
|
+
vp_rake_core (~> 0.9.44)
|
103
|
+
vp_ivy (1.0.11)
|
104
|
+
rubyzip (~> 1.1.6)
|
105
|
+
vp_rake_core (~> 0.9.44)
|
106
|
+
vp_puppet (0.9.36)
|
107
|
+
ffi (~> 1.9.6)
|
108
|
+
librarian-puppet (>= 1.3.2, < 3.0)
|
109
|
+
puppet (= 3.7.5)
|
110
|
+
vp_rake_core (~> 0.9.52)
|
111
|
+
vp_rake_core (0.9.56)
|
112
|
+
rake
|
113
|
+
simplecov (= 0.7.1)
|
114
|
+
win32-dir (0.4.9)
|
115
|
+
ffi (>= 1.0.0)
|
116
|
+
win32-eventlog (0.6.7)
|
117
|
+
ffi
|
118
|
+
win32-process (0.7.5)
|
119
|
+
ffi (>= 1.0.0)
|
120
|
+
win32-security (0.2.5)
|
121
|
+
ffi
|
122
|
+
win32-service (0.8.10)
|
123
|
+
ffi
|
124
|
+
ffi-win32-extensions
|
125
|
+
win32console (1.3.2-x86-mingw32)
|
126
|
+
|
127
|
+
PLATFORMS
|
128
|
+
x86-mingw32
|
129
|
+
|
130
|
+
DEPENDENCIES
|
131
|
+
aws-sdk (~> 2.9.25)
|
132
|
+
colorize (~> 0.8.1)
|
133
|
+
rspec
|
134
|
+
vp_devkit_dotnet
|
135
|
+
|
136
|
+
BUNDLED WITH
|
137
|
+
1.13.6
|
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2017 Cimpress
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
# AwsLogCleaner
|
2
|
+
|
3
|
+
A gem to clean up aws lambda and Api Gateway log groups that contain a search term.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'aws_log_cleaner'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install aws_log_cleaner
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
### Requirements
|
24
|
+
|
25
|
+
* `AWS_SECRET_KEY` & `AWS_ACCESS_KEY` env variables must be set.
|
26
|
+
OR
|
27
|
+
* A credentials file must be present in `~\.aws`
|
28
|
+
|
29
|
+
The application defaults to `eu-west-1` but this can be overridden using `AWS_DEFAULT_REGION` environment variable.
|
30
|
+
|
31
|
+
### From the command line
|
32
|
+
|
33
|
+
* Get help by using:
|
34
|
+
|
35
|
+
`aws_log_cleaner -h`
|
36
|
+
|
37
|
+
* Run the plan command in eu-west-1 where prefix like some_text.
|
38
|
+
|
39
|
+
`aws_log_cleaner -p -l 'some_text' -r 'eu-west-1' -k [ACCESS_KEY] -s [SECRET]`
|
40
|
+
|
41
|
+
## Code
|
42
|
+
|
43
|
+
Initialise a new LogCleaner
|
44
|
+
|
45
|
+
`log_cleaner = AwsLogCleaner::AwsLogCleaner.new`
|
46
|
+
|
47
|
+
Run a plan
|
48
|
+
|
49
|
+
`log_cleaner.plan('some_text')`
|
50
|
+
|
51
|
+
Or run a delete
|
52
|
+
|
53
|
+
`log_cleaner.delete('some_text')`
|
54
|
+
|
55
|
+
## Development
|
56
|
+
|
57
|
+
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
58
|
+
|
59
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
data/Rakefile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require 'aws_log_cleaner/version'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = 'aws_log_cleaner'
|
9
|
+
spec.version = AwsLogCleaner::VERSION
|
10
|
+
spec.authors = ['dbaker']
|
11
|
+
spec.email = ['dbaker@vistaprint.com']
|
12
|
+
spec.summary = 'Clean up lambda and API Gateway logs in AWS that contain a search term ' \
|
13
|
+
'in their log group name.'
|
14
|
+
spec.description = 'Cleans all logs in AWS that contain a search term in the log group name. ' \
|
15
|
+
'Also searches for api gateway instances that contain the search term ' \
|
16
|
+
'and finds their corresponding logs.'
|
17
|
+
spec.homepage = 'https://github.com/daniebker/aws_log_cleaner'
|
18
|
+
spec.license = 'Apache-2.0'
|
19
|
+
|
20
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
21
|
+
f.match(%r{^(test|spec|features)/})
|
22
|
+
end
|
23
|
+
spec.executables << 'aws_log_cleaner'
|
24
|
+
|
25
|
+
spec.add_development_dependency 'bundler', '~> 1.13'
|
26
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.6.0'
|
28
|
+
|
29
|
+
spec.add_dependency 'aws-sdk', '~> 2.9.25'
|
30
|
+
spec.add_dependency 'colorize', '~> 0.8.1'
|
31
|
+
end
|
data/bin/aws_log_cleaner
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'aws_log_cleaner'
|
4
|
+
require 'colorize'
|
5
|
+
|
6
|
+
USAGE = <<ENDUSAGE.freeze
|
7
|
+
Usage:
|
8
|
+
aws_log_cleaner [-h] ([-p] or [-a]) [-r region] [-l like]
|
9
|
+
ENDUSAGE
|
10
|
+
|
11
|
+
HELP = <<ENDHELP.freeze
|
12
|
+
-h, --help Show this help.
|
13
|
+
-p, --plan Runs the plan command.
|
14
|
+
-a, --apply Applys the deletion. Preview what will be deleted with [-p]
|
15
|
+
-l, --like The search term returned logs should contain.
|
16
|
+
-r, --region The region to run in.
|
17
|
+
-k, --access_key The access key to use for AWS.
|
18
|
+
-s, --secret The secret to use for AWS.
|
19
|
+
ENDHELP
|
20
|
+
|
21
|
+
args = { region: nil, access_key: nil, secret: nil }
|
22
|
+
unflagged_args = []
|
23
|
+
next_arg = unflagged_args.first
|
24
|
+
ARGV.each do |arg|
|
25
|
+
case arg
|
26
|
+
when '-h', '--help' then args[:help] = true
|
27
|
+
when '-p', '--plan' then args[:plan] = true
|
28
|
+
when '-a', '--apply' then args[:delete] = true
|
29
|
+
when '-l', '--like' then next_arg = :like
|
30
|
+
when '-r', '--region' then next_arg = :region
|
31
|
+
when '-k', '--access_key' then next_arg = :access_key
|
32
|
+
when '-s', '--secret' then next_arg = :secret
|
33
|
+
else
|
34
|
+
if next_arg
|
35
|
+
args[next_arg] = arg
|
36
|
+
unflagged_args.delete(next_arg)
|
37
|
+
end
|
38
|
+
next_arg = unflagged_args.first
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
if args[:help] || (!args[:plan] && !args[:delete])
|
43
|
+
puts USAGE
|
44
|
+
puts HELP
|
45
|
+
exit
|
46
|
+
end
|
47
|
+
|
48
|
+
like = args[:like].nil? ? '' : args[:like]
|
49
|
+
|
50
|
+
log_cleaner = \
|
51
|
+
AwsLogCleaner::AwsLogCleaner.new(args[:access_key], args[:secret], args[:region])
|
52
|
+
|
53
|
+
if args[:plan]
|
54
|
+
puts "== PLAN MODE ==\n\n"
|
55
|
+
|
56
|
+
result = log_cleaner.plan(like)
|
57
|
+
|
58
|
+
puts "Found #{result.count} item(s) to remove\n\n"
|
59
|
+
elsif args[:delete] && !args[:plan]
|
60
|
+
puts "== DELETE MODE ==\n\n"
|
61
|
+
|
62
|
+
result = log_cleaner.delete(like)
|
63
|
+
|
64
|
+
puts "Deleted #{result.count} item(s)\n\n"
|
65
|
+
end
|
66
|
+
|
67
|
+
result.each do |item|
|
68
|
+
puts item.to_s.red
|
69
|
+
end
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "aws_log_cleaner"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require_relative 'aws_log_cleaner/cloud_watch_logs'
|
2
|
+
require_relative 'aws_log_cleaner/log_group_cleaner_commands'
|
3
|
+
require_relative 'aws_log_cleaner/api_gateway'
|
4
|
+
require_relative 'aws_log_cleaner/api_gateway_retriever'
|
5
|
+
require_relative 'aws_log_cleaner/log_group_filterer'
|
6
|
+
require_relative 'aws_log_cleaner/aws_credentials'
|
7
|
+
require_relative 'aws_log_cleaner/version'
|
8
|
+
|
9
|
+
|
10
|
+
module AwsLogCleaner
|
11
|
+
|
12
|
+
# Application wrapper
|
13
|
+
class AwsLogCleaner
|
14
|
+
|
15
|
+
def initialize(access_key = nil, secret = nil, region = nil)
|
16
|
+
credentials = AwsCredentials.new(access_key, secret, region)
|
17
|
+
cloud_watch_logs = CloudWatchLogs.new(credentials)
|
18
|
+
api_gateway = ApiGateway.new(credentials)
|
19
|
+
|
20
|
+
api_retriever = ApiGatewayRetriever.new(api_gateway)
|
21
|
+
log_group_filterer = LogGroupFilterer.new(cloud_watch_logs)
|
22
|
+
|
23
|
+
@log_cleaner = LogGroupCleanerCommands.new(
|
24
|
+
cloud_watch_logs,
|
25
|
+
api_retriever,
|
26
|
+
log_group_filterer
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
def plan(like)
|
31
|
+
@log_cleaner.plan(like)
|
32
|
+
end
|
33
|
+
|
34
|
+
def delete(like)
|
35
|
+
@log_cleaner.delete(like)
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
Aws.use_bundled_cert!
|
4
|
+
|
5
|
+
module AwsLogCleaner
|
6
|
+
|
7
|
+
# Class responsible for interacting with the Aws
|
8
|
+
# ApiGatewayClient
|
9
|
+
class ApiGateway
|
10
|
+
|
11
|
+
def initialize(credentials)
|
12
|
+
@api_client = Aws::APIGateway::Client.new(
|
13
|
+
region: credentials.region,
|
14
|
+
credentials: credentials.credentials
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
def list_all_apis
|
19
|
+
Enumerator.new do |enum|
|
20
|
+
request = { limit: 25 }
|
21
|
+
loop do
|
22
|
+
response = @api_client.get_rest_apis(request)
|
23
|
+
response.items.each do |item|
|
24
|
+
enum.yield item
|
25
|
+
end
|
26
|
+
break if response.position.nil?
|
27
|
+
request[:position] = response.position
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
module AwsLogCleaner
|
3
|
+
|
4
|
+
# Responsible for retrieving API gateway instances filtered by prefix
|
5
|
+
class ApiGatewayRetriever
|
6
|
+
|
7
|
+
def initialize(api_gateway)
|
8
|
+
@api_gateway = api_gateway
|
9
|
+
end
|
10
|
+
|
11
|
+
def retrieve(prefix)
|
12
|
+
apis = @api_gateway.list_all_apis
|
13
|
+
apis.select{ |item| item.name.to_s.downcase.start_with?(prefix) }
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
Aws.use_bundled_cert!
|
4
|
+
|
5
|
+
module AwsLogCleaner
|
6
|
+
|
7
|
+
# wrapper for aws credentials
|
8
|
+
class AwsCredentials
|
9
|
+
|
10
|
+
attr_reader :credentials, :region
|
11
|
+
|
12
|
+
def initialize(access_key = nil, secret = nil, region = nil)
|
13
|
+
key_id = access_key || ENV['AWS_ACCESS_KEY_ID']
|
14
|
+
secret = secret || ENV['AWS_SECRET_ACCESS_KEY']
|
15
|
+
@region = region || ENV['AWS_DEFAULT_REGION']
|
16
|
+
|
17
|
+
@credentials =
|
18
|
+
if key_id.nil? && secret.nil?
|
19
|
+
Aws::SharedCredentials.new
|
20
|
+
else
|
21
|
+
Aws::Credentials.new(key_id, secret)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module AwsLogCleaner
|
4
|
+
|
5
|
+
# Class responsible for interacting with AWS Cloudwatch
|
6
|
+
class CloudWatchLogs
|
7
|
+
|
8
|
+
# Required for Windows users.
|
9
|
+
Aws.use_bundled_cert!
|
10
|
+
|
11
|
+
def initialize(credentials)
|
12
|
+
@cloud_watch_logs = Aws::CloudWatchLogs::Client.new(
|
13
|
+
region: credentials.region,
|
14
|
+
credentials: credentials.credentials
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
def list_all_log_groups(token = nil)
|
19
|
+
log_groups = []
|
20
|
+
resp = describe_log_groups(token)
|
21
|
+
log_groups.concat(resp.log_groups)
|
22
|
+
|
23
|
+
if resp.next_token.to_s.empty?
|
24
|
+
log_groups
|
25
|
+
else
|
26
|
+
log_groups.concat(
|
27
|
+
list_all_log_groups(resp.next_token)
|
28
|
+
)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete_log_groups(log_group_names)
|
33
|
+
log_group_names.each do |name|
|
34
|
+
@cloud_watch_logs.delete_log_group(
|
35
|
+
log_group_name: name
|
36
|
+
)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def describe_log_groups(token)
|
43
|
+
if token.nil?
|
44
|
+
@cloud_watch_logs.describe_log_groups
|
45
|
+
else
|
46
|
+
@cloud_watch_logs.describe_log_groups(
|
47
|
+
next_token: token
|
48
|
+
)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require_relative 'cloud_watch_logs'
|
2
|
+
|
3
|
+
module AwsLogCleaner
|
4
|
+
|
5
|
+
# Responsible for cleaning log groups in AWS.
|
6
|
+
class LogGroupCleanerCommands
|
7
|
+
|
8
|
+
def initialize(cloud_watch_logs, api_retriever, log_group_filterer)
|
9
|
+
@cloud_watch_logs = cloud_watch_logs
|
10
|
+
@api_retriever = api_retriever
|
11
|
+
@log_group_filterer = log_group_filterer
|
12
|
+
end
|
13
|
+
|
14
|
+
def plan(text)
|
15
|
+
result(text)
|
16
|
+
end
|
17
|
+
|
18
|
+
def delete(text)
|
19
|
+
@cloud_watch_logs.delete_log_groups(log_group_names_containing(text))
|
20
|
+
result(text)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def log_group_names_containing(text)
|
26
|
+
if @log_group_names.nil?
|
27
|
+
@log_group_names = []
|
28
|
+
log_groups =
|
29
|
+
@log_group_filterer.filter_by_name_includes(text)
|
30
|
+
|
31
|
+
apis = @api_retriever.retrieve(text)
|
32
|
+
apis.each do |api|
|
33
|
+
log_groups.concat(
|
34
|
+
@log_group_filterer.filter_by_name_includes(api.id.to_s)
|
35
|
+
)
|
36
|
+
end
|
37
|
+
|
38
|
+
log_groups.each do |log_group|
|
39
|
+
@log_group_names.push(log_group.log_group_name)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
@log_group_names
|
44
|
+
end
|
45
|
+
|
46
|
+
def result(text)
|
47
|
+
to_delete = []
|
48
|
+
log_group_names_containing(text).each do |name|
|
49
|
+
to_delete.push("(-) #{name}")
|
50
|
+
end
|
51
|
+
to_delete
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
module AwsLogCleaner
|
3
|
+
|
4
|
+
class LogGroupFilterer
|
5
|
+
|
6
|
+
def initialize(cloud_watch_logs)
|
7
|
+
@cloud_watch_logs = cloud_watch_logs
|
8
|
+
end
|
9
|
+
|
10
|
+
def filter_by_name_includes(text)
|
11
|
+
log_groups = @cloud_watch_logs.list_all_log_groups
|
12
|
+
log_groups.select{ |item| item.log_group_name.to_s.downcase.include?(text) }
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
metadata
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: aws_log_cleaner
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- dbaker
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-06-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.13'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.13'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 3.6.0
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 3.6.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: aws-sdk
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 2.9.25
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 2.9.25
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: colorize
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.8.1
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.8.1
|
83
|
+
description: Cleans all logs in AWS that contain a search term in the log group name.
|
84
|
+
Also searches for api gateway instances that contain the search term and finds their
|
85
|
+
corresponding logs.
|
86
|
+
email:
|
87
|
+
- dbaker@vistaprint.com
|
88
|
+
executables:
|
89
|
+
- aws_log_cleaner
|
90
|
+
extensions: []
|
91
|
+
extra_rdoc_files: []
|
92
|
+
files:
|
93
|
+
- ".gitignore"
|
94
|
+
- ".rspec"
|
95
|
+
- Gemfile
|
96
|
+
- Gemfile.lock
|
97
|
+
- LICENSE
|
98
|
+
- README.md
|
99
|
+
- Rakefile
|
100
|
+
- aws_log_cleaner.gemspec
|
101
|
+
- bin/aws_log_cleaner
|
102
|
+
- bin/console
|
103
|
+
- bin/setup
|
104
|
+
- lib/aws_log_cleaner.rb
|
105
|
+
- lib/aws_log_cleaner/api_gateway.rb
|
106
|
+
- lib/aws_log_cleaner/api_gateway_retriever.rb
|
107
|
+
- lib/aws_log_cleaner/aws_credentials.rb
|
108
|
+
- lib/aws_log_cleaner/cloud_watch_logs.rb
|
109
|
+
- lib/aws_log_cleaner/log_group_cleaner_commands.rb
|
110
|
+
- lib/aws_log_cleaner/log_group_filterer.rb
|
111
|
+
- lib/aws_log_cleaner/version.rb
|
112
|
+
homepage: https://github.com/daniebker/aws_log_cleaner
|
113
|
+
licenses:
|
114
|
+
- Apache-2.0
|
115
|
+
metadata: {}
|
116
|
+
post_install_message:
|
117
|
+
rdoc_options: []
|
118
|
+
require_paths:
|
119
|
+
- lib
|
120
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '0'
|
130
|
+
requirements: []
|
131
|
+
rubyforge_project:
|
132
|
+
rubygems_version: 2.6.8
|
133
|
+
signing_key:
|
134
|
+
specification_version: 4
|
135
|
+
summary: Clean up lambda and API Gateway logs in AWS that contain a search term in
|
136
|
+
their log group name.
|
137
|
+
test_files: []
|