aws_log_cleaner 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +126 -0
- data/.rspec +1 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +137 -0
- data/LICENSE +13 -0
- data/README.md +59 -0
- data/Rakefile +13 -0
- data/aws_log_cleaner.gemspec +31 -0
- data/bin/aws_log_cleaner +69 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/aws_log_cleaner.rb +39 -0
- data/lib/aws_log_cleaner/api_gateway.rb +33 -0
- data/lib/aws_log_cleaner/api_gateway_retriever.rb +17 -0
- data/lib/aws_log_cleaner/aws_credentials.rb +25 -0
- data/lib/aws_log_cleaner/cloud_watch_logs.rb +53 -0
- data/lib/aws_log_cleaner/log_group_cleaner_commands.rb +56 -0
- data/lib/aws_log_cleaner/log_group_filterer.rb +17 -0
- data/lib/aws_log_cleaner/version.rb +3 -0
- metadata +137 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: e0253331366894b4b77ece9c5d7bc1fd3d9d652f
|
|
4
|
+
data.tar.gz: 00cd865249540f4268a55303f748e4a15e6a6127
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 3534b15480fb89f62168e5800c791303c26fa5fb54f8dd8a0a347637d91d39fe85d4798a2cdad8651c157253a90b2f048048c3fceed6b363cfc28b816d5dba52
|
|
7
|
+
data.tar.gz: 9fc84b6ade388df9e2449ad368990b4ee14254d26df212c5906e5085977aacf5bbcb99998818ec09fe763f2ee33c3cc7fb77a1fd820d3857211d266e99ab6fac
|
data/.gitignore
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
|
|
2
|
+
# Created by https://www.gitignore.io/api/ruby,visualstudiocode,rubymine
|
|
3
|
+
|
|
4
|
+
### Ruby ###
|
|
5
|
+
*.gem
|
|
6
|
+
*.rbc
|
|
7
|
+
/.config
|
|
8
|
+
/coverage/
|
|
9
|
+
/InstalledFiles
|
|
10
|
+
/pkg/
|
|
11
|
+
/spec/reports/
|
|
12
|
+
/spec/examples.txt
|
|
13
|
+
/test/tmp/
|
|
14
|
+
/test/version_tmp/
|
|
15
|
+
/tmp/
|
|
16
|
+
|
|
17
|
+
# Used by dotenv library to load environment variables.
|
|
18
|
+
# .env
|
|
19
|
+
|
|
20
|
+
## Specific to RubyMotion:
|
|
21
|
+
.dat*
|
|
22
|
+
.repl_history
|
|
23
|
+
build/
|
|
24
|
+
*.bridgesupport
|
|
25
|
+
build-iPhoneOS/
|
|
26
|
+
build-iPhoneSimulator/
|
|
27
|
+
|
|
28
|
+
## Specific to RubyMotion (use of CocoaPods):
|
|
29
|
+
#
|
|
30
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
|
31
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
|
32
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
|
33
|
+
#
|
|
34
|
+
# vendor/Pods/
|
|
35
|
+
|
|
36
|
+
## Documentation cache and generated files:
|
|
37
|
+
/.yardoc/
|
|
38
|
+
/_yardoc/
|
|
39
|
+
/doc/
|
|
40
|
+
/rdoc/
|
|
41
|
+
|
|
42
|
+
## Environment normalization:
|
|
43
|
+
/.bundle/
|
|
44
|
+
/vendor/bundle
|
|
45
|
+
/lib/bundler/man/
|
|
46
|
+
|
|
47
|
+
# for a library or gem, you might want to ignore these files since the code is
|
|
48
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
49
|
+
# Gemfile.lock
|
|
50
|
+
# .ruby-version
|
|
51
|
+
# .ruby-gemset
|
|
52
|
+
|
|
53
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
|
54
|
+
.rvmrc
|
|
55
|
+
|
|
56
|
+
### RubyMine ###
|
|
57
|
+
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
|
|
58
|
+
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
|
59
|
+
|
|
60
|
+
# User-specific stuff:
|
|
61
|
+
.idea/**/workspace.xml
|
|
62
|
+
.idea/**/tasks.xml
|
|
63
|
+
.idea/dictionaries
|
|
64
|
+
|
|
65
|
+
# Sensitive or high-churn files:
|
|
66
|
+
.idea/**/dataSources/
|
|
67
|
+
.idea/**/dataSources.ids
|
|
68
|
+
.idea/**/dataSources.xml
|
|
69
|
+
.idea/**/dataSources.local.xml
|
|
70
|
+
.idea/**/sqlDataSources.xml
|
|
71
|
+
.idea/**/dynamic.xml
|
|
72
|
+
.idea/**/uiDesigner.xml
|
|
73
|
+
|
|
74
|
+
# Gradle:
|
|
75
|
+
.idea/**/gradle.xml
|
|
76
|
+
.idea/**/libraries
|
|
77
|
+
|
|
78
|
+
# CMake
|
|
79
|
+
cmake-build-debug/
|
|
80
|
+
|
|
81
|
+
# Mongo Explorer plugin:
|
|
82
|
+
.idea/**/mongoSettings.xml
|
|
83
|
+
|
|
84
|
+
## File-based project format:
|
|
85
|
+
*.iws
|
|
86
|
+
|
|
87
|
+
## Plugin-specific files:
|
|
88
|
+
|
|
89
|
+
# IntelliJ
|
|
90
|
+
/out/
|
|
91
|
+
|
|
92
|
+
# mpeltonen/sbt-idea plugin
|
|
93
|
+
.idea_modules/
|
|
94
|
+
|
|
95
|
+
# JIRA plugin
|
|
96
|
+
atlassian-ide-plugin.xml
|
|
97
|
+
|
|
98
|
+
# Cursive Clojure plugin
|
|
99
|
+
.idea/replstate.xml
|
|
100
|
+
|
|
101
|
+
# Crashlytics plugin (for Android Studio and IntelliJ)
|
|
102
|
+
com_crashlytics_export_strings.xml
|
|
103
|
+
crashlytics.properties
|
|
104
|
+
crashlytics-build.properties
|
|
105
|
+
fabric.properties
|
|
106
|
+
|
|
107
|
+
### RubyMine Patch ###
|
|
108
|
+
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
|
|
109
|
+
|
|
110
|
+
.idea/
|
|
111
|
+
*.iml
|
|
112
|
+
modules.xml
|
|
113
|
+
.idea/misc.xml
|
|
114
|
+
*.ipr
|
|
115
|
+
|
|
116
|
+
# Sonarlint plugin
|
|
117
|
+
.idea/sonarlint
|
|
118
|
+
|
|
119
|
+
### VisualStudioCode ###
|
|
120
|
+
.vscode/*
|
|
121
|
+
!.vscode/settings.json
|
|
122
|
+
!.vscode/tasks.json
|
|
123
|
+
!.vscode/launch.json
|
|
124
|
+
!.vscode/extensions.json
|
|
125
|
+
|
|
126
|
+
# End of https://www.gitignore.io/api/ruby,visualstudiocode,rubymine
|
data/.rspec
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
--require spec_helper
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
GEM
|
|
2
|
+
remote: https://rubygems.org/
|
|
3
|
+
remote: http://vbuartifactory.vistaprint.svc/artifactory/api/gems/gems-release-local/
|
|
4
|
+
specs:
|
|
5
|
+
artifactory (2.2.1)
|
|
6
|
+
artifactory-client-mcp (0.0.9)
|
|
7
|
+
artifactory (~> 2.2.0)
|
|
8
|
+
vp_rake_core (~> 0.9.44)
|
|
9
|
+
asciidoctor (0.1.4)
|
|
10
|
+
aws-sdk (2.9.25)
|
|
11
|
+
aws-sdk-resources (= 2.9.25)
|
|
12
|
+
aws-sdk-core (2.9.25)
|
|
13
|
+
aws-sigv4 (~> 1.0)
|
|
14
|
+
jmespath (~> 1.0)
|
|
15
|
+
aws-sdk-resources (2.9.25)
|
|
16
|
+
aws-sdk-core (= 2.9.25)
|
|
17
|
+
aws-sigv4 (1.0.0)
|
|
18
|
+
colorize (0.8.1)
|
|
19
|
+
diff-lcs (1.3)
|
|
20
|
+
facter (2.4.6-x86-mingw32)
|
|
21
|
+
ffi (~> 1.9.5)
|
|
22
|
+
win32-dir (~> 0.4.8)
|
|
23
|
+
win32-security (~> 0.2.5)
|
|
24
|
+
win32console (~> 1.3.2)
|
|
25
|
+
faraday (0.9.2)
|
|
26
|
+
multipart-post (>= 1.2, < 3)
|
|
27
|
+
faraday_middleware (0.10.1)
|
|
28
|
+
faraday (>= 0.7.4, < 1.0)
|
|
29
|
+
fast_gettext (1.1.0)
|
|
30
|
+
ffi (1.9.18-x86-mingw32)
|
|
31
|
+
ffi-win32-extensions (1.0.3)
|
|
32
|
+
ffi
|
|
33
|
+
gettext (3.2.2)
|
|
34
|
+
locale (>= 2.0.5)
|
|
35
|
+
text (>= 1.3.0)
|
|
36
|
+
gettext-setup (0.25)
|
|
37
|
+
fast_gettext (~> 1.1.0)
|
|
38
|
+
gettext (>= 3.0.2)
|
|
39
|
+
locale
|
|
40
|
+
hiera (1.3.4)
|
|
41
|
+
json_pure
|
|
42
|
+
jmespath (1.3.1)
|
|
43
|
+
json_pure (1.8.6)
|
|
44
|
+
librarian-puppet (2.2.3)
|
|
45
|
+
librarianp (>= 0.6.3)
|
|
46
|
+
puppet_forge (~> 2.1)
|
|
47
|
+
rsync
|
|
48
|
+
librarianp (0.6.3)
|
|
49
|
+
thor (~> 0.15)
|
|
50
|
+
locale (2.1.2)
|
|
51
|
+
minitar (0.5.4)
|
|
52
|
+
multi_json (1.12.1)
|
|
53
|
+
multipart-post (2.0.0)
|
|
54
|
+
puppet (3.7.5-x86-mingw32)
|
|
55
|
+
facter (> 1.6, < 3)
|
|
56
|
+
ffi (~> 1.9.5)
|
|
57
|
+
hiera (~> 1.0)
|
|
58
|
+
json_pure
|
|
59
|
+
minitar (~> 0.5.4)
|
|
60
|
+
win32-dir (~> 0.4.9)
|
|
61
|
+
win32-eventlog (~> 0.6.1)
|
|
62
|
+
win32-process (~> 0.7.4)
|
|
63
|
+
win32-security (~> 0.2.5)
|
|
64
|
+
win32-service (~> 0.8.6)
|
|
65
|
+
win32console (= 1.3.2)
|
|
66
|
+
puppet_forge (2.2.4)
|
|
67
|
+
faraday (~> 0.9.0)
|
|
68
|
+
faraday_middleware (>= 0.9.0, < 0.11.0)
|
|
69
|
+
gettext-setup (~> 0.11)
|
|
70
|
+
minitar
|
|
71
|
+
semantic_puppet (~> 0.1.0)
|
|
72
|
+
rake (12.0.0)
|
|
73
|
+
rspec (3.6.0)
|
|
74
|
+
rspec-core (~> 3.6.0)
|
|
75
|
+
rspec-expectations (~> 3.6.0)
|
|
76
|
+
rspec-mocks (~> 3.6.0)
|
|
77
|
+
rspec-core (3.6.0)
|
|
78
|
+
rspec-support (~> 3.6.0)
|
|
79
|
+
rspec-expectations (3.6.0)
|
|
80
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
81
|
+
rspec-support (~> 3.6.0)
|
|
82
|
+
rspec-mocks (3.6.0)
|
|
83
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
84
|
+
rspec-support (~> 3.6.0)
|
|
85
|
+
rspec-support (3.6.0)
|
|
86
|
+
rsync (1.0.9)
|
|
87
|
+
rubyzip (1.1.7)
|
|
88
|
+
semantic_puppet (0.1.4)
|
|
89
|
+
gettext-setup (>= 0.3)
|
|
90
|
+
simplecov (0.7.1)
|
|
91
|
+
multi_json (~> 1.0)
|
|
92
|
+
simplecov-html (~> 0.7.1)
|
|
93
|
+
simplecov-html (0.7.1)
|
|
94
|
+
text (1.3.1)
|
|
95
|
+
thor (0.19.4)
|
|
96
|
+
vp_devkit_dotnet (1.0.45)
|
|
97
|
+
artifactory-client-mcp (~> 0.0.1)
|
|
98
|
+
asciidoctor (~> 0.1.4)
|
|
99
|
+
json_pure (~> 1.7)
|
|
100
|
+
vp_ivy (~> 1.0.1)
|
|
101
|
+
vp_puppet (~> 0.9.25)
|
|
102
|
+
vp_rake_core (~> 0.9.44)
|
|
103
|
+
vp_ivy (1.0.11)
|
|
104
|
+
rubyzip (~> 1.1.6)
|
|
105
|
+
vp_rake_core (~> 0.9.44)
|
|
106
|
+
vp_puppet (0.9.36)
|
|
107
|
+
ffi (~> 1.9.6)
|
|
108
|
+
librarian-puppet (>= 1.3.2, < 3.0)
|
|
109
|
+
puppet (= 3.7.5)
|
|
110
|
+
vp_rake_core (~> 0.9.52)
|
|
111
|
+
vp_rake_core (0.9.56)
|
|
112
|
+
rake
|
|
113
|
+
simplecov (= 0.7.1)
|
|
114
|
+
win32-dir (0.4.9)
|
|
115
|
+
ffi (>= 1.0.0)
|
|
116
|
+
win32-eventlog (0.6.7)
|
|
117
|
+
ffi
|
|
118
|
+
win32-process (0.7.5)
|
|
119
|
+
ffi (>= 1.0.0)
|
|
120
|
+
win32-security (0.2.5)
|
|
121
|
+
ffi
|
|
122
|
+
win32-service (0.8.10)
|
|
123
|
+
ffi
|
|
124
|
+
ffi-win32-extensions
|
|
125
|
+
win32console (1.3.2-x86-mingw32)
|
|
126
|
+
|
|
127
|
+
PLATFORMS
|
|
128
|
+
x86-mingw32
|
|
129
|
+
|
|
130
|
+
DEPENDENCIES
|
|
131
|
+
aws-sdk (~> 2.9.25)
|
|
132
|
+
colorize (~> 0.8.1)
|
|
133
|
+
rspec
|
|
134
|
+
vp_devkit_dotnet
|
|
135
|
+
|
|
136
|
+
BUNDLED WITH
|
|
137
|
+
1.13.6
|
data/LICENSE
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright 2017 Cimpress
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# AwsLogCleaner
|
|
2
|
+
|
|
3
|
+
A gem to clean up aws lambda and Api Gateway log groups that contain a search term.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem 'aws_log_cleaner'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
$ bundle
|
|
16
|
+
|
|
17
|
+
Or install it yourself as:
|
|
18
|
+
|
|
19
|
+
$ gem install aws_log_cleaner
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
### Requirements
|
|
24
|
+
|
|
25
|
+
* `AWS_SECRET_KEY` & `AWS_ACCESS_KEY` env variables must be set.
|
|
26
|
+
OR
|
|
27
|
+
* A credentials file must be present in `~\.aws`
|
|
28
|
+
|
|
29
|
+
The application defaults to `eu-west-1` but this can be overridden using `AWS_DEFAULT_REGION` environment variable.
|
|
30
|
+
|
|
31
|
+
### From the command line
|
|
32
|
+
|
|
33
|
+
* Get help by using:
|
|
34
|
+
|
|
35
|
+
`aws_log_cleaner -h`
|
|
36
|
+
|
|
37
|
+
* Run the plan command in eu-west-1 where prefix like some_text.
|
|
38
|
+
|
|
39
|
+
`aws_log_cleaner -p -l 'some_text' -r 'eu-west-1' -k [ACCESS_KEY] -s [SECRET]`
|
|
40
|
+
|
|
41
|
+
## Code
|
|
42
|
+
|
|
43
|
+
Initialise a new LogCleaner
|
|
44
|
+
|
|
45
|
+
`log_cleaner = AwsLogCleaner::AwsLogCleaner.new`
|
|
46
|
+
|
|
47
|
+
Run a plan
|
|
48
|
+
|
|
49
|
+
`log_cleaner.plan('some_text')`
|
|
50
|
+
|
|
51
|
+
Or run a delete
|
|
52
|
+
|
|
53
|
+
`log_cleaner.delete('some_text')`
|
|
54
|
+
|
|
55
|
+
## Development
|
|
56
|
+
|
|
57
|
+
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
58
|
+
|
|
59
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
data/Rakefile
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
|
+
require 'aws_log_cleaner/version'
|
|
6
|
+
|
|
7
|
+
Gem::Specification.new do |spec|
|
|
8
|
+
spec.name = 'aws_log_cleaner'
|
|
9
|
+
spec.version = AwsLogCleaner::VERSION
|
|
10
|
+
spec.authors = ['dbaker']
|
|
11
|
+
spec.email = ['dbaker@vistaprint.com']
|
|
12
|
+
spec.summary = 'Clean up lambda and API Gateway logs in AWS that contain a search term ' \
|
|
13
|
+
'in their log group name.'
|
|
14
|
+
spec.description = 'Cleans all logs in AWS that contain a search term in the log group name. ' \
|
|
15
|
+
'Also searches for api gateway instances that contain the search term ' \
|
|
16
|
+
'and finds their corresponding logs.'
|
|
17
|
+
spec.homepage = 'https://github.com/daniebker/aws_log_cleaner'
|
|
18
|
+
spec.license = 'Apache-2.0'
|
|
19
|
+
|
|
20
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
21
|
+
f.match(%r{^(test|spec|features)/})
|
|
22
|
+
end
|
|
23
|
+
spec.executables << 'aws_log_cleaner'
|
|
24
|
+
|
|
25
|
+
spec.add_development_dependency 'bundler', '~> 1.13'
|
|
26
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.6.0'
|
|
28
|
+
|
|
29
|
+
spec.add_dependency 'aws-sdk', '~> 2.9.25'
|
|
30
|
+
spec.add_dependency 'colorize', '~> 0.8.1'
|
|
31
|
+
end
|
data/bin/aws_log_cleaner
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'aws_log_cleaner'
|
|
4
|
+
require 'colorize'
|
|
5
|
+
|
|
6
|
+
USAGE = <<ENDUSAGE.freeze
|
|
7
|
+
Usage:
|
|
8
|
+
aws_log_cleaner [-h] ([-p] or [-a]) [-r region] [-l like]
|
|
9
|
+
ENDUSAGE
|
|
10
|
+
|
|
11
|
+
HELP = <<ENDHELP.freeze
|
|
12
|
+
-h, --help Show this help.
|
|
13
|
+
-p, --plan Runs the plan command.
|
|
14
|
+
-a, --apply Applys the deletion. Preview what will be deleted with [-p]
|
|
15
|
+
-l, --like The search term returned logs should contain.
|
|
16
|
+
-r, --region The region to run in.
|
|
17
|
+
-k, --access_key The access key to use for AWS.
|
|
18
|
+
-s, --secret The secret to use for AWS.
|
|
19
|
+
ENDHELP
|
|
20
|
+
|
|
21
|
+
args = { region: nil, access_key: nil, secret: nil }
|
|
22
|
+
unflagged_args = []
|
|
23
|
+
next_arg = unflagged_args.first
|
|
24
|
+
ARGV.each do |arg|
|
|
25
|
+
case arg
|
|
26
|
+
when '-h', '--help' then args[:help] = true
|
|
27
|
+
when '-p', '--plan' then args[:plan] = true
|
|
28
|
+
when '-a', '--apply' then args[:delete] = true
|
|
29
|
+
when '-l', '--like' then next_arg = :like
|
|
30
|
+
when '-r', '--region' then next_arg = :region
|
|
31
|
+
when '-k', '--access_key' then next_arg = :access_key
|
|
32
|
+
when '-s', '--secret' then next_arg = :secret
|
|
33
|
+
else
|
|
34
|
+
if next_arg
|
|
35
|
+
args[next_arg] = arg
|
|
36
|
+
unflagged_args.delete(next_arg)
|
|
37
|
+
end
|
|
38
|
+
next_arg = unflagged_args.first
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
if args[:help] || (!args[:plan] && !args[:delete])
|
|
43
|
+
puts USAGE
|
|
44
|
+
puts HELP
|
|
45
|
+
exit
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
like = args[:like].nil? ? '' : args[:like]
|
|
49
|
+
|
|
50
|
+
log_cleaner = \
|
|
51
|
+
AwsLogCleaner::AwsLogCleaner.new(args[:access_key], args[:secret], args[:region])
|
|
52
|
+
|
|
53
|
+
if args[:plan]
|
|
54
|
+
puts "== PLAN MODE ==\n\n"
|
|
55
|
+
|
|
56
|
+
result = log_cleaner.plan(like)
|
|
57
|
+
|
|
58
|
+
puts "Found #{result.count} item(s) to remove\n\n"
|
|
59
|
+
elsif args[:delete] && !args[:plan]
|
|
60
|
+
puts "== DELETE MODE ==\n\n"
|
|
61
|
+
|
|
62
|
+
result = log_cleaner.delete(like)
|
|
63
|
+
|
|
64
|
+
puts "Deleted #{result.count} item(s)\n\n"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
result.each do |item|
|
|
68
|
+
puts item.to_s.red
|
|
69
|
+
end
|
data/bin/console
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "aws_log_cleaner"
|
|
5
|
+
|
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
8
|
+
|
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
10
|
+
# require "pry"
|
|
11
|
+
# Pry.start
|
|
12
|
+
|
|
13
|
+
require "irb"
|
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
require_relative 'aws_log_cleaner/cloud_watch_logs'
|
|
2
|
+
require_relative 'aws_log_cleaner/log_group_cleaner_commands'
|
|
3
|
+
require_relative 'aws_log_cleaner/api_gateway'
|
|
4
|
+
require_relative 'aws_log_cleaner/api_gateway_retriever'
|
|
5
|
+
require_relative 'aws_log_cleaner/log_group_filterer'
|
|
6
|
+
require_relative 'aws_log_cleaner/aws_credentials'
|
|
7
|
+
require_relative 'aws_log_cleaner/version'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
module AwsLogCleaner
|
|
11
|
+
|
|
12
|
+
# Application wrapper
|
|
13
|
+
class AwsLogCleaner
|
|
14
|
+
|
|
15
|
+
def initialize(access_key = nil, secret = nil, region = nil)
|
|
16
|
+
credentials = AwsCredentials.new(access_key, secret, region)
|
|
17
|
+
cloud_watch_logs = CloudWatchLogs.new(credentials)
|
|
18
|
+
api_gateway = ApiGateway.new(credentials)
|
|
19
|
+
|
|
20
|
+
api_retriever = ApiGatewayRetriever.new(api_gateway)
|
|
21
|
+
log_group_filterer = LogGroupFilterer.new(cloud_watch_logs)
|
|
22
|
+
|
|
23
|
+
@log_cleaner = LogGroupCleanerCommands.new(
|
|
24
|
+
cloud_watch_logs,
|
|
25
|
+
api_retriever,
|
|
26
|
+
log_group_filterer
|
|
27
|
+
)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def plan(like)
|
|
31
|
+
@log_cleaner.plan(like)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def delete(like)
|
|
35
|
+
@log_cleaner.delete(like)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
require 'aws-sdk'
|
|
2
|
+
|
|
3
|
+
Aws.use_bundled_cert!
|
|
4
|
+
|
|
5
|
+
module AwsLogCleaner
|
|
6
|
+
|
|
7
|
+
# Class responsible for interacting with the Aws
|
|
8
|
+
# ApiGatewayClient
|
|
9
|
+
class ApiGateway
|
|
10
|
+
|
|
11
|
+
def initialize(credentials)
|
|
12
|
+
@api_client = Aws::APIGateway::Client.new(
|
|
13
|
+
region: credentials.region,
|
|
14
|
+
credentials: credentials.credentials
|
|
15
|
+
)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def list_all_apis
|
|
19
|
+
Enumerator.new do |enum|
|
|
20
|
+
request = { limit: 25 }
|
|
21
|
+
loop do
|
|
22
|
+
response = @api_client.get_rest_apis(request)
|
|
23
|
+
response.items.each do |item|
|
|
24
|
+
enum.yield item
|
|
25
|
+
end
|
|
26
|
+
break if response.position.nil?
|
|
27
|
+
request[:position] = response.position
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
|
|
2
|
+
module AwsLogCleaner
|
|
3
|
+
|
|
4
|
+
# Responsible for retrieving API gateway instances filtered by prefix
|
|
5
|
+
class ApiGatewayRetriever
|
|
6
|
+
|
|
7
|
+
def initialize(api_gateway)
|
|
8
|
+
@api_gateway = api_gateway
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def retrieve(prefix)
|
|
12
|
+
apis = @api_gateway.list_all_apis
|
|
13
|
+
apis.select{ |item| item.name.to_s.downcase.start_with?(prefix) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
require 'aws-sdk'
|
|
2
|
+
|
|
3
|
+
Aws.use_bundled_cert!
|
|
4
|
+
|
|
5
|
+
module AwsLogCleaner
|
|
6
|
+
|
|
7
|
+
# wrapper for aws credentials
|
|
8
|
+
class AwsCredentials
|
|
9
|
+
|
|
10
|
+
attr_reader :credentials, :region
|
|
11
|
+
|
|
12
|
+
def initialize(access_key = nil, secret = nil, region = nil)
|
|
13
|
+
key_id = access_key || ENV['AWS_ACCESS_KEY_ID']
|
|
14
|
+
secret = secret || ENV['AWS_SECRET_ACCESS_KEY']
|
|
15
|
+
@region = region || ENV['AWS_DEFAULT_REGION']
|
|
16
|
+
|
|
17
|
+
@credentials =
|
|
18
|
+
if key_id.nil? && secret.nil?
|
|
19
|
+
Aws::SharedCredentials.new
|
|
20
|
+
else
|
|
21
|
+
Aws::Credentials.new(key_id, secret)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
require 'aws-sdk'
|
|
2
|
+
|
|
3
|
+
module AwsLogCleaner
|
|
4
|
+
|
|
5
|
+
# Class responsible for interacting with AWS Cloudwatch
|
|
6
|
+
class CloudWatchLogs
|
|
7
|
+
|
|
8
|
+
# Required for Windows users.
|
|
9
|
+
Aws.use_bundled_cert!
|
|
10
|
+
|
|
11
|
+
def initialize(credentials)
|
|
12
|
+
@cloud_watch_logs = Aws::CloudWatchLogs::Client.new(
|
|
13
|
+
region: credentials.region,
|
|
14
|
+
credentials: credentials.credentials
|
|
15
|
+
)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def list_all_log_groups(token = nil)
|
|
19
|
+
log_groups = []
|
|
20
|
+
resp = describe_log_groups(token)
|
|
21
|
+
log_groups.concat(resp.log_groups)
|
|
22
|
+
|
|
23
|
+
if resp.next_token.to_s.empty?
|
|
24
|
+
log_groups
|
|
25
|
+
else
|
|
26
|
+
log_groups.concat(
|
|
27
|
+
list_all_log_groups(resp.next_token)
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def delete_log_groups(log_group_names)
|
|
33
|
+
log_group_names.each do |name|
|
|
34
|
+
@cloud_watch_logs.delete_log_group(
|
|
35
|
+
log_group_name: name
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def describe_log_groups(token)
|
|
43
|
+
if token.nil?
|
|
44
|
+
@cloud_watch_logs.describe_log_groups
|
|
45
|
+
else
|
|
46
|
+
@cloud_watch_logs.describe_log_groups(
|
|
47
|
+
next_token: token
|
|
48
|
+
)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
require_relative 'cloud_watch_logs'
|
|
2
|
+
|
|
3
|
+
module AwsLogCleaner
|
|
4
|
+
|
|
5
|
+
# Responsible for cleaning log groups in AWS.
|
|
6
|
+
class LogGroupCleanerCommands
|
|
7
|
+
|
|
8
|
+
def initialize(cloud_watch_logs, api_retriever, log_group_filterer)
|
|
9
|
+
@cloud_watch_logs = cloud_watch_logs
|
|
10
|
+
@api_retriever = api_retriever
|
|
11
|
+
@log_group_filterer = log_group_filterer
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def plan(text)
|
|
15
|
+
result(text)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def delete(text)
|
|
19
|
+
@cloud_watch_logs.delete_log_groups(log_group_names_containing(text))
|
|
20
|
+
result(text)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def log_group_names_containing(text)
|
|
26
|
+
if @log_group_names.nil?
|
|
27
|
+
@log_group_names = []
|
|
28
|
+
log_groups =
|
|
29
|
+
@log_group_filterer.filter_by_name_includes(text)
|
|
30
|
+
|
|
31
|
+
apis = @api_retriever.retrieve(text)
|
|
32
|
+
apis.each do |api|
|
|
33
|
+
log_groups.concat(
|
|
34
|
+
@log_group_filterer.filter_by_name_includes(api.id.to_s)
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
log_groups.each do |log_group|
|
|
39
|
+
@log_group_names.push(log_group.log_group_name)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
@log_group_names
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def result(text)
|
|
47
|
+
to_delete = []
|
|
48
|
+
log_group_names_containing(text).each do |name|
|
|
49
|
+
to_delete.push("(-) #{name}")
|
|
50
|
+
end
|
|
51
|
+
to_delete
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
|
|
2
|
+
module AwsLogCleaner
|
|
3
|
+
|
|
4
|
+
class LogGroupFilterer
|
|
5
|
+
|
|
6
|
+
def initialize(cloud_watch_logs)
|
|
7
|
+
@cloud_watch_logs = cloud_watch_logs
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def filter_by_name_includes(text)
|
|
11
|
+
log_groups = @cloud_watch_logs.list_all_log_groups
|
|
12
|
+
log_groups.select{ |item| item.log_group_name.to_s.downcase.include?(text) }
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: aws_log_cleaner
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.3
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- dbaker
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2017-06-14 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: bundler
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '1.13'
|
|
20
|
+
type: :development
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '1.13'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: rake
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '10.0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '10.0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: rspec
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 3.6.0
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 3.6.0
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: aws-sdk
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 2.9.25
|
|
62
|
+
type: :runtime
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 2.9.25
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: colorize
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: 0.8.1
|
|
76
|
+
type: :runtime
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: 0.8.1
|
|
83
|
+
description: Cleans all logs in AWS that contain a search term in the log group name.
|
|
84
|
+
Also searches for api gateway instances that contain the search term and finds their
|
|
85
|
+
corresponding logs.
|
|
86
|
+
email:
|
|
87
|
+
- dbaker@vistaprint.com
|
|
88
|
+
executables:
|
|
89
|
+
- aws_log_cleaner
|
|
90
|
+
extensions: []
|
|
91
|
+
extra_rdoc_files: []
|
|
92
|
+
files:
|
|
93
|
+
- ".gitignore"
|
|
94
|
+
- ".rspec"
|
|
95
|
+
- Gemfile
|
|
96
|
+
- Gemfile.lock
|
|
97
|
+
- LICENSE
|
|
98
|
+
- README.md
|
|
99
|
+
- Rakefile
|
|
100
|
+
- aws_log_cleaner.gemspec
|
|
101
|
+
- bin/aws_log_cleaner
|
|
102
|
+
- bin/console
|
|
103
|
+
- bin/setup
|
|
104
|
+
- lib/aws_log_cleaner.rb
|
|
105
|
+
- lib/aws_log_cleaner/api_gateway.rb
|
|
106
|
+
- lib/aws_log_cleaner/api_gateway_retriever.rb
|
|
107
|
+
- lib/aws_log_cleaner/aws_credentials.rb
|
|
108
|
+
- lib/aws_log_cleaner/cloud_watch_logs.rb
|
|
109
|
+
- lib/aws_log_cleaner/log_group_cleaner_commands.rb
|
|
110
|
+
- lib/aws_log_cleaner/log_group_filterer.rb
|
|
111
|
+
- lib/aws_log_cleaner/version.rb
|
|
112
|
+
homepage: https://github.com/daniebker/aws_log_cleaner
|
|
113
|
+
licenses:
|
|
114
|
+
- Apache-2.0
|
|
115
|
+
metadata: {}
|
|
116
|
+
post_install_message:
|
|
117
|
+
rdoc_options: []
|
|
118
|
+
require_paths:
|
|
119
|
+
- lib
|
|
120
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - ">="
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '0'
|
|
125
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
126
|
+
requirements:
|
|
127
|
+
- - ">="
|
|
128
|
+
- !ruby/object:Gem::Version
|
|
129
|
+
version: '0'
|
|
130
|
+
requirements: []
|
|
131
|
+
rubyforge_project:
|
|
132
|
+
rubygems_version: 2.6.8
|
|
133
|
+
signing_key:
|
|
134
|
+
specification_version: 4
|
|
135
|
+
summary: Clean up lambda and API Gateway logs in AWS that contain a search term in
|
|
136
|
+
their log group name.
|
|
137
|
+
test_files: []
|