volume_sweeper 1.0.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -2
- data/lib/volume_sweeper/cli.rb +1 -0
- data/lib/volume_sweeper/core.rb +1 -0
- data/lib/volume_sweeper/providers/aws.rb +78 -4
- data/lib/volume_sweeper/providers/base.rb +1 -0
- data/lib/volume_sweeper/providers/oci.rb +1 -1
- data/lib/volume_sweeper/utils/notification.rb +30 -0
- data/lib/volume_sweeper/utils/notification_formatter.rb +3 -3
- data/lib/volume_sweeper/version.rb +1 -1
- metadata +33 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 69061fddbb0ab861a6e3061356f675e420dc37b899c6e997555be5f1f55bcf13
|
|
4
|
+
data.tar.gz: 51c187ffb879566b2faf1d4e71e8fb6651d9bee3e4bccd4240348f8d7de2139e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8e089dee93af6b64d99c02328087732ef139bc3b91fae9e0b3ab3a7c14b895e8060e2e0c2b1942099cf3e26bfc482cc1c457c094eaee96a37ccc6eaa5908c5e9
|
|
7
|
+
data.tar.gz: 29e41d99d2f45464d6cb281a4821e06dfc2454886c1355f8ae30103dfca7ad3d34ecc1fc7a524f87add8f5af859062fd23f9cda2d463b0ad535ccc1ee71c0948
|
data/README.md
CHANGED
|
@@ -7,7 +7,7 @@ A tool to scan and clean cloud infrastruture for unattached block volumes withou
|
|
|
7
7
|
## Supported Clouds
|
|
8
8
|
|
|
9
9
|
- [x] OCI
|
|
10
|
-
- [
|
|
10
|
+
- [x] AWS.
|
|
11
11
|
- [ ] GCP.
|
|
12
12
|
|
|
13
13
|
## Supported Kubernetes
|
|
@@ -31,7 +31,7 @@ $ gem install volume_sweeper
|
|
|
31
31
|
To scan and generate a report:
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
|
-
volume_sweeper --account-id <ID> --cloud aws|oci
|
|
34
|
+
volume_sweeper --account-id <ID> --cloud aws|oci --region <region>
|
|
35
35
|
```
|
|
36
36
|
|
|
37
37
|
To apply deletion for unattached block volumes:
|
|
@@ -40,6 +40,12 @@ To apply deletion for unattached block volumes:
|
|
|
40
40
|
volume_sweeper --mode delete
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
For all options:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
volume_sweeper -h
|
|
47
|
+
```
|
|
48
|
+
|
|
43
49
|
## Contributing
|
|
44
50
|
|
|
45
51
|
Bug reports and pull requests are welcome on GitHub at https://github.com/abarrak/volume_sweeper.
|
data/lib/volume_sweeper/cli.rb
CHANGED
|
@@ -29,6 +29,7 @@ module VolumeSweeper
|
|
|
29
29
|
opt.on('-m', '--mode [MODE]', 'The run modes: either audit, or delete.') { |o| options.mode = o }
|
|
30
30
|
opt.on('-c', '--cloud [CLOUD]', 'Supported clouds: aws, oci.') { |o| options.cloud = o }
|
|
31
31
|
opt.on('-f', '--config-path [PATH]', 'The file location for cloud config file') { |o| options.config_path = o }
|
|
32
|
+
opt.on('-s', '--creds-path [PATH]', 'The file location for cloud crednetials file') { |o| options.creds_path = o }
|
|
32
33
|
opt.on('-r', '--region [REGION]', 'The provider region of the account.') { |o| options.region = o }
|
|
33
34
|
opt.on('-a', '--account-id [Id]', 'The account or compartment Id.') { |o| options.account_id = o }
|
|
34
35
|
opt.on('-d', '--released-since [DAYS]', 'Volumes threshold duration') { |o| options.released_in_days = o }
|
data/lib/volume_sweeper/core.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'active_support/core_ext/object/blank'
|
|
2
|
+
require 'aws-sdk-ec2'
|
|
2
3
|
require_relative 'base'
|
|
3
4
|
require_relative '../utils/log'
|
|
4
5
|
|
|
@@ -6,26 +7,80 @@ module VolumeSweeper
|
|
|
6
7
|
module Providers
|
|
7
8
|
|
|
8
9
|
class Aws < Base
|
|
10
|
+
|
|
9
11
|
DEFAULT_REGION = 'us-west-2'
|
|
12
|
+
DEFAULT_CONFIF_PATH = '~/.aws/config'
|
|
13
|
+
DEFAULT_CREDS_PATH = '~/.aws/credentials'
|
|
14
|
+
|
|
15
|
+
BASE_CONSOLE_URL = "console.aws.amazon.com/ec2/home"
|
|
16
|
+
VOLUME_ATTRS = %i{ volume_id displayName state size attachments create_time availability_zone tags }
|
|
10
17
|
|
|
11
18
|
def initialize config_path: nil, region: nil, mode: :audit, **kwargs
|
|
12
19
|
super
|
|
13
20
|
@region ||= DEFAULT_REGION
|
|
21
|
+
set_console_base_url
|
|
14
22
|
validate_attrs
|
|
23
|
+
prepare_config config_path, kwargs[:creds_path]
|
|
15
24
|
end
|
|
16
25
|
|
|
17
26
|
def scan_block_volumes
|
|
18
|
-
|
|
27
|
+
volumes = Array.new
|
|
28
|
+
next_token = nil
|
|
29
|
+
opts = { max_results: 200 }
|
|
30
|
+
|
|
31
|
+
run_api_call do |client|
|
|
32
|
+
loop do
|
|
33
|
+
response = client.describe_volumes opts.merge(next_token: next_token)
|
|
34
|
+
response&.volumes&.map do |v|
|
|
35
|
+
volumes << v.to_hash.compact.slice(*VOLUME_ATTRS).transform_keys(volume_id: :id)
|
|
36
|
+
end
|
|
37
|
+
break if response.nil? || response.next_token.nil?
|
|
38
|
+
next_token = response.next_token
|
|
39
|
+
sleep 2
|
|
40
|
+
end
|
|
41
|
+
@log.msg "aws: collected #{volumes.size} block volumes from the account."
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
@log.msg "aws: filtering out any block volume with an active attachment."
|
|
45
|
+
result = volumes&.reject { |v| v[:state] != 'available' || v[:attachments]&.count > 0 } || []
|
|
46
|
+
|
|
47
|
+
@log.msg "aws: found #{result.count} unattached block volumes."
|
|
48
|
+
[volumes.size, result]
|
|
19
49
|
end
|
|
20
50
|
|
|
21
51
|
def delete_block_volumes ids_list
|
|
22
|
-
|
|
23
|
-
|
|
52
|
+
@log.msg "aws: #{ids_list&.count || 0} block volumes are eligible for cleanup."
|
|
53
|
+
return if ids_list.blank?
|
|
54
|
+
|
|
55
|
+
unless @run_mode == :delete
|
|
56
|
+
@log.msg "aws: running in :#{@run_mode} mode, exiting without delete operations."
|
|
57
|
+
return
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
@log.msg "aws: unused volume clean-up operation started."
|
|
61
|
+
|
|
62
|
+
ids_list.each do |id|
|
|
63
|
+
@log.msg "aws: deleting block volume #{id} .."
|
|
64
|
+
run_api_call do |client|
|
|
65
|
+
output = client.delete_volume({ volume_id: id.to_s })
|
|
66
|
+
if output&.successful?
|
|
67
|
+
@log.msg "aws: block volume #{id} is deleted successfully."
|
|
68
|
+
else
|
|
69
|
+
@log.msg "aws: block volume #{id} has failed."
|
|
70
|
+
end
|
|
71
|
+
sleep 2.5
|
|
72
|
+
end
|
|
73
|
+
end
|
|
24
74
|
end
|
|
25
75
|
|
|
26
76
|
private
|
|
27
77
|
|
|
28
|
-
def prepare_config
|
|
78
|
+
def prepare_config config_path, creds_path
|
|
79
|
+
# SDK automtically picks up config and credentails files in the
|
|
80
|
+
# default place for various methods.
|
|
81
|
+
@config_location ||= DEFAULT_CONFIF_PATH
|
|
82
|
+
@creds_location ||= DEFAULT_CREDS_PATH
|
|
83
|
+
::Aws.config.update({ region: @region })
|
|
29
84
|
end
|
|
30
85
|
|
|
31
86
|
def validate_attrs
|
|
@@ -33,6 +88,25 @@ module VolumeSweeper
|
|
|
33
88
|
@log.msg "provider error: aws account id is not assigned", level: :error
|
|
34
89
|
exit 1
|
|
35
90
|
end
|
|
91
|
+
|
|
92
|
+
def run_api_call
|
|
93
|
+
current_tries = 0
|
|
94
|
+
ec2_client = ::Aws::EC2::Client.new(region: @region, account_id: @account_id)
|
|
95
|
+
|
|
96
|
+
yield ec2_client if block_given?
|
|
97
|
+
|
|
98
|
+
rescue ::Aws::EC2::Errors::ServiceError => err
|
|
99
|
+
# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EBS/Errors.html
|
|
100
|
+
@log.msg errr&.context&.data, level: :error
|
|
101
|
+
raise if err&.code.to_s != '304'
|
|
102
|
+
rescue StandardError => err
|
|
103
|
+
@log.msg err, level: :error
|
|
104
|
+
raise
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def set_console_base_url
|
|
108
|
+
@base_link = "https://#{@region}.#{BASE_CONSOLE_URL}?region=#{@region}#VolumeDetails:volumeId="
|
|
109
|
+
end
|
|
36
110
|
end
|
|
37
111
|
|
|
38
112
|
end
|
|
@@ -7,6 +7,7 @@ module VolumeSweeper
|
|
|
7
7
|
def initialize **kwargs
|
|
8
8
|
@run_mode = kwargs[:mode]&.to_sym || :audit
|
|
9
9
|
@config_location = kwargs[:config_path]
|
|
10
|
+
@creds_location = kwargs[:creds_location]
|
|
10
11
|
@account_id = kwargs[:account_id]
|
|
11
12
|
@compartment_id = kwargs[:account_id]
|
|
12
13
|
@region = kwargs[:region]
|
|
@@ -19,7 +19,7 @@ module VolumeSweeper
|
|
|
19
19
|
def initialize config_path: nil, region: nil, mode: :audit, **kwargs
|
|
20
20
|
super
|
|
21
21
|
@region ||= DEFAULT_REGION
|
|
22
|
-
@base_link = "https://cloud.oracle.com/block-storage/volumes"
|
|
22
|
+
@base_link = "https://cloud.oracle.com/block-storage/volumes/"
|
|
23
23
|
validate_attrs
|
|
24
24
|
end
|
|
25
25
|
|
|
@@ -113,20 +113,50 @@ module VolumeSweeper
|
|
|
113
113
|
}
|
|
114
114
|
.content {
|
|
115
115
|
margin: 20px 20px 0 20px;
|
|
116
|
+
margin: 0 20px 0 20px;
|
|
116
117
|
padding: 10px;
|
|
117
118
|
background-color: white;
|
|
118
119
|
}
|
|
120
|
+
.header {
|
|
121
|
+
margin: 20px 20px 0 20px;
|
|
122
|
+
padding: 10px 10px 42px 10px;
|
|
123
|
+
border: 0;
|
|
124
|
+
text-align: right;
|
|
125
|
+
background-color: rgb(226, 225, 225);
|
|
126
|
+
color: rgb(69, 69, 69);
|
|
127
|
+
font-family: Georgia, 'Times New Roman', Times, serif, Times;
|
|
128
|
+
font-size: 16px;
|
|
129
|
+
}
|
|
130
|
+
.header p {
|
|
131
|
+
text-align: right;
|
|
132
|
+
margin: 0;
|
|
133
|
+
margin-top: 6px;
|
|
134
|
+
margin-right: 20px;
|
|
135
|
+
float: right;
|
|
136
|
+
display: block;
|
|
137
|
+
}
|
|
138
|
+
.header img {
|
|
139
|
+
margin: 0;
|
|
140
|
+
padding: 0;
|
|
141
|
+
border: 0;
|
|
142
|
+
float: left;
|
|
143
|
+
}
|
|
119
144
|
.foot {
|
|
120
145
|
margin: 0 20px 20px 20px;
|
|
121
146
|
padding: 5px;
|
|
122
147
|
border: 0;
|
|
123
148
|
text-align: center;
|
|
149
|
+
text-align: left;
|
|
124
150
|
background-color: black;
|
|
125
151
|
color: white;
|
|
126
152
|
font-family: Arial, Times;
|
|
127
153
|
font-size: 13px;
|
|
128
154
|
}
|
|
129
155
|
</style>
|
|
156
|
+
<div class="header">
|
|
157
|
+
<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAAAeFBMVEX///8AAADl5eVdXV1jY2Pd3d2xsbH4+Pi2trbz8/Ph4eGampr8/PwwMDDw8PDMzMyJiYmrq6vGxsalpaWPj4+Dg4PV1dW9vb1XV1dAQEAQEBA3NzegoKBra2twcHDPz89RUVF6enofHx8+Pj4XFxd9fX0kJCRJSUmiWCO7AAAEmElEQVR4nO3c6XaqMBQFYIMoooBYwXmonXz/N7zVgGYg2isHEl37+wsrnq2IIYOdDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAUTQe7fzdaBzZLqQhYcpKaWi7mCYMmGhguxx6KZOltguiNmKqke2SaO21gIztbRdF6qMi4YftoijNKwIyNrddFqFFZcKF7bLoBJUBGQtsF0ZmaEjo2S6MzJsh4dh2YWRMCd9sF0amZ0jYs10YmdCQ8IX635+VAZe2yyI0rUw4tV0Woagy4Us9CPcrAvZtF0XL1wL6tkui9q4EfLddEL2DFPBgu5wmbISAG9vFNEK827zYXaaAhM8PCZ8fEj4/JHx+SPj8Bi+SMEyGvUpDcYJtZDopcXx0an6IWV3xwdkZmzCrna6UOTmjUT2e9ij3xuECdZSirnfHPsZkRRyQsVViO5QoEQo7+nUchbfKpYiXO2haf7Kld1mXEhNURmRXlHSg+e4El/ZImiNQLiWhW+dUdn9cWZCy5uVQzncWc6prwiZrGPNqaIdAi8FVN+bBt+daPolb5RfGlrjVhwTNfGWKK8OF3/19Q3f22Jl7De+PZuTt8n68C/3Tw7kS+ieeuTM/iXzuc0jeLl9I5cI8avdcCf0CLu/cbpe83f+HhI9Cwva8fkK/0YR276VeP8vzNDYkHG/ybPKXH5HhJMs3et+FJ4zTPM/6dpbaetLibbWGcuRtObvTzGxZnKn2Xjyx+YWFjMo6LrmCYHk9crs/J4yxLuVetie/QOvzAOpCNTmhNPJ9K6I0iCz33pWEbUdUX15OeJCPmR9jx/KJUif09ks0TluIJ768ujTf/GSlznKINyYtYau31UR9dSmhuk2Nme42M/VEcUOblrDV8VN9l4+YcK0eNG1T0za0iUMhesI256T0vXZCQn1B8M7QzFY7U1g2rCdsc7mfPo8mJNT3AJm+QfqyWuEHQ09IP45gNrmVsPOlHswNzeTqieKGPT3hpMFEKu0WISXsqgdNg+Ha6m+xn60nvNc9IqV9TGJCrXDT3Ly200R8K7SEXw3m0WkTvtLPsTKZ+G1s5ls+cSUe0xK2eZH+WiovLyWUuyor876KSH4vpM6PmrDtHTbBz42E8kd8q7clxZiaDzH20/rodyTf6pUY1/14x9s9keR4OVOZu5ITbm1ssJnthB3a6gcVTU+lr/z7U25v/ulSPU7VCELCj12rt1FRGIbRwnQpBr8H/9RKFIYVlyBPuPg9aHuZ1OuPRCHho5CwPUj4KHcS8sdY+h8s/vziwlIF/jRM3y3mz6BtPvWa8PVL9ANhvFfowv9JFb0r6o5HWN0btCJu5HLiF78byxOLgTfa0cxiRNaNv8sqxtZoV9kVY64uLInqXMa4Kf8UqZi5c+VP3coh4DXVhZqUo+bO/K/EZVwmo7ijXvdtuLH28uw6yu9PZl4d+8l1dMSN20xBG70mYBopt0Sfq6nLqU/wZHy/5v/i0HewFGkTozWkztxFJcGo/t68k3jkyA99FW+Qbbt1bLOBE51tAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgGf2D4WAKMqRFhM1AAAAAElFTkSuQmCC" width="35px" height="35px">
|
|
158
|
+
<p>< Maintenance Bot ></p>
|
|
159
|
+
</div>
|
|
130
160
|
<div class="content">
|
|
131
161
|
Hello,
|
|
132
162
|
<br><br>
|
|
@@ -27,13 +27,13 @@ module VolumeSweeper
|
|
|
27
27
|
<<~HTML
|
|
28
28
|
The environment is scanned.<br>
|
|
29
29
|
* Active volumes: #{active_count}<br>
|
|
30
|
-
* Unused volumes: #{unused_list&.count || 0}<br>
|
|
30
|
+
* Unused volumes: #{unused_list&.count || 0}<br><br>
|
|
31
31
|
|
|
32
32
|
Found the following volumes without instance bound or K8S PV relation.<br>
|
|
33
|
-
<u>(#{notice})
|
|
33
|
+
<u>(#{notice}).</u> <br>
|
|
34
34
|
<ul style="color: #400707">
|
|
35
35
|
<% unused_list.each do |vol| %>
|
|
36
|
-
<li
|
|
36
|
+
<li><a href="#{@provider_base_url}<%= vol %>"><%= vol %></a>.</li>
|
|
37
37
|
<% end %>
|
|
38
38
|
</ul>
|
|
39
39
|
HTML
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: volume_sweeper
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Abdullah Barrak
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2025-01-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: cowsay
|
|
@@ -24,6 +24,20 @@ dependencies:
|
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: 0.3.0
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: aws-sdk-ec2
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '1.498'
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '1.498'
|
|
27
41
|
- !ruby/object:Gem::Dependency
|
|
28
42
|
name: oci
|
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -108,6 +122,20 @@ dependencies:
|
|
|
108
122
|
- - "~>"
|
|
109
123
|
- !ruby/object:Gem::Version
|
|
110
124
|
version: '3'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: nokogiri
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - "~>"
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: '1.18'
|
|
132
|
+
type: :runtime
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - "~>"
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: '1.18'
|
|
111
139
|
- !ruby/object:Gem::Dependency
|
|
112
140
|
name: rspec
|
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -206,8 +234,8 @@ dependencies:
|
|
|
206
234
|
- - ">="
|
|
207
235
|
- !ruby/object:Gem::Version
|
|
208
236
|
version: '0'
|
|
209
|
-
description:
|
|
210
|
-
|
|
237
|
+
description: This is a scanning tool for unused block volumes in kubernetes clusters
|
|
238
|
+
(AWS, GCP, OCI).
|
|
211
239
|
email:
|
|
212
240
|
- abdullah@abarrak.com
|
|
213
241
|
executables: []
|
|
@@ -264,5 +292,5 @@ requirements: []
|
|
|
264
292
|
rubygems_version: 3.3.26
|
|
265
293
|
signing_key:
|
|
266
294
|
specification_version: 4
|
|
267
|
-
summary: A CLI for
|
|
295
|
+
summary: A CLI for kubernetes volumes cleanup in multiple cloud provides.
|
|
268
296
|
test_files: []
|