volume_sweeper 1.0.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27bd3d28a1fb0df057bd544b2d56c83ce90cb2523d91d83070babb79b34623db
-  data.tar.gz: d361c6ca00453bc2d26118ad2db0825a76030e366e2b7bd75edeb0996a99baee
+  metadata.gz: 69061fddbb0ab861a6e3061356f675e420dc37b899c6e997555be5f1f55bcf13
+  data.tar.gz: 51c187ffb879566b2faf1d4e71e8fb6651d9bee3e4bccd4240348f8d7de2139e
 SHA512:
-  metadata.gz: f172d17d2117fa70c2f41c15029e9b620d4eea15fe3f64942914e626c9c1e4b31498d0c7357a67c28f2b754c17c5872c1f8be7f2da35ac60fb2eb10c57f40bb0
-  data.tar.gz: 3766a085343bf9a0c2da75986409f0204aec7d9e54e5d09db5d0b853e4889c566b81c808579777431510cc0ddc6bc55747b85d2b5c5b2e570126e8167bcd0c75
+  metadata.gz: 8e089dee93af6b64d99c02328087732ef139bc3b91fae9e0b3ab3a7c14b895e8060e2e0c2b1942099cf3e26bfc482cc1c457c094eaee96a37ccc6eaa5908c5e9
+  data.tar.gz: 29e41d99d2f45464d6cb281a4821e06dfc2454886c1355f8ae30103dfca7ad3d34ecc1fc7a524f87add8f5af859062fd23f9cda2d463b0ad535ccc1ee71c0948

data/README.md

@@ -7,7 +7,7 @@ A tool to scan and clean cloud infrastruture for unattached block volumes withou
 ## Supported Clouds
 
 - [x] OCI
-- [ ] AWS.
+- [x] AWS.
 - [ ] GCP.
 
 ## Supported Kubernetes
@@ -31,7 +31,7 @@ $ gem install volume_sweeper
 To scan and generate a report:
 
 ```bash
-volume_sweeper --account-id <ID> --cloud aws|oci
+volume_sweeper --account-id <ID> --cloud aws|oci --region <region>
 ```
 
 To apply deletion for unattached block volumes:
@@ -40,6 +40,12 @@ To apply deletion for unattached block volumes:
 volume_sweeper --mode delete
 ```
 
+For all options:
+
+```bash
+volume_sweeper -h
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/abarrak/volume_sweeper.

data/lib/volume_sweeper/cli.rb

@@ -29,6 +29,7 @@ module VolumeSweeper
           opt.on('-m', '--mode [MODE]', 'The run modes: either audit, or delete.') { |o| options.mode = o }
           opt.on('-c', '--cloud [CLOUD]', 'Supported clouds: aws, oci.') { |o| options.cloud = o }
           opt.on('-f', '--config-path [PATH]', 'The file location for cloud config file') { |o| options.config_path = o }
+          opt.on('-s', '--creds-path [PATH]', 'The file location for cloud crednetials file') { |o| options.creds_path = o }
           opt.on('-r', '--region [REGION]', 'The provider region of the account.') { |o| options.region = o }
           opt.on('-a', '--account-id [Id]', 'The account or compartment Id.') { |o| options.account_id = o }
           opt.on('-d', '--released-since [DAYS]', 'Volumes threshold duration') { |o| options.released_in_days = o }

data/lib/volume_sweeper/core.rb

@@ -17,6 +17,7 @@ module VolumeSweeper
       mode = opts.mode&.to_sym
       options = {
         config_path: opts.config_path,
+        creds_path: opts.creds_path,
         account_id: opts.account_id,
         region: opts.region,
         mode: opts.mode,

data/lib/volume_sweeper/providers/aws.rb

@@ -1,4 +1,5 @@
 require 'active_support/core_ext/object/blank'
+require 'aws-sdk-ec2'
 require_relative 'base'
 require_relative '../utils/log'
 
@@ -6,26 +7,80 @@ module VolumeSweeper
   module Providers
 
     class Aws < Base
+
       DEFAULT_REGION = 'us-west-2'
+      DEFAULT_CONFIF_PATH = '~/.aws/config'
+      DEFAULT_CREDS_PATH  = '~/.aws/credentials'
+
+      BASE_CONSOLE_URL = "console.aws.amazon.com/ec2/home"
+      VOLUME_ATTRS = %i{ volume_id displayName state size attachments create_time availability_zone tags }
 
       def initialize config_path: nil, region: nil, mode: :audit, **kwargs
         super
         @region ||= DEFAULT_REGION
+        set_console_base_url
         validate_attrs
+        prepare_config config_path, kwargs[:creds_path]
       end
 
       def scan_block_volumes
-        raise NotImplementedError
+        volumes = Array.new
+        next_token = nil
+        opts = { max_results: 200 }
+
+        run_api_call do |client|
+          loop do
+            response = client.describe_volumes opts.merge(next_token: next_token)
+            response&.volumes&.map do |v|
+              volumes << v.to_hash.compact.slice(*VOLUME_ATTRS).transform_keys(volume_id: :id)
+            end
+            break if response.nil? || response.next_token.nil?
+            next_token = response.next_token
+            sleep 2
+          end
+          @log.msg "aws: collected #{volumes.size} block volumes from the account."
+        end
+
+        @log.msg "aws: filtering out any block volume with an active attachment."
+        result = volumes&.reject { |v| v[:state] != 'available' || v[:attachments]&.count > 0 } || []
+
+        @log.msg "aws: found #{result.count} unattached block volumes."
+        [volumes.size, result]
       end
 
       def delete_block_volumes ids_list
-        return if ids_list.blank? || @run_mode != :delete
-        raise NotImplementedError
+        @log.msg "aws: #{ids_list&.count || 0} block volumes are eligible for cleanup."
+        return if ids_list.blank?
+
+        unless @run_mode == :delete
+          @log.msg "aws: running in :#{@run_mode} mode, exiting without delete operations."
+          return
+        end
+
+        @log.msg "aws: unused volume clean-up operation started."
+
+        ids_list.each do |id|
+          @log.msg "aws: deleting block volume #{id} .."
+          run_api_call do |client|
+            output = client.delete_volume({ volume_id: id.to_s })
+            if output&.successful?
+              @log.msg "aws: block volume #{id} is deleted successfully."
+            else
+              @log.msg "aws: block volume #{id} has failed."
+            end
+            sleep 2.5
+          end
+        end
       end
 
       private
 
-      def prepare_config
+      def prepare_config config_path, creds_path
+        # SDK automtically picks up config and credentails files in the
+        # default place for various methods.
+        @config_location ||= DEFAULT_CONFIF_PATH
+        @creds_location  ||= DEFAULT_CREDS_PATH
+        ::Aws.config.update({ region: @region })
       end
 
       def validate_attrs
@@ -33,6 +88,25 @@ module VolumeSweeper
         @log.msg "provider error: aws account id is not assigned", level: :error
         exit 1
       end
+
+      def run_api_call
+        current_tries = 0
+        ec2_client = ::Aws::EC2::Client.new(region: @region, account_id: @account_id)
+
+        yield ec2_client if block_given?
+
+      rescue ::Aws::EC2::Errors::ServiceError => err
+        # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EBS/Errors.html
+        @log.msg errr&.context&.data, level: :error
+        raise if err&.code.to_s != '304'
+      rescue StandardError => err
+        @log.msg err, level: :error
+        raise
+      end
+
+      def set_console_base_url
+        @base_link = "https://#{@region}.#{BASE_CONSOLE_URL}?region=#{@region}#VolumeDetails:volumeId="
+      end
     end
 
   end

data/lib/volume_sweeper/providers/base.rb

@@ -7,6 +7,7 @@ module VolumeSweeper
       def initialize **kwargs
         @run_mode = kwargs[:mode]&.to_sym || :audit
         @config_location = kwargs[:config_path]
+        @creds_location = kwargs[:creds_location]
         @account_id = kwargs[:account_id]
         @compartment_id = kwargs[:account_id]
         @region = kwargs[:region]

data/lib/volume_sweeper/providers/oci.rb

@@ -19,7 +19,7 @@ module VolumeSweeper
       def initialize config_path: nil, region: nil, mode: :audit, **kwargs
         super
         @region ||= DEFAULT_REGION
-        @base_link = "https://cloud.oracle.com/block-storage/volumes"
+        @base_link = "https://cloud.oracle.com/block-storage/volumes/"
         validate_attrs
       end
 

data/lib/volume_sweeper/utils/notification_formatter.rb

@@ -33,7 +33,7 @@ module VolumeSweeper
               <u>(#{notice}).</u> <br>
               <ul style="color: #400707">
                 <% unused_list.each do |vol| %>
-                <li><a href="#{@provider_base_url}/<%= vol %>"><%= vol %></a>.</li>
+                <li><a href="#{@provider_base_url}<%= vol %>"><%= vol %></a>.</li>
                 <% end %>
               </ul>
             HTML

data/lib/volume_sweeper/version.rb

@@ -1,3 +1,3 @@
 module VolumeSweeper
-  VERSION = "1.0.2"
+  VERSION = "2.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: volume_sweeper
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Abdullah Barrak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-12 00:00:00.000000000 Z
+date: 2025-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cowsay
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ec2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.498'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.498'
 - !ruby/object:Gem::Dependency
   name: oci
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -206,8 +234,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 'This is a scanning tool for cloud infrastructure cross referenced with
-  related clusters. '
+description: This is a scanning tool for unused block volumes in kubernetes clusters
+  (AWS, GCP, OCI).
 email:
 - abdullah@abarrak.com
 executables: []
@@ -264,5 +292,5 @@ requirements: []
 rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
-summary: A CLI for block volumes sweeping and cleanup
+summary: A CLI for kubernetes volumes cleanup in multiple cloud provides.
 test_files: []