volume_sweeper 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3b347010ebacef693dd8fe9917e0ef3165dadeed8f79076e61ab5fa051ae6f6a
+  data.tar.gz: 613ff41df273c38ae7b8a3c55176c284716aa64be804b1fd86b4d824cbdfff90
+SHA512:
+  metadata.gz: 3ac1b14b8a25b824a0832b423a20ba0ff8d00653f70eebb83d6e266fced137096b80f8ce82c5496eeffa3f967d194b77a32f72641411c97bd10b14a8f35cb523
+  data.tar.gz: 8f3212d22053ba1c3ed803dd659929f635e7b4ac7c73f618263ce27a3bc83439fd72f47f358344c5e276d59be817f29f6f5b73cf1bb2d2bb5aee96816e1793ba

data/.dockerignore

@@ -0,0 +1,220 @@
+.git
+.gitignore
+
+### Git ###
+
+# $ git config --global mergetool.keepBackup false
+*.orig
+
+
+*.BACKUP.*
+*.BASE.*
+*.LOCAL.*
+*.REMOTE.*
+*_BACKUP_*.txt
+*_BASE_*.txt
+*_LOCAL_*.txt
+*_REMOTE_*.txt
+
+### JetBrains+all ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### JetBrains+all Patch ###
+# Ignores the whole .idea folder and all .iml files
+# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
+
+.idea/
+
+# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
+
+*.iml
+modules.xml
+.idea/misc.xml
+*.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Rails ###
+*.rbc
+capybara-*.html
+.rspec
+/db/*.sqlite3
+/db/*.sqlite3-journal
+/public/system
+/coverage/
+/spec/tmp
+rerun.txt
+pickle-email-*.html
+
+# Ignore all logfiles and tempfiles.
+/log/*
+/tmp/*
+!/log/.keep
+!/tmp/.keep
+
+# TODO Comment out this rule if you are OK with secrets being uploaded to the repo
+config/initializers/secret_token.rb
+config/master.key
+
+# Only include if you have production secrets in this file, which is no longer a Rails default
+# config/secrets.yml
+
+# dotenv
+# TODO Comment out this rule if environment variables can be committed
+.env
+
+## Environment normalization:
+/.bundle
+/vendor/bundle
+
+# these should all be checked in to normalize the environment:
+# Gemfile.lock, .ruby-version, .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# if using bower-rails ignore default bower_components path bower.json files
+/vendor/assets/bower_components
+*.bowerrc
+bower.json
+
+# Ignore pow environment settings
+.powenv
+
+# Ignore Byebug command history file.
+.byebug_history
+
+# Ignore node_modules
+node_modules/
+
+# Ignore precompiled javascript packs
+/public/packs
+/public/packs-test
+/public/assets
+
+# Ignore yarn files
+/yarn-error.log
+yarn-debug.log*
+.yarn-integrity
+
+# Ignore uploaded files in development
+/storage/*
+!/storage/.keep
+
+### Ruby ###
+*.gem
+/.config
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+# Ignore Byebug command history file.
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+
+# End of https://www.gitignore.io/api/git,ruby,rails,jetbrains+all

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+volume_sweeper

data/.ruby-version

@@ -0,0 +1 @@
+3.1.4

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/testdouble/standard
+ruby_version: 3.1.4

data/Dockerfile

@@ -0,0 +1,20 @@
+# syntax=docker/dockerfile:1
+
+FROM ruby:3.1.4 AS base
+
+FROM base AS dependencies
+RUN apt-get update -qq && apt-get install -y gcc cron
+
+FROM dependencies as build
+USER root
+WORKDIR /app
+COPY . .
+RUN gem install bundler
+RUN bundle install --without development
+
+FROM build as runtime
+
+COPY --from=build /app /app
+
+EXPOSE 3000
+CMD ["sh", "-c", "./bin/volume_sweeper --help"]

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Abdullah Barrak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# Volume Sweeper
+[![CI (tests)](https://github.com/abarrak/volume_sweeper/actions/workflows/ci.yml/badge.svg)](https://github.com/abarrak/volume_sweeper/actions/workflows/ci.yml) [![Gem Version](https://badge.fury.io/rb/volume_sweeper.svg)](https://badge.fury.io/rb/volume_sweeper) [![Test Coverage](https://api.codeclimate.com/v1/badges/b9d24a336e67236937dd/test_coverage)](https://codeclimate.com/github/abarrak/volume_sweeper/test_coverage) [![Maintainability](https://api.codeclimate.com/v1/badges/b9d24a336e67236937dd/maintainability)](https://codeclimate.com/github/abarrak/volume_sweeper/maintainability) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+
+A tool to scan and clean cloud infrastruture for unattached block volumes without kubernetes clusters persistent volumes.
+
+## Supported Clouds
+
+- [x] OCI
+- [ ] AWS.
+- [ ] GCP.
+
+## Supported Kubernetes
+
+Any distributions + v1.19.
+
+## Prerequisits
+
+1. Kubernetes: a service account with read/update access to the cluster is required, scoped to `PV` resources.
+2. Cloud: access is required for block volumes service (BV) with read and delete roles.
+
+
+## Installation
+
+```bash
+$ gem install volume_sweeper
+```
+
+## Usage
+
+To scan and generate a report:
+
+```bash
+volume_sweeper --account-id <ID> --cloud aws|oci
+```
+
+To apply deletion for unattached block volumes:
+
+```bash
+volume_sweeper --mode delete
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/abarrak/volume_sweeper.
+
+## License
+
+[MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "standard/rake"
+
+task default: %i[spec standard]

data/lib/volume_sweeper/cli.rb

@@ -0,0 +1,57 @@
+require 'optparse'
+require 'ostruct'
+require 'cowsay'
+require_relative 'utils/log'
+
+module VolumeSweeper
+  class Cli
+    class << self
+      attr_reader :options
+      ##
+      # A simple cli scriptlet to proccess command line arguments and pass them to
+      # the core component to run.
+      #
+      def run
+        print_banner
+        set_default_options
+        process_user_input
+        options
+      end
+
+      private
+
+      def print_banner
+        puts Cowsay::say('Volume Sweeper 1.0', 'cow'), ""
+      end
+
+      def process_user_input
+        OptionParser.new("== Usage: volume_sweeper.rb [options]") do |opt|
+          opt.on('-m', '--mode [MODE]', 'The run modes: either audit, or delete.') { |o| options.mode = o }
+          opt.on('-c', '--cloud [CLOUD]', 'Supported clouds: aws, oci.') { |o| options.cloud = o }
+          opt.on('-f', '--config-path [PATH]', 'The file location for cloud config file') { |o| options.config_path = o }
+          opt.on('-r', '--region [REGION]', 'The provider region of the account.') { |o| options.region = o }
+          opt.on('-a', '--account-id [Id]', 'The account or compartment Id.') { |o| options.account_id = o }
+          opt.on('-d', '--released-since [DAYS]', 'Volumes threshold duration') { |o| options.released_in_days = o }
+          opt.on('--kube-api-url [URL]', 'Kubernetes API URL') { |o| options.kube_api_url = o }
+          opt.on('--kube-api-ca-path [PATH]', 'Kubernetes API CA Cert Path') { |o| options.kube_api_ca_path = o }
+          opt.on('--kube-api-token [TOKEN]', 'Kubernetes API TOKEN (base64 formatted)') { |o| options.kube_api_token = o }
+          opt.on('--notification_subject [TEXT]', 'The message subject.') { |o| options.notification_subject = o }
+          opt.on('--smtp-host [HOST]', '') { |o| options.smtp_host = o }
+          opt.on('--smtp-port [PORT]', '') { |o| options.smtp_port = o }
+          opt.on('--smtp-username [USER]', '') { |o| options.smtp_username = o }
+          opt.on('--smtp-password [PASS]', '') { |o| options.smtp_password = o }
+          opt.on('--smtp-tls [ENABLE_TLS_FLAG]', '') { |o| options.smtp_tls = o }
+          opt.on('--smtp-sender [SENDER]', '') { |o| options.smtp_sender = o }
+          opt.on('--smtp-receiver [RECEIVER]', '') { |o| options.smtp_receiver = o }
+          opt.on('--ms-teams-webhook [URL]', '') { |o| options.ms_teams_webhook = o }
+          opt.on('-h', '--help') { |_| puts "", opt, "-" * 34; exit 0 }
+        end.parse!
+      end
+
+      def set_default_options
+        @options = OpenStruct.new
+      end
+    end
+
+  end
+end

data/lib/volume_sweeper/comparer.rb

@@ -0,0 +1,56 @@
+require "active_support/core_ext/object/blank"
+require_relative "utils/log"
+
+module VolumeSweeper
+  module Comparer
+    LIST_SEP = "\n     "
+    ##
+    # Compare unattached block volumes against kubernetes persistent volumes
+    # in order to avoid any block volume that as volumeHandler reference,
+    # even if not bound to instance.
+    #
+    # === Docs:
+    #
+    # The algorithm goes as follows:
+    # ```
+    #   FOR each_cluster IN oci:
+    #     A] Fetch PVs (name, ocid)
+    #     B] Fetch BLOCK VOL where attachment = nil
+    #     C] Compare A ^ B to Extract Bx NOT IN Ax
+    #        THEN:
+    #       DEL [C] result
+    #   End
+    # ```
+    #
+    def self.process  block_volumes, persistent_volumes
+      unused_volumes = []
+      active_volumes = []
+      counters = { active: 0, unused: 0 }
+
+      return {} if block_volumes.blank? || persistent_volumes.blank?
+
+      block_volumes.each do |vol|
+        if persistent_volumes.any? { |p| p[:volumeHandle]&.strip == vol&.strip  }
+          counters[:active] += 1
+          active_volumes << vol
+        else
+          counters[:unused] += 1
+          unused_volumes << vol
+        end
+      end
+
+      seperator = -> (str) { str.join(LIST_SEP).prepend LIST_SEP }
+
+      active_list = active_volumes.any? ? seperator.call(active_volumes) : 'None'
+      unused_list = unused_volumes.any? ? seperator.call(unused_volumes) : 'None'
+
+      Utils::Log.instance.msg "=> Found #{counters[:active]} still in use."
+      Utils::Log.instance.msg "=> Found #{counters[:unused]} unused and should be terminated."
+      Utils::Log.instance.msg "=> Details:"
+      Utils::Log.instance.msg "===> Active: ", active_list
+      Utils::Log.instance.msg "===> Unused: ", unused_list
+
+      { active_ids: active_volumes, unused_ids: unused_volumes }
+    end
+  end
+end

data/lib/volume_sweeper/core.rb

@@ -0,0 +1,73 @@
+require 'active_support/core_ext/string/inflections'
+require_relative 'cli'
+require_relative 'comparer'
+require_relative 'providers/aws'
+require_relative 'providers/oci'
+require_relative 'kube/client'
+require_relative 'utils/log'
+require_relative 'utils/notification'
+require_relative 'utils/notification_formatter'
+
+module VolumeSweeper
+  module Core
+    def self.process
+      # Process user input ..
+      opts = VolumeSweeper::Cli.run
+      cloud = opts.cloud
+      mode = opts.mode&.to_sym
+      options = {
+        config_path: opts.config_path,
+        account_id: opts.account_id,
+        region: opts.region,
+        mode: opts.mode,
+        kube_api_url: opts.kube_api_url,
+        kube_api_ca_path: opts.kube_api_ca_path,
+        kube_api_token: opts.kube_api_token,
+        notification_subject: opts.notification_subject,
+        smtp_host: opts.smtp_host,
+        smtp_port: opts.smtp_port,
+        smtp_username: opts.smtp_username,
+        smtp_password: opts.smtp_password,
+        smtp_tls: opts.smtp_tls,
+        smtp_sender: opts.smtp_sender,
+        smtp_receiver: opts.smtp_receiver,
+        ms_teams_webhook: opts.ms_teams_webhook
+      }
+
+      unless cloud.nil? || %w{oci aws}.include?(cloud)
+        Utils::Log.instance.msg "No could provider is chosen", level: :fatal
+        exit
+      end
+
+      # Build and run provider checks ..
+      klass = cloud.capitalize
+      provider = "VolumeSweeper::Providers::#{klass}".constantize.new **options
+      active_count, block_vols = provider.scan_block_volumes
+      block_vols.map! { |v| v[:id] }
+
+      # Build and run kubernetes checks ..
+      kube_client = VolumeSweeper::Kube::Client.new **options
+      cluster_vols = kube_client.fetch_pesistent_volumes
+
+      # Prepare notification layer ..
+      notifier  = VolumeSweeper::Utils::Notification.new **options
+      formatter = VolumeSweeper::Utils::NotificationFormatter.new provider.base_link, mode
+
+      # Run cross reference checks.
+      results = Comparer.process block_vols, cluster_vols
+
+      # Send notice messages.
+      message = formatter.formlate_meessage results, active_count: active_count
+      notifier.send_ms_teams_notice message
+      notifier.send_mail message if results[:unused_ids]&.any?
+
+      # Then, clean up any unattached block volume without PV bound to.
+      provider.delete_block_volumes results[:unused_ids]
+
+      # Wait for plaform logs aggregation.
+      sleep 30
+
+      VolumeSweeper::Utils::Log.instance.msg "Done !"
+    end
+  end
+end

data/lib/volume_sweeper/kube/client.rb

@@ -0,0 +1,138 @@
+require "active_support/core_ext/object/blank"
+require "base64"
+require "kubeclient"
+require_relative "../utils/log"
+
+module VolumeSweeper
+  module Kube
+    ##
+    #
+    # This class enables interation with the kube apis using 2 different modes.
+    #
+    #   1. In-cluster acccess.
+    #        Assuming the code will run in the cluster it targets,
+    #        So the main defaults that pods that mount from service account
+    #        secret like `token` and `ca.cert` are used in this case.
+    #
+    #   2. External cluster access.
+    #       Pass the environment variables below to access outside cluster
+    #         * KUBE_API_URL
+    #         * KUBE_API_TOKEN
+    #         * KUBE_API_CA_PATH
+    #       Or their kwargs counterpart to constructor.
+    #         * :kube_api_url
+    #         * :kube_api_token [base64 formatted]
+    #         * :kube_api_ca_path
+    #
+    #   The env vars take the highest precedence.
+    #
+    class Client
+      def initialize **kwargs
+        @run_mode = kwargs[:mode]&.to_sym || :audit
+        @api_url = kwargs[:kube_api_url]
+        @api_token = kwargs[:kube_api_token]
+        @api_ca_path = kwargs[:kube_api_ca_path]
+
+        @log = Utils::Log.instance
+        prepare_kube_client
+      end
+
+      def fetch_pesistent_volumes
+        resources = []
+        make_api_call :get_persistent_volumes do |i|
+          resources << format_response_attrs(i)
+        end
+        @log.msg "kube: Collected #{resources.size} persisted volumes from the cluster."
+
+        resources
+      end
+
+      def delete_released_persistent_volumes age_in_days: 10
+        names = []
+        make_api_call :get_persistent_volumes do |i|
+          names << i[:metadat][:name] if i.dig(:status, :phase) == "Released"
+        end
+        @log.msg "kube: Collected #{resources.size} released persisted volumes."
+
+        return if names.blank? || @run_mode != :delete
+
+        # Do actual deletion for released persistent volumes.
+        # TODO: use last transistion timestamp if possible.
+        @log.msg "kube: Looping over #{names.size} persisted volumes to be deleted sequentially."
+        names.each do |pv|
+          @log.msg "kube: deleting #{names} persisted volume .."
+          @client.delete_pesistent_volume pv
+          sleep 2
+        end
+        @log.msg "kube: completed deletion of #{names.size} persisted volume."
+      end
+
+      protected
+
+      def prepare_kube_client
+        url = "#{build_cluster_url}/api"
+        @client ||= Kubeclient::Client.new url, "v1",
+                                           auth_options: build_auth_config,
+                                           ssl_options: build_ssl_config
+        @log.msg "Kube: running in :#{@run_mode} mode"
+      end
+
+      def build_cluster_url
+        default_cluster_url = "https://kubernetes.default.svc"
+        ENV.fetch("KUBE_API_URL", @api_url || default_cluster_url)
+      end
+
+      def build_ssl_config
+        default_ca_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        cluster_ca_path = ENV.fetch("KUBE_API_CA_PATH", @api_ca_path || default_ca_path)
+
+        Hash.new.tap do |h|
+          if File.exist? cluster_ca_path
+            h[:ca_file] = cluster_ca_path
+          else
+            h[:verify_ssl] = OpenSSL::SSL::VERIFY_NONE
+          end
+        end
+      end
+
+      def build_auth_config
+        cluster_token = ENV["KUBE_API_SECRET"]
+        default_token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+        Hash.new.tap do |h|
+          if cluster_token.present?
+            h[:bearer_token] = Base64.decode64 cluster_token
+          elsif @api_token.present?
+            h[:bearer_token] = Base64.decode64 @api_token
+          elsif File.exist? default_token_path
+            h[:bearer_token] = File.read default_token_path
+          end
+        end
+      end
+
+      def make_api_call method, **opts
+        continue = nil
+        { limit: 30 }.merge! opts
+        loop do
+          opts[:continue] = continue
+          output = @client.send method, **opts
+          continue = output.continue
+          output.map(&:to_h).collect do |i|
+            yield i
+          end
+          break if output.last?
+        end
+      end
+
+      def format_response_attrs item
+        {
+          name: item.dig(:metadat, :name),
+          status: item.dig(:status, :phase),
+          volumeHandle: item.dig(:spec, :csi, :volumeHandle),
+          pvc: item.dig(:spec, :claimRef, :name),
+          namespace: item.dig(:spec, :claimRef, :namespace)
+        }
+      end
+    end
+
+  end
+end