gitlab-janitor 0.0.3 → 1.0.2.92939

data/lib/gitlab_janitor/container_cleaner.rb

@@ -0,0 +1,120 @@
+module GitlabJanitor
+  class ContainerCleaner < BaseCleaner
+
+    class Model < BaseCleaner::Model
+
+      def initialize(model)
+        super(model)
+
+        info['_Age'] = (Time.now - Time.at(created_at)).round(0)
+      end
+
+      def created_at
+        info['Created']
+      end
+
+      def name
+        @name ||= info['Names'].first.sub(%r{^/}, '')
+      end
+
+      def age
+        info['_Age']
+      end
+
+      def age_text
+        Fugit::Duration.parse(age).deflate.to_plain_s
+      end
+
+    end
+
+    attr_reader :excludes, :includes
+
+    def initialize(includes: [''], excludes: [''], **kwargs)
+      super(**kwargs)
+      @includes = includes
+      @excludes = excludes
+      @deadline = deadline
+    end
+
+    def do_clean(remove: false)
+      to_remove, keep = prepare(Docker::Container.all(all: true).map{|m| Model.new(m) })
+
+      return if to_remove.empty?
+
+      keep.each {|m| logger.debug("  KEEP #{m.name}") }
+
+      if remove
+        logger.info 'Removing containers...'
+        to_remove.each do |model|
+          return false if exiting?
+
+          logger.tagged(model.name) do
+            logger.debug '   Removing...'
+            log_exception('Stop') { model.stop }
+            log_exception('Wait') { model.wait(15) }
+            log_exception('Remove') { model.remove }
+            logger.debug '   Removing COMPLETED'
+          end
+        end
+      else
+        logger.info 'Skip removal due to dry run'
+      end
+    end
+
+    def prepare(containers)
+      @logger.debug("Selecting containers by includes #{@includes}...")
+      to_remove = select_by_name(containers)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], containers
+      end
+      @logger.info("Selected containers: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
+      @logger.debug("Filtering containers by excludes #{@excludes}...")
+      to_remove = reject_by_name(to_remove)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], containers
+      end
+      @logger.info("Filtered containers: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
+      @logger.info("Filtering containers by deadline: older than #{Fugit::Duration.parse(@deadline).deflate.to_plain_s}...")
+      to_remove = select_by_deadline(to_remove)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], containers
+      end
+      @logger.info("Filtered containers: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
+      [to_remove, containers - to_remove]
+    end
+
+    def format_item(model)
+      "#{Time.at(model.created_at)} Age:#{model.age_text.ljust(10)} #{model.name.first(60).ljust(60)}"
+    end
+
+    def select_by_name(containers)
+      containers.select do |model|
+        @includes.any? do |pattern|
+          File.fnmatch(pattern, model.name)
+        end
+      end
+    end
+
+    def reject_by_name(containers)
+      containers.reject do |model|
+        @excludes.any? do |pattern|
+          File.fnmatch(pattern, model.name)
+        end
+      end
+    end
+
+    def select_by_deadline(containers)
+      containers.select do |model|
+        model.age > deadline
+      end
+    end
+
+  end
+end
+

data/lib/gitlab_janitor/image_cleaner/store.rb

@@ -0,0 +1,66 @@
+module GitlabJanitor
+  class ImageCleaner
+    class Store
+
+      attr_reader :logger, :filename, :images
+
+      def initialize(logger:, filename: './images.txt')
+        @filename = filename
+        @logger = logger
+      end
+
+      def parse_images
+        Docker::Image.all.map do |m|
+          tags = m.info.fetch('RepoTags', []) || []
+          tags = [m.info['id']] if tags.empty? || tags.first == '<none>:<none>'
+          m.info['RepoTags'] = tags
+
+          tags.map do |name|
+            Model.new(m, name, self)
+          end
+        end.flatten
+      end
+
+      def load
+        containers = Docker::Container.all(all: true).each_with_object({}) do |c, res|
+          res[c.info['ImageID']] = true
+          res[c.info['Image']] = true
+        end
+
+        File.open(@filename, 'a+') do |file|
+          i = 0
+          @images = file.readlines.select(&:present?).each_with_object({}) do |line, imgs|
+            i += 1
+            name, image_id, loaded_at = line.strip.split(' ')
+            loaded_at = Time.now.to_i if containers[name] || containers[image_id]
+
+            imgs[name] = { id: image_id, loaded_at: Time.at(loaded_at.to_i) }
+          rescue StandardError => e
+            logger.error "Unable to load from line #{i} '#{line}': #{e}"
+          end
+        end
+        @images
+      end
+
+      def image(img)
+        @images[img.name] ||= { id: img.id, loaded_at: Time.now }
+      end
+
+      def save(imgs = parse_images, skip_older: Time.at(0))
+        @images = @images.delete_if do |_k, img|
+          img[:loaded_at] < skip_older
+        end
+
+        imgs.each {|m| image(m) }
+
+        File.open(@filename, 'w+') do |file|
+          @images.each do |name, data|
+            file.puts("#{name} #{data[:id]} #{data[:loaded_at].to_i}")
+          end
+        end
+      end
+
+    end
+  end
+end
+

data/lib/gitlab_janitor/image_cleaner.rb

@@ -0,0 +1,140 @@
+require 'open3'
+require 'redis'
+
+module GitlabJanitor
+  class ImageCleaner < BaseCleaner
+
+    class Model < BaseCleaner::Model
+
+      attr_reader :store
+
+      def initialize(model, name, store)
+        super(model)
+        @store = store
+        @name = name
+
+        info['_Age'] = (Time.now - Time.at(loaded_at)).round(0)
+      end
+
+      def loaded_at
+        store.image(self)[:loaded_at]
+      end
+
+      def name
+        @name || id
+      end
+
+      def age
+        info['_Age']
+      end
+
+      def age_text
+        Fugit::Duration.parse(age).deflate.to_plain_s
+      end
+
+      def id
+        info['id']
+      end
+
+    end
+
+    require_relative 'image_cleaner/store'
+
+    attr_reader :store
+
+    def initialize(image_store:, redis: nil, redis_list: 'gitlab-janitor:images_force_clean', **kwargs)
+      super(**kwargs)
+      @store = Store.new(filename: image_store, logger: logger)
+      @redis_url = redis
+      @redis_list = redis_list
+    end
+
+    def do_clean(remove: false)
+      store.load
+
+      force_clean(remove: remove)
+
+      to_remove, keep = prepare(store.parse_images)
+      store.save(skip_older: Time.now - @deadline)
+
+      return if to_remove.empty?
+
+      keep.each {|m| logger.debug("  KEEP #{m.name}") }
+
+      if remove
+        logger.info 'Removing images...'
+        to_remove.each do |model|
+          return false if exiting?
+
+          logger.tagged(model.name) do
+            logger.debug '   Removing...'
+            log_exception('Remove') { out, _status = Open3.capture2e("docker rmi #{model.name}"); logger.info(out) }
+            logger.debug '   Removing COMPLETED'
+          end
+        end
+      else
+        logger.info 'Skip removal due to dry run'
+      end
+    ensure
+      if remove
+        logger.info 'docker image prune -f'
+        out, _status = Open3.capture2e("docker image prune -f")
+        logger.info(out)
+      end
+    end
+
+    def prepare(images)
+      to_remove = images
+
+      @logger.info("Selected images: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
+      @logger.info("Filtering images by deadline: older than #{Fugit::Duration.parse(@deadline).deflate.to_plain_s}...")
+      to_remove = select_by_deadline(to_remove)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], images
+      end
+      @logger.info("Filtered images: \n#{to_remove.map{|c| "  !! #{format_item(c)}" }.join("\n")}")
+
+      [to_remove, (images - to_remove)]
+    end
+
+    def format_item(model)
+      "#{model.loaded_at} Age:#{model.age_text.ljust(13)} #{model.name.first(60).ljust(60)}"
+    end
+
+    def select_by_deadline(images)
+      images.select do |model|
+        model.age > deadline
+      end
+    end
+
+    def force_clean(remove: false)
+      return if @redis_url.nil?
+      logger.info("Force clean image from #{@redis_url}/#{@redis_url}...")
+
+      redis = Redis.new(url: @redis_url)
+      redis.ltrim(@redis_list, 0, 10)
+      now = Time.now
+      redis.lrange(@redis_list, 0, -1).each do |pair|
+        image, ts = pair.split('|')
+        if (now - Time.at(ts.to_i)) > 10.seconds
+          if remove
+            logger.info("Force clean #{image}")
+            log_exception('Remove') { out, _status = Open3.capture2e("docker rmi #{image}"); logger.info(out) }
+          else
+            logger.info("Skip Force clean #{image} due to dry run")
+          end
+        else
+          logger.info("Delay force clean #{image} by time")
+        end
+      rescue StandardError => e
+        logger.error("Error from force line: '#{pair}': #{e.inspect}")
+      end
+    rescue StandardError => e
+      logger.error("Unable to retrieve data from redis: #{e}")
+    end
+
+  end
+end
+

data/lib/{gitlab-janitor → gitlab_janitor}/utils.rb

@@ -1,11 +1,7 @@
-module GitlabJanitor
-
-  class Fmt < ActiveSupport::Logger::SimpleFormatter       
-    def call(severity, timestamp, progname, msg)
-      super
-    end
-  end
+require 'logger'
+require 'active_support/all'
 
+module GitlabJanitor
   class Util
 
     TERM_SIGNALS = %w[INT TERM].freeze
@@ -21,27 +17,25 @@ module GitlabJanitor
       end
 
       def logger
-        $logger
+        $logger ||= ActiveSupport::TaggedLogging.new(Logger.new(STDOUT)).tap do |logger|
+          logger.level = ENV.fetch('LOG_LEVEL', Logger::INFO)
+          formatter = Logger::Formatter.new
+          formatter.extend ActiveSupport::TaggedLogging::Formatter
+          logger.formatter = formatter
+        end
       end
 
       def setup
         STDOUT.sync = true
         STDERR.sync = true
 
-        $logger = ActiveSupport::TaggedLogging.new(Logger.new(STDOUT))
-        $logger.level = ENV.fetch('LOG_LEVEL', Logger::INFO)
-        formatter = Logger::Formatter.new
-        formatter.extend ActiveSupport::TaggedLogging::Formatter
-        $logger.formatter = formatter
-
-         
-
         initialize_signal_handlers
 
         String.class_eval do
           def to_bool
             return true   if self == true   || self =~ (/(true|t|yes|y|1)$/i)
             return false  if self == false  || self.blank? || self =~ (/(false|f|no|n|0)$/i)
+
             raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
           end
         end
@@ -50,10 +44,10 @@ module GitlabJanitor
       def initialize_signal_handlers
         TERM_SIGNALS.each do |sig|
           trap(sig) do |*_args|
-            TERM_SIGNALS.each do |sig|
-              trap(sig) do |*_args|
-                STDERR.puts 'Forcing exit!'
-                Kernel::exit!(1)
+            TERM_SIGNALS.each do |s|
+              trap(s) do |*_args|
+                warn 'Forcing exit!'
+                Kernel.exit!(1)
               end
             end
 
@@ -66,4 +60,5 @@ module GitlabJanitor
     end
 
   end
-end
+end
+

data/lib/gitlab_janitor/version.rb

@@ -0,0 +1,6 @@
+module GitlabJanitor
+
+  VERSION = '1.0.2'.freeze
+
+end
+

data/lib/gitlab_janitor/volume_cleaner.rb

@@ -0,0 +1,120 @@
+module GitlabJanitor
+  class VolumeCleaner < BaseCleaner
+
+    class Model < BaseCleaner::Model
+
+      def initialize(model)
+        super(model)
+
+        info['_Age'] = (Time.now - Time.parse(created_at)).round(0)
+      end
+
+      def created_at
+        info['CreatedAt']
+      end
+
+      def name
+        info['Name']
+      end
+
+      def age
+        info['_Age']
+      end
+
+      def age_text
+        Fugit::Duration.parse(age).deflate.to_plain_s
+      end
+
+      def mountpoint
+        info['Mountpoint']
+      end
+
+    end
+
+    attr_reader :includes
+
+    def initialize(includes: [''], **kwargs)
+      super(**kwargs)
+      @includes = includes
+    end
+
+    def do_clean(remove: false)
+      to_remove, keep = prepare(Docker::Volume.all.map{|m| Model.new(m) })
+
+      return if to_remove.empty?
+
+      keep.each {|m| logger.debug("  KEEP #{m.name}") }
+
+      if remove
+        logger.info 'Removing volumes...'
+        to_remove.each do |model|
+          return false if exiting?
+
+          logger.tagged(model.name.first(10)) do
+            logger.debug '   Removing...'
+            log_exception('Remove') { model.remove }
+            logger.debug '   Removing COMPLETED'
+          end
+        end
+      else
+        logger.info 'Skip removal due to dry run'
+      end
+    end
+
+    def prepare(volumes)
+      @logger.debug('Selecting unnamed volumes...')
+      to_remove = select_unnamed(volumes)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], volumes
+      end
+      @logger.info("Selected volumes: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
+      @logger.debug("Selecting volumes by includes #{@includes}...")
+      to_remove += select_by_name(volumes)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], images
+      end
+      @logger.info("Selected volumes: \n#{to_remove.map{|c| "  + #{format_item(c)}" }.join("\n")}")
+
+      @logger.info("Filtering volumes by deadline: older than #{Fugit::Duration.parse(@deadline).deflate.to_plain_s}...")
+      to_remove = select_by_deadline(to_remove)
+      if to_remove.empty?
+        @logger.info('Noting to remove.')
+        return [], volumes
+      end
+      @logger.info("Filtered volumes: \n#{to_remove.map{|c| "  !! #{format_item(c)}" }.join("\n")}")
+
+      [to_remove, (volumes - to_remove)]
+    end
+
+    def format_item(model)
+      "#{Time.parse(model.created_at)} Age:#{model.age_text.ljust(13)} #{model.name.first(10).ljust(10)} #{model.mountpoint}"
+    end
+
+    def select_by_name(volumes)
+      volumes.select do |model|
+        @includes.any? do |pattern|
+          File.fnmatch(pattern, model.name)
+        end
+      end
+    end
+
+    SHA_RX = /^[a-zA-Z0-9]{64}$/.freeze
+
+    def select_unnamed(volumes)
+      volumes.select do |model|
+        SHA_RX.match(model.name)
+      end
+    end
+
+    def select_by_deadline(volumes)
+      volumes.select do |model|
+        model.age > deadline
+      end
+    end
+
+  end
+end
+

data/lib/gitlab_janitor.rb

@@ -0,0 +1,8 @@
+require_relative 'gitlab_janitor/version'
+require_relative 'gitlab_janitor/utils'
+require_relative 'gitlab_janitor/base_cleaner'
+require_relative 'gitlab_janitor/container_cleaner'
+require_relative 'gitlab_janitor/volume_cleaner'
+require_relative 'gitlab_janitor/image_cleaner'
+require_relative 'gitlab_janitor/cache_cleaner'
+

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-janitor
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 1.0.2.92939
 platform: ruby
 authors:
 - Samoilenko Yuri
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-01 00:00:00.000000000 Z
+date: 2022-08-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -101,13 +115,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: docker-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -115,21 +143,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: fugit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: docker-api
+  name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,7 +171,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fugit
+  name: optparse
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -157,7 +185,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: optparse
+  name: tzinfo-data
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -188,11 +216,15 @@ files:
 - docker-compose.yml
 - gitlab-janitor.gemspec
 - lib/gitlab-janitor.rb
-- lib/gitlab-janitor/base-cleaner.rb
-- lib/gitlab-janitor/container-cleaner.rb
-- lib/gitlab-janitor/utils.rb
-- lib/gitlab-janitor/version.rb
-- lib/gitlab-janitor/volume-cleaner.rb
+- lib/gitlab_janitor.rb
+- lib/gitlab_janitor/base_cleaner.rb
+- lib/gitlab_janitor/cache_cleaner.rb
+- lib/gitlab_janitor/container_cleaner.rb
+- lib/gitlab_janitor/image_cleaner.rb
+- lib/gitlab_janitor/image_cleaner/store.rb
+- lib/gitlab_janitor/utils.rb
+- lib/gitlab_janitor/version.rb
+- lib/gitlab_janitor/volume_cleaner.rb
 homepage: https://github.com/RnD-Soft/gitlab-janitor
 licenses:
 - MIT
@@ -212,7 +244,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.8
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: GitLab Janitor is a tool to automatically manage stalled containers when