artifactory-cleaner 1.0.1

data/lib/artifactory/cleaner/discovered_artifact.rb

@@ -0,0 +1,71 @@
+require 'uri'
+require 'artifactory'
+
+module Artifactory
+  module Cleaner
+    ##
+    # An Artifact discovered during a repository search
+    #
+    # This class is a wrapper of Artifactory::Resource::Artifact because the parent class does not have a concept of
+    # `last_downloaded` nor the most recent date for any action on an Artifact. These are important to deciding if an
+    # Artifcat should be deleted
+    class DiscoveredArtifact < Artifactory::Resource::Artifact
+      ##
+      # Time representing the date and time this artifact was last downloaded by a client (presumably to be installed)
+      attribute :last_downloaded
+
+      ##
+      # What's the earliest Time of any of the date/time properties on this object?
+      def earliest_date
+        self.class.earliest_date_from(self)
+      end
+
+      ##
+      # What's the most recent Time of any of the date/time properties on this object?
+      def latest_date
+        self.class.latest_date_from(self)
+      end
+
+      ##
+      # The filename componet (basename) of this artifact's URL
+      def filename
+        uri = URI(self.uri)
+        File.basename(uri.path)
+      end
+
+      ##
+      # A string representation of this artifact
+      def to_s
+        "#<DiscoveredArtifact #{filename} last accessed #{latest_date}>"
+      end
+
+      ##
+      # Given an Artifactory::Resource::Artifact, return the value of the earliest date property on that object
+      #
+      # Designed to answer the question "what's the first time anything happened to a given Artifact?", this method
+      # returns the earliest (longest ago) date from the given artifact's created, last modified and last downloaded
+      # timestamps.
+      def self.earliest_date_from(artifact)
+        [
+            artifact.created,
+            artifact.last_modified,
+            artifact.respond_to?(:last_downloaded) ? artifact.last_downloaded : nil,
+        ].compact.sort.first
+      end
+
+      ##
+      # Given an Artifactory::Resource::Artifact, return the value of the latest date property on that object
+      #
+      # Designed to answer the question "what's the most recent interaction with a given Artifact?", this method
+      # returns the latest (most recent) date from the given artifact's created, last modified and last downloaded
+      # timestamps.
+      def self.latest_date_from(artifact)
+        [
+            artifact.created,
+            artifact.last_modified,
+            artifact.respond_to?(:last_downloaded) ? artifact.last_downloaded : nil,
+        ].compact.sort.last
+      end
+    end
+  end
+end

data/lib/artifactory/cleaner/discovery_worker.rb

@@ -0,0 +1,126 @@
+module Artifactory
+  module Cleaner
+    ##
+    # Helper class representing Threads spawned to discover Artifacts
+    class DiscoveryWorker
+      def initialize(processing_queues, artifactory_client)
+        @running = false
+        @working = false
+        @thread = nil
+        @queues = processing_queues
+        @artifactory_client = artifactory_client
+      end
+
+      ##
+      # Is this DiscoveryWorker running, listening to the queue and processing requests
+      def running?
+        @running
+      end
+
+      ##
+      # Is this DiscoveryWorker currently processing a request?
+      #
+      # when #running? is true and #working? is not, then the worker is idle, blocked, waiting for an action.
+      #
+      # when #working? is true, there will be at least one more result pushed to the outgoing queue when the current
+      # request finishes
+      def working?
+        @working
+      end
+
+      ##
+      # Is the Thread for this worker alive?
+      def alive?
+        @thread ? @thread.alive? : false
+      end
+
+      ##
+      # Start the DiscoveryWorker and begin processing from the queue
+      def start
+        @running = true
+        @thread = Thread.new do
+          while running?
+            process @queues.incoming.pop(false)
+          end
+        end
+        self
+      end
+
+      ##
+      # Stop the Thread and re-join the parent
+      def shutdown(timeout = 300)
+        @running = false
+        @thread.join(timeout) if @thread and @thread.alive?
+      end
+
+      ##
+      # Forcibly kill the Thread and destroy it
+      def stop
+        @running = false
+        @thread.kill if @thread and @thread.alive?
+        @thread = nil
+      end
+      alias_method :kill, :stop
+
+      ##
+      # String representation of this DiscoveryWorker and it's status
+      def to_s
+        "#<#{self.class}:#{self.object_id}; #{running? ? 'running' : 'not running'}, #{working? ? 'working' : 'idle'}, #{alive? ? 'alive' : 'dead'}>"
+      end
+
+      private
+
+      ##
+      # debug / verbose logging
+      # TODO: add a verbose conditional, maybe a different log destination
+      def log(msg)
+        STDERR.puts msg
+      end
+
+      ##
+      # pop from the incoming queue, process, push result to outgoing
+      def process(payload)
+        @working = true
+        begin
+          @queues.outgoing.push discover_artifact_from payload
+        rescue => ex
+          return ex # TODO: Should this be pushed to the outgoing queue instead?
+        ensure
+          @working = false
+        end
+      end
+
+      ##
+      # main method, given artifact info, generate a Artifactory::Cleaner::DiscoveredArtifact
+      #
+      # this method makes HTTP calls to the Artifactory API
+      def discover_artifact_from(artifact_info, retries = 10)
+        artifact = nil
+        while retries > 0
+          begin
+            retries -= 1
+            artifact = Artifactory::Cleaner::DiscoveredArtifact.from_url(artifact_info["uri"], client: @artifactory_client)
+            artifact.last_downloaded = Time.parse(artifact_info["lastDownloaded"]) unless artifact_info["lastDownloaded"].to_s.empty?
+            return artifact
+          rescue Net::OpenTimeout, Artifactory::Error::ConnectionError => err
+            artifact = err
+            if retries
+              log "[WARN] Connection Failure attempting to reach Artifactory API: #{err}; Retrying in 10 seconds"
+              sleep 10
+            end
+          rescue Artifactory::Error::HTTPError => err
+            artifact = err
+            if err.code == 404
+              log "[WARN] HTTP 404 Not Found fetching: #{artifact["uri"]}"
+              return nil
+            else
+              retries = min(retries, 1)
+              log "[ERROR] HTTP Error while fetching an artifact from a usage search: #{err}; #{retries ? 'Will' : 'Will not'} retry"
+            end
+          end
+        end
+        artifact
+      end
+    end
+  end
+end

data/lib/artifactory/cleaner/util.rb

@@ -0,0 +1,21 @@
+module Artifactory
+  module Cleaner
+    module Util
+      ##
+      # Given a size in bytes, return a "human readable" size with a unit suffix (MiB, GiB)
+      #
+      # Taken from https://stackoverflow.com/a/47486815/75801
+      # Used instead of a gem due to simplicity and speed
+      def self.filesize(size)
+        units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'Pib', 'EiB']
+
+        return '0.0 B' if size == 0
+        exp = (Math.log(size) / Math.log(1024)).to_i
+        exp += 1 if (size.to_f / 1024 ** exp >= 1024 - 0.05)
+        exp = 6 if exp > 6
+
+        '%.1f %s' % [size.to_f / 1024 ** exp, units[exp]]
+      end
+    end
+  end
+end

data/lib/artifactory/cleaner/version.rb

@@ -0,0 +1,7 @@
+module Artifactory
+  module Cleaner
+    ##
+    # Version number of the Artifactory::Cleaner gem
+    VERSION = "1.0.1"
+  end
+end

metadata

@@ -0,0 +1,252 @@
+--- !ruby/object:Gem::Specification
+name: artifactory-cleaner
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Josh Gitlin
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.20
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.20
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: sysexits
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: artifactory
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Will provide maintainence tasks for Artifactory, like cleaning up old
+  artifacts
+email:
+- jgitlin@pinnacle21.com
+executables:
+- artifactory-cleaner
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".idea/.gitignore"
+- ".idea/checkstyle-idea.xml"
+- ".idea/dictionaries/jgitlin.xml"
+- ".idea/misc.xml"
+- ".idea/modules.xml"
+- ".idea/vcs.xml"
+- ".rspec"
+- ".rspec_status"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- artifactory-cleaner.gemspec
+- artifactory-cleaner.iml
+- bin/console
+- bin/setup
+- doc/rdoc/Artifactory.html
+- doc/rdoc/Artifactory/Cleaner.html
+- doc/rdoc/Artifactory/Cleaner/ArtifactBucket.html
+- doc/rdoc/Artifactory/Cleaner/ArtifactBucketCollection.html
+- doc/rdoc/Artifactory/Cleaner/ArtifactFilter.html
+- doc/rdoc/Artifactory/Cleaner/ArtifactFilterRule.html
+- doc/rdoc/Artifactory/Cleaner/CLI.html
+- doc/rdoc/Artifactory/Cleaner/Controller.html
+- doc/rdoc/Artifactory/Cleaner/DiscoveredArtifact.html
+- doc/rdoc/Artifactory/Cleaner/DiscoveryWorker.html
+- doc/rdoc/Artifactory/Cleaner/Error.html
+- doc/rdoc/Artifactory/Cleaner/SpecHelpers.html
+- doc/rdoc/Artifactory/Cleaner/Util.html
+- doc/rdoc/CODE_OF_CONDUCT_md.html
+- doc/rdoc/Float.html
+- doc/rdoc/Gemfile.html
+- doc/rdoc/Gemfile_lock.html
+- doc/rdoc/Object.html
+- doc/rdoc/README_md.html
+- doc/rdoc/Rakefile.html
+- doc/rdoc/artifactory-cleaner_gemspec.html
+- doc/rdoc/artifactory-cleaner_iml.html
+- doc/rdoc/bin/setup.html
+- doc/rdoc/created.rid
+- doc/rdoc/css/fonts.css
+- doc/rdoc/css/rdoc.css
+- doc/rdoc/filterlist_yaml.html
+- doc/rdoc/filters/clean-amzn_yaml.html
+- doc/rdoc/filters/snapshots_yaml.html
+- doc/rdoc/filters/test-filter_yaml.html
+- doc/rdoc/filters/yum-test_yaml.html
+- doc/rdoc/fonts/Lato-Light.ttf
+- doc/rdoc/fonts/Lato-LightItalic.ttf
+- doc/rdoc/fonts/Lato-Regular.ttf
+- doc/rdoc/fonts/Lato-RegularItalic.ttf
+- doc/rdoc/fonts/SourceCodePro-Bold.ttf
+- doc/rdoc/fonts/SourceCodePro-Regular.ttf
+- doc/rdoc/images/add.png
+- doc/rdoc/images/arrow_up.png
+- doc/rdoc/images/brick.png
+- doc/rdoc/images/brick_link.png
+- doc/rdoc/images/bug.png
+- doc/rdoc/images/bullet_black.png
+- doc/rdoc/images/bullet_toggle_minus.png
+- doc/rdoc/images/bullet_toggle_plus.png
+- doc/rdoc/images/date.png
+- doc/rdoc/images/delete.png
+- doc/rdoc/images/find.png
+- doc/rdoc/images/loadingAnimation.gif
+- doc/rdoc/images/macFFBgHack.png
+- doc/rdoc/images/package.png
+- doc/rdoc/images/page_green.png
+- doc/rdoc/images/page_white_text.png
+- doc/rdoc/images/page_white_width.png
+- doc/rdoc/images/plugin.png
+- doc/rdoc/images/ruby.png
+- doc/rdoc/images/tag_blue.png
+- doc/rdoc/images/tag_green.png
+- doc/rdoc/images/transparent.png
+- doc/rdoc/images/wrench.png
+- doc/rdoc/images/wrench_orange.png
+- doc/rdoc/images/zoom.png
+- doc/rdoc/index.html
+- doc/rdoc/js/darkfish.js
+- doc/rdoc/js/jquery.js
+- doc/rdoc/js/navigation.js
+- doc/rdoc/js/navigation.js.gz
+- doc/rdoc/js/search.js
+- doc/rdoc/js/search_index.js
+- doc/rdoc/js/search_index.js.gz
+- doc/rdoc/js/searcher.js
+- doc/rdoc/js/searcher.js.gz
+- doc/rdoc/results/archive-test-4_log.html
+- doc/rdoc/results/buckets-2020-01-31_txt.html
+- doc/rdoc/results/clean-test-2_log.html
+- doc/rdoc/results/clean-test-3_log.html
+- doc/rdoc/results/clean-test-5_log.html
+- doc/rdoc/results/clean-test-6_log.html
+- doc/rdoc/results/clean-test-7_log.html
+- doc/rdoc/results/clean-test-8-real_log.html
+- doc/rdoc/results/clean-test-9_log.html
+- doc/rdoc/results/clean-test1_log.html
+- doc/rdoc/results/yum-test_2020-01-31_log.html
+- doc/rdoc/results/yum-test_dry-run_log.html
+- doc/rdoc/table_of_contents.html
+- exe/artifactory-cleaner
+- lib/artifactory/cleaner.rb
+- lib/artifactory/cleaner/artifact_bucket.rb
+- lib/artifactory/cleaner/artifact_bucket_collection.rb
+- lib/artifactory/cleaner/artifact_filter.rb
+- lib/artifactory/cleaner/artifact_filter_rule.rb
+- lib/artifactory/cleaner/cli.rb
+- lib/artifactory/cleaner/controller.rb
+- lib/artifactory/cleaner/discovered_artifact.rb
+- lib/artifactory/cleaner/discovery_worker.rb
+- lib/artifactory/cleaner/util.rb
+- lib/artifactory/cleaner/version.rb
+homepage: https://bitbucket.org/pinnacle21/artifactory-cleaner/src/master/
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://bitbucket.org/pinnacle21/artifactory-cleaner/src/master/
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6.2
+signing_key: 
+specification_version: 4
+summary: Performs maintainence tasks on Artifactory repositories
+test_files: []