artifactory-cleaner 1.0.1

data/exe/artifactory-cleaner

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+##
+# Artifactory Cleaner CLI
+# This file provides the command-line interface to Artifactory Cleaner. It is just an entrypoint to the
+# Artifactory::Cleaner::CLI class; all the work is done there
+# @see Artifactory::Cleaner::CLI
+
+require 'bundler/setup'
+require 'artifactory/cleaner'
+
+Artifactory::Cleaner::CLI.start(ARGV)

data/lib/artifactory/cleaner.rb

@@ -0,0 +1,17 @@
+require "artifactory/cleaner/version"
+require "artifactory/cleaner/util"
+require "artifactory/cleaner/discovered_artifact"
+require "artifactory/cleaner/artifact_bucket"
+require "artifactory/cleaner/artifact_bucket_collection"
+require "artifactory/cleaner/artifact_filter"
+require "artifactory/cleaner/artifact_filter_rule"
+require "artifactory/cleaner/discovery_worker"
+require "artifactory/cleaner/controller"
+require "artifactory/cleaner/cli"
+
+module Artifactory
+  module Cleaner
+    class Error < StandardError; end
+
+  end
+end

data/lib/artifactory/cleaner/artifact_bucket.rb

@@ -0,0 +1,102 @@
+require 'forwardable'
+require 'artifactory'
+
+module Artifactory
+  module Cleaner
+
+    ##
+    # A collection of Artifacts within a date range
+    #
+    # An Artifactory::Cleaner::ArtifactBucket represents an "age bucket" when analyzing Artifact usage; Artifacts are
+    # grouped into buckets of time to aid in developing an archive strategy.
+    #
+    # Artifactory::Cleaner::ArtifactBucket is largely just an Array of Artifactory::Resource::Artifact instances, with
+    # logic to maintain a filesize count and properties fr the age of the artifacts within.
+    #
+    # This class works with the Artifactory::Cleaner::ArtifactBucketCollection class, which maintains a collection of
+    # Artifactory::Cleaner::ArtifactBucket instances and handles selecting the proper one for a given Artifact
+    class ArtifactBucket
+      extend Forwardable
+      include Enumerable
+
+      attr_reader :min
+      attr_reader :max
+      attr_reader :filesize
+
+      ##
+      # ArtifactBucket constructor
+      #
+      # Params:
+      # +min+:: Lower bound (in days) for the age of artifacts this bucket should contain
+      # +max+:: Upper bound (in days) for the age of artifacts this bucket should contain, defaults to none (infinity)
+      def initialize(min,max=nil)
+        @min = min
+        @max = max.nil? ? Float::INFINITY : max
+        @filesize = 0
+        @collection = []
+      end
+
+      delegate [:[], :slice, :clear, :first, :last, :delete, :shift, :length, :empty?, :each] => :@collection
+
+      ##
+      # Given an age (in days) return true if this bucket covers that age (if it's within the min and max of this bucket)
+      def covers?(age)
+        age >= @min && age < @max
+      end
+
+      ##
+      # Update an artifact in the bucket
+      #
+      # TODO: This method does not validate if the artifact still belongs in this bucket by age
+      def []=(key, artifact)
+        raise TypeError, "expected Artifactory::Resource::Artifact, got #{artifact.class.name}" unless artifact.is_a? Artifactory::Resource::Artifact
+        @filesize -= @collection[key].size if @collection[key].is_a? Artifactory::Resource::Artifact
+        @filesize += artifact.size
+        @collection[key] = artifact
+      end
+
+      ##
+      # Add an artifact to the end of this bucket
+      #
+      # Calls push on the Array which backs this bucket
+      #
+      # Aliased as method `<<`
+      #
+      # TODO: This method does not validate if the artifact belongs in this bucket by age
+      #
+      # @see: Array#push
+      def push(artifact)
+        raise TypeError, "expected Artifactory::Resource::Artifact, got #{artifact.class.name}" unless artifact.is_a? Artifactory::Resource::Artifact
+        @collection.push artifact
+        @filesize += artifact.size
+        self
+      end
+      alias_method :<<, :push
+
+      ##
+      # Add an artifact to the beginning of this bucket
+      #
+      # Calls unshift on the Array which backs this bucket
+      #
+      # TODO: This method does not validate if the artifact belongs in this bucket by age
+      #
+      # @see Array#unshift
+      def unshift(artifact)
+        raise TypeError, "expected Artifactory::Resource::Artifact, got #{artifact.class.name}" unless artifact.is_a? Artifactory::Resource::Artifact
+        @collection.unshift artifact
+        @filesize += artifact.size
+        self
+      end
+
+      ##
+      # Recalculate the file size of this bucket by adding up the size of all artifacts it contains
+      #
+      # This method forces recalculation of the total fize size of all artifacts within this bucket; the filesize is
+      # tracked automatically as artifacts are added, so thi method should be unnecessary. It is av available in case
+      # the tracking built in to `push`/`unshift`/`[]=` hsa a bug, or in case artifact sizes somehow change
+      def recalculate_filesize
+        @filesize = @collection.reduce(0) {|sum,asset| sum + asset.size}
+      end
+    end
+  end
+end

data/lib/artifactory/cleaner/artifact_bucket_collection.rb

@@ -0,0 +1,118 @@
+require 'forwardable'
+require 'artifactory'
+
+module Artifactory
+  module Cleaner
+
+    ##
+    # Organize Artifacts by age bucket for analysis
+    #
+    # An Artifactory::Cleaner::ArtifactBucketCollection represents "age buckets" used for analyzing Artifact usage.
+    # Artifacts are grouped into buckets of time to aid in developing an archive strategy. This class maintains a
+    # list of buckets and handles the logic for sorting Artifacts into those buckets.
+    #
+    # Artifactory::Cleaner::ArtifactBucketCollection is largely just an Array of Artifactory::Cleaner::ArtifactBucket
+    # instances, with logic to sort and select them and logic to distribute Artifactory::Resource::Artifact instances
+    # into the proper Artifactory::Cleaner::ArtifactBucket
+    class ArtifactBucketCollection
+      extend Forwardable
+      include Enumerable
+
+      def initialize(buckets = [30,60,90,180,365,730,1095,nil])
+        @buckets = []
+        define_buckets(buckets)
+      end
+
+      delegate [:length, :each, :first, :last, :each] => :@buckets
+
+      ##
+      # Remove all Artifacts from this collection
+      #
+      # Calls `clear` on every bucket within this collection
+      def clear
+        @buckets.each &:clear
+      end
+
+      ##
+      # Adjust the bucket sizes within this collection
+      #
+      # Given an Enumerable of ages (as integer values of days) define buckets representing those periods within this
+      # collection. This method is similar to the constructor: provide an Enumerable where each value represents a
+      # bucket size and new buckets will be added to this collection representing the ages (in days) contained within
+      # `bucket_list`
+      #
+      # TODO: This will not update older buckets or move artifacts around, so if buckets were already defined then this
+      # method may result in an invalid configuration, E.G. overlapping buckets or artifacts which are no longer in
+      # the desired buckets. For best results, call this method on an ArtifactBucketCollection for which you already
+      # know the bucket sizes and to which no artifacts have yet been added
+      def define_buckets(bucket_list)
+        last_size = 0
+        bucket_list.each do |size|
+          @buckets << Artifactory::Cleaner::ArtifactBucket.new(last_size,size)
+          last_size = size
+        end
+      end
+
+      ##
+      # Return an Array containing the bucket sizes of this collection.
+      #
+      # Returns the `max` property from every bucket within this collection, thus representing the bucket sizes this
+      # collection contains (as a properly configured ArtifactBucketCollection has the min of each bucket set to the max
+      # of the previous bucket, thus covering an entire time range)
+      def bucket_sizes
+        @buckets.map &:max
+      end
+
+      ##
+      # Total number of Artifacts within this collection
+      #
+      # Returns the sum of the length of all buckets within this collection
+      def artifact_count
+        @buckets.reduce(0) { |sum, bkt| sum + bkt.length }
+      end
+
+      ##
+      # Add a new artifact to this collection
+      #
+      # Given an Artifactory::Resource::Artifact `artifact`, find the proper ArtifactBucket within this ArtifactBucketCollection
+      # and add the artifact ton that bucket
+      #
+      # Aliased as `<<`
+      def add(artifact)
+        age = (Time.now - Artifactory::Cleaner::DiscoveredArtifact.latest_date_from(artifact))/(3600*24)
+
+        if (bucket = @buckets.find {|b| b.covers? age})
+          bucket << artifact
+        else
+          raise RangeError, "No bucket available for an artifact of age #{age.floor} days"
+        end
+        self
+      end
+      alias_method :<<, :add
+
+      ##
+      # Accessor for a bucket of a given age
+      #
+      # Returns the bucket which covers the period `age` (represented as an artifact age, in days)
+      #
+      # Aliased as `[]`
+      def bucket(age)
+        @buckets.find {|b| b.covers? age}
+      end
+      alias_method :[], :bucket
+
+      ##
+      # Human-readable summary of this collection
+      #
+      # Returns a string summarizing each bucket within this collection: how many packages and what filesize each bucket
+      # contains. Used when analyzing artifact searches: artifacts discovered from a search are placed into an
+      # ArtifactBucketCollection and then this report can be produced to describe how old the artifacts are and where
+      # opportunities for cleaning exist.
+      def report
+        buckets.map {|bucket|
+          "#{bucket.length} packages between #{bucket.min} and #{bucket.max} days, totaling #{Artifactory::Cleaner::Util::filesize bucket.filesize}"
+        }.join("\n")
+      end
+    end
+  end
+end

data/lib/artifactory/cleaner/artifact_filter.rb

@@ -0,0 +1,146 @@
+module Artifactory
+  module Cleaner
+    ##
+    # Filter a list of artifacts based on a series of include/exclude rules
+    #
+    # Artifactory::Cleaner::ArtifactFilter is used to filter a list of artifacts based on rules. It is both a whitelist
+    # and a blacklist: it maintains a list of rules in sorted priority order, and the first rule which matches a given
+    # artifact determines the action for that artifact (include or exclude)
+    #
+    # Rules are stored in ascending priority order with lower numbers being greater priority. (Think "Priority 1" or
+    # process queue scheduling via `nice` value under Linux.
+    class ArtifactFilter
+      extend Forwardable
+      include Enumerable
+
+      ##
+      # ArtifactFilter constructor
+      def initialize()
+        @rules = []
+        @sorted = false
+        @default_action = :include
+      end
+
+      attr_accessor :default_action
+
+      delegate [:length, :clear, :empty?] => :@rules
+
+      ##
+      # Access a rule by index
+      def [](*args)
+        sort_if_needed
+        @rules[*args]
+      end
+
+      ##
+      # Update a given rule
+      def []=(key, rule)
+        raise TypeError, "expected Artifactory::Cleaner::ArtifactFilterRule, got #{rule.class.name}" unless rule.is_a? Artifactory::Cleaner::ArtifactFilterRule
+        @rules[key] = rule
+        @sorted = false
+      end
+
+      ##
+      # Slice the filter rules, see Array#slice
+      def slice(*args, &block)
+        sort_if_needed
+        @rules.slice(*args, &block)
+      end
+
+      ##
+      # Search for a rule
+      def bsearch(*args, &block)
+        sort_if_needed
+        @rules.bsearch(*args, &block)
+      end
+
+      ##
+      # Get the first (numerically first priority) rule
+      def first
+        sort_if_needed
+        @rules.first
+      end
+
+      ##
+      # Get the last (numerically last priority) rule
+      def last
+        sort_if_needed
+        @rules.last
+      end
+
+      ##
+      # Iterate over all rules (See Enumerable#each)
+      def each(&block)
+        sort_if_needed
+        @rules.each(&block)
+      end
+
+      ##
+      # Add rules to this filter
+      #
+      # Like Array#push this method adds a rule to the end of the array, however the array will be sorted in priority
+      # order before usage so addition at the end or the beginning is somewhat meaningless
+      def push(rule)
+        raise TypeError, "expected Artifactory::Cleaner::ArtifactFilterRule, got #{rule.class.name}" unless rule.is_a? Artifactory::Cleaner::ArtifactFilterRule
+        @rules.push rule
+        @sorted = false
+        self
+      end
+      alias_method :<<, :push
+
+      ##
+      # Add a rule to this filter
+      #
+      # Like Array#unshift this method adds a rule to the beginning of the array, however the array will be sorted in
+      # priority order before usage so addition at the end or the beginning is somewhat meaningless
+      def unshift(rule)
+        raise TypeError, "expected Artifactory::Cleaner::ArtifactFilterRule, got #{rule.class.name}" unless rule.is_a? Artifactory::Cleaner::ArtifactFilterRule
+        @rules.unshift rule
+        @sorted = false
+        self
+      end
+
+      ##
+      # Ensure the filterset is sorted properly. Should not need to be called manually
+      def sort!
+        sort_if_needed
+        self
+      end
+
+      ##
+      # Filter a given Artifactory::Resource::Artifact and return the action which should be taken
+      #
+      # Returns a symbol from the rule which matches this artifact, or the default action (:include) if no rules matched
+      def action_for(artifact)
+        sort_if_needed
+        @rules.each do |rule|
+          action = rule.action_for artifact
+          return action if action
+        end
+        @default_action
+      end
+
+
+      ##
+      # Filter a collection of Artifactory::Resource::Artifact instances, returning the ones for which the action matches
+      #
+      # Takes an Enumerable filled with Artifactory::Resource::Artifact instances and returns a filtered enumerable for
+      # which all artifacts matched the desired action after applying the filter rules to each item. `action` defaults
+      # to :include but can be changed if desired
+      #
+      # Unlike Array#filter +this method does not take a block+
+      def filter(artifacts, action = :include)
+        artifacts.filter {|artifact| action_for(artifact) == action}
+      end
+
+      private
+
+      ##
+      # Sort the array but avoid needless sorting
+      def sort_if_needed
+        @rules.sort! unless @sorted
+        @sorted = true
+      end
+    end
+  end
+end

data/lib/artifactory/cleaner/artifact_filter_rule.rb

@@ -0,0 +1,81 @@
+module Artifactory
+  module Cleaner
+    ##
+    # Filter a collection of artifacts based on include/deny rules
+    #
+    # The Artifactory::Cleaner::ArtifactFilterRile class represents a whitelist or blacklist entry. It matches a package
+    # and then targets that package for inclusion or exclusion.
+    class ArtifactFilterRule
+      include Comparable
+
+      def initialize(action: :include, priority: 0, property: :uri, regex: //)
+        @regex = regex if regex.is_a? Regexp
+        @action = action
+        @priority = priority.to_i
+        @property = property.to_sym
+      end
+
+      attr_reader :regex
+
+      ##
+      # Change the regex of this rule
+      def regex=(re)
+        raise TypeError, 'Expected a Regexp' unless re.is_a? Regexp
+        @regex = re
+      end
+      alias_method :regexp, :regex
+      alias_method :regexp=, :regex=
+
+      # TODO: Allow changing priority. Right now this would cause problems because if the rule is in a filer, the filter won't be sorted properly after this change.
+      attr_reader :priority
+      attr_accessor :property
+      attr_accessor :action
+
+      ##
+      # Does this rule trigger an action on a given artifact?
+      #
+      # This method returns true if the given artifact matches the criteria of this rule, and the rule is of type :include
+      def action_for(artifact)
+        if matches?(artifact)
+          @action
+        else
+          false
+        end
+      end
+
+      ##
+      # Does this rule determine that an artifact should be included?
+      #
+      # This method returns true if the given artifact matches the criteria of this rule, and the rule is of type :include
+      def includes?(artifact)
+        @type == :include && matches?(artifact)
+      end
+
+      ##
+      # Does this rule determine that an artifact should be excluded?
+      #
+      # This method returns true if the given artifact matches the criteria of this rule, and the rule is of type :exclude
+      def excludes?(artifact)
+        @type == :exclude && matches?(artifact)
+      end
+
+      ##
+      # Does this rule match a given package?
+      #
+      # Returns true if the `property` of a given artifact matches the `regex`
+      def matches?(artifact)
+        @regex.is_a? Regexp and @regex.match?(artifact.send(@property).to_s)
+      end
+
+      ##
+      # Compare priority with another rule
+      def <=>(other_rule)
+        if other_rule.is_a? ArtifactFilterRule
+          @priority <=> other_rule.priority
+        else
+          nil
+        end
+      end
+    end
+  end
+end