katello 4.7.4 → 4.8.0.rc1

data/app/models/katello/glue/pulp/repo.rb

@@ -1,353 +0,0 @@
-module Katello
-  module Glue::Pulp::Repo
-    # TODO: move into submodules
-    def self.included(base)
-      base.send :include, LazyAccessor
-      base.send :include, InstanceMethods
-
-      base.class_eval do
-        lazy_accessor :pulp_repo_facts,
-                      :initializer => (lambda do |_s|
-                                         if pulp_id
-                                           begin
-                                             Katello.pulp_server.extensions.repository.retrieve_with_details(pulp_id)
-                                           rescue RestClient::ResourceNotFound
-                                             nil # not found = it was not orchestrated yet
-                                           end
-                                         end
-                                       end)
-
-        lazy_accessor :importers,
-                      :initializer => lambda { |_s| pulp_repo_facts["importers"] if pulp_id }
-
-        lazy_accessor :distributors,
-                      :initializer => lambda { |_s| pulp_repo_facts["distributors"] if pulp_id }
-
-        def self.delete_orphaned_content
-          Katello.pulp_server.resources.content.remove_orphans
-        end
-
-        def self.needs_importer_updates(repos, smart_proxy)
-          repos.select do |repo|
-            repo_details = repo.backend_service(smart_proxy).backend_data
-            next unless repo_details
-            capsule_importer = repo_details["importers"][0]
-            !repo.importer_matches?(capsule_importer, smart_proxy)
-          end
-        end
-
-        def self.needs_distributor_updates(repos, smart_proxy)
-          repos.select do |repo|
-            repo_details = repo.backend_service(smart_proxy).backend_data
-            next unless repo_details
-            !repo.distributors_match?(repo_details["distributors"], smart_proxy)
-          end
-        end
-      end
-    end
-
-    module InstanceMethods
-      # TODO: This module is too long. See https://projects.theforeman.org/issues/12584.
-      def last_sync
-        last = self.latest_dynflow_sync
-        last.nil? ? nil : last.to_s
-      end
-
-      def initialize(attrs = nil)
-        if attrs.nil?
-          super
-        else
-          #rename "type" to "cp_type" (activerecord and candlepin variable name conflict)
-          #if attrs.has_key?(type_key) && !(attrs.has_key?(:cp_type) || attrs.has_key?('cp_type'))
-          #  attrs[:cp_type] = attrs[type_key]
-          #end
-
-          attrs_used_by_model = attrs.reject do |k, _v|
-            !self.class.column_defaults.keys.member?(k.to_s) && (!respond_to?(:"#{k.to_s}=") rescue true)
-          end
-          super(attrs_used_by_model)
-        end
-      end
-
-      def srpm_count
-        pulp_repo_facts['content_unit_counts']['srpm']
-      end
-
-      def to_hash
-        pulp_repo_facts.merge(as_json).merge(:sync_state => sync_state)
-      end
-
-      def pulp_scratchpad_checksum_type
-        if SmartProxy.pulp_primary.has_feature?(SmartProxy::PULP_FEATURE)
-          pulp_repo_facts&.dig('scratchpad', 'checksum_type')
-        end
-      end
-
-      def pulp_counts_differ?
-        pulp_counts = pulp_repo_facts[:content_unit_counts]
-        rpms.count != pulp_counts['rpm'].to_i ||
-          srpms.count != pulp_counts['srpm'].to_i ||
-          errata.count != pulp_counts['erratum'].to_i ||
-          package_groups.count != pulp_counts['package_group'].to_i ||
-          docker_manifests.count != pulp_counts['docker_manifest'].to_i ||
-          docker_tags.count != pulp_counts['docker_tag'].to_i
-      end
-
-      def empty_in_pulp?
-        pulp_repo_facts[:content_unit_counts].values.all? { |value| value == 0 }
-      end
-
-      def generate_importer(capsule = ::SmartProxy.default_capsule!)
-        backend_service(capsule).generate_importer
-      end
-
-      def generate_distributors(capsule = ::SmartProxy.default_capsule!)
-        backend_service(capsule).generate_distributors
-      end
-
-      def package_group_count
-        content_unit_counts = 0
-        if self.pulp_repo_facts
-          content_unit_counts = self.pulp_repo_facts[:content_unit_counts][:package_group]
-        end
-        content_unit_counts
-      end
-
-      def clone_file_metadata(to_repo)
-        Katello.pulp_server.extensions.yum_repo_metadata_file.copy(self.pulp_id, to_repo.pulp_id)
-      end
-
-      def unassociate_by_filter(content_type, filter_clauses)
-        criteria = {:type_ids => [content_type], :filters => {:unit => filter_clauses}}
-        case content_type
-        when Katello.pulp_server.extensions.rpm.content_type
-          criteria[:fields] = { :unit => Pulp::Rpm::PULP_SELECT_FIELDS}
-        when Katello.pulp_server.extensions.errata.content_type
-          criteria[:fields] = { :unit => Pulp::Erratum::PULP_SELECT_FIELDS}
-        end
-        Katello.pulp_server.extensions.repository.unassociate_units(self.pulp_id, criteria)
-      end
-
-      def clear_contents
-        tasks = content_types.flat_map { |type| type.unassociate_from_repo(self.pulp_id, {}) }
-
-        tasks << Katello.pulp_server.extensions.repository.unassociate_units(self.pulp_id,
-                   :type_ids => ['rpm'], :filters => {}, :fields => { :unit => Pulp::Rpm::PULP_SELECT_FIELDS})
-        tasks
-      end
-
-      def content_types
-        [Katello.pulp_server.extensions.errata,
-         Katello.pulp_server.extensions.package_group,
-         Katello.pulp_server.extensions.module_stream
-        ]
-      end
-
-      def sync_status
-        self._get_most_recent_sync_status if @sync_status.nil?
-      end
-
-      def sync_state
-        status = sync_status
-        return PulpSyncStatus::Status::NOT_SYNCED if status.nil?
-        status.state
-      end
-
-      def synced?
-        sync_history = self.sync_status
-        !sync_history.nil? && successful_sync?(sync_history)
-      end
-
-      def successful_sync?(sync_history_item)
-        sync_history_item['state'] == PulpTaskStatus::Status::FINISHED.to_s
-      end
-
-      def find_distributor(use_clone_distributor = false)
-        dist_type_id = if use_clone_distributor
-                         case self.content_type
-                         when Repository::YUM_TYPE
-                           Runcible::Models::YumCloneDistributor.type_id
-                         when Repository::PUPPET_TYPE
-                           Runcible::Models::PuppetInstallDistributor.type_id
-                         end
-                       else
-                         case self.content_type
-                         when Repository::YUM_TYPE
-                           Runcible::Models::YumDistributor.type_id
-                         when Repository::PUPPET_TYPE
-                           Runcible::Models::PuppetInstallDistributor.type_id
-                         end
-                       end
-
-        distributors.detect { |dist| dist["distributor_type_id"] == dist_type_id }
-      end
-
-      def sort_sync_status(statuses)
-        statuses.sort! do |a, b|
-          if a['finish_time'].nil? && b['finish_time'].nil?
-            if a['start_time'].nil?
-              1
-            elsif b['start_time'].nil?
-              -1
-            else
-              a['start_time'] <=> b['start_time']
-            end
-          elsif a['finish_time'].nil?
-            if a['start_time'].nil?
-              1
-            else
-              -1
-            end
-          elsif b['finish_time'].nil?
-            if b['start_time'].nil?
-              -1
-            else
-              1
-            end
-          else
-            b['finish_time'] <=> a['finish_time']
-          end
-        end
-        return statuses
-      end
-
-      def unit_type_id
-        case content_type
-        when Repository::YUM_TYPE
-          "rpm"
-        when Repository::DOCKER_TYPE
-          "docker_manifest"
-        when Repository::OSTREE_TYPE
-          "ostree"
-        when Repository::FILE_TYPE
-          "iso"
-        when Repository::DEB_TYPE
-          "deb"
-        when Repository::ANSIBLE_COLLECTION_TYPE
-          "ansible_collection"
-        end
-      end
-
-      def unit_search(options = {})
-        Katello.pulp_server.extensions.repository.unit_search(self.pulp_id, options)
-      end
-
-      def docker?
-        self.content_type == Repository::DOCKER_TYPE
-      end
-
-      def file?
-        self.content_type == Repository::FILE_TYPE
-      end
-
-      def yum?
-        self.content_type == Repository::YUM_TYPE
-      end
-
-      def ostree?
-        self.content_type == Repository::OSTREE_TYPE
-      end
-
-      def deb?
-        self.content_type == Repository::DEB_TYPE
-      end
-
-      def ansible_collection?
-        self.content_type == Repository::ANSIBLE_COLLECTION_TYPE
-      end
-
-      def published?
-        distributors.map { |dist| dist['last_publish'] }.compact.any?
-      end
-
-      def capsule_download_policy(capsule)
-        policy = capsule.download_policy || Setting[:default_proxy_download_policy]
-        if self.yum?
-          if policy == ::SmartProxy::DOWNLOAD_INHERIT
-            self.root.download_policy
-          else
-            policy
-          end
-        end
-      end
-
-      def distributors_match?(capsule_distributors, capsule)
-        generated_distributor_configs = self.generate_distributors(capsule)
-        generated_distributor_configs.all? do |gen_dist|
-          type = gen_dist.class.type_id
-          found_on_capsule = capsule_distributors.find { |dist| dist['distributor_type_id'] == type }
-          found_on_capsule && filtered_distribution_config_equal?(gen_dist.config, found_on_capsule['config'])
-        end
-      end
-
-      def needs_metadata_publish?
-        last_publish = last_publish_task.try(:[], 'finish_time')
-        last_sync = last_sync_task.try(:[], 'finish_time')
-        return false if last_sync.nil?
-        return true if last_publish.nil?
-
-        Time.parse(last_sync) >= Time.parse(last_publish)
-      end
-
-      def last_sync_task
-        tasks = Katello.pulp_server.extensions.repository.sync_status(self.pulp_id)
-        most_recent_task(tasks)
-      end
-
-      def last_publish_task
-        tasks = Katello.pulp_server.extensions.repository.publish_status(self.pulp_id)
-        most_recent_task(tasks, true)
-      end
-
-      def most_recent_task(tasks, only_successful = false)
-        tasks = tasks.select { |t| t['finish_time'] }.sort_by { |t| t['finish_time'] }
-        tasks = tasks.select { |task| task['error'].nil? } if only_successful
-        tasks.last
-      end
-
-      def filtered_distribution_config_equal?(generated_config, actual_config)
-        generated = generated_config.clone
-        actual = actual_config.clone
-        #We store 'default' checksum type as nil, but pulp will default to sha256, so if we haven't set it, ignore it
-        if generated.keys.include?('checksum_type') && generated['checksum_type'].nil?
-          generated.delete('checksum_type')
-          actual.delete('checksum_type')
-        end
-        generated.compact == actual.compact
-      end
-
-      def importer_matches?(capsule_importer, capsule)
-        generated_importer = self.generate_importer(capsule)
-        capsule_importer.try(:[], 'importer_type_id') == generated_importer.id &&
-            generated_importer.config.compact == capsule_importer['config'].compact
-      end
-
-      protected
-
-      def object_to_hash(object)
-        hash = {}
-        object.instance_variables.each { |var| hash[var.to_s.delete("@")] = object.instance_variable_get(var) }
-        hash
-      end
-
-      def _get_most_recent_sync_status
-        begin
-          history = Katello.pulp_server.extensions.repository.sync_status(pulp_id)
-
-          if history.blank?
-            history = PulpSyncStatus.convert_history(Katello.pulp_server.extensions.repository.sync_history(pulp_id))
-          end
-        rescue
-          history = PulpSyncStatus.convert_history(Katello.pulp_server.extensions.repository.sync_history(pulp_id))
-        end
-
-        if history.blank?
-          return PulpSyncStatus.new(:state => PulpSyncStatus::Status::NOT_SYNCED)
-        else
-          history = sort_sync_status(history)
-          return PulpSyncStatus.pulp_task(history.first.with_indifferent_access)
-        end
-      end
-    end
-  end
-end

data/app/models/katello/pulp_sync_status.rb

@@ -1,165 +0,0 @@
-module Katello
-  class PulpSyncProgress
-    attr_reader :total_size, :size_left, :total_count, :items_left, :error_details, :step
-
-    def initialize(progress_attrs = {})
-      @total_size = @size_left = @total_count = @items_left = 0
-
-      unless progress_attrs.nil?
-        #depending on whether this is a history item, or current sync structure may be different
-        ht = HashUtil.new
-
-        details = ht.null_safe_get(progress_attrs, nil, %w(progress_report yum_importer content)) ||
-                  ht.null_safe_get(progress_attrs, nil, %w(progress_report details packages sync_report)) ||
-                  ht.null_safe_get(progress_attrs, nil, %w(progress_report iso_importer))
-
-        #if the task is waiting, it wont have a progress report
-        progress_attrs['progress_report'] ||= {}
-
-        if progress_attrs['progress_report']['iso_importer']
-          @total_size  = ht.null_safe_get(details, 0, ['total_bytes'])
-          @size_left   = @total_size - ht.null_safe_get(details, 0, ['finished_bytes'])
-          @total_count = ht.null_safe_get(details, 0, ['num_isos'])
-          @items_left  = @total_count - ht.null_safe_get(details, 0, ['num_isos_finished'])
-
-        else
-          @total_size  = ht.null_safe_get(details, 0, ['size_total'])
-          @size_left   = ht.null_safe_get(details, 0, ['size_left'])
-        end
-
-        @error_details = errors(progress_attrs['progress_report'])
-        @step = ht.null_safe_get(progress_attrs, 0, ['step'])
-      end
-    end
-
-    private
-
-    def errors(progress)
-      if progress[:yum_importer]
-        details = progress[:yum_importer]
-      elsif progress[:details]
-        details = progress[:details]
-      end
-
-      format_errors(details)
-    end
-
-    # Possible formats coming from pulp
-    #
-    # We ignore this case:
-    #   {'finished_count' => {}}
-    #
-    # We extract from this case:
-    #   {'content' => {'error' => ''},
-    #    'errata' => {'error' => ''},
-    #    'packages' => {'error' => ''},
-    #    'metadata' => {'error_details => ''}
-    #   }
-    def format_errors(details)
-      errors = {messages: [], details: []}
-
-      if details && !details.key?(:finished_count)
-        details.each do |step, report|
-          if step == "content"
-            parse_content(report, errors)
-          else
-            parse_generic(report, errors)
-          end
-        end
-      end
-
-      errors
-    end
-
-    def parse_content(details, errors)
-      timeout = false
-
-      details['error_details'].each do |error|
-        if error['error_code'] == 37
-          timeout = true
-        end
-      end
-
-      if timeout
-        errors[:messages] << _('One or more packages failed to sync properly.')
-        errors[:details].concat(details[:error_details]) if details[:error_details].present?
-      else
-        parse_generic(details, errors)
-      end
-    end
-
-    def parse_generic(details, errors)
-      errors[:messages] << details[:error] if details[:error].present?
-      errors[:details].concat(details[:error_details]) if details[:error_details].present?
-    end
-  end
-end
-
-module Katello
-  class PulpSyncStatus < PulpTaskStatus
-    HISTORY_ERROR = 'failed'.freeze
-    HISTORY_SUCCESS = 'success'.freeze
-    FINISHED  = "finished".freeze
-    ERROR     = "error".freeze
-    RUNNING   = "running".freeze
-    WAITING   = "waiting".freeze
-    CANCELED  = "canceled".freeze
-
-    class Status < TaskStatus::Status
-      NOT_SYNCED = :not_synced
-    end
-
-    def progress
-      PulpSyncProgress.new(attributes['progress'])
-    end
-
-    def after_refresh
-      correct_state
-    end
-
-    def self.pulp_task(pulp_status)
-      task_status = PulpSyncStatus.find_by(:uuid => pulp_status[:id])
-      task_status = self.new { |t| yield t if block_given? } if task_status.nil?
-      task_status.update_state(pulp_status)
-    end
-
-    def update_state(pulp_status)
-      PulpSyncStatus.dump_state(pulp_status, self)
-      correct_state
-      self
-    end
-
-    def correct_state
-      if [FINISHED].include?(self.state) && self.progress.error_details[:messages].present?
-        self.state = ERROR
-        self.save! unless self.new_record?
-      end
-    end
-
-    # Pulp history items are moved from the task item, but are different
-    #  and as a result we need to convert the structure
-    # @option [Array] history_list list of pulp sync history hashes
-    def self.convert_history(history_list)
-      #history item attributes
-      #["_id", "_ns", "added_count", "completed", "details", "error_message", "exception", "id",
-      # "importer_id", "importer_type_id", "removed_count", "repo_id", "result", "started", "summary",
-      # "traceback", "updated_count"]
-
-      #task status attributes
-      #["task_group_id", "exception", "traceback", "_href", "task_id", "call_request_tags", "reasons",
-      # "start_time", "tags", "state", "finish_time", "dependency_failures", "schedule_id", "progress",
-      # "call_request_group_id", "call_request_id", "principal_login", "response", "result"]
-      history_list.collect do |history|
-        result = history['result']
-        result = ERROR if result == HISTORY_ERROR
-        result = FINISHED if result == HISTORY_SUCCESS
-        {
-          :state => result,
-          :progress => {:details => history["details"]},
-          :finish_time => history['completed'],
-          :start_time => history['started']
-        }.with_indifferent_access
-      end
-    end
-  end
-end

data/app/models/katello/pulp_task_status.rb

@@ -1,63 +0,0 @@
-module Katello
-  class PulpTaskStatus < TaskStatus
-    WAIT_TIMES = [0.5, 1, 2, 4, 8, 16].freeze
-    WAIT_TIME_STEP = 5
-
-    validates_lengths_from_database
-
-    def refresh
-      PulpTaskStatus.refresh(self)
-    end
-
-    def after_refresh
-      #potentially used by child class, see PulpSyncStatus for example
-    end
-
-    def affected_units
-      self.result['num_changes']
-    end
-
-    def error
-      self.result[:errors][0] if self.error? && self.result[:errors]
-    end
-
-    def self.using_pulp_task(pulp_status)
-      if pulp_status.is_a? TaskStatus
-        pulp_status
-      else
-        task_id = pulp_status[:task_id] || pulp_status[:spawned_tasks].first[:task_id]
-        pulp_status = Katello.pulp_server.resources.task.poll(task_id)
-
-        task_status = TaskStatus.find_by(:uuid => task_id)
-        task_status = self.new { |t| yield t if block_given? } if task_status.nil?
-        PulpTaskStatus.dump_state(pulp_status, task_status)
-      end
-    end
-
-    def self.dump_state(pulp_status, task_status)
-      if !pulp_status.key?(:state) && pulp_status[:result] == "success"
-        # Note: if pulp_status doesn't contain a state, the status is coming from pulp sync history
-        pulp_status[:state] = Status::FINISHED.to_s
-      end
-
-      task_status.attributes = {
-        :uuid => pulp_status[:task_id],
-        :state => pulp_status[:state] || pulp_status[:result],
-        :start_time => pulp_status[:start_time] || pulp_status[:start_time],
-        :finish_time => pulp_status[:finish_time],
-        :progress => pulp_status,
-        :result => pulp_status[:result].nil? ? {:errors => [pulp_status[:exception], pulp_status[:traceback]]} : pulp_status[:result]
-      }
-      task_status.save! unless task_status.new_record?
-      task_status
-    end
-
-    def self.refresh(task_status)
-      pulp_task = Katello.pulp_server.resources.task.poll(task_status.uuid)
-
-      self.dump_state(pulp_task, task_status)
-      task_status.after_refresh
-      task_status
-    end
-  end
-end

data/app/services/katello/pulp/content.rb

@@ -1,24 +0,0 @@
-module Katello
-  module Pulp
-    class Content
-      extend Katello::Abstract::Pulp::Content
-      class << self
-        def create_upload(_size = 0, _checksum = nil, _content_type = nil, _repository = nil)
-          pulp_content.create_upload_request
-        end
-
-        def delete_upload(upload_id)
-          pulp_content.delete_upload_request(upload_id)
-        end
-
-        def upload_chunk(upload_id, offset, content, _size)
-          pulp_content.upload_bits(upload_id, offset, content)
-        end
-
-        private def pulp_content
-          SmartProxy.pulp_primary.pulp_api.resources.content
-        end
-      end
-    end
-  end
-end

data/app/services/katello/pulp/importer_comparison.rb

@@ -1,28 +0,0 @@
-module Katello
-  module Pulp
-    module ImporterComparison
-      def importer_matches?(generated_importer, capsule_importer)
-        if capsule_importer.try(:[], 'importer_type_id') != generated_importer.id
-          return false
-        end
-
-        generated_config = generated_importer.config
-        capsule_config = capsule_importer['config']
-        if generated_config['proxy_host'] == ""
-          proxy_keys = %w(proxy_host proxy_username proxy_password)
-          proxy_keys.each do |key|
-            generated_config.delete(key)
-            capsule_config.delete(key)
-          end
-        end
-
-        if generated_config['proxy_password'] == "" && capsule_config['proxy_password'] == "*****"
-          generated_config.delete('proxy_password')
-          capsule_config.delete('proxy_password')
-        end
-
-        generated_config.compact == capsule_config.compact
-      end
-    end
-  end
-end

data/app/services/katello/pulp/repository/deb.rb

@@ -1,61 +0,0 @@
-module Katello
-  module Pulp
-    class Repository
-      class Deb < ::Katello::Pulp::Repository
-        REPOSITORY_TYPE = 'deb'.freeze
-
-        def generate_primary_importer
-          config = {
-            feed: root.url,
-            remove_missing: root.mirror_on_sync?,
-            releases: root.deb_releases,
-            components: root.deb_components,
-            architectures: root.deb_architectures,
-            gpg_keys: root&.gpg_key&.content,
-            require_signature: root.gpg_key.present?
-          }
-          importer_class.new(config.merge(primary_importer_connection_options))
-        end
-
-        def generate_mirror_importer
-          config = {
-            feed: external_url,
-            remove_missing: true,
-            releases: root.deb_releases,
-            components: root.deb_components,
-            architectures: root.deb_architectures,
-            gpg_keys: root&.gpg_key&.content,
-            require_signature: root.gpg_key.present?
-          }
-          importer_class.new(config.merge(mirror_importer_connection_options))
-        end
-
-        def generate_distributors
-          [Runcible::Models::DebDistributor.new(repo.relative_path, root.unprotected, true,
-                                                id: repo.pulp_id,
-                                                auto_publish: true)]
-        end
-
-        def distributors_to_publish(_options)
-          {Runcible::Models::DebDistributor => {}}
-        end
-
-        def partial_repo_path
-          "/pulp/deb/#{repo.relative_path}/"
-        end
-
-        def importer_class
-          Runcible::Models::DebImporter
-        end
-
-        def copy_contents(destination_repo, _options = {})
-          [
-            @smart_proxy.pulp_api.extensions.deb.copy(@repo.pulp_id, destination_repo.pulp_id, fields: ::Katello::Pulp::Deb::PULP_SELECT_FIELDS),
-            @smart_proxy.pulp_api.extensions.deb_release.copy(@repo.pulp_id, destination_repo.pulp_id, {}),
-            @smart_proxy.pulp_api.extensions.deb_component.copy(@repo.pulp_id, destination_repo.pulp_id, {})
-          ]
-        end
-      end
-    end
-  end
-end