katello 3.11.2 → 3.12.0.rc1

data/app/services/katello/pulp/docker_manifest_list.rb

@@ -2,6 +2,14 @@ module Katello
   module Pulp
     class DockerManifestList < PulpContentUnit
       CONTENT_TYPE = "docker_manifest_list".freeze
+
+      def update_model(model)
+        model.update_attributes(:schema_version => backend_data[:schema_version],
+                          :digest => backend_data[:digest],
+                          :downloaded => backend_data[:downloaded],
+                          :docker_manifests => ::Katello::DockerManifest.where(:digest => backend_data[:manifests].pluck(:digest))
+                         )
+      end
     end
   end
 end

data/app/services/katello/pulp/docker_tag.rb

@@ -2,6 +2,14 @@ module Katello
   module Pulp
     class DockerTag < PulpContentUnit
       CONTENT_TYPE = "docker_tag".freeze
+
+      def update_model(model)
+        taggable_class = backend_data['manifest_type'] == "list" ? ::Katello::DockerManifestList : ::Katello::DockerManifest
+        model.docker_taggable = taggable_class.find_by(:digest => backend_data['manifest_digest'])
+        model.repository_id = ::Katello::Repository.find_by(:pulp_id => backend_data['repo_id']).try(:id)
+        model.name = backend_data['name']
+        model.save!
+      end
     end
   end
 end

data/app/services/katello/pulp/erratum.rb

@@ -7,6 +7,123 @@ module Katello
       def self.unit_handler
         Katello.pulp_server.extensions.errata
       end
+
+      def update_model(model)
+        keys = %w(title id severity issued type description reboot_suggested solution updated summary)
+        custom_json = backend_data.slice(*keys)
+
+        # handle SUSE epoch dates
+        custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])
+        custom_json["updated"] = convert_date_if_epoch(custom_json["updated"]) unless custom_json["updated"].blank?
+
+        if model.updated.blank? || (custom_json['updated'].to_datetime != model.updated.to_datetime)
+          custom_json['errata_id'] = custom_json.delete('id')
+          custom_json['errata_type'] = custom_json.delete('type')
+          custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated']
+          model.update_attributes!(custom_json)
+
+          unless backend_data['references'].blank?
+            update_bugzillas(model, backend_data['references'].select { |r| r['type'] == 'bugzilla' })
+            update_cves(model, backend_data['references'].select { |r| r['type'] == 'cve' })
+          end
+        end
+        update_packages(model, backend_data['pkglist']) unless backend_data['pkglist'].blank?
+        update_modules(model, backend_data['pkglist']) unless backend_data['pkglist'].blank?
+      end
+
+      def update_bugzillas(model, json)
+        needed_function = lambda do
+          existing_names = model.bugzillas.pluck(:bug_id)
+          json.select { |bz| !existing_names.include?(bz['id']) }
+        end
+        action_function = lambda do |needed|
+          model.bugzillas.create!(needed.map { |bug| {:bug_id => bug['id'], :href => bug['href']} })
+        end
+        run_until(model, needed_function, action_function)
+      end
+
+      def update_cves(model, json)
+        needed_function = lambda do
+          existing_names = model.cves.pluck(:cve_id)
+          json.select { |cve| !existing_names.include?(cve['id']) }
+        end
+        action_function = lambda do |needed|
+          model.cves.create!(needed.map { |cve| {:cve_id => cve['id'], :href => cve['href']} })
+        end
+        run_until(model, needed_function, action_function)
+      end
+
+      def update_packages(model, json)
+        needed_function = lambda do
+          package_hashes = json.map { |list| list['packages'] }.flatten
+          package_attributes = package_hashes.map do |hash|
+            nvrea = Util::Package.build_nvra(hash)
+            {'name' => hash['name'], 'nvrea' => nvrea, 'filename' => hash['filename']}
+          end
+          existing_nvreas = model.packages.pluck(:nvrea)
+          package_attributes.delete_if { |pkg| existing_nvreas.include?(pkg['nvrea']) }
+          package_attributes.uniq { |pkg| pkg['nvrea'] }
+        end
+        action_function = lambda do |needed|
+          model.packages.create!(needed)
+        end
+        run_until(model, needed_function, action_function)
+      end
+
+      def update_modules(model, json)
+        needed_function = lambda do
+          module_stream_attributes = []
+          json.each do |package_item|
+            if package_item['module']
+              module_stream = ::Katello::ModuleStream.where(package_item['module']).first
+              next if module_stream.blank?
+              nvreas = package_item["packages"].map { |hash| Util::Package.build_nvra(hash) }
+              module_stream_id_column = "#{ModuleStreamErratumPackage.table_name}.module_stream_id"
+              existing = ErratumPackage.joins(:module_streams).
+                                        where(module_stream_id_column => module_stream.id,
+                                              :nvrea => nvreas).pluck(:nvrea)
+
+              (nvreas - existing).each do |nvrea|
+                package = model.packages.find_by(:nvrea => nvrea)
+                module_stream_attributes << { :module_stream_id => module_stream.id,
+                                              :erratum_package_id => package.id }
+              end
+            end
+          end
+          module_stream_attributes.uniq
+        end
+
+        action_function = lambda do |needed|
+          needed.each do |msep|
+            ModuleStreamErratumPackage.create!(msep)
+          end
+        end
+
+        run_until(model, needed_function, action_function)
+      end
+
+      def run_until(model, needed_function, action_function)
+        needed = needed_function.call
+        retries = needed.length
+        until needed.empty? || retries == 0
+          begin
+            action_function.call(needed)
+          rescue ActiveRecord::RecordNotUnique
+            model.reload
+          end
+          needed = needed_function.call
+          retries -= 1
+        end
+        fail _('Failed indexing errata, maximum retries encountered') if retries == 0 && needed.any?
+      end
+
+      def convert_date_if_epoch(date)
+        date.to_i.to_s == date ? epoch_to_date(date) : date
+      end
+
+      def epoch_to_date(epoch)
+        Time.at(epoch.to_i).to_s
+      end
     end
   end
 end

data/app/services/katello/pulp/file_unit.rb

@@ -8,6 +8,14 @@ module Katello
       CONTENT_TYPE = "iso".freeze
 
       lazy_accessor :pulp_facts, :initializer => :backend_data
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['checksum'] = backend_data['checksum']
+        custom_json['path'] = backend_data['name']
+        custom_json['name'] = File.basename(backend_data['name'])
+        model.update_attributes!(custom_json)
+      end
     end
   end
 end

data/app/services/katello/pulp/module_stream.rb

@@ -4,6 +4,36 @@ module Katello
       include LazyAccessor
 
       CONTENT_TYPE = "modulemd".freeze
+
+      def update_model(model)
+        shared_attributes = backend_data.keys & model.class.column_names
+        shared_json = backend_data.select { |key, _v| shared_attributes.include?(key) }
+        model.update_attributes!(shared_json)
+
+        create_stream_artifacts(model, backend_data['artifacts']) if backend_data.key?('artifacts')
+        create_profiles(model, backend_data['profiles']) if backend_data.key?('profiles')
+      end
+
+      def create_stream_artifacts(model, artifacts_json)
+        artifacts_json.each do |name|
+          Katello::Util::Support.active_record_retry do
+            model.artifacts.where(name: name).first_or_create!
+          end
+        end
+      end
+
+      def create_profiles(model, profiles_json)
+        profiles_json.each do |profile, rpms|
+          Katello::Util::Support.active_record_retry do
+            profile = model.profiles.where(name: profile).first_or_create!
+          end
+          rpms.each do |rpm|
+            Katello::Util::Support.active_record_retry do
+              profile.rpms.where(name: rpm).first_or_create!
+            end
+          end
+        end
+      end
     end
   end
 end

data/app/services/katello/pulp/ostree_branch.rb

@@ -2,6 +2,13 @@ module Katello
   module Pulp
     class OstreeBranch < PulpContentUnit
       CONTENT_TYPE = "ostree".freeze
+
+      def update_model(model)
+        model.update_attributes(:name => backend_data[:branch],
+                          :version => backend_data[:metadata][:version],
+                          :commit => backend_data[:commit]
+                         )
+      end
     end
   end
 end

data/app/services/katello/pulp/package_category.rb

@@ -0,0 +1,7 @@
+module Katello
+  module Pulp
+    class PackageCategory < PulpContentUnit
+      CONTENT_TYPE = "package_category".freeze
+    end
+  end
+end

data/app/services/katello/pulp/package_group.rb

@@ -11,6 +11,10 @@ module Katello
                     :conditional_package_names, :default_package_names, :_id,
                     :repository_memberships,
                     :initializer => :pulp_facts
+
+      def update_model(model)
+        model.update_attributes!(backend_data.slice('name', 'description'))
+      end
     end
   end
 end

data/app/services/katello/pulp/pulp_content_unit.rb

@@ -5,12 +5,14 @@ module Katello
     class PulpContentUnit
       # Any class that extends this class should define:
       # Class::CONTENT_TYPE
+      # Class#update_model
 
       # Any class that extends this class can define:
       # Class.unit_handler (optional)
       # Class::PULP_INDEXED_FIELDS (optional)
 
       attr_accessor :uuid
+      attr_writer :backend_data
 
       def initialize(uuid)
         self.uuid = uuid
@@ -77,8 +79,23 @@ module Katello
       end
 
       def backend_data
+        @backend_data ||= fetch_backend_data
+        @backend_data.try(:with_indifferent_access)
+      end
+
+      def fetch_backend_data
         self.class.pulp_data(self.uuid)
       end
+
+      def self.remove(repo, uuids = nil)
+        fields = self.const_get(:PULP_SELECT_FIELDS) if self.constants.include?(:PULP_SELECT_FIELDS)
+        clauses = {:association => {'unit_id' => {'$in' => uuids}}}
+        clause = { type_ids: [const_get(:CONTENT_TYPE)]}
+        clause = clause.merge(filters: clauses) if uuids
+        clause = clause.merge(fields: { :unit => fields}) if fields
+        Katello.pulp_server.resources.repository.unassociate_units(repo.pulp_id,
+                                                                   clause)
+      end
     end
   end
 end

data/app/services/katello/pulp/puppet_module.rb

@@ -11,6 +11,11 @@ module Katello
                     :_ns, :project_page, :source, :dependencies,
                     :_content_type_id, :checksums, :_id, :types,
                     :initializer => :pulp_facts
+
+      def update_model(model)
+        custom_json = backend_data.slice('name', 'author', 'title', 'version', 'summary')
+        model.update_attributes!(custom_json)
+      end
     end
   end
 end

data/app/services/katello/pulp/repository.rb

@@ -1,7 +1,7 @@
 module Katello
   module Pulp
     class Repository < ::Actions::Pulp::Abstract
-      attr_accessor :repo, :input, :pulp_api
+      attr_accessor :repo, :input
       attr_accessor :smart_proxy
       delegate :root, to: :repo
 
@@ -51,6 +51,10 @@ module Katello
         fail NotImplementedError
       end
 
+      def distributors_to_publish
+        fail NotImplementedError
+      end
+
       def generate_master_importer
         fail NotImplementedError
       end
@@ -90,6 +94,13 @@ module Katello
                                                                               generate_distributors, display_name: root.name)
       end
 
+      def delete
+        smart_proxy.pulp_api.extensions.repository.delete(repo.pulp_id)
+      rescue RestClient::ResourceNotFound
+        Rails.logger.warn("Tried to delete repository #{repo.pulp_id}, but it did not exist.")
+        []
+      end
+
       def external_url(force_https = false)
         uri = URI.parse(::SmartProxy.pulp_master.pulp_url)
         uri.scheme = (root.unprotected && !force_https) ? 'http' : 'https'
@@ -144,6 +155,21 @@ module Katello
         }
       end
 
+      def distributor_publish(options)
+        distributors_to_publish(options).map do |distributor_class, config|
+          config[:force_full] = options.fetch(:force, false)
+          smart_proxy.pulp_api.extensions.repository.publish(repo.pulp_id, lookup_distributor_id(distributor_class.type_id),
+                                                             override_config: config)
+        end
+      end
+
+      def lookup_distributor_id(distributor_type_id)
+        distributor = backend_data["distributors"].find do |dist|
+          dist["distributor_type_id"] == distributor_type_id
+        end
+        distributor['id']
+      end
+
       def proxy_host_importer_value
         root.ignore_global_proxy ? "" : nil
       end
@@ -204,15 +230,22 @@ module Katello
         tasks
       end
 
-      def copy_units(destination_repo, units, options = {})
+      def copy_units(destination_repo, units, override_config = {})
         content_type = units.first.class::CONTENT_TYPE
         unit_ids = units.pluck(:pulp_id)
-        override_config = {}
-        override_config[:recursive] = true if options[:recursive]
-        override_config[:resolve_dependencies] = true if options[:resolve_dependencies]
 
         smart_proxy.pulp_api.extensions.send(content_type).copy(repo.pulp_id, destination_repo.pulp_id, ids: unit_ids, override_config: override_config)
       end
+
+      def content_unit_types
+        Katello::RepositoryTypeManager.find(repo.content_type)
+      end
+
+      def index_content_units
+        content_unit_types.each do |content_type|
+          content_type.pulp2_service_class
+        end
+      end
     end
   end
 end

data/app/services/katello/pulp/repository/deb.rb

@@ -34,6 +34,10 @@ module Katello
                                                 auto_publish: true)]
         end
 
+        def distributors_to_publish(_options)
+          {Runcible::Models::DebDistributor => {}}
+        end
+
         def partial_repo_path
           "/pulp/deb/#{repo.relative_path}/"
         end

data/app/services/katello/pulp/repository/docker.rb

@@ -39,6 +39,10 @@ module Katello
                                                   :repo_registry_id => repo.container_repository_name)]
         end
 
+        def distributors_to_publish(_options)
+          {Runcible::Models::DockerDistributor => {}}
+        end
+
         def external_url(_force_https = false)
           pulp_uri = URI.parse(SmartProxy.pulp_master.pulp_url)
           "#{pulp_uri.host.downcase}:#{Setting['pulp_docker_registry_port']}/#{repo.container_repository_name}"
@@ -51,10 +55,14 @@ module Katello
         def copy_contents(destination_repo, options = {})
           if options[:filters]
             clause_gen = ::Katello::Util::DockerManifestClauseGenerator.new(@repo, options[:filters])
+            clause_gen.generate
             criteria = {filters: {:unit => clause_gen.copy_clause } }
           end
 
-          @smart_proxy.pulp_api.extensions.docker_tag.copy(@repo.pulp_id, destination_repo.pulp_id, criteria || {})
+          @smart_proxy.pulp_api.extensions.docker_tag.copy(
+            @repo.pulp_id,
+            destination_repo.pulp_id,
+            criteria || {})
         end
       end
     end

data/app/services/katello/pulp/repository/file.rb

@@ -32,6 +32,10 @@ module Katello
           Runcible::Models::IsoImporter
         end
 
+        def distributors_to_publish(_options)
+          {Runcible::Models::IsoDistributor => {}}
+        end
+
         def copy_contents(destination_repo, _options = {})
           @smart_proxy.pulp_api.extensions.file.copy(@repo.pulp_id, destination_repo.pulp_id, {})
         end

data/app/services/katello/pulp/repository/ostree.rb

@@ -14,7 +14,7 @@ module Katello
 
         def generate_mirror_importer
           config = {
-            feed: root.url,
+            feed: external_url(true),
             depth: PULP_MIRROR_SYNC_DEPTH
           }
           Runcible::Models::OstreeImporter.new(config.merge(mirror_importer_connection_options))
@@ -27,8 +27,12 @@ module Katello
                                                   :depth => root.compute_ostree_upstream_sync_depth)]
         end
 
+        def distributors_to_publish(_options)
+          {Runcible::Models::OstreeDistributor => {}}
+        end
+
         def partial_repo_path
-          "/pulp/puppet/#{pulp_id}/"
+          "/pulp/ostree/web/#{repo.relative_path}".sub('//', '/')
         end
 
         def importer_class