foreman_inventory_upload 1.0.0.beta7 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85837a58f5c5a76038117ea20c6a10873e8c87add4841c8fb08d29e69564b7f8
-  data.tar.gz: 652f7cac5cc2a3b861d82092257f0f683563aca3301f0c8a547ffc1cdfce89d6
+  metadata.gz: 4b18615a8878127f74e382f34b99c409d2a3cea180d40aaf4600660bf3e32875
+  data.tar.gz: fdabaddefa7099573546fe4f80a5ac3e406d2f52b1816efb7e433a958b73fe14
 SHA512:
-  metadata.gz: 751a08aec0fafcc07496027d72dd087756f366acd6da686ea67833a3b2502fbffbd677734a122addd77b1213b35c61eee35736026617c43ae0ae866f2abd6632
-  data.tar.gz: bb8ba3a8aaad6e5ef7994aff64d42b3848d0855187b6bd1c91205c010b2c7aaeb89236678f69bd81d4f179bcd9e2a5faab8c4842410f6908d15c167bc45ecb07
+  metadata.gz: 1552fbef1b4ffadb0271a804d54b8782ebe9b39c0557686398e28fb72a159cddee7eea26e4ca65b86224a2fa596fa56b7e1d7237b03b9ef4eebf0f776cd32520
+  data.tar.gz: e6ddf50d580f4be2ef5575eed02cf5f448e273239f33449c80f79fb68fefb124ba3c92ec45e58ed6fcb4f2fedb6cc5a6c6a45577e8b00247ca4bbeab810e30c9

data/app/controllers/foreman_inventory_upload/accounts_controller.rb

@@ -0,0 +1,39 @@
+module ForemanInventoryUpload
+  class AccountsController < ::ApplicationController
+    # override default "welcome screen behavior, since we don't have a model"
+    def welcome
+      true
+    end
+
+    def index
+      labels = Organization.all.pluck(:id, :name)
+
+      accounts = Hash[
+        labels.map do |id, label|
+          generate_report_status = status_for(id, ForemanInventoryUpload::Async::GenerateReportJob)
+          upload_report_status = status_for(id, ForemanInventoryUpload::Async::UploadReportJob)
+
+          [
+            id,
+            {
+              generate_report_status: generate_report_status,
+              upload_report_status: upload_report_status,
+              label: label,
+            },
+          ]
+        end
+      ]
+
+      render json: {
+        accounts: accounts,
+      }, status: :ok
+    end
+
+    private
+
+    def status_for(label, job_class)
+      label = job_class.output_label(label)
+      ForemanInventoryUpload::Async::ProgressOutput.get(label)&.status
+    end
+  end
+end

data/app/controllers/foreman_inventory_upload/reports_controller.rb

@@ -3,7 +3,7 @@
 module ForemanInventoryUpload
   class ReportsController < ::ApplicationController
     def last
-      label = ForemanInventoryUpload::Async::GenerateReportJob.output_label(params[:portal_user])
+      label = ForemanInventoryUpload::Async::GenerateReportJob.output_label(params[:organization_id])
       output = ForemanInventoryUpload::Async::ProgressOutput.get(label)&.full_output
       task_label = ForemanInventoryUpload::Async::GenerateAllReportsJob.singleton_job_name
       scheduled = ForemanTasks::Task.where(
@@ -18,10 +18,9 @@ module ForemanInventoryUpload
     end
 
     def generate
-      portal_user = params[:portal_user]
+      organization_id = params[:organization_id]
 
-      generated_file_name = File.join(ForemanInventoryUpload.base_folder, "#{portal_user}.tar.gz")
-      ForemanInventoryUpload::Async::GenerateReportJob.perform_later(generated_file_name, portal_user)
+      ForemanInventoryUpload::Async::GenerateReportJob.perform_later(ForemanInventoryUpload.generated_reports_folder, organization_id)
 
       render json: {
         action_status: 'success',

data/app/controllers/foreman_inventory_upload/uploads_controller.rb

@@ -1,7 +1,7 @@
 module ForemanInventoryUpload
   class UploadsController < ::ApplicationController
     def last
-      label = ForemanInventoryUpload::Async::UploadReportJob.output_label(params[:portal_user])
+      label = ForemanInventoryUpload::Async::UploadReportJob.output_label(params[:organization_id])
       output = ForemanInventoryUpload::Async::ProgressOutput.get(label)&.full_output
 
       render json: {
@@ -10,8 +10,8 @@ module ForemanInventoryUpload
     end
 
     def download_file
-      filename = 'hosts_report.tar.gz'
-      path = Rails.root.join(ForemanInventoryUpload.uploads_folder(params[:portal_user]), filename)
+      filename = ForemanInventoryUpload.facts_archive_name(params[:organization_id])
+      path = Rails.root.join(ForemanInventoryUpload.uploads_folder, filename)
       unless File.exist? path
         return throw_flash_error(
           "Path doesn't exist: #{path}"

data/app/views/scripts/uploader.sh.erb

@@ -3,26 +3,38 @@
 DEST=<%= @upload_url %>
 RH_USERNAME=<%= @rh_username %>
 
-if [ -z "$RH_USERNAME" ]
+if [ -z "$FILES" ]
 then
-        IFS= read -rp "Enter username: " RH_USERNAME
+        FILES=./*.tar.gz
 fi
 
-if [ -z "$RH_PASSWORD" ]
+if [ -n "$CER_PATH" ]
 then
-        IFS= read -rsp "Enter password: " RH_PASSWORD
-fi
+        AUTH_KEY="--cert"
+        AUTH_VAL="$CER_PATH"
+else
+        if [ -z "$RH_USERNAME" ]
+        then
+                IFS= read -rp "Enter username: " RH_USERNAME
+        fi
 
-if [ -z "$FILES" ]
-then
-        FILES=./*.tar.gz
+        if [ -z "$RH_PASSWORD" ]
+        then
+                IFS= read -rsp "Enter password: " RH_PASSWORD
+        fi
+
+        AUTH_KEY="-u"
+        AUTH_VAL="\"$RH_USERNAME\":\"$RH_PASSWORD\""
 fi
 
+# /tmp/a b/x.pem
+# curl --cert /tmp/a\ b/x.pem
+
 mkdir -p done
 
 for f in $FILES
 do
-  curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST -u "$RH_USERNAME":"$RH_PASSWORD"
+  curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST "$AUTH_KEY" "$AUTH_VAL"
   if [ $? -eq 0 ]; then
     mv $f done/
     echo "Done: $f"

data/config/routes.rb

@@ -1,10 +1,10 @@
 Rails.application.routes.draw do
   namespace :foreman_inventory_upload do
     get 'index', to: 'react#index'
-    get ':portal_user/reports/last', to: 'reports#last', constraints: { portal_user: %r{[^\/]+} }
-    post ':portal_user/reports', to: 'reports#generate', constraints: { portal_user: %r{[^\/]+} }
-    get ':portal_user/uploads/last', to: 'uploads#last', constraints: { portal_user: %r{[^\/]+} }
-    get ':portal_user/uploads/file', to: 'uploads#download_file', constraints: { portal_user: %r{[^\/]+} }
-    get 'statuses', to: 'statuses#index'
+    get ':organization_id/reports/last', to: 'reports#last', constraints: { organization_id: %r{[^\/]+} }
+    post ':organization_id/reports', to: 'reports#generate', constraints: { organization_id: %r{[^\/]+} }
+    get ':organization_id/uploads/last', to: 'uploads#last', constraints: { organization_id: %r{[^\/]+} }
+    get ':organization_id/uploads/file', to: 'uploads#download_file', constraints: { organization_id: %r{[^\/]+} }
+    get 'accounts', to: 'accounts#index'
   end
 end

data/lib/foreman_inventory_upload.rb

@@ -11,16 +11,20 @@ module ForemanInventoryUpload
     )
   end
 
-  def self.uploads_folder(group)
-    @uploads_folders ||= {}
-    cache = @uploads_folders[group]
-    return cache if cache
+  def self.uploads_folder
+    @uploads_folder ||= ensure_folder(
+      File.join(
+        ForemanInventoryUpload.base_folder,
+        'uploads/'
+      )
+    )
+  end
 
-    @uploads_folders[group] = ensure_folder(
+  def self.generated_reports_folder
+    @generated_reports_folder ||= ensure_folder(
       File.join(
         ForemanInventoryUpload.base_folder,
-        'uploads/',
-        "#{group}/"
+        'generated_reports/'
       )
     )
   end
@@ -33,12 +37,12 @@ module ForemanInventoryUpload
     'uploader.sh'
   end
 
-  def self.facts_archive_name
-    'hosts_report.tar.gz'
+  def self.facts_archive_name(organization)
+    "report_for_#{organization}.tar.gz"
   end
 
   def self.upload_url
-    # for testing set ENV to 'https://ci.cloud.paas.psi.redhat.com/api/ingress/v1/upload'
+    # for testing set ENV to 'https://ci.cloud.redhat.com/api/ingress/v1/upload'
     @upload_url ||= ENV['SATELLITE_INVENTORY_UPLOAD_URL'] || 'https://cloud.redhat.com/api/ingress/v1/upload'
   end
 

data/lib/foreman_inventory_upload/async/generate_all_reports_job.rb

@@ -2,13 +2,10 @@ module ForemanInventoryUpload
   module Async
     class GenerateAllReportsJob < ::ApplicationJob
       def perform
-        portal_users = RedhatAccess::TelemetryConfiguration
-                       .where(enable_telemetry: true)
-                       .distinct
-                       .pluck(:portal_user)
-        portal_users.map do |portal_user|
-          generated_file_name = File.join(ForemanInventoryUpload.base_folder, "#{portal_user}.tar.gz")
-          GenerateReportJob.perform_later(generated_file_name, portal_user)
+        organizations = Organization.unscoped.all
+
+        organizations.map do |organization|
+          GenerateReportJob.perform_later(ForemanInventoryUpload.generated_reports_folder, organization.id)
         end
       ensure
         self.class.set(:wait => 24.hours).perform_later

data/lib/foreman_inventory_upload/async/generate_report_job.rb

@@ -1,17 +1,21 @@
 module ForemanInventoryUpload
   module Async
     class GenerateReportJob < ShellProcess
-      def self.output_label(portal_user)
-        "report_for_#{portal_user}"
+      def self.output_label(label)
+        "report_for_#{label}"
       end
 
-      def perform(result_file, portal_user)
-        @result_file = result_file
-        @portal_user = portal_user
+      def perform(base_folder, organization)
+        @base_folder = base_folder
+        @organization = organization
 
-        super(GenerateReportJob.output_label(portal_user))
+        super(GenerateReportJob.output_label(organization))
 
-        QueueForUploadJob.perform_later(result_file, portal_user)
+        QueueForUploadJob.perform_later(
+          base_folder,
+          ForemanInventoryUpload.facts_archive_name(organization),
+          organization
+        )
       end
 
       def command
@@ -20,8 +24,8 @@ module ForemanInventoryUpload
 
       def env
         super.merge(
-          'target' => @result_file,
-          'portal_user' => @portal_user
+          'target' => @base_folder,
+          'organization_id' => @organization
         )
       end
     end

data/lib/foreman_inventory_upload/async/queue_for_upload_job.rb

@@ -1,21 +1,22 @@
 module ForemanInventoryUpload
   module Async
     class QueueForUploadJob < ::ApplicationJob
-      def perform(report_file, portal_user)
-        @portal_user = portal_user
+      def perform(base_folder, report_file, organization_id)
+        @base_folder = base_folder
+        @report_file = report_file
         logger.debug('Ensuring objects')
         ensure_ouput_folder
         ensure_output_script
         logger.debug("Copying #{report_file} to #{uploads_folder}")
-        enqueued_file_name = File.join(uploads_folder, ForemanInventoryUpload.facts_archive_name)
-        FileUtils.mv(report_file, enqueued_file_name)
+        enqueued_file_name = File.join(uploads_folder, report_file)
+        FileUtils.mv(File.join(base_folder, report_file), enqueued_file_name)
         logger.debug("Done copying #{report_file} to #{enqueued_file_name}")
 
-        UploadReportJob.perform_later(enqueued_file_name, portal_user)
+        UploadReportJob.perform_later(enqueued_file_name, organization_id)
       end
 
       def uploads_folder
-        @uploads_folder ||= ForemanInventoryUpload.uploads_folder(@portal_user)
+        @uploads_folder ||= ForemanInventoryUpload.uploads_folder
       end
 
       def script_file
@@ -36,7 +37,6 @@ module ForemanInventoryUpload
           source: template_src,
           variables: {
             upload_url: ForemanInventoryUpload.upload_url,
-            rh_username: @portal_user,
           }
         )
         script_source = Foreman::Renderer.render(template_src, scope)

data/lib/foreman_inventory_upload/async/upload_report_job.rb

@@ -1,15 +1,23 @@
+require 'tempfile'
+
 module ForemanInventoryUpload
   module Async
     class UploadReportJob < ShellProcess
-      def self.output_label(portal_user)
-        "upload_for_#{portal_user}"
+      def self.output_label(label)
+        "upload_for_#{label}"
       end
 
-      def perform(filename, portal_user)
-        @portal_user = portal_user
+      def perform(filename, organization_id)
         @filename = filename
+        @organization = Organization.find(organization_id)
 
-        super(UploadReportJob.output_label(portal_user))
+        Tempfile.create([@organization.name, '.pem']) do |cer_file|
+          cer_file.write(rh_credentials[:cert])
+          cer_file.write(rh_credentials[:key])
+          cer_file.flush
+          @cer_path = cer_file.path
+          super(UploadReportJob.output_label(organization_id))
+        end
       end
 
       def command
@@ -18,22 +26,19 @@ module ForemanInventoryUpload
 
       def env
         super.merge(
-          'RH_USERNAME' => rh_username,
-          'RH_PASSWORD' => rh_password,
-          'FILES' => @filename
+          'FILES' => @filename,
+          'CER_PATH' => @cer_path
         )
       end
 
       def rh_credentials
-        @rh_credentials ||= RedhatAccess::TelemetryConfiguration.where(portal_user: @portal_user).last
-      end
-
-      def rh_username
-        @portal_user
-      end
-
-      def rh_password
-        rh_credentials.portal_password
+        @rh_credentials ||= begin
+          candlepin_id_certificate = @organization.owner_details['upstreamConsumer']['idCert']
+          {
+            cert: candlepin_id_certificate['cert'],
+            key: candlepin_id_certificate['key'],
+          }
+        end
       end
     end
   end

data/lib/foreman_inventory_upload/generators/archived_report.rb

@@ -6,10 +6,10 @@ module ForemanInventoryUpload
         @logger = logger
       end
 
-      def render(portal_user)
+      def render(organization:)
         Dir.mktmpdir do |tmpdir|
           @logger.info "Started generating hosts report in #{tmpdir}"
-          host_batches = ForemanInventoryUpload::Generators::Queries.for_report(portal_user)
+          host_batches = ForemanInventoryUpload::Generators::Queries.for_org(organization)
           File.open(File.join(tmpdir, 'metadata.json'), 'w') do |metadata_out|
             metadata_generator = ForemanInventoryUpload::Generators::Metadata.new(metadata_out)
             metadata_generator.render do |inner_generator|

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -17,6 +17,7 @@ module ForemanInventoryUpload
               'distribution::name',
               'uname::release',
               'lscpu::flags',
+              'distribution::version',
             ]).pluck(:name, :id)
           ]
       end
@@ -37,15 +38,23 @@ module ForemanInventoryUpload
       end
 
       def self.for_report(portal_user)
-        org_ids = Organization
-                  .joins(:telemetry_configuration)
-                  .where(
-                    redhat_access_telemetry_configurations: {
-                      portal_user: portal_user,
-                      enable_telemetry: true,
-                    }
-                  ).pluck(:id)
-        for_slice(Host.where(organization_id: org_ids)).in_batches(of: 1_000)
+        org_ids = organizations_for_user(portal_user).pluck(:id)
+        for_slice(Host.unscoped.where(organization_id: org_ids)).in_batches(of: 1_000)
+      end
+
+      def self.for_org(organization_id)
+        for_slice(Host.unscoped.where(organization_id: organization_id)).in_batches(of: 1_000)
+      end
+
+      def self.organizations_for_user(portal_user)
+        Organization
+          .joins(:telemetry_configuration)
+          .where(
+            redhat_access_telemetry_configurations: {
+              portal_user: portal_user,
+              enable_telemetry: true,
+            }
+          )
       end
     end
   end

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -23,14 +23,14 @@ module ForemanInventoryUpload
             first = true
             hosts_batch.each do |host|
               @stream.comma unless first
-              first = false
-              report_host(host)
+              first = false if report_host(host)
             end
           end
         end
       end
 
       def report_host(host)
+        return nil unless host&.subscription_facet&.pools&.first
         @stream.object do
           @stream.simple_field('display_name', host.name)
           @stream.simple_field('fqdn', host.fqdn)
@@ -56,13 +56,7 @@ module ForemanInventoryUpload
             @stream.object do
               @stream.simple_field('namespace', 'satellite')
               @stream.object_field('facts', :last) do
-                @stream.simple_field('virtual_host_name', host.subscription_facet.hypervisor_host&.name)
-                @stream.simple_field('virtual_host_uuid', host.subscription_facet.hypervisor_host&.subscription_facet&.uuid)
-                if defined?(ForemanThemeSatellite)
-                  @stream.simple_field('satellite_version', ForemanThemeSatellite::SATELLITE_VERSION)
-                end
-                @stream.simple_field('satellite_instance_id', Foreman.respond_to?(:instance_id) ? Foreman.instance_id : nil)
-                @stream.simple_field('organization_id', host.organization_id, :last)
+                report_satellite_facts(host)
               end
             end
           end
@@ -120,6 +114,19 @@ module ForemanInventoryUpload
         end
       end
 
+      def report_satellite_facts(host)
+        @stream.simple_field('virtual_host_name', host.subscription_facet.hypervisor_host&.name)
+        @stream.simple_field('virtual_host_uuid', host.subscription_facet.hypervisor_host&.subscription_facet&.uuid)
+        if defined?(ForemanThemeSatellite)
+          @stream.simple_field('satellite_version', ForemanThemeSatellite::SATELLITE_VERSION)
+        end
+        @stream.simple_field('system_purpose_usage', host.subscription_facet.purpose_usage)
+        @stream.simple_field('system_purpose_role', host.subscription_facet.purpose_role)
+        @stream.simple_field('distribution_version', fact_value(host, 'distribution::version'))
+        @stream.simple_field('satellite_instance_id', Foreman.respond_to?(:instance_id) ? Foreman.instance_id : nil)
+        @stream.simple_field('organization_id', host.organization_id, :last)
+      end
+
       def fact_value(host, fact_name)
         value_record = host.fact_values.find do |fact_value|
           fact_value.fact_name_id == ForemanInventoryUpload::Generators::Queries.fact_names[fact_name]