foreman_inventory_upload 0.0.1.dev1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 42f516b5764fdf3d298592d73e3349a3aac827bb
-  data.tar.gz: 68d8d1d23a8a455b4c7659ea4b4c18e082dd83e0
+SHA256:
+  metadata.gz: a633e9716013237bbbcce5cba1748fad264cea4b4b965e5f19bb0f0be8d41e4e
+  data.tar.gz: 58de7c1e9666e4373fb44db980bacfb4392131bfddbd31a86123de34b1d456bd
 SHA512:
-  metadata.gz: 63f68d4d635e776d3414f450f4a7174f9163c4160c20bb50eceb1bd09b6311bc415b5e66e4d31756f38eb2da89554428ff7ab81561274e322871d70f2a41dcaa
-  data.tar.gz: 3df2740f66a0d01532d029c8ed1be571678504e8ab3e17e31a1992df01b1915191a5b1dc7788ac275a0e96ed17cac4b8fdd88caf723f06add20e747d90ee17e6
+  metadata.gz: 033077e27db0e59ec53895ab8038ff4da2c1cf8e223ebe7296dce00d7961e3d79edf5ac807b438e2eb0c71afb42a93bf9923d6aa1ce647a3df7323190c4dcf3d
+  data.tar.gz: ca759475503697dfe741a92b11d9beb8f8520056cb19aea59745d350103c56dded3f6641087b77448652bdfe06ead35c6916d6d356447bb8da31271699849eca

data/Rakefile

@@ -20,7 +20,7 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-APP_RAKEFILE = File.expand_path('../test/dummy/Rakefile', __FILE__)
+APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
 
 Bundler::GemHelper.install_tasks
 

data/app/controllers/foreman_inventory_upload/accounts_controller.rb

@@ -0,0 +1,39 @@
+module ForemanInventoryUpload
+  class AccountsController < ::ApplicationController
+    # override default "welcome screen behavior, since we don't have a model"
+    def welcome
+      true
+    end
+
+    def index
+      labels = Organization.all.pluck(:id, :name)
+
+      accounts = Hash[
+        labels.map do |id, label|
+          generate_report_status = status_for(id, ForemanInventoryUpload::Async::GenerateReportJob)
+          upload_report_status = status_for(id, ForemanInventoryUpload::Async::UploadReportJob)
+
+          [
+            id,
+            {
+              generate_report_status: generate_report_status,
+              upload_report_status: upload_report_status,
+              label: label,
+            },
+          ]
+        end
+      ]
+
+      render json: {
+        accounts: accounts,
+      }, status: :ok
+    end
+
+    private
+
+    def status_for(label, job_class)
+      label = job_class.output_label(label)
+      ForemanInventoryUpload::Async::ProgressOutput.get(label)&.status
+    end
+  end
+end

data/app/controllers/foreman_inventory_upload/reports_controller.rb

@@ -1,22 +1,29 @@
+# frozen_string_literal: true
+
 module ForemanInventoryUpload
   class ReportsController < ::ApplicationController
     def last
-      label = ForemanInventoryUpload::Async::GenerateReportJob.output_label(params[:portal_user])
+      label = ForemanInventoryUpload::Async::GenerateReportJob.output_label(params[:organization_id])
       output = ForemanInventoryUpload::Async::ProgressOutput.get(label)&.full_output
+      task_label = ForemanInventoryUpload::Async::GenerateAllReportsJob.singleton_job_name
+      scheduled = ForemanTasks::Task.where(
+        :label => task_label,
+        :state => 'scheduled'
+      ).first&.start_at || nil
 
       render json: {
-        output: output
+        output: output,
+        scheduled: scheduled,
       }, status: :ok
     end
 
     def generate
-      portal_user = params[:portal_user]
+      organization_id = params[:organization_id]
 
-      generated_file_name = File.join(ForemanInventoryUpload.base_folder, "#{portal_user}.tar.gz")
-      ForemanInventoryUpload::Async::GenerateReportJob.perform_later(generated_file_name, portal_user)
+      ForemanInventoryUpload::Async::GenerateReportJob.perform_later(ForemanInventoryUpload.generated_reports_folder, organization_id)
 
       render json: {
-        action_status: 'success'
+        action_status: 'success',
       }, status: :ok
     end
   end

data/app/controllers/foreman_inventory_upload/uploads_controller.rb

@@ -1,17 +1,17 @@
 module ForemanInventoryUpload
   class UploadsController < ::ApplicationController
     def last
-      label = ForemanInventoryUpload::Async::UploadReportJob.output_label(params[:portal_user])
+      label = ForemanInventoryUpload::Async::UploadReportJob.output_label(params[:organization_id])
       output = ForemanInventoryUpload::Async::ProgressOutput.get(label)&.full_output
 
       render json: {
-        output: output
+        output: output,
       }, status: :ok
     end
 
     def download_file
-      filename = 'hosts_report.tar.gz'
-      path = Rails.root.join(ForemanInventoryUpload.uploads_folder(params[:portal_user]), filename)
+      filename = ForemanInventoryUpload.facts_archive_name(params[:organization_id])
+      path = Rails.root.join(ForemanInventoryUpload.uploads_folder, filename)
       unless File.exist? path
         return throw_flash_error(
           "Path doesn't exist: #{path}"

data/app/helpers/foreman_inventory_upload_helper.rb

@@ -10,4 +10,4 @@ module ForemanInventoryUploadHelper
       end
     end
   end
-end
+end

data/app/views/scripts/uploader.sh.erb

@@ -1,28 +1,39 @@
 #! /bin/sh
 
 DEST=<%= @upload_url %>
-RH_USERNAME=<%= @rh_username %>
 
-if [ -z "$RH_USERNAME" ]
+if [ -z "$FILES" ]
 then
-        IFS= read -rp "Enter username: " RH_USERNAME
+        FILES=./*.tar.gz
 fi
 
-if [ -z "$RH_PASSWORD" ]
+if [ -n "$CER_PATH" ]
 then
-        IFS= read -rsp "Enter password: " RH_PASSWORD
-fi
+        AUTH_KEY="--cert"
+        AUTH_VAL="$CER_PATH"
+else
+        if [ -z "$RH_USERNAME" ]
+        then
+                IFS= read -rp "Enter username: " RH_USERNAME
+        fi
 
-if [ -z "$FILES" ]
-then
-        FILES=./*.tar.gz
+        if [ -z "$RH_PASSWORD" ]
+        then
+                IFS= read -rsp "Enter password: " RH_PASSWORD
+        fi
+
+        AUTH_KEY="-u"
+        AUTH_VAL="\"$RH_USERNAME\":\"$RH_PASSWORD\""
 fi
 
+# /tmp/a b/x.pem
+# curl --cert /tmp/a\ b/x.pem
+
 mkdir -p done
 
 for f in $FILES
 do
-  curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST -u "$RH_USERNAME":"$RH_PASSWORD"
+  curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST "$AUTH_KEY" "$AUTH_VAL"
   if [ $? -eq 0 ]; then
     mv $f done/
     echo "Done: $f"

data/config/routes.rb

@@ -1,10 +1,10 @@
 Rails.application.routes.draw do
   namespace :foreman_inventory_upload do
     get 'index', to: 'react#index'
-    get ':portal_user/reports/last', to: 'reports#last', constraints: { portal_user: %r{[^\/]+} }
-    post ':portal_user/reports', to: 'reports#generate', constraints: { portal_user: %r{[^\/]+} }
-    get ':portal_user/uploads/last', to: 'uploads#last', constraints: { portal_user: %r{[^\/]+} }
-    get ':portal_user/uploads/file', to: 'uploads#download_file', constraints: { portal_user: %r{[^\/]+} }
-    get 'statuses', to: 'statuses#index'
+    get ':organization_id/reports/last', to: 'reports#last', constraints: { organization_id: %r{[^\/]+} }
+    post ':organization_id/reports', to: 'reports#generate', constraints: { organization_id: %r{[^\/]+} }
+    get ':organization_id/uploads/last', to: 'uploads#last', constraints: { organization_id: %r{[^\/]+} }
+    get ':organization_id/uploads/file', to: 'uploads#download_file', constraints: { organization_id: %r{[^\/]+} }
+    get 'accounts', to: 'accounts#index'
   end
 end

data/lib/foreman_inventory_upload/async/generate_all_reports_job.rb

@@ -0,0 +1,19 @@
+module ForemanInventoryUpload
+  module Async
+    class GenerateAllReportsJob < ::ApplicationJob
+      def perform
+        organizations = Organization.unscoped.all
+
+        organizations.map do |organization|
+          GenerateReportJob.perform_later(ForemanInventoryUpload.generated_reports_folder, organization.id)
+        end
+      ensure
+        self.class.set(:wait => 24.hours).perform_later
+      end
+
+      def self.singleton_job_name
+        name
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/generate_report_job.rb

@@ -1,17 +1,21 @@
 module ForemanInventoryUpload
   module Async
     class GenerateReportJob < ShellProcess
-      def self.output_label(portal_user)
-        "report_for_#{portal_user}"
+      def self.output_label(label)
+        "report_for_#{label}"
       end
 
-      def perform(result_file, portal_user)
-        @result_file = result_file
-        @portal_user = portal_user
+      def perform(base_folder, organization)
+        @base_folder = base_folder
+        @organization = organization
 
-        super(GenerateReportJob.output_label(portal_user))
+        super(GenerateReportJob.output_label(organization))
 
-        QueueForUploadJob.perform_later(result_file, portal_user)
+        QueueForUploadJob.perform_later(
+          base_folder,
+          ForemanInventoryUpload.facts_archive_name(organization),
+          organization
+        )
       end
 
       def command
@@ -20,8 +24,8 @@ module ForemanInventoryUpload
 
       def env
         super.merge(
-          'target' => @result_file,
-          'portal_user' => @portal_user
+          'target' => @base_folder,
+          'organization_id' => @organization
         )
       end
     end

data/lib/foreman_inventory_upload/async/progress_output.rb

@@ -50,7 +50,7 @@ module ForemanInventoryUpload
       private
 
       def file_mode
-        @mode == :reader ? 'r' : 'w'
+        (@mode == :reader) ? 'r' : 'w'
       end
 
       def file_name(type = 'out')

data/lib/foreman_inventory_upload/async/queue_for_upload_job.rb

@@ -1,21 +1,22 @@
 module ForemanInventoryUpload
   module Async
     class QueueForUploadJob < ::ApplicationJob
-      def perform(report_file, portal_user)
-        @portal_user = portal_user
+      def perform(base_folder, report_file, organization_id)
+        @base_folder = base_folder
+        @report_file = report_file
         logger.debug('Ensuring objects')
         ensure_ouput_folder
         ensure_output_script
         logger.debug("Copying #{report_file} to #{uploads_folder}")
-        enqueued_file_name = File.join(uploads_folder, ForemanInventoryUpload.facts_archive_name)
-        FileUtils.mv(report_file, enqueued_file_name)
+        enqueued_file_name = File.join(uploads_folder, report_file)
+        FileUtils.mv(File.join(base_folder, report_file), enqueued_file_name)
         logger.debug("Done copying #{report_file} to #{enqueued_file_name}")
 
-        UploadReportJob.perform_later(enqueued_file_name, portal_user)
+        UploadReportJob.perform_later(enqueued_file_name, organization_id)
       end
 
       def uploads_folder
-        @uploads_folder ||= ForemanInventoryUpload.uploads_folder(@portal_user)
+        @uploads_folder ||= ForemanInventoryUpload.uploads_folder
       end
 
       def script_file
@@ -29,17 +30,11 @@ module ForemanInventoryUpload
       def ensure_output_script
         return if File.exist?(script_file)
 
-        script_source = File.join(ForemanInventoryUpload::Engine.root, 'app/views/scripts/uploader.sh.erb')
-
-        template_src = Foreman::Renderer::Source::String.new(content: File.read(script_source))
-        scope = Foreman::Renderer::Scope::Base.new(
-          source: template_src,
-          variables: {
-            upload_url: ForemanInventoryUpload.upload_url,
-            rh_username: @portal_user
-          }
-        )
-        script_source = Foreman::Renderer.render(template_src, scope)
+        script_source_file = File.join(ForemanInventoryUpload::Engine.root, 'app/views/scripts/uploader.sh.erb')
+
+        script_source = File.read(script_source_file)
+        script_source.gsub!('<%= @upload_url %>', ForemanInventoryUpload.upload_url)
+
         File.write(script_file, script_source)
         FileUtils.chmod('+x', script_file)
       end

data/lib/foreman_inventory_upload/async/shell_process.rb

@@ -21,7 +21,8 @@ module ForemanInventoryUpload
         progress_output.close
       end
 
-      def command; end
+      def command
+      end
 
       def env
         {}

data/lib/foreman_inventory_upload/async/upload_report_job.rb

@@ -1,15 +1,23 @@
+require 'tempfile'
+
 module ForemanInventoryUpload
   module Async
     class UploadReportJob < ShellProcess
-      def self.output_label(portal_user)
-        "upload_for_#{portal_user}"
+      def self.output_label(label)
+        "upload_for_#{label}"
       end
 
-      def perform(filename, portal_user)
-        @portal_user = portal_user
+      def perform(filename, organization_id)
         @filename = filename
+        @organization = Organization.find(organization_id)
 
-        super(UploadReportJob.output_label(portal_user))
+        Tempfile.create([@organization.name, '.pem']) do |cer_file|
+          cer_file.write(rh_credentials[:cert])
+          cer_file.write(rh_credentials[:key])
+          cer_file.flush
+          @cer_path = cer_file.path
+          super(UploadReportJob.output_label(organization_id))
+        end
       end
 
       def command
@@ -18,22 +26,19 @@ module ForemanInventoryUpload
 
       def env
         super.merge(
-          'RH_USERNAME' => rh_username,
-          'RH_PASSWORD' => rh_password,
-          'FILES' => @filename
+          'FILES' => @filename,
+          'CER_PATH' => @cer_path
         )
       end
 
       def rh_credentials
-        @rh_credentials ||= RedhatAccess::TelemetryConfiguration.where(portal_user: @portal_user).last
-      end
-
-      def rh_username
-        @portal_user
-      end
-
-      def rh_password
-        rh_credentials.portal_password
+        @rh_credentials ||= begin
+          candlepin_id_certificate = @organization.owner_details['upstreamConsumer']['idCert']
+          {
+            cert: candlepin_id_certificate['cert'],
+            key: candlepin_id_certificate['key'],
+          }
+        end
       end
     end
   end

data/lib/foreman_inventory_upload/engine.rb

@@ -11,6 +11,8 @@ module ForemanInventoryUpload
     config.autoload_paths += Dir["#{config.root}/app/overrides"]
     config.autoload_paths += Dir["#{config.root}/lib"]
 
+    config.eager_load_paths += Dir["#{config.root}/lib"]
+
     # Add any db migrations
     initializer 'foreman_inventory_upload.load_app_instance_data' do |app|
       ForemanInventoryUpload::Engine.paths['db/migrate'].existent.each do |path|
@@ -32,7 +34,15 @@ module ForemanInventoryUpload
 
         # Adding a sub menu after hosts menu
         sub_menu :top_menu, :foreman_inventory_upload, :caption => N_('RH Inventory'), :icon => 'fa fa-cloud-upload' do
-          menu :top_menu, :level1, :caption => N_('Manage'), :url_hash => { controller: :'foreman_inventory_upload/react', :action=>:index}
+          menu :top_menu, :level1, :caption => N_('Manage'), :url_hash => { controller: :'foreman_inventory_upload/react', :action => :index}
+        end
+      end
+    end
+
+    initializer "foreman_inventory_upload.set_dynflow.config.on_init", :before => :finisher_hook do |_app|
+      unless Rails.env.test?
+        ForemanTasks.dynflow.config.on_init do |world|
+          ForemanInventoryUpload::Async::GenerateAllReportsJob.spawn_if_missing(world)
         end
       end
     end

data/lib/foreman_inventory_upload/generators/archived_report.rb

@@ -6,10 +6,10 @@ module ForemanInventoryUpload
         @logger = logger
       end
 
-      def render(portal_user)
+      def render(organization:)
         Dir.mktmpdir do |tmpdir|
           @logger.info "Started generating hosts report in #{tmpdir}"
-          host_batches = ForemanInventoryUpload::Generators::Queries.for_report(portal_user)
+          host_batches = ForemanInventoryUpload::Generators::Queries.for_org(organization)
           File.open(File.join(tmpdir, 'metadata.json'), 'w') do |metadata_out|
             metadata_generator = ForemanInventoryUpload::Generators::Metadata.new(metadata_out)
             metadata_generator.render do |inner_generator|

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -16,7 +16,8 @@ module ForemanInventoryUpload
               'dmi::bios::relase_date',
               'distribution::name',
               'uname::release',
-              'lscpu::flags'
+              'lscpu::flags',
+              'distribution::version',
             ]).pluck(:name, :id)
           ]
       end
@@ -31,21 +32,29 @@ module ForemanInventoryUpload
             :installed_packages,
             :content_facet,
             :host_statuses,
-            subscription_facet: %i[pools installed_products]
+            subscription_facet: [:pools, :installed_products, :hypervisor_host]
           )
           .merge(fact_values)
       end
 
       def self.for_report(portal_user)
-        org_ids = Organization
-                  .joins(:telemetry_configuration)
-                  .where(
-                    redhat_access_telemetry_configurations: {
-                      portal_user: portal_user,
-                      enable_telemetry: true
-                    }
-                  ).pluck(:id)
-        for_slice(Host.where(organization_id: org_ids)).in_batches(of: 1_000)
+        org_ids = organizations_for_user(portal_user).pluck(:id)
+        for_slice(Host.unscoped.where(organization_id: org_ids)).in_batches(of: 1_000)
+      end
+
+      def self.for_org(organization_id)
+        for_slice(Host.unscoped.where(organization_id: organization_id)).in_batches(of: 1_000)
+      end
+
+      def self.organizations_for_user(portal_user)
+        Organization
+          .joins(:telemetry_configuration)
+          .where(
+            redhat_access_telemetry_configurations: {
+              portal_user: portal_user,
+              enable_telemetry: true,
+            }
+          )
       end
     end
   end

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -23,14 +23,14 @@ module ForemanInventoryUpload
             first = true
             hosts_batch.each do |host|
               @stream.comma unless first
-              first = false
-              report_host(host)
+              first = false if report_host(host)
             end
           end
         end
       end
 
       def report_host(host)
+        return nil unless host&.subscription_facet&.pools&.first
         @stream.object do
           @stream.simple_field('display_name', host.name)
           @stream.simple_field('fqdn', host.fqdn)
@@ -49,9 +49,17 @@ module ForemanInventoryUpload
               @stream.stringify_value(nic.mac) if nic.mac
             end.compact.join(', '))
           end
-          @stream.object_field('system_profile', :last) do
+          @stream.object_field('system_profile') do
             report_system_profile(host)
           end
+          @stream.array_field('facts', :last) do
+            @stream.object do
+              @stream.simple_field('namespace', 'satellite')
+              @stream.object_field('facts', :last) do
+                report_satellite_facts(host)
+              end
+            end
+          end
         end
       end
 
@@ -67,7 +75,7 @@ module ForemanInventoryUpload
               'ipv6_addresses': [nic.ip6].compact,
               'mtu': nic.mtu,
               'mac_address': nic.mac,
-              'name': nic.identifier
+              'name': nic.identifier,
             }.compact.to_json
           end.join(', '))
         end
@@ -92,7 +100,7 @@ module ForemanInventoryUpload
             @stream.raw(installed_products.map do |product|
               {
                 'name': product.name,
-                'id': product.cp_product_id
+                'id': product.cp_product_id,
               }.to_json
             end.join(', '))
           end
@@ -106,6 +114,19 @@ module ForemanInventoryUpload
         end
       end
 
+      def report_satellite_facts(host)
+        @stream.simple_field('virtual_host_name', host.subscription_facet.hypervisor_host&.name)
+        @stream.simple_field('virtual_host_uuid', host.subscription_facet.hypervisor_host&.subscription_facet&.uuid)
+        if defined?(ForemanThemeSatellite)
+          @stream.simple_field('satellite_version', ForemanThemeSatellite::SATELLITE_VERSION)
+        end
+        @stream.simple_field('system_purpose_usage', host.subscription_facet.purpose_usage)
+        @stream.simple_field('system_purpose_role', host.subscription_facet.purpose_role)
+        @stream.simple_field('distribution_version', fact_value(host, 'distribution::version'))
+        @stream.simple_field('satellite_instance_id', Foreman.respond_to?(:instance_id) ? Foreman.instance_id : nil)
+        @stream.simple_field('organization_id', host.organization_id, :last)
+      end
+
       def fact_value(host, fact_name)
         value_record = host.fact_values.find do |fact_value|
           fact_value.fact_name_id == ForemanInventoryUpload::Generators::Queries.fact_names[fact_name]

data/lib/foreman_inventory_upload/version.rb

@@ -1,3 +1,3 @@
 module ForemanInventoryUpload
-  VERSION = '0.0.1.dev1'.freeze
+  VERSION = '0.9.0'.freeze
 end

data/lib/foreman_inventory_upload.rb

@@ -11,16 +11,20 @@ module ForemanInventoryUpload
     )
   end
 
-  def self.uploads_folder(group)
-    @uploads_folders ||= {}
-    cache = @uploads_folders[group]
-    return cache if cache
+  def self.uploads_folder
+    @uploads_folder ||= ensure_folder(
+      File.join(
+        ForemanInventoryUpload.base_folder,
+        'uploads/'
+      )
+    )
+  end
 
-    @uploads_folders[group] = ensure_folder(
+  def self.generated_reports_folder
+    @generated_reports_folder ||= ensure_folder(
       File.join(
         ForemanInventoryUpload.base_folder,
-        'uploads/',
-        "#{group}/"
+        'generated_reports/'
       )
     )
   end
@@ -33,12 +37,13 @@ module ForemanInventoryUpload
     'uploader.sh'
   end
 
-  def self.facts_archive_name
-    'hosts_report.tar.gz'
+  def self.facts_archive_name(organization)
+    "report_for_#{organization}.tar.gz"
   end
 
   def self.upload_url
-    'https://ci.cloud.paas.psi.redhat.com/api/ingress/v1/upload'
+    # for testing set ENV to 'https://ci.cloud.redhat.com/api/ingress/v1/upload'
+    @upload_url ||= ENV['SATELLITE_INVENTORY_UPLOAD_URL'] || 'https://cloud.redhat.com/api/ingress/v1/upload'
   end
 
   def self.ensure_folder(folder)

data/lib/tasks/generator.rake

@@ -4,12 +4,26 @@ namespace :foreman_inventory_upload do
   namespace :report do
     desc 'Generate inventory report to be sent to Red Hat cloud'
     task generate: :environment do
-      target = ENV['target'] || ForemanInventoryUpload.facts_archive_name
-      portal_user = ENV['portal_user'] || 'anonymous'
+      portal_user = ENV['portal_user']
+      organizations = [ENV['organization_id']]
+      base_folder = ENV['target'] || Dir.pwd
 
-      archived_report_generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
-      archived_report_generator.render(portal_user)
-      puts "Successfully generated #{target} for #{portal_user}"
+      unless portal_user || organizations.empty?
+        puts "Must specify either portal_user or organization_id"
+      end
+
+      if portal_user
+        puts "Generating report for all organizations associated with #{portal_user}"
+        base_folder = File.join(base_folder, portal_user)
+        organizations = ForemanInventoryUpload::Generators::Queries.organizations_for_user(portal_user).pluck(:id)
+      end
+
+      organizations.each do |organization|
+        target = File.join(base_folder, ForemanInventoryUpload.facts_archive_name(organization))
+        archived_report_generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
+        archived_report_generator.render(organization: organization)
+        puts "Successfully generated #{target} for organization id #{organization}"
+      end
     end
   end
 end