foreman_rh_cloud 0.9.4

data/lib/foreman_inventory_upload/async/shell_process.rb

@@ -0,0 +1,36 @@
+require 'open3'
+
+module ForemanInventoryUpload
+  module Async
+    class ShellProcess < ::ApplicationJob
+      def perform(instance_label)
+        klass_name = self.class.name
+        logger.debug("Starting #{klass_name} with label #{instance_label}")
+        progress_output = ProgressOutput.register(instance_label)
+        Open3.popen2e(env, command) do |_stdin, stdout_stderr, wait_thread|
+          progress_output.status = "Running in pid #{wait_thread.pid}"
+
+          stdout_stderr.each do |out_line|
+            progress_output.write_line(out_line)
+          end
+
+          progress_output.status = wait_thread.value.to_s
+        end
+        logger.debug("Finished job #{klass_name} with label #{instance_label}")
+      ensure
+        progress_output.close
+      end
+
+      def command
+      end
+
+      def env
+        {}
+      end
+
+      def logger
+        Foreman::Logging.logger('background')
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/upload_report_job.rb

@@ -0,0 +1,58 @@
+require 'tempfile'
+
+module ForemanInventoryUpload
+  module Async
+    class UploadReportJob < ShellProcess
+      def self.output_label(label)
+        "upload_for_#{label}"
+      end
+
+      def perform(filename, organization_id)
+        @filename = filename
+        @organization = Organization.find(organization_id)
+
+        Tempfile.create([@organization.name, '.pem']) do |cer_file|
+          cer_file.write(rh_credentials[:cert])
+          cer_file.write(rh_credentials[:key])
+          cer_file.flush
+          @cer_path = cer_file.path
+          super(UploadReportJob.output_label(organization_id))
+        end
+      end
+
+      def command
+        File.join(File.dirname(@filename), ForemanInventoryUpload.upload_script_file)
+      end
+
+      def env
+        env_vars = super.merge(
+          'FILES' => @filename,
+          'CER_PATH' => @cer_path
+        )
+        if http_proxy_string
+          env_vars['http_proxy'] = http_proxy_string
+          env_vars['https_proxy'] = http_proxy_string
+        end
+        env_vars
+      end
+
+      def http_proxy_string
+        @http_proxy_string ||= begin
+          if Setting[:content_default_http_proxy]
+            HttpProxy.unscoped.find_by(name: Setting[:content_default_http_proxy])&.full_url
+          end
+        end
+      end
+
+      def rh_credentials
+        @rh_credentials ||= begin
+          candlepin_id_certificate = @organization.owner_details['upstreamConsumer']['idCert']
+          {
+            cert: candlepin_id_certificate['cert'],
+            key: candlepin_id_certificate['key'],
+          }
+        end
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/archived_report.rb

@@ -0,0 +1,53 @@
+module ForemanInventoryUpload
+  module Generators
+    class ArchivedReport
+      def initialize(target, logger = Rails.logger)
+        @target = target
+        @logger = logger
+      end
+
+      def render(organization:)
+        Dir.mktmpdir do |tmpdir|
+          @logger.info "Started generating hosts report in #{tmpdir}"
+          host_batches = ForemanInventoryUpload::Generators::Queries.for_org(organization)
+          File.open(File.join(tmpdir, 'metadata.json'), 'w') do |metadata_out|
+            metadata_generator = ForemanInventoryUpload::Generators::Metadata.new(metadata_out)
+            metadata_generator.render do |inner_generator|
+              first = true
+              host_batches.each do |hosts_batch|
+                slice_id = Foreman.uuid
+                hosts_count = hosts_batch.count
+                @logger.info "Adding slice #{slice_id} with #{hosts_count} hosts"
+                generate_slice(tmpdir, slice_id, hosts_batch)
+                inner_generator.add_slice(slice_id, hosts_count, first)
+                first = false
+              end
+            end
+          end
+          @logger.info 'Report generation finished'
+
+          @logger.info 'Archiving generated report'
+          # success = system('tar', '-zcvf', @target, '-C', tmpdir, '.')
+          Open3.popen2e('tar', '-zcvf', @target, '-C', tmpdir, '.') do |_in, out, wait_thr|
+            @logger.info("tar: #{out.read}")
+
+            if wait_thr.value.success?
+              @logger.info 'Report archived successfully'
+            else
+              @logger.info "Tar command failed: #{$CHILD_STATUS}"
+            end
+          end
+        end
+      end
+
+      private
+
+      def generate_slice(tmpdir, slice_id, hosts_batch)
+        File.open(File.join(tmpdir, "#{slice_id}.json"), 'w') do |slice_out|
+          slice_generator = ForemanInventoryUpload::Generators::Slice.new(hosts_batch, slice_out, slice_id)
+          slice_generator.render
+        end
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/fact_helpers.rb

@@ -0,0 +1,18 @@
+module ForemanInventoryUpload
+  module Generators
+    module FactHelpers
+      extend ActiveSupport::Concern
+
+      def fact_value(host, fact_name)
+        value_record = host.fact_values.find do |fact_value|
+          fact_value.fact_name_id == ForemanInventoryUpload::Generators::Queries.fact_names[fact_name]
+        end
+        value_record&.value
+      end
+
+      def kilobytes_to_bytes(kilobytes)
+        kilobytes * 1024
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/json_stream.rb

@@ -0,0 +1,55 @@
+module ForemanInventoryUpload
+  module Generators
+    class JsonStream
+      attr_reader :out
+      def initialize(out)
+        @out = out
+      end
+
+      def array
+        @out << '['
+        yield
+        @out << ']'
+      end
+
+      def object
+        @out << '{'
+        yield
+        @out << '}'
+      end
+
+      def comma
+        @out << ', '
+      end
+
+      def raw(string)
+        @out << string
+      end
+
+      def simple_field(name, value, last = false)
+        @out << "\"#{name}\": #{stringify_value(value)}#{last ? '' : ','}" unless value.nil?
+      end
+
+      def array_field(name, last = false, &block)
+        @out << "\"#{name}\": "
+        array(&block)
+        @out << ',' unless last
+      end
+
+      def object_field(name, last = false, &block)
+        @out << "\"#{name}\": "
+        object(&block)
+        @out << ',' unless last
+      end
+
+      def stringify_value(value)
+        return value if value.is_a?(Integer)
+        return value if value.is_a?(TrueClass)
+        return value if value.is_a?(FalseClass)
+        return value.to_json if value.is_a?(Hash)
+
+        "\"#{value}\""
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/metadata.rb

@@ -0,0 +1,36 @@
+module ForemanInventoryUpload
+  module Generators
+    class Metadata
+      def initialize(output = [])
+        @stream = JsonStream.new(output)
+      end
+
+      def add_slice(slice_id, hosts_count, first)
+        @stream.comma unless first
+
+        @stream.object_field(slice_id, :last) do
+          @stream.simple_field('number_hosts', hosts_count, :last)
+        end
+      end
+
+      def render(metadata = nil, &block)
+        render_report(metadata, &block)
+        @stream.out
+      end
+
+      private
+
+      def render_report(metadata)
+        @stream.object do
+          @stream.simple_field('report_id', Foreman.uuid)
+          @stream.simple_field('host_inventory_api_version', '1.0')
+          @stream.simple_field('source', 'Satellite')
+          @stream.simple_field('source_metadata', metadata)
+          @stream.object_field('report_slices', :last) do
+            yield(self)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -0,0 +1,61 @@
+module ForemanInventoryUpload
+  module Generators
+    class Queries
+      def self.fact_names
+        @fact_names ||= Hash[
+          Katello::RhsmFactName.where(name:
+            [
+              'dmi::system::uuid',
+              'virt::uuid',
+              'cpu::cpu(s)',
+              'cpu::cpu_socket(s)',
+              'cpu::core(s)_per_socket',
+              'memory::memtotal',
+              'dmi::bios::vendor',
+              'dmi::bios::version',
+              'dmi::bios::relase_date',
+              'distribution::name',
+              'uname::release',
+              'lscpu::flags',
+              'distribution::version',
+            ]).pluck(:name, :id)
+          ]
+      end
+
+      def self.for_slice(base)
+        fact_values = FactValue.where(fact_name_id: fact_names.values)
+        base
+          .joins(:subscription_facet)
+          .eager_load(:fact_values)
+          .preload(
+            :interfaces,
+            :installed_packages,
+            :content_facet,
+            :host_statuses,
+            subscription_facet: [:pools, :installed_products, :hypervisor_host]
+          )
+          .merge(fact_values)
+      end
+
+      def self.for_report(portal_user)
+        org_ids = organizations_for_user(portal_user).pluck(:id)
+        for_slice(Host.unscoped.where(organization_id: org_ids)).in_batches(of: 1_000)
+      end
+
+      def self.for_org(organization_id)
+        for_slice(Host.unscoped.where(organization_id: organization_id)).in_batches(of: 1_000)
+      end
+
+      def self.organizations_for_user(portal_user)
+        Organization
+          .joins(:telemetry_configuration)
+          .where(
+            redhat_access_telemetry_configurations: {
+              portal_user: portal_user,
+              enable_telemetry: true,
+            }
+          )
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -0,0 +1,147 @@
+module ForemanInventoryUpload
+  module Generators
+    class Slice
+      include FactHelpers
+
+      attr_accessor :slice_id
+
+      def initialize(hosts, output = [], slice_id = Foreman.uuid)
+        @stream = JsonStream.new(output)
+        @hosts = hosts
+        @slice_id = slice_id
+      end
+
+      def render
+        report_slice(@hosts)
+        @stream.out
+      end
+
+      private
+
+      def report_slice(hosts_batch)
+        @stream.object do
+          @stream.simple_field('report_slice_id', @slice_id)
+          @stream.array_field('hosts', :last) do
+            first = true
+            hosts_batch.each do |host|
+              next unless host&.subscription_facet&.pools&.first
+              @stream.comma unless first
+              first = false if report_host(host)
+            end
+          end
+        end
+      end
+
+      def report_host(host)
+        @stream.object do
+          @stream.simple_field('display_name', host.name)
+          @stream.simple_field('fqdn', host.fqdn)
+          @stream.simple_field('account', host.subscription_facet.pools.first.account_number.to_s)
+          @stream.simple_field('subscription_manager_id', host.subscription_facet.uuid)
+          @stream.simple_field('satellite_id', host.subscription_facet.uuid)
+          @stream.simple_field('bios_uuid', fact_value(host, 'dmi::system::uuid'))
+          @stream.simple_field('vm_uuid', fact_value(host, 'virt::uuid'))
+          @stream.array_field('ip_addresses') do
+            @stream.raw(host.interfaces.map do |nic|
+              @stream.stringify_value(nic.ip) if nic.ip
+            end.compact.join(', '))
+          end
+          @stream.array_field('mac_addresses') do
+            @stream.raw(host.interfaces.map do |nic|
+              @stream.stringify_value(nic.mac) if nic.mac
+            end.compact.join(', '))
+          end
+          @stream.object_field('system_profile') do
+            report_system_profile(host)
+          end
+          @stream.array_field('facts') do
+            @stream.object do
+              @stream.simple_field('namespace', 'satellite')
+              @stream.object_field('facts', :last) do
+                report_satellite_facts(host)
+              end
+            end
+          end
+
+          @stream.array_field('tags', :last) do
+            report_tag('satellite', 'satellite_instance_id', Foreman.instance_id) if Foreman.respond_to?(:instance_id)
+            report_tag('satellite', 'organization_id', host.organization_id, :last)
+          end
+        end
+      end
+
+      def report_tag(namespace, key, value, last = nil)
+        @stream.object do
+          @stream.simple_field('namespace', namespace)
+          @stream.simple_field('key', key)
+          @stream.simple_field('value', value, :last)
+        end
+        @stream.comma unless last
+      end
+
+      def report_system_profile(host)
+        @stream.simple_field('number_of_cpus', fact_value(host, 'cpu::cpu(s)').to_i)
+        @stream.simple_field('number_of_sockets', fact_value(host, 'cpu::cpu_socket(s)').to_i)
+        @stream.simple_field('cores_per_socket', fact_value(host, 'cpu::core(s)_per_socket').to_i)
+        @stream.simple_field('system_memory_bytes', kilobytes_to_bytes(fact_value(host, 'memory::memtotal').to_i))
+        @stream.array_field('network_interfaces') do
+          @stream.raw(host.interfaces.map do |nic|
+            {
+              'ipv4_addresses': [nic.ip].compact,
+              'ipv6_addresses': [nic.ip6].compact,
+              'mtu': nic.try(:mtu),
+              'mac_address': nic.mac,
+              'name': nic.identifier,
+            }.compact.to_json
+          end.join(', '))
+        end
+        @stream.simple_field('bios_vendor', fact_value(host, 'dmi::bios::vendor'))
+        @stream.simple_field('bios_version', fact_value(host, 'dmi::bios::version'))
+        @stream.simple_field('bios_release_date', fact_value(host, 'dmi::bios::relase_date'))
+        if (cpu_flags = fact_value(host, 'lscpu::flags'))
+          @stream.array_field('cpu_flags') do
+            @stream.raw(cpu_flags.split.map do |flag|
+              @stream.stringify_value(flag)
+            end.join(', '))
+          end
+        end
+        @stream.simple_field('os_release', fact_value(host, 'distribution::name'))
+        @stream.simple_field('os_kernel_version', fact_value(host, 'uname::release'))
+        @stream.simple_field('arch', host.architecture&.name)
+        @stream.simple_field('subscription_status', host.subscription_status_label)
+        @stream.simple_field('katello_agent_running', host.content_facet&.katello_agent_installed?)
+        @stream.simple_field('satellite_managed', true)
+        unless (installed_products = host.subscription_facet&.installed_products).empty?
+          @stream.array_field('installed_products') do
+            @stream.raw(installed_products.map do |product|
+              {
+                'name': product.name,
+                'id': product.cp_product_id,
+              }.to_json
+            end.join(', '))
+          end
+        end
+        @stream.array_field('installed_packages', :last) do
+          first = true
+          host.installed_packages.each do |package|
+            @stream.raw("#{first ? '' : ', '}#{@stream.stringify_value(package.nvra)}")
+            first = false
+          end
+        end
+      end
+
+      def report_satellite_facts(host)
+        @stream.simple_field('virtual_host_name', host.subscription_facet.hypervisor_host&.name)
+        @stream.simple_field('virtual_host_uuid', host.subscription_facet.hypervisor_host&.subscription_facet&.uuid)
+        if defined?(ForemanThemeSatellite)
+          @stream.simple_field('satellite_version', ForemanThemeSatellite::SATELLITE_VERSION)
+        end
+        @stream.simple_field('system_purpose_usage', host.subscription_facet.purpose_usage)
+        @stream.simple_field('system_purpose_role', host.subscription_facet.purpose_role)
+        @stream.simple_field('distribution_version', fact_value(host, 'distribution::version'))
+        @stream.simple_field('satellite_instance_id', Foreman.respond_to?(:instance_id) ? Foreman.instance_id : nil)
+        @stream.simple_field('organization_id', host.organization_id, :last)
+      end
+    end
+  end
+end