foreman_inventory_upload 0.0.1.dev1

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -0,0 +1,52 @@
+module ForemanInventoryUpload
+  module Generators
+    class Queries
+      def self.fact_names
+        @fact_names ||= Hash[
+          Katello::RhsmFactName.where(name:
+            [
+              'dmi::system::uuid',
+              'virt::uuid',
+              'cpu::cpu(s)',
+              'cpu::cpu_socket(s)',
+              'cpu::core(s)_per_socket',
+              'memory::memtotal',
+              'dmi::bios::vendor',
+              'dmi::bios::version',
+              'dmi::bios::relase_date',
+              'distribution::name',
+              'uname::release',
+              'lscpu::flags'
+            ]).pluck(:name, :id)
+          ]
+      end
+
+      def self.for_slice(base)
+        fact_values = FactValue.where(fact_name_id: fact_names.values)
+        base
+          .joins(:subscription_facet)
+          .eager_load(:fact_values)
+          .preload(
+            :interfaces,
+            :installed_packages,
+            :content_facet,
+            :host_statuses,
+            subscription_facet: %i[pools installed_products]
+          )
+          .merge(fact_values)
+      end
+
+      def self.for_report(portal_user)
+        org_ids = Organization
+                  .joins(:telemetry_configuration)
+                  .where(
+                    redhat_access_telemetry_configurations: {
+                      portal_user: portal_user,
+                      enable_telemetry: true
+                    }
+                  ).pluck(:id)
+        for_slice(Host.where(organization_id: org_ids)).in_batches(of: 1_000)
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -0,0 +1,117 @@
+module ForemanInventoryUpload
+  module Generators
+    class Slice
+      attr_accessor :slice_id
+
+      def initialize(hosts, output = [], slice_id = Foreman.uuid)
+        @stream = JsonStream.new(output)
+        @hosts = hosts
+        @slice_id = slice_id
+      end
+
+      def render
+        report_slice(@hosts)
+        @stream.out
+      end
+
+      private
+
+      def report_slice(hosts_batch)
+        @stream.object do
+          @stream.simple_field('report_slice_id', @slice_id)
+          @stream.array_field('hosts', :last) do
+            first = true
+            hosts_batch.each do |host|
+              @stream.comma unless first
+              first = false
+              report_host(host)
+            end
+          end
+        end
+      end
+
+      def report_host(host)
+        @stream.object do
+          @stream.simple_field('display_name', host.name)
+          @stream.simple_field('fqdn', host.fqdn)
+          @stream.simple_field('account', host.subscription_facet.pools.first.account_number.to_s)
+          @stream.simple_field('subscription_manager_id', host.subscription_facet.uuid)
+          @stream.simple_field('satellite_id', host.subscription_facet.uuid)
+          @stream.simple_field('bios_uuid', fact_value(host, 'dmi::system::uuid'))
+          @stream.simple_field('vm_uuid', fact_value(host, 'virt::uuid'))
+          @stream.array_field('ip_addresses') do
+            @stream.raw(host.interfaces.map do |nic|
+              @stream.stringify_value(nic.ip) if nic.ip
+            end.compact.join(', '))
+          end
+          @stream.array_field('mac_addresses') do
+            @stream.raw(host.interfaces.map do |nic|
+              @stream.stringify_value(nic.mac) if nic.mac
+            end.compact.join(', '))
+          end
+          @stream.object_field('system_profile', :last) do
+            report_system_profile(host)
+          end
+        end
+      end
+
+      def report_system_profile(host)
+        @stream.simple_field('number_of_cpus', fact_value(host, 'cpu::cpu(s)').to_i)
+        @stream.simple_field('number_of_sockets', fact_value(host, 'cpu::cpu_socket(s)').to_i)
+        @stream.simple_field('cores_per_socket', fact_value(host, 'cpu::core(s)_per_socket').to_i)
+        @stream.simple_field('system_memory_bytes', fact_value(host, 'memory::memtotal').to_i)
+        @stream.array_field('network_interfaces') do
+          @stream.raw(host.interfaces.map do |nic|
+            {
+              'ipv4_addresses': [nic.ip].compact,
+              'ipv6_addresses': [nic.ip6].compact,
+              'mtu': nic.mtu,
+              'mac_address': nic.mac,
+              'name': nic.identifier
+            }.compact.to_json
+          end.join(', '))
+        end
+        @stream.simple_field('bios_vendor', fact_value(host, 'dmi::bios::vendor'))
+        @stream.simple_field('bios_version', fact_value(host, 'dmi::bios::version'))
+        @stream.simple_field('bios_release_date', fact_value(host, 'dmi::bios::relase_date'))
+        if (cpu_flags = fact_value(host, 'lscpu::flags'))
+          @stream.array_field('cpu_flags') do
+            @stream.raw(cpu_flags.split.map do |flag|
+              @stream.stringify_value(flag)
+            end.join(', '))
+          end
+        end
+        @stream.simple_field('os_release', fact_value(host, 'distribution::name'))
+        @stream.simple_field('os_kernel_version', fact_value(host, 'uname::release'))
+        @stream.simple_field('arch', host.architecture&.name)
+        @stream.simple_field('subscription_status', host.subscription_status_label)
+        @stream.simple_field('katello_agent_running', host.content_facet&.katello_agent_installed?)
+        @stream.simple_field('satellite_managed', true)
+        unless (installed_products = host.subscription_facet&.installed_products).empty?
+          @stream.array_field('installed_products') do
+            @stream.raw(installed_products.map do |product|
+              {
+                'name': product.name,
+                'id': product.cp_product_id
+              }.to_json
+            end.join(', '))
+          end
+        end
+        @stream.array_field('installed_packages', :last) do
+          first = true
+          host.installed_packages.each do |package|
+            @stream.raw("#{first ? '' : ', '}#{@stream.stringify_value(package.nvra)}")
+            first = false
+          end
+        end
+      end
+
+      def fact_value(host, fact_name)
+        value_record = host.fact_values.find do |fact_value|
+          fact_value.fact_name_id == ForemanInventoryUpload::Generators::Queries.fact_names[fact_name]
+        end
+        value_record&.value
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/version.rb

@@ -0,0 +1,3 @@
+module ForemanInventoryUpload
+  VERSION = '0.0.1.dev1'.freeze
+end

data/lib/foreman_inventory_upload.rb

@@ -0,0 +1,48 @@
+require 'foreman_inventory_upload/engine'
+
+module ForemanInventoryUpload
+  def self.base_folder
+    # in production setup, where selinux is enabled, we only have rights to
+    # create folders under /ver/lib/foreman. If the folder does not exist, it's
+    # a dev setup, where we can use our current root
+    @base_folder ||= File.join(
+      Dir.glob('/var/lib/foreman').first || Dir.getwd,
+      'red_hat_inventory/'
+    )
+  end
+
+  def self.uploads_folder(group)
+    @uploads_folders ||= {}
+    cache = @uploads_folders[group]
+    return cache if cache
+
+    @uploads_folders[group] = ensure_folder(
+      File.join(
+        ForemanInventoryUpload.base_folder,
+        'uploads/',
+        "#{group}/"
+      )
+    )
+  end
+
+  def self.outputs_folder
+    @outputs_folder ||= ensure_folder(File.join(ForemanInventoryUpload.base_folder, 'outputs/'))
+  end
+
+  def self.upload_script_file
+    'uploader.sh'
+  end
+
+  def self.facts_archive_name
+    'hosts_report.tar.gz'
+  end
+
+  def self.upload_url
+    'https://ci.cloud.paas.psi.redhat.com/api/ingress/v1/upload'
+  end
+
+  def self.ensure_folder(folder)
+    FileUtils.mkdir_p(folder)
+    folder
+  end
+end

data/lib/tasks/foreman_inventory_upload_tasks.rake

@@ -0,0 +1,37 @@
+require 'rake/testtask'
+
+# Tests
+namespace :test do
+  desc 'Test ForemanInventoryUpload'
+  Rake::TestTask.new(:foreman_inventory_upload) do |t|
+    test_dir = File.join(File.dirname(__FILE__), '../..', 'test')
+    t.libs << ['test', test_dir]
+    t.pattern = "#{test_dir}/**/*_test.rb"
+    t.verbose = true
+    t.warning = false
+  end
+end
+
+namespace :foreman_inventory_upload do
+  task :rubocop do
+    begin
+      require 'rubocop/rake_task'
+      RuboCop::RakeTask.new(:rubocop_foreman_inventory_upload) do |task|
+        task.patterns = ["#{ForemanInventoryUpload::Engine.root}/app/**/*.rb",
+                         "#{ForemanInventoryUpload::Engine.root}/lib/**/*.rb",
+                         "#{ForemanInventoryUpload::Engine.root}/test/**/*.rb"]
+      end
+    rescue
+      puts 'Rubocop not loaded.'
+    end
+
+    Rake::Task['rubocop_foreman_inventory_upload'].invoke
+  end
+end
+
+Rake::Task[:test].enhance ['test:foreman_inventory_upload']
+
+load 'tasks/jenkins.rake'
+if Rake::Task.task_defined?(:'jenkins:unit')
+  Rake::Task['jenkins:unit'].enhance ['test:foreman_inventory_upload', 'foreman_inventory_upload:rubocop']
+end

data/lib/tasks/generator.rake

@@ -0,0 +1,15 @@
+require 'tempfile'
+
+namespace :foreman_inventory_upload do
+  namespace :report do
+    desc 'Generate inventory report to be sent to Red Hat cloud'
+    task generate: :environment do
+      target = ENV['target'] || ForemanInventoryUpload.facts_archive_name
+      portal_user = ENV['portal_user'] || 'anonymous'
+
+      archived_report_generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
+      archived_report_generator.render(portal_user)
+      puts "Successfully generated #{target} for #{portal_user}"
+    end
+  end
+end

data/locale/Makefile

@@ -0,0 +1,60 @@
+#
+# Makefile for PO merging and MO generation. More info in the README.
+#
+# make all-mo (default) - generate MO files
+# make check - check translations using translate-tool
+# make tx-update - download and merge translations from Transifex
+# make clean - clean everything
+#
+DOMAIN = foreman_yupana
+VERSION = $(shell ruby -e 'require "rubygems";spec = Gem::Specification::load(Dir.glob("../*.gemspec")[0]);puts spec.version')
+POTFILE = $(DOMAIN).pot
+MOFILE = $(DOMAIN).mo
+POFILES = $(shell find . -name '$(DOMAIN).po')
+MOFILES = $(patsubst %.po,%.mo,$(POFILES))
+POXFILES = $(patsubst %.po,%.pox,$(POFILES))
+EDITFILES = $(patsubst %.po,%.edit.po,$(POFILES))
+
+%.mo: %.po
+	mkdir -p $(shell dirname $@)/LC_MESSAGES
+	msgfmt -o $(shell dirname $@)/LC_MESSAGES/$(MOFILE) $<
+
+# Generate MO files from PO files
+all-mo: $(MOFILES)
+
+# Check for malformed strings
+%.pox: %.po
+	msgfmt -c $<
+	pofilter --nofuzzy -t variables -t blank -t urls -t emails -t long -t newlines \
+		-t endwhitespace -t endpunc -t puncspacing -t options -t printf -t validchars --gnome $< > $@
+	cat $@
+	! grep -q msgid $@
+
+%.edit.po:
+	touch $@
+
+check: $(POXFILES)
+
+# Unify duplicate translations
+uniq-po:
+	for f in $(shell find ./ -name "*.po") ; do \
+		msguniq $$f -o $$f ; \
+	done
+
+tx-pull: $(EDITFILES)
+	tx pull -f
+	for f in $(EDITFILES) ; do \
+		sed -i 's/^\("Project-Id-Version: \).*$$/\1$(DOMAIN) $(VERSION)\\n"/' $$f; \
+	done
+
+tx-update: tx-pull
+	@echo
+	@echo Run rake plugin:gettext[$(DOMAIN)] from the Foreman installation, then make -C locale mo-files to finish
+	@echo
+
+mo-files: $(MOFILES)
+	git add $(POFILES) $(POTFILE) ../locale/*/LC_MESSAGES
+	git commit -m "i18n - pulling from tx"
+	@echo
+	@echo Changes commited!
+	@echo

data/locale/en/foreman_yupana.po

@@ -0,0 +1,19 @@
+# foreman_yupana
+#
+# This file is distributed under the same license as foreman_yupana.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: version 0.0.1\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2014-08-20 08:46+0100\n"
+"PO-Revision-Date: 2014-08-20 08:54+0100\n"
+"Last-Translator: Foreman Team <foreman-dev@googlegroups.com>\n"
+"Language-Team: Foreman Team <foreman-dev@googlegroups.com>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+

data/locale/foreman_yupana.pot

@@ -0,0 +1,19 @@
+# foreman_yupana
+#
+# This file is distributed under the same license as foreman_yupana.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: version 0.0.1\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2014-08-20 08:46+0100\n"
+"PO-Revision-Date: 2014-08-20 08:46+0100\n"
+"Last-Translator: Foreman Team <foreman-dev@googlegroups.com>\n"
+"Language-Team: Foreman Team <foreman-dev@googlegroups.com>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n"
+

data/locale/gemspec.rb

@@ -0,0 +1,2 @@
+# Matches foreman_yupana.gemspec
+_('TODO: Description of ForemanYupana.')

data/test/controllers/reports_controller_test.rb

@@ -0,0 +1,21 @@
+require 'test_plugin_helper'
+
+class ReportsControllerTest < ActionController::TestCase
+  tests ForemanInventoryUpload::ReportsController
+
+  test 'Returns latest report generation status' do
+    progress_output = mock('progress_output')
+    test_portal_user = 'test_portal_user'
+    ForemanInventoryUpload::Async::ProgressOutput
+      .expects(:get)
+      .with(ForemanInventoryUpload::Async::GenerateReportJob.output_label(test_portal_user))
+      .returns(progress_output)
+    progress_output.expects(:full_output).returns('test output')
+
+    get :last, params: { portal_user: test_portal_user }, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal 'test output', actual['output']
+  end
+end

data/test/controllers/statuses_controller_test.rb

@@ -0,0 +1,26 @@
+require 'test_plugin_helper'
+
+class StatusesControllerTest < ActionController::TestCase
+  tests ForemanInventoryUpload::StatusesController
+
+  include FolderIsolation
+
+  test 'Returns statuses for each process type' do
+    configuration = RedhatAccess::TelemetryConfiguration.new(enable_telemetry: true, portal_user: 'test')
+    configuration.save!
+
+    generate_label = ForemanInventoryUpload::Async::GenerateReportJob.output_label('test')
+    generate_output = ForemanInventoryUpload::Async::ProgressOutput.register(generate_label)
+    generate_output.status = 'generate_status_test'
+    upload_label = ForemanInventoryUpload::Async::UploadReportJob.output_label('test')
+    upload_output = ForemanInventoryUpload::Async::ProgressOutput.register(upload_label)
+    upload_output.status = 'upload_status_test'
+
+    get :index, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)['statuses']['test']
+    assert_equal 'generate_status_test', actual['generate_report_status']
+    assert_equal 'upload_status_test', actual['upload_report_status']
+  end
+end

data/test/controllers/uploads_controller_test.rb

@@ -0,0 +1,21 @@
+require 'test_plugin_helper'
+
+class UploadsControllerTest < ActionController::TestCase
+  tests ForemanInventoryUpload::UploadsController
+
+  test 'Returns latest upload status' do
+    progress_output = mock('progress_output')
+    test_portal_user = 'test_portal_user'
+    ForemanInventoryUpload::Async::ProgressOutput
+      .expects(:get)
+      .with(ForemanInventoryUpload::Async::UploadReportJob.output_label(test_portal_user))
+      .returns(progress_output)
+    progress_output.expects(:full_output).returns('test output')
+
+    get :last, params: { portal_user: test_portal_user }, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal 'test output', actual['output']
+  end
+end

data/test/factories/inventory_upload_factories.rb

@@ -0,0 +1,88 @@
+# redefine katello factories, as long as katello is not compatible with dynamic properties
+FactoryBot.define do
+  factory :katello_organization, :class => "Organization" do
+    type {"Organization"}
+    sequence(:name) { |n| "Organization#{n}" }
+    sequence(:label) { |n| "org#{n}" }
+    sequence(:id) { |n| n }
+
+    trait :acme_corporation do
+      name {"ACME_Corporation"}
+      type {"Organization"}
+      description {"This is the first Organization."}
+      label {"acme_corporation_label"}
+    end
+
+    trait :with_library do
+      association :library, :factory => :katello_library
+    end
+
+    factory :acme_corporation, :traits => [:acme_corporation]
+  end
+end
+
+FactoryBot.define do
+  factory :katello_content_view, :class => Katello::ContentView do
+    sequence(:name) { |n| "Database#{n}" }
+    description {"This content view is for database content"}
+    association :organization, :factory => :katello_organization
+
+    trait :composite do
+      composite {true}
+    end
+  end
+end
+
+FactoryBot.define do
+  factory :katello_k_t_environment, :aliases => [:katello_environment], :class => Katello::KTEnvironment do
+    sequence(:name) { |n| "Environment#{n}" }
+    sequence(:label) { |n| "environment#{n}" }
+    association :organization, :factory => :katello_organization
+  end
+end
+
+FactoryBot.define do
+  factory :katello_content_facets, :aliases => [:content_facet], :class => ::Katello::Host::ContentFacet do
+    sequence(:uuid) { |n| "uuid-#{n}" }
+  end
+end
+
+FactoryBot.define do
+  factory :katello_subscription_facets, :aliases => [:subscription_facet], :class => ::Katello::Host::SubscriptionFacet do
+    sequence(:uuid) { |n| "uuid-#{n}-#{rand(500)}" }
+    facts { {'memory.memtotal' => "12 GB"} }
+  end
+end
+
+FactoryBot.define do
+  factory :katello_pool, :class => Katello::Pool do
+    active {true}
+    end_date {Date.today + 1.year}
+  end
+end
+
+FactoryBot.modify do
+  factory :host do
+    transient do
+      content_view {nil}
+      lifecycle_environment {nil}
+      content_source {nil}
+    end
+
+    trait :with_content do
+      association :content_facet, :factory => :content_facet, :strategy => :build
+
+      after(:build) do |host, evaluator|
+        if host.content_facet
+          host.content_facet.content_view = evaluator.content_view if evaluator.content_view
+          host.content_facet.lifecycle_environment = evaluator.lifecycle_environment if evaluator.lifecycle_environment
+          host.content_facet.content_source = evaluator.content_source if evaluator.content_source
+        end
+      end
+    end
+
+    trait :with_subscription do
+      association :subscription_facet, :factory => :subscription_facet, :strategy => :build
+    end
+  end
+end

data/test/test_plugin_helper.rb

@@ -0,0 +1,26 @@
+# This calls the main test_helper in Foreman-core
+require 'test_helper'
+
+# Add plugin to FactoryBot's paths
+FactoryBot.definition_file_paths << File.join(File.dirname(__FILE__), 'factories')
+# FactoryBot.definition_file_paths << "#{Katello::Engine.root}/test/factories"
+FactoryBot.reload
+
+module FolderIsolation
+  extend ActiveSupport::Concern
+
+  included do
+    setup do
+      @tmpdir = Dir.mktmpdir(self.class.name.underscore)
+
+      ForemanInventoryUpload.stubs(:base_folder).returns(@tmpdir)
+      ForemanInventoryUpload.instance_variable_set(:@outputs_folder, nil)
+      ForemanInventoryUpload.instance_variable_set(:@uploads_folders, nil)
+    end
+
+    teardown do
+      FileUtils.remove_entry @tmpdir
+      ForemanInventoryUpload.unstub(:base_folder)
+    end
+  end
+end

data/test/unit/archived_report_generator_test.rb

@@ -0,0 +1,61 @@
+require 'test_plugin_helper'
+
+class ArchivedReportGeneratorTest < ActiveSupport::TestCase
+  setup do
+    User.current = User.find_by(login: 'secret_admin')
+
+    env = FactoryBot.create(:katello_k_t_environment)
+    cv = env.content_views << FactoryBot.create(:katello_content_view, organization: env.organization)
+
+    @host = FactoryBot.create(
+      :host,
+      :with_subscription,
+      :with_content,
+      content_view: cv.first,
+      lifecycle_environment: env,
+      organization: env.organization
+    )
+
+    @host.subscription_facet.pools << FactoryBot.create(:katello_pool, account_number: '1234', cp_id: 1)
+  end
+
+  def interesting_facts
+    [
+      'dmi::system::uuid',
+      'virt::uuid',
+      'cpu::cpu(s)',
+      'cpu::cpu_socket(s)',
+      'cpu::core(s)_per_socket',
+      'memory::memtotal',
+      'dmi::bios::vendor',
+      'dmi::bios::version',
+      'dmi::bios::relase_date',
+      'distribution::name',
+      'uname::release',
+      'lscpu::flags'
+    ]
+  end
+
+  def fact_names
+    @fact_names ||= Hash[
+      interesting_facts.map do |fact|
+        [fact, FactoryBot.create(:fact_name, name: fact, type: 'Katello::RhsmFactName')]
+      end
+    ]
+  end
+
+  test 'generates a report for a single host' do
+    batches = Host.where(id: @host.id).in_batches
+    portal_user = 'test_portal_user'
+
+    ForemanInventoryUpload::Generators::Queries.expects(:for_report).with(portal_user).returns(batches)
+    Dir.mktmpdir do |tmpdir|
+      target = File.join(tmpdir, 'test.tar.gz')
+      generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
+      generator.render(portal_user)
+
+      files = Dir["#{tmpdir}/*"]
+      assert_equal "#{tmpdir}/test.tar.gz", files.first
+    end
+  end
+end

data/test/unit/metadata_generator_test.rb

@@ -0,0 +1,40 @@
+require 'test_plugin_helper'
+
+class MetadataGeneratorTest < ActiveSupport::TestCase
+  setup do
+  end
+
+  test 'generates an empty report' do
+    generator = ForemanInventoryUpload::Generators::Metadata.new
+
+    json_str = generator.render do
+    end
+    actual = JSON.parse(json_str.join("\n"))
+
+    assert_not_nil actual['report_id']
+    assert_equal 'Satellite', actual['source']
+    assert_equal({}, actual['report_slices'])
+  end
+
+  test 'generates a report for a single slice' do
+    generator = ForemanInventoryUpload::Generators::Metadata.new
+
+    json_str = generator.render do |gen|
+      gen.add_slice('test_123', 1, true)
+      gen.add_slice('test_1234', 2, false)
+      gen.add_slice('test_12345', 3, false)
+    end
+
+    actual = JSON.parse(json_str.join("\n"))
+
+    assert_not_nil actual['report_id']
+    assert_equal 'Satellite', actual['source']
+    assert_not_nil(slices = actual['report_slices'])
+    assert_not_nil(slice = slices['test_123'])
+    assert_equal 1, slice['number_hosts']
+    assert_not_nil(slice = slices['test_1234'])
+    assert_equal 2, slice['number_hosts']
+    assert_not_nil(slice = slices['test_12345'])
+    assert_equal 3, slice['number_hosts']
+  end
+end