foreman_inventory_upload 0.0.1.dev1

data/Rakefile

@@ -0,0 +1,47 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ForemanInventoryUpload'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path('../test/dummy/Rakefile', __FILE__)
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test
+
+begin
+  require 'rubocop/rake_task'
+  RuboCop::RakeTask.new
+rescue => _
+  puts 'Rubocop not loaded.'
+end
+
+task :default do
+  Rake::Task['rubocop'].execute
+end

data/app/controllers/foreman_inventory_upload/react_controller.rb

@@ -0,0 +1,7 @@
+module ForemanInventoryUpload
+  class ReactController < ::ApplicationController
+    def index
+      render 'foreman_inventory_upload/layouts/react', :layout => false
+    end
+  end
+end

data/app/controllers/foreman_inventory_upload/reports_controller.rb

@@ -0,0 +1,23 @@
+module ForemanInventoryUpload
+  class ReportsController < ::ApplicationController
+    def last
+      label = ForemanInventoryUpload::Async::GenerateReportJob.output_label(params[:portal_user])
+      output = ForemanInventoryUpload::Async::ProgressOutput.get(label)&.full_output
+
+      render json: {
+        output: output
+      }, status: :ok
+    end
+
+    def generate
+      portal_user = params[:portal_user]
+
+      generated_file_name = File.join(ForemanInventoryUpload.base_folder, "#{portal_user}.tar.gz")
+      ForemanInventoryUpload::Async::GenerateReportJob.perform_later(generated_file_name, portal_user)
+
+      render json: {
+        action_status: 'success'
+      }, status: :ok
+    end
+  end
+end

data/app/controllers/foreman_inventory_upload/statuses_controller.rb

@@ -0,0 +1,41 @@
+module ForemanInventoryUpload
+  class StatusesController < ::ApplicationController
+    # override default "welcome screen behavior, since we don't have a model"
+    def welcome
+      true
+    end
+
+    def index
+      portal_users = RedhatAccess::TelemetryConfiguration
+                     .where(enable_telemetry: true)
+                     .distinct
+                     .pluck(:portal_user)
+
+      statuses = Hash[
+        portal_users.map do |portal_user|
+          generate_report_status = status_for(portal_user, ForemanInventoryUpload::Async::GenerateReportJob)
+          upload_report_status = status_for(portal_user, ForemanInventoryUpload::Async::UploadReportJob)
+
+          [
+            portal_user,
+            {
+              generate_report_status: generate_report_status,
+              upload_report_status: upload_report_status
+            }
+          ]
+        end
+      ]
+
+      render json: {
+        statuses: statuses
+      }, status: :ok
+    end
+
+    private
+
+    def status_for(portal_user, job_class)
+      label = job_class.output_label(portal_user)
+      ForemanInventoryUpload::Async::ProgressOutput.get(label)&.status
+    end
+  end
+end

data/app/controllers/foreman_inventory_upload/uploads_controller.rb

@@ -0,0 +1,31 @@
+module ForemanInventoryUpload
+  class UploadsController < ::ApplicationController
+    def last
+      label = ForemanInventoryUpload::Async::UploadReportJob.output_label(params[:portal_user])
+      output = ForemanInventoryUpload::Async::ProgressOutput.get(label)&.full_output
+
+      render json: {
+        output: output
+      }, status: :ok
+    end
+
+    def download_file
+      filename = 'hosts_report.tar.gz'
+      path = Rails.root.join(ForemanInventoryUpload.uploads_folder(params[:portal_user]), filename)
+      unless File.exist? path
+        return throw_flash_error(
+          "Path doesn't exist: #{path}"
+        )
+      end
+
+      send_file path, disposition: 'attachment', filename: filename
+    end
+
+    def throw_flash_error(message)
+      process_error(
+        :redirect => foreman_inventory_upload_index_path,
+        :error_msg => message
+      )
+    end
+  end
+end

data/app/helpers/foreman_inventory_upload_helper.rb

@@ -0,0 +1,13 @@
+module ForemanInventoryUploadHelper
+  if Foreman::Version.new.minor.to_i < 23
+    def webpacked_plugins_css_for(*plugin_names)
+      css_tags_for(select_requested_plugins(plugin_names)).join.html_safe
+    end
+
+    def css_tags_for(requested_plugins)
+      requested_plugins.map do |plugin|
+        stylesheet_link_tag(*webpack_asset_paths(plugin.to_s, :extension => 'css'), "data-turbolinks-track" => true)
+      end
+    end
+  end
+end

data/app/views/dashboard/_inventory_upload_widget.html.erb

@@ -0,0 +1,2 @@
+<h4 class="header ca"><%= _('ForemanInventoryUpload') %></h4>
+<%= _('Widget content') %>

data/app/views/foreman_inventory_upload/layouts/react.html.erb

@@ -0,0 +1,16 @@
+<% content_for(:javascripts) do %>
+  <%= webpacked_plugins_js_for :'foreman_inventory_upload' %>
+<% end %>
+
+<% content_for(:stylesheets) do %>
+    <%= webpacked_plugins_css_for :'foreman_inventory_upload' %>
+<% end %>
+
+<% content_for(:content) do %>
+  <%= notifications %>
+  <div id="organization-id" data-id="<%= Organization.current.id if Organization.current %>" ></div>
+  <div id="user-id" data-id="<%= User.current.id if User.current %>" ></div>
+  <div id="ForemanInventoryUploadReactRoot"></div>
+<% end %>
+<%= render file: "layouts/base" %>
+<%= mount_react_component('ForemanInventoryUpload', '#ForemanInventoryUploadReactRoot') %>

data/app/views/scripts/uploader.sh.erb

@@ -0,0 +1,31 @@
+#! /bin/sh
+
+DEST=<%= @upload_url %>
+RH_USERNAME=<%= @rh_username %>
+
+if [ -z "$RH_USERNAME" ]
+then
+        IFS= read -rp "Enter username: " RH_USERNAME
+fi
+
+if [ -z "$RH_PASSWORD" ]
+then
+        IFS= read -rsp "Enter password: " RH_PASSWORD
+fi
+
+if [ -z "$FILES" ]
+then
+        FILES=./*.tar.gz
+fi
+
+mkdir -p done
+
+for f in $FILES
+do
+  curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST -u "$RH_USERNAME":"$RH_PASSWORD"
+  if [ $? -eq 0 ]; then
+    mv $f done/
+    echo "Done: $f"
+  fi
+done
+echo "Uploaded files moved to done/ folder"

data/config/routes.rb

@@ -0,0 +1,10 @@
+Rails.application.routes.draw do
+  namespace :foreman_inventory_upload do
+    get 'index', to: 'react#index'
+    get ':portal_user/reports/last', to: 'reports#last', constraints: { portal_user: %r{[^\/]+} }
+    post ':portal_user/reports', to: 'reports#generate', constraints: { portal_user: %r{[^\/]+} }
+    get ':portal_user/uploads/last', to: 'uploads#last', constraints: { portal_user: %r{[^\/]+} }
+    get ':portal_user/uploads/file', to: 'uploads#download_file', constraints: { portal_user: %r{[^\/]+} }
+    get 'statuses', to: 'statuses#index'
+  end
+end

data/lib/foreman_inventory_upload/async/generate_report_job.rb

@@ -0,0 +1,29 @@
+module ForemanInventoryUpload
+  module Async
+    class GenerateReportJob < ShellProcess
+      def self.output_label(portal_user)
+        "report_for_#{portal_user}"
+      end
+
+      def perform(result_file, portal_user)
+        @result_file = result_file
+        @portal_user = portal_user
+
+        super(GenerateReportJob.output_label(portal_user))
+
+        QueueForUploadJob.perform_later(result_file, portal_user)
+      end
+
+      def command
+        'rake foreman_inventory_upload:report:generate'
+      end
+
+      def env
+        super.merge(
+          'target' => @result_file,
+          'portal_user' => @portal_user
+        )
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/progress_output.rb

@@ -0,0 +1,61 @@
+module ForemanInventoryUpload
+  module Async
+    class ProgressOutput
+      def self.get(label)
+        ProgressOutput.new(label, :reader)
+      end
+
+      def self.register(label)
+        ProgressOutput.new(label, :writer)
+      end
+
+      def initialize(label, mode)
+        @label = label
+        @mode = mode
+      end
+
+      def buffer
+        @buffer ||= begin
+                      File.open(file_name, file_mode)
+                    rescue Errno::ENOENT
+                      StringIO.new
+                    end
+      end
+
+      def full_output
+        buffer.read
+      end
+
+      def write_line(line)
+        buffer << line
+        buffer.fsync
+      end
+
+      def close
+        @buffer&.close
+      end
+
+      def status
+        File.read(file_name(:status))
+      rescue Errno::ENOENT
+        ''
+      end
+
+      def status=(status)
+        File.atomic_write(file_name(:status)) do |status_file|
+          status_file.write(status)
+        end
+      end
+
+      private
+
+      def file_mode
+        @mode == :reader ? 'r' : 'w'
+      end
+
+      def file_name(type = 'out')
+        File.join(ForemanInventoryUpload.outputs_folder, "#{@label}.#{type}")
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/queue_for_upload_job.rb

@@ -0,0 +1,52 @@
+module ForemanInventoryUpload
+  module Async
+    class QueueForUploadJob < ::ApplicationJob
+      def perform(report_file, portal_user)
+        @portal_user = portal_user
+        logger.debug('Ensuring objects')
+        ensure_ouput_folder
+        ensure_output_script
+        logger.debug("Copying #{report_file} to #{uploads_folder}")
+        enqueued_file_name = File.join(uploads_folder, ForemanInventoryUpload.facts_archive_name)
+        FileUtils.mv(report_file, enqueued_file_name)
+        logger.debug("Done copying #{report_file} to #{enqueued_file_name}")
+
+        UploadReportJob.perform_later(enqueued_file_name, portal_user)
+      end
+
+      def uploads_folder
+        @uploads_folder ||= ForemanInventoryUpload.uploads_folder(@portal_user)
+      end
+
+      def script_file
+        @script_file ||= File.join(uploads_folder, ForemanInventoryUpload.upload_script_file)
+      end
+
+      def ensure_ouput_folder
+        FileUtils.mkdir_p(uploads_folder)
+      end
+
+      def ensure_output_script
+        return if File.exist?(script_file)
+
+        script_source = File.join(ForemanInventoryUpload::Engine.root, 'app/views/scripts/uploader.sh.erb')
+
+        template_src = Foreman::Renderer::Source::String.new(content: File.read(script_source))
+        scope = Foreman::Renderer::Scope::Base.new(
+          source: template_src,
+          variables: {
+            upload_url: ForemanInventoryUpload.upload_url,
+            rh_username: @portal_user
+          }
+        )
+        script_source = Foreman::Renderer.render(template_src, scope)
+        File.write(script_file, script_source)
+        FileUtils.chmod('+x', script_file)
+      end
+
+      def logger
+        Foreman::Logging.logger('background')
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/shell_process.rb

@@ -0,0 +1,35 @@
+require 'open3'
+
+module ForemanInventoryUpload
+  module Async
+    class ShellProcess < ::ApplicationJob
+      def perform(instance_label)
+        klass_name = self.class.name
+        logger.debug("Starting #{klass_name} with label #{instance_label}")
+        progress_output = ProgressOutput.register(instance_label)
+        Open3.popen2e(env, command) do |_stdin, stdout_stderr, wait_thread|
+          progress_output.status = "Running in pid #{wait_thread.pid}"
+
+          stdout_stderr.each do |out_line|
+            progress_output.write_line(out_line)
+          end
+
+          progress_output.status = wait_thread.value.to_s
+        end
+        logger.debug("Finished job #{klass_name} with label #{instance_label}")
+      ensure
+        progress_output.close
+      end
+
+      def command; end
+
+      def env
+        {}
+      end
+
+      def logger
+        Foreman::Logging.logger('background')
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/upload_report_job.rb

@@ -0,0 +1,40 @@
+module ForemanInventoryUpload
+  module Async
+    class UploadReportJob < ShellProcess
+      def self.output_label(portal_user)
+        "upload_for_#{portal_user}"
+      end
+
+      def perform(filename, portal_user)
+        @portal_user = portal_user
+        @filename = filename
+
+        super(UploadReportJob.output_label(portal_user))
+      end
+
+      def command
+        File.join(File.dirname(@filename), ForemanInventoryUpload.upload_script_file)
+      end
+
+      def env
+        super.merge(
+          'RH_USERNAME' => rh_username,
+          'RH_PASSWORD' => rh_password,
+          'FILES' => @filename
+        )
+      end
+
+      def rh_credentials
+        @rh_credentials ||= RedhatAccess::TelemetryConfiguration.where(portal_user: @portal_user).last
+      end
+
+      def rh_username
+        @portal_user
+      end
+
+      def rh_password
+        rh_credentials.portal_password
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/engine.rb

@@ -0,0 +1,52 @@
+require 'katello'
+require 'redhat_access'
+
+module ForemanInventoryUpload
+  class Engine < ::Rails::Engine
+    engine_name 'foreman_inventory_upload'
+
+    config.autoload_paths += Dir["#{config.root}/app/controllers/concerns"]
+    config.autoload_paths += Dir["#{config.root}/app/helpers/concerns"]
+    config.autoload_paths += Dir["#{config.root}/app/models/concerns"]
+    config.autoload_paths += Dir["#{config.root}/app/overrides"]
+    config.autoload_paths += Dir["#{config.root}/lib"]
+
+    # Add any db migrations
+    initializer 'foreman_inventory_upload.load_app_instance_data' do |app|
+      ForemanInventoryUpload::Engine.paths['db/migrate'].existent.each do |path|
+        app.config.paths['db/migrate'] << path
+      end
+    end
+
+    initializer 'foreman_inventory_upload.register_plugin', :before => :finisher_hook do |_app|
+      Foreman::Plugin.register :foreman_inventory_upload do
+        requires_foreman '>= 1.20'
+
+        # Add permissions
+        security_block :foreman_inventory_upload do
+          permission :view_foreman_inventory_upload, :'foreman_inventory_upload/reports' => [:last]
+        end
+
+        # Add a new role called 'Discovery' if it doesn't exist
+        role 'ForemanInventoryUpload', [:view_foreman_inventory_upload]
+
+        # Adding a sub menu after hosts menu
+        sub_menu :top_menu, :foreman_inventory_upload, :caption => N_('RH Inventory'), :icon => 'fa fa-cloud-upload' do
+          menu :top_menu, :level1, :caption => N_('Manage'), :url_hash => { controller: :'foreman_inventory_upload/react', :action=>:index}
+        end
+      end
+    end
+
+    rake_tasks do
+      Rake::Task['db:seed'].enhance do
+        ForemanInventoryUpload::Engine.load_seed
+      end
+    end
+
+    initializer 'foreman_inventory_upload.register_gettext', after: :load_config_initializers do |_app|
+      locale_dir = File.join(File.expand_path('../..', __dir__), 'locale')
+      locale_domain = 'foreman_inventory_upload'
+      Foreman::Gettext::Support.add_text_domain locale_domain, locale_dir
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/archived_report.rb

@@ -0,0 +1,53 @@
+module ForemanInventoryUpload
+  module Generators
+    class ArchivedReport
+      def initialize(target, logger = Rails.logger)
+        @target = target
+        @logger = logger
+      end
+
+      def render(portal_user)
+        Dir.mktmpdir do |tmpdir|
+          @logger.info "Started generating hosts report in #{tmpdir}"
+          host_batches = ForemanInventoryUpload::Generators::Queries.for_report(portal_user)
+          File.open(File.join(tmpdir, 'metadata.json'), 'w') do |metadata_out|
+            metadata_generator = ForemanInventoryUpload::Generators::Metadata.new(metadata_out)
+            metadata_generator.render do |inner_generator|
+              first = true
+              host_batches.each do |hosts_batch|
+                slice_id = Foreman.uuid
+                hosts_count = hosts_batch.count
+                @logger.info "Adding slice #{slice_id} with #{hosts_count} hosts"
+                generate_slice(tmpdir, slice_id, hosts_batch)
+                inner_generator.add_slice(slice_id, hosts_count, first)
+                first = false
+              end
+            end
+          end
+          @logger.info 'Report generation finished'
+
+          @logger.info 'Archiving generated report'
+          # success = system('tar', '-zcvf', @target, '-C', tmpdir, '.')
+          Open3.popen2e('tar', '-zcvf', @target, '-C', tmpdir, '.') do |_in, out, wait_thr|
+            @logger.info("tar: #{out.read}")
+
+            if wait_thr.value.success?
+              @logger.info 'Report archived successfully'
+            else
+              @logger.info "Tar command failed: #{$CHILD_STATUS}"
+            end
+          end
+        end
+      end
+
+      private
+
+      def generate_slice(tmpdir, slice_id, hosts_batch)
+        File.open(File.join(tmpdir, "#{slice_id}.json"), 'w') do |slice_out|
+          slice_generator = ForemanInventoryUpload::Generators::Slice.new(hosts_batch, slice_out, slice_id)
+          slice_generator.render
+        end
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/json_stream.rb

@@ -0,0 +1,55 @@
+module ForemanInventoryUpload
+  module Generators
+    class JsonStream
+      attr_reader :out
+      def initialize(out)
+        @out = out
+      end
+
+      def array
+        @out << '['
+        yield
+        @out << ']'
+      end
+
+      def object
+        @out << '{'
+        yield
+        @out << '}'
+      end
+
+      def comma
+        @out << ', '
+      end
+
+      def raw(string)
+        @out << string
+      end
+
+      def simple_field(name, value, last = false)
+        @out << "\"#{name}\": #{stringify_value(value)}#{last ? '' : ','}" unless value.nil?
+      end
+
+      def array_field(name, last = false, &block)
+        @out << "\"#{name}\": "
+        array(&block)
+        @out << ',' unless last
+      end
+
+      def object_field(name, last = false, &block)
+        @out << "\"#{name}\": "
+        object(&block)
+        @out << ',' unless last
+      end
+
+      def stringify_value(value)
+        return value if value.is_a?(Integer)
+        return value if value.is_a?(TrueClass)
+        return value if value.is_a?(FalseClass)
+        return value.to_json if value.is_a?(Hash)
+
+        "\"#{value}\""
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/metadata.rb

@@ -0,0 +1,36 @@
+module ForemanInventoryUpload
+  module Generators
+    class Metadata
+      def initialize(output = [])
+        @stream = JsonStream.new(output)
+      end
+
+      def add_slice(slice_id, hosts_count, first)
+        @stream.comma unless first
+
+        @stream.object_field(slice_id, :last) do
+          @stream.simple_field('number_hosts', hosts_count, :last)
+        end
+      end
+
+      def render(metadata = nil, &block)
+        render_report(metadata, &block)
+        @stream.out
+      end
+
+      private
+
+      def render_report(metadata)
+        @stream.object do
+          @stream.simple_field('report_id', Foreman.uuid)
+          @stream.simple_field('host_inventory_api_version', '1.0')
+          @stream.simple_field('source', 'Satellite')
+          @stream.simple_field('source_metadata', metadata)
+          @stream.object_field('report_slices', :last) do
+            yield(self)
+          end
+        end
+      end
+    end
+  end
+end