foreman_acd 0.4.0 → 0.9.0

data/app/models/concerns/foreman_acd/host_managed_extensions.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ForemanAcd
+  # Extends the Host Managed
+  module HostManagedExtensions
+    extend ActiveSupport::Concern
+
+    def self.prepended(base)
+      base.instance_eval do
+        before_provision :initiate_acd_app_configurator_after_host_deployment, :if => :deployed_via_acd?
+        before_destroy :check_deletable, :prepend => true, :if => :deployed_via_acd?
+      end
+    end
+
+    def deployed_via_acd?
+      find_foreman_host
+      @foreman_host.present?
+    end
+
+    private
+
+    def check_deletable
+      return if @foreman_host.blank?
+      ::Foreman::Logging.logger('foreman_acd').warn "Could not delete host '#{name}' because it is used in Applications > App Instances '#{@foreman_host.app_instance.name}'"
+      raise _("Could not delete host '%{host_name}' because it is used in Applications > App Instances '%{app_instance_name}'") % {
+        :host_name => name, :app_instance_name => @foreman_host.app_instance.name
+      }
+    end
+
+    def find_foreman_host
+      @foreman_host = ForemanAcd::ForemanHost.find_by(:host_id => id)
+    end
+
+    def initiate_acd_app_configurator_after_host_deployment
+      return if @foreman_host.blank?
+      ForemanAcd.initiate_acd_app_configurator(@foreman_host.app_instance)
+    end
+  end
+end

data/app/models/foreman_acd/acd_provider.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module ForemanAcd
+  # Implement a RemoteExecutionProvider
+  class AcdProvider < ::RemoteExecutionProvider
+    class << self
+      def supports_effective_user?
+        true
+      end
+
+      def proxy_operation_name
+        'acd'
+      end
+
+      def humanized_name
+        'ACD'
+      end
+
+      def proxy_command_options(template_invocation, host)
+        super(template_invocation, host).merge(:name => host.name)
+      end
+
+      def ssh_password(_host); end
+
+      def ssh_key_passphrase(_host); end
+
+      def sudo_password(_host); end
+
+      # Workaround till infrastructure jobs on proxies are possible. See
+      # configure in services/foreman_acd/app_configurator.rb for more details.
+      # def required_proxy_selector_for(_template)
+      #   AcdProxyProxySelector.new
+      # end
+    end
+  end
+end

data/app/models/foreman_acd/ansible_playbook.rb

@@ -4,14 +4,36 @@ module ForemanAcd
   # Ansible playbook
   class AnsiblePlaybook < ApplicationRecord
     include Authorizable
+    include Taxonomix
     extend FriendlyId
     friendly_id :name
+    include Parameterizable::ByIdName
 
     self.table_name = 'acd_ansible_playbooks'
-    has_many :app_definitions, :inverse_of => :ansible_playbook, :foreign_key => 'acd_ansible_playbook_id', dependent: :restrict_with_error
+    has_many :app_definitions, :inverse_of => :ansible_playbook, :foreign_key => 'acd_ansible_playbook_id', :dependent => :restrict_with_error
     validates :name, :presence => true, :uniqueness => true
+    validates :scm_type, :presence => true
     scoped_search :on => :name
-    default_scope -> { order("acd_ansible_playbooks.name") }
+
+    default_scope do
+      with_taxonomy_scope do
+        order('acd_ansible_playbooks.name')
+      end
+    end
+
+    def used_location_ids
+      Location.joins(:taxable_taxonomies).where(
+        'taxable_taxonomies.taxable_type' => 'ForemanAcd::AnsiblePlaybook',
+        'taxable_taxonomies.taxable_id' => id
+      ).pluck("#{Taxonomy.table_name}.id")
+    end
+
+    def used_organization_ids
+      Organization.joins(:taxable_taxonomies).where(
+        'taxable_taxonomies.taxable_type' => 'ForemanAcd::AnsiblePlaybook',
+        'taxable_taxonomies.taxable_id' => id
+      ).pluck("#{Taxonomy.table_name}.id")
+    end
 
     def self.humanize_class_name(_name = nil)
       _('Ansible playbook')
@@ -23,7 +45,7 @@ module ForemanAcd
 
     def content
       case scm_type
-      when 'directory'
+      when 'directory' || 'git'
         File.read(File.join(path, playfile))
       else
         raise NotImplementedError.new "scm_type #{scm_type.inspect} not supported!"
@@ -31,20 +53,16 @@ module ForemanAcd
     end
 
     def as_unified_structobj
-      groups = []
-
-      JSON.load(self.vars).each do |group_name, vars|
-        groups << OpenStruct.new(
-          :name => group_name,
-          :vars => vars.map { |k,v| OpenStruct.new(:name => k, :value => v) })
+      # FIXME: For now, we convert all values to string - even booleans and dicts
+      pretty_groups = JSON.parse(vars).each do |_, params|
+        params.transform_values!(&:to_s)
       end
 
-      adata = OpenStruct.new(
-        :id => self.id,
-        :name => self.name,
-        :groups => JSON.load(self.vars)
+      OpenStruct.new(
+        :id => id,
+        :name => name,
+        :groups => pretty_groups
       )
-      adata
     end
   end
 end

data/app/models/foreman_acd/app_definition.rb

@@ -4,14 +4,37 @@ module ForemanAcd
   # Application Definition
   class AppDefinition < ApplicationRecord
     include Authorizable
+    include Taxonomix
     extend FriendlyId
     friendly_id :name
+    include Parameterizable::ByIdName
 
+    self.table_name = 'acd_app_definitions'
     validates :name, :presence => true, :uniqueness => true
+    validates :ansible_playbook, :presence => true
     has_many :app_instances, :inverse_of => :app_definition, :dependent => :destroy
     belongs_to :ansible_playbook, :inverse_of => :app_definitions, :foreign_key => :acd_ansible_playbook_id
     scoped_search :on => :name
-    default_scope -> { order("app_definitions.name") }
+
+    default_scope do
+      with_taxonomy_scope do
+        order('acd_app_definitions.name')
+      end
+    end
+
+    def used_location_ids
+      Location.joins(:taxable_taxonomies).where(
+        'taxable_taxonomies.taxable_type' => 'ForemanAcd::AppDefinition',
+        'taxable_taxonomies.taxable_id' => id
+      ).pluck("#{Taxonomy.table_name}.id")
+    end
+
+    def used_organization_ids
+      Organization.joins(:taxable_taxonomies).where(
+        'taxable_taxonomies.taxable_type' => 'ForemanAcd::AppDefinition',
+        'taxable_taxonomies.taxable_id' => id
+      ).pluck("#{Taxonomy.table_name}.id")
+    end
 
     def self.humanize_class_name(_name = nil)
       _('App Definition')

data/app/models/foreman_acd/app_instance.rb

@@ -4,12 +4,25 @@ module ForemanAcd
   # Application Instance
   class AppInstance < ApplicationRecord
     include Authorizable
+    include ForemanTasks::Concerns::ActionSubject
     extend FriendlyId
     friendly_id :name
+    include Parameterizable::ByIdName
+
+    self.table_name = 'acd_app_instances'
+    belongs_to :last_deploy_task, :class_name => 'ForemanTasks::Task'
+    belongs_to :initial_configure_task, :class_name => 'ForemanTasks::Task'
     validates :name, :presence => true, :uniqueness => true
+    validates :app_definition, :presence => true
     belongs_to :app_definition, :inverse_of => :app_instances
+    belongs_to :organization
+    validates :organization, :presence => true
+    belongs_to :location
+    validates :location, :presence => true
+    has_many :foreman_hosts, :inverse_of => :app_instance, :dependent => :destroy
     scoped_search :on => :name
-    default_scope -> { order("app_instances.name") }
+    default_scope -> { order('acd_app_instances.name') }
+    attr_accessor :hosts
 
     def self.humanize_class_name(_name = nil)
       _('App Instance')
@@ -19,10 +32,77 @@ module ForemanAcd
       'app_instances'
     end
 
-    def foreman_hosts
-      app_hosts = JSON.parse(self.hosts)
-      host_ids = app_hosts.select{ |h| h&.key?('foreman_host_id') }.map{ |h| h['foreman_host_id'] }
-      ::Host.find(host_ids)
+    def clean_all_hosts
+      remember_host_ids = foreman_hosts.select(&:fresh_host?).map(&:host_id)
+
+      # Clean the app instance association first
+      foreman_hosts.update_all(:host_id => nil)
+
+      # Remove all hosts afterwards
+      delete_hosts(remember_host_ids)
+    end
+
+    def clean_hosts_by_id(ids = [])
+      # Clean the app instance association first
+      foreman_hosts.where(:host_id => ids, :is_existing_host => false).update_all(:host_id => nil)
+
+      # Remove all hosts afterwards
+      delete_hosts(ids)
+    end
+
+    def hosts_deployment_finished?
+      return true if all_hosts_deployed?
+
+      ::Foreman::Logging.logger('foreman_acd').info('Another host is still in build-phase. Wait for it...')
+      false
+    end
+
+    def deployment_state
+      return :new if last_deploy_task.nil?
+      return :initiated if !last_deploy_task.nil? && last_deploy_task.ended_at.nil?
+
+      state = if all_hosts_deployed?
+                :finished
+              elsif last_deploy_task.ended_at? && last_deploy_task.result != 'success'
+                :failed
+              else
+                :pending
+              end
+      state
+    end
+
+    def initial_configure_job
+      return nil if initial_configure_task.nil?
+      return JobInvocation.find(initial_configure_task.output['configure_job_id']) if initial_configure_task.output.key?('configure_job_id') &&
+                                                                                      !initial_configure_task.output['configure_job_id'].nil?
+      nil
+    end
+
+    def initial_configure_state
+      return :unconfigured if initial_configure_job.nil? && initial_configure_task.nil?
+      return :scheduled if initial_configure_job.nil?
+      return :pending unless initial_configure_job.finished?
+      initial_configure_job.status_label.to_sym
+    end
+
+    private
+
+    def all_hosts_deployed?
+      foreman_hosts.each do |foreman_host|
+        return false if foreman_host.host.nil? || foreman_host.host.build?
+      end
+      true
+    end
+
+    def delete_hosts(ids = [])
+      return if ids.empty?
+      ids.each do |host_id|
+        h = ::Host.find(host_id) unless host_id.nil?
+        if h
+          Katello::RegistrationManager.unregister_host(h, :unregistering => false) if ForemanAcd.with_katello?
+          h.destroy
+        end
+      end
     end
   end
 end

data/app/models/foreman_acd/foreman_host.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ForemanAcd
+  # Foreman Host
+  class ForemanHost < ApplicationRecord
+    include Authorizable
+
+    self.table_name = 'acd_foreman_hosts'
+    validates :hostname, :presence => true
+    belongs_to :app_instance, :inverse_of => :foreman_hosts
+    belongs_to :host, :class_name => '::Host::Managed'
+    scoped_search :on => :hostname
+    default_scope -> { order('acd_foreman_hosts.hostname') }
+
+    def self.humanize_class_name(_name = nil)
+      _('Foreman Host')
+    end
+
+    def fresh_host?
+      !is_existing_host
+    end
+
+    def existing_host?
+      is_existing_host
+    end
+
+    def self.permission_name
+      'foreman_hosts'
+    end
+  end
+end

data/app/models/foreman_acd/taxonomy_extensions.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ForemanAcd
+  # This is a TaxonomyExtensions
+  module TaxonomyExtensions
+    extend ActiveSupport::Concern
+
+    included do
+      if SETTINGS[:version].to_s.to_f <= 1.7
+        def self.enabled_taxonomies
+          %w[locations organizations].select { |taxonomy| SETTINGS["#{taxonomy}_enabled".to_sym] }
+        end
+      end
+    end
+  end
+end
+# To be removed after 1.7 compatibility is no longer required

data/app/services/foreman_acd/acd_proxy_proxy_selector.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ForemanAcd
+  # AcdProxy Selector implementing a RemoteExecutionProxySelector which
+  # only returns the Smart Proxy of the given host
+  class AcdProxyProxySelector < ::RemoteExecutionProxySelector
+    def determine_proxy(*args)
+      host, _provider = args
+
+      # We already did the determine_proxy in app_configurator. We want that REX
+      # isn't doing the determination of the proxy twice.
+      # Therefore, we will just return the host, which is the proxy!
+
+      ::SmartProxy.find_by(:name => host.name)
+    end
+  end
+end

data/app/services/foreman_acd/app_configurator.rb

@@ -3,7 +3,6 @@
 module ForemanAcd
   # application instances configurator
   class AppConfigurator
-
     delegate :logger, :to => :Rails
 
     def initialize(app_instance)
@@ -13,58 +12,87 @@ module ForemanAcd
     def configure
       job_input = {}
       job_input['application_name'] = @app_instance.name
-      job_input['playbook_name'] = @app_instance.app_definition.ansible_playbook.name
-      job_input['playbook_path'] = File.join(@app_instance.app_definition.ansible_playbook.path,
-                                            @app_instance.app_definition.ansible_playbook.playfile)
-
-      # TODO should or do we really need it to a file?
-      #inventory_file = File.new("/tmp/acd_inventory_file", "w") # we can also use Tempfile.new() but a tempfile will be deleted soon (after transaction finished)
-      #inventory_file << inventory.to_yaml
-      #inventory_file.close
+      job_input['playbook_id'] = @app_instance.app_definition.ansible_playbook.id
+      job_input['playbook_file'] = @app_instance.app_definition.ansible_playbook.playfile
 
       logger.info("Use inventory to configure #{@app_instance.name} with ansible playbook #{@app_instance.app_definition.ansible_playbook.name}")
 
       begin
         proxy_hosts = {}
-        jobs = []
+        job = nil
+        result = OpenStruct.new
 
         hosts = @app_instance.foreman_hosts
+
+        # Important: This uses the REX Proxy Selector and not
+        #  the AcdProxySelection because the AcdProxySelector
+        #  does not find the proxy but only return the given
+        #  host. This is required because we want to run the
+        #  ansible playbook for a group of hosts on the smart proxy.
+        #  So the process need to be:
+        #  1. get the proxy which is required to connect to host a,b,c
+        #  2. run the job on the proxy
+        #  In 2. we need to make sure that REX doesn't try to find the
+        #  proxy which is necessary to connect to the proxy
         proxy_selector = RemoteExecutionProxySelector.new
         hosts.each do |h|
-          proxy = proxy_selector.determine_proxy(h, 'SSH')
-          unless proxy_hosts.has_key?(proxy.name)
-            proxy_hosts[proxy.name] = Array.new
+          begin
+            unless h.host
+              result.success = false
+              result.error = 'App Instance is not deployed'
+              return [result, job]
+            end
+            proxy = proxy_selector.determine_proxy(h.host, 'ACD')
+            result.success = true
+          rescue NoMethodError => e
+            result.success = false
+            result.error = "#{e}, Install/Update smart-proxies for ACD"
+            return [result, job]
           end
-          proxy_hosts[proxy.name] << h.id
+          proxy_hosts[proxy.name] = [] unless proxy_hosts.key?(proxy.name)
+          proxy_hosts[proxy.name] << h
         end
 
-        # TODO just for testing...
-        #proxy_hosts = { Host.first.name => [ Host.first.id] }
+        # TODO: just for testing...
+        # proxy_hosts = { Host.first.name => [ Host.first.id] }
 
-        # we need to compose multiple jobs. for each proxy one job.
-        proxy_hosts.each do |proxy_name, host_names|
-          # create the inventory file
-          inventory = ForemanAcd::InventoryCreator.new(@app_instance, host_names).create_inventory
-          job_input['inventory'] = YAML.dump(inventory)
+        proxy_inventories = {}
+        proxy_host_ids =  []
+        proxy_hosts.each do |proxy_name, foreman_hosts|
+          proxy_inventories[proxy_name] = ForemanAcd::InventoryCreator.new(@app_instance, foreman_hosts).create_inventory
 
-          # Unfortunately, we can not use "JobInvocationComposer.for_feature" method
-          # because then its not possible to the set effective_user
-          job = JobTemplate.find(RemoteExecutionFeature.feature('run_acd_ansible_playbook').job_template_id)
-          params = {
-            :job_category => job.job_category,
-            :job_template_id => job.id,
-            :targeting_type => 'static_query',
-            :search_query => "name = #{proxy_name}",
-            :effective_user => 'foreman-proxy',
-            :inputs => job_input.to_hash
-          }
-          composer = JobInvocationComposer.from_api_params(params)
-          jobs << composer
+          # Workaround till infrastructure jobs on proxies are possible. See
+          #     https://community.theforeman.org/t/infrastructure-roles/22001/33
+          #
+          # Why do we use 'foreman_hosts.first.host'? REX is 'host-centric'. During
+          # job execution, REX will try to find the proxy on which the job should be executed.
+          # We throw the first host at REX's feet so that the resolution of the
+          # proxy works (... which we have already done above).
+          # ACD on the smart proxy side will call the ansible-playbook for ALL hosts
+          # which are behind this smart proxy.
+          #
+          # Additionally, we disable that  AcdProxyProxySelector is used.
+          # See required_proxy_selector_for in models/foreman_acd/acd_provider.rb
+          proxy_host_ids << foreman_hosts.first.host.id
+
+          # Actually, we want this:
+          # proxy_host_ids << Host.find_by(:name => proxy_name).id
         end
+
+        job_input['inventory'] = YAML.dump(proxy_inventories)
+        composer = JobInvocationComposer.for_feature(
+          :run_acd_ansible_playbook,
+          proxy_host_ids,
+          job_input.to_hash
+        )
+        job = composer
       rescue StandardError => e
-        logger.error("Failed to configure hosts: #{e.class}: #{e.message}\n#{e.backtrace.join($INPUT_RECORD_SEPARATOR)}")
+        result.success = false
+        result.error = _('Failed to configure hosts: %{err_msg}' % { :err_msg => e.message })
+        logger.error('Failed to configure hosts: %{err_class}: %{err_msg}' % { :err_class => e.class, :err_msg => e.message })
+        job = nil
       end
-      jobs
+      [result, job]
     end
   end
 end

data/app/services/foreman_acd/app_deployer.rb

@@ -3,59 +3,94 @@
 module ForemanAcd
   # application instances deployer
   class AppDeployer
-
     delegate :logger, :to => :Rails
 
     def initialize(app_instance)
       @app_instance = app_instance
     end
 
-    def deploy
+    def deploy(safe_deploy)
+      output = []
       services = JSON.parse(@app_instance.app_definition.services)
-      app_hosts = JSON.parse(@app_instance.hosts)
-
-      app_hosts.each do |host_data|
-        begin
-          service_data = services.select { |k| k['id'] == host_data['service'].to_i }.first
-          host_params = set_host_params(host_data, service_data)
-
-          host = nil
-          if host_data.has_key?('foreman_host_id')
-            logger.debug("Try to find host with id #{host_data['foreman_host_id']}")
-            begin
-              host = Host.find(host_data['foreman_host_id'])
-            rescue ActiveRecord::RecordNotFound
-              logger.info("Host with id #{host_data['foreman_host_id']} couldn\'t be found, create a new one!")
-              host = nil
-            end
-          end
-
-          if host.nil?
-            params = host_attributes(host_params)
-            logger.info("Host creation parameters for #{host_data['hostname']}:\n#{params}\n")
-            host = Host.new(params)
-          else
-            logger.info("Update parameters and re-deploy host #{host_data['hostname']}")
-            host.attributes = host_attributes(host_params, host)
-            host.setBuild
-            host.power.reset
-          end
-
-          # REMOVE ME (but very nice for testing)
-          # host.mac = "00:11:22:33:44:55"
-
-          apply_compute_profile(host)
-          host.suggest_default_pxe_loader
-          host.save
-
-          # save the foreman host id
-          host_data['foreman_host_id'] = host.id
-        rescue StandardError => e
-          logger.error("Failed to initiate host creation: #{e.class}: #{e.message}\n#{e.backtrace.join($INPUT_RECORD_SEPARATOR)}")
+      all_hosts = []
+
+      foreman_hosts = if safe_deploy
+                        @app_instance.foreman_hosts.find(safe_deploy)
+                      else
+                        @app_instance.foreman_hosts
+                      end
+
+      foreman_hosts.each do |foreman_host|
+        service_data = services.select { |k| k['id'] == foreman_host.service.to_i }.first
+
+        # Handle already deployed hosts
+        if foreman_host.existing_host?
+          domain = Hostgroup.find(service_data['hostgroup']).domain.name
+          fqdn = "#{foreman_host.hostname}.#{domain}"
+          h =  Host.find_by(:name => fqdn)
+          foreman_host.update!(:host_id => h.id)
+          next
+        end
+
+        host_params = set_host_params(foreman_host, service_data)
+        host = foreman_host.host.presence
+
+        is_rebuild = false
+
+        if host.blank?
+          params = host_attributes(host_params)
+          log_params = params.dup
+          log_params['root_pass'] = '****' if log_params.key?('root_pass')
+          msg = "Host creation parameters for #{foreman_host.hostname}:\n#{log_params}\n"
+          logger.info(msg)
+          output << msg
+          host = Host.new(params)
+        else
+          msg = "Update parameters and re-deploy host #{foreman_host.hostname}"
+          logger.info(msg)
+          output << msg
+          host.attributes = host_attributes(host_params, host)
+          is_rebuild = true
         end
+
+        # REMOVE ME (but very nice for testing)
+        # prng = Random.new
+        # x = prng.rand(100)
+        # y = prng.rand(100)
+        # host.mac = "00:11:22:33:#{x}:#{y}"
+
+        apply_compute_profile(host)
+        host.suggest_default_pxe_loader
+
+        all_hosts << OpenStruct.new(:foreman_host => foreman_host, :host => host, :rebuild => is_rebuild)
       end
 
-      return app_hosts
+      # do this in a second step, so that we get the progress report for all
+      all_hosts.each do |os_host|
+        # Save the host -> will initiate the deployment
+        os_host.host.save!
+        msg = "Saved and initiated/updated host #{os_host.foreman_host.hostname}"
+        logger.info(msg)
+        output << msg
+
+        os_host.host.power.reset if os_host.rebuild
+
+        # save the foreman host id
+        os_host.foreman_host.update!(:host_id => os_host.host.id)
+
+        progress_report = Rails.cache.fetch(os_host.host.progress_report_id)
+        os_host.foreman_host.update!(:last_progress_report => progress_report)
+        if progress_report.empty?
+          msg = "Progress report for #{os_host.foreman_host.hostname} is empty!"
+          output << msg
+        end
+      end
+
+      # Try to start the configuration, too. In case of a app instance including only already deployed hosts
+      # this would start the configuration job then.
+      ForemanAcd.initiate_acd_app_configurator(@app_instance)
+
+      output
     end
 
     private
@@ -91,20 +126,20 @@ module ForemanAcd
 
     def hardcoded_params
       result = {}
-      result['managed'] = true
-      result['enabled'] = true
       result['build'] = true
       result['compute_attributes'] = { 'start' => '1' }
+      result['enabled'] = true
       result['host_parameters_attributes'] = []
+      result['managed'] = true
       result
     end
 
-    def set_host_params(host_data, service_data)
+    def set_host_params(foreman_host, service_data)
       result = hardcoded_params
-      result['name'] = host_data['hostname']
+      result['name'] = foreman_host.hostname
       result['hostgroup_id'] = service_data['hostgroup']
 
-      host_data['foremanParameters'].each do |param|
+      JSON.parse(foreman_host.foremanParameters).each do |param|
         case param['type']
 
         when 'computeprofile'