foreman_ansible_core 2.2.0 → 3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e195d6690093de003b7ddaf9b807ac0d79ad775a
-  data.tar.gz: 4286d8f97ffca80c17be58c302fb2aeffbf1d02f
+SHA256:
+  metadata.gz: a45baed63d6c808d9df4b3717e427b332e45af0b49a6f652e197e7f151e5c023
+  data.tar.gz: da8b32e8eb3c6177299fd1fc2789f0db173e29a833323277edf4f89da9eb83de
 SHA512:
-  metadata.gz: b1555fd4cb45577b25aa1f6b6ff4ba0ac02d354fa3c74e33790dc63e3454de9d70c1a01b4cc929b1096437395dfbdbb519e82e22f333710da2a5b5cf0d12331b
-  data.tar.gz: 23e7b36347e64370135f75e18165d424ba6ee7bc429335be4e83a1778e05d6fd17c38c2ba1f76e8a0515cd424b4cf644c0b7995b876aa3f4055e4ccb5bcaf53c
+  metadata.gz: 49118a48bf6ebb4ede1e03b6a9278aa1d507da498620db057d0fc108ae7aabf4851b1ed8ee4f515553d64a5473ed6463835cc27e08e0606220804197b867fefa
+  data.tar.gz: 932fb010a258d877b038daa7d0e97f599242c25282dd7d7e7ed74fa77de1050b5251fd6f0690e60746b2bd6f25406cff63ac476983a90dccb929390d2b87f453

data/lib/foreman_ansible_core.rb

@@ -15,15 +15,20 @@ module ForemanAnsibleCore
 
   if ForemanTasksCore.dynflow_present?
     require 'foreman_tasks_core/runner'
-    require 'foreman_ansible_core/playbook_runner'
+    require 'foreman_ansible_core/runner/playbook'
+    require 'foreman_ansible_core/runner/ansible_runner'
     require 'foreman_ansible_core/actions'
   end
 
   require 'foreman_remote_execution_core/actions'
   require 'foreman_ansible_core/remote_execution_core/ansible_runner'
-  require 'foreman_ansible_core/remote_execution_core/settings_override'
-  ForemanRemoteExecutionCore::Actions::RunScript.send(
-    :prepend,
-    ForemanAnsibleCore::RemoteExecutionCore::SettingsOverride
-  )
+  require 'foreman_ansible_core/task_launcher/playbook'
+  require 'foreman_ansible_core/task_launcher/ansible_runner'
+
+  if defined?(SmartProxyDynflowCore)
+    SmartProxyDynflowCore::TaskLauncherRegistry.register('ansible-runner',
+                                                         ForemanAnsibleCore::TaskLauncher::AnsibleRunner)
+    SmartProxyDynflowCore::TaskLauncherRegistry.register('ansible-playbook',
+                                                         ForemanAnsibleCore::TaskLauncher::Playbook)
+  end
 end

data/lib/foreman_ansible_core/actions.rb

@@ -8,7 +8,7 @@ module ForemanAnsibleCore
     # to execute the playbook run
     class RunPlaybook < ForemanTasksCore::Runner::Action
       def initiate_runner
-        ForemanAnsibleCore::PlaybookRunner.new(
+        ForemanAnsibleCore::Runner::Playbook.new(
           input[:inventory],
           input[:playbook],
           input[:options]

data/lib/foreman_ansible_core/remote_execution_core/ansible_runner.rb

@@ -5,13 +5,15 @@ module ForemanAnsibleCore
     # Takes an inventory and runs it through REXCore CommandRunner
     class AnsibleRunner < ::ForemanTasksCore::Runner::CommandRunner
       DEFAULT_REFRESH_INTERVAL = 1
+      CONNECTION_PROMPT = 'Are you sure you want to continue connecting (yes/no)? '
 
-      def initialize(options)
-        super(options)
-        @playbook_runner = ForemanAnsibleCore::PlaybookRunner.new(
+      def initialize(options, suspended_action:)
+        super(options, :suspended_action => suspended_action)
+        @playbook_runner = ForemanAnsibleCore::Runner::Playbook.new(
           options['ansible_inventory'],
           options['script'],
-          options
+          options,
+          :suspended_action => suspended_action
         )
       end
 
@@ -39,6 +41,21 @@ module ForemanAnsibleCore
         @command_in = @playbook_runner.command_in
         @command_pid = @playbook_runner.command_pid
         super
+        kill if unknown_host_key_fingerprint?
+      end
+
+      def kill
+        publish_exit_status(1)
+        ::Process.kill('SIGTERM', @command_pid)
+        close
+      end
+
+      private
+
+      def unknown_host_key_fingerprint?
+        last_output = @continuous_output.raw_outputs.last
+        return if last_output.nil?
+        last_output['output']&.lines&.last == CONNECTION_PROMPT
       end
     end
   end

data/lib/foreman_ansible_core/runner/ansible_runner.rb

@@ -0,0 +1,160 @@
+require 'shellwords'
+
+module ForemanAnsibleCore
+  module Runner
+    class AnsibleRunner < ForemanTasksCore::Runner::Parent
+      include ForemanTasksCore::Runner::Command
+
+      def initialize(input, suspended_action:)
+        super input, :suspended_action => suspended_action
+        @inventory = rebuild_secrets(rebuild_inventory(input), input)
+        @playbook = input.values.first[:input][:action_input][:script]
+        @root = working_dir
+      end
+
+      def start
+        prepare_directory_structure
+        write_inventory
+        write_playbook
+        start_ansible_runner
+      end
+
+      def refresh
+        return unless super
+        @counter ||= 1
+        @uuid ||= File.basename(Dir["#{@root}/artifacts/*"].first)
+        job_event_dir = File.join(@root, 'artifacts', @uuid, 'job_events')
+        loop do
+          files = Dir["#{job_event_dir}/*.json"].map do |file|
+            num = File.basename(file)[/\A\d+/].to_i unless file.include?('partial')
+            [file, num]
+          end
+          files_with_nums = files.select { |(_, num)| num && num >= @counter }.sort_by(&:last)
+          break if files_with_nums.empty?
+          logger.debug("[foreman_ansible] - processing event files: #{files_with_nums.map(&:first).inspect}}")
+          files_with_nums.map(&:first).each { |event_file| handle_event_file(event_file) }
+          @counter = files_with_nums.last.last + 1
+        end
+      end
+
+      private
+
+      def handle_event_file(event_file)
+        logger.debug("[foreman_ansible] - parsing event file #{event_file}")
+        begin
+          event = JSON.parse(File.read(event_file))
+          if (hostname = event.dig('event_data', 'host'))
+            handle_host_event(hostname, event)
+          else
+            handle_broadcast_data(event)
+          end
+          true
+        rescue JSON::ParserError => e
+          logger.error("[foreman_ansible] - Error parsing runner event at #{event_file}: #{e.class}: #{e.message}")
+          logger.debug(e.backtrace.join("\n"))
+        end
+      end
+
+      def handle_host_event(hostname, event)
+        log_event("for host: #{hostname.inspect}", event)
+        publish_data_for(hostname, event['stdout'] + "\n", 'stdout') if event['stdout']
+        case event['event']
+        when 'runner_on_ok'
+          publish_exit_status_for(hostname, 0) if @exit_statuses[hostname].nil?
+        when 'runner_on_unreachable'
+          publish_exit_status_for(hostname, 1)
+        when 'runner_on_failed'
+          publish_exit_status_for(hostname, 2) if event.dig('event_data', 'ignore_errors').nil?
+        end
+      end
+
+      def handle_broadcast_data(event)
+        log_event("broadcast", event)
+        if event['event'] == 'playbook_on_stats'
+          header, *rows = event['stdout'].strip.lines.map(&:chomp)
+          @outputs.keys.select { |key| key.is_a? String }.each do |host|
+            line = rows.find { |row| row =~ /#{host}/ }
+            publish_data_for(host, [header, line].join("\n"), 'stdout')
+          end
+        else
+          broadcast_data(event['stdout'] + "\n", 'stdout')
+        end
+      end
+
+      def write_inventory
+        path = File.join(@root, 'inventory', 'hosts')
+        data_path = File.join(@root, 'data')
+        inventory_script = <<~INVENTORY_SCRIPT
+          #!/bin/sh
+          cat #{::Shellwords.escape data_path}
+        INVENTORY_SCRIPT
+        File.write(path, inventory_script)
+        File.write(data_path, JSON.dump(@inventory))
+        File.chmod(0o0755, path)
+      end
+
+      def write_playbook
+        File.write(File.join(@root, 'project', 'playbook.yml'), @playbook)
+      end
+
+      def start_ansible_runner
+        command = ['ansible-runner', 'run', @root, '-p', 'playbook.yml']
+        initialize_command(*command)
+        logger.debug("[foreman_ansible] - Running command '#{command.join(' ')}'")
+      end
+
+      def prepare_directory_structure
+        inner = %w[inventory project].map { |part| File.join(@root, part) }
+        ([@root] + inner).each do |path|
+          FileUtils.mkdir_p path
+        end
+      end
+
+      def log_event(description, event)
+        # TODO: replace this ugly code with block variant once https://github.com/Dynflow/dynflow/pull/323
+        # arrives in production
+        logger.debug("[foreman_ansible] - handling event #{description}: #{JSON.pretty_generate(event)}") if logger.level <= ::Logger::DEBUG
+      end
+
+      # Each per-host task has inventory only for itself, we must
+      # collect all the partial inventories into one large inventory
+      # containing all the hosts.
+      def rebuild_inventory(input)
+        action_inputs = input.values.map { |hash| hash[:input][:action_input] }
+        hostnames = action_inputs.map { |hash| hash[:name] }
+        inventories = action_inputs.map { |hash| hash[:ansible_inventory] }
+        host_vars = inventories.map { |i| i['_meta']['hostvars'] }.reduce(&:merge)
+
+        { '_meta' => { 'hostvars' => host_vars },
+          'all' => { 'hosts' => hostnames,
+                     'vars' => inventories.first['all']['vars'] } }
+      end
+
+      def working_dir
+        return @root if @root
+        dir = ForemanAnsibleCore.settings[:working_dir]
+        @tmp_working_dir = true
+        if dir.nil?
+          Dir.mktmpdir
+        else
+          Dir.mktmpdir(nil, File.expand_path(dir))
+        end
+      end
+
+      def rebuild_secrets(inventory, input)
+        input.each do |host, host_input|
+          secrets = host_input['input']['action_input']['secrets']
+          per_host = secrets['per-host'][host]
+
+          new_secrets = {
+            'ansible_ssh_pass' => inventory['ssh_password'] || per_host['ansible_ssh_pass'],
+            'ansible_sudo_pass' => inventory['sudo_password'] || per_host['ansible_sudo_pass']
+          }
+          inventory['_meta']['hostvars'][host].update(new_secrets)
+        end
+
+        inventory
+      end
+    end
+  end
+end

data/lib/foreman_ansible_core/{command_creator.rb → runner/command_creator.rb}

@@ -34,7 +34,7 @@ module ForemanAnsibleCore
     end
 
     def json_inventory_script
-      File.expand_path('../../bin/json_inventory.sh', File.dirname(__FILE__))
+      File.expand_path('../../../bin/json_inventory.sh', File.dirname(__FILE__))
     end
 
     def setup_verbosity

data/lib/foreman_ansible_core/runner/playbook.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'foreman_tasks_core/runner/command_runner'
+require_relative 'command_creator'
+require 'tmpdir'
+require 'net/ssh'
+
+module ForemanAnsibleCore
+  module Runner
+    # Implements ForemanTasksCore::Runner::Base interface for running
+    # Ansible playbooks, used by the Foreman Ansible plugin and Ansible proxy
+    class Playbook < ForemanTasksCore::Runner::CommandRunner
+      attr_reader :command_out, :command_in, :command_pid
+
+      def initialize(inventory, playbook, options = {}, suspended_action:)
+        super :suspended_action => suspended_action
+        @inventory = rebuild_secrets(inventory, options[:secrets])
+        unknown_hosts.each do |host|
+          add_to_known_hosts(host)
+        end
+        @playbook  = playbook
+        @options   = options
+        initialize_dirs
+      end
+
+      def start
+        write_inventory
+        write_playbook
+        command = CommandCreator.new(inventory_file,
+                                     playbook_file,
+                                     @options).command
+        logger.debug('[foreman_ansible] - Initializing Ansible Runner')
+        Dir.chdir(@ansible_dir) do
+          initialize_command(*command)
+          logger.debug("[foreman_ansible] - Running command '#{command.join(' ')}'")
+        end
+      end
+
+      def kill
+        publish_data('== TASK ABORTED BY USER ==', 'stdout')
+        publish_exit_status(1)
+        ::Process.kill('SIGTERM', @command_pid)
+        close
+      end
+
+      def close
+        super
+        FileUtils.remove_entry(@working_dir) if @tmp_working_dir
+      end
+
+      private
+
+      def write_inventory
+        ensure_directory(File.dirname(inventory_file))
+        File.write(inventory_file, JSON.dump(@inventory))
+      end
+
+      def write_playbook
+        ensure_directory(File.dirname(playbook_file))
+        File.write(playbook_file, @playbook)
+      end
+
+      def inventory_file
+        File.join(@working_dir, 'foreman-inventories', id)
+      end
+
+      def playbook_file
+        File.join(@working_dir, "foreman-playbook-#{id}.yml")
+      end
+
+      def events_dir
+        File.join(@working_dir, 'events', id.to_s)
+      end
+
+      def ensure_directory(path)
+        if File.exist?(path)
+          raise "#{path} expected to be a directory" unless File.directory?(path)
+        else
+          FileUtils.mkdir_p(path)
+        end
+        path
+      end
+
+      def initialize_dirs
+        settings = ForemanAnsibleCore.settings
+        initialize_working_dir(settings[:working_dir])
+        initialize_ansible_dir(settings[:ansible_dir])
+      end
+
+      def initialize_working_dir(working_dir)
+        if working_dir.nil?
+          @working_dir = Dir.mktmpdir
+          @tmp_working_dir = true
+        else
+          @working_dir = File.expand_path(working_dir)
+        end
+      end
+
+      def initialize_ansible_dir(ansible_dir)
+        raise "Ansible dir #{ansible_dir} does not exist" unless
+          !ansible_dir.nil? && File.exist?(ansible_dir)
+        @ansible_dir = ansible_dir
+      end
+
+      def unknown_hosts
+        @inventory['all']['hosts'].select do |host|
+          Net::SSH::KnownHosts.search_for(host).empty?
+        end
+      end
+
+      def add_to_known_hosts(host)
+        logger.warn("[foreman_ansible] - Host #{host} not found in known_hosts")
+        Net::SSH::Transport::Session.new(host).host_keys.each do |host_key|
+          Net::SSH::KnownHosts.add(host, host_key)
+        end
+        logger.warn("[foreman_ansible] - Added host key #{host} to known_hosts")
+      rescue StandardError => e
+        logger.error('[foreman_ansible] - Failed to save host key for '\
+          "#{host}: #{e}")
+      end
+
+      def rebuild_secrets(inventory, secrets)
+        inventory['all']['hosts'].each do |name|
+          per_host = secrets['per-host'][name]
+
+          new_secrets = {
+            'ansible_ssh_pass' => inventory['ssh_password'] || per_host['ansible_ssh_pass'],
+            'ansible_sudo_pass' => inventory['sudo_password'] || per_host['ansible_sudo_pass']
+          }
+          inventory['_meta']['hostvars'][name].update(new_secrets)
+        end
+
+        inventory
+      end
+    end
+  end
+end

data/lib/foreman_ansible_core/task_launcher/ansible_runner.rb

@@ -0,0 +1,38 @@
+require 'fileutils'
+
+module ForemanAnsibleCore
+  module TaskLauncher
+    class AnsibleRunner < ForemanTasksCore::TaskLauncher::AbstractGroup
+      def runner_input(input)
+        super(input).reduce({}) do |acc, (_id, data)|
+          acc.merge(data[:input]['action_input']['name'] => data)
+        end
+      end
+
+      def operation
+        'ansible-runner'
+      end
+
+      def self.runner_class
+        Runner::AnsibleRunner
+      end
+
+      # def self.input_format
+      #   {
+      #     $UUID => {
+      #       :execution_plan_id => $EXECUTION_PLAN_UUID,
+      #       :run_step_id => Integer,
+      #       :input => {
+      #         :action_class => Class,
+      #         :action_input => {
+      #           "ansible_inventory"=> String,
+      #           "hostname"=>"127.0.0.1",
+      #           "script"=>"---\n- hosts: all\n  tasks:\n    - shell: |\n        true\n      register: out\n    - debug: var=out"
+      #         }
+      #       }
+      #     }
+      #   }
+      # end
+    end
+  end
+end

data/lib/foreman_ansible_core/task_launcher/playbook.rb

@@ -0,0 +1,23 @@
+module ForemanAnsibleCore
+  module TaskLauncher
+    class Playbook < ForemanTasksCore::TaskLauncher::Batch
+      class PlaybookRunnerAction < ForemanTasksCore::Runner::Action
+        def initiate_runner
+          additional_options = {
+            :step_id => run_step_id,
+            :uuid => execution_plan_id
+          }
+          ::ForemanAnsibleCore::RemoteExecutionCore::AnsibleRunner.new(
+            input.merge(additional_options),
+            :suspended_action => suspended_action
+          )
+        end
+      end
+
+      def child_launcher(parent)
+        ForemanTasksCore::TaskLauncher::Single.new(world, callback, :parent => parent,
+                                                                    :action_class_override => PlaybookRunnerAction)
+      end
+    end
+  end
+end

data/lib/foreman_ansible_core/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ForemanAnsibleCore
-  VERSION = '2.2.0'
+  VERSION = '3.0.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: foreman_ansible_core
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 3.0.3
 platform: ruby
 authors:
 - Daniel Lobato Garcia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-22 00:00:00.000000000 Z
+date: 2020-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   name: foreman_remote_execution_core
   requirement: !ruby/object:Gem::Requirement
@@ -79,11 +79,13 @@ files:
 - bin/json_inventory.sh
 - lib/foreman_ansible_core.rb
 - lib/foreman_ansible_core/actions.rb
-- lib/foreman_ansible_core/command_creator.rb
 - lib/foreman_ansible_core/exception.rb
-- lib/foreman_ansible_core/playbook_runner.rb
 - lib/foreman_ansible_core/remote_execution_core/ansible_runner.rb
-- lib/foreman_ansible_core/remote_execution_core/settings_override.rb
+- lib/foreman_ansible_core/runner/ansible_runner.rb
+- lib/foreman_ansible_core/runner/command_creator.rb
+- lib/foreman_ansible_core/runner/playbook.rb
+- lib/foreman_ansible_core/task_launcher/ansible_runner.rb
+- lib/foreman_ansible_core/task_launcher/playbook.rb
 - lib/foreman_ansible_core/version.rb
 homepage: https://github.com/theforeman/foreman_ansible
 licenses:
@@ -104,8 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: 'Ansible integration with Foreman (theforeman.org): core bits'

data/lib/foreman_ansible_core/playbook_runner.rb

@@ -1,121 +0,0 @@
-# frozen_string_literal: true
-
-require 'foreman_tasks_core/runner/command_runner'
-require_relative 'command_creator'
-require 'tmpdir'
-require 'net/ssh'
-
-module ForemanAnsibleCore
-  # Implements ForemanTasksCore::Runner::Base interface for running
-  # Ansible playbooks, used by the Foreman Ansible plugin and Ansible proxy
-  class PlaybookRunner < ForemanTasksCore::Runner::CommandRunner
-    attr_reader :command_out, :command_in, :command_pid
-
-    def initialize(inventory, playbook, options = {})
-      super
-      @inventory = inventory
-      unknown_hosts.each do |host|
-        add_to_known_hosts(host)
-      end
-      @playbook  = playbook
-      @options   = options
-      initialize_dirs
-    end
-
-    def start
-      write_inventory
-      write_playbook
-      command = CommandCreator.new(inventory_file,
-                                   playbook_file,
-                                   @options).command
-      logger.debug('[foreman_ansible] - Initializing Ansible Runner')
-      Dir.chdir(@ansible_dir) do
-        initialize_command(*command)
-        logger.debug("[foreman_ansible] - Running command #{command}")
-      end
-    end
-
-    def kill
-      publish_data('== TASK ABORTED BY USER ==', 'stdout')
-      publish_exit_status(1)
-      ::Process.kill('SIGTERM', @command_pid)
-      close
-    end
-
-    def close
-      super
-      FileUtils.remove_entry(@working_dir) if @tmp_working_dir
-    end
-
-    private
-
-    def write_inventory
-      ensure_directory(File.dirname(inventory_file))
-      File.write(inventory_file, @inventory)
-    end
-
-    def write_playbook
-      ensure_directory(File.dirname(playbook_file))
-      File.write(playbook_file, @playbook)
-    end
-
-    def inventory_file
-      File.join(@working_dir, 'foreman-inventories', id)
-    end
-
-    def playbook_file
-      File.join(@working_dir, "foreman-playbook-#{id}.yml")
-    end
-
-    def events_dir
-      File.join(@working_dir, 'events', id.to_s)
-    end
-
-    def ensure_directory(path)
-      if File.exist?(path)
-        raise "#{path} expected to be a directory" unless File.directory?(path)
-      else
-        FileUtils.mkdir_p(path)
-      end
-      path
-    end
-
-    def initialize_dirs
-      settings = ForemanAnsibleCore.settings
-      initialize_working_dir(settings[:working_dir])
-      initialize_ansible_dir(settings[:ansible_dir])
-    end
-
-    def initialize_working_dir(working_dir)
-      if working_dir.nil?
-        @working_dir = Dir.mktmpdir
-        @tmp_working_dir = true
-      else
-        @working_dir = File.expand_path(working_dir)
-      end
-    end
-
-    def initialize_ansible_dir(ansible_dir)
-      raise "Ansible dir #{ansible_dir} does not exist" unless
-        !ansible_dir.nil? && File.exist?(ansible_dir)
-      @ansible_dir = ansible_dir
-    end
-
-    def unknown_hosts
-      JSON.parse(@inventory)['all']['hosts'].select do |host|
-        Net::SSH::KnownHosts.search_for(host).empty?
-      end
-    end
-
-    def add_to_known_hosts(host)
-      logger.warn("[foreman_ansible] - Host #{host} not found in known_hosts")
-      Net::SSH::Transport::Session.new(host).host_keys.each do |host_key|
-        Net::SSH::KnownHosts.add(host, host_key)
-      end
-      logger.warn("[foreman_ansible] - Added host key #{host} to known_hosts")
-    rescue StandardError => e
-      logger.error('[foreman_ansible] - Failed to save host key for '\
-                   "#{host}: #{e}")
-    end
-  end
-end

data/lib/foreman_ansible_core/remote_execution_core/settings_override.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module ForemanAnsibleCore
-  module RemoteExecutionCore
-    # Ensure the Ansible provider is used whenever a JobTemplate using this
-    # provider is called.
-    module SettingsOverride
-      def initiate_runner
-        return super unless input['ansible_inventory']
-        additional_options = {
-          :step_id => run_step_id,
-          :uuid => execution_plan_id
-        }
-        ::ForemanAnsibleCore::RemoteExecutionCore::AnsibleRunner.new(
-          input.merge(additional_options)
-        )
-      end
-    end
-  end
-end