smart_proxy_ansible 3.3.0 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a1934350dedb103c779cdf61196cff80a43751d260fd5f550d26289dc63f388
-  data.tar.gz: cee72032c8d95fde92021ef8c312e9c4f0408b9ac1d88b5a0ebf9cdd97bc906c
+  metadata.gz: 7e690f2c224c01d97752317ca99ae54ac84a01f45fc2d613c727477d0acbcdcf
+  data.tar.gz: 0ec2124072f46f08a9820dee2311dce4a72c44980701ed3ab12697788be1e2ca
 SHA512:
-  metadata.gz: a473838108c4b9cbcef9da311410a2fda5bc39047a2cd4c623533ef718918583de1c830105b92a52303bfc41e1eeb933f1d4a898380a0cf9b01c32d7fb07cb76
-  data.tar.gz: cf1aa8ca722850fd39d312cefcbf178e193fcf391ca8058638dd26db01c288f7350ea6c80e5fb004e87cdda664f5200f24b83b61a2dc3226d88006d0efff7cb3
+  metadata.gz: 1b9e32a10bec5a241ea8e6d5aa9727f67e31b7cac1f47b6181ab14177942a48d722d23d54dc324a1fdfb19e0f57d442f80126a93e06331551700210628d8b7c8
+  data.tar.gz: 83f106b39297d92d521a2472db2c6c8c5a8d60b6391ea88b16a455e2d01070df6acb03ce2138474e8790a6d2495b897a1e118d4954af5115cba6f914a1e0de13

data/README.md

@@ -1,6 +1,6 @@
 # Smart-proxy Ansible plugin
 
-Proxy plugin to make [foreman_ansible](https://github.com/theforeman/foreman_ansible) actions run in the proxy
+Proxy plugin to make [foreman_ansible](https://github.com/theforeman/foreman_ansible) actions run on the proxy.
 
 ## Compatibility
 
@@ -11,8 +11,8 @@ This plugin requires at least Foreman Proxy 2.3.
 ### Prerequisites
 
 We expect your proxy to also have
-[smart_proxy_dynflow](https://github.com/theforeman/smart_proxy_dynflow) 0.1.5
-at least, and [foreman-tasks-core](https://github.com/theforeman/foreman-tasks) as
+[smart_proxy_dynflow](https://github.com/theforeman/smart_proxy_dynflow) 0.5
+at least, and [smart_proxy_remote_execution_ssh](https://github.com/theforeman/smart_proxy_remote_execution_ssh) 0.4 as
 a gem requirement.
 
 ### Get the code

data/lib/smart_proxy_ansible/configuration_loader.rb

@@ -4,17 +4,10 @@ module Proxy::Ansible
       require 'smart_proxy_dynflow'
       require 'smart_proxy_dynflow/continuous_output'
       require 'smart_proxy_ansible/task_launcher/ansible_runner'
-      require 'smart_proxy_ansible/task_launcher/playbook'
-      require 'smart_proxy_ansible/actions'
-      require 'smart_proxy_ansible/remote_execution_core/ansible_runner'
       require 'smart_proxy_ansible/runner/ansible_runner'
-      require 'smart_proxy_ansible/runner/command_creator'
-      require 'smart_proxy_ansible/runner/playbook'
 
       Proxy::Dynflow::TaskLauncherRegistry.register('ansible-runner',
                                                     TaskLauncher::AnsibleRunner)
-      Proxy::Dynflow::TaskLauncherRegistry.register('ansible-playbook',
-                                                    TaskLauncher::Playbook)
     end
   end
 end

data/lib/smart_proxy_ansible/playbooks_reader.rb

@@ -24,7 +24,7 @@ module Proxy
             name = ReaderHelper.playbook_or_role_full_name(path)
             {
               name: name,
-              playbooks_content: File.readlines(path)
+              playbooks_content: File.read(path)
             } if playbooks_to_import.nil? || playbooks_to_import.include?(name)
           end.compact
         rescue Errno::ENOENT, Errno::EACCES => e

data/lib/smart_proxy_ansible/reader_helper.rb

@@ -32,7 +32,7 @@ module Proxy
 
         def playbook_or_role_full_name(path)
           parts = path.split('/')
-          playbook = parts.pop
+          playbook = parts.pop.sub(/\.ya?ml/, '')
           parts.pop
           collection = parts.pop
           author = parts.pop

data/lib/smart_proxy_ansible/runner/ansible_runner.rb

@@ -1,17 +1,17 @@
 require 'shellwords'
 require 'yaml'
 
-require 'smart_proxy_dynflow/runner/command'
+require 'smart_proxy_dynflow/runner/process_manager_command'
 require 'smart_proxy_dynflow/runner/base'
 require 'smart_proxy_dynflow/runner/parent'
 module Proxy::Ansible
   module Runner
     class AnsibleRunner < ::Proxy::Dynflow::Runner::Parent
-      include ::Proxy::Dynflow::Runner::Command
-      attr_reader :execution_timeout_interval, :command_pid
+      include ::Proxy::Dynflow::Runner::ProcessManagerCommand
+      attr_reader :execution_timeout_interval
 
-      def initialize(input, suspended_action:)
-        super input, :suspended_action => suspended_action
+      def initialize(input, suspended_action:, id: nil)
+        super input, :suspended_action => suspended_action, :id => id
         @inventory = rebuild_secrets(rebuild_inventory(input), input)
         action_input = input.values.first[:input][:action_input]
         @playbook = action_input[:script]
@@ -34,22 +34,10 @@ module Proxy::Ansible
         start_ansible_runner
       end
 
-      def refresh
-        return unless super
-        @counter ||= 1
-        @uuid ||= File.basename(Dir["#{@root}/artifacts/*"].first)
-        job_event_dir = File.join(@root, 'artifacts', @uuid, 'job_events')
-        loop do
-          files = Dir["#{job_event_dir}/*.json"].map do |file|
-            num = File.basename(file)[/\A\d+/].to_i unless file.include?('partial')
-            [file, num]
-          end
-          files_with_nums = files.select { |(_, num)| num && num >= @counter }.sort_by(&:last)
-          break if files_with_nums.empty?
-          logger.debug("[foreman_ansible] - processing event files: #{files_with_nums.map(&:first).inspect}}")
-          files_with_nums.map(&:first).each { |event_file| handle_event_file(event_file) }
-          @counter = files_with_nums.last.last + 1
-        end
+      def run_refresh_output
+        logger.debug('refreshing runner on demand')
+        process_artifacts
+        generate_updates
       end
 
       def timeout
@@ -62,7 +50,7 @@ module Proxy::Ansible
       end
 
       def kill
-        ::Process.kill('SIGTERM', @command_pid)
+        ::Process.kill('SIGTERM', @process_manager.pid)
         publish_exit_status(2)
         @inventory['all']['hosts'].each { |hostname| @exit_statuses[hostname] = 2 }
         broadcast_data('Timeout for execution passed, stopping the job', 'stderr')
@@ -74,13 +62,44 @@ module Proxy::Ansible
         FileUtils.remove_entry(@root) if @tmp_working_dir && Dir.exist?(@root) && @cleanup_working_dirs
       end
 
+      def publish_exit_status(status)
+        process_artifacts
+        super
+        @targets.each_key { |host| publish_exit_status_for(host, status) } if status != 0
+      end
+
+      def initialize_command(*command)
+        super
+        @process_manager.stdin.close unless @process_manager.done?
+      end
+
       private
 
+      def process_artifacts
+        @counter ||= 1
+        @uuid ||= if (f = Dir["#{@root}/artifacts/*"].first)
+                    File.basename(f)
+                  end
+        return unless @uuid
+        job_event_dir = File.join(@root, 'artifacts', @uuid, 'job_events')
+        loop do
+          files = Dir["#{job_event_dir}/*.json"].map do |file|
+            num = File.basename(file)[/\A\d+/].to_i unless file.include?('partial')
+            [file, num]
+          end
+          files_with_nums = files.select { |(_, num)| num && num >= @counter }.sort_by(&:last)
+          break if files_with_nums.empty?
+          logger.debug("[foreman_ansible] - processing event files: #{files_with_nums.map(&:first).inspect}}")
+          files_with_nums.map(&:first).each { |event_file| handle_event_file(event_file) }
+          @counter = files_with_nums.last.last + 1
+        end
+      end
+
       def handle_event_file(event_file)
         logger.debug("[foreman_ansible] - parsing event file #{event_file}")
         begin
           event = JSON.parse(File.read(event_file))
-          if (hostname = event.dig('event_data', 'host'))
+          if (hostname = hostname_for_event(event))
             handle_host_event(hostname, event)
           else
             handle_broadcast_data(event)
@@ -92,6 +111,17 @@ module Proxy::Ansible
         end
       end
 
+      def hostname_for_event(event)
+        hostname = event.dig('event_data', 'host') || event.dig('event_data', 'remote_addr')
+        return nil if hostname.nil? || hostname.empty?
+
+        unless @targets.key?(hostname)
+          logger.warn("handle_host_event: unknown host #{hostname} for event '#{event['event']}', broadcasting")
+          return nil
+        end
+        hostname
+      end
+
       def handle_host_event(hostname, event)
         log_event("for host: #{hostname.inspect}", event)
         publish_data_for(hostname, event['stdout'] + "\n", 'stdout') if event['stdout']

data/lib/smart_proxy_ansible/task_launcher/ansible_runner.rb

@@ -24,7 +24,8 @@ module Proxy::Ansible
       # Discard everything apart from hostname to be able to tell the actions
       # apart when debugging
       def transform_input(input)
-        { 'action_input' => super['action_input'].slice('name', :task_id) }
+        action_input = super['action_input']
+        { 'action_input' => { 'name' => action_input['name'], :task_id => action_input[:task_id], :runner_id => action_input[:runner_id] } }
       end
 
       # def self.input_format

data/lib/smart_proxy_ansible/version.rb

@@ -2,6 +2,6 @@ module Proxy
   # Version, this allows the proxy and other plugins know
   # what version of the Ansible plugin is running
   module Ansible
-    VERSION = '3.3.0'
+    VERSION = '3.4.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_ansible
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.1
 platform: ruby
 authors:
 - Ivan Nečas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-17 00:00:00.000000000 Z
+date: 2022-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -115,14 +115,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: smart_proxy_remote_execution_ssh
   requirement: !ruby/object:Gem::Requirement
@@ -152,7 +152,6 @@ files:
 - bin/json_inventory.sh
 - bundler.d/ansible.rb
 - lib/smart_proxy_ansible.rb
-- lib/smart_proxy_ansible/actions.rb
 - lib/smart_proxy_ansible/api.rb
 - lib/smart_proxy_ansible/configuration_loader.rb
 - lib/smart_proxy_ansible/exception.rb
@@ -160,13 +159,9 @@ files:
 - lib/smart_proxy_ansible/playbooks_reader.rb
 - lib/smart_proxy_ansible/plugin.rb
 - lib/smart_proxy_ansible/reader_helper.rb
-- lib/smart_proxy_ansible/remote_execution_core/ansible_runner.rb
 - lib/smart_proxy_ansible/roles_reader.rb
 - lib/smart_proxy_ansible/runner/ansible_runner.rb
-- lib/smart_proxy_ansible/runner/command_creator.rb
-- lib/smart_proxy_ansible/runner/playbook.rb
 - lib/smart_proxy_ansible/task_launcher/ansible_runner.rb
-- lib/smart_proxy_ansible/task_launcher/playbook.rb
 - lib/smart_proxy_ansible/validate_settings.rb
 - lib/smart_proxy_ansible/variables_extractor.rb
 - lib/smart_proxy_ansible/version.rb
@@ -190,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Smart-Proxy Ansible plugin

data/lib/smart_proxy_ansible/actions.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require 'smart_proxy_dynflow/action/runner'
-
-module Proxy::Ansible
-  module Actions
-    # Action that can be run both on Foreman or Foreman proxy side
-    # to execute the playbook run
-    class RunPlaybook < Proxy::Dynflow::Action::Runner
-      def initiate_runner
-        Proxy::Ansible::Runner::Playbook.new(
-          input[:inventory],
-          input[:playbook],
-          input[:options]
-        )
-      end
-    end
-  end
-end

data/lib/smart_proxy_ansible/remote_execution_core/ansible_runner.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-require 'smart_proxy_dynflow/runner/command_runner'
-
-module Proxy::Ansible
-  module RemoteExecutionCore
-    # Takes an inventory and runs it through REXCore CommandRunner
-    class AnsibleRunner < ::Proxy::Dynflow::Runner::CommandRunner
-      DEFAULT_REFRESH_INTERVAL = 1
-      CONNECTION_PROMPT = 'Are you sure you want to continue connecting (yes/no)? '
-
-      def initialize(options, suspended_action:)
-        super(options, :suspended_action => suspended_action)
-        @playbook_runner = Proxy::Ansible::Runner::Playbook.new(
-          options['ansible_inventory'],
-          options['script'],
-          options,
-          :suspended_action => suspended_action
-        )
-      end
-
-      def start
-        @playbook_runner.logger = logger
-        @playbook_runner.start
-      rescue StandardError => e
-        logger.error(
-          'error while initalizing command'\
-          " #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}"
-        )
-        publish_exception('Error initializing command', e)
-      end
-
-      def fill_continuous_output(continuous_output)
-        delegated_output.fetch('result', []).each do |raw_output|
-          continuous_output.add_raw_output(raw_output)
-        end
-      rescue StandardError => e
-        continuous_output.add_exception(_('Error loading data from proxy'), e)
-      end
-
-      def refresh
-        @command_out = @playbook_runner.command_out
-        @command_in = @playbook_runner.command_in
-        @command_pid = @playbook_runner.command_pid
-        super
-        kill if unknown_host_key_fingerprint?
-      end
-
-      def kill
-        publish_exit_status(1)
-        ::Process.kill('SIGTERM', @command_pid)
-        close
-      end
-
-      private
-
-      def unknown_host_key_fingerprint?
-        last_output = @continuous_output.raw_outputs.last
-        return if last_output.nil?
-        last_output['output']&.lines&.last == CONNECTION_PROMPT
-      end
-    end
-  end
-end

data/lib/smart_proxy_ansible/runner/command_creator.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Proxy::Ansible
-  # Creates the actual command to be passed to smart_proxy_dynflow to run
-  class CommandCreator
-    def initialize(inventory_file, playbook_file, options = {})
-      @options = options
-      @playbook_file = playbook_file
-      @inventory_file = inventory_file
-      command
-    end
-
-    def command
-      parts = [environment_variables]
-      parts << 'ansible-playbook'
-      parts.concat(command_options)
-      parts << @playbook_file
-      parts
-    end
-
-    private
-
-    def environment_variables
-      defaults = { 'JSON_INVENTORY_FILE' => @inventory_file }
-      defaults['ANSIBLE_CALLBACK_WHITELIST'] = '' if rex_command?
-      defaults
-    end
-
-    def command_options
-      opts = ['-i', json_inventory_script]
-      opts.concat([setup_verbosity]) if verbose?
-      opts.concat(['-T', @options[:timeout]]) unless @options[:timeout].nil?
-      opts
-    end
-
-    def json_inventory_script
-      File.expand_path('../../../bin/json_inventory.sh', File.dirname(__FILE__))
-    end
-
-    def setup_verbosity
-      verbosity_level = @options[:verbosity_level].to_i
-      verbosity = '-'
-      verbosity_level.times do
-        verbosity += 'v'
-      end
-      verbosity
-    end
-
-    def verbose?
-      verbosity_level = @options[:verbosity_level]
-      # rubocop:disable Rails/Present
-      !verbosity_level.nil? && !verbosity_level.empty? &&
-        verbosity_level.to_i.positive?
-      # rubocop:enable Rails/Present
-    end
-
-    def rex_command?
-      @options[:remote_execution_command]
-    end
-  end
-end

data/lib/smart_proxy_ansible/runner/playbook.rb

@@ -1,138 +0,0 @@
-# frozen_string_literal: true
-
-require 'smart_proxy_dynflow/continuous_output'
-require 'smart_proxy_dynflow/runner/command_runner'
-require_relative 'command_creator'
-require 'tmpdir'
-require 'net/ssh'
-
-module Proxy::Ansible
-  module Runner
-    # Implements Proxy::Dynflow::Runner::Base interface for running
-    # Ansible playbooks, used by the Foreman Ansible plugin and Ansible proxy
-    class Playbook < Proxy::Dynflow::Runner::CommandRunner
-      attr_reader :command_out, :command_in, :command_pid
-
-      def initialize(inventory, playbook, options = {}, suspended_action:)
-        super :suspended_action => suspended_action
-        @inventory = rebuild_secrets(inventory, options[:secrets])
-        unknown_hosts.each do |host|
-          add_to_known_hosts(host)
-        end
-        @playbook  = playbook
-        @options   = options
-        initialize_dirs
-      end
-
-      def start
-        write_inventory
-        write_playbook
-        command = CommandCreator.new(inventory_file,
-                                     playbook_file,
-                                     @options).command
-        logger.debug('[foreman_ansible] - Initializing Ansible Runner')
-        Dir.chdir(@ansible_dir) do
-          initialize_command(*command)
-          logger.debug("[foreman_ansible] - Running command '#{command.join(' ')}'")
-        end
-      end
-
-      def kill
-        publish_data('== TASK ABORTED BY USER ==', 'stdout')
-        publish_exit_status(1)
-        ::Process.kill('SIGTERM', @command_pid)
-        close
-      end
-
-      def close
-        super
-        FileUtils.remove_entry(@working_dir) if @tmp_working_dir
-      end
-
-      private
-
-      def write_inventory
-        ensure_directory(File.dirname(inventory_file))
-        File.write(inventory_file, JSON.dump(@inventory))
-      end
-
-      def write_playbook
-        ensure_directory(File.dirname(playbook_file))
-        File.write(playbook_file, @playbook)
-      end
-
-      def inventory_file
-        File.join(@working_dir, 'foreman-inventories', id)
-      end
-
-      def playbook_file
-        File.join(@working_dir, "foreman-playbook-#{id}.yml")
-      end
-
-      def events_dir
-        File.join(@working_dir, 'events', id.to_s)
-      end
-
-      def ensure_directory(path)
-        if File.exist?(path)
-          raise "#{path} expected to be a directory" unless File.directory?(path)
-        else
-          FileUtils.mkdir_p(path)
-        end
-        path
-      end
-
-      def initialize_dirs
-        settings = Proxy::Ansible::Plugin.settings
-        initialize_working_dir(settings[:working_dir])
-        initialize_ansible_dir(settings[:ansible_dir])
-      end
-
-      def initialize_working_dir(working_dir)
-        if working_dir.nil?
-          @working_dir = Dir.mktmpdir
-          @tmp_working_dir = true
-        else
-          @working_dir = File.expand_path(working_dir)
-        end
-      end
-
-      def initialize_ansible_dir(ansible_dir)
-        raise "Ansible dir #{ansible_dir} does not exist" unless
-          !ansible_dir.nil? && File.exist?(ansible_dir)
-        @ansible_dir = ansible_dir
-      end
-
-      def unknown_hosts
-        @inventory['all']['hosts'].select do |host|
-          Net::SSH::KnownHosts.search_for(host).empty?
-        end
-      end
-
-      def add_to_known_hosts(host)
-        logger.warn("[foreman_ansible] - Host #{host} not found in known_hosts")
-        Net::SSH::Transport::Session.new(host).host_keys.each do |host_key|
-          Net::SSH::KnownHosts.add(host, host_key)
-        end
-        logger.warn("[foreman_ansible] - Added host key #{host} to known_hosts")
-      rescue StandardError => e
-        logger.error('[foreman_ansible] - Failed to save host key for '\
-          "#{host}: #{e}")
-      end
-
-      def rebuild_secrets(inventory, secrets)
-        inventory['all']['hosts'].each do |name|
-          per_host = secrets['per-host'][name]
-
-          new_secrets = {
-            'ansible_password' => inventory['ssh_password'] || per_host['ansible_password'],
-            'ansible_become_password' => inventory['effective_user_password'] || per_host['ansible_become_password']
-          }
-          inventory['_meta']['hostvars'][name].update(new_secrets)
-        end
-
-        inventory
-      end
-    end
-  end
-end

data/lib/smart_proxy_ansible/task_launcher/playbook.rb

@@ -1,25 +0,0 @@
-require 'smart_proxy_dynflow/action/runner'
-
-module Proxy::Ansible
-  module TaskLauncher
-    class Playbook < Proxy::Dynflow::TaskLauncher::Batch
-      class PlaybookRunnerAction < Proxy::Dynflow::Action::Runner
-        def initiate_runner
-          additional_options = {
-            :step_id => run_step_id,
-            :uuid => execution_plan_id
-          }
-          ::Proxy::Ansible::RemoteExecutionCore::AnsibleRunner.new(
-            input.merge(additional_options),
-            :suspended_action => suspended_action
-          )
-        end
-      end
-
-      def child_launcher(parent)
-        ::Proxy::Dynflow::TaskLauncher::Single.new(world, callback, :parent => parent,
-                                                                    :action_class_override => PlaybookRunnerAction)
-      end
-    end
-  end
-end