smart_proxy_ansible 3.2.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ae7dee62b09f9ea49553d1e2efab2b3f535940be697cf132af28fe0096da30f
-  data.tar.gz: 1997fbe1d05cedb27ee1a8c05d38b2644a25b84e0fbd11a93f201c8f81647643
+  metadata.gz: 1139197a8fd9b944e4bec3a0dcf1ed18951bffb88457663927ba427ca4f1b372
+  data.tar.gz: 0b774027ceedca7a36714f7725704261996efdbfe835f08bcc6fbe6a7beabc22
 SHA512:
-  metadata.gz: 30b1c1acfd7ca623ab1bc8ccbbd83cfa0b31c9c99516ea8414e500470fa67b9755a579f3f87d77abaca565f9b31ff1503b335224f7664c40bda8d3e7e4707ddd
-  data.tar.gz: 9135ef0523d5246271da5b559384d7a53b952a9da67301c00b975d3544e365fd73af156b1cdd77989af75ea98990b1d78c33cd8f7ad040dfd0fa62cfb6103645
+  metadata.gz: 2e5535a4a9d2100e9d68321be99e23b331100e4bb8b2b1e4d7104ab7216064db9f9361c68ac6989fe7197217441fe574087278606c848be65a2362634001d818
+  data.tar.gz: 2704d69243f82a099282d5289de914bb754826a9d32a92a93d1ad0c747f512ebe006ac2869554c26616126e3085233a6a166d0670130b8844d0e52e89f4182f5

data/bin/json_inventory.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# An inventory script to load the inventory from JSON file.
+#
+
+if [ -z "$JSON_INVENTORY_FILE" ]; then
+   echo "JSON_INVENTORY_FILE not specified"
+   exit 1
+fi
+
+cat $JSON_INVENTORY_FILE

data/lib/smart_proxy_ansible/api.rb

@@ -28,15 +28,23 @@ module Proxy
         {}.to_json
       end
 
+      get '/playbooks_names' do
+        PlaybooksReader.playbooks_names.to_json
+      end
+
+      get '/playbooks/:playbooks_names?' do
+        PlaybooksReader.playbooks(params[:playbooks_names]).to_json
+      end
+
       private
 
       def extract_variables(role_name)
         variables = {}
         role_name_parts = role_name.split('.')
         if role_name_parts.count == 3
-          RolesReader.collections_paths.split(':').each do |path|
-            variables[role_name] ||= VariablesExtractor
-                                   .extract_variables("#{path}/ansible_collections/#{role_name_parts[0]}/#{role_name_parts[1]}/roles/#{role_name_parts[2]}")
+          ReaderHelper.collections_paths.split(':').each do |path|
+            variables[role_name] = VariablesExtractor
+                                   .extract_variables("#{path}/ansible_collections/#{role_name_parts[0]}/#{role_name_parts[1]}/roles/#{role_name_parts[2]}") if variables[role_name].nil? || variables[role_name].empty?
           end
         else
           RolesReader.roles_path.split(':').each do |path|

data/lib/smart_proxy_ansible/configuration_loader.rb

@@ -0,0 +1,20 @@
+module Proxy::Ansible
+  class ConfigurationLoader
+    def load_classes
+      require 'smart_proxy_dynflow'
+      require 'smart_proxy_dynflow/continuous_output'
+      require 'smart_proxy_ansible/task_launcher/ansible_runner'
+      require 'smart_proxy_ansible/task_launcher/playbook'
+      require 'smart_proxy_ansible/actions'
+      require 'smart_proxy_ansible/remote_execution_core/ansible_runner'
+      require 'smart_proxy_ansible/runner/ansible_runner'
+      require 'smart_proxy_ansible/runner/command_creator'
+      require 'smart_proxy_ansible/runner/playbook'
+
+      Proxy::Dynflow::TaskLauncherRegistry.register('ansible-runner',
+                                                    TaskLauncher::AnsibleRunner)
+      Proxy::Dynflow::TaskLauncherRegistry.register('ansible-playbook',
+                                                    TaskLauncher::Playbook)
+    end
+  end
+end

data/lib/smart_proxy_ansible/exception.rb

@@ -37,5 +37,8 @@ module Proxy
     class ReadConfigFileException < Proxy::Ansible::Exception; end
     class ReadRolesException < Proxy::Ansible::Exception; end
     class ReadVariablesException < Proxy::Ansible::Exception; end
+    class NotExistingWorkingDirException < Proxy::Ansible::Exception; end
+    class ReadPlaybooksNamesException < Proxy::Ansible::Exception; end
+    class ReadPlaybooksException < Proxy::Ansible::Exception; end
   end
 end

data/lib/smart_proxy_ansible/playbooks_reader.rb

@@ -0,0 +1,37 @@
+module Proxy
+  module Ansible
+    # Implements the logic needed to read the playbooks and associated information
+    class PlaybooksReader
+      class << self
+        def playbooks_names
+          ReaderHelper.collections_paths.split(':').flat_map { |path| get_playbooks_names(path) }
+        end
+
+        def playbooks(playbooks_to_import)
+          ReaderHelper.collections_paths.split(':').reduce([]) do |playbooks, path|
+            playbooks.concat(read_collection_playbooks(path, playbooks_to_import))
+          end
+        end
+
+        def get_playbooks_names(collections_path)
+          Dir.glob("#{collections_path}/ansible_collections/*/*/playbooks/*").map do |path|
+            ReaderHelper.playbook_or_role_full_name(path)
+          end
+        end
+
+        def read_collection_playbooks(collections_path, playbooks_to_import = nil)
+          Dir.glob("#{collections_path}/ansible_collections/*/*/playbooks/*").map do |path|
+            name = ReaderHelper.playbook_or_role_full_name(path)
+            {
+              name: name,
+              playbooks_content: File.readlines(path)
+            } if playbooks_to_import.nil? || playbooks_to_import.include?(name)
+          end.compact
+        rescue Errno::ENOENT, Errno::EACCES => e
+          message = "Could not read Ansible playbooks #{collections_path} - #{e.message}"
+          raise ReadPlaybooksException, message
+        end
+      end
+    end
+  end
+end

data/lib/smart_proxy_ansible/plugin.rb

@@ -8,22 +8,9 @@ module Proxy
       default_settings :ansible_dir => Dir.home
                        # :working_dir => nil
 
-      after_activation do
-        require 'smart_proxy_dynflow'
-        require 'smart_proxy_dynflow/continuous_output'
-        require 'smart_proxy_ansible/task_launcher/ansible_runner'
-        require 'smart_proxy_ansible/task_launcher/playbook'
-        require 'smart_proxy_ansible/actions'
-        require 'smart_proxy_ansible/remote_execution_core/ansible_runner'
-        require 'smart_proxy_ansible/runner/ansible_runner'
-        require 'smart_proxy_ansible/runner/command_creator'
-        require 'smart_proxy_ansible/runner/playbook'
-
-        Proxy::Dynflow::TaskLauncherRegistry.register('ansible-runner',
-          TaskLauncher::AnsibleRunner)
-        Proxy::Dynflow::TaskLauncherRegistry.register('ansible-playbook',
-          TaskLauncher::Playbook)
-      end
+      load_classes ::Proxy::Ansible::ConfigurationLoader
+      load_validators :validate_settings => ::Proxy::Ansible::ValidateSettings
+      validate :validate!, :validate_settings => nil
     end
   end
 end

data/lib/smart_proxy_ansible/reader_helper.rb

@@ -0,0 +1,44 @@
+module Proxy
+  module Ansible
+    # Helper for Playbooks Reader
+    class ReaderHelper
+      class << self
+        DEFAULT_COLLECTIONS_PATHS = '/etc/ansible/collections:/usr/share/ansible/collections'.freeze
+        DEFAULT_CONFIG_FILE = '/etc/ansible/ansible.cfg'.freeze
+
+        def collections_paths
+          config_path(path_from_config('collections_paths'), DEFAULT_COLLECTIONS_PATHS)
+        end
+
+        def config_path(config_line, default)
+          return default if config_line.empty?
+
+          config_line_key = config_line.first.split('=').first.strip
+          # In case of commented roles_path key "#roles_path" or #collections_paths, return default
+          return default if ['#roles_path', '#collections_paths'].include?(config_line_key)
+
+          config_line.first.split('=').last.strip
+        end
+
+        def path_from_config(config_key)
+          File.readlines(DEFAULT_CONFIG_FILE).select do |line|
+            line =~ /^\s*#{config_key}/
+          end
+        rescue Errno::ENOENT, Errno::EACCES => e
+          RolesReader.logger.debug(e.backtrace)
+          message = "Could not read Ansible config file #{DEFAULT_CONFIG_FILE} - #{e.message}"
+          raise ReadConfigFileException.new(message), message
+        end
+
+        def playbook_or_role_full_name(path)
+          parts = path.split('/')
+          playbook = parts.pop
+          parts.pop
+          collection = parts.pop
+          author = parts.pop
+          "#{author}.#{collection}.#{playbook}"
+        end
+      end
+    end
+  end
+end

data/lib/smart_proxy_ansible/roles_reader.rb

@@ -5,33 +5,16 @@ module Proxy
     # Implements the logic needed to read the roles and associated information
     class RolesReader
       class << self
-        DEFAULT_CONFIG_FILE = '/etc/ansible/ansible.cfg'.freeze
         DEFAULT_ROLES_PATH = '/etc/ansible/roles:/usr/share/ansible/roles'.freeze
-        DEFAULT_COLLECTIONS_PATHS = '/etc/ansible/collections:/usr/share/ansible/collections'.freeze
 
         def list_roles
           roles = roles_path.split(':').map { |path| read_roles(path) }.flatten
-          collection_roles = collections_paths.split(':').map { |path| read_collection_roles(path) }.flatten
+          collection_roles = ReaderHelper.collections_paths.split(':').map { |path| read_collection_roles(path) }.flatten
           roles + collection_roles
         end
 
         def roles_path
-          config_path(path_from_config('roles_path'), DEFAULT_ROLES_PATH)
-        end
-
-        def collections_paths
-          config_path(path_from_config('collections_paths'), DEFAULT_COLLECTIONS_PATHS)
-        end
-
-        def config_path(config_line, default)
-          # Default to /etc/ansible/roles if config_line is empty
-          return default if config_line.empty?
-
-          config_line_key = config_line.first.split('=').first.strip
-          # In case of commented roles_path key "#roles_path" or #collections_paths, return default
-          return default if ['#roles_path', '#collections_paths'].include?(config_line_key)
-
-          config_line.first.split('=').last.strip
+          ReaderHelper.config_path(ReaderHelper.path_from_config('roles_path'), DEFAULT_ROLES_PATH)
         end
 
         def logger
@@ -58,33 +41,17 @@ module Proxy
 
         def glob_path(path)
           Dir.glob path
-
         end
 
         def read_collection_roles(collections_path)
           Dir.glob("#{collections_path}/ansible_collections/*/*/roles/*").map do |path|
-            parts = path.split('/')
-            role = parts.pop
-            parts.pop
-            collection = parts.pop
-            author = parts.pop
-            "#{author}.#{collection}.#{role}"
+            ReaderHelper.playbook_or_role_full_name(path)
           end
         rescue Errno::ENOENT, Errno::EACCES => e
           logger.debug(e.backtrace)
           message = "Could not read Ansible roles #{collections_path} - #{e.message}"
           raise ReadRolesException.new(message), message
         end
-
-        def path_from_config(config_key)
-          File.readlines(DEFAULT_CONFIG_FILE).select do |line|
-            line =~ /^\s*#{config_key}/
-          end
-        rescue Errno::ENOENT, Errno::EACCES => e
-          logger.debug(e.backtrace)
-          message = "Could not read Ansible config file #{DEFAULT_CONFIG_FILE} - #{e.message}"
-          raise ReadConfigFileException.new(message), message
-        end
       end
     end
   end

data/lib/smart_proxy_ansible/runner/ansible_runner.rb

@@ -1,4 +1,5 @@
 require 'shellwords'
+require 'yaml'
 
 require 'smart_proxy_dynflow/runner/command'
 require 'smart_proxy_dynflow/runner/base'
@@ -7,6 +8,7 @@ module Proxy::Ansible
   module Runner
     class AnsibleRunner < ::Proxy::Dynflow::Runner::Parent
       include ::Proxy::Dynflow::Runner::Command
+      attr_reader :execution_timeout_interval, :command_pid
 
       def initialize(input, suspended_action:)
         super input, :suspended_action => suspended_action
@@ -19,19 +21,36 @@ module Proxy::Ansible
         @check_mode = action_input[:check_mode]
         @tags = action_input[:tags]
         @tags_flag = action_input[:tags_flag]
+        @passphrase = action_input['secrets']['key_passphrase']
+        @execution_timeout_interval = action_input[:execution_timeout_interval]
+        @cleanup_working_dirs = action_input.fetch(:cleanup_working_dirs, true)
       end
 
       def start
         prepare_directory_structure
         write_inventory
         write_playbook
+        write_ssh_key if !@passphrase.nil? && !@passphrase.empty?
         start_ansible_runner
       end
 
+      def initialize_command(*command)
+        r, w = IO.pipe
+
+        @command_pid = spawn(*command, :out => w, :err => w, :in => '/dev/null')
+        @command_out = r
+        w.close
+      rescue Errno::ENOENT => e
+        publish_exception("Error running command '#{command.join(' ')}'", e)
+      end
+
       def refresh
-        return unless super
+        super
+        @uuid ||= if (f = Dir["#{@root}/artifacts/*"].first)
+                    File.basename(f)
+                  end
+        return unless @uuid
         @counter ||= 1
-        @uuid ||= File.basename(Dir["#{@root}/artifacts/*"].first)
         job_event_dir = File.join(@root, 'artifacts', @uuid, 'job_events')
         loop do
           files = Dir["#{job_event_dir}/*.json"].map do |file|
@@ -46,9 +65,26 @@ module Proxy::Ansible
         end
       end
 
+      def timeout
+        logger.debug('job timed out')
+        super
+      end
+
+      def timeout_interval
+        execution_timeout_interval
+      end
+
+      def kill
+        ::Process.kill('SIGTERM', @command_pid)
+        publish_exit_status(2)
+        @inventory['all']['hosts'].each { |hostname| @exit_statuses[hostname] = 2 }
+        broadcast_data('Timeout for execution passed, stopping the job', 'stderr')
+        close
+      end
+
       def close
         super
-        FileUtils.remove_entry(@root) if @tmp_working_dir
+        FileUtils.remove_entry(@root) if @tmp_working_dir && Dir.exist?(@root) && @cleanup_working_dirs
       end
 
       private
@@ -85,10 +121,17 @@ module Proxy::Ansible
       def handle_broadcast_data(event)
         log_event("broadcast", event)
         if event['event'] == 'playbook_on_stats'
+          failures = event.dig('event_data', 'failures') || {}
+          unreachable = event.dig('event_data', 'dark') || {}
           header, *rows = event['stdout'].strip.lines.map(&:chomp)
           @outputs.keys.select { |key| key.is_a? String }.each do |host|
             line = rows.find { |row| row =~ /#{host}/ }
             publish_data_for(host, [header, line].join("\n"), 'stdout')
+
+            # If the task has been rescued, it won't consider a failure
+            if @exit_statuses[host].to_i != 0 && failures[host].to_i <= 0 && unreachable[host].to_i <= 0
+              publish_exit_status_for(host, 0)
+            end
           end
         else
           broadcast_data(event['stdout'] + "\n", 'stdout')
@@ -111,6 +154,19 @@ module Proxy::Ansible
         File.write(File.join(@root, 'project', 'playbook.yml'), @playbook)
       end
 
+      def write_ssh_key
+        key_path = File.join(@root, 'env', 'ssh_key')
+        File.symlink(File.expand_path(Proxy::RemoteExecution::Ssh::Plugin.settings[:ssh_identity_key_file]), key_path)
+
+        passwords_path = File.join(@root, 'env', 'passwords')
+        # here we create a secrets file for ansible-runner, which uses the key as regexp
+        # to match line asking for password, given the limitation to match only first 100 chars
+        # and the fact the line contains dynamically created temp directory, the regexp
+        # mentions only things that are always there, such as artifacts directory and the key name
+        secrets = YAML.dump({ "for.*/artifacts/.*/ssh_key_data:" => @passphrase })
+        File.write(passwords_path, secrets, perm: 0o600)
+      end
+
       def start_ansible_runner
         env = {}
         env['FOREMAN_CALLBACK_DISABLE'] = '1' if @rex_command
@@ -133,7 +189,7 @@ module Proxy::Ansible
       end
 
       def check_cmd
-        check_mode? ? '--check' : ''
+        check_mode? ? '"--check"' : ''
       end
 
       def verbosity
@@ -145,11 +201,11 @@ module Proxy::Ansible
       end
 
       def check_mode?
-        @check_mode == true
+        @check_mode == true && @rex_command == false
       end
 
       def prepare_directory_structure
-        inner = %w[inventory project].map { |part| File.join(@root, part) }
+        inner = %w[inventory project env].map { |part| File.join(@root, part) }
         ([@root] + inner).each do |path|
           FileUtils.mkdir_p path
         end

data/lib/smart_proxy_ansible/task_launcher/ansible_runner.rb

@@ -24,7 +24,8 @@ module Proxy::Ansible
       # Discard everything apart from hostname to be able to tell the actions
       # apart when debugging
       def transform_input(input)
-        { 'action_input' => input['action_input'].slice('name') }
+        action_input = super['action_input']
+        { 'action_input' => { 'name' => action_input['name'], :task_id => action_input[:task_id] } }
       end
 
       # def self.input_format

data/lib/smart_proxy_ansible/validate_settings.rb

@@ -0,0 +1,7 @@
+module Proxy::Ansible
+  class ValidateSettings < ::Proxy::PluginValidators::Base
+    def validate!(settings)
+      raise NotExistingWorkingDirException.new("Working directory does not exist") unless settings[:working_dir].nil? || File.directory?(File.expand_path(settings[:working_dir]))
+    end
+  end
+end

data/lib/smart_proxy_ansible/version.rb

@@ -2,6 +2,6 @@ module Proxy
   # Version, this allows the proxy and other plugins know
   # what version of the Ansible plugin is running
   module Ansible
-    VERSION = '3.2.0'
+    VERSION = '3.3.1'
   end
 end

data/lib/smart_proxy_ansible.rb

@@ -4,8 +4,12 @@ module Proxy
   # Basic requires for this plugin
   module Ansible
     require 'smart_proxy_ansible/version'
+    require 'smart_proxy_ansible/configuration_loader'
+    require 'smart_proxy_ansible/validate_settings'
     require 'smart_proxy_ansible/plugin'
     require 'smart_proxy_ansible/roles_reader'
+    require 'smart_proxy_ansible/playbooks_reader'
+    require 'smart_proxy_ansible/reader_helper'
     require 'smart_proxy_ansible/variables_extractor'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_ansible
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Ivan Nečas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2022-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -149,13 +149,17 @@ extra_rdoc_files:
 files:
 - LICENSE
 - README.md
+- bin/json_inventory.sh
 - bundler.d/ansible.rb
 - lib/smart_proxy_ansible.rb
 - lib/smart_proxy_ansible/actions.rb
 - lib/smart_proxy_ansible/api.rb
+- lib/smart_proxy_ansible/configuration_loader.rb
 - lib/smart_proxy_ansible/exception.rb
 - lib/smart_proxy_ansible/http_config.ru
+- lib/smart_proxy_ansible/playbooks_reader.rb
 - lib/smart_proxy_ansible/plugin.rb
+- lib/smart_proxy_ansible/reader_helper.rb
 - lib/smart_proxy_ansible/remote_execution_core/ansible_runner.rb
 - lib/smart_proxy_ansible/roles_reader.rb
 - lib/smart_proxy_ansible/runner/ansible_runner.rb
@@ -163,6 +167,7 @@ files:
 - lib/smart_proxy_ansible/runner/playbook.rb
 - lib/smart_proxy_ansible/task_launcher/ansible_runner.rb
 - lib/smart_proxy_ansible/task_launcher/playbook.rb
+- lib/smart_proxy_ansible/validate_settings.rb
 - lib/smart_proxy_ansible/variables_extractor.rb
 - lib/smart_proxy_ansible/version.rb
 - settings.d/ansible.yml.example
@@ -185,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.4
 signing_key:
 specification_version: 4
 summary: Smart-Proxy Ansible plugin