choria-colt 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d2f9c64ee41ebc49a88deff71b470baab7c4e83b921102518969fe2a35b77f1
-  data.tar.gz: 8f69562d201ff4addc3b44aa693d508ceca1a1623455d2ac57b273e59cd300f1
+  metadata.gz: fbb608f3080843c72765bb15bb6b375d1e3e02370c59b00b1d34fb9d5c02e57c
+  data.tar.gz: cc292639012cdde4ea87d35522f207e792e2727275d6c331bebfe7332948f905
 SHA512:
-  metadata.gz: 8fd9cc5e888843d6c8954fe7d2543b4496fe861354162f499831f4ebe7cc8ccd3def73604c6d44e6f83273f71eb8364fc1b01c99581c16e8b2635adbee071aea
-  data.tar.gz: 5a912066de4c960ffc85debf3cc53a678ac3312f2e75daec0d999f50af7f24215f1347b6c4984de55db9f3bb357075486310ab11db685e8f866c61a06049940f
+  metadata.gz: e3ef3078f635f8223eb3e2fbc9d5d043cdb536f0acc34008f61c80a4efc5af760b23021bb1c023a3aeed402903c1f59e66628e78e7087b398cb42bd8bf62d97a
+  data.tar.gz: 5b05929aae9c9414c1af2daa68fe7f17d5c8da758282e0d30ec9e27e69ba919eb9e2ec4ab59469aa94d7c7244e45e2f8f3c5a6ee4a9f35eafcd7c09ba7b0a9ae

data/.rubocop.yml

@@ -11,6 +11,7 @@ Metrics/AbcSize:
   Max: 20
 
 Metrics/ClassLength:
+  Max: 120
   Exclude:
     - 'lib/choria/colt/cli.rb'
 

data/.rubocop_todo.yml

@@ -1,17 +1,17 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-04-07 15:43:34 UTC using RuboCop version 1.26.0.
+# on 2022-04-28 09:34:37 UTC using RuboCop version 1.26.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
 # Offense count: 1
-# Configuration parameters: IgnoredMethods.
-Metrics/CyclomaticComplexity:
-  Max: 9
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 22
 
 # Offense count: 1
 # Configuration parameters: IgnoredMethods.
-Metrics/PerceivedComplexity:
-  Max: 9
+Metrics/CyclomaticComplexity:
+  Max: 8

data/lib/choria/colt/cli/formatter.rb

@@ -5,6 +5,33 @@ module Choria
   class Colt
     class CLI < Thor
       class Formatter
+        module Result
+          def sender
+            self[:sender]
+          end
+
+          def exitcode
+            dig(:data, :exitcode)
+          end
+
+          def ok?
+            exitcode&.zero?
+          end
+
+          def runtime
+            dig(:data, :runtime)
+          end
+
+          # CLI
+          def output
+            if dig(:result, :_output).nil?
+              JSON.pretty_generate(self[:result]).split("\n")
+            else
+              dig(:result, :_output)
+            end
+          end
+        end
+
         attr_reader :pastel
 
         def initialize(colored:)
@@ -13,27 +40,36 @@ module Choria
         end
 
         def process_result(result)
-          return process_error(result) unless result.dig(:data, :exitcode)&.zero?
+          result.extend Formatter::Result
 
-          process_success(result)
+          case result[:statuscode]
+          when 0
+            # 0 	OK
+            process_success(result)
+          when 1
+            # 1 	OK, failed. All the data parsed ok, we have a action matching the request but the requested action could not be completed. 	RPCAborted
+            process_error(result) # unless result.ok?
+          else
+            # 2 	Unknown action 	UnknownRPCAction
+            # 3 	Missing data 	MissingRPCData
+            # 4 	Invalid data 	InvalidRPCData
+            # 5 	Other error
+            process_rpc_error(result)
+          end
         end
 
         def process_success(result)
-          output_lines = [
-            "#{pastel.host(result[:sender]).ljust(60, ' ')}duration: #{pastel.bright_white format('%.2fs', result[:data][:runtime])}",
-          ]
-
-          output_lines += if result.dig(:result, :_output).nil?
-                            JSON.pretty_generate(result[:result]).split("\n").map { |line| "  #{line}" }
-                          else
-                            result.dig(:result, :_output).map { |line| "  #{line}" }
-                          end
+          host = format_host(result, "#{pastel.bright_green '√'} ")
+          headline = "#{pastel.on_green ' '} "
 
-          output_lines.join("\n")
+          [
+            host,
+            result.output.map { |line| "#{headline}#{line}" },
+          ].flatten.join("\n")
         end
 
         def process_error(result) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-          host = "#{pastel.bright_red '⨯'} #{pastel.host(result[:sender]).ljust(60, ' ')}duration: #{pastel.bright_white format('%.2fs', result[:data][:runtime])}"
+          host = format_host(result, "#{pastel.bright_red '⨯'} ")
           output = result.dig(:result, '_output')
           error_details = JSON.pretty_generate(result.dig(:result, :_error, :details)).split "\n"
           error_description = [
@@ -61,6 +97,26 @@ module Choria
             ].flatten.map { |line| "#{headline}#{line}" },
           ].flatten.join("\n")
         end
+
+        def process_rpc_error(result)
+          host = "#{pastel.bright_red '⨯'} #{pastel.host(result.sender)}"
+          headline = "#{pastel.on_red ' '} "
+
+          [
+            host,
+            "#{headline}#{pastel.bright_red "RPC error (#{result[:statuscode]})"}: #{pastel.bright_white result[:statusmsg]}",
+          ].join("\n")
+        end
+
+        private
+
+        def format_duration(result)
+          result.runtime.nil? ? '' : "duration: #{pastel.bright_white format('%.2fs', result.runtime)}"
+        end
+
+        def format_host(result, headline)
+          "#{headline}#{pastel.host(result.sender).ljust(60, ' ')}#{format_duration(result)}"
+        end
       end
     end
   end

data/lib/choria/colt/cli.rb

@@ -27,6 +27,7 @@ module Choria
                desc: 'Select the targets which have the specified Puppet classes.'
         def run(*args) # rubocop:disable Metrics/AbcSize
           input = extract_task_parameters_from_args(args)
+          targets = extract_targets_from_options
 
           raise Thor::Error, 'Task name is required' if args.empty?
           raise Thor::Error, "Too many arguments: #{args}" unless args.count == 1
@@ -35,8 +36,7 @@ module Choria
 
           task_name = args.shift
 
-          targets = options['targets']&.split(',')
-          targets = nil if options['targets'] == 'all'
+          logger.debug "Targets: #{targets}"
 
           targets_with_classes = options['targets_with_classes']&.split(',')
 
@@ -45,8 +45,8 @@ module Choria
           end
 
           File.write 'last_run.json', JSON.pretty_generate(results)
-        rescue Choria::Orchestrator::Error => e
-          raise Thor::Error, "#{e.class}: #{e}"
+        rescue Choria::Orchestrator::Error
+          # This error is already logged and displayed.
         end
 
         desc 'show [task name] [options]', 'Show available tasks and task documentation'
@@ -90,8 +90,8 @@ module Choria
           end
 
           File.write 'last_run.json', JSON.pretty_generate(results)
-        rescue Choria::Orchestrator::Error => e
-          raise Thor::Error, "#{e.class}: #{e}"
+        rescue Choria::Orchestrator::Error
+          # This error is already logged and displayed.
         end
 
         no_commands do # rubocop:disable Metrics/BlockLength
@@ -130,6 +130,16 @@ module Choria
             end.to_h
           end
 
+          def extract_targets_from_options
+            return nil if options['targets'] == 'all'
+
+            targets = options['targets']&.split(',')
+            targets&.map do |t|
+              t = File.read(t.sub(/^@/, '')).lines.map(&:strip) if t.start_with? '@'
+              t
+            end&.flatten
+          end
+
           def show_tasks_summary(tasks)
             tasks.reject! { |_task, metadata| metadata['metadata']['private'] }
 

data/lib/choria/colt/data_structurer.rb

@@ -2,6 +2,8 @@ module Choria
   class Colt
     module DataStructurer
       def self.structure(res) # rubocop:disable Metrics/AbcSize
+        return res unless [0, 1].include? res[:statuscode]
+
         # data.stdout seems to always be JSON, so parse it once.
         res[:result] = JSON.parse res[:data][:stdout]
         res[:data].delete :stdout

data/lib/choria/colt/version.rb

@@ -2,6 +2,6 @@
 
 module Choria
   class Colt
-    VERSION = '0.4.0'
+    VERSION = '0.5.0'
   end
 end

data/lib/choria/orchestrator/task/result_set.rb

@@ -0,0 +1,28 @@
+require 'choria/colt/data_structurer'
+
+module Choria
+  class Orchestrator
+    class Task
+      class ResultSet
+        attr_reader :results
+
+        def initialize(on_result:)
+          @results = []
+          @on_result = on_result
+        end
+
+        def integrate_rpc_error(rpc_error)
+          result = rpc_error[:body]
+          result[:sender] = rpc_error[:senderid]
+          integrate_result(result)
+        end
+
+        def integrate_result(result)
+          structured_result = Choria::Colt::DataStructurer.structure(result).with_indifferent_access
+          @results << structured_result
+          @on_result&.call(structured_result)
+        end
+      end
+    end
+  end
+end

data/lib/choria/orchestrator/task.rb

@@ -1,4 +1,4 @@
-require 'choria/colt/data_structurer'
+require_relative 'task/result_set'
 
 require 'active_support'
 require 'active_support/core_ext/hash/indifferent_access'
@@ -7,17 +7,15 @@ module Choria
   class Orchestrator
     class Task
       class Error < Orchestrator::Error; end
+      class NoNodesLeftError < Error; end
 
-      attr_reader :id, :name, :input, :environment, :rpc_results, :results
-      attr_accessor :rpc_responses
+      attr_reader :id, :name, :input, :environment
 
       def initialize(orchestrator:, id: nil, name: nil, input: {}, environment: 'production')
         @id = id
         @name = name
         @environment = environment
         @orchestrator = orchestrator
-        @results = []
-
         return if @name.nil?
 
         @input = default_input.merge input
@@ -33,71 +31,82 @@ module Choria
         metadata['files'].to_json
       end
 
-      def wait # rubocop:disable Metrics/AbcSize
-        if @id.nil?
-          rpc_responses_ok, rpc_responses_error = rpc_responses.partition { |res| (res[:body][:statuscode]).zero? }
-          rpc_responses_error.each do |res|
-            logger.error "Task request failed on '#{res[:senderid]}':\n#{pp res}"
-          end
+      def results
+        result_set.results
+      end
 
-          task_ids = rpc_responses_ok.map { |res| res[:body][:data][:task_id] }.uniq
+      def run
+        raise Error, 'Unable to run a task by ID' if name.nil?
 
-          raise NotImplementedError, "Multiple task IDs: #{task_ids}" unless task_ids.count == 1
+        @pending_targets = rpc_client.discover
+        _download
+        _run_no_wait
+      end
 
-          @id = task_ids.first
-        end
+      def wait
+        raise Error, 'Task ID is required!' if @id.nil?
 
-        wait_results
+        logger.info "Waiting task #{@id} results…"
+        @rpc_results = []
+        loop do
+          self.rpc_results = rpc_client.task_status(task_id: @id).map(&:results)
+          break if @pending_targets.empty?
+        end
       end
 
       def on_result(&block)
-        @on_result = lambda { |result|
-          block.call(result)
-        }
+        @on_result = ->(result) { block.call(result) }
       end
 
       private
 
-      def rpc_results=(results)
-        new_result_hosts = (results.map { |res| res[:sender] }) - (@results.map { |res| res[:sender] })
-
-        new_result_hosts.each do |host|
-          result = results.find { |res| res[:sender] == host }
-
-          next unless result[:data][:exitcode] != -1
-
-          logger.debug "New result for task ##{@id}: #{result}"
-          structured_result = Choria::Colt::DataStructurer.structure(result).with_indifferent_access
-
-          @on_result&.call(structured_result)
+      def result_set
+        @result_set ||= ResultSet.new(on_result: @on_result)
+      end
 
-          @results << structured_result
+      def rpc_results=(results)
+        pending_results, completed_results = results.partition { |res| res[:data][:exitcode] == -1 }
+        @pending_targets ||= pending_results.map { |res| res[:sender] }
+
+        new_results = completed_results.select { |res| @pending_targets.include? res[:sender] }
+        new_results.each do |res|
+          logger.debug "New result for task ##{@id}: #{res}"
+          result_set.integrate_result(res)
+          @pending_targets.delete res[:sender]
         end
-
-        @rpc_results = results
       end
 
-      def wait_results
-        raise 'Task ID is required!' if @id.nil?
-
-        logger.info "Waiting task #{@id} results…"
-
-        @results = []
-        @rpc_results = []
+      def process_rpc_response(rpc_response)
+        rpc_response.extend Orchestrator::RpcResponse
+        logger.debug "  RPC Response: '#{rpc_response}'"
+        return unless rpc_response.rpc_error?
 
-        loop do
-          self.rpc_results = @orchestrator.rpc_client.task_status(task_id: @id).map(&:results)
+        @pending_targets.delete rpc_response.sender
+        result_set.integrate_rpc_error(rpc_response)
+      end
 
-          break if terminated?
+      def _download
+        logger.info "Downloading task '#{name}' on #{rpc_client.discover.size} nodes…"
+        rpc_client.download(task: name, files: files, verbose: false) do |rpc_response|
+          process_rpc_response(rpc_response)
         end
+
+        raise NoNodesLeftError, "No nodes left to continue after 'download' action" if @pending_targets.empty?
       end
 
-      def terminated?
-        @rpc_results.each do |result|
-          return false if result[:data][:exitcode] == -1
+      def _run_no_wait # rubocop:disable Metrics/AbcSize
+        logger.info "Starting task '#{name}' on #{rpc_client.discover.size} nodes…"
+        task_ids = []
+        rpc_client.run_no_wait(task: name, files: files, input: input.to_json, verbose: false) do |rpc_response|
+          process_rpc_response(rpc_response)
+          task_ids << rpc_response.task_id
         end
+        raise NoNodesLeftError, "No nodes left to continue after 'run_no_wait' action" if @pending_targets.empty?
 
-        true
+        task_ids.uniq!
+        raise NotImplementedError, "Multiple task IDs: #{task_ids}" unless task_ids.count == 1
+
+        @id = task_ids.first
       end
 
       def _metadata
@@ -122,6 +131,10 @@ module Choria
       def logger
         @orchestrator.logger
       end
+
+      def rpc_client
+        @orchestrator.rpc_client
+      end
     end
   end
 end

data/lib/choria/orchestrator.rb

@@ -6,6 +6,24 @@ module Choria
     class Error < StandardError; end
     class DiscoverError < Error; end
 
+    module RpcResponse
+      def sender
+        self[:senderid]
+      end
+
+      def rpc_error?
+        !rpc_success?
+      end
+
+      def rpc_success?
+        [0, 1].include? self[:body][:statuscode]
+      end
+
+      def task_id
+        self[:body][:data][:task_id]
+      end
+    end
+
     include MCollective::RPC
 
     attr_reader :logger
@@ -33,51 +51,13 @@ module Choria
       end
 
       logger.info 'Discovering targets…'
-      raise DiscoverError, 'No request sent, no node discovered' if rpc_client.discover.size.zero?
+      raise DiscoverError, 'No requests sent, no nodes discovered' if rpc_client.discover.size.zero?
 
-      logger.info "Downloading task '#{task.name}' on #{rpc_client.discover.size} nodes…"
-      rpc_client.download(task: task.name, files: task.files, verbose: verbose)
-
-      responses = []
-      logger.info "Starting task '#{task.name}' on #{rpc_client.discover.size} nodes…"
-      rpc_client.run_no_wait(task: task.name, files: task.files, input: task.input.to_json, verbose: verbose) do |response|
-        logger.debug "  Response: '#{response}'"
-        responses << response
-      end
-
-      # TODO: Include stats in logs when logger will be available (see MCollective::RPC#printrpcstats)
-
-      task.rpc_responses = responses
-    end
-
-    def validate_rpc_result(result)
-      raise Error, "The RPC agent returned an error: #{result[:statusmsg]}" unless (result[:statuscode]).zero?
+      task.run
     end
 
     def rpc_client
-      @rpc_client ||= rpcclient('bolt_tasks', options: rpc_options)
-    end
-
-    private
-
-    def rpc_options
-      {
-        verbose: false,
-        disctimeout: nil,
-        timeout: 5,
-        config: '/etc/choria/client.conf',
-        collective: 'mcollective',
-        discovery_method: nil,
-        discovery_options: [],
-        filter: {
-          'fact' => [], 'cf_class' => [], 'agent' => [], 'identity' => [], 'compound' => []
-        },
-        progress_bar: false,
-        mcollective_limit_targets: false,
-        batch_size: nil,
-        batch_sleep_time: 1,
-        output_format: :json,
-      }
+      @rpc_client ||= rpcclient('bolt_tasks', options: {})
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: choria-colt
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Romuald Conty
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-04-25 00:00:00.000000000 Z
+date: 2022-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -218,6 +218,7 @@ files:
 - lib/choria/colt/version.rb
 - lib/choria/orchestrator.rb
 - lib/choria/orchestrator/task.rb
+- lib/choria/orchestrator/task/result_set.rb
 - sig/choria/colt.rbs
 homepage: https://github.com/opus-codium/choria-colt
 licenses: []