choria-colt 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e76e667917bf1958b48d45148712fa5e0d6f2c15280bbbda0d4899b0537e9f8
-  data.tar.gz: 17637d3bf6738bef6c28a282aabb1c4ac7680e4b2e8a6f0fe873a27b59079cc1
+  metadata.gz: 5791ce130daf3f0d0300f6ca56fa07f88c6a94e08ff4979d84282526220a3eb4
+  data.tar.gz: d4ce31439d50ba72bff357e13901c4621a1a8f79ce0a8458ed1b0c948e22f7e0
 SHA512:
-  metadata.gz: bacccbdfd46669936f1271d05e4fdceae1864e07ae7f14b260f6de3ec2ed1e90d0c3270f57ea3faaa424b267eab59e1460abcdf4262feec421c6ea510bc811d5
-  data.tar.gz: ad8aaf32f6f71f349d04955cb795c0439c4ecd30103f290da82b3ce7037c6dce330b7e89123a416c120b61bc194c754debfb97412e6e200af8d53264aaba169c
+  metadata.gz: 427751ad2b8a07a0426c74aad97c2f67c2fa27b151f63db96e63d8116c15f39615ae40dd76d6aa3c1f6ad92c95c27219eb312bdb0f43f281b05ab8bd3cd07c12
+  data.tar.gz: bd056cdb49de93481735ba4ece34ed40c48ac7d728c31ec8d0066aa300ecb7bed055f5d541c6973e3459b292475497ce4ef5357ae5cb8e3ec681a66e29b4684f

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## [v0.8.0](https://github.com/opus-codium/choria-colt/tree/v0.8.0) (2022-11-25)
+
+[Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.7.0...v0.8.0)
+
+**Merged pull requests:**
+
+- CLI: Summarize task status results by default [\#33](https://github.com/opus-codium/choria-colt/pull/33) ([neomilium](https://github.com/neomilium))
+- CLI: Always display stderr if available [\#32](https://github.com/opus-codium/choria-colt/pull/32) ([neomilium](https://github.com/neomilium))
+
+## [v0.7.0](https://github.com/opus-codium/choria-colt/tree/v0.7.0) (2022-09-26)
+
+[Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.6.0...v0.7.0)
+
+**Fixed bugs:**
+
+- Task: Fix IDs array processing [\#30](https://github.com/opus-codium/choria-colt/pull/30) ([neomilium](https://github.com/neomilium))
+
+**Merged pull requests:**
+
+- Use ActiveSuport::Cache instead our own Cache mecanism for tasks metadata [\#31](https://github.com/opus-codium/choria-colt/pull/31) ([neomilium](https://github.com/neomilium))
+- Improve robustness when nodes are unhealthy [\#29](https://github.com/opus-codium/choria-colt/pull/29) ([neomilium](https://github.com/neomilium))
+
 ## [v0.6.0](https://github.com/opus-codium/choria-colt/tree/v0.6.0) (2022-05-16)
 
 [Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.5.1...v0.6.0)

data/choria-colt.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'puppet'
   spec.add_dependency 'thor'
   spec.add_dependency 'tty-logger'
+  spec.add_dependency 'tty-progressbar'
 
   # spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'github_changelog_generator'

data/lib/choria/colt/cli/formatter.rb

@@ -22,6 +22,10 @@ module Choria
             dig(:data, :runtime)
           end
 
+          def statuscode
+            self[:statuscode]
+          end
+
           # CLI
           def output
             if dig(:result, :_output).nil?
@@ -30,92 +34,131 @@ module Choria
               dig(:result, :_output)
             end
           end
+
+          def stderr
+            dig(:result, :_stderr)
+          end
         end
 
-        attr_reader :pastel
+        module FormattedResult
+          attr_accessor :pastel
 
-        def initialize(colored:)
-          @pastel = Pastel.new(enabled: colored)
-          pastel.alias_color(:host, :cyan)
-        end
+          def host
+            if statuscode.zero?
+              format_host pastel.bright_green('√ ').to_s
+            else
+              format_host pastel.bright_red('⨯ ').to_s
+            end
+          end
 
-        def process_result(result)
-          result.extend Formatter::Result
+          def content
+            case statuscode
+            when 0
+              # 0 	OK
+              format_success
+            when 1
+              # 1 	OK, failed. All the data parsed ok, we have a action matching the request but the requested action could not be completed. 	RPCAborted
+              format_error
+            else
+              # 2 	Unknown action 	UnknownRPCAction
+              # 3 	Missing data 	MissingRPCData
+              # 4 	Invalid data 	InvalidRPCData
+              # 5 	Other error
+              format_rpc_error
+            end
+          end
 
-          case result[:statuscode]
-          when 0
-            # 0 	OK
-            process_success(result)
-          when 1
-            # 1 	OK, failed. All the data parsed ok, we have a action matching the request but the requested action could not be completed. 	RPCAborted
-            process_error(result) # unless result.ok?
-          else
-            # 2 	Unknown action 	UnknownRPCAction
-            # 3 	Missing data 	MissingRPCData
-            # 4 	Invalid data 	InvalidRPCData
-            # 5 	Other error
-            process_rpc_error(result)
+          def to_s
+            [
+              host,
+              content,
+            ].join("\n")
           end
-        end
 
-        def process_success(result)
-          host = format_host(result, "#{pastel.bright_green '√'} ")
-          headline = "#{pastel.on_green ' '} "
+          private
 
-          [
-            host,
-            result.output.map { |line| "#{headline}#{line}" },
-          ].flatten.join("\n")
-        end
+          def stderr_description
+            if stderr.nil? || stderr.empty?
+              []
+            else
+              [
+                nil,
+                pastel.bright_red('stderr:'),
+                stderr,
+              ]
+            end
+          end
+
+          def output_description
+            if output.nil? || output.empty?
+              []
+            else
+              [
+                nil,
+                pastel.bright_red('output:'),
+                output,
+              ]
+            end
+          end
+
+          def format_duration
+            runtime.nil? ? '' : "duration: #{pastel.bright_white format('%.2fs', runtime)}"
+          end
+
+          def format_host(headline)
+            "#{headline}#{pastel.host(sender).ljust(60, ' ')}#{format_duration}".strip
+          end
+
+          def format_success
+            headline = "#{pastel.on_green ' '} "
+            warning_headline = "#{pastel.on_yellow ' '} "
 
-        def process_error(result) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-          host = format_host(result, "#{pastel.bright_red '⨯'} ")
-          output = result.dig(:result, '_output')
-          error_details = JSON.pretty_generate(result.dig(:result, :_error, :details)).split "\n"
-          error_description = [
-            "#{pastel.bright_red result.dig(:result, :_error, :kind)}: #{pastel.bright_white result.dig(:result, :_error, :msg)}",
-            "  details: #{error_details.shift}",
-            error_details.map { |line| "  #{line}" },
-          ]
-          output_description = if output.nil? || output.empty?
-                                 []
-                               else
-                                 [
-                                   nil,
-                                   pastel.bright_red('output:'),
-                                   output,
-                                 ]
-                               end
-
-          headline = "#{pastel.on_red ' '} "
-
-          [
-            host,
             [
-              error_description,
-              output_description,
-            ].flatten.map { |line| "#{headline}#{line}" },
-          ].flatten.join("\n")
-        end
+              output.map { |line| "#{headline}#{line}" },
+              stderr_description.flatten.map { |line| "#{warning_headline}#{line}" },
+            ].flatten.join("\n")
+          end
+
+          def format_error
+            error_details = JSON.pretty_generate(dig(:result, :_error, :details)).split "\n"
+            error_description = [
+              "#{pastel.bright_red dig(:result, :_error, :kind)}: #{pastel.bright_white dig(:result, :_error, :msg)}",
+              "  details: #{error_details.shift}",
+              error_details.map { |line| "  #{line}" },
+            ]
 
-        def process_rpc_error(result)
-          host = "#{pastel.bright_red '⨯'} #{pastel.host(result.sender)}"
-          headline = "#{pastel.on_red ' '} "
+            headline = "#{pastel.on_red ' '} "
 
-          [
-            host,
-            "#{headline}#{pastel.bright_red "RPC error (#{result[:statuscode]})"}: #{pastel.bright_white result[:statusmsg]}",
-          ].join("\n")
+            [
+              [
+                error_description,
+                output_description,
+                stderr_description,
+              ].flatten.map { |line| "#{headline}#{line}" },
+            ].flatten.join("\n")
+          end
+
+          def format_rpc_error
+            headline = "#{pastel.on_red ' '} "
+
+            [
+              "#{headline}#{pastel.bright_red "RPC error (#{statuscode})"}: #{pastel.bright_white self[:statusmsg]}",
+            ].join("\n")
+          end
         end
 
-        private
+        attr_reader :pastel
 
-        def format_duration(result)
-          result.runtime.nil? ? '' : "duration: #{pastel.bright_white format('%.2fs', result.runtime)}"
+        def initialize(colored:)
+          @pastel = Pastel.new(enabled: colored)
+          pastel.alias_color(:host, :cyan)
         end
 
-        def format_host(result, headline)
-          "#{headline}#{pastel.host(result.sender).ljust(60, ' ')}#{format_duration(result)}"
+        def format(result)
+          result.extend Formatter::Result
+          result.extend Formatter::FormattedResult
+          result.pastel = pastel
+          result
         end
       end
     end

data/lib/choria/colt/cli.rb

@@ -44,7 +44,7 @@ module Choria
 
           environment = options['environment']
           results = colt.run_bolt_task task_name, input: input, targets: targets, targets_with_classes: targets_with_classes, environment: environment do |result|
-            $stdout.puts formatter.process_result(result)
+            $stdout.puts formatter.format(result)
           end
 
           File.write 'last_run.json', JSON.pretty_generate(results)
@@ -68,10 +68,9 @@ module Choria
                default: 'production'
         def show(*tasks_names)
           environment = options['environment']
-          cache_directory = File.expand_path('.cache/colt/tasks')
-          FileUtils.mkdir_p cache_directory
-          cache = Cache.new(path: File.join(cache_directory, "#{environment}.yaml"))
 
+          require 'active_support/cache/file_store'
+          cache = ActiveSupport::Cache::FileStore.new File.expand_path('~/.cache/colt/tasks')
           tasks = colt.tasks(environment: environment, cache: cache)
 
           if tasks_names.empty?
@@ -87,12 +86,24 @@ module Choria
 
           A task ID is required to request Choria services and retrieve results.
         DESC
+        option :style,
+               aliases: ['-S'],
+               desc: "Output style; can be 'continuous' or 'summary'",
+               default: 'summary'
         define_targets_and_filters_options
         def status(task_id)
+          supported_styles = %i[summary continous]
+          raise Thor::Error, "Invalid style: '#{options['style']}' (available: #{supported_styles})" unless supported_styles.include? options['style'].to_sym
+
           targets, targets_with_classes = extract_targets_and_filters_from_options
 
-          results = colt.wait_bolt_task(task_id, targets: targets, targets_with_classes: targets_with_classes) do |result|
-            $stdout.puts formatter.process_result(result)
+          case options['style']
+          when 'continous'
+            results = colt.wait_bolt_task(task_id, targets: targets, targets_with_classes: targets_with_classes) do |result|
+              $stdout.puts formatter.format(result)
+            end
+          when 'summary'
+            show_summarized_status(task_id, targets: targets, targets_with_classes: targets_with_classes)
           end
 
           File.write 'last_run.json', JSON.pretty_generate(results)
@@ -112,6 +123,7 @@ module Choria
                 [:stream, { output: File.open('colt-debug.log', 'a'), level: :debug }],
               ]
               config.metadata = %i[date time]
+              Choria::Colt::Debugger.enabled = true
             end
           end
 
@@ -186,6 +198,33 @@ module Choria
             end.join "\n"
           end
 
+          def summarize(results)
+            results_grouped_by_result_content = results.group_by { |result| result[:result] }
+            results_grouped_by_result_content.each do |_content, grouped_results|
+              # Display each host
+              grouped_results.each do |result|
+                $stdout.puts formatter.format(result).host
+              end
+              # Display the result content
+              content = formatter.format(grouped_results.first).content
+              content = pastel.bright_white '(no output)' if content.nil? || content.empty?
+              $stdout.puts content
+            end
+          end
+
+          def show_summarized_status(task_id, targets:, targets_with_classes:)
+            require 'tty-progressbar'
+
+            bar = TTY::ProgressBar.new('[:bar] :current/:total ET::elapsed ETA::eta :rate/s')
+            results = colt.wait_bolt_task(task_id, targets: targets, targets_with_classes: targets_with_classes) do |_result, count, total_count|
+              bar.update total: total_count
+              bar.current = count
+            end
+
+            puts "\nSummary:"
+            summarize(results)
+          end
+
           def pastel
             @pastel ||= _pastel
           end

data/lib/choria/colt/data_structurer.rb

@@ -4,22 +4,29 @@ module Choria
       def self.structure(res) # rubocop:disable Metrics/AbcSize
         return res unless [0, 1].include? res[:statuscode]
 
+        # If data is empty, status message is an RPC error
+        if res[:data].empty?
+          res[:result] = {
+            _error: {
+              kind: 'choria/rpc',
+              msg: res[:statusmsg],
+            },
+          }
+          return res
+        end
+
         # data.stdout seems to always be JSON, so parse it once.
-        res[:result] = JSON.parse res[:data][:stdout]
+        res[:result] = JSON.parse res[:data][:stdout] unless res.dig(:data, :stdout).nil?
         res[:data].delete :stdout
 
         # On one side, data.stderr is filled by the remote execution stderr.
         # On the other side, error description is in JSON (ie. '_error')
         # So merge data.stderr in '_error'.'details'
-        unless res[:data][:stderr].empty?
-          raise NotImplementedError, 'What to do when res[:data][:stderr] contains something?' if res[:result]['_error'].empty?
-
-          res[:result]['_error']['details'].merge!({ 'stderr' => res[:data][:stderr].split("\n") })
-        end
+        res[:result]['_stderr'] = res[:data][:stderr].split("\n") unless res.dig(:data, :stderr).nil? || res[:data][:stderr].empty?
         res[:data].delete :stderr
 
         # Convert '_output' (ie. stdout) lines into array
-        res[:result]['_output'] = res[:result]['_output'].split("\n") unless res[:result]['_output'].nil?
+        res[:result]['_output'] = res[:result]['_output'].split("\n") unless res.dig(:result, '_output').nil?
 
         res
       end

data/lib/choria/colt/debugger.rb

@@ -0,0 +1,28 @@
+module Choria
+  class Colt
+    module Debugger
+      class << self
+        attr_writer :enabled
+
+        def enabled
+          @enabled ||= false
+        end
+
+        def root_directory
+          'colt-debug'
+        end
+
+        # This method is helpful to grab raw content to be used as test fixture
+        # To do so, copy the generated result set (ie. directory) in relevant fixture directory (e.g. `spec/fixtures/orchestrator/task/result_sets`)
+        def save_file(result_set:, filename:, content:)
+          directory = File.join Colt::Debugger.root_directory, result_set
+          FileUtils.mkdir_p directory
+          path = File.join directory, filename
+          File.write(path, content)
+
+          path
+        end
+      end
+    end
+  end
+end

data/lib/choria/colt/version.rb

@@ -2,6 +2,6 @@
 
 module Choria
   class Colt
-    VERSION = '0.6.0'
+    VERSION = '0.8.0'
   end
 end

data/lib/choria/colt.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
-require 'choria/colt/cache'
 require 'choria/colt/version'
 require 'choria/orchestrator'
 require 'choria/orchestrator/task'
 
 require 'logger'
 
+require 'active_support/cache/memory_store'
+
 module Choria
   class Colt
     class Error < StandardError; end
@@ -45,32 +46,30 @@ module Choria
       raise
     end
 
-    def tasks(environment:, cache: nil)
+    def tasks(environment:, cache: nil, force_cache_refresh: false)
+      cache ||= ActiveSupport::Cache::MemoryStore.new
+
       tasks_names = orchestrator.tasks_support.tasks(environment).map do |task|
         task['name']
       end
 
-      return tasks_metadata(tasks_names, environment) if cache.nil?
-
-      cached_tasks = cache.load
-      return cached_tasks if cache.clean? && cached_tasks.keys.sort == tasks_names.sort
-
-      updated_tasks = tasks_metadata(tasks_names, environment)
-      cache.save updated_tasks
-
-      updated_tasks
+      tasks_names.map do |task_name|
+        metadata = cache.fetch(task_name, force: force_cache_refresh) do
+          task_metadata(task_name, environment)
+        end
+        [task_name, metadata]
+      end.to_h
     end
 
     private
 
-    def tasks_metadata(tasks, environment)
-      logger.info "Fetching metadata for tasks (environment: '#{environment}')"
+    def task_metadata(name, environment)
+      logger.debug "Fetching metadata for task '#{name}' (environment: '#{environment}')"
 
-      tasks.map do |task|
-        logger.debug "Fetching metadata for task '#{task}' (environment: '#{environment}')"
-        metadata = orchestrator.tasks_support.task_metadata(task, environment)
-        [task, metadata]
-      end.to_h
+      # FIXME: Remove this workaround when the Puppet bug is fixed
+      return { 'metadata' => { 'private' => true } } if ['application::utils'].include? name
+
+      orchestrator.tasks_support.task_metadata(name, environment)
     end
   end
 end

data/lib/choria/orchestrator/task/result_set.rb

@@ -6,9 +6,12 @@ module Choria
       class ResultSet
         attr_reader :results
 
+        attr_accessor :pending_count
+
         def initialize(on_result:)
           @results = []
           @on_result = on_result
+          @pending_count = 0
         end
 
         def integrate_rpc_error(rpc_error)
@@ -20,7 +23,7 @@ module Choria
         def integrate_result(result)
           structured_result = Choria::Colt::DataStructurer.structure(result).with_indifferent_access
           @results << structured_result
-          @on_result&.call(structured_result)
+          @on_result&.call(structured_result, @results.count, pending_count + @results.count)
         end
       end
     end

data/lib/choria/orchestrator/task.rb

@@ -3,6 +3,8 @@ require_relative 'task/result_set'
 require 'active_support'
 require 'active_support/core_ext/hash/indifferent_access'
 
+require 'choria/colt/debugger'
+
 module Choria
   class Orchestrator
     class Task
@@ -55,7 +57,7 @@ module Choria
       end
 
       def on_result(&block)
-        @on_result = ->(result) { block.call(result) }
+        @on_result = ->(result, count, total_count) { block.call(result, count, total_count) }
       end
 
       private
@@ -64,15 +66,26 @@ module Choria
         @result_set ||= ResultSet.new(on_result: @on_result)
       end
 
+      def log_new_result(res)
+        if Colt::Debugger.enabled
+          debug_file = Colt::Debugger.save_file(result_set: @id, filename: "#{Time.now.iso8601}-#{res[:sender]}.json", content: JSON.pretty_generate(res))
+          logger.debug "New result for task ##{@id} saved in '#{debug_file}'"
+        else
+          logger.debug "New result for task ##{@id} from '#{res[:sender]}'"
+        end
+      end
+
       def rpc_results=(results)
         completed_results = results.reject { |res| res[:data][:exitcode] == -1 }
         @pending_targets ||= results.map { |res| res[:sender] }
 
         new_results = completed_results.select { |res| @pending_targets.include? res[:sender] }
         new_results.each do |res|
-          logger.debug "New result for task ##{@id}: #{res}"
-          result_set.integrate_result(res)
+          log_new_result res
+
           @pending_targets.delete res[:sender]
+          result_set.pending_count = @pending_targets.count
+          result_set.integrate_result res
         end
       end
 
@@ -103,6 +116,7 @@ module Choria
         end
         raise NoNodesLeftError, "No nodes left to continue after 'run_no_wait' action" if @pending_targets.empty?
 
+        task_ids.compact!
         task_ids.uniq!
         raise NotImplementedError, "Multiple task IDs: #{task_ids}" unless task_ids.count == 1
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: choria-colt
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Romuald Conty
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-05-16 00:00:00.000000000 Z
+date: 2022-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tty-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: github_changelog_generator
   requirement: !ruby/object:Gem::Requirement
@@ -210,11 +224,11 @@ files:
 - choria-colt.gemspec
 - exe/colt
 - lib/choria/colt.rb
-- lib/choria/colt/cache.rb
 - lib/choria/colt/cli.rb
 - lib/choria/colt/cli/formatter.rb
 - lib/choria/colt/cli/thor.rb
 - lib/choria/colt/data_structurer.rb
+- lib/choria/colt/debugger.rb
 - lib/choria/colt/version.rb
 - lib/choria/orchestrator.rb
 - lib/choria/orchestrator/task.rb

data/lib/choria/colt/cache.rb

@@ -1,33 +0,0 @@
-require 'yaml'
-
-module Choria
-  class Colt
-    class Cache
-      def initialize(path:, force_refresh: false)
-        @path = path
-        @data = YAML.safe_load File.read(@path)
-        @clean = true unless force_refresh
-      rescue Errno::ENOENT
-        @clean = false
-      end
-
-      def dirty?
-        !clean?
-      end
-
-      def clean?
-        @clean
-      end
-
-      def load
-        @data
-      end
-
-      def save(data)
-        @data = data
-        File.write @path, data.to_yaml
-        @clean = true
-      end
-    end
-  end
-end