choria-colt 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/.rubocop.yml +3 -1
- data/.rubocop_todo.yml +22 -0
- data/choria-colt.gemspec +6 -1
- data/lib/choria/colt/cli/formatter.rb +67 -0
- data/lib/choria/colt/cli.rb +63 -27
- data/lib/choria/colt/data_structurer.rb +26 -0
- data/lib/choria/colt/version.rb +1 -1
- data/lib/choria/colt.rb +16 -10
- data/lib/choria/orchestrator/task.rb +78 -14
- data/lib/choria/orchestrator.rb +12 -29
- metadata +76 -2
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 17cd0f4a8fd7bc14aed8bd1f702405d257c8ab18b31406eb13eb3b3b3d6623cd
         | 
| 4 | 
            +
              data.tar.gz: 9ad55b6d635820f4486476ed9722e23f68f85037f0b94c0ed93d500a64e10e29
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: cac0c586629af294b520c29e4967824cd832b257405be6a95ba4e2a552f1a92cba936b22e219ed0342b8b1cfff34b3257a5c3cf5ab6607f66450d864b636ba12
         | 
| 7 | 
            +
              data.tar.gz: 6d6cf405dd1973a4e7d813a58de872a8b37514e821ea90c216a42b3242240895ed1fb4adda8dd24d065fbbf926b7ae186fea610b9e54f5c2e5df3a7fa75b94ef
         | 
    
        data/.rspec
    ADDED
    
    | @@ -0,0 +1 @@ | |
| 1 | 
            +
            --require spec_helper
         | 
    
        data/.rubocop.yml
    CHANGED
    
    
    
        data/.rubocop_todo.yml
    ADDED
    
    | @@ -0,0 +1,22 @@ | |
| 1 | 
            +
            # This configuration was generated by
         | 
| 2 | 
            +
            # `rubocop --auto-gen-config`
         | 
| 3 | 
            +
            # on 2022-04-07 15:43:34 UTC using RuboCop version 1.26.0.
         | 
| 4 | 
            +
            # The point is for the user to remove these configuration records
         | 
| 5 | 
            +
            # one by one as the offenses are removed from the code base.
         | 
| 6 | 
            +
            # Note that changes in the inspected code, or installation of new
         | 
| 7 | 
            +
            # versions of RuboCop, may require this file to be generated again.
         | 
| 8 | 
            +
             | 
| 9 | 
            +
            # Offense count: 1
         | 
| 10 | 
            +
            # Configuration parameters: CountComments, CountAsOne.
         | 
| 11 | 
            +
            Metrics/ClassLength:
         | 
| 12 | 
            +
              Max: 109
         | 
| 13 | 
            +
             | 
| 14 | 
            +
            # Offense count: 1
         | 
| 15 | 
            +
            # Configuration parameters: IgnoredMethods.
         | 
| 16 | 
            +
            Metrics/CyclomaticComplexity:
         | 
| 17 | 
            +
              Max: 9
         | 
| 18 | 
            +
             | 
| 19 | 
            +
            # Offense count: 1
         | 
| 20 | 
            +
            # Configuration parameters: IgnoredMethods.
         | 
| 21 | 
            +
            Metrics/PerceivedComplexity:
         | 
| 22 | 
            +
              Max: 9
         | 
    
        data/choria-colt.gemspec
    CHANGED
    
    | @@ -28,15 +28,20 @@ Gem::Specification.new do |spec| | |
| 28 28 | 
             
              spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
         | 
| 29 29 | 
             
              spec.require_paths = ['lib']
         | 
| 30 30 |  | 
| 31 | 
            +
              spec.add_dependency 'activesupport'
         | 
| 31 32 | 
             
              spec.add_dependency 'choria-mcorpc-support'
         | 
| 32 33 | 
             
              spec.add_dependency 'deep_merge'
         | 
| 34 | 
            +
              spec.add_dependency 'pastel'
         | 
| 33 35 | 
             
              spec.add_dependency 'puppet'
         | 
| 34 36 | 
             
              spec.add_dependency 'thor'
         | 
| 37 | 
            +
              spec.add_dependency 'tty-logger'
         | 
| 35 38 |  | 
| 39 | 
            +
              # spec.add_development_dependency 'byebug'
         | 
| 36 40 | 
             
              spec.add_development_dependency 'rake'
         | 
| 37 41 | 
             
              spec.add_development_dependency 'rspec'
         | 
| 38 42 | 
             
              spec.add_development_dependency 'rubocop'
         | 
| 39 | 
            -
               | 
| 43 | 
            +
              spec.add_development_dependency 'rubocop-rake'
         | 
| 44 | 
            +
              spec.add_development_dependency 'rubocop-rspec'
         | 
| 40 45 |  | 
| 41 46 | 
             
              # For more information and examples about making a new gem, check out our
         | 
| 42 47 | 
             
              # guide at: https://bundler.io/guides/creating_gem.html
         | 
| @@ -0,0 +1,67 @@ | |
| 1 | 
            +
            require 'choria/colt/cli'
         | 
| 2 | 
            +
            require 'choria/colt/cli/thor'
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            module Choria
         | 
| 5 | 
            +
              class Colt
         | 
| 6 | 
            +
                class CLI < Thor
         | 
| 7 | 
            +
                  class Formatter
         | 
| 8 | 
            +
                    attr_reader :pastel
         | 
| 9 | 
            +
             | 
| 10 | 
            +
                    def initialize(colored:)
         | 
| 11 | 
            +
                      @pastel = Pastel.new(enabled: colored)
         | 
| 12 | 
            +
                      pastel.alias_color(:host, :cyan)
         | 
| 13 | 
            +
                    end
         | 
| 14 | 
            +
             | 
| 15 | 
            +
                    def process_result(result)
         | 
| 16 | 
            +
                      return process_error(result) unless result.dig(:data, :exitcode)&.zero?
         | 
| 17 | 
            +
             | 
| 18 | 
            +
                      process_success(result)
         | 
| 19 | 
            +
                    end
         | 
| 20 | 
            +
             | 
| 21 | 
            +
                    def process_success(result)
         | 
| 22 | 
            +
                      output_lines = [
         | 
| 23 | 
            +
                        "#{pastel.host(result[:sender]).ljust(60, ' ')}duration: #{pastel.bright_white result[:data][:runtime]}",
         | 
| 24 | 
            +
                      ]
         | 
| 25 | 
            +
             | 
| 26 | 
            +
                      output_lines += if result.dig(:result, :_output).nil?
         | 
| 27 | 
            +
                                        JSON.pretty_generate(result[:result]).split("\n").map { |line| "  #{line}" }
         | 
| 28 | 
            +
                                      else
         | 
| 29 | 
            +
                                        result.dig(:result, :_output).map { |line| "  #{line}" }
         | 
| 30 | 
            +
                                      end
         | 
| 31 | 
            +
             | 
| 32 | 
            +
                      output_lines.join("\n")
         | 
| 33 | 
            +
                    end
         | 
| 34 | 
            +
             | 
| 35 | 
            +
                    def process_error(result) # rubocop:disable Metrics/AbcSize
         | 
| 36 | 
            +
                      host = "#{pastel.bright_red '⨯'} #{pastel.host(result[:sender]).ljust(60, ' ')}duration: #{pastel.bright_white result[:data][:runtime]}"
         | 
| 37 | 
            +
                      output = result.dig(:result, '_output')
         | 
| 38 | 
            +
                      error_details = JSON.pretty_generate(result.dig(:result, :_error, :details)).split "\n"
         | 
| 39 | 
            +
                      error_description = [
         | 
| 40 | 
            +
                        "#{pastel.bright_red result.dig(:result, :_error, :kind)}: #{pastel.bright_white result.dig(:result, :_error, :msg)}",
         | 
| 41 | 
            +
                        "  details: #{error_details.shift}",
         | 
| 42 | 
            +
                        error_details.map { |line| "  #{line}" },
         | 
| 43 | 
            +
                      ]
         | 
| 44 | 
            +
                      output_description = if output.nil? || output.empty?
         | 
| 45 | 
            +
                                             []
         | 
| 46 | 
            +
                                           else
         | 
| 47 | 
            +
                                             [
         | 
| 48 | 
            +
                                               nil,
         | 
| 49 | 
            +
                                               pastel.bright_red('output:'),
         | 
| 50 | 
            +
                                               output,
         | 
| 51 | 
            +
                                             ]
         | 
| 52 | 
            +
                                           end
         | 
| 53 | 
            +
             | 
| 54 | 
            +
                      headline = "#{pastel.on_red ' '} "
         | 
| 55 | 
            +
             | 
| 56 | 
            +
                      [
         | 
| 57 | 
            +
                        host,
         | 
| 58 | 
            +
                        [
         | 
| 59 | 
            +
                          error_description,
         | 
| 60 | 
            +
                          output_description,
         | 
| 61 | 
            +
                        ].flatten.map { |line| "#{headline}#{line}" },
         | 
| 62 | 
            +
                      ].flatten.join("\n")
         | 
| 63 | 
            +
                    end
         | 
| 64 | 
            +
                  end
         | 
| 65 | 
            +
                end
         | 
| 66 | 
            +
              end
         | 
| 67 | 
            +
            end
         | 
    
        data/lib/choria/colt/cli.rb
    CHANGED
    
    | @@ -1,13 +1,17 @@ | |
| 1 1 | 
             
            require 'choria/colt'
         | 
| 2 | 
            +
            require 'choria/colt/cli/formatter'
         | 
| 2 3 | 
             
            require 'choria/colt/cli/thor'
         | 
| 3 4 |  | 
| 4 5 | 
             
            require 'json'
         | 
| 5 | 
            -
            require 'logger'
         | 
| 6 | 
            +
            require 'tty/logger'
         | 
| 6 7 |  | 
| 7 8 | 
             
            module Choria
         | 
| 8 9 | 
             
              class Colt
         | 
| 9 10 | 
             
                class CLI < Thor
         | 
| 10 11 | 
             
                  class Tasks < Thor
         | 
| 12 | 
            +
                    class_option :log_level,
         | 
| 13 | 
            +
                                 desc: 'Set log level for CLI',
         | 
| 14 | 
            +
                                 default: 'info'
         | 
| 11 15 | 
             
                    # BOLT: desc 'run <task name> [parameters] {--targets TARGETS | --query QUERY | --rerun FILTER} [options]', 'Run a Bolt task'
         | 
| 12 16 | 
             
                    desc 'run <task name> [parameters] --targets TARGETS [options]', 'Run a Bolt task'
         | 
| 13 17 | 
             
                    long_desc <<~DESC
         | 
| @@ -17,24 +21,30 @@ module Choria | |
| 17 21 | 
             
                    DESC
         | 
| 18 22 | 
             
                    option :targets,
         | 
| 19 23 | 
             
                           aliases: ['--target', '-t'],
         | 
| 20 | 
            -
                           desc: 'Identifies the targets of the command.' | 
| 21 | 
            -
             | 
| 22 | 
            -
             | 
| 24 | 
            +
                           desc: 'Identifies the targets of the command.'
         | 
| 25 | 
            +
                    option :targets_with_classes,
         | 
| 26 | 
            +
                           aliases: ['--targets-with-class', '-C'],
         | 
| 27 | 
            +
                           desc: 'Select the targets which have the specified Puppet classes.'
         | 
| 28 | 
            +
                    def run(*args) # rubocop:disable Metrics/AbcSize
         | 
| 23 29 | 
             
                      input = extract_task_parameters_from_args(args)
         | 
| 24 30 |  | 
| 25 31 | 
             
                      raise Thor::Error, 'Task name is required' if args.empty?
         | 
| 26 32 | 
             
                      raise Thor::Error, "Too many arguments: #{args}" unless args.count == 1
         | 
| 27 33 |  | 
| 34 | 
            +
                      raise Thor::Error, 'Flag --targets or --targets-with-class is required' if options['targets'].nil? && options['targets_with_classes'].nil?
         | 
| 35 | 
            +
             | 
| 28 36 | 
             
                      task_name = args.shift
         | 
| 29 37 |  | 
| 30 | 
            -
                      targets = options['targets'] | 
| 38 | 
            +
                      targets = options['targets']&.split(',')
         | 
| 31 39 | 
             
                      targets = nil if options['targets'] == 'all'
         | 
| 32 40 |  | 
| 33 | 
            -
                       | 
| 41 | 
            +
                      targets_with_classes = options['targets_with_classes']&.split(',')
         | 
| 34 42 |  | 
| 35 | 
            -
                       | 
| 43 | 
            +
                      results = colt.run_bolt_task task_name, input: input, targets: targets, targets_with_classes: targets_with_classes do |result|
         | 
| 44 | 
            +
                        $stdout.puts formatter.process_result(result)
         | 
| 45 | 
            +
                      end
         | 
| 36 46 |  | 
| 37 | 
            -
                       | 
| 47 | 
            +
                      File.write 'last_run.json', JSON.pretty_generate(results)
         | 
| 38 48 | 
             
                    rescue Choria::Orchestrator::Error => e
         | 
| 39 49 | 
             
                      raise Thor::Error, "#{e.class}: #{e}"
         | 
| 40 50 | 
             
                    end
         | 
| @@ -68,9 +78,23 @@ module Choria | |
| 68 78 | 
             
                      end
         | 
| 69 79 | 
             
                    end
         | 
| 70 80 |  | 
| 71 | 
            -
                    no_commands do
         | 
| 81 | 
            +
                    no_commands do # rubocop:disable Metrics/BlockLength
         | 
| 72 82 | 
             
                      def colt
         | 
| 73 | 
            -
                        @colt ||= Choria::Colt.new logger:  | 
| 83 | 
            +
                        @colt ||= Choria::Colt.new logger: logger
         | 
| 84 | 
            +
                      end
         | 
| 85 | 
            +
             | 
| 86 | 
            +
                      def logger
         | 
| 87 | 
            +
                        @logger ||= TTY::Logger.new do |config|
         | 
| 88 | 
            +
                          config.handlers = [
         | 
| 89 | 
            +
                            [:console, { output: $stderr, level: options['log_level'].to_sym }],
         | 
| 90 | 
            +
                            [:stream, { output: File.open('colt-debug.log', 'a'), level: :debug }],
         | 
| 91 | 
            +
                          ]
         | 
| 92 | 
            +
                          config.metadata = %i[date time]
         | 
| 93 | 
            +
                        end
         | 
| 94 | 
            +
                      end
         | 
| 95 | 
            +
             | 
| 96 | 
            +
                      def formatter
         | 
| 97 | 
            +
                        @formatter ||= Formatter.new(colored: $stdout.tty?)
         | 
| 74 98 | 
             
                      end
         | 
| 75 99 |  | 
| 76 100 | 
             
                      def extract_task_parameters_from_args(args)
         | 
| @@ -78,7 +102,14 @@ module Choria | |
| 78 102 | 
             
                        args.reject! { |arg| arg =~ /^\w+=/ }
         | 
| 79 103 |  | 
| 80 104 | 
             
                        parameters.map do |parameter|
         | 
| 81 | 
            -
                          key, value = parameter.split('=')
         | 
| 105 | 
            +
                          key, value = parameter.split('=', 2)
         | 
| 106 | 
            +
             | 
| 107 | 
            +
                          # TODO: Convert to boolean only if the expected type of parameter is boolean
         | 
| 108 | 
            +
                          # TODO: Support String to integer convertion
         | 
| 109 | 
            +
                          # TODO: Support @notation from parameter and/or whole input
         | 
| 110 | 
            +
                          value = true if value == 'true'
         | 
| 111 | 
            +
                          value = false if value == 'false'
         | 
| 112 | 
            +
             | 
| 82 113 | 
             
                          [key, value]
         | 
| 83 114 | 
             
                        end.to_h
         | 
| 84 115 | 
             
                      end
         | 
| @@ -87,7 +118,7 @@ module Choria | |
| 87 118 | 
             
                        tasks.reject! { |_task, metadata| metadata['metadata']['private'] }
         | 
| 88 119 |  | 
| 89 120 | 
             
                        puts <<~OUTPUT
         | 
| 90 | 
            -
                          Tasks
         | 
| 121 | 
            +
                          #{pastel.title 'Tasks'}
         | 
| 91 122 | 
             
                          #{tasks.map { |task, metadata| "#{task}#{' ' * (60 - task.size)}#{metadata['metadata']['description']}" }.join("\n").gsub(/^/, '  ')}
         | 
| 92 123 | 
             
                        OUTPUT
         | 
| 93 124 | 
             
                      end
         | 
| @@ -95,30 +126,35 @@ module Choria | |
| 95 126 | 
             
                      def show_task_details(task_name, tasks)
         | 
| 96 127 | 
             
                        metadata = tasks[task_name]
         | 
| 97 128 | 
             
                        puts <<~OUTPUT
         | 
| 98 | 
            -
                          Task:  | 
| 129 | 
            +
                          #{pastel.title "Task: #{task_name}"}
         | 
| 99 130 | 
             
                            #{metadata['metadata']['description']}
         | 
| 100 131 |  | 
| 101 | 
            -
                          Parameters | 
| 102 | 
            -
                          #{ | 
| 132 | 
            +
                          #{pastel.title 'Parameters'}
         | 
| 133 | 
            +
                          #{format_task_parameters(metadata['metadata']['parameters']).gsub(/^/, '  ')}
         | 
| 103 134 | 
             
                        OUTPUT
         | 
| 104 135 | 
             
                      end
         | 
| 105 136 |  | 
| 106 | 
            -
                      def  | 
| 107 | 
            -
                         | 
| 137 | 
            +
                      def format_task_parameters(parameters)
         | 
| 138 | 
            +
                        parameters.map do |parameter, metadata|
         | 
| 139 | 
            +
                          output = <<~OUTPUT
         | 
| 140 | 
            +
                            #{pastel.parameter(parameter)}  #{pastel.parameter_type metadata['type']}
         | 
| 141 | 
            +
                              #{metadata['description']}
         | 
| 142 | 
            +
                          OUTPUT
         | 
| 143 | 
            +
                          output += "  Default: #{metadata['default']}" unless metadata['default'].nil?
         | 
| 144 | 
            +
                          output
         | 
| 145 | 
            +
                        end.join "\n"
         | 
| 108 146 | 
             
                      end
         | 
| 109 147 |  | 
| 110 | 
            -
                      def  | 
| 111 | 
            -
                         | 
| 112 | 
            -
             | 
| 113 | 
            -
                        $stdout.puts JSON.pretty_generate(result)
         | 
| 148 | 
            +
                      def pastel
         | 
| 149 | 
            +
                        @pastel ||= _pastel
         | 
| 114 150 | 
             
                      end
         | 
| 115 151 |  | 
| 116 | 
            -
                      def  | 
| 117 | 
            -
                         | 
| 118 | 
            -
             | 
| 119 | 
            -
                         | 
| 120 | 
            -
                         | 
| 121 | 
            -
                         | 
| 152 | 
            +
                      def _pastel
         | 
| 153 | 
            +
                        pastel = Pastel.new(enabled: $stdout.tty?)
         | 
| 154 | 
            +
                        pastel.alias_color(:title, :cyan)
         | 
| 155 | 
            +
                        pastel.alias_color(:parameter, :yellow)
         | 
| 156 | 
            +
                        pastel.alias_color(:parameter_type, :bright_white)
         | 
| 157 | 
            +
                        pastel
         | 
| 122 158 | 
             
                      end
         | 
| 123 159 | 
             
                    end
         | 
| 124 160 | 
             
                  end
         | 
| @@ -0,0 +1,26 @@ | |
| 1 | 
            +
            module Choria
         | 
| 2 | 
            +
              class Colt
         | 
| 3 | 
            +
                module DataStructurer
         | 
| 4 | 
            +
                  def self.structure(res) # rubocop:disable Metrics/AbcSize
         | 
| 5 | 
            +
                    # data.stdout seems to always be JSON, so parse it once.
         | 
| 6 | 
            +
                    res[:result] = JSON.parse res[:data][:stdout]
         | 
| 7 | 
            +
                    res[:data].delete :stdout
         | 
| 8 | 
            +
             | 
| 9 | 
            +
                    # On one side, data.stderr is filled by the remote execution stderr.
         | 
| 10 | 
            +
                    # On the other side, error description is in JSON (ie. '_error')
         | 
| 11 | 
            +
                    # So merge data.stderr in '_error'.'details'
         | 
| 12 | 
            +
                    unless res[:data][:stderr].empty?
         | 
| 13 | 
            +
                      raise NotImplementedError, 'What to do when res[:data][:stderr] contains something?' if res[:result]['_error'].empty?
         | 
| 14 | 
            +
             | 
| 15 | 
            +
                      res[:result]['_error']['details'].merge!({ 'stderr' => res[:data][:stderr].split("\n") })
         | 
| 16 | 
            +
                    end
         | 
| 17 | 
            +
                    res[:data].delete :stderr
         | 
| 18 | 
            +
             | 
| 19 | 
            +
                    # Convert '_output' (ie. stdout) lines into array
         | 
| 20 | 
            +
                    res[:result]['_output'] = res[:result]['_output'].split("\n") unless res[:result]['_output'].nil?
         | 
| 21 | 
            +
             | 
| 22 | 
            +
                    res
         | 
| 23 | 
            +
                  end
         | 
| 24 | 
            +
                end
         | 
| 25 | 
            +
              end
         | 
| 26 | 
            +
            end
         | 
    
        data/lib/choria/colt/version.rb
    CHANGED
    
    
    
        data/lib/choria/colt.rb
    CHANGED
    
    | @@ -19,11 +19,13 @@ module Choria | |
| 19 19 | 
             
                  @orchestrator = Choria::Orchestrator.new logger: @logger
         | 
| 20 20 | 
             
                end
         | 
| 21 21 |  | 
| 22 | 
            -
                def run_bolt_task(task_name, input: {}, targets: nil)
         | 
| 22 | 
            +
                def run_bolt_task(task_name, input: {}, targets: nil, targets_with_classes: nil, &block)
         | 
| 23 23 | 
             
                  logger.debug "Instantiate task '#{task_name}' and validate input"
         | 
| 24 24 | 
             
                  task = Choria::Orchestrator::Task.new(task_name, input: input, orchestrator: orchestrator)
         | 
| 25 25 |  | 
| 26 | 
            -
                   | 
| 26 | 
            +
                  task.on_result(&block) if block_given?
         | 
| 27 | 
            +
             | 
| 28 | 
            +
                  orchestrator.run(task, targets: targets, targets_with_classes: targets_with_classes)
         | 
| 27 29 | 
             
                  task.wait
         | 
| 28 30 | 
             
                  task.results
         | 
| 29 31 | 
             
                rescue Choria::Orchestrator::Error => e
         | 
| @@ -36,14 +38,6 @@ module Choria | |
| 36 38 | 
             
                    task['name']
         | 
| 37 39 | 
             
                  end
         | 
| 38 40 |  | 
| 39 | 
            -
                  def tasks_metadata(tasks, environment)
         | 
| 40 | 
            -
                    tasks.map do |task|
         | 
| 41 | 
            -
                      logger.debug "Fetching metadata for task '#{task}' (environment: '#{environment}')"
         | 
| 42 | 
            -
                      metadata = orchestrator.tasks_support.task_metadata(task, environment)
         | 
| 43 | 
            -
                      [task, metadata]
         | 
| 44 | 
            -
                    end.to_h
         | 
| 45 | 
            -
                  end
         | 
| 46 | 
            -
             | 
| 47 41 | 
             
                  return tasks_metadata(tasks_names, environment) if cache.nil?
         | 
| 48 42 |  | 
| 49 43 | 
             
                  cached_tasks = cache.load
         | 
| @@ -54,5 +48,17 @@ module Choria | |
| 54 48 |  | 
| 55 49 | 
             
                  updated_tasks
         | 
| 56 50 | 
             
                end
         | 
| 51 | 
            +
             | 
| 52 | 
            +
                private
         | 
| 53 | 
            +
             | 
| 54 | 
            +
                def tasks_metadata(tasks, environment)
         | 
| 55 | 
            +
                  logger.info "Fetching metadata for tasks (environment: '#{environment}')"
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                  tasks.map do |task|
         | 
| 58 | 
            +
                    logger.debug "Fetching metadata for task '#{task}' (environment: '#{environment}')"
         | 
| 59 | 
            +
                    metadata = orchestrator.tasks_support.task_metadata(task, environment)
         | 
| 60 | 
            +
                    [task, metadata]
         | 
| 61 | 
            +
                  end.to_h
         | 
| 62 | 
            +
                end
         | 
| 57 63 | 
             
              end
         | 
| 58 64 | 
             
            end
         | 
| @@ -1,17 +1,25 @@ | |
| 1 | 
            +
            require 'choria/colt/data_structurer'
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            require 'active_support'
         | 
| 4 | 
            +
            require 'active_support/core_ext/hash/indifferent_access'
         | 
| 5 | 
            +
             | 
| 1 6 | 
             
            module Choria
         | 
| 2 7 | 
             
              class Orchestrator
         | 
| 3 8 | 
             
                class Task
         | 
| 4 9 | 
             
                  class Error < Orchestrator::Error; end
         | 
| 5 10 |  | 
| 6 | 
            -
                  attr_reader :name, :input, :environment, :rpc_results
         | 
| 11 | 
            +
                  attr_reader :name, :input, :environment, :rpc_results, :results
         | 
| 7 12 | 
             
                  attr_accessor :rpc_responses
         | 
| 8 13 |  | 
| 9 14 | 
             
                  def initialize(name, orchestrator:, input: {}, environment: 'production')
         | 
| 10 15 | 
             
                    @name = name
         | 
| 11 | 
            -
                    @input = input
         | 
| 12 16 | 
             
                    @environment = environment
         | 
| 13 17 | 
             
                    @orchestrator = orchestrator
         | 
| 18 | 
            +
                    @input = default_input.merge input
         | 
| 14 19 |  | 
| 20 | 
            +
                    @results = []
         | 
| 21 | 
            +
             | 
| 22 | 
            +
                    logger.debug "Task inputs: #{input}"
         | 
| 15 23 | 
             
                    validate_inputs
         | 
| 16 24 | 
             
                  end
         | 
| 17 25 |  | 
| @@ -23,37 +31,93 @@ module Choria | |
| 23 31 | 
             
                    metadata['files'].to_json
         | 
| 24 32 | 
             
                  end
         | 
| 25 33 |  | 
| 26 | 
            -
                  def wait
         | 
| 27 | 
            -
                     | 
| 34 | 
            +
                  def wait # rubocop:disable Metrics/AbcSize
         | 
| 35 | 
            +
                    rpc_responses_ok, rpc_responses_error = rpc_responses.partition { |res| (res[:body][:statuscode]).zero? }
         | 
| 36 | 
            +
                    rpc_responses_error.each do |res|
         | 
| 37 | 
            +
                      logger.error "Task request failed on '#{res[:senderid]}':\n#{pp res}"
         | 
| 38 | 
            +
                    end
         | 
| 39 | 
            +
             | 
| 40 | 
            +
                    task_ids = rpc_responses_ok.map { |res| res[:body][:data][:task_id] }.uniq
         | 
| 41 | 
            +
             | 
| 28 42 | 
             
                    raise NotImplementedError, "Multiple task IDs: #{task_ids}" unless task_ids.count == 1
         | 
| 29 43 |  | 
| 30 | 
            -
                    @ | 
| 44 | 
            +
                    @id = task_ids.first
         | 
| 45 | 
            +
             | 
| 46 | 
            +
                    wait_results
         | 
| 47 | 
            +
                  end
         | 
| 48 | 
            +
             | 
| 49 | 
            +
                  def on_result(&block)
         | 
| 50 | 
            +
                    @on_result = lambda { |result|
         | 
| 51 | 
            +
                      block.call(result)
         | 
| 52 | 
            +
                    }
         | 
| 53 | 
            +
                  end
         | 
| 54 | 
            +
             | 
| 55 | 
            +
                  private
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                  def rpc_results=(results)
         | 
| 58 | 
            +
                    new_result_hosts = (results.map { |res| res[:sender] }) - (@results.map { |res| res[:sender] })
         | 
| 59 | 
            +
             | 
| 60 | 
            +
                    new_result_hosts.each do |host|
         | 
| 61 | 
            +
                      result = results.find { |res| res[:sender] == host }
         | 
| 62 | 
            +
             | 
| 63 | 
            +
                      next unless result[:data][:exitcode] != -1
         | 
| 64 | 
            +
             | 
| 65 | 
            +
                      logger.debug "New result for task ##{@id}: #{result}"
         | 
| 66 | 
            +
                      structured_result = Choria::Colt::DataStructurer.structure(result).with_indifferent_access
         | 
| 67 | 
            +
             | 
| 68 | 
            +
                      @on_result&.call(structured_result)
         | 
| 69 | 
            +
             | 
| 70 | 
            +
                      @results << structured_result
         | 
| 71 | 
            +
                    end
         | 
| 72 | 
            +
             | 
| 73 | 
            +
                    @rpc_results = results
         | 
| 31 74 | 
             
                  end
         | 
| 32 75 |  | 
| 33 | 
            -
                  def  | 
| 34 | 
            -
                    @ | 
| 35 | 
            -
                      raise NotImplementedError, 'What to do when res[:data][:stderr] contains something?' unless res[:data][:stderr].empty?
         | 
| 76 | 
            +
                  def wait_results
         | 
| 77 | 
            +
                    raise 'Task ID is required!' if @id.nil?
         | 
| 36 78 |  | 
| 37 | 
            -
             | 
| 38 | 
            -
             | 
| 39 | 
            -
             | 
| 40 | 
            -
             | 
| 79 | 
            +
                    logger.info 'Waiting task results…'
         | 
| 80 | 
            +
             | 
| 81 | 
            +
                    @results = []
         | 
| 82 | 
            +
                    @rpc_results = []
         | 
| 83 | 
            +
             | 
| 84 | 
            +
                    loop do
         | 
| 85 | 
            +
                      self.rpc_results = @orchestrator.rpc_client.task_status(task_id: @id).map(&:results)
         | 
| 86 | 
            +
             | 
| 87 | 
            +
                      break if terminated?
         | 
| 41 88 | 
             
                    end
         | 
| 42 89 | 
             
                  end
         | 
| 43 90 |  | 
| 44 | 
            -
                   | 
| 91 | 
            +
                  def terminated?
         | 
| 92 | 
            +
                    @rpc_results.each do |result|
         | 
| 93 | 
            +
                      return false if result[:data][:exitcode] == -1
         | 
| 94 | 
            +
                    end
         | 
| 95 | 
            +
             | 
| 96 | 
            +
                    true
         | 
| 97 | 
            +
                  end
         | 
| 45 98 |  | 
| 46 99 | 
             
                  def _metadata
         | 
| 47 | 
            -
                     | 
| 100 | 
            +
                    logger.info 'Downloading task metadata from the Puppet Server…'
         | 
| 48 101 | 
             
                    @orchestrator.tasks_support.task_metadata(@name, @environment)
         | 
| 49 102 | 
             
                  rescue RuntimeError => e
         | 
| 50 103 | 
             
                    raise Error, e.message
         | 
| 51 104 | 
             
                  end
         | 
| 52 105 |  | 
| 106 | 
            +
                  def default_input
         | 
| 107 | 
            +
                    parameters_with_defaults = metadata['metadata']['parameters'].reject { |_k, v| v['default'].nil? }
         | 
| 108 | 
            +
                    parameters_with_defaults.transform_values do |meta|
         | 
| 109 | 
            +
                      meta['default']
         | 
| 110 | 
            +
                    end
         | 
| 111 | 
            +
                  end
         | 
| 112 | 
            +
             | 
| 53 113 | 
             
                  def validate_inputs
         | 
| 54 114 | 
             
                    ok, reason = @orchestrator.tasks_support.validate_task_inputs(@input, metadata)
         | 
| 55 115 | 
             
                    raise Error, reason.sub(/^\n/, '') unless ok
         | 
| 56 116 | 
             
                  end
         | 
| 117 | 
            +
             | 
| 118 | 
            +
                  def logger
         | 
| 119 | 
            +
                    @orchestrator.logger
         | 
| 120 | 
            +
                  end
         | 
| 57 121 | 
             
                end
         | 
| 58 122 | 
             
              end
         | 
| 59 123 | 
             
            end
         | 
    
        data/lib/choria/orchestrator.rb
    CHANGED
    
    | @@ -21,21 +21,25 @@ module Choria | |
| 21 21 | 
             
                  @tasks_support ||= MCollective::Util::Choria.new.tasks_support
         | 
| 22 22 | 
             
                end
         | 
| 23 23 |  | 
| 24 | 
            -
                def run(task, targets: nil, verbose: false)
         | 
| 25 | 
            -
                  logger.debug "Running task: '#{task.name}' (targets: #{targets.nil? ? 'all' : targets})"
         | 
| 24 | 
            +
                def run(task, targets: nil, targets_with_classes: nil, verbose: false) # rubocop:disable Metrics/AbcSize
         | 
| 26 25 | 
             
                  rpc_client.progress = verbose
         | 
| 27 26 |  | 
| 27 | 
            +
                  logger.debug "Running task: '#{task.name}' (targets: #{targets.nil? ? 'all' : targets})"
         | 
| 28 28 | 
             
                  targets&.each { |target| rpc_client.identity_filter target }
         | 
| 29 29 |  | 
| 30 | 
            -
                   | 
| 30 | 
            +
                  unless targets_with_classes.nil?
         | 
| 31 | 
            +
                    logger.debug "Filtering targets with classes: #{targets_with_classes}"
         | 
| 32 | 
            +
                    targets_with_classes.each { |klass| rpc_client.class_filter klass }
         | 
| 33 | 
            +
                  end
         | 
| 31 34 |  | 
| 32 | 
            -
                  logger.info  | 
| 35 | 
            +
                  logger.info 'Discovering targets…'
         | 
| 36 | 
            +
                  raise DiscoverError, 'No request sent, no node discovered' if rpc_client.discover.size.zero?
         | 
| 33 37 |  | 
| 38 | 
            +
                  logger.info "Downloading task '#{task.name}' on #{rpc_client.discover.size} nodes…"
         | 
| 34 39 | 
             
                  rpc_client.download(task: task.name, files: task.files, verbose: verbose)
         | 
| 35 40 |  | 
| 36 | 
            -
                  # TODO: Extract error from 'rpc' (see MCollective::RPC#printrpc)
         | 
| 37 | 
            -
             | 
| 38 41 | 
             
                  responses = []
         | 
| 42 | 
            +
                  logger.info "Starting task '#{task.name}' on #{rpc_client.discover.size} nodes…"
         | 
| 39 43 | 
             
                  rpc_client.run_no_wait(task: task.name, files: task.files, input: task.input.to_json, verbose: verbose) do |response|
         | 
| 40 44 | 
             
                    logger.debug "  Response: '#{response}'"
         | 
| 41 45 | 
             
                    responses << response
         | 
| @@ -46,37 +50,16 @@ module Choria | |
| 46 50 | 
             
                  task.rpc_responses = responses
         | 
| 47 51 | 
             
                end
         | 
| 48 52 |  | 
| 49 | 
            -
                def wait_results(task_id:)
         | 
| 50 | 
            -
                  raise 'Task ID is required!' if task_id.nil?
         | 
| 51 | 
            -
             | 
| 52 | 
            -
                  task_status_results = nil
         | 
| 53 | 
            -
                  loop do
         | 
| 54 | 
            -
                    task_status_results = rpc_client.task_status(task_id: task_id).map(&:results)
         | 
| 55 | 
            -
                    logger.debug "Task ##{task_id} status: #{task_status_results}"
         | 
| 56 | 
            -
                    break if task_completed? task_status_results
         | 
| 57 | 
            -
                  end
         | 
| 58 | 
            -
             | 
| 59 | 
            -
                  task_status_results
         | 
| 60 | 
            -
                end
         | 
| 61 | 
            -
             | 
| 62 | 
            -
                def task_completed?(results)
         | 
| 63 | 
            -
                  results.each do |result|
         | 
| 64 | 
            -
                    return false unless result[:data][:completed]
         | 
| 65 | 
            -
                  end
         | 
| 66 | 
            -
             | 
| 67 | 
            -
                  true
         | 
| 68 | 
            -
                end
         | 
| 69 | 
            -
             | 
| 70 53 | 
             
                def validate_rpc_result(result)
         | 
| 71 54 | 
             
                  raise Error, "The RPC agent returned an error: #{result[:statusmsg]}" unless (result[:statuscode]).zero?
         | 
| 72 55 | 
             
                end
         | 
| 73 56 |  | 
| 74 | 
            -
                private
         | 
| 75 | 
            -
             | 
| 76 57 | 
             
                def rpc_client
         | 
| 77 58 | 
             
                  @rpc_client ||= rpcclient('bolt_tasks', options: rpc_options)
         | 
| 78 59 | 
             
                end
         | 
| 79 60 |  | 
| 61 | 
            +
                private
         | 
| 62 | 
            +
             | 
| 80 63 | 
             
                def rpc_options
         | 
| 81 64 | 
             
                  {
         | 
| 82 65 | 
             
                    verbose: false,
         | 
    
        metadata
    CHANGED
    
    | @@ -1,15 +1,29 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: choria-colt
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.3.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Romuald Conty
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: exe
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2022- | 
| 11 | 
            +
            date: 2022-04-12 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 14 | 
            +
              name: activesupport
         | 
| 15 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 16 | 
            +
                requirements:
         | 
| 17 | 
            +
                - - ">="
         | 
| 18 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 19 | 
            +
                    version: '0'
         | 
| 20 | 
            +
              type: :runtime
         | 
| 21 | 
            +
              prerelease: false
         | 
| 22 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 23 | 
            +
                requirements:
         | 
| 24 | 
            +
                - - ">="
         | 
| 25 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 26 | 
            +
                    version: '0'
         | 
| 13 27 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 28 | 
             
              name: choria-mcorpc-support
         | 
| 15 29 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -38,6 +52,20 @@ dependencies: | |
| 38 52 | 
             
                - - ">="
         | 
| 39 53 | 
             
                  - !ruby/object:Gem::Version
         | 
| 40 54 | 
             
                    version: '0'
         | 
| 55 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 56 | 
            +
              name: pastel
         | 
| 57 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 58 | 
            +
                requirements:
         | 
| 59 | 
            +
                - - ">="
         | 
| 60 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 61 | 
            +
                    version: '0'
         | 
| 62 | 
            +
              type: :runtime
         | 
| 63 | 
            +
              prerelease: false
         | 
| 64 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 65 | 
            +
                requirements:
         | 
| 66 | 
            +
                - - ">="
         | 
| 67 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 68 | 
            +
                    version: '0'
         | 
| 41 69 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 42 70 | 
             
              name: puppet
         | 
| 43 71 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -66,6 +94,20 @@ dependencies: | |
| 66 94 | 
             
                - - ">="
         | 
| 67 95 | 
             
                  - !ruby/object:Gem::Version
         | 
| 68 96 | 
             
                    version: '0'
         | 
| 97 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 98 | 
            +
              name: tty-logger
         | 
| 99 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 100 | 
            +
                requirements:
         | 
| 101 | 
            +
                - - ">="
         | 
| 102 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 103 | 
            +
                    version: '0'
         | 
| 104 | 
            +
              type: :runtime
         | 
| 105 | 
            +
              prerelease: false
         | 
| 106 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 107 | 
            +
                requirements:
         | 
| 108 | 
            +
                - - ">="
         | 
| 109 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 110 | 
            +
                    version: '0'
         | 
| 69 111 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 70 112 | 
             
              name: rake
         | 
| 71 113 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -108,6 +150,34 @@ dependencies: | |
| 108 150 | 
             
                - - ">="
         | 
| 109 151 | 
             
                  - !ruby/object:Gem::Version
         | 
| 110 152 | 
             
                    version: '0'
         | 
| 153 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 154 | 
            +
              name: rubocop-rake
         | 
| 155 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 156 | 
            +
                requirements:
         | 
| 157 | 
            +
                - - ">="
         | 
| 158 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 159 | 
            +
                    version: '0'
         | 
| 160 | 
            +
              type: :development
         | 
| 161 | 
            +
              prerelease: false
         | 
| 162 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 163 | 
            +
                requirements:
         | 
| 164 | 
            +
                - - ">="
         | 
| 165 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 166 | 
            +
                    version: '0'
         | 
| 167 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 168 | 
            +
              name: rubocop-rspec
         | 
| 169 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 170 | 
            +
                requirements:
         | 
| 171 | 
            +
                - - ">="
         | 
| 172 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 173 | 
            +
                    version: '0'
         | 
| 174 | 
            +
              type: :development
         | 
| 175 | 
            +
              prerelease: false
         | 
| 176 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 177 | 
            +
                requirements:
         | 
| 178 | 
            +
                - - ">="
         | 
| 179 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 180 | 
            +
                    version: '0'
         | 
| 111 181 | 
             
            description: Colt eases the Bolt tasks run through Choria
         | 
| 112 182 | 
             
            email:
         | 
| 113 183 | 
             
            - romuald@opus-codium.fr
         | 
| @@ -116,7 +186,9 @@ executables: | |
| 116 186 | 
             
            extensions: []
         | 
| 117 187 | 
             
            extra_rdoc_files: []
         | 
| 118 188 | 
             
            files:
         | 
| 189 | 
            +
            - ".rspec"
         | 
| 119 190 | 
             
            - ".rubocop.yml"
         | 
| 191 | 
            +
            - ".rubocop_todo.yml"
         | 
| 120 192 | 
             
            - Gemfile
         | 
| 121 193 | 
             
            - README.md
         | 
| 122 194 | 
             
            - Rakefile
         | 
| @@ -125,7 +197,9 @@ files: | |
| 125 197 | 
             
            - lib/choria/colt.rb
         | 
| 126 198 | 
             
            - lib/choria/colt/cache.rb
         | 
| 127 199 | 
             
            - lib/choria/colt/cli.rb
         | 
| 200 | 
            +
            - lib/choria/colt/cli/formatter.rb
         | 
| 128 201 | 
             
            - lib/choria/colt/cli/thor.rb
         | 
| 202 | 
            +
            - lib/choria/colt/data_structurer.rb
         | 
| 129 203 | 
             
            - lib/choria/colt/version.rb
         | 
| 130 204 | 
             
            - lib/choria/orchestrator.rb
         | 
| 131 205 | 
             
            - lib/choria/orchestrator/task.rb
         |