choria-colt 0.5.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5a3d94cfa899f41ced9fc0405ed5f4fb41924267d4b56f6d31ba28420c9b0e7
-  data.tar.gz: df5bef6793badb31de0ee0185d0f8d3ca44288fedcf64b34084dbb60d1a6de30
+  metadata.gz: e86ae8b9274433cf146d571d2a7466a2c7c131e826ec5e4a8df9540d37f7994d
+  data.tar.gz: 9e4717f11b4f1f3f93c555818b9281e0118ac59a4bbf7b2573d6a85ed6e74fd9
 SHA512:
-  metadata.gz: bba0206e6f84679cd6c7ed5fd1831dc8e2beba052b096b265ac272baadeaf20a69585d6165a37ca1f7db2ca48a6fe532eeb77171c75f656bb0aa9f2c50796ccb
-  data.tar.gz: 02d6f79e80a664445793596ea9d96f106a14523ba9824893e7c1d43cc2c3484f63e27a1a20173d30e608e48f4d59872db9878385da2b197021d56374c502defa
+  metadata.gz: 3754982bc70a01d44fc4115c76693f67201d5a618d2f61b229d9b3408bea3ff2c2ac187ea5c0bd55fdbf8ca391c668f09086339ab366a85f93100c0ac1bd8973
+  data.tar.gz: c1a1dd4eaf6287d28d09baa7f45ff7ab297ffff5a8bd78d519d9a814dfcab74512d6f300fe6696789d1687e1a9f04aa021f90531a98143e263fae4d3a8d53688

data/.rubocop_todo.yml

@@ -1,16 +1,11 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-04-28 09:34:37 UTC using RuboCop version 1.26.0.
+# on 2022-05-03 07:08:32 UTC using RuboCop version 1.26.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 1
-# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
-Metrics/AbcSize:
-  Max: 22
-
 # Offense count: 1
 # Configuration parameters: IgnoredMethods.
 Metrics/CyclomaticComplexity:

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## [v0.7.0](https://github.com/opus-codium/choria-colt/tree/v0.7.0) (2022-09-26)
+
+[Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.6.0...v0.7.0)
+
+**Fixed bugs:**
+
+- Task: Fix IDs array processing [\#30](https://github.com/opus-codium/choria-colt/pull/30) ([neomilium](https://github.com/neomilium))
+
+**Merged pull requests:**
+
+- Use ActiveSuport::Cache instead our own Cache mecanism for tasks metadata [\#31](https://github.com/opus-codium/choria-colt/pull/31) ([neomilium](https://github.com/neomilium))
+- Improve robustness when nodes are unhealthy [\#29](https://github.com/opus-codium/choria-colt/pull/29) ([neomilium](https://github.com/neomilium))
+
+## [v0.6.0](https://github.com/opus-codium/choria-colt/tree/v0.6.0) (2022-05-16)
+
+[Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.5.1...v0.6.0)
+
+**Merged pull requests:**
+
+- Allow to run a task from any puppet environment [\#28](https://github.com/opus-codium/choria-colt/pull/28) ([neomilium](https://github.com/neomilium))
+- CLI: Use symbol notation to target `:all` nodes [\#27](https://github.com/opus-codium/choria-colt/pull/27) ([neomilium](https://github.com/neomilium))
+- CLI: Allow targeting on task status requests [\#26](https://github.com/opus-codium/choria-colt/pull/26) ([neomilium](https://github.com/neomilium))
+- Fix `task status` command [\#25](https://github.com/opus-codium/choria-colt/pull/25) ([neomilium](https://github.com/neomilium))
+
+## [v0.5.1](https://github.com/opus-codium/choria-colt/tree/v0.5.1) (2022-05-02)
+
+[Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.5.0...v0.5.1)
+
 ## [v0.5.0](https://github.com/opus-codium/choria-colt/tree/v0.5.0) (2022-05-02)
 
 [Full Changelog](https://github.com/opus-codium/choria-colt/compare/v0.4.0...v0.5.0)

data/README.md

@@ -51,8 +51,8 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 
 To release a new version:
-  1. update the version number in `version.rb`
-  1. generate CHANGELOG.md with `bundle exec rake changelog`
+  1. update the version number in `lib/choria/colt/version.rb`
+  1. generate `CHANGELOG.md` with `bundle exec rake changelog`
   1. commit changes
   1. run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/lib/choria/colt/cli.rb

@@ -9,6 +9,15 @@ module Choria
   class Colt
     class CLI < Thor
       class Tasks < Thor
+        def self.define_targets_and_filters_options
+          option :targets,
+                 aliases: ['--target', '-t'],
+                 desc: 'Identifies the targets of the command.'
+          option :targets_with_classes,
+                 aliases: ['--targets-with-class', '-C'],
+                 desc: 'Select the targets which have the specified Puppet classes.'
+        end
+
         class_option :log_level,
                      desc: 'Set log level for CLI',
                      default: 'info'
@@ -19,28 +28,22 @@ module Choria
 
           Parameters take the form parameter=value.
         DESC
-        option :targets,
-               aliases: ['--target', '-t'],
-               desc: 'Identifies the targets of the command.'
-        option :targets_with_classes,
-               aliases: ['--targets-with-class', '-C'],
-               desc: 'Select the targets which have the specified Puppet classes.'
-        def run(*args) # rubocop:disable Metrics/AbcSize
-          input = extract_task_parameters_from_args(args)
-          targets = extract_targets_from_options
-
+        define_targets_and_filters_options
+        option :environment,
+               aliases: ['-E'],
+               desc: 'Puppet environment to grab tasks from',
+               default: 'production'
+        def run(*args)
           raise Thor::Error, 'Task name is required' if args.empty?
-          raise Thor::Error, "Too many arguments: #{args}" unless args.count == 1
 
-          raise Thor::Error, 'Flag --targets or --targets-with-class is required' if options['targets'].nil? && options['targets_with_classes'].nil?
+          input = extract_task_parameters_from_args(args)
+          raise Thor::Error, "Too many arguments: #{args}" unless args.count == 1
 
           task_name = args.shift
+          targets, targets_with_classes = extract_targets_and_filters_from_options
 
-          logger.debug "Targets: #{targets}"
-
-          targets_with_classes = options['targets_with_classes']&.split(',')
-
-          results = colt.run_bolt_task task_name, input: input, targets: targets, targets_with_classes: targets_with_classes do |result|
+          environment = options['environment']
+          results = colt.run_bolt_task task_name, input: input, targets: targets, targets_with_classes: targets_with_classes, environment: environment do |result|
             $stdout.puts formatter.process_result(result)
           end
 
@@ -65,10 +68,9 @@ module Choria
                default: 'production'
         def show(*tasks_names)
           environment = options['environment']
-          cache_directory = File.expand_path('.cache/colt/tasks')
-          FileUtils.mkdir_p cache_directory
-          cache = Cache.new(path: File.join(cache_directory, "#{environment}.yaml"))
 
+          require 'active_support/cache/file_store'
+          cache = ActiveSupport::Cache::FileStore.new File.expand_path('~/.cache/colt/tasks')
           tasks = colt.tasks(environment: environment, cache: cache)
 
           if tasks_names.empty?
@@ -84,8 +86,11 @@ module Choria
 
           A task ID is required to request Choria services and retrieve results.
         DESC
+        define_targets_and_filters_options
         def status(task_id)
-          results = colt.wait_bolt_task task_id do |result|
+          targets, targets_with_classes = extract_targets_and_filters_from_options
+
+          results = colt.wait_bolt_task(task_id, targets: targets, targets_with_classes: targets_with_classes) do |result|
             $stdout.puts formatter.process_result(result)
           end
 
@@ -130,8 +135,17 @@ module Choria
             end.to_h
           end
 
+          def extract_targets_and_filters_from_options
+            raise Thor::Error, 'Flag --targets or --targets-with-class is required' if options['targets'].nil? && options['targets_with_classes'].nil?
+
+            targets = extract_targets_from_options
+            targets_with_classes = options['targets_with_classes']&.split(',')
+
+            [targets, targets_with_classes]
+          end
+
           def extract_targets_from_options
-            return nil if options['targets'] == 'all'
+            return nil if options['targets'] == ':all'
 
             targets = options['targets']&.split(',')
             targets&.map do |t|

data/lib/choria/colt/data_structurer.rb

@@ -4,14 +4,25 @@ module Choria
       def self.structure(res) # rubocop:disable Metrics/AbcSize
         return res unless [0, 1].include? res[:statuscode]
 
+        # If data is empty, status message is an RPC error
+        if res[:data].empty?
+          res[:result] = {
+            _error: {
+              kind: 'choria/rpc',
+              msg: res[:statusmsg],
+            },
+          }
+          return res
+        end
+
         # data.stdout seems to always be JSON, so parse it once.
-        res[:result] = JSON.parse res[:data][:stdout]
+        res[:result] = JSON.parse res[:data][:stdout] unless res.dig(:data, :stdout).nil?
         res[:data].delete :stdout
 
         # On one side, data.stderr is filled by the remote execution stderr.
         # On the other side, error description is in JSON (ie. '_error')
         # So merge data.stderr in '_error'.'details'
-        unless res[:data][:stderr].empty?
+        unless res.dig(:data, :stderr).nil? || res[:data][:stderr].empty?
           raise NotImplementedError, 'What to do when res[:data][:stderr] contains something?' if res[:result]['_error'].empty?
 
           res[:result]['_error']['details'].merge!({ 'stderr' => res[:data][:stderr].split("\n") })
@@ -19,7 +30,7 @@ module Choria
         res[:data].delete :stderr
 
         # Convert '_output' (ie. stdout) lines into array
-        res[:result]['_output'] = res[:result]['_output'].split("\n") unless res[:result]['_output'].nil?
+        res[:result]['_output'] = res[:result]['_output'].split("\n") unless res.dig(:result, '_output').nil?
 
         res
       end

data/lib/choria/colt/version.rb

@@ -2,6 +2,6 @@
 
 module Choria
   class Colt
-    VERSION = '0.5.1'
+    VERSION = '0.7.0'
   end
 end

data/lib/choria/colt.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
-require 'choria/colt/cache'
 require 'choria/colt/version'
 require 'choria/orchestrator'
 require 'choria/orchestrator/task'
 
 require 'logger'
 
+require 'active_support/cache/memory_store'
+
 module Choria
   class Colt
     class Error < StandardError; end
@@ -19,9 +20,9 @@ module Choria
       @orchestrator = Choria::Orchestrator.new logger: @logger
     end
 
-    def run_bolt_task(task_name, input: {}, targets: nil, targets_with_classes: nil, &block)
+    def run_bolt_task(task_name, input: {}, targets: nil, targets_with_classes: nil, environment: 'production', &block)
       logger.debug "Instantiate task '#{task_name}' and validate input"
-      task = Choria::Orchestrator::Task.new(name: task_name, input: input, orchestrator: orchestrator)
+      task = Choria::Orchestrator::Task.new(name: task_name, input: input, environment: environment, orchestrator: orchestrator)
 
       task.on_result(&block) if block_given?
 
@@ -33,11 +34,11 @@ module Choria
       raise
     end
 
-    def wait_bolt_task(task_id, &block)
-      task = Choria::Orchestrator::Task.new(id: task_id, orchestrator: orchestrator)
+    def wait_bolt_task(task_id, targets: nil, targets_with_classes: nil, &block)
+      orchestrator.discover(targets: targets, targets_with_classes: targets_with_classes)
 
+      task = Choria::Orchestrator::Task.new(id: task_id, orchestrator: orchestrator)
       task.on_result(&block) if block_given?
-
       task.wait
       task.results
     rescue Choria::Orchestrator::Error => e
@@ -45,32 +46,30 @@ module Choria
       raise
     end
 
-    def tasks(environment:, cache: nil)
+    def tasks(environment:, cache: nil, force_cache_refresh: false)
+      cache ||= ActiveSupport::Cache::MemoryStore.new
+
       tasks_names = orchestrator.tasks_support.tasks(environment).map do |task|
         task['name']
       end
 
-      return tasks_metadata(tasks_names, environment) if cache.nil?
-
-      cached_tasks = cache.load
-      return cached_tasks if cache.clean? && cached_tasks.keys.sort == tasks_names.sort
-
-      updated_tasks = tasks_metadata(tasks_names, environment)
-      cache.save updated_tasks
-
-      updated_tasks
+      tasks_names.map do |task_name|
+        metadata = cache.fetch(task_name, force: force_cache_refresh) do
+          task_metadata(task_name, environment)
+        end
+        [task_name, metadata]
+      end.to_h
     end
 
     private
 
-    def tasks_metadata(tasks, environment)
-      logger.info "Fetching metadata for tasks (environment: '#{environment}')"
+    def task_metadata(name, environment)
+      logger.debug "Fetching metadata for task '#{name}' (environment: '#{environment}')"
 
-      tasks.map do |task|
-        logger.debug "Fetching metadata for task '#{task}' (environment: '#{environment}')"
-        metadata = orchestrator.tasks_support.task_metadata(task, environment)
-        [task, metadata]
-      end.to_h
+      # FIXME: Remove this workaround when the Puppet bug is fixed
+      return { 'metadata' => { 'private' => true } } if ['application::utils'].include? name
+
+      orchestrator.tasks_support.task_metadata(name, environment)
     end
   end
 end

data/lib/choria/orchestrator/task.rb

@@ -65,8 +65,8 @@ module Choria
       end
 
       def rpc_results=(results)
-        pending_results, completed_results = results.partition { |res| res[:data][:exitcode] == -1 }
-        @pending_targets ||= pending_results.map { |res| res[:sender] }
+        completed_results = results.reject { |res| res[:data][:exitcode] == -1 }
+        @pending_targets ||= results.map { |res| res[:sender] }
 
         new_results = completed_results.select { |res| @pending_targets.include? res[:sender] }
         new_results.each do |res|
@@ -103,6 +103,7 @@ module Choria
         end
         raise NoNodesLeftError, "No nodes left to continue after 'run_no_wait' action" if @pending_targets.empty?
 
+        task_ids.compact!
         task_ids.uniq!
         raise NotImplementedError, "Multiple task IDs: #{task_ids}" unless task_ids.count == 1
 

data/lib/choria/orchestrator.rb

@@ -39,10 +39,16 @@ module Choria
       @tasks_support ||= MCollective::Util::Choria.new.tasks_support
     end
 
-    def run(task, targets: nil, targets_with_classes: nil, verbose: false) # rubocop:disable Metrics/AbcSize
+    def run(task, targets: nil, targets_with_classes: nil, verbose: false)
       rpc_client.progress = verbose
+      discover(targets: targets, targets_with_classes: targets_with_classes)
+      raise DiscoverError, 'No requests sent, no nodes discovered' if rpc_client.discover.size.zero?
+
+      task.run
+    end
 
-      logger.debug "Running task: '#{task.name}' (targets: #{targets.nil? ? 'all' : targets})"
+    def discover(targets: nil, targets_with_classes: nil)
+      logger.debug "Targets: #{targets.nil? ? 'all' : targets})"
       targets&.each { |target| rpc_client.identity_filter target }
 
       unless targets_with_classes.nil?
@@ -51,9 +57,7 @@ module Choria
       end
 
       logger.info 'Discovering targets…'
-      raise DiscoverError, 'No requests sent, no nodes discovered' if rpc_client.discover.size.zero?
-
-      task.run
+      rpc_client.discover
     end
 
     def rpc_client

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: choria-colt
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Romuald Conty
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-05-02 00:00:00.000000000 Z
+date: 2022-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -210,7 +210,6 @@ files:
 - choria-colt.gemspec
 - exe/colt
 - lib/choria/colt.rb
-- lib/choria/colt/cache.rb
 - lib/choria/colt/cli.rb
 - lib/choria/colt/cli/formatter.rb
 - lib/choria/colt/cli/thor.rb

data/lib/choria/colt/cache.rb

@@ -1,33 +0,0 @@
-require 'yaml'
-
-module Choria
-  class Colt
-    class Cache
-      def initialize(path:, force_refresh: false)
-        @path = path
-        @data = YAML.safe_load File.read(@path)
-        @clean = true unless force_refresh
-      rescue Errno::ENOENT
-        @clean = false
-      end
-
-      def dirty?
-        !clean?
-      end
-
-      def clean?
-        @clean
-      end
-
-      def load
-        @data
-      end
-
-      def save(data)
-        @data = data
-        File.write @path, data.to_yaml
-        @clean = true
-      end
-    end
-  end
-end