choria-colt 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fa21249d732edfc1a2391bd20dcfc33cd8eee373329f90ffbf95ad6e9feb0a5c
+  data.tar.gz: 1ee9fbda91dd047ca932e8d117b65c2aab0252fdc96c30bf4b3be7a841ef3af5
+SHA512:
+  metadata.gz: d6b9e92533f0911ba97d3032f46a6d6b7c8f779317cf471dae1ee70a5457cca53ee62b1410a8ecba691cba504dadc3df54f92f9dfb4a63238e11cbc16c04d261
+  data.tar.gz: cc601e0c25747af5ff818a32eff04138138a66277a2daf430016ac5755a2906a3016f918fe43c23cffd3a5a3a66cc40267267a3cfe65bef211b178c1cc5ca8ef

data/.rubocop.yml

@@ -0,0 +1,38 @@
+AllCops:
+  TargetRubyVersion: 2.5
+  Exclude:
+    - 'bin/colt'
+    - 'vendor/**/*'
+  NewCops: enable
+
+Metrics/AbcSize:
+  Max: 20
+
+Metrics/MethodLength:
+  Max: 17
+
+Layout/LineLength:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: comma
+
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+
+Metrics/BlockLength:
+  Exclude:
+    - '*.gemspec'
+    - 'spec/**/*_spec.rb'

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in choria-colt.gemspec
+gemspec

data/README.md

@@ -0,0 +1,54 @@
+# Choria::Colt
+
+This _gem_ provides:
+ * `colt` executable, a [Bolt](https://puppet.com/docs/bolt/latest/bolt.html)-like command line interface to run _Bolt_ task through [Choria](https://puppet.com/docs/bolt/latest/bolt.html)
+ * helpers functions and classes to run _Bolt_ tasks through _Choria_ infrastructure from any _ruby_ code
+
+## Usage
+
+### CLI
+
+```shell-session
+$ colt tasks run exec=hostname -t vm012345.example.net,vm543210.example.net
+```
+
+### Code
+
+```ruby
+require 'choria/colt'
+
+targets = [ 'vm012345.example.net' ]
+task_name = 'exec'
+task_input = { 'command' => 'hostname' }
+
+colt = Choria::Colt.new(logger: Logger.new($stdout))
+results = colt.run_bolt_task task_name, input: task_input, targets: targets
+
+$stdout.puts JSON.pretty_generate(results)
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'choria-colt'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install choria-colt
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/opus-codium/choria-colt.

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: :rubocop

data/choria-colt.gemspec

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative 'lib/choria/colt/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'choria-colt'
+  spec.version = Choria::Colt::VERSION
+  spec.authors = ['Romuald Conty']
+  spec.email = ['romuald@opus-codium.fr']
+
+  spec.summary = 'Bolt-like CLI to run Bolt tasks, through Choria'
+  spec.description = 'Colt eases the Bolt tasks run through Choria'
+  spec.homepage = 'https://github.com/opus-codium/choria-colt'
+  spec.required_ruby_version = '>= 2.5.0'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/opus-codium/choria-colt'
+  spec.metadata['changelog_uri'] = 'https://github.com/opus-codium/choria-colt/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'choria-mcorpc-support'
+  spec.add_dependency 'deep_merge'
+  spec.add_dependency 'puppet'
+  spec.add_dependency 'thor'
+
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  #spec.add_development_dependency 'byebug'
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+  spec.metadata['rubygems_mfa_required'] = 'true'
+end

data/exe/colt

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'choria/colt/cli'
+
+Choria::Colt::CLI.start ARGV

data/lib/choria/colt/cache.rb

@@ -0,0 +1,33 @@
+require 'yaml'
+
+module Choria
+  class Colt
+    class Cache
+      def initialize(path:, force_refresh: false)
+        @path = path
+        @data = YAML.safe_load File.read(@path)
+        @clean = true unless force_refresh
+      rescue Errno::ENOENT
+        @clean = false
+      end
+
+      def dirty?
+        !clean?
+      end
+
+      def clean?
+        @clean
+      end
+
+      def load
+        @data
+      end
+
+      def save(data)
+        @data = data
+        File.write @path, data.to_yaml
+        @clean = true
+      end
+    end
+  end
+end

data/lib/choria/colt/cli/thor.rb

@@ -0,0 +1,42 @@
+require 'thor'
+require 'choria/colt/cli'
+
+module Choria
+  class Colt
+    # Workaround some, still unfixed, Thor behaviors
+    #
+    # This class extends ::Thor class to
+    # - exit with status code sets to `1` on Thor failure (e.g. missing required option)
+    # - exit with status code sets to `1` when user calls `msync` (or a subcommand) without required arguments
+    # - show subcommands help using `msync subcommand --help`
+    class Thor < ::Thor
+      def self.start(*args)
+        if (Thor::HELP_MAPPINGS & ARGV).any? && subcommands.none? { |command| command.start_with?(ARGV[0]) }
+          Thor::HELP_MAPPINGS.each do |cmd|
+            if (match = ARGV.delete(cmd))
+              ARGV.unshift match
+            end
+          end
+        end
+        super
+      end
+
+      desc '_invalid_command_call', 'Invalid command', hide: true
+      def _invalid_command_call
+        self.class.new.help
+        exit 1
+      end
+      default_task :_invalid_command_call
+
+      def self.exit_on_failure?
+        true
+      end
+
+      def self.is_thor_reserved_word?(word, type) # rubocop:disable Naming/PredicateName
+        return false if word == 'run'
+
+        super
+      end
+    end
+  end
+end

data/lib/choria/colt/cli.rb

@@ -0,0 +1,109 @@
+require 'choria/colt'
+require 'choria/colt/cli/thor'
+
+require 'json'
+require 'logger'
+
+module Choria
+  class Colt
+    class CLI < Thor
+      class Tasks < Thor
+        # BOLT: desc 'run <task name> [parameters] {--targets TARGETS | --query QUERY | --rerun FILTER} [options]', 'Run a Bolt task'
+        desc 'run <task name> [parameters] --targets TARGETS [options]', 'Run a Bolt task'
+        long_desc <<~DESC
+          Run a task on the specified targets.
+
+          Parameters take the form parameter=value.
+        DESC
+        option :targets,
+               aliases: ['--target', '-t'],
+               desc: 'Identifies the targets of the command.',
+               required: true
+        def run(*args)
+          input = extract_task_parameters_from_args(args)
+
+          raise Thor::Error, 'Task name is required' if args.empty?
+          raise Thor::Error, "Too many arguments: #{args}" unless args.count == 1
+
+          task_name = args.shift
+
+          targets = options['targets'].split ','
+          targets = nil if options['targets'] == 'all'
+
+          results = colt.run_bolt_task task_name, input: input, targets: targets
+          $stdout.puts JSON.pretty_generate(results)
+        rescue Choria::Orchestrator::Error => e
+          raise Thor::Error, "#{e.class}: #{e}"
+        end
+
+        desc 'show [task name] [options]', 'Show available tasks and task documentation'
+        long_desc <<~DESC
+          Show available tasks and task documentation.
+
+          Omitting the name of a task will display a list of tasks available
+          in the Bolt project.
+
+          Providing the name of a task will display detailed documentation for
+          the task, including a list of available parameters.
+        DESC
+        option :environment,
+               aliases: ['-E'],
+               desc: 'Puppet environment to grab tasks from',
+               default: 'production'
+        def show(*tasks_names)
+          environment = options['environment']
+          cache_directory = File.expand_path('.cache/colt/tasks')
+          FileUtils.mkdir_p cache_directory
+          # TODO: Support multiple infrastructure
+          cache = Cache.new(path: File.join(cache_directory, "#{environment}.yaml"))
+
+          tasks = colt.tasks(environment: environment, cache: cache)
+
+          if tasks_names.nil?
+            show_tasks_summary
+          else
+            tasks_names.each { |task_name| show_task_details(task_name, tasks) }
+          end
+        end
+
+        no_commands do
+          def colt
+            @colt ||= Choria::Colt.new logger: Logger.new($stdout)
+          end
+
+          def extract_task_parameters_from_args(args)
+            parameters = args.grep(/^\w+=/)
+            args.reject! { |arg| arg =~ /^\w+=/ }
+
+            parameters.map do |parameter|
+              key, value = parameter.split('=')
+              [key, value]
+            end.to_h
+          end
+
+          def show_tasks_summary(tasks)
+            tasks.reject! { |_task, metadata| metadata['metadata']['private'] }
+
+            puts <<~OUTPUT
+              Tasks
+              #{tasks.map { |task, metadata| "#{task}#{' ' * (60 - task.size)}#{metadata['metadata']['description']}" }.join("\n").gsub(/^/, '  ')}
+            OUTPUT
+          end
+
+          def show_task_details(task_name, tasks)
+            metadata = tasks[task_name]
+            puts <<~OUTPUT
+              Task: '#{task_name}'
+                #{metadata['metadata']['description']}
+
+                #{metadata}
+            OUTPUT
+          end
+        end
+      end
+
+      desc 'tasks', 'Show and run Bolt tasks.'
+      subcommand 'tasks', CLI::Tasks
+    end
+  end
+end

data/lib/choria/colt/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Choria
+  class Colt
+    VERSION = '0.1.0'
+  end
+end

data/lib/choria/colt.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'choria/colt/cache'
+require 'choria/colt/version'
+require 'choria/orchestrator'
+require 'choria/orchestrator/task'
+
+require 'logger'
+
+module Choria
+  class Colt
+    class Error < StandardError; end
+
+    attr_reader :logger, :orchestrator
+
+    def initialize(logger: nil)
+      @logger = logger
+
+      @orchestrator = Choria::Orchestrator.new logger: @logger
+    end
+
+    def run_bolt_task(task_name, input: {}, targets: nil)
+      logger.debug "Instantiate task '#{task_name}' and validate input"
+      task = Choria::Orchestrator::Task.new(task_name, input: input, orchestrator: orchestrator)
+
+      orchestrator.run(task, targets: targets)
+      task.wait
+      task.results
+    rescue Choria::Orchestrator::Error => e
+      logger.error e.message
+      raise
+    end
+
+    def tasks(environment:, cache: nil)
+      tasks_names = orchestrator.tasks_support.tasks(environment).map do |task|
+        task['name']
+      end
+
+      def tasks_metadata(tasks, environment)
+        tasks.map do |task|
+          logger.debug "Fetching metadata for task '#{task}' (environment: '#{environment}')"
+          metadata = orchestrator.tasks_support.task_metadata(task, environment)
+          [task, metadata]
+        end.to_h
+      end
+
+      return tasks_metadata(tasks_names, environment) if cache.nil?
+
+      cached_tasks = cache.load
+      return cached_tasks if cache.clean? && cached_tasks.keys.sort == tasks_names.sort
+
+      updated_tasks = tasks_metadata(tasks_names, environment)
+      cache.save updated_tasks
+
+      updated_tasks
+    end
+  end
+end

data/lib/choria/orchestrator/task.rb

@@ -0,0 +1,59 @@
+module Choria
+  class Orchestrator
+    class Task
+      class Error < Orchestrator::Error; end
+
+      attr_reader :name, :input, :environment, :rpc_results
+      attr_accessor :rpc_responses
+
+      def initialize(name, orchestrator:, input: {}, environment: 'production')
+        @name = name
+        @input = input
+        @environment = environment
+        @orchestrator = orchestrator
+
+        validate_inputs
+      end
+
+      def metadata
+        @metadata ||= _metadata
+      end
+
+      def files
+        metadata['files'].to_json
+      end
+
+      def wait
+        task_ids = rpc_responses.map { |res| res[:body][:data][:task_id] }.uniq
+        raise NotImplementedError, "Multiple task IDs: #{task_ids}" unless task_ids.count == 1
+
+        @rpc_results = @orchestrator.wait_results task_id: task_ids.first
+      end
+
+      def results
+        @rpc_results.map do |res|
+          raise NotImplementedError, 'What to do when res[:data][:stderr] contains something?' unless res[:data][:stderr].empty?
+
+          res[:result] = JSON.parse res[:data][:stdout]
+          res[:data].delete :stderr
+          res[:data].delete :stdout
+          res
+        end
+      end
+
+      private
+
+      def _metadata
+        # puts 'Retrieving task metadata for task %s from the Puppet Server' % task if verbose
+        @orchestrator.tasks_support.task_metadata(@name, @environment)
+      rescue RuntimeError => e
+        raise Error, e.message
+      end
+
+      def validate_inputs
+        ok, reason = @orchestrator.tasks_support.validate_task_inputs(@input, metadata)
+        raise Error, reason.sub(/^\n/, '') unless ok
+      end
+    end
+  end
+end

data/lib/choria/orchestrator.rb

@@ -0,0 +1,99 @@
+require 'English'
+require 'mcollective'
+
+module Choria
+  class Orchestrator
+    class Error < StandardError; end
+    class DiscoverError < Error; end
+
+    include MCollective::RPC
+
+    attr_reader :logger
+
+    def initialize(logger:)
+      @logger = logger
+
+      configfile ||= MCollective::Util.config_file_for_user
+      MCollective::Config.instance.loadconfig(configfile)
+    end
+
+    def tasks_support
+      @tasks_support ||= MCollective::Util::Choria.new.tasks_support
+    end
+
+    def run(task, targets: nil, verbose: false)
+      logger.debug "Running task: '#{task.name}' (targets: #{targets.nil? ? 'all' : targets})"
+      rpc_client.progress = verbose
+
+      targets&.each { |target| rpc_client.identity_filter target }
+
+      raise DiscoverError, 'No request sent, no node discovered' if rpc_client.discover.size.zero?
+
+      logger.info "Attempting to download and run task '#{task.name}' on #{rpc_client.discover.size} nodes"
+
+      rpc_client.download(task: task.name, files: task.files, verbose: verbose)
+
+      # TODO: Extract error from 'rpc' (see MCollective::RPC#printrpc)
+
+      responses = []
+      rpc_client.run_no_wait(task: task.name, files: task.files, input: task.input.to_json, verbose: verbose) do |response|
+        logger.debug "  Response: '#{response}'"
+        responses << response
+      end
+
+      # TODO: Include stats in logs when logger will be available (see MCollective::RPC#printrpcstats)
+
+      task.rpc_responses = responses
+    end
+
+    def wait_results(task_id:)
+      raise 'Task ID is required!' if task_id.nil?
+
+      loop do
+        task_status_results = rpc_client.task_status(task_id: task_id).map(&:results)
+        logger.debug "Task ##{task_id} status: #{task_status_results}"
+        break if task_completed? task_status_results
+      end
+
+      task_status_results
+    end
+
+    def task_completed?(results)
+      results.each do |result|
+        return false unless result[:data][:completed]
+      end
+
+      true
+    end
+
+    def validate_rpc_result(result)
+      raise Error, "The RPC agent returned an error: #{result[:statusmsg]}" unless (result[:statuscode]).zero?
+    end
+
+    private
+
+    def rpc_client
+      @rpc_client ||= rpcclient('bolt_tasks', options: rpc_options)
+    end
+
+    def rpc_options
+      {
+        verbose: false,
+        disctimeout: nil,
+        timeout: 5,
+        config: '/etc/choria/client.conf',
+        collective: 'mcollective',
+        discovery_method: nil,
+        discovery_options: [],
+        filter: {
+          'fact' => [], 'cf_class' => [], 'agent' => [], 'identity' => [], 'compound' => []
+        },
+        progress_bar: false,
+        mcollective_limit_targets: false,
+        batch_size: nil,
+        batch_sleep_time: 1,
+        output_format: :json,
+      }
+    end
+  end
+end

data/sig/choria/colt.rbs

@@ -0,0 +1,6 @@
+module Choria
+  module Colt
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: choria-colt
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Romuald Conty
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2022-03-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: choria-mcorpc-support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puppet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Colt eases the Bolt tasks run through Choria
+email:
+- romuald@opus-codium.fr
+executables:
+- colt
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- Gemfile
+- README.md
+- Rakefile
+- choria-colt.gemspec
+- exe/colt
+- lib/choria/colt.rb
+- lib/choria/colt/cache.rb
+- lib/choria/colt/cli.rb
+- lib/choria/colt/cli/thor.rb
+- lib/choria/colt/version.rb
+- lib/choria/orchestrator.rb
+- lib/choria/orchestrator/task.rb
+- sig/choria/colt.rbs
+homepage: https://github.com/opus-codium/choria-colt
+licenses: []
+metadata:
+  homepage_uri: https://github.com/opus-codium/choria-colt
+  source_code_uri: https://github.com/opus-codium/choria-colt
+  changelog_uri: https://github.com/opus-codium/choria-colt/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Bolt-like CLI to run Bolt tasks, through Choria
+test_files: []