openbolt 5.4.0 → 5.5.0

data/lib/bolt/transport/choria.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'concurrent/map'
+require 'digest/sha2'
+require 'json'
+require 'securerandom'
+require 'shellwords'
+require_relative '../../bolt/transport/base'
+
+module Bolt
+  module Transport
+    # Choria transport for OpenBolt. Communicates with nodes via Choria's NATS
+    # pub/sub messaging infrastructure using the choria-mcorpc-support gem as
+    # the client library. Extends Transport::Base directly (not Simple) because
+    # Choria's pub/sub model doesn't fit the persistent connection/shell
+    # abstraction that Simple assumes.
+    #
+    # Available capabilities depend on which agents are installed on the
+    # target node:
+    #
+    #   bolt_tasks agent only: Only run_task works, via the bolt_tasks agent
+    #     which downloads task files from an OpenVox/Puppet Server and executes
+    #     them via task_wrapper. All other operations fail with an actionable
+    #     error directing the user to install the shell agent.
+    #
+    #   shell agent installed (>= 1.2.1): run_command, run_script, and
+    #     run_task work. run_task uses the bolt_tasks agent by default.
+    #     To run local tasks via the shell agent, set task-agent to 'shell'
+    #     in project config or specify --choria-task-agent shell.
+    #
+    #   Upload, download, and plans are not yet supported.
+    class Choria < Base
+      def initialize
+        super
+        @config_mutex = Mutex.new
+        @config_error = nil
+        @client_configured = false
+        # Serializes RPC calls across batch threads. See the comment on
+        # rpc_request in helpers.rb for why this is necessary.
+        @rpc_mutex = Mutex.new
+        # Multiple batch threads write to this map concurrently when we
+        # have more than one collective.
+        @agent_cache = Concurrent::Map.new
+        @default_collective = nil
+      end
+
+      # Advertise both shell and powershell so tasks with either requirement
+      # can be selected. The per-target selection happens in
+      # select_implementation below, which picks the right feature set based
+      # on the target's detected OS.
+      def provided_features
+        %w[shell powershell]
+      end
+
+      # Override to select task implementation based on the target's OS.
+      # Other transports rely on inventory features to pick the right
+      # implementation, but Choria discovers the OS at runtime via the
+      # os.family fact. We pass only the detected platform's feature so
+      # task.select_implementation picks the correct .ps1 or .sh file.
+      #
+      # @param target [Bolt::Target] Target whose OS determines the implementation
+      # @param task [Bolt::Task] Task with platform-specific implementations
+      # @return [Hash] Selected implementation hash with 'path', 'name', 'input_method', 'files' keys
+      def select_implementation(target, task)
+        features = windows_target?(target) ? ['powershell'] : ['shell']
+        impl = task.select_implementation(target, features)
+        impl['input_method'] ||= default_input_method(impl['path'])
+        impl
+      end
+
+      # Group targets by collective so each batch uses a single RPC client
+      # scope. MCollective RPC calls are published to a collective-specific
+      # NATS subject, so targets in different collectives must be in separate
+      # batches. Most deployments have one collective, yielding one batch.
+      # Bolt runs each batch in its own thread and @rpc_mutex serializes
+      # the RPC calls across threads to prevent response misrouting.
+      #
+      # @param targets [Array<Bolt::Target>] All targets for this operation
+      # @return [Array<Array<Bolt::Target>>] Targets grouped by collective
+      def batches(targets)
+        # Populates @default_collective from the Choria config so targets
+        # without an explicit collective are grouped correctly.
+        configure_client(targets.first)
+        targets.group_by { |target| collective_for(target) }.values
+      end
+
+      # Override batch_task to handle multiple targets in one thread using the RPC.
+      # Implementation grouping (mixed-platform support) is handled internally
+      # by run_task_via_bolt_tasks and run_task_via_shell.
+      #
+      # @param targets [Array<Bolt::Target>] Targets in a single collective batch
+      # @param task [Bolt::Task] Task to execute
+      # @param arguments [Hash] Task parameter names to values
+      # @param options [Hash] Execution options (unused currently, passed through from Base)
+      # @param position [Array] Positional info for result tracking
+      # @param callback [Proc] Called with :node_start and :node_result events
+      # @return [Array<Bolt::Result>] Results for all targets (successes and failures)
+      def batch_task(targets, task, arguments, _options = {}, position = [], &callback)
+        chosen_agent = targets.first.options['task-agent'] || 'bolt_tasks'
+        result_opts = { action: 'task', name: task.name, position: position }
+
+        # The results var here is the error results for incapable targets, to which we'll add in
+        # the successful results from the capable targets as we go.
+        capable, results = prepare_targets(targets, chosen_agent, result_opts, &callback)
+
+        logger.debug { "Task #{task.name} routing: agent: #{chosen_agent}, #{capable.size} capable / #{targets.size - capable.size} incapable" }
+
+        unless capable.empty?
+          capable.each { |target| callback&.call(type: :node_start, target: target) }
+          arguments = unwrap_sensitive_args(arguments)
+
+          results += case chosen_agent
+                     when 'bolt_tasks'
+                       run_task_via_bolt_tasks(capable, task, arguments, result_opts, &callback)
+                     when 'shell'
+                       run_task_via_shell(capable, task, arguments, result_opts, &callback)
+                     else
+                       raise Bolt::Error.new(
+                         "Unsupported task-agent '#{chosen_agent}'",
+                         'bolt/choria-unsupported-agent'
+                       )
+                     end
+        end
+
+        results
+      end
+
+      # Override batch_task_with for per-target arguments. Only called
+      # from the run_task_with Puppet plan function (no CLI or Ruby API
+      # path uses this). Discovery is batched upfront, but execution is
+      # sequential per-target because MCollective RPC calls send the
+      # same arguments to all targets. A future optimization could batch
+      # the download/infra-setup/polling steps while keeping only the
+      # start step per-target.
+      #
+      # THIS IS NOT YET READY FOR PRODUCTION. The API is stable, but we don't
+      # yet have full plan support and this runs the task sequentially across
+      # targets, which is very inefficient. It had to be implemented now, though,
+      # in order to prevent the assert_batch_size_one from the Base interface
+      # from blowing things up.
+      #
+      # @param targets [Array<Bolt::Target>] Targets in a single collective batch
+      # @param task [Bolt::Task] Task to execute
+      # @param target_mapping [Hash{Bolt::Target => Hash}] Per-target argument hashes
+      # @param options [Hash] Execution options (passed through from Base)
+      # @param position [Array] Positional info for result tracking
+      # @param callback [Proc] Called with :node_start and :node_result events
+      # @return [Array<Bolt::Result>] Results for all targets
+      def batch_task_with(targets, task, target_mapping, options = {}, position = [], &callback)
+        # Pre-warm the agent cache so individual batch_task calls are cache hits
+        configure_client(targets.first)
+        discover_agents(targets)
+
+        results = []
+        targets.each do |target|
+          results += batch_task([target], task, target_mapping[target], options, position, &callback)
+        end
+        results
+      end
+
+      # Override batch_connected? to check all targets in one RPC call. Only
+      # used for wait_until_available in plans.
+      #
+      # @param targets [Array<Bolt::Target>] Targets to check connectivity for
+      # @return [Boolean] True if all targets responded to ping
+      def batch_connected?(targets)
+        logger.debug { "Checking connectivity for #{target_count(targets)}" }
+        first_target = targets.first
+        configure_client(first_target)
+
+        response = rpc_request('rpcutil', targets, 'rpcutil.ping') do |client|
+          client.ping
+        end
+        response[:responded].length == targets.length
+      rescue StandardError => e
+        raise if e.is_a?(Bolt::Error)
+
+        logger.warn { "Batch connectivity check failed: #{e.class}: #{e.message}" }
+        false
+      end
+
+      def upload(_target, _source, _destination, _options = {}, _position = [])
+        raise Bolt::Error.new(
+          'The Choria transport does not yet support upload.',
+          'bolt/choria-unsupported-operation'
+        )
+      end
+
+      def download(_target, _source, _destination, _options = {}, _position = [])
+        raise Bolt::Error.new(
+          'The Choria transport does not yet support download.',
+          'bolt/choria-unsupported-operation'
+        )
+      end
+
+      # Returns the Choria node identity for a target. Uses the transport
+      # 'host' config if set, falling back to target.host (which Bolt
+      # derives from the URI or target name).
+      def choria_identity(target)
+        target.options['host'] || target.host
+      end
+
+      # Returns the collective for a target, used by batches() to group
+      # targets. Falls back to the default collective from the loaded config.
+      def collective_for(target)
+        target.options['collective'] || @default_collective
+      end
+    end
+  end
+end
+
+require_relative 'choria/agent_discovery'
+require_relative 'choria/bolt_tasks'
+require_relative 'choria/client'
+require_relative 'choria/command_builders'
+require_relative 'choria/helpers'
+require_relative 'choria/shell'

data/lib/bolt/transport/winrm/connection.rb

@@ -162,9 +162,20 @@ module Bolt
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
-        def upload_file_smb(source, destination)
+        def require_ruby_smb
           # lazy-load expensive gem code
+          # In BinData 2.5.0+, the below commit makes things VERY noisy when loading RubySMB, so
+          # we temporarily disable warnings while we load it. If this ever gets fixed, get rid
+          # of this function and restore the plain 'require' where this function is called.
+          # https://github.com/dmendel/bindata/commit/2c8588a1ae5959080fffa429e07027f2ff20161c
+          prev = $VERBOSE
+          $VERBOSE = nil
           require 'ruby_smb'
+          $VERBOSE = prev
+        end
+
+        def upload_file_smb(source, destination)
+          require_ruby_smb
 
           win_dest = destination.tr('/', '\\')
           if (md = win_dest.match(/^([a-z]):\\(.*)/i))
@@ -215,8 +226,7 @@ module Bolt
         end
 
         def download_file_smb(source, destination)
-          # lazy-load expensive gem code
-          require 'ruby_smb'
+          require_ruby_smb
 
           win_source = source.tr('/', '\\')
           if (md = win_source.match(/^([a-z]):\\(.*)/i))

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '5.4.0'
+  VERSION = '5.5.0'
 end

data/lib/mcollective/agent/shell.ddl

@@ -0,0 +1,154 @@
+metadata    :name        => "shell",
+            :description => "Run commands with the local shell",
+            :author      => "Puppet Labs",
+            :license     => "Apache-2.0",
+            :version     => "1.2.1",
+            :url         => "https://github.com/choria-plugins/shell-agent",
+            :timeout     => 180
+
+action "run", :description => "Run a command" do
+    display :always
+
+    input   :command,
+            :prompt      => "Command",
+            :description => "Command to run",
+            :type        => :string,
+            :validation  => '.*',
+            :maxlength   => 10 * 1024,
+            :optional    => false
+
+    input   :user,
+            :prompt      => "User",
+            :description => "User to run command as",
+            :type        => :string,
+            :validation  => '.*',
+            :maxlength   => 1024,
+            :optional    => true
+
+    input   :timeout,
+            :prompt      => "Timeout",
+            :description => "Timeout to wait for the command to complete",
+            :type        => :float,
+            :optional    => true
+            # TODO(richardc): validate positive.  May need another validator class
+
+    output  :stdout,
+            :description => "stdout from the command",
+            :display_as  => "stdout"
+
+    output  :stderr,
+            :description => "stderr from the command",
+            :display_as  => "stderr"
+
+    output  :success,
+            :description  => "did the process exit successfully",
+            :display_as   => "success"
+
+    output  :exitcode,
+            :description => "exit code of the command",
+            :display_as  => "exitcode"
+end
+
+action "start", :description => "Spawn a command" do
+    display :always
+
+    input   :command,
+            :prompt      => "Command",
+            :description => "Command to run",
+            :type        => :string,
+            :validation  => '.*',
+            :maxlength   => 10 * 1024,
+            :optional    => false
+
+    input   :user,
+            :prompt      => "User",
+            :description => "User to run command as",
+            :type        => :string,
+            :validation  => '.*',
+            :maxlength   => 1024,
+            :optional    => true
+
+    output  :handle,
+            :description => "identifier to a running command",
+            :display_as  => "handle"
+end
+
+action "status", :description => "Get status of managed command" do
+    display :always
+
+    input   :handle,
+            :prompt      => "Handle",
+            :description => "Handle of the command",
+            :type        => :string,
+            :validation  => '^[0-9a-z\-]*$',
+            :maxlength   => 36,
+            :optional    => false
+
+    input   :stdout_offset,
+            :prompt      => "stdout_offset",
+            :description => "stdout_offset",
+            :type        => :integer,
+            :optional    => true
+
+    input   :stderr_offset,
+            :prompt      => "stderr_offset",
+            :description => "stderr_offset",
+            :type        => :integer,
+            :optional    => true
+
+    # Running, Exited
+    output :status,
+        :description => "status of the command",
+        :display_as  => "status"
+
+    # Stdout to this point - resets internal state
+    output :stdout,
+        :description => "stdout of the command",
+        :display_as  => "stdout"
+
+    # Stderr to this point - resets internal state
+    output :stderr,
+        :description => "stderr of the command",
+        :display_as  => "stderr"
+
+    # Only meaningful if status == Exited
+    output :exitcode,
+        :description => "exitcode of the command",
+        :display_as  => "exitcode"
+
+end
+
+action "list", :description => "Get a list of all running commands" do
+    display :always
+
+    output   :jobs,
+        :description => "state of managed jobs",
+        :display_as  => "jobs"
+
+end
+
+action "statuses", :description => "Get status and output of multiple managed commands" do
+    display :always
+
+    input   :handles,
+            :prompt      => "Handles",
+            :description => "Array of command handles to query",
+            :type        => :array,
+            :optional    => false
+
+    output  :statuses,
+            :description => "status and output keyed by handle",
+            :display_as  => "statuses"
+end
+
+action "kill", :description => "Kill a command by handle" do
+    display :always
+
+    input   :handle,
+            :prompt      => "Handle",
+            :description => "Handle of the command",
+            :type        => :string,
+            :validation  => '^[0-9a-z\-]*$',
+            :maxlength   => 36,
+            :optional    => false
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openbolt
 version: !ruby/object:Gem::Version
-  version: 5.4.0
+  version: 5.5.0
 platform: ruby
 authors:
 - OpenVox Project
@@ -51,6 +51,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: choria-mcorpc-support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -327,16 +341,16 @@ dependencies:
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: winrm
   requirement: !ruby/object:Gem::Requirement
@@ -445,14 +459,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 5.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 5.2.0
 description: Execute commands remotely over SSH and WinRM
 email:
 - openvox@voxpupuli.org
@@ -550,6 +564,7 @@ files:
 - lib/bolt/config/modulepath.rb
 - lib/bolt/config/options.rb
 - lib/bolt/config/transport/base.rb
+- lib/bolt/config/transport/choria.rb
 - lib/bolt/config/transport/docker.rb
 - lib/bolt/config/transport/jail.rb
 - lib/bolt/config/transport/local.rb
@@ -618,7 +633,6 @@ files:
 - lib/bolt/plugin/env_var.rb
 - lib/bolt/plugin/module.rb
 - lib/bolt/plugin/prompt.rb
-- lib/bolt/plugin/puppet_connect_data.rb
 - lib/bolt/plugin/puppetdb.rb
 - lib/bolt/plugin/task.rb
 - lib/bolt/project.rb
@@ -646,6 +660,13 @@ files:
 - lib/bolt/task/puppet_server.rb
 - lib/bolt/task/run.rb
 - lib/bolt/transport/base.rb
+- lib/bolt/transport/choria.rb
+- lib/bolt/transport/choria/agent_discovery.rb
+- lib/bolt/transport/choria/bolt_tasks.rb
+- lib/bolt/transport/choria/client.rb
+- lib/bolt/transport/choria/command_builders.rb
+- lib/bolt/transport/choria/helpers.rb
+- lib/bolt/transport/choria/shell.rb
 - lib/bolt/transport/docker.rb
 - lib/bolt/transport/docker/connection.rb
 - lib/bolt/transport/jail.rb
@@ -681,6 +702,7 @@ files:
 - lib/bolt_spec/plans/publish_stub.rb
 - lib/bolt_spec/run.rb
 - lib/logging_extensions/logging.rb
+- lib/mcollective/agent/shell.ddl
 - libexec/apply_catalog.rb
 - libexec/bolt_catalog
 - libexec/custom_facts.rb
@@ -694,7 +716,6 @@ files:
 - modules/canary/lib/puppet/functions/canary/random_split.rb
 - modules/canary/lib/puppet/functions/canary/skip.rb
 - modules/canary/plans/init.pp
-- modules/puppet_connect/plans/test_input_data.pp
 - modules/puppetdb_fact/plans/init.pp
 - resources/bolt_bash_completion.sh
 homepage: https://github.com/OpenVoxProject/openbolt/
@@ -715,7 +736,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Execute commands remotely over SSH and WinRM
 test_files: []

data/lib/bolt/plugin/puppet_connect_data.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-module Bolt
-  class Plugin
-    class PuppetConnectData
-      INPUT_DATA_VAR = 'PUPPET_CONNECT_INPUT_DATA'
-
-      def initialize(context:, **_opts)
-        if ENV.key?(INPUT_DATA_VAR)
-          # The user provided input data that they will copy-paste into the Puppet Connect UI
-          # for inventory syncing. This environment variable will likely be set when invoking a
-          # general "test Puppet Connect input data" command. That command tests that parsing
-          # the inventory with the given input data results in connectable targets. Part of
-          # that requires validating that the input data contains all of the referenced keys,
-          # which is what this plugin will do in validate_resolve_reference.
-          @input_data_path = ENV[INPUT_DATA_VAR]
-          data_path = @input_data_path
-        else
-          # The user is using this plugin during a regular Bolt invocation, so fetch the (minimal)
-          # required data from the default location. This data should typically be non-autoloadable
-          # secrets like WinRM passwords.
-          #
-          # Note that any unspecified keys will be resolved to nil.
-          data_path = File.join(context.boltdir, 'puppet_connect_data.yaml')
-        end
-
-        @data = Bolt::Util.read_optional_yaml_hash(
-          data_path,
-          File.basename(data_path)
-        )
-
-        if @input_data_path
-          # Validate that the data does not contain any plugin-reference
-          # values
-          @data.each do |key, toplevel_value|
-            # Use walk_vals to check for nested plugin references
-            Bolt::Util.walk_vals(toplevel_value) do |current_value|
-              if current_value.is_a?(Hash) && current_value.key?('_plugin')
-                raise invalid_input_data_err("the #{key} key's value contains a plugin reference")
-              end
-
-              current_value
-            end
-          end
-        end
-      end
-
-      def name
-        'puppet_connect_data'
-      end
-
-      def hooks
-        hook_descriptions.keys
-      end
-
-      def hook_descriptions
-        {
-          resolve_reference: nil,
-          validate_resolve_reference: nil
-        }
-      end
-
-      def resolve_reference(opts)
-        key = opts['key']
-        @data[key]
-      end
-
-      def validate_resolve_reference(opts)
-        unless opts['key']
-          raise Bolt::ValidationError,
-                "puppet_connect_data plugin requires that 'key' be specified"
-        end
-        if @input_data_path && !@data.key?(opts['key'])
-          # Input data for Puppet Connect was provided and opts['key'] does not have a
-          # value specified. Raise an error for this case.
-          raise invalid_input_data_err("a value for the #{opts['key']} key is not specified")
-        end
-      end
-
-      def invalid_input_data_err(msg)
-        Bolt::ValidationError.new("invalid input data #{@input_data_path}: #{msg}")
-      end
-    end
-  end
-end