opswalrus 1.0.51 → 1.0.53

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be0988f270b372d5ad10f8ed1abb90db64ce6d07c34a1f17854d24e397c80d67
-  data.tar.gz: bdf5f694430d8178f3a4799d53cad884fc3a8adaecc74c14bf76b2a1a451a756
+  metadata.gz: 007f1ac465143fdd629539509b94a8bf7230b034817526d1265768f97eae0786
+  data.tar.gz: 9d0cc607ce7c9fcd06d9e402254b3eb5a75a68687a1ee485aa915b3cd488e2b6
 SHA512:
-  metadata.gz: 54cf7193022eb5fec71c71a4417668567e5134a7fcb5f72a31ed10ecee7d52765c3fbe5ecb33ee70052d5f5c3e17f60076352beec4a16397944493ccd25cf317
-  data.tar.gz: 293e8f8948d3c0d7ec7f64ea40551519fe5f14be4317185603f49e33e74f5ec188e1121933b51c1234a261fbd56e18a2bfa6ecc78d74e313d4c996763715edf4
+  metadata.gz: 83b67721de074efa3ad368ed21ec469d40928351c4ccc6f32d530e3bbda2cdd81b62f2b72692a4796b74d1efb7bbbfa31721db0eefb93864069dc277f8aa66fe
+  data.tar.gz: b54d628df099de1f4c2eea591b86449b5fd254d495ccae38f89dd95f5d1137171aaf9e6c1bd2023613f1c33561b5e49bd12d5696d6bc197bc065ead0a79aed1d

data/Gemfile

@@ -7,3 +7,5 @@ gemspec
 
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+
+gem 'solargraph', group: :development

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    opswalrus (1.0.51)
+    opswalrus (1.0.53)
+      activesupport (~> 7.0)
       bcrypt_pbkdf (~> 1.1)
       binding_of_caller (~> 1.0)
       citrus (~> 3.0)
@@ -17,28 +18,61 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.0.8)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
     addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
+    ast (2.4.2)
+    backport (1.2.0)
+    base64 (0.1.1)
     bcrypt_pbkdf (1.1.0)
+    benchmark (0.2.1)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
     citrus (3.0.2)
     concurrent-ruby (1.2.2)
     debug_inspector (1.1.0)
     diff-lcs (1.5.0)
+    e2mmap (0.1.0)
     ed25519 (1.3.0)
     git (1.18.0)
       addressable (~> 2.8)
       rchardet (~> 1.8)
     gli (2.21.1)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.6)
+    json (2.6.3)
+    kramdown (2.4.0)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    language_server-protocol (3.17.0.3)
+    minitest (5.20.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
     net-ssh (7.2.0)
+    nokogiri (1.15.4-x86_64-linux)
+      racc (~> 1.4)
+    parallel (1.23.0)
+    parser (3.2.2.3)
+      ast (~> 2.4.1)
+      racc
     pastel (0.8.0)
       tty-color (~> 0.5)
     public_suffix (5.0.3)
+    racc (1.7.1)
+    rainbow (3.1.1)
     rake (13.0.6)
+    rbs (2.8.4)
     rchardet (1.8.0)
+    regexp_parser (2.8.1)
+    reverse_markdown (2.1.1)
+      nokogiri
+    rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -52,12 +86,45 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.1)
+    rubocop (1.56.3)
+      base64 (~> 0.1.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.29.0)
+      parser (>= 3.2.1.0)
+    ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
     semantic_logger (4.14.0)
       concurrent-ruby (~> 1.0)
+    solargraph (0.49.0)
+      backport (~> 1.2)
+      benchmark
+      bundler (~> 2.0)
+      diff-lcs (~> 1.4)
+      e2mmap
+      jaro_winkler (~> 1.5)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.1)
+      parser (~> 3.0)
+      rbs (~> 2.0)
+      reverse_markdown (~> 2.0)
+      rubocop (~> 1.38)
+      thor (~> 1.0)
+      tilt (~> 2.0)
+      yard (~> 0.9, >= 0.9.24)
     sshkit (1.21.5)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
+    thor (1.2.2)
+    tilt (2.3.0)
     tty-color (0.6.0)
     tty-cursor (0.7.1)
     tty-editor (0.7.0)
@@ -70,7 +137,11 @@ GEM
       tty-screen (~> 0.8)
       wisper (~> 2.0)
     tty-screen (0.8.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.4.2)
     wisper (2.0.1)
+    yard (0.9.34)
 
 PLATFORMS
   x86_64-linux
@@ -79,6 +150,7 @@ DEPENDENCIES
   opswalrus!
   rake (~> 13.0)
   rspec (~> 3.0)
+  solargraph
 
 BUNDLED WITH
    2.4.10

data/lib/opswalrus/_reboot.ops

@@ -0,0 +1,64 @@
+params:
+  delay: integer?     # default: 1 - 1 second delay before reboot
+  sync: boolean?      # default: true - wait for the remote host to become available again before returning success/failure
+  timeout: integer?   # default: 300 - 300 seconds (5 minutes)
+...
+
+delay = params.delay.integer!(default: 1)
+sync = params.sync.boolean!(default: true)
+timeout = params.timeout.integer!(default: 300)
+
+delay = 1 if delay < 1
+
+ssh_noprep in: :sequence do
+# ssh_noprep do
+
+  # survey of command options:
+  # sudo reboot
+  # sudo systemctl reboot
+
+  # desc "Rebooting #{to_s} (alias=#{self.alias})"
+  # reboot_success = sh? 'sudo /bin/sh -c "(sleep {{ delay }} && reboot) &"'.mustache
+  # puts reboot_success
+
+  # reconnect_time = nil
+  # reconnect_success = if sync
+  #   desc "Waiting for #{to_s} (alias=#{self.alias}) to finish rebooting"
+  #   initial_reconnect_delay = delay + 10
+  #   sleep initial_reconnect_delay
+
+  #   reconnected = false
+  #   give_up = false
+  #   t1 = Time.now
+  #   until reconnected || give_up
+  #     begin
+  #       reconnected = sh?('true')
+  #       # while trying to reconnect, we expect the following exceptions:
+  #       # 1. Net::SSH::Disconnect < Net::SSH::Exception with message: "connection closed by remote host"
+  #       # 2. Errno::ECONNRESET < SystemCallError with message: "Connection reset by peer"
+  #     rescue Net::SSH::Disconnect, Errno::ECONNRESET => e
+  #       # noop; we expect these while we're trying to reconnect
+  #     rescue => e
+  #       puts "#{e.class} < #{e.class.superclass}"
+  #       puts e.message
+  #       puts e.backtrace.take(5).join("\n")
+  #     end
+
+  #     wait_time_elapsed_in_seconds = Time.now - t1
+  #     give_up = wait_time_elapsed_in_seconds > timeout
+  #     sleep 5
+  #   end
+  #   reconnect_time = initial_reconnect_delay + (Time.now - t1)
+  #   reconnected
+  # else
+  #   false
+  # end
+
+  # {
+  #   success: reboot_success && (sync == reconnect_success),
+  #   rebooted: reboot_success,
+  #   reconnected: reconnect_success,
+  #   reboot_duration: reconnect_time
+  # }
+  reboot(delay: delay, sync: sync, timeout: timeout)
+end

data/lib/opswalrus/_run_remote.ops

@@ -0,0 +1,12 @@
+params:
+  ops_file: OpsFile
+  operation_kv_args: array string
+...
+ssh in: :sequence do
+# ssh_noprep do
+  puts params.stringify_keys!
+  desc "Running `#{params.ops_file.ops_file_path} #{params.operation_kv_args.join(' ')}` on #{to_s} (alias=#{self.alias})"
+  # run_ops(ops_command, ops_command_options = nil, command_arguments, in_bundle_root_dir: true, ops_prompt_for_sudo_password: false)
+  puts self.inspect
+  self._invoke_remote(params.ops_file, *params.operation_kv_args)
+end

data/lib/opswalrus/app.rb

@@ -38,11 +38,11 @@ module OpsWalrus
     attr_reader :identity_file_paths
 
     def initialize(pwd = Dir.pwd)
-      SemanticLogger.default_level = :info
+      SemanticLogger.default_level = :warn
       # SemanticLogger.add_appender(file_name: 'development.log', formatter: :color)   # Log to a file, and use the colorized formatter
       SemanticLogger.add_appender(io: $stdout, formatter: :color)       # Log errors and above to standard error:
       @logger = SemanticLogger[OpsWalrus]    # Logger.new($stdout, level: Logger::INFO)
-      @logger.level = :info    # , :trace or 'trace'
+      @logger.level = :warn    # :trace or 'trace'
 
       # @logger.warn Style.yellow("warn"), foo: "bar", baz: {qux: "quux"}
       # @logger.info Style.yellow("info"), foo: "bar", baz: {qux: "quux"}
@@ -216,18 +216,22 @@ module OpsWalrus
     end
 
     def bootstrap()
-      set_pwd(__FILE__.to_pathname.dirname)
-      bootstrap_ops_file = OpsFile.new(self, __FILE__.to_pathname.dirname.join("_bootstrap.ops"))
-      op = OperationRunner.new(self, bootstrap_ops_file)
-      op.run([], params_json_hash: @params)
+      run_internal("_bootstrap.ops")
     end
 
     def shell(command)
+      run_internal("_shell.ops", {"command" => command})
+    end
+
+    def reboot()
+      run_internal("_reboot.ops")
+    end
+
+    def run_internal(ops_file_name, params = @params)
       set_pwd(__FILE__.to_pathname.dirname)
-      shell_ops_file = OpsFile.new(self, __FILE__.to_pathname.dirname.join("_shell.ops"))
-      op = OperationRunner.new(self, shell_ops_file)
-      puts "running #{command}"
-      result = op.run([], params_json_hash: {"command" => command})
+      internal_ops_file = OpsFile.new(self, __FILE__.to_pathname.dirname.join(ops_file_name))
+      op = OperationRunner.new(self, internal_ops_file)
+      result = op.run([], params_json_hash: params)
       puts "result class=#{result.class}"
       exit_status = result.exit_status
       stdout = JSON.pretty_generate(result.value)
@@ -243,7 +247,45 @@ module OpsWalrus
       1
     end
 
-    # args is of the form ["github.com/davidkellis/my-package/sub-package1", "operation1", "arg1:val1", "arg2:val2", "arg3:val3"]
+    # package_operation_and_args is of the form ["github.com/davidkellis/my-package/sub-package1", "operation1", "arg1:val1", "arg2:val2", "arg3:val3"]
+    # if the first argument is the path to a .ops file, then treat it as a local path, and add the containing package
+    #   to the load path
+    # otherwise, copy the
+    # returns the exit status code that the script should terminate with
+    def run_remote(package_operation_and_args, update_bundle: false)
+      return 0 if package_operation_and_args.empty?
+
+      ops_file_path, operation_kv_args, tmp_bundle_root_dir = get_entry_point_ops_file_and_args(package_operation_and_args)
+
+      ops_file = load_entry_point_ops_file(ops_file_path, tmp_bundle_root_dir)
+
+      bundler.update if update_bundle
+
+      debug "Running: #{ops_file.ops_file_path}"
+
+      internal_ops_file = OpsFile.new(self, __FILE__.to_pathname.dirname.join("_run_remote.ops"))
+
+      op = OperationRunner.new(self, internal_ops_file)
+      result = op.run([], params_json_hash: {ops_file: ops_file, operation_kv_args: operation_kv_args})
+      exit_status = result.exit_status
+
+      debug "Op exit_status"
+      debug exit_status
+
+      debug "Op output"
+      debug JSON.pretty_generate(result.value)
+
+      puts JSON.pretty_generate(result.value)
+
+      exit_status
+    rescue Error => e
+      puts "Error: #{e.message}"
+      1
+    ensure
+      FileUtils.remove_entry(tmp_bundle_root_dir) if tmp_bundle_root_dir
+    end
+
+    # package_operation_and_args is of the form ["github.com/davidkellis/my-package/sub-package1", "operation1", "arg1:val1", "arg2:val2", "arg3:val3"]
     # if the first argument is the path to a .ops file, then treat it as a local path, and add the containing package
     #   to the load path
     # otherwise, copy the

data/lib/opswalrus/cli.rb

@@ -30,13 +30,11 @@ module OpsWalrus
 
     program_desc 'ops is an operation runner'
 
-    switch [:v, :verbose], desc: "Verbose output"
-    switch :debug, desc: "Debug output"
-    switch :trace, desc: "Trace output"
+    switch :loud, desc: "Verbose output"
+    switch :louder, desc: "Debug output"
+    switch :loudest, desc: "Trace output"
 
-    switch :noop, desc: "Perform a dry run"
-    switch :dryrun, desc: "Perform a dry run"
-    switch :dry_run, desc: "Perform a dry run"
+    switch :dry, desc: "Perform a dry run"
 
     flag [:h, :hosts], multiple: true, desc: "Specify the hosts.yaml file"
     flag [:t, :tags], multiple: true, desc: "Specify a set of tags to filter the hosts by"
@@ -61,7 +59,7 @@ module OpsWalrus
           hosts = global_options[:hosts]
           tags = global_options[:tags]
 
-          $app.set_log_level(global_options[:trace] && :trace || global_options[:debug] && :debug || global_options[:verbose] && :info || :warn)
+          $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
 
           $app.report_inventory(hosts, tags: tags)
         end
@@ -123,12 +121,10 @@ module OpsWalrus
     long_desc 'Bootstrap a set of hotss to run opswalrus: install dependencies, ruby, opswalrus gem'
     command :bootstrap do |c|
       # dry run
-      c.switch :noop, desc: "Perform a dry run"
-      c.switch :dryrun, desc: "Perform a dry run"
-      c.switch :dry_run, desc: "Perform a dry run"
+      c.switch :dry, desc: "Perform a dry run"
 
       c.action do |global_options, options, args|
-        $app.set_log_level(global_options[:trace] && :trace || global_options[:debug] && :debug || global_options[:verbose] && :info || :warn)
+        $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
 
         hosts = global_options[:hosts]
         $app.set_inventory_hosts(hosts)
@@ -141,7 +137,7 @@ module OpsWalrus
 
         $app.set_identity_files(id_files)
 
-        dry_run = [:noop, :dryrun, :dry_run].any? {|sym| global_options[sym] || options[sym] }
+        dry_run = global_options[:dry] || options[:dry]
         $app.dry_run! if dry_run
 
         $app.bootstrap()
@@ -155,12 +151,10 @@ module OpsWalrus
       c.flag [:u, :user], desc: "Specify the user that the operation will run as"
 
       # dry run
-      c.switch :noop, desc: "Perform a dry run"
-      c.switch :dryrun, desc: "Perform a dry run"
-      c.switch :dry_run, desc: "Perform a dry run"
+      c.switch :dry, desc: "Perform a dry run"
 
       c.action do |global_options, options, args|
-        $app.set_log_level(global_options[:trace] && :trace || global_options[:debug] && :debug || global_options[:verbose] && :info || :warn)
+        $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
 
         hosts = global_options[:hosts]
         $app.set_inventory_hosts(hosts)
@@ -176,7 +170,7 @@ module OpsWalrus
 
         $app.set_identity_files(id_files)
 
-        dry_run = [:noop, :dryrun, :dry_run].any? {|sym| global_options[sym] || options[sym] }
+        dry_run = global_options[:dry] || options[:dry]
         $app.dry_run! if dry_run
 
         if options[:pass]
@@ -189,6 +183,35 @@ module OpsWalrus
       end
     end
 
+    desc "Reboot one or more remote hosts"
+    long_desc 'Reboot one or more remote hosts'
+    command :reboot do |c|
+      # dry run
+      c.switch :dry, desc: "Perform a dry run"
+
+      c.action do |global_options, options, args|
+        $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
+
+        hosts = global_options[:hosts]
+        $app.set_inventory_hosts(hosts)
+
+        tags = global_options[:tags]
+        $app.set_inventory_tags(tags)
+
+        id_files = global_options[:id]
+        id_files = OpsWalrus.env_specified_age_ids if id_files.empty?
+
+        $app.set_identity_files(id_files)
+
+        dry_run = global_options[:dry] || options[:dry]
+        $app.dry_run! if dry_run
+
+        exit_status = $app.reboot()
+
+        exit_now!("error", exit_status) unless exit_status == 0
+      end
+    end
+
     desc 'Run an operation from a package'
     long_desc 'Run the specified operation found within the specified package'
     arg 'args', :multiple
@@ -196,17 +219,16 @@ module OpsWalrus
       c.switch [:b, :bundle], desc: "Update bundle prior to running the specified operation"
       c.switch :pass, desc: "Prompt for a sudo password"
       c.switch :script, desc: "Script mode"
+      c.switch [:r, :remote], desc: "Run the operation on the remote hosts"
 
       c.flag [:u, :user], desc: "Specify the user that the operation will run as"
       c.flag [:p, :params], desc: "Either specify a file that contains JSON OR specify a JSON encoded string. In both cases, the JSON represents the runtime arguments (i.e. the params) for the operation. The JSON string must conform to the params schema for the operation being run."
 
       # dry run
-      c.switch :noop, desc: "Perform a dry run"
-      c.switch :dryrun, desc: "Perform a dry run"
-      c.switch :dry_run, desc: "Perform a dry run"
+      c.switch :dry, desc: "Perform a dry run"
 
       c.action do |global_options, options, args|
-        $app.set_log_level(global_options[:trace] && :trace || global_options[:debug] && :debug || global_options[:verbose] && :info || :warn)
+        $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
 
         hosts = global_options[:hosts]
         $app.set_inventory_hosts(hosts)
@@ -225,7 +247,7 @@ module OpsWalrus
 
         $app.set_identity_files(id_files)
 
-        dry_run = [:noop, :dryrun, :dry_run].any? {|sym| global_options[sym] || options[sym] }
+        dry_run = global_options[:dry] || options[:dry]
         $app.dry_run! if dry_run
 
         if options[:pass]
@@ -236,7 +258,11 @@ module OpsWalrus
           $app.script_mode!
         end
 
-        exit_status = $app.run(args, update_bundle: options[:bundle])
+        exit_status = if options[:remote]
+          $app.run_remote(args, update_bundle: options[:bundle])
+        else
+          $app.run(args, update_bundle: options[:bundle])
+        end
 
         exit_now!("error", exit_status) unless exit_status == 0
       end
@@ -250,7 +276,7 @@ module OpsWalrus
       long_desc 'Download and bundle the latest versions of dependencies for the current package'
       c.command :update do |update|
         update.action do |global_options, options, args|
-          $app.set_log_level(global_options[:trace] && :trace || global_options[:debug] && :debug || global_options[:verbose] && :info || :warn)
+          $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
 
           $app.bundle_update
         end
@@ -270,7 +296,7 @@ module OpsWalrus
         unzip.flag [:o, :output], desc: "Specify the output directory"
 
         unzip.action do |global_options, options, args|
-          $app.set_log_level(global_options[:trace] && :trace || global_options[:debug] && :debug || global_options[:verbose] && :info || :warn)
+          $app.set_log_level(global_options[:loudest] && :trace || global_options[:louder] && :debug || global_options[:loud] && :info || :warn)
 
           output_dir = options[:output]
           zip_file_path = args.first

data/lib/opswalrus/host.rb

@@ -81,6 +81,13 @@ module OpsWalrus
       @_host.to_s
     end
 
+    def _invoke_remote(ops_file, *args, **kwargs)
+      puts "boom"
+      is_invocation_a_call_to_package_in_bundle_dir = true
+      invocation_context = RemoteImportInvocationContext.new(@runtime_env, self, ops_file, is_invocation_a_call_to_package_in_bundle_dir, ops_prompt_for_sudo_password: !!ssh_password)
+      invocation_context._invoke(*args, **kwargs)
+    end
+
     # returns [stdout, stderr, exit_status]
     def _bootstrap_host(print_report = true)
       # copy over bootstrap shell script
@@ -119,7 +126,7 @@ module OpsWalrus
       raise Error, "Unable to upload ops bundle to remote host" unless upload_success
 
       stdout, _stderr, exit_status = @_host.run_ops(:bundle, "unzip tmpops.zip", in_bundle_root_dir: false)
-      raise Error, "Unable to unzip ops bundle on remote host" unless exit_status == 0
+      raise Error, "Unable to unzip ops bundle on remote host: #{stdout}" unless exit_status == 0
       tmp_bundle_root_dir = stdout.strip
       @_host.set_ssh_session_tmp_bundle_root_dir(tmp_bundle_root_dir)
 
@@ -143,6 +150,58 @@ module OpsWalrus
 
 
   module HostDSL
+    # delay: integer?     # default: 1 - 1 second delay before reboot
+    # sync: boolean?      # default: true - wait for the remote host to become available again before returning success/failure
+    # timeout: integer?   # default: 300 - 300 seconds (5 minutes)
+    def reboot(delay: 1, sync: true, timeout: 300)
+      delay = 1 if delay < 1
+
+      desc "Rebooting #{to_s} (alias=#{self.alias})"
+      reboot_success = sh? 'sudo /bin/sh -c "(sleep {{ delay }} && reboot) &"'.mustache
+      puts reboot_success
+
+      reconnect_time = nil
+      reconnect_success = if sync
+        desc "Waiting for #{to_s} (alias=#{self.alias}) to finish rebooting"
+        initial_reconnect_delay = delay + 10
+        sleep initial_reconnect_delay
+
+        reconnected = false
+        give_up = false
+        t1 = Time.now
+        until reconnected || give_up
+          begin
+            reconnected = sh?('true')
+            # while trying to reconnect, we expect the following exceptions:
+            # 1. Net::SSH::Disconnect < Net::SSH::Exception with message: "connection closed by remote host"
+            # 2. Errno::ECONNRESET < SystemCallError with message: "Connection reset by peer"
+            # 3. Errno::ECONNREFUSED < SystemCallError with message: "Connection refused - connect(2) for 192.168.56.10:22"
+          rescue Net::SSH::Disconnect, Errno::ECONNRESET, Errno::ECONNREFUSED => e
+            # noop; we expect these while we're trying to reconnect
+          rescue => e
+            puts "#{e.class} < #{e.class.superclass}"
+            puts e.message
+            puts e.backtrace.take(5).join("\n")
+          end
+
+          wait_time_elapsed_in_seconds = Time.now - t1
+          give_up = wait_time_elapsed_in_seconds > timeout
+          sleep 5
+        end
+        reconnect_time = initial_reconnect_delay + (Time.now - t1)
+        reconnected
+      else
+        false
+      end
+
+      {
+        success: reboot_success && (sync == reconnect_success),
+        rebooted: reboot_success,
+        reconnected: reconnect_success,
+        reboot_duration: reconnect_time
+      }
+    end
+
     # runs the given command
     # returns the stdout from the command
     def sh(desc_or_cmd = nil, cmd = nil, input: nil, &block)
@@ -163,7 +222,7 @@ module OpsWalrus
     end
 
     # returns the tuple: [stdout, stderr, exit_status]
-    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil, log_level: nil, ops_prompt_for_sudo_password: false)
+    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil, ops_prompt_for_sudo_password: false)
       # description = nil
 
       return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
@@ -179,7 +238,7 @@ module OpsWalrus
         else
           offset = 3    # 3, because 1 references the stack frame corresponding to the caller of WalrusLang.eval,
                         # 2 references the stack frame corresponding to the caller of shell!,
-                        # and 3 references the stack frame corresponding to teh caller of either sh/sh?/shell
+                        # and 3 references the stack frame corresponding to the caller of either sh/sh?/shell
           WalrusLang.eval(cmd, offset)
         end
       else
@@ -190,28 +249,24 @@ module OpsWalrus
       #cmd = Shellwords.escape(cmd)
 
       cmd_id = Random.uuid.split('-').first
-      # if App.instance.report_mode?
       output_block = StringIO.open do |io|
-        io.print Style.blue(host)
-        io.print " (#{Style.blue(self.alias)})" if self.alias
-        io.print " | #{Style.magenta(description)}" if description
-        io.puts
-        io.print Style.yellow(cmd_id)
-        io.print Style.green.bold(" > ")
-        io.puts Style.yellow(cmd)
+        if App.instance.info?   # this is true if log_level is trace, debug, info
+          io.print Style.blue(host)
+          io.print " (#{Style.blue(self.alias)})" if self.alias
+          io.print " | #{Style.magenta(description)}" if description
+          io.puts
+          io.print Style.yellow(cmd_id)
+          io.print Style.green.bold(" > ")
+          io.puts Style.yellow(cmd)
+        elsif App.instance.warn? && description
+          io.print Style.blue(host)
+          io.print " (#{Style.blue(self.alias)})" if self.alias
+          io.print " | #{Style.magenta(description)}" if description
+          io.puts
+        end
         io.string
       end
-      puts output_block
-
-        # puts Style.green("*" * 80)
-        # if self.alias
-        #   print "[#{Style.blue(self.alias)} | #{Style.blue(host)}] "
-        # else
-        #   print "[#{Style.blue(host)}] "
-        # end
-        # print "#{description}: " if description
-        # puts Style.yellow("[#{cmd_id}] #{cmd}")
-      # end
+      puts output_block unless output_block.empty?
 
       return unless cmd && !cmd.strip.empty?
 
@@ -226,27 +281,37 @@ module OpsWalrus
       seconds = t2 - t1
 
       output_block = StringIO.open do |io|
-        if App.instance.info? || log_level == :info
-          io.puts Style.cyan(out)
-          io.puts Style.red(err)
-        elsif App.instance.debug? || log_level == :debug
-          io.puts Style.cyan(out)
-          io.puts Style.red(err)
-        elsif App.instance.trace? || log_level == :trace
-          io.puts Style.cyan(out)
-          io.puts Style.red(err)
-        end
-        io.print Style.yellow(cmd_id)
-        io.print Style.blue(" | Finished in #{seconds} seconds with exit status ")
-        if exit_status == 0
-          io.puts Style.green("#{exit_status} (#{exit_status == 0 ? 'success' : 'failure'})")
-        else
-          io.puts Style.red("#{exit_status} (#{exit_status == 0 ? 'success' : 'failure'})")
+        if App.instance.info?   # this is true if log_level is trace, debug, info
+          if App.instance.trace?
+            io.puts Style.cyan(out)
+            io.puts Style.red(err)
+          elsif App.instance.debug?
+            io.puts Style.cyan(out)
+            io.puts Style.red(err)
+          elsif App.instance.info?
+            # io.puts Style.cyan(out)
+            # io.puts Style.red(err)
+          end
+          io.print Style.yellow(cmd_id)
+          io.print Style.blue(" | Finished in #{seconds} seconds with exit status ")
+          if exit_status == 0
+            io.puts Style.green("#{exit_status} (success)")
+          else
+            io.puts Style.red("#{exit_status} (failure)")
+          end
+          io.puts Style.green("*" * 80)
+        elsif App.instance.warn? && description
+          io.print Style.blue(" | Finished in #{seconds} seconds with exit status ")
+          if exit_status == 0
+            io.puts Style.green("#{exit_status} (success)")
+          else
+            io.puts Style.red("#{exit_status} (failure)")
+          end
+          io.puts Style.green("*" * 80)
         end
-        io.puts Style.green("*" * 80)
         io.string
       end
-      puts output_block
+      puts output_block unless output_block.empty?
 
       [out, err, exit_status]
     end
@@ -262,18 +327,18 @@ module OpsWalrus
       # cmd = "OPS_GEM=\"#{OPS_GEM}\" OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}'; $OPS_GEM exec --conservative -g opswalrus ops"
       cmd = "OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}' eval #{OPS_CMD}"
       if App.instance.trace?
-        cmd << " --trace"
+        cmd << " --loudest"
       elsif App.instance.debug?
-        cmd << " --debug"
+        cmd << " --louder"
       elsif App.instance.info?
-        cmd << " --verbose"
+        cmd << " --loud"
       end
       cmd << " #{ops_command.to_s}"
       cmd << " #{ops_command_options.to_s}" if ops_command_options
       cmd << " #{@tmp_bundle_root_dir}" if in_bundle_root_dir
       cmd << " #{command_arguments}" unless command_arguments.empty?
 
-      shell!(cmd, log_level: :info, ops_prompt_for_sudo_password: ops_prompt_for_sudo_password)
+      shell!(cmd, ops_prompt_for_sudo_password: ops_prompt_for_sudo_password)
     end
 
     def desc(msg)

data/lib/opswalrus/interaction_handlers.rb

@@ -8,9 +8,7 @@ module OpsWalrus
 
     attr_accessor :input_mappings   # Hash[ String | Regex => String ]
 
-    # log_level is one of: :fatal, :error, :warn, :info, :debug, :trace
-    def initialize(mapping, log_level = nil)
-      @log_level = log_level
+    def initialize(mapping)
       @input_mappings = mapping
     end
 
@@ -63,7 +61,7 @@ module OpsWalrus
       if new_mapping.empty? || new_mapping == @input_mappings
         yield self
       else
-        yield ScopedMappingInteractionHandler.new(new_mapping, @log_level)
+        yield ScopedMappingInteractionHandler.new(new_mapping)
       end
     end
 

data/lib/opswalrus/operation_runner.rb

@@ -110,12 +110,12 @@ module OpsWalrus
         Invocation::Error.new(e)
       end
 
-      if result.failure?
-        App.instance.debug "Ops script error details:"
-        App.instance.debug "Error: #{result.value}"
-        App.instance.debug "Status code: #{result.exit_status}"
-        App.instance.debug @entry_point_ops_file.script.to_s
-      end
+      # if result.failure?
+      #   App.instance.debug "Ops script error details:"
+      #   App.instance.debug "Error: #{result.value}"
+      #   App.instance.debug "Status code: #{result.exit_status}"
+      #   App.instance.debug @entry_point_ops_file.script.to_s
+      # end
 
       result
     end

data/lib/opswalrus/ops_file_script_dsl.rb

@@ -55,6 +55,9 @@ module OpsWalrus
       sshkit_hosts = hosts.map(&:sshkit_host)
       sshkit_host_to_ops_host_map = sshkit_hosts.zip(hosts).to_h
       ops_file_script = local_host = self
+
+      results_lock = Thread::Mutex.new
+      results = {}
       # on sshkit_hosts do |sshkit_host|
       SSHKit::Coordinator.new(sshkit_hosts).each(in: kwargs[:in] || :parallel) do |sshkit_host|
         # in this context, self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
@@ -73,6 +76,10 @@ module OpsWalrus
             # we run the block in the context of the host proxy object, s.t. `self` within the block evaluates to the host proxy object
             retval = host.instance_exec(local_host, &block)    # local_host is passed as the argument to the block
 
+            results_lock.synchronize do
+              results[host] = retval
+            end
+
             retval
           rescue SSHKit::Command::Failed => e
             App.instance.error "[!] Command failed:"
@@ -102,6 +109,7 @@ module OpsWalrus
           end
         end # runtime_env.handle_input
       end # SSHKit::Coordinator
+      results
     end # def ssh
 
     def ssh(*args, **kwargs, &block)
@@ -111,6 +119,9 @@ module OpsWalrus
       sshkit_hosts = hosts.map(&:sshkit_host)
       sshkit_host_to_ops_host_map = sshkit_hosts.zip(hosts).to_h
       ops_file_script = local_host = self
+
+      results_lock = Thread::Mutex.new
+      results = {}
       # bootstrap_shell_script = BootstrapLinuxHostShellScript
       # on sshkit_hosts do |sshkit_host|
       SSHKit::Coordinator.new(sshkit_hosts).each(in: kwargs[:in] || :parallel) do |sshkit_host|
@@ -134,6 +145,10 @@ module OpsWalrus
               puts "Failed to bootstrap #{host}. Unable to run operation."
             end
 
+            results_lock.synchronize do
+              results[host] = retval
+            end
+
             retval
           rescue SSHKit::Command::Failed => e
             App.instance.error "[!] Command failed:"
@@ -163,6 +178,7 @@ module OpsWalrus
           end
         end # runtime_env.handle_input
       end # SSHKit::Coordinator
+      results
     end # def ssh
 
     def current_dir
@@ -247,7 +263,7 @@ module OpsWalrus
     end
 
     # returns the tuple: [stdout, stderr, exit_status]
-    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil, log_level: nil)
+    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil)
       return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
 
       description = desc_or_cmd if cmd || block
@@ -261,7 +277,7 @@ module OpsWalrus
         else
           offset = 3    # 3, because 1 references the stack frame corresponding to the caller of WalrusLang.eval,
                         # 2 references the stack frame corresponding to the caller of shell!,
-                        # and 3 references the stack frame corresponding to teh caller of either sh/sh?/shell
+                        # and 3 references the stack frame corresponding to the caller of either sh/sh?/shell
           WalrusLang.eval(cmd, offset)
         end
       else
@@ -269,7 +285,7 @@ module OpsWalrus
       end
       #cmd = Shellwords.escape(cmd)
 
-      report_on(@runtime_env.local_hostname, description, cmd, log_level: log_level) do
+      report_on(@runtime_env.local_hostname, description, cmd) do
         if App.instance.dry_run?
           ["", "", 0]
         else
@@ -284,19 +300,25 @@ module OpsWalrus
       end
     end
 
-    def report_on(hostname, description = nil, cmd, log_level: nil)
+    def report_on(hostname, description = nil, cmd)
       cmd_id = Random.uuid.split('-').first
 
       output_block = StringIO.open do |io|
-        io.print Style.blue(hostname)
-        io.print " | #{Style.magenta(description)}" if description
-        io.puts
-        io.print Style.yellow(cmd_id)
-        io.print Style.green.bold(" > ")
-        io.puts Style.yellow(cmd)
+        if App.instance.info?   # this is true if log_level is trace, debug, info
+          io.print Style.blue(hostname)
+          io.print " | #{Style.magenta(description)}" if description
+          io.puts
+          io.print Style.yellow(cmd_id)
+          io.print Style.green.bold(" > ")
+          io.puts Style.yellow(cmd)
+        elsif App.instance.warn? && description
+          io.print Style.blue(hostname)
+          io.print " | #{Style.magenta(description)}" if description
+          io.puts
+        end
         io.string
       end
-      puts output_block
+      puts output_block unless output_block.empty?
 
       t1 = Time.now
       out, err, exit_status = yield
@@ -304,27 +326,37 @@ module OpsWalrus
       seconds = t2 - t1
 
       output_block = StringIO.open do |io|
-        if App.instance.info? || log_level == :info
-          io.puts Style.cyan(out)
-          io.puts Style.red(err)
-        elsif App.instance.debug? || log_level == :debug
-          io.puts Style.cyan(out)
-          io.puts Style.red(err)
-        elsif App.instance.trace? || log_level == :trace
-          io.puts Style.cyan(out)
-          io.puts Style.red(err)
-        end
-        io.print Style.yellow(cmd_id)
-        io.print Style.blue(" | Finished in #{seconds} seconds with exit status ")
-        if exit_status == 0
-          io.puts Style.green("#{exit_status} (#{exit_status == 0 ? 'success' : 'failure'})")
-        else
-          io.puts Style.red("#{exit_status} (#{exit_status == 0 ? 'success' : 'failure'})")
+        if App.instance.info?   # this is true if log_level is trace, debug, info
+          if App.instance.trace?
+            io.puts Style.cyan(out)
+            io.puts Style.red(err)
+          elsif App.instance.debug?
+            io.puts Style.cyan(out)
+            io.puts Style.red(err)
+          elsif App.instance.info?
+            # io.puts Style.cyan(out)
+            # io.puts Style.red(err)
+          end
+          io.print Style.yellow(cmd_id)
+          io.print Style.blue(" | Finished in #{seconds} seconds with exit status ")
+          if exit_status == 0
+            io.puts Style.green("#{exit_status} (success)")
+          else
+            io.puts Style.red("#{exit_status} (failure)")
+          end
+          io.puts Style.green("*" * 80)
+        elsif App.instance.warn? && description
+          io.print Style.blue(" | Finished in #{seconds} seconds with exit status ")
+          if exit_status == 0
+            io.puts Style.green("#{exit_status} (success)")
+          else
+            io.puts Style.red("#{exit_status} (failure)")
+          end
+          io.puts Style.green("*" * 80)
         end
-        io.puts Style.green("*" * 80)
         io.string
       end
-      puts output_block
+      puts output_block unless output_block.empty?
 
       [out, err, exit_status]
     end

data/lib/opswalrus/patches.rb

@@ -1,5 +1,7 @@
 require 'json'
 require 'pathname'
+require 'active_support'
+require 'active_support/core_ext/hash'
 
 class String
   def escape_single_quotes

data/lib/opswalrus/version.rb

@@ -1,3 +1,3 @@
 module OpsWalrus
-  VERSION = "1.0.51"
+  VERSION = "1.0.53"
 end

data/opswalrus.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # gem dependencies
+  spec.add_dependency "activesupport", "~> 7.0"
   spec.add_dependency "binding_of_caller", "~> 1.0"
   spec.add_dependency "citrus", "~> 3.0"
   spec.add_dependency "gli", "~> 2.21"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: opswalrus
 version: !ruby/object:Gem::Version
-  version: 1.0.51
+  version: 1.0.53
 platform: ruby
 authors:
 - David Ellis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-20 00:00:00.000000000 Z
+date: 2023-09-22 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: binding_of_caller
   requirement: !ruby/object:Gem::Requirement
@@ -185,6 +199,8 @@ files:
 - exe/ops
 - lib/opswalrus.rb
 - lib/opswalrus/_bootstrap.ops
+- lib/opswalrus/_reboot.ops
+- lib/opswalrus/_run_remote.ops
 - lib/opswalrus/_shell.ops
 - lib/opswalrus/app.rb
 - lib/opswalrus/bootstrap.sh