opswalrus 1.0.16 → 1.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6d990ac1fac1a918ebca3eac342377ee61702f0864e635e378ad8e69d7ec0d5
-  data.tar.gz: 900661d04774a42b0a960624fd7fc51b9690c23ad15fba5aaf94f8a3e1241de6
+  metadata.gz: 27cea4b01102c518b26f6b47e47a6461ff76aac39f4ab0094955aa6413d3e9f7
+  data.tar.gz: ce1b064a905fea231d4d551513a74f33f726cfcfdede0e4a2660acefc17c1b62
 SHA512:
-  metadata.gz: 88e94b27da26afb8b925f1b363c6ccf889f3e105d28e3f7d2ccfd1d2d8fe1b61d443d64dc1829c42b8644216e19bad1c4e384757246414b62dc18104421cb8fe
-  data.tar.gz: 3e5b2b2062a592102bed054b7bae779a66b676ca8d47beb42cbab0a6cc6096e191129b3fd9a2c6fc815f8964bf78270b15687cbba99787c0323fb51c87e82c9c
+  metadata.gz: 5bd3daea688e85fe18e8f61bb455b238478b8be1f6caf9be2bed096af0d20187155d4882a2fb2732044eaee3f93303f1592b3025fbdfc74775a323007bd1a544
+  data.tar.gz: 027a8423627f1af58966e9064157c72cdfca7aec5e74f04130cd8b563ec23217ee9dc0da9ccf43afdc1b07f39bd2ab69e3d744d7355b9f27ae1536d16180caea

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    opswalrus (1.0.16)
+    opswalrus (1.0.18)
       amazing_print (~> 1.5)
       bcrypt_pbkdf (~> 1.1)
       citrus (~> 3.0)
@@ -9,8 +9,10 @@ PATH
       git (~> 1.18)
       gli (~> 2.21)
       ougai (~> 2.0)
+      pastel (~> 0.8)
       rubyzip (~> 2.3)
       sshkit (~> 1.21)
+      tty-editor (~> 0.7)
 
 GEM
   remote: https://rubygems.org/
@@ -32,6 +34,8 @@ GEM
     oj (3.16.0)
     ougai (2.0.0)
       oj (~> 3.10)
+    pastel (0.8.0)
+      tty-color (~> 0.5)
     public_suffix (5.0.3)
     rake (13.0.6)
     rchardet (1.8.0)
@@ -52,6 +56,19 @@ GEM
     sshkit (1.21.5)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
+    tty-color (0.6.0)
+    tty-cursor (0.7.1)
+    tty-editor (0.7.0)
+      tty-prompt (~> 0.22)
+    tty-prompt (0.23.1)
+      pastel (~> 0.8)
+      tty-reader (~> 0.8)
+    tty-reader (0.9.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      wisper (~> 2.0)
+    tty-screen (0.8.1)
+    wisper (2.0.1)
 
 PLATFORMS
   x86_64-linux

data/lib/opswalrus/app.rb

@@ -1,14 +1,16 @@
+require "amazing_print"
 require "citrus"
 require "io/console"
 require "json"
 # require "logger"
 require "random/formatter"
 require "ougai"
+require "pastel"
+require "pathname"
 require "shellwords"
 require "socket"
 require "stringio"
 require "yaml"
-require "pathname"
 require_relative "errors"
 require_relative "patches"
 require_relative "git"
@@ -21,6 +23,8 @@ require_relative "version"
 
 
 module OpsWalrus
+  Style = Pastel.new(enabled: $stdout.tty?)
+
   class App
     def self.instance(*args)
       @instance ||= new(*args)
@@ -32,9 +36,14 @@ module OpsWalrus
     attr_reader :local_hostname
 
     def initialize(pwd = Dir.pwd)
-      @logger = Ougai::Logger.new($stdout, level: Logger::INFO)    # Logger.new($stdout, level: Logger::INFO)
+      @logger = Ougai::Logger.new($stdout)    # Logger.new($stdout, level: Logger::INFO)
+      @logger.level = :info    # , :trace or 'trace'
       @logger.formatter = Ougai::Formatters::Readable.new
 
+      # @logger.warn Style.yellow("warn"), foo: "bar", baz: {qux: "quux"}
+      # @logger.info Style.yellow("info"), foo: "bar", baz: {qux: "quux"}
+      # @logger.debug Style.yellow("debug"), foo: "bar", baz: {qux: "quux"}
+      # @logger.trace Style.yellow("trace"), foo: "bar", baz: {qux: "quux"}
 
       @verbose = false
       @sudo_user = nil
@@ -46,6 +55,7 @@ module OpsWalrus
       @bundler = Bundler.new(@pwd)
       @local_hostname = "localhost"
       @mode = :report     # :report | :script
+      @dry_run = false
     end
 
     def to_s
@@ -68,6 +78,14 @@ module OpsWalrus
       @mode == :script
     end
 
+    def dry_run?
+      @dry_run
+    end
+
+    def dry_run!
+      @dry_run = true
+    end
+
     def set_local_hostname(hostname)
       hostname = hostname.strip
       @local_hostname = hostname.empty? ? "localhost" : hostname
@@ -91,37 +109,54 @@ module OpsWalrus
       @bundler.bundle_dir
     end
 
-    def set_verbose(verbose)
-      @verbose = verbose
-      @logger.debug! if verbose?
+    # log_level = :fatal, :error, :warn, :info, :debug, :trace
+    # irb(main):018:0> Ougai::Logger::TRACE
+    # => -1
+    # irb(main):019:0> Ougai::Logger::DEBUG
+    # => 0
+    # irb(main):020:0> Ougai::Logger::INFO
+    # => 1
+    # irb(main):021:0> Ougai::Logger::WARN
+    # => 2
+    # irb(main):022:0> Ougai::Logger::ERROR
+    # => 3
+    # irb(main):023:0> Ougai::Logger::FATAL
+    # => 4
+    def set_log_level(log_level)
+      @logger.level = log_level
     end
 
     def verbose?
-      @verbose
+      @logger.level <= 1
     end
 
     def debug?
-      @verbose == 2
+      @logger.level <= 0
+    end
+
+    def fatal(*args)
+      @logger.fatal(*args)
     end
 
-    def error(msg)
-      @logger.error(msg)
+    def error(*args)
+      @logger.error(*args)
     end
 
-    def warn(msg)
-      @logger.warn(msg)
+    def warn(*args)
+      @logger.warn(*args)
     end
 
-    def log(msg)
-      @logger.info(msg)
+    def log(*args)
+      @logger.info(*args)
     end
+    alias_method :info, :log
 
-    def debug(msg)
-      @logger.debug(msg)
+    def debug(*args)
+      @logger.debug(*args)
     end
 
-    def trace(msg)
-      @logger.trace(msg)
+    def trace(*args)
+      @logger.trace(*args)
     end
 
     def set_pwd(pwd)

data/lib/opswalrus/bootstrap.sh

@@ -9,6 +9,7 @@ if [ -x "$(command -v /home/linuxbrew/.linuxbrew/bin/brew)" ]; then
     echo 'Ruby is already installed.' >&2
 
     # make sure the latest opswalrus gem is installed
+    # todo: figure out how to install this differently, so that test versions will work
     gem install opswalrus
 
     exit 0

data/lib/opswalrus/bundler.rb

@@ -26,6 +26,10 @@ module OpsWalrus
       FileUtils.mkdir_p(@bundle_dir) unless @bundle_dir.exist?
     end
 
+    def delete_pwd_bundle_directory
+      FileUtils.remove_dir(@bundle_dir) if @bundle_dir.exist?
+    end
+
     # # returns the OpsFile within the bundle directory that represents the given ops_file (which is outside of the bundle directory)
     # def build_bundle_for_ops_file(ops_file)
     #   if ops_file.package_file   # ops_file is part of larger package
@@ -50,6 +54,7 @@ module OpsWalrus
     # end
 
     def update
+      delete_pwd_bundle_directory
       ensure_pwd_bundle_directory_exists
 
       package_yaml_files = pwd.glob("./**/package.yaml") - pwd.glob("./**/#{BUNDLE_DIR}/**/package.yaml")
@@ -110,36 +115,16 @@ module OpsWalrus
       version = package_reference.version
 
       destination_package_path = @bundle_dir.join(package_reference.import_resolution_dirname)
-      FileUtils.remove_dir(destination_package_path) if destination_package_path.exist?
+
+      # we return early here under the assumption that an already downloaded package/version combo will not
+      # differ if we download it again multiple times to the same location
+      if destination_package_path.exist?
+        App.instance.log("Skipping #{package_reference} referenced in #{package_file.package_file_path} since it already has been downloaded to #{destination_package_path}")
+        return destination_package_path
+      end
+      # FileUtils.remove_dir(destination_package_path) if destination_package_path.exist?
 
       download_package_contents(package_file, local_name, package_url, version, destination_package_path)
-      # case
-      # when package_url =~ /\.git/                               # git reference
-      #   download_git_package(package_url, version, destination_package_path)
-      # when package_url.start_with?("file://")                   # local path
-      #   path = package_url.sub("file://", "")
-      #   path = path.to_pathname
-      #   package_path_to_download = if path.relative?            # relative path
-      #     package_file.containing_directory.join(path)
-      #   else                                                    # absolute path
-      #     path.realpath
-      #   end
-
-      #   raise Error, "Package not found: #{package_path_to_download}" unless package_path_to_download.exist?
-      #   FileUtils.cp_r(package_path_to_download, destination_package_path)
-      # when package_url.to_pathname.exist? || package_file.containing_directory.join(package_url).exist?     # local path
-      #   path = package_url.to_pathname
-      #   package_path_to_download = if path.relative?            # relative path
-      #     package_file.containing_directory.join(path)
-      #   else                                                    # absolute path
-      #     path.realpath
-      #   end
-
-      #   raise Error, "Package not found: #{package_path_to_download}" unless File.exist?(package_path_to_download)
-      #   FileUtils.cp_r(package_path_to_download, destination_package_path)
-      # else                                                      # git reference
-      #   download_git_package(package_url, version, destination_package_path)
-      # end
 
       destination_package_path
     end
@@ -147,10 +132,12 @@ module OpsWalrus
     def download_package_contents(package_file, local_name, package_url, version, destination_package_path)
       package_path = package_url.to_pathname
       package_path = package_path.to_s.gsub(/^~/, Dir.home).to_pathname
+      App.instance.trace("download_package_contents #{package_path}")
       if package_path.absolute? && package_path.exist?                                   # absolute path reference
         return case
         when package_path.directory?
           package_path_to_download = package_path.realpath
+          App.instance.debug("Copying #{package_path_to_download} to #{destination_package_path}")
           FileUtils.cp_r(package_path_to_download, destination_package_path)
         when package_path.file?
           raise Error, "Package reference must be a directory, not a file:: #{local_name}: #{package_path}"
@@ -163,6 +150,7 @@ module OpsWalrus
         if rebased_path.exist?
           return case
           when rebased_path.directory?
+            App.instance.debug("Copying #{package_path_to_download} to #{destination_package_path}")
             package_path_to_download = rebased_path.realpath
             FileUtils.cp_r(package_path_to_download, destination_package_path)
           when rebased_path.file?
@@ -174,6 +162,7 @@ module OpsWalrus
       end
 
       if package_uri = Git.repo?(package_url)                                                          # git repo
+        App.instance.debug("Cloning repo #{package_uri} into #{destination_package_path}")
         download_git_package(package_uri, version, destination_package_path)
       end
     end
@@ -195,15 +184,10 @@ module OpsWalrus
     end
 
     def dynamic_package_path_for_git_package(package_url, version = nil)
-      package_reference_dirname = sanitize_path(package_url)
+      package_reference_dirname = DynamicPackageReference.import_resolution_dirname(package_url, version)
       bundle_dir.join(package_reference_dirname)
     end
 
-    def sanitize_path(path)
-      # found this at https://apidock.com/rails/v5.2.3/ActiveStorage/Filename/sanitized
-      path.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-")
-    end
-
     # returns the directory that the zip file is unzipped into
     def unzip(zip_bundle_file, output_dir = nil)
       if zip_bundle_file.to_pathname.exist?

data/lib/opswalrus/cli.rb

@@ -9,7 +9,7 @@ module OpsWalrus
 
     pre do |global_options, command, options, args|
       $app = App.instance(Dir.pwd)
-      $app.set_local_hostname(ENV["WALRUS_LOCAL_HOSTNAME"]) if ENV["WALRUS_LOCAL_HOSTNAME"]
+      $app.set_local_hostname(ENV["OPSWALRUS_LOCAL_HOSTNAME"]) if ENV["OPSWALRUS_LOCAL_HOSTNAME"]
       true
     end
 
@@ -31,6 +31,10 @@ module OpsWalrus
     desc 'Turn on debug mode'
     switch [:d, :debug]
 
+    switch :noop, desc: "Perform a dry run"
+    switch :dryrun, desc: "Perform a dry run"
+    switch :dry_run, desc: "Perform a dry run"
+
     flag [:h, :hosts], multiple: true, desc: "Specify the hosts.yaml file"
     flag [:t, :tags], multiple: true, desc: "Specify a set of tags to filter the hosts by"
 
@@ -48,7 +52,8 @@ module OpsWalrus
         hosts = global_options[:hosts]
         tags = global_options[:tags]
 
-        $app.set_verbose(global_options[:debug] || global_options[:verbose])
+        log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
+        $app.set_log_level(log_level)
 
         $app.report_inventory(hosts, tags: tags)
       end
@@ -63,28 +68,31 @@ module OpsWalrus
       c.flag [:p, :params], desc: "JSON string that represents the input parameters for the operation. The JSON string must conform to the params schema for the operation."
       c.switch :script, desc: "Script mode"
 
+      # dry run
+      c.switch :noop, desc: "Perform a dry run"
+      c.switch :dryrun, desc: "Perform a dry run"
+      c.switch :dry_run, desc: "Perform a dry run"
+
       c.action do |global_options, options, args|
+        log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
+        $app.set_log_level(log_level)
+
         hosts = global_options[:hosts] || []
         tags = global_options[:tags] || []
 
         $app.set_inventory_hosts(hosts)
         $app.set_inventory_tags(tags)
 
-        verbose = case
-        when global_options[:debug]
-          2
-        when global_options[:verbose]
-          1
-        end
-
         user = options[:user]
         params = options[:params]
 
-        $app.set_verbose(verbose)
         $app.set_params(params)
 
         $app.set_sudo_user(user) if user
 
+        dry_run = [:noop, :dryrun, :dry_run].any? {|sym| global_options[sym] || options[sym] }
+        $app.dry_run! if dry_run
+
         if options[:pass]
           $app.prompt_sudo_password
         end
@@ -107,6 +115,9 @@ module OpsWalrus
       long_desc 'Download and bundle the latest versions of dependencies for the current package'
       c.command :update do |update|
         update.action do |global_options, options, args|
+          log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
+          $app.set_log_level(log_level)
+
           $app.bundle_update
         end
       end
@@ -125,6 +136,9 @@ module OpsWalrus
         unzip.flag [:o, :output], desc: "Specify the output directory"
 
         unzip.action do |global_options, options, args|
+          log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
+          $app.set_log_level(log_level)
+
           output_dir = options[:output]
           zip_file_path = args.first
 

data/lib/opswalrus/host.rb

@@ -18,20 +18,20 @@ module OpsWalrus
       ops_file.local_symbol_table.each do |symbol_name, import_reference|
         unless methods_defined.include? symbol_name
           klass.define_method(symbol_name) do |*args, **kwargs, &block|
-            # puts "resolving local symbol table entry: #{symbol_name}"
+            App.instance.trace "resolving local symbol table entry: #{symbol_name}"
             namespace_or_ops_file = @runtime_env.resolve_import_reference(ops_file, import_reference)
-            # puts "namespace_or_ops_file=#{namespace_or_ops_file.to_s}"
+            App.instance.trace "namespace_or_ops_file=#{namespace_or_ops_file.to_s}"
 
             invocation_context = case import_reference
             # we know we're dealing with a package dependency reference, so we want to run an ops file contained within the bundle directory,
             # therefore, we want to reference the specified ops file with respect to the bundle dir
             when PackageDependencyReference
-              RemoteImportInvocationContext.new(@runtime_env, self, namespace_or_ops_file, true)
+              RemoteImportInvocationContext.new(@runtime_env, self, namespace_or_ops_file, true, prompt_for_sudo_password: !!ssh_password)
 
             # we know we're dealing with a directory reference or OpsFile reference outside of the bundle dir, so we want to reference
             # the specified ops file with respect to the root directory, and not with respect to the bundle dir
             when DirectoryReference, OpsFileReference
-              RemoteImportInvocationContext.new(@runtime_env, self, namespace_or_ops_file, false)
+              RemoteImportInvocationContext.new(@runtime_env, self, namespace_or_ops_file, false, prompt_for_sudo_password: !!ssh_password)
             end
 
             invocation_context._invoke(*args, **kwargs)
@@ -49,7 +49,11 @@ module OpsWalrus
         unless methods_defined.include? symbol_name
           # puts "2. defining: #{symbol_name}(...)"
           klass.define_method(symbol_name) do |*args, **kwargs, &block|
-            invocation_context = RemoteImportInvocationContext.new(@runtime_env, self, namespace_or_ops_file, false)
+            App.instance.trace "resolving implicit import: #{symbol_name}"
+            namespace_or_ops_file = @runtime_env.resolve_sibling_symbol(ops_file, symbol_name)
+            App.instance.trace "namespace_or_ops_file=#{namespace_or_ops_file.to_s}"
+
+            invocation_context = RemoteImportInvocationContext.new(@runtime_env, self, namespace_or_ops_file, false, prompt_for_sudo_password: !!ssh_password)
             invocation_context._invoke(*args, **kwargs)
           end
           methods_defined << symbol_name
@@ -107,13 +111,14 @@ module OpsWalrus
       #cmd = Shellwords.escape(cmd)
 
       if App.instance.report_mode?
+        puts Style.green("*" * 80)
         if self.alias
-          print "[#{self.alias} | #{host}] "
+          print "[#{Style.blue(self.alias)} | #{Style.blue(host)}] "
         else
-          print "[#{host}] "
+          print "[#{Style.blue(host)}] "
         end
         print "#{description}: " if description
-        puts cmd
+        puts Style.yellow(cmd)
       end
 
       return unless cmd && !cmd.strip.empty?
@@ -122,9 +127,12 @@ module OpsWalrus
       # puts "shell: #{cmd.inspect}"
       # puts "sudo_password: #{sudo_password}"
 
-      sshkit_cmd = execute_cmd(cmd, input: input)
-
-      [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
+      if App.instance.dry_run?
+        ["", "", 0]
+      else
+        sshkit_cmd = execute_cmd(cmd, input: input)
+        [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
+      end
     end
 
     # def init_brew
@@ -137,7 +145,13 @@ module OpsWalrus
       # e.g. /home/linuxbrew/.linuxbrew/bin/gem exec -g opswalrus ops run echo.ops args:foo args:bar
 
       # cmd = "/home/linuxbrew/.linuxbrew/bin/gem exec -g opswalrus ops"
-      cmd = "/home/linuxbrew/.linuxbrew/bin/gem exec -g opswalrus ops"
+      local_hostname_for_remote_host = if self.alias
+        "#{self.alias} | #{host}"
+      else
+        host
+      end
+
+      cmd = "OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}'; /home/linuxbrew/.linuxbrew/bin/gem exec -g opswalrus ops"
       cmd << " -v" if verbose
       cmd << " #{ops_command.to_s}"
       cmd << " #{ops_command_options.to_s}" if ops_command_options

data/lib/opswalrus/interaction_handlers.rb

@@ -5,6 +5,7 @@ module OpsWalrus
   class ScopedMappingInteractionHandler
     attr_accessor :input_mappings   # Hash[ String | Regex => String ]
 
+    # log_level is one of: :fatal, :error, :warn, :info, :debug, :trace
     def initialize(mapping, log_level = nil)
       @log_level = log_level
       @input_mappings = mapping
@@ -21,11 +22,10 @@ module OpsWalrus
     # end
 
     # sudo_password : String
-    def mapping_for_sudo_password(sudo_password)
+    def self.mapping_for_sudo_password(sudo_password)
       {
         /\[sudo\] password for .*?:\s*/ => "#{sudo_password}\n",
         App::LOCAL_SUDO_PASSWORD_PROMPT => "#{sudo_password}\n",
-        # /\s+/ => nil,     # unnecessary
       }
     end
 
@@ -39,7 +39,7 @@ module OpsWalrus
       raise ArgumentError.new("mapping must be a Hash") unless mapping.is_a?(Hash)
 
       if sudo_password
-        mapping.merge!(mapping_for_sudo_password(sudo_password))
+        mapping.merge!(ScopedMappingInteractionHandler.mapping_for_sudo_password(sudo_password))
       end
 
       if mapping.empty?
@@ -57,17 +57,16 @@ module OpsWalrus
     end
 
     def on_data(_command, stream_name, data, channel)
-      log("Looking up response for #{stream_name} message #{data.inspect}")
-
       response_data = begin
         first_matching_key_value_pair = @input_mappings.find {|k, _v| k === data }
         first_matching_key_value_pair&.last
       end
 
       if response_data.nil?
-        log("Unable to find interaction handler mapping for #{stream_name}: #{data.inspect} so no response was sent")
+        trace(Style.red("No interaction handler mapping for #{stream_name}: #{data} so no response was sent"))
       else
-        log("Sending #{response_data.inspect}")
+        debug(Style.cyan("Handling #{stream_name} message #{data}"))
+        debug(Style.cyan("Sending response #{response_data}"))
         if channel.respond_to?(:send_data)  # Net SSH Channel
           channel.send_data(response_data)
         elsif channel.respond_to?(:write)   # Local IO
@@ -80,16 +79,21 @@ module OpsWalrus
 
     private
 
-    def log(message)
-      # puts message
-      SSHKit.config.output.send(@log_level, message) unless @log_level.nil?
+    def trace(message)
+      App.instance.trace(message)
+    end
+
+    def debug(message)
+      App.instance.debug(message)
+      if [:fatal, :error, :warn, :info, :debug, :trace].include? @log_level
+        SSHKit.config.output.send(@log_level, message)
+      end
     end
 
   end
 
   class PasswdInteractionHandler
     def on_data(command, stream_name, data, channel)
-      # puts data
       case data
       when '(current) UNIX password: '
         channel.send_data("old_pw\n")
@@ -118,8 +122,6 @@ module OpsWalrus
 
   class SudoPromptInteractionHandler
     def on_data(command, stream_name, data, channel)
-      # puts "0" * 80
-      # puts data.inspect
       case data
       when /\[sudo\] password for/
         if channel.respond_to?(:send_data)  # Net::SSH channel
@@ -128,12 +130,11 @@ module OpsWalrus
           channel.write("conquer\n")
         end
       when /\s+/
-        puts 'space, do nothing'
+        nil
       else
         raise "Unexpected prompt: #{data} on stream #{stream_name} and channel #{channel.inspect}"
-    end
+      end
     end
   end
 
-
 end