opswalrus 1.0.5 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 727948ba554b3da5510261c8ddc8983a8113b7df7c30fb769a2291bc0af77195
-  data.tar.gz: 70a553c15a613d41300775b0f063cb2c34fcc9998d2f8d26d310a1c88fec3a17
+  metadata.gz: b0c1c00f88ff00a4f7eb735a634ea35e1e8818b44fffdccea7a2347bac78e199
+  data.tar.gz: fe54254076c753cfeaf26906fa0f7475bd11807c0ac70d4067da03653b0f774c
 SHA512:
-  metadata.gz: d1d4198f362160b5d0f1baba4fe0999f382b3cf1250cd8a608859901e55bb8f307673761c6699f60c81a70814d8eec3a67b66de241e94f95856dbdf69f703dc2
-  data.tar.gz: 3fd8074d05b9c50976ef81a700fe95217046e7e7ced70799e6f65eac46b5d61f14f1ee267191e64264663a4d11bb3b61d202d4eab32c80e08e92b0d9769ca7ff
+  metadata.gz: 28244a51a465941731663e9db3901049d36d1fa385a16b4c5ed9254c31fa62ca44012c2d776ef5b6d1d06906c012802b1352e166d72d28b3ec0aa6f9e27073f5
+  data.tar.gz: 39ccb70ca9dc5b2ecf67b1ab29adb7ccc79cb43ad57d256be0deb68b49909bc4c5cd4ddc2871e68aeeb374ed3d0202c365094536e7933f61c3ecf700a2178076

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    opswalrus (1.0.4)
-      bcrypt_pbkdf
-      citrus
-      ed25519
-      git
-      gli
-      rubyzip
-      sshkit
+    opswalrus (1.0.7)
+      bcrypt_pbkdf (~> 1.1)
+      citrus (~> 3.0)
+      ed25519 (~> 1.3)
+      git (~> 1.18)
+      gli (~> 2.21)
+      rubyzip (~> 2.3)
+      sshkit (~> 1.21)
 
 GEM
   remote: https://rubygems.org/

data/build.ops

@@ -0,0 +1,40 @@
+params:
+  version: string
+
+imports:
+  core: "https://github.com/opswalrus/core.git"
+...
+
+version = params.version
+
+exit 1, "version parameter must be specified" unless version
+
+template = <<TEMPLATE
+module OpsWalrus
+  VERSION = "{{ version }}"
+end
+TEMPLATE
+
+puts "Write version.rb for version #{version}"
+core.template.write(path: "./lib/opswalrus/version.rb", template: template, variables: {version: version})
+
+sh("Build gem") { 'gem build opswalrus.gemspec' }
+bw_status_output = sh("Check whether Bitwarden is locked or not") { 'bw status' }
+# the status command currently exhibits an error in which it emits 'mac failed.' some number of times, so we need to filter that out
+# see:
+# - https://community.bitwarden.com/t/what-does-mac-failed-mean-exactly/29208
+# - https://github.com/bitwarden/cli/issues/88
+# - https://github.com/vwxyzjn/portwarden/issues/22
+# ❯ bw status
+# mac failed.
+# {"serverUrl":"...","lastSync":"2023-08-17T19:14:09.384Z","userEmail":"...","userId":"...","status":"locked"}
+bw_status_output = bw_status_output.gsub('mac failed.', '').strip
+bw_status_json = bw_status_output.parse_json
+
+if bw_status_json['status'] != 'unlocked'
+  exit 0, "Bitwarden is not unlocked. Please unlock bitwarden with: bw unlock"
+end
+
+totp = sh("Get Rubygems OTP") { 'bw get totp Rubygems' }
+sh("Push gem", input: {"You have enabled multi-factor authentication. Please enter OTP code." => "#{totp}\n"}) { 'gem push opswalrus-{{ version }}.gem' }
+sh("Build docker image") { 'docker build -t opswalrus/ops:{{ version }} .' }

data/lib/opswalrus/app.rb

@@ -1,5 +1,4 @@
 require "citrus"
-require "git"
 require "io/console"
 require "json"
 require "random/formatter"
@@ -8,27 +7,14 @@ require "socket"
 require "stringio"
 require "yaml"
 require "pathname"
+require_relative "patches"
+require_relative "git"
 require_relative "host"
 require_relative "hosts_file"
 require_relative "operation_runner"
 require_relative "bundler"
 require_relative "package_file"
 
-class String
-  def escape_single_quotes
-    gsub("'"){"\\'"}
-  end
-
-  def to_pathname
-    Pathname.new(self)
-  end
-end
-
-class Pathname
-  def to_pathname
-    self
-  end
-end
 
 module OpsWalrus
   class Error < StandardError
@@ -131,6 +117,7 @@ module OpsWalrus
     def prompt_sudo_password
       password = IO::console.getpass(LOCAL_SUDO_PASSWORD_PROMPT)
       set_sudo_password(password)
+      # puts "sudo password = |#{password}|"
       nil
     end
 
@@ -158,24 +145,11 @@ module OpsWalrus
       ops_file = OpsFile.new(self, ops_file_path)
 
       # if the ops file is part of a package, then set the package directory as the app's pwd
-      # puts "run1: #{ops_file.ops_file_path}"
       if ops_file.package_file && ops_file.package_file.dirname.to_s !~ /#{Bundler::BUNDLE_DIR}/
-        # puts "set pwd: #{ops_file.package_file.dirname}"
-        set_pwd(ops_file.package_file.dirname)
-        rebased_ops_file_relative_path = ops_file.ops_file_path.relative_path_from(ops_file.package_file.dirname)
-        # note: rebased_ops_file_relative_path is a relative path that is relative to ops_file.package_file.dirname
-        # puts "rebased path: #{rebased_ops_file_relative_path}"
-        absolute_ops_file_path = ops_file.package_file.dirname.join(rebased_ops_file_relative_path)
-        # puts "absolute path: #{absolute_ops_file_path}"
-        ops_file = OpsFile.new(self, absolute_ops_file_path)
+        ops_file = set_pwd_to_ops_file_package_directory(ops_file)
       end
-      # puts "run2: #{ops_file.ops_file_path}"
 
       op = OperationRunner.new(self, ops_file)
-      # if op.requires_sudo?
-      #   prompt_sudo_password unless sudo_password
-      # end
-      # exit_status, out, err, script_output_structure = op.run(operation_kv_args, params_json_hash: @params, verbose: @verbose)
       result = op.run(operation_kv_args, params_json_hash: @params, verbose: @verbose)
       exit_status = result.exit_status
 
@@ -183,14 +157,7 @@ module OpsWalrus
         puts "Op exit_status"
         puts exit_status
 
-        # puts "Op stdout"
-        # puts out
-
-        # puts "Op stderr"
-        # puts err
-
         puts "Op output"
-        # puts script_output_structure ? JSON.pretty_generate(script_output_structure) : nil.inspect
         puts JSON.pretty_generate(result.value)
       end
 
@@ -203,6 +170,19 @@ module OpsWalrus
       FileUtils.remove_entry(tmp_dir) if tmp_dir
     end
 
+    # sets the App's pwd to the ops file's package directory and
+    # returns a new OpsFile that points at the revised pathname with considered as relative to the package file's directory
+    def set_pwd_to_ops_file_package_directory(ops_file)
+      # puts "set pwd: #{ops_file.package_file.dirname}"
+      set_pwd(ops_file.package_file.dirname)
+      rebased_ops_file_relative_path = ops_file.ops_file_path.relative_path_from(ops_file.package_file.dirname)
+      # note: rebased_ops_file_relative_path is a relative path that is relative to ops_file.package_file.dirname
+      # puts "rebased path: #{rebased_ops_file_relative_path}"
+      absolute_ops_file_path = ops_file.package_file.dirname.join(rebased_ops_file_relative_path)
+      # puts "absolute path: #{absolute_ops_file_path}"
+      OpsFile.new(self, absolute_ops_file_path)
+    end
+
     # package_operation_and_args can take one of the following forms:
     # - ["github.com/davidkellis/my-package", "operation1", "arg1:val1", "arg2:val2", "arg3:val3"]
     # - ["foo.zip", "foo/myfile.ops", "arg1:val1", "arg2:val2", "arg3:val3"]
@@ -244,7 +224,7 @@ module OpsWalrus
 
         # operation_kv_args = package_operation_and_args
         [ops_file_path, operation_kv_args, tmp_dir]
-      when repo_url = git_repo?(package_or_ops_file_reference)
+      when repo_url = Git.repo?(package_or_ops_file_reference)
         destination_package_path = bundler.download_git_package(repo_url)
 
         ops_file_path, operation_kv_args = find_entry_point_ops_file_in_dir(destination_package_path, package_operation_and_args)
@@ -301,22 +281,6 @@ module OpsWalrus
       [ops_file_path, operation_kv_args]
     end
 
-    # git_repo?("davidkellis/arborist") -> "https://github.com/davidkellis/arborist"
-    # returns the repo URL
-    def git_repo?(repo_reference)
-      candidate_repo_references = [
-        repo_reference,
-        repo_reference =~ /(\.(com|net|org|dev|io|local))\// && "https://#{repo_reference}",
-        repo_reference !~ /github\.com\// && repo_reference =~ /^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38}\/([\w\.@\:\-~]+)$/i && "https://github.com/#{repo_reference}"    # this regex is from https://www.npmjs.com/package/github-username-regex and https://www.debuggex.com/r/H4kRw1G0YPyBFjfm
-      ].compact
-      working_repo_reference = candidate_repo_references.find {|reference| Git.ls_remote(reference) rescue nil }
-      working_repo_reference
-    end
-
-    # def is_dir_git_repo?(dir_path)
-    #   Git.ls_remote(reference) rescue nil
-    # end
-
     def bundle_status
     end
 

data/lib/opswalrus/bundler.rb

@@ -22,8 +22,8 @@ module OpsWalrus
       @bundle_dir
     end
 
-    def ensure_package_bundle_directory_exists
-      FileUtils.mkdir_p(@bundle_dir)
+    def ensure_pwd_bundle_directory_exists
+      FileUtils.mkdir_p(@bundle_dir) unless @bundle_dir.exist?
     end
 
     # # returns the OpsFile within the bundle directory that represents the given ops_file (which is outside of the bundle directory)
@@ -50,7 +50,7 @@ module OpsWalrus
     # end
 
     def update
-      ensure_package_bundle_directory_exists
+      ensure_pwd_bundle_directory_exists
 
       package_yaml_files = pwd.glob("./**/package.yaml") - pwd.glob("./**/#{BUNDLE_DIR}/**/package.yaml")
       package_files_within_pwd = package_yaml_files.map {|path| PackageFile.new(path.realpath) }
@@ -83,7 +83,7 @@ module OpsWalrus
 
     # returns the self_pkg directory within the bundle directory
     # def include_directory_in_bundle_as_self_pkg(dirname = pwd)
-    #   ensure_package_bundle_directory_exists
+    #   ensure_pwd_bundle_directory_exists
 
     #   destination_package_path = @bundle_dir.join("self_pkg")
 
@@ -103,7 +103,7 @@ module OpsWalrus
     # This method downloads a package_url that is a dependency referenced in the specified package_file
     # returns the destination directory that the package was downloaded to
     def download_package(package_file, package_reference)
-      ensure_package_bundle_directory_exists
+      ensure_pwd_bundle_directory_exists
 
       local_name = package_reference.local_name
       package_url = package_reference.package_uri
@@ -144,19 +144,26 @@ module OpsWalrus
     end
 
     def download_git_package(package_url, version = nil, destination_package_path = nil)
-      destination_package_path ||= begin
-        package_reference_dirname = sanitize_path(package_url)
-        bundle_dir.join(package_reference_dirname)
-      end
-      FileUtils.remove_dir(destination_package_path) if File.exist?(destination_package_path)
+      ensure_pwd_bundle_directory_exists
+
+      destination_package_path ||= dynamic_package_path_for_git_package(package_url, version)
+
+      return destination_package_path if destination_package_path.exist?
+
       if version
-        Git.clone(package_url, destination_package_path, branch: version, config: ['submodule.recurse=true'])
+        ::Git.clone(package_url, destination_package_path, branch: version, config: ['submodule.recurse=true'])
       else
-        Git.clone(package_url, destination_package_path, config: ['submodule.recurse=true'])
+        ::Git.clone(package_url, destination_package_path, config: ['submodule.recurse=true'])
       end
+
       destination_package_path
     end
 
+    def dynamic_package_path_for_git_package(package_url, version = nil)
+      package_reference_dirname = sanitize_path(package_url)
+      bundle_dir.join(package_reference_dirname)
+    end
+
     def sanitize_path(path)
       # found this at https://apidock.com/rails/v5.2.3/ActiveStorage/Filename/sanitized
       path.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-")

data/lib/opswalrus/git.rb

@@ -0,0 +1,22 @@
+require "git"
+
+module OpsWalrus
+  class Git
+    # repo?("davidkellis/arborist") -> "https://github.com/davidkellis/arborist"
+    # returns the repo URL or directory path
+    def self.repo?(repo_reference)
+      if Dir.exist?(repo_reference)
+        ::Git.ls_remote(repo_reference) && repo_reference rescue nil
+      else
+        candidate_repo_references = [
+          repo_reference,
+          repo_reference =~ /(\.(com|net|org|dev|io|local))\// && "https://#{repo_reference}",
+          repo_reference !~ /github\.com\// && repo_reference =~ /^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38}\/([\w\.@\:\-~]+)$/i && "https://github.com/#{repo_reference}"    # this regex is from https://www.npmjs.com/package/github-username-regex and https://www.debuggex.com/r/H4kRw1G0YPyBFjfm
+        ].compact
+        working_repo_reference = candidate_repo_references.find {|reference| ::Git.ls_remote(reference) rescue nil }
+        working_repo_reference
+      end
+    end
+
+  end
+end

data/lib/opswalrus/host.rb

@@ -7,18 +7,18 @@ module OpsWalrus
 
   module HostDSL
     # returns the stdout from the command
-    def sh(desc_or_cmd = nil, cmd = nil, stdin: nil, &block)
-      out, err, status = *shell!(desc_or_cmd, cmd, block, stdin: stdin)
+    def sh(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+      out, err, status = *shell!(desc_or_cmd, cmd, block, input: input)
       out
     end
 
     # returns the tuple: [stdout, stderr, exit_status]
-    def shell(desc_or_cmd = nil, cmd = nil, stdin: nil, &block)
-      shell!(desc_or_cmd, cmd, block, stdin: stdin)
+    def shell(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+      shell!(desc_or_cmd, cmd, block, input: input)
     end
 
     # returns the tuple: [stdout, stderr, exit_status]
-    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, stdin: nil)
+    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil)
       # description = nil
 
       return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
@@ -27,6 +27,10 @@ module OpsWalrus
       cmd = block.call if block
       cmd ||= desc_or_cmd
 
+      cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
+
+      #cmd = Shellwords.escape(cmd)
+
       if self.alias
         print "[#{self.alias} | #{host}] "
       else
@@ -35,16 +39,13 @@ module OpsWalrus
       print "#{description}: " if description
       puts cmd
 
-      cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
       return unless cmd && !cmd.strip.empty?
 
-      #cmd = Shellwords.escape(cmd)
-
       # puts "shell: #{cmd}"
       # puts "shell: #{cmd.inspect}"
       # puts "sudo_password: #{sudo_password}"
 
-      sshkit_cmd = execute_cmd(cmd)
+      sshkit_cmd = execute_cmd(cmd, input: input)
 
       [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
     end
@@ -202,6 +203,10 @@ module OpsWalrus
       })
     end
 
+    def set_runtime_env(runtime_env)
+      @runtime_env = runtime_env
+    end
+
     def set_ssh_session_connection(sshkit_backend)
       @sshkit_backend = sshkit_backend
     end
@@ -211,18 +216,27 @@ module OpsWalrus
     end
 
     def clear_ssh_session
+      @runtime_env = nil
       @sshkit_backend = nil
       @tmp_bundle_root_dir = nil
     end
 
-    def execute(*args)
+    def execute(*args, input: nil)
       # puts "interaction handler responds with: #{ssh_password}"
-      @sshkit_backend.capture(*args, interaction_handler: SudoPasswordMapper.new(ssh_password).interaction_handler, verbosity: :info)
+      # @sshkit_backend.capture(*args, interaction_handler: SudoPasswordMapper.new(ssh_password).interaction_handler, verbosity: :info)
       # @sshkit_backend.capture(*args, interaction_handler: SudoPromptInteractionHandler.new, verbosity: :info)
+
+      @runtime_env.handle_input(input, ssh_password) do |interaction_handler|
+        @sshkit_backend.capture(*args, interaction_handler: interaction_handler, verbosity: :info)
+      end
+
     end
 
-    def execute_cmd(*args)
-      @sshkit_backend.execute_cmd(*args, interaction_handler: SudoPasswordMapper.new(ssh_password).interaction_handler, verbosity: :info)
+    def execute_cmd(*args, input: nil)
+      # @sshkit_backend.execute_cmd(*args, interaction_handler: SudoPasswordMapper.new(ssh_password).interaction_handler, verbosity: :info)
+      @runtime_env.handle_input(input, ssh_password) do |interaction_handler|
+        @sshkit_backend.execute_cmd(*args, interaction_handler: interaction_handler, verbosity: :info)
+      end
     end
 
     def upload(local_path_or_io, remote_path)

data/lib/opswalrus/interaction_handlers.rb

@@ -2,6 +2,91 @@ require 'sshkit'
 
 module OpsWalrus
 
+  class ScopedMappingInteractionHandler
+    attr_accessor :input_mappings   # Hash[ String | Regex => String ]
+
+    def initialize(mapping, log_level = nil)
+      @log_level = log_level
+      @input_mappings = mapping
+    end
+
+    # temporarily adds a sudo password mapping to the interaction handler while the given block is being evaluated
+    # when the given block returns, then the temporary mapping is removed from the interaction handler
+    # def with_sudo_password(password, &block)
+    #   with_mapping({
+    #     /\[sudo\] password for .*?:\s*/ => "#{password}\n",
+    #     App::LOCAL_SUDO_PASSWORD_PROMPT => "#{password}\n",
+    #     # /\s+/ => nil,     # unnecessary
+    #   }, &block)
+    # end
+
+    # sudo_password : String
+    def mapping_for_sudo_password(sudo_password)
+      {
+        /\[sudo\] password for .*?:\s*/ => "#{sudo_password}\n",
+        App::LOCAL_SUDO_PASSWORD_PROMPT => "#{sudo_password}\n",
+        # /\s+/ => nil,     # unnecessary
+      }
+    end
+
+    # temporarily adds the specified input mapping to the interaction handler while the given block is being evaluated
+    # when the given block returns, then the temporary mapping is removed from the interaction handler
+    #
+    # mapping : Hash[ String | Regex => String ]
+    def with_mapping(mapping, sudo_password = nil)
+      mapping ||= {}
+
+      raise ArgumentError.new("mapping must be a Hash") unless mapping.is_a?(Hash)
+
+      if sudo_password
+        mapping.merge!(mapping_for_sudo_password(sudo_password))
+      end
+
+      if mapping.empty?
+        yield self
+      else
+        yield ScopedMappingInteractionHandler.new(@input_mappings.merge(mapping), @log_level)
+      end
+    end
+
+    # adds the specified input mapping to the interaction handler
+    #
+    # mapping : Hash[ String | Regex => String ]
+    def add_mapping(mapping)
+      @input_mappings.merge!(mapping)
+    end
+
+    def on_data(_command, stream_name, data, channel)
+      log("Looking up response for #{stream_name} message #{data.inspect}")
+
+      response_data = begin
+        first_matching_key_value_pair = @input_mappings.find {|k, _v| k === data }
+        first_matching_key_value_pair&.last
+      end
+
+      if response_data.nil?
+        log("Unable to find interaction handler mapping for #{stream_name}: #{data.inspect} so no response was sent")
+      else
+        log("Sending #{response_data.inspect}")
+        if channel.respond_to?(:send_data)  # Net SSH Channel
+          channel.send_data(response_data)
+        elsif channel.respond_to?(:write)   # Local IO
+          channel.write(response_data)
+        else
+          raise "Unable to write response data to channel #{channel.inspect} - does not support '#send_data' or '#write'"
+        end
+      end
+    end
+
+    private
+
+    def log(message)
+      # puts message
+      SSHKit.config.output.send(@log_level, message) unless @log_level.nil?
+    end
+
+  end
+
   class PasswdInteractionHandler
     def on_data(command, stream_name, data, channel)
       # puts data

data/lib/opswalrus/local_pty_backend.rb

@@ -16,12 +16,16 @@ module SSHKit
         cmd.started = Time.now
         PTY.spawn(cmd.to_command) do |stdout, stdin, pid|
           stdout_thread = Thread.new do
+            # debug_log = StringIO.new
             buffer = ""
             partial_buffer = ""
             while !stdout.closed?
+              # debug_log.puts "!!!\nbuffer=#{buffer}|EOL|\npartial=#{partial_buffer}|EOL|"
               # puts "9" * 80
               begin
-                stdout.read_nonblock(4096, partial_buffer)
+                # partial_buffer = ""
+                # stdout.read_nonblock(4096, partial_buffer)
+                partial_buffer = stdout.read_nonblock(4096)
                 buffer << partial_buffer
                 # puts "nonblocking1. buffer=#{buffer} partial_buffer=#{partial_buffer}"
                 buffer = handle_data_for_stdout(output, cmd, buffer, stdin, false)
@@ -48,6 +52,10 @@ module SSHKit
               end
             end
             # puts "end!"
+            # debug_log.puts "!!!\nbuffer=#{buffer}|EOL|\npartial=#{partial_buffer}|EOL|"
+
+            # puts "*" * 80
+            # puts debug_log.string
 
           end
           stdout_thread.join
@@ -59,12 +67,14 @@ module SSHKit
 
       # returns [complete lines, new buffer]
       def split_buffer(buffer)
-        lines = buffer.split(/(\r\n)\r|\n/)
+        lines = buffer.split(/(\r\n)|\r|\n/)
         buffer = lines.pop
         [lines, buffer]
       end
 
       def handle_data_for_stdout(output, cmd, buffer, stdin, is_blocked)
+        # puts "handling data for stdout: #{buffer}"
+
         # we're blocked on reading, so let's process the buffer
         lines, buffer = split_buffer(buffer)
         lines.each do |line|

data/lib/opswalrus/operation_runner.rb

@@ -72,7 +72,7 @@ module OpsWalrus
         Invocation::Error.new(e)
       end
 
-      if verbose && result.failure?
+      if verbose == 2 && result.failure?
         puts "Ops script error details:"
         puts "Error: #{result.value}"
         puts "Status code: #{result.exit_status}"

data/lib/opswalrus/ops_file.rb

@@ -1,5 +1,6 @@
 require 'pathname'
 require 'yaml'
+require_relative 'git'
 require_relative 'ops_file_script'
 
 module OpsWalrus
@@ -91,12 +92,16 @@ module OpsWalrus
         imports_hash.map do |local_name, yaml_import_reference|
           local_name = local_name.to_s
           import_reference = case yaml_import_reference
+
+          # when the imports line says:
+          # imports:
+          #   my_package: my_package
           in String => import_str
             case
             when package_reference = package_file&.dependency(import_str)     # package dependency reference
               # in this context, import_str is the local package name documented in the package's dependencies
               PackageDependencyReference.new(local_name, package_reference)
-            when import_str.to_pathname.exist?            # path reference
+            when import_str.to_pathname.exist?                                # path reference
               path = import_str.to_pathname
               case
               when path.directory?
@@ -106,13 +111,26 @@ module OpsWalrus
               else
                 raise Error, "Unknown import reference: #{local_name} -> #{import_str.inspect}"
               end
+            when Git.repo?(import_str)                                        # ops file has imported an ad-hoc git repo
+              package_uri = import_str
+              destination_package_path = app.bundler.dynamic_package_path_for_git_package(package_uri)
+              # puts "DynamicPackageImportReference: #{local_name} -> #{destination_package_path}"
+              DynamicPackageImportReference.new(local_name, DynamicPackageReference.new(local_name, package_uri, nil))
+            else
+              raise Error, "Unknown import reference: #{local_name}: #{yaml_import_reference.inspect}"
             end
-          # in Hash
+
+          # when the imports line says:
+          # imports:
+          #   my_package:
+          #     url: https://...
+          #     version: 2.1
+          # in Hash => package_defn
           #   url = package_defn["url"]
           #   version = package_defn["version"]
           #   PackageReference.new(local_name, url, version&.to_s)
           else
-            raise Error, "Unknown package reference: #{package_defn.inspect}"
+            raise Error, "Unknown import reference: #{local_name}: #{yaml_import_reference.inspect}"
           end
           [local_name, import_reference]
         end.to_h
@@ -205,36 +223,5 @@ module OpsWalrus
       dirname.glob("*").select(&:directory?)
     end
 
-
-
-
-
-
-
-
-    def supporting_library_include_dir_and_require_lib
-      if Dir.exist?(ops_file_helper_library_directory)
-        [ops_file_helper_library_directory, ops_file_helper_library_basename]
-      elsif File.exist?(ops_file_sibling_helper_library_file)
-        [dirname, ops_file_helper_library_basename]
-      else
-        [nil, nil]
-      end
-    end
-
-    def ops_file_helper_library_basename
-      basename.sub_ext(".rb")
-    end
-
-    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "/home/david/sync/projects/ops/ops/core/host/info"
-    def ops_file_helper_library_directory
-      File.join(dirname, basename)
-    end
-
-    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "/home/david/sync/projects/ops/ops/core/host/info.rb"
-    def ops_file_sibling_helper_library_file
-      "#{ops_file_helper_library_directory}.rb"
-    end
-
   end
 end

data/lib/opswalrus/ops_file_script.rb

@@ -20,6 +20,9 @@ module OpsWalrus
     def [](index, *args, **kwargs, &block)
       @obj.method(:[]).call(index, *args, **kwargs, &block)
     end
+    def respond_to_missing?(method, *)
+      @obj.is_a?(Hash) && @obj.respond_to?(method)
+    end
     def method_missing(name, *args, **kwargs, &block)
       case @obj
       when Array
@@ -118,13 +121,15 @@ module OpsWalrus
       # bootstrap_shell_script = BootstrapLinuxHostShellScript
       # on sshkit_hosts do |sshkit_host|
       SSHKit::Coordinator.new(sshkit_hosts).each(in: kwargs[:in] || :parallel) do |sshkit_host|
-        host = sshkit_host_to_ops_host_map[sshkit_host]
 
+        # in this context, self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
+
+        host = sshkit_host_to_ops_host_map[sshkit_host]
         # puts "#{host.alias} / #{host}:"
 
         begin
-          # in this context, self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
-          host.set_ssh_session_connection(self)
+          host.set_runtime_env(runtime_env)
+          host.set_ssh_session_connection(self)  # self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
 
           # copy over bootstrap shell script
           # io = StringIO.new(bootstrap_shell_script)
@@ -196,11 +201,14 @@ module OpsWalrus
       @runtime_env.app.inventory(tags)
     end
 
-    def exit(exit_status)
+    def exit(exit_status, message = nil)
+      if message
+        puts message
+      end
       result = if exit_status == 0
-        Success.new(nil)
+        Invocation::Success.new(nil)
       else
-        Error.new(nil, exit_status)
+        Invocation::Error.new(nil, exit_status)
       end
       throw :exit_now, result
     end
@@ -214,6 +222,9 @@ module OpsWalrus
       end
     end
 
+    # currently, import may only be used to import a package that is referenced in the script's package file
+    # I may decide to extend this to work with dynamic package references
+    #
     # local_package_name is the local package name defined for the package dependency that is attempting to be referenced
     def import(local_package_name)
       local_package_name = local_package_name.to_s
@@ -234,18 +245,18 @@ module OpsWalrus
     end
 
     # returns the stdout from the command
-    def sh(desc_or_cmd = nil, cmd = nil, stdin: nil, &block)
-      out, err, status = *shell!(desc_or_cmd, cmd, block, stdin: stdin)
+    def sh(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+      out, err, status = *shell!(desc_or_cmd, cmd, block, input: input)
       out
     end
 
     # returns the tuple: [stdout, stderr, exit_status]
-    def shell(desc_or_cmd = nil, cmd = nil, stdin: nil, &block)
-      shell!(desc_or_cmd, cmd, block, stdin: stdin)
+    def shell(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+      shell!(desc_or_cmd, cmd, block, input: input)
     end
 
     # returns the tuple: [stdout, stderr, exit_status]
-    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, stdin: nil)
+    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil)
       # description = nil
 
       return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
@@ -254,19 +265,20 @@ module OpsWalrus
       cmd = block.call if block
       cmd ||= desc_or_cmd
 
+      cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
+
+      #cmd = Shellwords.escape(cmd)
+
       # puts "shell! self: #{self.inspect}"
 
       print "[#{@runtime_env.local_hostname}] "
       print "#{description}: " if description
       puts cmd
 
-      cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
       return unless cmd && !cmd.strip.empty?
 
-      #cmd = Shellwords.escape(cmd)
-
-      sudo_password = @runtime_env.sudo_password
-      sudo_password &&= sudo_password.gsub(/\n+$/,'')     # remove trailing newlines from sudo_password
+      # sudo_password = @runtime_env.sudo_password
+      # sudo_password &&= sudo_password.gsub(/\n+$/,'')     # remove trailing newlines from sudo_password
 
       # puts "shell: #{cmd}"
       # puts "shell: #{cmd.inspect}"
@@ -274,10 +286,14 @@ module OpsWalrus
 
       # sshkit_cmd = SSHKit::Backend::LocalNonBlocking.new {
       # sshkit_cmd = SSHKit::Backend::LocalPty.new {
-      sshkit_cmd = backend.execute_cmd(cmd, interaction_handler: SudoPasswordMapper.new(sudo_password).interaction_handler, verbosity: :info)
+      # sshkit_cmd = backend.execute_cmd(cmd, interaction_handler: SudoPasswordMapper.new(sudo_password).interaction_handler, verbosity: :info)
         # execute_cmd(cmd, interaction_handler: SudoPromptInteractionHandler.new, verbosity: :info)
       # }.run
 
+      sshkit_cmd = @runtime_env.handle_input(input) do |interaction_handler|
+        backend.execute_cmd(cmd, interaction_handler: interaction_handler, verbosity: :info)
+      end
+
       [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
     end
 

data/lib/opswalrus/package_file.rb

@@ -5,6 +5,7 @@ require_relative 'bundler'
 
 module OpsWalrus
 
+  # these are static package references defined ahead of time in the package file
   class PackageReference
     attr_accessor :local_name
     attr_accessor :package_uri
@@ -23,7 +24,7 @@ module OpsWalrus
       path.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;/\t\r\n\\", "-")
     end
 
-    # important: the dirname implemented as the local_name is critical because Bundler#download_package downloads
+    # important: the import_resolution_dirname implemented as the local_name is critical because Bundler#download_package downloads
     # package dependencies to the name that this method returns, which must match the package reference's local name
     # so that later, when the package is being looked up on the load path (in LoadPath#resolve_import_reference),
     # the package reference's referenced git repo or file path may not exist or be available, and so the package
@@ -32,12 +33,20 @@ module OpsWalrus
     # If this implementation changes, then Bundler#download_package and LoadPath#resolve_import_reference must also
     # change in order for the three things to reconcile with respect to one another, since all three bits of logic are
     # what make bundling package dependencies and loading them function properly.
-    def dirname
+    def import_resolution_dirname
       local_name
     end
 
   end
 
+  # these are dynamic package references defined at runtime when an OpsFile's imports are being evaluated.
+  # this will usually be the case when an ops file does not belong to a package
+  class DynamicPackageReference < PackageReference
+    def import_resolution_dirname
+      sanitized_package_uri
+    end
+  end
+
   class PackageFile
     attr_accessor :package_file_path
     attr_accessor :yaml

data/lib/opswalrus/runtime_environment.rb

@@ -3,6 +3,7 @@ require 'shellwords'
 require 'socket'
 require 'sshkit'
 
+require_relative 'interaction_handlers'
 require_relative 'traversable'
 require_relative 'walrus_lang'
 
@@ -32,6 +33,14 @@ module OpsWalrus
     end
   end
 
+  class DynamicPackageImportReference < ImportReference
+    attr_accessor :package_reference
+    def initialize(local_name, package_reference)
+      super(local_name)
+      @package_reference = package_reference
+    end
+  end
+
   class OpsFileReference < ImportReference
     attr_accessor :ops_file_path
     def initialize(local_name, ops_file_path)
@@ -86,6 +95,8 @@ module OpsWalrus
       @dir = dir
       @root_namespace = build_symbol_resolution_tree(@dir)
       @path_map = build_path_map(@root_namespace)
+
+      @dynamic_package_additions_memo = {}
     end
 
     # returns a tree of Namespace -> {Namespace* -> {Namespace* -> ..., OpsFile*}, OpsFile*}
@@ -129,6 +140,19 @@ module OpsWalrus
       path_map
     end
 
+    def dynamically_add_new_package_dir(new_package_dir)
+      # patch the symbol resolution (namespace) tree
+      dir_basename = new_package_dir.basename
+      unless @root_namespace.resolve_symbol(dir_basename)
+        new_child_namespace = build_symbol_resolution_tree(new_package_dir)
+        @root_namespace.add(dir_basename, new_child_namespace)
+
+        # patch the path map
+        new_partial_path_map = build_path_map(new_child_namespace)
+        @path_map.merge!(new_partial_path_map)
+      end
+    end
+
     # returns a Namespace
     def lookup_namespace(ops_file)
       @path_map[ops_file.dirname]
@@ -144,7 +168,16 @@ module OpsWalrus
       case import_reference
       when PackageDependencyReference
         # puts "root namespace: #{@root_namespace.symbol_table}"
-        @root_namespace.resolve_symbol(import_reference.package_reference.local_name)   # returns the Namespace associated with the bundled package dirname (i.e. the local name)
+        @root_namespace.resolve_symbol(import_reference.package_reference.import_resolution_dirname)   # returns the Namespace associated with the bundled package dirname (i.e. the local name)
+      when DynamicPackageImportReference
+        dynamic_package_reference = import_reference.package_reference
+        @dynamic_package_additions_memo[dynamic_package_reference] ||= begin
+          # puts "Downloading dynamic package: #{dynamic_package_reference.inspect}"
+          dynamically_added_package_dir = @runtime_env.app.bundler.download_git_package(dynamic_package_reference.package_uri, dynamic_package_reference.version)
+          dynamically_add_new_package_dir(dynamically_added_package_dir)
+          dynamically_added_package_dir
+        end
+        @root_namespace.resolve_symbol(import_reference.package_reference.import_resolution_dirname)   # returns the Namespace associated with the bundled package dirname (i.e. the sanitized package uri)
       when DirectoryReference
         @path_map[import_reference.dirname]
       when OpsFileReference
@@ -164,9 +197,27 @@ module OpsWalrus
       @bundle_load_path = LoadPath.new(self, @app.bundle_dir)
       @app_load_path = LoadPath.new(self, @app.pwd)
 
+      @interaction_handler = ScopedMappingInteractionHandler.new({
+        /\[sudo\] password for .*?:\s*/ => "#{sudo_password}\n",
+      })
+
       configure_sshkit
     end
 
+    # def with_sudo_password(password, &block)
+    #   @interaction_handler.with_sudo_password(password, &block)
+    # end
+
+    # input_mapping : Hash[ String | Regex => String ]
+    # sudo_password : String
+    def handle_input(input_mapping, sudo_password = nil, &block)
+      @interaction_handler.with_mapping(input_mapping, sudo_password, &block)
+    end
+
+    # def handle_input_with_sudo_password(input_mapping, password, &block)
+    #   @interaction_handler.with_mapping(input_mapping, password, &block)
+    # end
+
     # configure sshkit globally
     def configure_sshkit
       SSHKit.config.use_format :blackhole
@@ -244,7 +295,7 @@ module OpsWalrus
       case import_reference
 
       # we know we're dealing with a package dependency reference, so we want to do the lookup in the bundle load path, where package dependencies live
-      when PackageDependencyReference
+      when PackageDependencyReference, DynamicPackageImportReference
         @bundle_load_path.resolve_import_reference(origin_ops_file, import_reference)
 
       # we know we're dealing with a directory reference or OpsFile reference outside of the bundle dir, so we want to do the lookup in the app load path

data/lib/opswalrus/version.rb

@@ -1,3 +1,3 @@
 module OpsWalrus
-  VERSION = "1.0.5"
+  VERSION = "1.0.7"
 end

data/lib/opswalrus/walrus_lang.rb

@@ -57,7 +57,7 @@ module WalrusLang
 
   # binding_obj : Binding | Hash
   def self.render(template, binding_obj)
-    binding_obj = binding_obj.to_binding if binding_obj.is_a?(Hash)
+    binding_obj = binding_obj.to_binding if binding_obj.respond_to?(:to_binding)
     ast = WalrusLang::Parser.parse(template)
     ast.render(binding_obj)
   end

data/opswalrus.gemspec

@@ -11,10 +11,10 @@ Gem::Specification.new do |spec|
   spec.summary               = "opswalrus is a tool that runs scripts against a fleet of hosts"
   spec.description           = "opswalrus is a tool that runs scripts against a fleet of hosts hosts. It's kind of like Ansible, but aims to be simpler to use."
   spec.homepage              = "https://github.com/opswalrus/opswalrus"
-  spec.license               = 'EPL-2.0'
+  spec.license               = "EPL-2.0"
   spec.required_ruby_version = ">= 2.6.0"
 
-  # spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'"
+  # spec.metadata["allowed_push_host"] = "TODO: Set to your gem server - https://example.com"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/opswalrus/opswalrus"
@@ -33,13 +33,13 @@ Gem::Specification.new do |spec|
 
   # gem dependencies
   spec.add_dependency "citrus", "~> 3.0"
-  spec.add_dependency "gli", '~> 2.21'
-  spec.add_dependency "git", '~> 1.18'
-  spec.add_dependency "rubyzip", '~> 2.3'
+  spec.add_dependency "gli", "~> 2.21"
+  spec.add_dependency "git", "~> 1.18"
+  spec.add_dependency "rubyzip", "~> 2.3"
 
-  spec.add_dependency "bcrypt_pbkdf", '~> 1.1'
-  spec.add_dependency "ed25519", '~> 1.3'
-  spec.add_dependency "sshkit", '~> 1.21'
+  spec.add_dependency "bcrypt_pbkdf", "~> 1.1"
+  spec.add_dependency "ed25519", "~> 1.3"
+  spec.add_dependency "sshkit", "~> 1.21"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opswalrus
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.7
 platform: ruby
 authors:
 - David Ellis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-16 00:00:00.000000000 Z
+date: 2023-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: citrus
@@ -124,15 +124,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- build.ops
 - exe/ops
 - lib/opswalrus.rb
 - lib/opswalrus/app.rb
 - lib/opswalrus/bootstrap.sh
-- lib/opswalrus/bootstrap_linux_host1.sh
-- lib/opswalrus/bootstrap_linux_host2.sh
-- lib/opswalrus/bootstrap_linux_host3.sh
 - lib/opswalrus/bundler.rb
 - lib/opswalrus/cli.rb
+- lib/opswalrus/git.rb
 - lib/opswalrus/host.rb
 - lib/opswalrus/hosts_file.rb
 - lib/opswalrus/interaction_handlers.rb

data/lib/opswalrus/bootstrap_linux_host1.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-# update package list
-sudo apt update -y
-
-# update OS
-# sudo DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -o Dpkg::Options::="--force-confdef" dist-upgrade -q -y --allow-downgrades --allow-remove-essential --allow-change-held-packages
-
-if [ -f /var/run/reboot-required ]; then
-echo 'A system reboot is required!'
-sudo reboot
-fi

data/lib/opswalrus/bootstrap_linux_host2.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-# there are probably some services that need restarting because they're using old libraries, so we'll just do the easy thing and reboot
-sudo DEBIAN_FRONTEND=noninteractive apt install -yq needrestart
-
-# install basic development tools
-sudo DEBIAN_FRONTEND=noninteractive apt install -yq build-essential
-
-# install ruby dependencies
-sudo DEBIAN_FRONTEND=noninteractive apt install -yq autoconf patch rustc libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libgmp-dev libncurses5-dev libffi-dev libgdbm6 libgdbm-dev libdb-dev uuid-dev
-
-# restart services that need it
-sudo needrestart -q -r a
-
-# vagrant@ubuntu-jammy:~$ sudo needrestart -q -r a
-#  systemctl restart unattended-upgrades.service
-
-# vagrant@ubuntu-jammy:~$ sudo needrestart -r l
-# Scanning processes...
-# Scanning candidates...
-# Scanning linux images...
-#
-# Running kernel seems to be up-to-date.
-#
-# Services to be restarted:
-#
-# Service restarts being deferred:
-#  systemctl restart unattended-upgrades.service
-#
-# No containers need to be restarted.
-#
-# No user sessions are running outdated binaries.
-#
-# No VM guests are running outdated hypervisor (qemu) binaries on this host.
-
-# reboot just in case
-# sudo reboot

data/lib/opswalrus/bootstrap_linux_host3.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-# install homebrew
-NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-
-# install gcc, rust, ruby
-brew install gcc
-brew install rust
-brew install ruby
-
-# brew install rtx
-# eval "$(rtx activate bash)"   # register a shell hook
-# rtx use -g ruby@3.2   # install ruby via rtx
-
-# download frum for ruby version management
-# curl -L -o frum.tar.gz https://github.com/TaKO8Ki/frum/releases/download/v0.1.2/frum-v0.1.2-x86_64-unknown-linux-musl.tar.gz
-# tar -zxf frum.tar.gz
-# mv frum-v0.1.2-x86_64-unknown-linux-musl/frum ~/bin
-# chmod 755 ~/bin/frum
-