opswalrus 1.0.7 → 1.0.9

data/lib/opswalrus/ops_file_script_dsl.rb

@@ -0,0 +1,297 @@
+require 'shellwords'
+require 'stringio'
+
+require 'sshkit'
+require 'sshkit/dsl'
+
+require_relative 'host'
+require_relative 'sshkit_ext'
+require_relative 'walrus_lang'
+
+# this file contains all of the logic associated with the invocation of the dynamically defined OpsFileScript#_invoke method
+
+module OpsWalrus
+
+  class ArrayOrHashNavigationProxy
+    def initialize(array_or_hash)
+      @obj = array_or_hash
+    end
+    def [](index, *args, **kwargs, &block)
+      @obj.method(:[]).call(index, *args, **kwargs, &block)
+    end
+    def respond_to_missing?(method, *)
+      @obj.is_a?(Hash) && @obj.respond_to?(method)
+    end
+    def method_missing(name, *args, **kwargs, &block)
+      case @obj
+      when Array
+        @obj.method(name).call(*args, **kwargs, &block)
+      when Hash
+        if @obj.respond_to?(name)
+          @obj.method(name).call(*args, **kwargs, &block)
+        else
+          value = self[name.to_s]
+          case value
+          when Array, Hash
+            ArrayOrHashNavigationProxy.new(value)
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+
+  class InvocationParams
+    # params : Hash
+    def initialize(params)
+      @params = params
+    end
+
+    def [](key)
+      key = key.to_s if key.is_a? Symbol
+      @params[key]
+    end
+
+    def dig(*keys)
+      # keys = keys.map {|key| key.is_a?(Integer) ? key : key.to_s }
+      @params.dig(*keys)
+    end
+
+    def method_missing(name, *args, **kwargs, &block)
+      if @params.respond_to?(name)
+        @params.method(name).call(*args, **kwargs, &block)
+      else
+        value = self[name]
+        case value
+        when Array, Hash
+          ArrayOrHashNavigationProxy.new(value)
+        else
+          value
+        end
+      end
+    end
+  end
+
+  module Invocation
+    class Result
+      attr_accessor :value
+      attr_accessor :exit_status
+      def initialize(value, exit_status = 0)
+        @value = value
+        @exit_status = exit_status
+      end
+      def success?
+        !failure?
+      end
+      def failure?
+        !success?
+      end
+    end
+    class Success < Result
+      def initialize(value)
+        super(value, 0)
+      end
+      def success?
+        true
+      end
+    end
+    class Error < Result
+      def initialize(value, exit_status = 1)
+        super(value, exit_status == 0 ? 1 : exit_status)
+      end
+      def failure?
+        true
+      end
+    end
+  end
+
+
+  # BootstrapLinuxHostShellScript = <<~SCRIPT
+  #   #!/usr/bin/env bash
+  #   ...
+  # SCRIPT
+
+  module OpsFileScriptDSL
+    def ssh(*args, **kwargs, &block)
+      hosts = inventory(*args, **kwargs).map {|host| host_proxy_class.new(host) }
+      sshkit_hosts = hosts.map(&:sshkit_host)
+      sshkit_host_to_ops_host_map = sshkit_hosts.zip(hosts).to_h
+      runtime_env = @runtime_env
+      local_host = self
+      # bootstrap_shell_script = BootstrapLinuxHostShellScript
+      # on sshkit_hosts do |sshkit_host|
+      SSHKit::Coordinator.new(sshkit_hosts).each(in: kwargs[:in] || :parallel) do |sshkit_host|
+
+        # in this context, self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
+
+        host = sshkit_host_to_ops_host_map[sshkit_host]
+        # puts "#{host.alias} / #{host}:"
+
+        begin
+          host.set_runtime_env(runtime_env)
+          host.set_ssh_session_connection(self)  # self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
+
+          # copy over bootstrap shell script
+          # io = StringIO.new(bootstrap_shell_script)
+          io = File.open(__FILE__.to_pathname.dirname.join("bootstrap.sh"))
+          upload_success = host.upload(io, "tmpopsbootstrap.sh")
+          io.close
+          raise Error, "Unable to upload bootstrap shell script to remote host" unless upload_success
+          host.execute(:chmod, "755", "tmpopsbootstrap.sh")
+          host.execute(:sh, "tmpopsbootstrap.sh")
+
+          # copy over ops bundle zip file
+          zip_bundle_path = runtime_env.zip_bundle_path
+          upload_success = host.upload(zip_bundle_path, "tmpops.zip")
+          raise Error, "Unable to upload ops bundle to remote host" unless upload_success
+
+          stdout, stderr, exit_status = host.run_ops(:bundle, "unzip tmpops.zip", in_bundle_root_dir: false)
+          raise Error, "Unable to unzip ops bundle on remote host" unless exit_status == 0
+          tmp_bundle_root_dir = stdout.strip
+          host.set_ssh_session_tmp_bundle_root_dir(tmp_bundle_root_dir)
+
+          # we run the block in the context of the host, s.t. `self` within the block evaluates to `host`
+          retval = host.instance_exec(local_host, &block)    # host is passed as the argument to the block
+
+          # puts retval.inspect
+
+          # cleanup
+          if tmp_bundle_root_dir =~ /tmp/   # sanity check the temp path before we blow away something we don't intend
+            host.execute(:rm, "-rf", "tmpopsbootstrap.sh", "tmpops.zip", tmp_bundle_root_dir)
+          else
+            host.execute(:rm, "-rf", "tmpopsbootstrap.sh", "tmpops.zip")
+          end
+
+          retval
+        rescue SSHKit::Command::Failed => e
+          puts "[!] Command failed:"
+          puts e.message
+        rescue Net::SSH::ConnectionTimeout
+          puts "[!] The host '#{host}' not alive!"
+        rescue Net::SSH::Timeout
+          puts "[!] The host '#{host}' disconnected/timeouted unexpectedly!"
+        rescue Errno::ECONNREFUSED
+          puts "[!] Incorrect port #{port} for #{host}"
+        rescue Net::SSH::HostKeyMismatch => e
+          puts "[!] The host fingerprint does not match the last observed fingerprint for #{host}"
+          puts e.message
+          puts "You might try `ssh-keygen -f ~/.ssh/known_hosts -R \"#{host}\"`"
+        rescue Net::SSH::AuthenticationFailed
+          puts "Wrong Password: #{host} | #{user}:#{password}"
+        rescue Net::SSH::Authentication::DisallowedMethod
+          puts "[!] The host '#{host}' doesn't accept password authentication method."
+        rescue Errno::EHOSTUNREACH => e
+          puts "[!] The host '#{host}' is unreachable"
+        rescue => e
+          puts e.class
+          puts e.message
+          # puts e.backtrace.join("\n")
+        ensure
+          host.clear_ssh_session
+        end
+      end
+    end
+
+    def inventory(*args, **kwargs)
+      tags = args.map(&:to_s)
+
+      kwargs = kwargs.transform_keys(&:to_s)
+      tags.concat(kwargs["tags"]) if kwargs["tags"]
+
+      @runtime_env.app.inventory(tags)
+    end
+
+    def exit(exit_status, message = nil)
+      if message
+        puts message
+      end
+      result = if exit_status == 0
+        Invocation::Success.new(nil)
+      else
+        Invocation::Error.new(nil, exit_status)
+      end
+      throw :exit_now, result
+    end
+
+    def env(*keys)
+      keys = keys.map(&:to_s)
+      if keys.empty?
+        @env
+      else
+        @env.dig(*keys)
+      end
+    end
+
+    # currently, import may only be used to import a package that is referenced in the script's package file
+    # I may decide to extend this to work with dynamic package references
+    #
+    # local_package_name is the local package name defined for the package dependency that is attempting to be referenced
+    def import(local_package_name)
+      local_package_name = local_package_name.to_s
+      package_reference = ops_file.package_file&.dependency(local_package_name)
+      raise Error, "Unknown package reference: #{local_package_name}" unless package_reference
+      import_reference = PackageDependencyReference.new(local_package_name, package_reference)
+      # puts "import: #{import_reference.inspect}"
+      @runtime_env.resolve_import_reference(ops_file, import_reference)
+    end
+
+    def params(*keys, default: nil)
+      keys = keys.map(&:to_s)
+      if keys.empty?
+        @params
+      else
+        @params.dig(*keys) || default
+      end
+    end
+
+    # returns the stdout from the command
+    def sh(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+      out, err, status = *shell!(desc_or_cmd, cmd, block, input: input)
+      out
+    end
+
+    # returns the tuple: [stdout, stderr, exit_status]
+    def shell(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+      shell!(desc_or_cmd, cmd, block, input: input)
+    end
+
+    # returns the tuple: [stdout, stderr, exit_status]
+    def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil)
+      # description = nil
+
+      return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
+
+      description = desc_or_cmd if cmd || block
+      cmd = block.call if block
+      cmd ||= desc_or_cmd
+
+      cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
+
+      #cmd = Shellwords.escape(cmd)
+
+      # puts "shell! self: #{self.inspect}"
+
+      if App.instance.report_mode?
+        print "[#{@runtime_env.local_hostname}] "
+        print "#{description}: " if description
+        puts cmd
+      end
+
+      return unless cmd && !cmd.strip.empty?
+
+      sshkit_cmd = @runtime_env.handle_input(input) do |interaction_handler|
+        # self is a Module instance that is serving as the evaluation context in an instance of a subclass of an Invocation; see Invocation#evaluate
+        backend.execute_cmd(cmd, interaction_handler: interaction_handler, verbosity: :info)
+      end
+
+      [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
+    end
+
+    # def init_brew
+    #   execute('eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"')
+    # end
+
+  end
+
+end

data/lib/opswalrus/patches.rb

@@ -0,0 +1,22 @@
+require 'json'
+require 'pathname'
+
+class String
+  def escape_single_quotes
+    gsub("'"){"\\'"}
+  end
+
+  def to_pathname
+    Pathname.new(self)
+  end
+
+  def parse_json
+    JSON.parse(self)
+  end
+end
+
+class Pathname
+  def to_pathname
+    self
+  end
+end

data/lib/opswalrus/runtime_environment.rb

@@ -62,6 +62,18 @@ module OpsWalrus
       @symbol_table = {}    # "symbol_name" => ops_file_or_child_namespace
     end
 
+    def to_s(indent = 0)
+      str = "Namespace: #{@dirname.to_s}\n"
+      @symbol_table.each do |k, v|
+        if v.is_a? Namespace
+          str << "#{'  ' * (indent)}|- #{k} : #{v.to_s(indent + 1)}\n"
+        else
+          str << "#{'  ' * (indent)}|- #{k} : #{v.to_s}\n"
+        end
+      end
+      str
+    end
+
     def add(symbol_name, ops_file_or_child_namespace)
       @symbol_table[symbol_name.to_s] = ops_file_or_child_namespace
     end
@@ -70,11 +82,27 @@ module OpsWalrus
       @symbol_table[symbol_name.to_s]
     end
 
+    # if this namespace contains an OpsFile of the same name as the namespace, e.g. pkg/install/install.ops, then this
+    # method invokes the OpsFile of that same name and returns the result;
+    # otherwise we return this namespace object
+    def _invoke_if_namespace_has_ops_file_of_same_name(*args, **kwargs, &block)
+      resolved_symbol = resolve_symbol(@dirname.basename)
+      if resolved_symbol.is_a? OpsFile
+        params_hash = resolved_symbol.build_params_hash(*args, **kwargs)
+        resolved_symbol.invoke(runtime_env, params_hash)
+      else
+        self
+      end
+    end
+
     def method_missing(name, *args, **kwargs, &block)
+      # puts "method_missing: #{name}"
+      # puts caller
       resolved_symbol = resolve_symbol(name)
       case resolved_symbol
       when Namespace
         resolved_symbol
+        resolved_symbol._invoke_if_namespace_has_ops_file_of_same_name(*args, **kwargs)
       when OpsFile
         params_hash = resolved_symbol.build_params_hash(*args, **kwargs)
         resolved_symbol.invoke(runtime_env, params_hash)
@@ -82,7 +110,7 @@ module OpsWalrus
     end
   end
 
-    # the assumption is that we have a bundle directory with all the packages in it
+  # the assumption is that we have a bundle directory with all the packages in it
   # and the bundle directory is the root directory
   class LoadPath
     include Traversable
@@ -96,6 +124,17 @@ module OpsWalrus
       @root_namespace = build_symbol_resolution_tree(@dir)
       @path_map = build_path_map(@root_namespace)
 
+      # puts "*" * 80
+      # puts "load path for #{@dir}"
+      # puts "-" * 80
+      # puts 'root namespace'
+      # puts @root_namespace.to_s
+      # puts "-" * 80
+      # puts 'path map'
+      # @path_map.each do |k,v|
+      #   puts "#{k.to_s}: #{v.to_s}"
+      # end
+
       @dynamic_package_additions_memo = {}
     end
 
@@ -204,20 +243,12 @@ module OpsWalrus
       configure_sshkit
     end
 
-    # def with_sudo_password(password, &block)
-    #   @interaction_handler.with_sudo_password(password, &block)
-    # end
-
     # input_mapping : Hash[ String | Regex => String ]
     # sudo_password : String
     def handle_input(input_mapping, sudo_password = nil, &block)
       @interaction_handler.with_mapping(input_mapping, sudo_password, &block)
     end
 
-    # def handle_input_with_sudo_password(input_mapping, password, &block)
-    #   @interaction_handler.with_mapping(input_mapping, password, &block)
-    # end
-
     # configure sshkit globally
     def configure_sshkit
       SSHKit.config.use_format :blackhole

data/lib/opswalrus/sshkit_ext.rb

@@ -9,10 +9,13 @@ require_relative 'local_pty_backend'
 module SSHKit
   module Backend
     class Abstract
+      def execute(*args)
+        options = { raise_on_non_zero_exit: false }.merge(args.extract_options!)
+        create_command_and_execute(args, options).success?
+      end
+
       def execute_cmd(*args)
-        # puts "args: #{args.inspect}"
-        options = { verbosity: :debug, strip: true }.merge(args.extract_options!)
-        # puts "options: #{options.inspect}"
+        options = { verbosity: :debug, strip: true, raise_on_non_zero_exit: false }.merge(args.extract_options!)
         create_command_and_execute(args, options)
       end
     end

data/lib/opswalrus/version.rb

@@ -1,3 +1,3 @@
 module OpsWalrus
-  VERSION = "1.0.7"
+  VERSION = "1.0.9"
 end

data/opswalrus.gemspec

@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "bcrypt_pbkdf", "~> 1.1"
   spec.add_dependency "ed25519", "~> 1.3"
-  spec.add_dependency "sshkit", "~> 1.21"
+  spec.add_dependency "sshkit", "~> 1.21"       # sshkit uses net-ssh, which depends on bcrypt_pbkdf and ed25519 to dynamically add support for ed25519 if those two gems are present
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opswalrus
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.9
 platform: ruby
 authors:
 - David Ellis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 2023-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: citrus
@@ -118,6 +118,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".rspec"
+- CNAME
 - Dockerfile
 - Gemfile
 - Gemfile.lock
@@ -135,12 +136,15 @@ files:
 - lib/opswalrus/host.rb
 - lib/opswalrus/hosts_file.rb
 - lib/opswalrus/interaction_handlers.rb
+- lib/opswalrus/invocation.rb
 - lib/opswalrus/local_non_blocking_backend.rb
 - lib/opswalrus/local_pty_backend.rb
 - lib/opswalrus/operation_runner.rb
 - lib/opswalrus/ops_file.rb
 - lib/opswalrus/ops_file_script.rb
+- lib/opswalrus/ops_file_script_dsl.rb
 - lib/opswalrus/package_file.rb
+- lib/opswalrus/patches.rb
 - lib/opswalrus/runtime_environment.rb
 - lib/opswalrus/sshkit_ext.rb
 - lib/opswalrus/traversable.rb