opswalrus 1.0.8 → 1.0.10

data/lib/opswalrus/invocation.rb

@@ -0,0 +1,446 @@
+# require 'json'
+# require 'set'
+# require 'shellwords'
+# require 'socket'
+# require 'stringio'
+
+# require 'sshkit'
+# require 'sshkit/dsl'
+
+# require_relative 'host'
+# require_relative 'sshkit_ext'
+# require_relative 'walrus_lang'
+
+# module OpsWalrus
+
+#   class ArrayOrHashNavigationProxy
+#     def initialize(array_or_hash)
+#       @obj = array_or_hash
+#     end
+#     def [](index, *args, **kwargs, &block)
+#       @obj.method(:[]).call(index, *args, **kwargs, &block)
+#     end
+#     def respond_to_missing?(method, *)
+#       @obj.is_a?(Hash) && @obj.respond_to?(method)
+#     end
+#     def method_missing(name, *args, **kwargs, &block)
+#       case @obj
+#       when Array
+#         @obj.method(name).call(*args, **kwargs, &block)
+#       when Hash
+#         if @obj.respond_to?(name)
+#           @obj.method(name).call(*args, **kwargs, &block)
+#         else
+#           value = self[name.to_s]
+#           case value
+#           when Array, Hash
+#             ArrayOrHashNavigationProxy.new(value)
+#           else
+#             value
+#           end
+#         end
+#       end
+#     end
+#   end
+
+#   class InvocationParams
+#     # params : Hash
+#     def initialize(params)
+#       @params = params
+#     end
+
+#     def [](key)
+#       key = key.to_s if key.is_a? Symbol
+#       @params[key]
+#     end
+
+#     def dig(*keys)
+#       # keys = keys.map {|key| key.is_a?(Integer) ? key : key.to_s }
+#       @params.dig(*keys)
+#     end
+
+#     def method_missing(name, *args, **kwargs, &block)
+#       if @params.respond_to?(name)
+#         @params.method(name).call(*args, **kwargs, &block)
+#       else
+#         value = self[name]
+#         case value
+#         when Array, Hash
+#           ArrayOrHashNavigationProxy.new(value)
+#         else
+#           value
+#         end
+#       end
+#     end
+#   end
+
+
+#   # BootstrapLinuxHostShellScript = <<~SCRIPT
+#   #   #!/usr/bin/env bash
+#   #   ...
+#   # SCRIPT
+
+#   module InvocationDSL
+#     def ssh(*args, **kwargs, &block)
+#       host_proxy_class = @ops_file_script.host_proxy_class
+#       hosts = inventory(*args, **kwargs).map {|host| host_proxy_class.new(host) }
+#       sshkit_hosts = hosts.map(&:sshkit_host)
+#       sshkit_host_to_ops_host_map = sshkit_hosts.zip(hosts).to_h
+#       runtime_env = @runtime_env
+#       local_host = self
+#       # bootstrap_shell_script = BootstrapLinuxHostShellScript
+#       # on sshkit_hosts do |sshkit_host|
+#       SSHKit::Coordinator.new(sshkit_hosts).each(in: kwargs[:in] || :parallel) do |sshkit_host|
+
+#         # in this context, self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
+
+#         host = sshkit_host_to_ops_host_map[sshkit_host]
+#         # puts "#{host.alias} / #{host}:"
+
+#         begin
+#           host.set_runtime_env(runtime_env)
+#           host.set_ssh_session_connection(self)  # self is an instance of one of the subclasses of SSHKit::Backend::Abstract, e.g. SSHKit::Backend::Netssh
+
+#           # copy over bootstrap shell script
+#           # io = StringIO.new(bootstrap_shell_script)
+#           io = File.open(__FILE__.to_pathname.dirname.join("bootstrap.sh"))
+#           upload_success = host.upload(io, "tmpopsbootstrap.sh")
+#           io.close
+#           raise Error, "Unable to upload bootstrap shell script to remote host" unless upload_success
+#           host.execute(:chmod, "755", "tmpopsbootstrap.sh")
+#           host.execute(:sh, "tmpopsbootstrap.sh")
+
+#           # copy over ops bundle zip file
+#           zip_bundle_path = runtime_env.zip_bundle_path
+#           upload_success = host.upload(zip_bundle_path, "tmpops.zip")
+#           raise Error, "Unable to upload ops bundle to remote host" unless upload_success
+
+#           stdout, stderr, exit_status = host.run_ops(:bundle, "unzip tmpops.zip", in_bundle_root_dir: false)
+#           raise Error, "Unable to unzip ops bundle on remote host" unless exit_status == 0
+#           tmp_bundle_root_dir = stdout.strip
+#           host.set_ssh_session_tmp_bundle_root_dir(tmp_bundle_root_dir)
+
+#           # we run the block in the context of the host, s.t. `self` within the block evaluates to `host`
+#           retval = host.instance_exec(local_host, &block)    # host is passed as the argument to the block
+
+#           # puts retval.inspect
+
+#           # cleanup
+#           if tmp_bundle_root_dir =~ /tmp/   # sanity check the temp path before we blow away something we don't intend
+#             host.execute(:rm, "-rf", "tmpopsbootstrap.sh", "tmpops.zip", tmp_bundle_root_dir)
+#           else
+#             host.execute(:rm, "-rf", "tmpopsbootstrap.sh", "tmpops.zip")
+#           end
+
+#           retval
+#         rescue SSHKit::Command::Failed => e
+#           puts "[!] Command failed:"
+#           puts e.message
+#         rescue Net::SSH::ConnectionTimeout
+#           puts "[!] The host '#{host}' not alive!"
+#         rescue Net::SSH::Timeout
+#           puts "[!] The host '#{host}' disconnected/timeouted unexpectedly!"
+#         rescue Errno::ECONNREFUSED
+#           puts "[!] Incorrect port #{port} for #{host}"
+#         rescue Net::SSH::HostKeyMismatch => e
+#           puts "[!] The host fingerprint does not match the last observed fingerprint for #{host}"
+#           puts e.message
+#           puts "You might try `ssh-keygen -f ~/.ssh/known_hosts -R \"#{host}\"`"
+#         rescue Net::SSH::AuthenticationFailed
+#           puts "Wrong Password: #{host} | #{user}:#{password}"
+#         rescue Net::SSH::Authentication::DisallowedMethod
+#           puts "[!] The host '#{host}' doesn't accept password authentication method."
+#         rescue Errno::EHOSTUNREACH => e
+#           puts "[!] The host '#{host}' is unreachable"
+#         rescue => e
+#           puts e.class
+#           puts e.message
+#           # puts e.backtrace.join("\n")
+#         ensure
+#           host.clear_ssh_session
+#         end
+#       end
+#     end
+
+#     def inventory(*args, **kwargs)
+#       tags = args.map(&:to_s)
+
+#       kwargs = kwargs.transform_keys(&:to_s)
+#       tags.concat(kwargs["tags"]) if kwargs["tags"]
+
+#       @runtime_env.app.inventory(tags)
+#     end
+
+#     def exit(exit_status, message = nil)
+#       if message
+#         puts message
+#       end
+#       result = if exit_status == 0
+#         Invocation::Success.new(nil)
+#       else
+#         Invocation::Error.new(nil, exit_status)
+#       end
+#       throw :exit_now, result
+#     end
+
+#     def env(*keys)
+#       keys = keys.map(&:to_s)
+#       if keys.empty?
+#         @env
+#       else
+#         @env.dig(*keys)
+#       end
+#     end
+
+#     # currently, import may only be used to import a package that is referenced in the script's package file
+#     # I may decide to extend this to work with dynamic package references
+#     #
+#     # local_package_name is the local package name defined for the package dependency that is attempting to be referenced
+#     def import(local_package_name)
+#       local_package_name = local_package_name.to_s
+#       package_reference = @ops_file_script.ops_file.package_file&.dependency(local_package_name)
+#       raise Error, "Unknown package reference: #{local_package_name}" unless package_reference
+#       import_reference = PackageDependencyReference.new(local_package_name, package_reference)
+#       # puts "import: #{import_reference.inspect}"
+#       @runtime_env.resolve_import_reference(@ops_file_script.ops_file, import_reference)
+#     end
+
+#     def params(*keys, default: nil)
+#       keys = keys.map(&:to_s)
+#       if keys.empty?
+#         @params
+#       else
+#         @params.dig(*keys) || default
+#       end
+#     end
+
+#     # returns the stdout from the command
+#     def sh(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+#       out, err, status = *shell!(desc_or_cmd, cmd, block, input: input)
+#       out
+#     end
+
+#     # returns the tuple: [stdout, stderr, exit_status]
+#     def shell(desc_or_cmd = nil, cmd = nil, input: nil, &block)
+#       shell!(desc_or_cmd, cmd, block, input: input)
+#     end
+
+#     # returns the tuple: [stdout, stderr, exit_status]
+#     def shell!(desc_or_cmd = nil, cmd = nil, block = nil, input: nil)
+#       # description = nil
+
+#       return ["", "", 0] if !desc_or_cmd && !cmd && !block    # we were told to do nothing; like hitting enter at the bash prompt; we can do nothing successfully
+
+#       description = desc_or_cmd if cmd || block
+#       cmd = block.call if block
+#       cmd ||= desc_or_cmd
+
+#       cmd = WalrusLang.render(cmd, block.binding) if block && cmd =~ /{{.*}}/
+
+#       #cmd = Shellwords.escape(cmd)
+
+#       # puts "shell! self: #{self.inspect}"
+
+#       if App.instance.report_mode?
+#         print "[#{@runtime_env.local_hostname}] "
+#         print "#{description}: " if description
+#         puts cmd
+#       end
+
+#       return unless cmd && !cmd.strip.empty?
+
+#       sshkit_cmd = @runtime_env.handle_input(input) do |interaction_handler|
+#         # self is a Module instance that is serving as the evaluation context in an instance of a subclass of an Invocation; see Invocation#evaluate
+#         backend.execute_cmd(cmd, interaction_handler: interaction_handler, verbosity: :info)
+#       end
+
+#       [sshkit_cmd.full_stdout, sshkit_cmd.full_stderr, sshkit_cmd.exit_status]
+#     end
+
+#     # def init_brew
+#     #   execute('eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"')
+#     # end
+
+#   end
+
+#   # An Invocation object represents a stack frame, and the params_hash represents the
+#   # arguments that the caller has supplied for that stack frame to reference
+#   class Invocation
+#     class Result
+#       attr_accessor :value
+#       attr_accessor :exit_status
+#       def initialize(value, exit_status = 0)
+#         @value = value
+#         @exit_status = exit_status
+#       end
+#       def success?
+#         !failure?
+#       end
+#       def failure?
+#         !success?
+#       end
+#     end
+#     class Success < Result
+#       def initialize(value)
+#         super(value, 0)
+#       end
+#       def success?
+#         true
+#       end
+#     end
+#     class Error < Result
+#       def initialize(value, exit_status = 1)
+#         super(value, exit_status == 0 ? 1 : exit_status)
+#       end
+#       def failure?
+#         true
+#       end
+#     end
+
+
+#     def self.define_invocation_class(ops_file)
+#       klass = Class.new(Invocation)
+
+#       methods_defined = Set.new
+
+#       # define methods for every import in the script
+#       ops_file.local_symbol_table.each do |symbol_name, import_reference|
+#         unless methods_defined.include? symbol_name
+#           klass.define_method(symbol_name) do |*args, **kwargs, &block|
+#             # puts "0" * 80
+#             # puts "@runtime_env.resolve_import_reference(@ops_file_script.ops_file, #{import_reference.inspect})"
+#             # puts @ops_file_script.ops_file.ops_file_path
+#             # puts symbol_name
+#             namespace_or_ops_file = @runtime_env.resolve_import_reference(@ops_file_script.ops_file, import_reference)
+#             # puts namespace_or_ops_file.inspect
+#             # puts "0" * 80
+#             case namespace_or_ops_file
+#             when Namespace
+#               namespace_or_ops_file
+#             when OpsFile
+#               params_hash = namespace_or_ops_file.build_params_hash(*args, **kwargs)
+#               namespace_or_ops_file.invoke(@runtime_env, params_hash)
+#             end
+#           end
+#           methods_defined << symbol_name
+#         end
+#       end
+
+#       # define methods for every Namespace or OpsFile within the namespace that the OpsFile resides within
+#       sibling_symbol_table = Set.new
+#       sibling_symbol_table |= ops_file.dirname.glob("*.ops").map {|ops_file_path| ops_file_path.basename(".ops").to_s }   # OpsFiles
+#       sibling_symbol_table |= ops_file.dirname.glob("*").select(&:directory?).map {|dir_path| dir_path.basename.to_s }    # Namespaces
+#       sibling_symbol_table.each do |symbol_name|
+#         unless methods_defined.include? symbol_name
+#           klass.define_method(symbol_name) do |*args, **kwargs, &block|
+#             # puts "0" * 80
+#             # puts "@runtime_env.resolve_symbol(@ops_file_script.ops_file, #{symbol_name})"
+#             # puts @ops_file_script.ops_file.ops_file_path
+#             # puts symbol_name
+#             namespace_or_ops_file = @runtime_env.resolve_sibling_symbol(@ops_file_script.ops_file, symbol_name)
+#             # puts namespace_or_ops_file.inspect
+#             # puts "0" * 80
+#             case namespace_or_ops_file
+#             when Namespace
+#               namespace_or_ops_file
+#             when OpsFile
+#               params_hash = namespace_or_ops_file.build_params_hash(*args, **kwargs)
+#               namespace_or_ops_file.invoke(@runtime_env, params_hash)
+#             end
+#           end
+#           methods_defined << symbol_name
+#         end
+#       end
+
+#       klass
+#     end
+
+#     include InvocationDSL
+
+#     def initialize(ops_file_script, runtime_env, params_hash)
+#       @ops_file_script = ops_file_script
+#       @runtime_env = runtime_env
+#       @params = InvocationParams.new(params_hash)
+#     end
+
+#     def backend
+#       @runtime_env.pty
+#     end
+
+#     def debug?
+#       @runtime_env.debug?
+#     end
+
+#     def verbose?
+#       @runtime_env.verbose?
+#     end
+
+#     # def evaluate
+#     #   # the evaluation context needs to be a module with all of the following:
+#     #   # - InvocationDSL methods
+#     #   # - @ops_file_script
+#     #   # - @runtime_env
+#     #   # - @params
+#     #   # - #backend
+#     #   # - #debug?
+#     #   # - #verbose?
+#     #   # - all the dynamically defined methods in the subclass of Invocation
+#     # end
+
+#     def evaluate
+#       eval(@ops_file_script.script, nil, @ops_file_script.ops_file.ops_file_path.to_s, @ops_file_script.ops_file.script_line_offset)
+#     end
+
+#     # def evaluate
+#     #   ops_file_script = @ops_file_script
+#     #   runtime_env = @runtime_env
+#     #   invocation_params = @params
+
+#     #   # we use a Module as the evaluation context instead of this #evaluate method so that classes and other structures may be defined in an .ops file
+#     #   evaluation_context = Module.new
+#     #   evaluation_context.module_exec {
+#     #     include InvocationDSL
+
+#     #     @ops_file_script = ops_file_script
+#     #     @runtime_env = runtime_env
+#     #     @params = invocation_params
+
+#     #     def backend
+#     #       @runtime_env.pty
+#     #     end
+
+#     #     def debug?
+#     #       @runtime_env.debug?
+#     #     end
+
+#     #     def verbose?
+#     #       @runtime_env.verbose?
+#     #     end
+#     #   }
+
+#     #   evaluation_context.module_eval(@ops_file_script.script, @ops_file_script.ops_file.ops_file_path.to_s, @ops_file_script.ops_file.script_line_offset)
+#     # end
+
+#     # def method_missing(name, *args, **kwargs, &block)
+#     #   puts "1" * 80
+#     #   import_reference = @ops_file_script.ops_file.resolve_import(name)
+#     #   if import_reference
+#     #     puts "2" * 80
+#     #     resolved_value = @runtime_env.resolve_import_reference(@ops_file_script.ops_file, import_reference)
+#     #     return resolved_value if resolved_value
+#     #   end
+
+#     #   puts "3" * 80
+#     #   case namespace_or_ops_file = @runtime_env.resolve_symbol(@ops_file_script.ops_file, name.to_s)
+#     #   when Namespace
+#     #     puts "4" * 80
+#     #     namespace_or_ops_file
+#     #   when OpsFile
+#     #     puts "5" * 80
+#     #     namespace_or_ops_file.invoke(@runtime_env, *args, **kwargs, &block)
+#     #   else
+#     #     raise NoMethodError, "No method named '#{name}'"
+#     #   end
+#     # end
+#   end
+# end

data/lib/opswalrus/operation_runner.rb

@@ -30,7 +30,7 @@ module OpsWalrus
 
     # runtime_kv_args is an Array(String) of the form: ["arg1:val1", "arg2:val2", ...]
     # params_json_hash is a Hash representation of a JSON string
-    def run(runtime_kv_args, params_json_hash: nil, verbose: false)
+    def run(runtime_kv_args, params_json_hash: nil)
       params_hash = runtime_kv_args.reduce(params_json_hash || {}) do |memo, kv_pair_string|
         str_key, str_value = kv_pair_string.split(":", 2)
         if pre_existing_value = memo[str_key]
@@ -43,7 +43,7 @@ module OpsWalrus
         memo
       end
 
-      if verbose == 2
+      if app.debug?
         puts "Script:"
         puts @entry_point_ops_file.script
       end
@@ -72,7 +72,7 @@ module OpsWalrus
         Invocation::Error.new(e)
       end
 
-      if verbose == 2 && result.failure?
+      if app.debug? && result.failure?
         puts "Ops script error details:"
         puts "Error: #{result.value}"
         puts "Status code: #{result.exit_status}"

data/lib/opswalrus/ops_file.rb

@@ -1,6 +1,7 @@
 require 'pathname'
 require 'yaml'
 require_relative 'git'
+require_relative 'host'
 require_relative 'ops_file_script'
 
 module OpsWalrus
@@ -9,7 +10,6 @@ module OpsWalrus
     attr_accessor :app
     attr_accessor :ops_file_path
     attr_accessor :yaml
-    attr_accessor :script
 
     def initialize(app, ops_file_path)
       @app = app
@@ -24,32 +24,46 @@ module OpsWalrus
       self.class == other.class && self.hash == other.hash
     end
 
+    def host_proxy_class
+      @host_proxy_class || (lazy_load_file && @host_proxy_class)
+    end
+
     def yaml
-      @yaml || (load_file && @yaml)
+      @yaml || (lazy_load_file && @yaml)
     end
 
     def script_line_offset
-      @script_line_offset || (load_file && @script_line_offset)
+      @script_line_offset || (lazy_load_file && @script_line_offset)
     end
 
+    # returns a subclass of OpsFileScript
+    def script_class
+      @script_class || (lazy_load_file && @script_class)
+    end
+
+    # returns an instance of the class returned by #script_class
     def script
-      @script || (load_file && @script)
+      @script || (lazy_load_file && @script)
     end
 
-    def load_file
+    def lazy_load_file
+      # puts "OpsFile#lazy_load_file for #{ops_file_path}"
+      # puts caller
       yaml, ruby_script = if @ops_file_path.exist?
         parse(File.read(@ops_file_path))
       end || ["", ""]
       @script_line_offset = yaml.lines.size + 1   # +1 to account for the ... line
       @yaml = YAML.load(yaml) || {}     # post_invariant: @yaml is a Hash
-      @script = OpsFileScript.new(self, ruby_script)
+      @script_class = OpsFileScript.define_for(self, ruby_script)
+      @script = @script_class.new(self, ruby_script)
+      @host_proxy_class = HostProxy.define_host_proxy_class(self)
     end
 
     def parse(script_string)
       file_halves = script_string.split(/^\.\.\.$/, 2)
       case file_halves.count
       when 1
-        yaml, ruby_script = "", file_halves
+        yaml, ruby_script = "", file_halves.first
       when 2
         yaml, ruby_script = *file_halves
       else
@@ -59,7 +73,7 @@ module OpsWalrus
     end
 
     def params
-      yaml["params"]
+      yaml["params"] || {}
     end
 
     def output
@@ -97,28 +111,7 @@ module OpsWalrus
           # imports:
           #   my_package: my_package
           in String => import_str
-            case
-            when package_reference = package_file&.dependency(import_str)     # package dependency reference
-              # in this context, import_str is the local package name documented in the package's dependencies
-              PackageDependencyReference.new(local_name, package_reference)
-            when import_str.to_pathname.exist?                                # path reference
-              path = import_str.to_pathname
-              case
-              when path.directory?
-                DirectoryReference.new(local_name, path.realpath)
-              when path.file? && path.extname.downcase == ".ops"
-                OpsFileReference.new(local_name, path.realpath)
-              else
-                raise Error, "Unknown import reference: #{local_name} -> #{import_str.inspect}"
-              end
-            when Git.repo?(import_str)                                        # ops file has imported an ad-hoc git repo
-              package_uri = import_str
-              destination_package_path = app.bundler.dynamic_package_path_for_git_package(package_uri)
-              # puts "DynamicPackageImportReference: #{local_name} -> #{destination_package_path}"
-              DynamicPackageImportReference.new(local_name, DynamicPackageReference.new(local_name, package_uri, nil))
-            else
-              raise Error, "Unknown import reference: #{local_name}: #{yaml_import_reference.inspect}"
-            end
+            import_string_to_import_reference(local_name, import_str)
 
           # when the imports line says:
           # imports:
@@ -137,9 +130,51 @@ module OpsWalrus
       end
     end
 
+    def import_string_to_import_reference(local_name, import_str)
+      if package_reference = package_file&.dependency(import_str)                       # package dependency reference
+        # in this context, import_str is the local package name documented in the package's dependencies
+        return PackageDependencyReference.new(local_name, package_reference)
+      end
+
+      import_path = import_str.to_pathname
+      if import_path.absolute? && import_path.exist?                                    # absolute path reference
+        return case
+        when import_path.directory?
+          DirectoryReference.new(local_name, import_path.realpath)
+        when import_path.file? && import_path.extname.downcase == ".ops"
+          OpsFileReference.new(local_name, import_path.realpath)
+        else
+          raise Error, "Unknown import reference for absolute path: #{local_name}: #{import_path}"
+        end
+      end
+      if import_path.relative?                                                          # relative path reference
+        rebased_path = dirname.join(import_path)
+        if rebased_path.exist?
+          return case
+          when rebased_path.directory?
+            DirectoryReference.new(local_name, rebased_path.realpath)
+          when rebased_path.file? && rebased_path.extname.downcase == ".ops"
+            OpsFileReference.new(local_name, rebased_path.realpath)
+          else
+            raise Error, "Unknown import reference for relative path: #{local_name}: #{import_path}"
+          end
+        elsif rebased_path.sub_ext(".ops").exist?
+          return OpsFileReference.new(local_name, rebased_path.sub_ext(".ops").realpath)
+        end
+      end
+
+      package_uri = import_str
+      if Git.repo?(package_uri)                                                         # ops file has imported an ad-hoc git repo
+        destination_package_path = app.bundler.dynamic_package_path_for_git_package(package_uri)
+        # puts "DynamicPackageImportReference: #{local_name} -> #{destination_package_path}"
+        return DynamicPackageImportReference.new(local_name, DynamicPackageReference.new(local_name, package_uri, nil))
+      end
+
+      raise Error, "Unknown import reference: #{local_name}: #{import_str.inspect}"
+    end
+
     def invoke(runtime_env, params_hash)
-      # puts "invoking: #{ops_file_path}"
-      script.invoke(runtime_env, params_hash)
+      script._invoke(runtime_env, params_hash)
     end
 
     def build_params_hash(*args, **kwargs)
@@ -223,5 +258,9 @@ module OpsWalrus
       dirname.glob("*").select(&:directory?)
     end
 
+    def to_s
+      "OpsFile: #{ops_file_path}"
+    end
+
   end
 end