opswalrus 1.0.0

data/lib/opswalrus/local_non_blocking_backend.rb

@@ -0,0 +1,132 @@
+require 'open3'
+require 'fileutils'
+
+module SSHKit
+
+  module Backend
+
+    # this backend is compatible with sudo if you use the -S flag, e.g.: sudo -S ...
+    class LocalNonBlocking < Local
+
+      private
+
+      def execute_command(cmd)
+        output.log_command_start(cmd.with_redaction)
+        cmd.started = Time.now
+        Open3.popen3(cmd.to_command) do |stdin, stdout, stderr, wait_thr|
+          stdout_thread = Thread.new do
+            buffer = ""
+            partial_buffer = ""
+            while !stdout.closed?
+              # puts "9" * 80
+              begin
+                stdout.read_nonblock(4096, partial_buffer)
+                buffer << partial_buffer
+                # puts "nonblocking1. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+                buffer = handle_data_for_stdout(output, cmd, buffer, stdin, false)
+                # puts "nonblocking2. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+              rescue IO::WaitReadable, Errno::EAGAIN, Errno::EWOULDBLOCK
+                # puts "blocking. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+                buffer = handle_data_for_stdout(output, cmd, buffer, stdin, true)
+                IO.select([stdout])
+                retry
+
+              # per https://stackoverflow.com/questions/1154846/continuously-read-from-stdout-of-external-process-in-ruby
+              # and https://stackoverflow.com/questions/10238298/ruby-on-linux-pty-goes-away-without-eof-raises-errnoeio
+              # the PTY can raise an Errno::EIO because the child process unexpectedly goes away
+              rescue EOFError, Errno::EIO
+                # puts "eof!"
+                handle_data_for_stdout(output, cmd, buffer, stdin, true)
+                stdout.close
+              rescue => e
+                puts "closing PTY due to unexpected error: #{e.message}"
+                handle_data_for_stdout(output, cmd, buffer, stdin, true)
+                stdout.close
+                # puts e.message
+                # puts e.backtrace.join("\n")
+              end
+            end
+            # puts "end!"
+          end
+          stderr_thread = Thread.new do
+            buffer = ""
+            partial_buffer = ""
+            while !stderr.closed?
+              # puts "9" * 80
+              begin
+                stderr.read_nonblock(4096, partial_buffer)
+                buffer << partial_buffer
+                # puts "nonblocking1. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+                buffer = handle_data_for_stderr(output, cmd, buffer, stdin, false)
+                # puts "nonblocking2. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+              rescue IO::WaitReadable, Errno::EAGAIN, Errno::EWOULDBLOCK
+                # puts "blocking. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+                buffer = handle_data_for_stderr(output, cmd, buffer, stdin, true)
+                IO.select([stderr])
+                retry
+
+              # per https://stackoverflow.com/questions/1154846/continuously-read-from-stdout-of-external-process-in-ruby
+              # and https://stackoverflow.com/questions/10238298/ruby-on-linux-pty-goes-away-without-eof-raises-errnoeio
+              # the PTY can raise an Errno::EIO because the child process unexpectedly goes away
+              rescue EOFError, Errno::EIO
+                # puts "eof!"
+                handle_data_for_stderr(output, cmd, buffer, stdin, true)
+                stderr.close
+              rescue => e
+                puts "closing PTY due to unexpected error: #{e.message}"
+                handle_data_for_stderr(output, cmd, buffer, stdin, true)
+                stderr.close
+                # puts e.message
+                # puts e.backtrace.join("\n")
+              end
+            end
+            # puts "end!"
+          end
+          stdout_thread.join
+          stderr_thread.join
+          cmd.exit_status = wait_thr.value.to_i
+          output.log_command_exit(cmd)
+        end
+      end
+
+
+      # returns [complete lines, new buffer]
+      def split_buffer(buffer)
+        lines = buffer.split(/(\r\n)\r|\n/)
+        buffer = lines.pop
+        [lines, buffer]
+      end
+
+      def handle_data_for_stdout(output, cmd, buffer, stdin, is_blocked)
+        # we're blocked on reading, so let's process the buffer
+        lines, buffer = split_buffer(buffer)
+        lines.each do |line|
+          cmd.on_stdout(stdin, line)
+          output.log_command_data(cmd, :stdout, line)
+        end
+        if is_blocked && buffer
+          cmd.on_stdout(stdin, buffer)
+          output.log_command_data(cmd, :stdout, buffer)
+          buffer = ""
+        end
+        buffer || ""
+      end
+
+
+      def handle_data_for_stderr(output, cmd, buffer, stdin, is_blocked)
+        # we're blocked on reading, so let's process the buffer
+        lines, buffer = split_buffer(buffer)
+        lines.each do |line|
+          cmd.on_stderr(stdin, line)
+          output.log_command_data(cmd, :stderr, line)
+        end
+        if is_blocked && buffer
+          cmd.on_stderr(stdin, buffer)
+          output.log_command_data(cmd, :stderr, buffer)
+          buffer = ""
+        end
+        buffer || ""
+      end
+    end
+  end
+end

data/lib/opswalrus/local_pty_backend.rb

@@ -0,0 +1,89 @@
+require 'open3'
+require 'pty'
+require 'fileutils'
+
+module SSHKit
+
+  module Backend
+
+    # this backend is compatible with sudo, even without the -S flag, e.g.: sudo ...
+    class LocalPty < Local
+
+      private
+
+      def execute_command(cmd)
+        output.log_command_start(cmd.with_redaction)
+        cmd.started = Time.now
+        PTY.spawn(cmd.to_command) do |stdout, stdin, pid|
+          stdout_thread = Thread.new do
+            buffer = ""
+            partial_buffer = ""
+            while !stdout.closed?
+              # puts "9" * 80
+              begin
+                stdout.read_nonblock(4096, partial_buffer)
+                buffer << partial_buffer
+                # puts "nonblocking1. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+                buffer = handle_data_for_stdout(output, cmd, buffer, stdin, false)
+                # puts "nonblocking2. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+              rescue IO::WaitReadable, Errno::EAGAIN, Errno::EWOULDBLOCK
+                # puts "blocking. buffer=#{buffer} partial_buffer=#{partial_buffer}"
+                buffer = handle_data_for_stdout(output, cmd, buffer, stdin, true)
+                IO.select([stdout])
+                retry
+
+              # per https://stackoverflow.com/questions/1154846/continuously-read-from-stdout-of-external-process-in-ruby
+              # and https://stackoverflow.com/questions/10238298/ruby-on-linux-pty-goes-away-without-eof-raises-errnoeio
+              # the PTY can raise an Errno::EIO because the child process unexpectedly goes away
+              rescue EOFError, Errno::EIO
+                # puts "eof!"
+                handle_data_for_stdout(output, cmd, buffer, stdin, true)
+                stdout.close
+              rescue => e
+                puts "closing PTY due to unexpected error: #{e.message}"
+                handle_data_for_stdout(output, cmd, buffer, stdin, true)
+                stdout.close
+                # puts e.message
+                # puts e.backtrace.join("\n")
+              end
+            end
+            # puts "end!"
+
+          end
+          stdout_thread.join
+          _pid, status = Process.wait2(pid)
+          cmd.exit_status = status.exitstatus
+          output.log_command_exit(cmd)
+        end
+      end
+
+      # returns [complete lines, new buffer]
+      def split_buffer(buffer)
+        lines = buffer.split(/(\r\n)\r|\n/)
+        buffer = lines.pop
+        [lines, buffer]
+      end
+
+      def handle_data_for_stdout(output, cmd, buffer, stdin, is_blocked)
+        # we're blocked on reading, so let's process the buffer
+        lines, buffer = split_buffer(buffer)
+        lines.each do |line|
+          # puts "1" * 80
+          # puts line
+          cmd.on_stdout(stdin, line)
+          output.log_command_data(cmd, :stdout, line)
+        end
+        if is_blocked && buffer
+          # puts "2" * 80
+          # puts buffer
+          cmd.on_stdout(stdin, buffer)
+          output.log_command_data(cmd, :stdout, buffer)
+          buffer = ""
+        end
+        buffer || ""
+      end
+
+
+    end
+  end
+end

data/lib/opswalrus/operation_runner.rb

@@ -0,0 +1,85 @@
+require "random/formatter"
+require "socket"
+
+require_relative "runtime_environment"
+require_relative "ops_file"
+
+module OpsWalrus
+  class OperationRunner
+    attr_accessor :app
+    attr_accessor :entry_point_ops_file
+
+    def initialize(app, entry_point_ops_file)
+      @app = app
+      @entry_point_ops_file = entry_point_ops_file
+      # @entry_point_ops_file_in_bundle_dir = bundle!(@entry_point_ops_file)
+    end
+
+    # def bundle!(entry_point_ops_file)
+    #   path_to_entry_point_ops_file_in_bundle_dir = @app.bundler.build_bundle_for_ops_file(entry_point_ops_file)
+    #   OpsFile.new(app, path_to_entry_point_ops_file_in_bundle_dir)
+    # end
+
+    def sudo_user
+      @app.sudo_user
+    end
+
+    def sudo_password
+      @app.sudo_password
+    end
+
+    # runtime_kv_args is an Array(String) of the form: ["arg1:val1", "arg2:val2", ...]
+    # params_json_hash is a Hash representation of a JSON string
+    def run(runtime_kv_args, params_json_hash: nil, verbose: false)
+      params_hash = runtime_kv_args.reduce(params_json_hash || {}) do |memo, kv_pair_string|
+        str_key, str_value = kv_pair_string.split(":", 2)
+        if pre_existing_value = memo[str_key]
+          array = pre_existing_value.is_a?(Array) ? pre_existing_value : [pre_existing_value]
+          array << str_value
+          memo[str_key] = array
+        else
+          memo[str_key] = str_value
+        end
+        memo
+      end
+
+      if verbose == 2
+        puts "Script:"
+        puts @entry_point_ops_file.script
+      end
+
+      result = begin
+        # update the bundle for the package
+        # @entry_point_ops_file.package_file&.bundle!   # we don't do this here because when the script is run
+                                                        # on a remote host, the package references may be invalid
+                                                        # so we will be unable to bundle at runtime on the remote host
+        catch(:exit_now) do
+          ruby_script_return = RuntimeEnvironment.new(app).run(@entry_point_ops_file, params_hash)
+          Invocation::Success.new(ruby_script_return)
+        end
+      rescue SSHKit::Command::Failed => e
+        puts "[!] Command failed: #{e.message}"
+      rescue Error => e
+        $stderr.puts "Error: Ops script crashed."
+        $stderr.puts e.message
+        $stderr.puts e.backtrace.join("\n")
+        Invocation::Error.new(e)
+      rescue => e
+        $stderr.puts "Unhandled Error: Ops script crashed."
+        $stderr.puts e.class
+        $stderr.puts e.message
+        $stderr.puts e.backtrace.join("\n")
+        Invocation::Error.new(e)
+      end
+
+      if verbose && result.failure?
+        puts "Ops script error details:"
+        puts "Error: #{result.value}"
+        puts "Status code: #{result.exit_status}"
+        puts @entry_point_ops_file.script
+      end
+
+      result
+    end
+  end
+end

data/lib/opswalrus/ops_file.rb

@@ -0,0 +1,235 @@
+require 'pathname'
+require 'yaml'
+require_relative 'ops_file_script'
+
+module OpsWalrus
+
+  class OpsFile
+    attr_accessor :app
+    attr_accessor :ops_file_path
+    attr_accessor :yaml
+    attr_accessor :script
+
+    def initialize(app, ops_file_path)
+      @app = app
+      @ops_file_path = ops_file_path.to_pathname.expand_path
+    end
+
+    def hash
+      @ops_file_path.hash
+    end
+
+    def eql?(other)
+      self.class == other.class && self.hash == other.hash
+    end
+
+    def yaml
+      @yaml || (load_file && @yaml)
+    end
+
+    def script
+      @script || (load_file && @script)
+    end
+
+    def load_file
+      yaml, ruby_script = if @ops_file_path.exist?
+        parse(File.read(@ops_file_path))
+      end || ["", ""]
+      @yaml = YAML.load(yaml) || {}     # post_invariant: @yaml is a Hash
+      @script = OpsFileScript.new(self, ruby_script)
+    end
+
+    def parse(script_string)
+      file_halves = script_string.split(/^\.\.\.$/, 2)
+      case file_halves.count
+      when 1
+        yaml, ruby_script = "", file_halves
+      when 2
+        yaml, ruby_script = *file_halves
+      else
+        raise Error, "Unexpected number of file sections: #{file_halves.inspect}"
+      end
+      [yaml, ruby_script]
+    end
+
+    def params
+      yaml["params"]
+    end
+
+    def output
+      yaml["output"]
+    end
+
+    def package_file
+      return @package_file if @package_file_evaluated
+      @package_file ||= begin
+        ops_file_path = @ops_file_path.realpath
+        ops_file_path.ascend.each do |path|
+          candidate_package_file_path = path.join("package.yaml")
+          return PackageFile.new(candidate_package_file_path) if candidate_package_file_path.exist?
+        end
+        nil
+      end
+      @package_file_evaluated = true
+      @package_file
+    end
+
+    # returns a map of the form: {"local_symbol" => import_reference, ... }
+    # import_reference is one of:
+    # 1. a package reference that matches one of the local package names in the dependencies captured in packages.yaml
+    # 2. a package reference that resolves to a relative path pointing at a package directory
+    # 3. a path that resolves to a directory containing ops files
+    # 4. a path that resolves to an ops file
+    def imports
+      @imports ||= begin
+        imports_hash = yaml["imports"] || {}
+        imports_hash.map do |local_name, yaml_import_reference|
+          local_name = local_name.to_s
+          import_reference = case yaml_import_reference
+          in String => import_str
+            case
+            when package_reference = package_file&.dependency(import_str)     # package dependency reference
+              # in this context, import_str is the local package name documented in the package's dependencies
+              PackageDependencyReference.new(local_name, package_reference)
+            when import_str.to_pathname.exist?            # path reference
+              path = import_str.to_pathname
+              case
+              when path.directory?
+                DirectoryReference.new(local_name, path.realpath)
+              when path.file? && path.extname.downcase == ".ops"
+                OpsFileReference.new(local_name, path.realpath)
+              else
+                raise Error, "Unknown import reference: #{local_name} -> #{import_str.inspect}"
+              end
+            end
+          # in Hash
+          #   url = package_defn["url"]
+          #   version = package_defn["version"]
+          #   PackageReference.new(local_name, url, version&.to_s)
+          else
+            raise Error, "Unknown package reference: #{package_defn.inspect}"
+          end
+          [local_name, import_reference]
+        end.to_h
+      end
+    end
+
+    def invoke(runtime_env, params_hash)
+      puts "invoking: #{ops_file_path}"
+      script.invoke(runtime_env, params_hash)
+    end
+
+    def build_params_hash(*args, **kwargs)
+      params_hash = {}
+
+      # if there is only one Hash object in args, treat that as the params hash
+      if args.size == 1 && args.first.is_a?(Hash)
+        tmp_params_hash = args.first.transform_keys(&:to_s)
+        params_hash.merge!(tmp_params_hash)
+      end
+
+      # if there are the same number of args as there are params, then treat each one as the corresponding param
+      if args.size == params.keys.size
+        tmp_params_hash = params.keys.zip(args).to_h.transform_keys(&:to_s)
+        params_hash.merge!(tmp_params_hash)
+      end
+
+      # merge in the kwargs as part of the params hash
+      params_hash.merge!(kwargs.transform_keys(&:to_s))
+
+      params_hash
+    end
+
+    # symbol table derived from explicit imports and the import for the private lib directory if it exists
+    # map of: "symbol_name" => ImportReference
+    def local_symbol_table
+      @local_symbol_table ||= begin
+        local_symbol_table = {}
+
+        local_symbol_table.merge(imports)
+
+        # this is the import for the private lib directory if it exists
+        if private_lib_dir.exist?
+          local_symbol_table[basename.to_s] = DirectoryReference.new(basename.to_s, private_lib_dir)
+        end
+
+        local_symbol_table
+      end
+    end
+
+    def resolve_import(symbol_name)
+      local_symbol_table[symbol_name]
+    end
+
+    # def namespace
+    #   @namespace ||= begin
+    #     ns = Namespace.new
+    #     sibling_ops_files.each do |ops_file|
+    #       ns.add(ops_file.basename, ops_file)
+    #     end
+    #     sibling_directories.each do |dir_path|
+    #       dir_basename = dir_path.basename
+    #       ns.add(dir_basename, ) unless resolve_symbol.resolve_symbol(dir_basename)
+    #     end
+    #     ns
+    #   end
+    # end
+
+    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "/home/david/sync/projects/ops/ops/core/host"
+    def dirname
+      @ops_file_path.dirname
+    end
+
+    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "info"
+    def basename
+      @ops_file_path.basename(".ops")
+    end
+
+    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "/home/david/sync/projects/ops/ops/core/host/info"
+    def private_lib_dir
+      dirname.join(basename)
+    end
+
+    def sibling_ops_files
+      dirname.glob("*.ops").map {|path| OpsFile.new(app, path) }
+    end
+
+    # irb(main):073:0> OpsFile.new("/home/david/sync/projects/ops/example/davidinfra/test.ops").sibling_directories
+    # => [#<Pathname:/home/david/sync/projects/ops/example/davidinfra/caddy>, #<Pathname:/home/david/sync/projects/ops/example/davidinfra/prepare_host>, #<Pathname:/home/david/sync/projects/ops/example/davidinfra/roles>]
+    def sibling_directories
+      dirname.glob("*").select(&:directory?)
+    end
+
+
+
+
+
+
+
+
+    def supporting_library_include_dir_and_require_lib
+      if Dir.exist?(ops_file_helper_library_directory)
+        [ops_file_helper_library_directory, ops_file_helper_library_basename]
+      elsif File.exist?(ops_file_sibling_helper_library_file)
+        [dirname, ops_file_helper_library_basename]
+      else
+        [nil, nil]
+      end
+    end
+
+    def ops_file_helper_library_basename
+      basename.sub_ext(".rb")
+    end
+
+    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "/home/david/sync/projects/ops/ops/core/host/info"
+    def ops_file_helper_library_directory
+      File.join(dirname, basename)
+    end
+
+    # "/home/david/sync/projects/ops/ops/core/host/info.ops" => "/home/david/sync/projects/ops/ops/core/host/info.rb"
+    def ops_file_sibling_helper_library_file
+      "#{ops_file_helper_library_directory}.rb"
+    end
+
+  end
+end