smart_proxy_remote_execution_ssh 0.2.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8dc1047c21ec17cf7485df70ef6bf56fa5eff1d05b1e1ba5cc38f334a4c6ed1
-  data.tar.gz: b262ea28dc1bd77278a83134afbe334d6c2a413977eacf3eb6fd6e877123164b
+  metadata.gz: f90b757baa9c530c89d5c6c960ebf69601b173886c2f74560513592ab78697e4
+  data.tar.gz: edf9bc88acd2f859f86dd15ddee26deb948f6b963ddcd6aee2c6d839e12198ed
 SHA512:
-  metadata.gz: 4e0b5ddece76bc8bbcf24300d3e80dfe75244f6e11333dc247c63a2f6a2216ca9915e7765f24aedcd781eb14e3dadd43142f64c02b44ab63a1ee4ec5c37fbd96
-  data.tar.gz: ef98b2d1813f30d0ce5f35c79497e40ffabc73534f770fd2b82f10bbd03d996833a5f999fdd937177b3c639daa0bacd3908d2610c9a3c63ea565369bf9cec0c5
+  metadata.gz: 56cb17a0d7ac5eb456533e72e83aaffb07892a725609685f84c95571afa4fb0868dbd4017c4cf5ceadafa35aaf01beca27bdadb424b5bf0c0a5280c704f21e2c
+  data.tar.gz: 04c1feec777172574e45369f48219a6d2ea80e7e704a695198ef56bb11b5d4c178c68650414a190f4e57751521a065ad5c0c649ad8842f99be03259f79f3c9f3

data/README.md

@@ -51,7 +51,7 @@ The simplest thing one can do is just to trigger a command:
 ```
 curl http://my-proxy.example.com:9292/dynflow/tasks \
       -X POST -H 'Content-Type: application/json'\
-      -d '{"action_name":  "Proxy::RemoteExecution::Ssh::CommandAction",
+      -d '{"action_name":  "ForemanRemoteExecutionCore::Actions::RunScript",
            "action_input": {"task_id" : "1234'$RANDOM'",
                             "script": "/usr/bin/ls",
                             "hostname": "localhost",

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -1,6 +1,7 @@
-require 'smart_proxy_remote_execution_ssh/version'
 require 'smart_proxy_dynflow'
+require 'smart_proxy_remote_execution_ssh/version'
 require 'smart_proxy_remote_execution_ssh/plugin'
+require 'smart_proxy_remote_execution_ssh/webrick_ext'
 
 module Proxy::RemoteExecution
   module Ssh
@@ -18,6 +19,8 @@ module Proxy::RemoteExecution
         unless File.exist?(public_key_file)
           raise "Ssh public key file #{public_key_file} doesn't exist"
         end
+
+        validate_ssh_log_level!
       end
 
       def private_key_file
@@ -27,6 +30,27 @@ module Proxy::RemoteExecution
       def public_key_file
         File.expand_path("#{private_key_file}.pub")
       end
+
+      def validate_ssh_log_level!
+        wanted_level = Plugin.settings.ssh_log_level.to_s
+        levels = Plugin::SSH_LOG_LEVELS
+        unless levels.include? wanted_level
+          raise "Wrong value '#{Plugin.settings.ssh_log_level}' for ssh_log_level, must be one of #{levels.join(', ')}"
+        end
+
+        current = ::Proxy::SETTINGS.log_level.to_s.downcase
+
+        # regular log levels correspond to upcased ssh logger levels
+        ssh, regular = [wanted_level, current].map do |wanted|
+          levels.each_with_index.find { |value, _index| value == wanted }.last
+        end
+
+        if ssh < regular
+          raise 'ssh_log_level cannot be more verbose than regular log level'
+        end
+
+        Plugin.settings.ssh_log_level = Plugin.settings.ssh_log_level.to_sym
+      end
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb

@@ -0,0 +1,21 @@
+require 'smart_proxy_dynflow/action/shareable'
+require 'smart_proxy_dynflow/action/runner'
+
+module Proxy::RemoteExecution::Ssh
+  module Actions
+    class RunScript < Proxy::Dynflow::Action::Runner
+      def initiate_runner
+        additional_options = {
+          :step_id => run_step_id,
+          :uuid => execution_plan_id,
+        }
+        Proxy::RemoteExecution::Ssh::Plugin.runner_class.build(input.merge(additional_options),
+                                                               suspended_action: suspended_action)
+      end
+
+      def runner_dispatcher
+        Dispatcher.instance
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -1,9 +1,42 @@
+require 'net/ssh'
+require 'base64'
+
 module Proxy::RemoteExecution
   module Ssh
+
     class Api < ::Sinatra::Base
+      include Sinatra::Authorization::Helpers
+
       get "/pubkey" do
         File.read(Ssh.public_key_file)
       end
+
+      post "/session" do
+        do_authorize_any
+        session = Cockpit::Session.new(env)
+        unless session.valid?
+          return [ 400, "Invalid request: /ssh/session requires connection upgrade to 'raw'" ]
+        end
+        session.hijack!
+        101
+      end
+
+      delete '/known_hosts/:name' do |name|
+        do_authorize_any
+        keys = Net::SSH::KnownHosts.search_for(name)
+        return [204] if keys.empty?
+        ssh_keys = keys.map { |key| Base64.strict_encode64 key.to_blob }
+        Net::SSH::KnownHosts.hostfiles({}, :user)
+          .map { |file| File.expand_path file }
+          .select { |file| File.readable?(file) && File.writable?(file) }
+          .each do |host_file|
+            lines = File.foreach(host_file).reject do |line|
+              ssh_keys.any? { |key| line.end_with? "#{key}\n" }
+            end
+            File.open(host_file, 'w') { |f| f.write lines.join }
+          end
+        204
+      end
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+#
+# Control script for the remote execution jobs.
+#
+# The initial script calls `$CONTROL_SCRIPT init-script-finish` once the original script exits.
+# In automatic mode, the exit code is sent back to the proxy on `init-script-finish`.
+#
+# What the script provides is also a manual mode, where the author of the rex script can take
+# full control of the job lifecycle. This allows keeping the marked as running even when
+# the initial script finishes.
+#
+# The manual mode is turned on by calling `$CONTROL_SCRIPT manual-control`. After calling this,
+# one can call `echo message | $CONTROL_SCRIPT update` to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code) to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code)
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+if ! command -v curl >/dev/null; then
+    echo 'curl is required' >&2
+    exit 1
+fi
+
+# send the callback data to proxy
+update() {
+    "$BASE_DIR/retrieve.sh" push_update
+}
+
+# wait for named pipe $1 to retrieve data. If $2 is provided, it serves as timeout
+# in seconds on how long to wait when reading.
+wait_for_pipe() {
+    pipe_path=$1
+    if [ -n "$2" ]; then
+        timeout="-t $2"
+    fi
+    if read $timeout <>"$pipe_path"; then
+        rm "$pipe_path"
+        return 0
+    else
+        return 1
+    fi
+}
+
+# function run in background, when receiving update data via STDIN.
+periodic_update() {
+    interval=1
+    # reading some data from periodic_update_control signals we're done
+    while ! wait_for_pipe "$BASE_DIR/periodic_update_control" "$interval"; do
+        update
+    done
+    # one more update before we finish
+    update
+    # signal the main process that we are finished
+    echo > "$BASE_DIR/periodic_update_finished"
+}
+
+# signal the periodic_update process that the main process is finishing
+periodic_update_finish() {
+    if [ -e "$BASE_DIR/periodic_update_control" ]; then
+       echo > "$BASE_DIR/periodic_update_control"
+    fi
+}
+
+ACTION=${1:-finish}
+
+case "$ACTION" in
+    init-script-finish)
+        if ! [ -e "$BASE_DIR/manual_mode" ]; then
+            # make the exit code of initialization script the exit code of the whole job
+            cp init_exit_code exit_code
+            update
+        fi
+        ;;
+    finish)
+        # take exit code passed via the command line, with fallback
+        # to the exit code of the initialization script
+        exit_code=${2:-$(cat "$BASE_DIR/init_exit_code")}
+        echo $exit_code > "$BASE_DIR/exit_code"
+        update
+        if [ -e "$BASE_DIR/manual_mode" ]; then
+            rm "$BASE_DIR/manual_mode"
+        fi
+        ;;
+    update)
+        # read data from input when redirected though a pipe
+        if ! [ -t 0 ]; then
+            # couple of named pipes to coordinate the main process with the periodic_update
+            mkfifo "$BASE_DIR/periodic_update_control"
+            mkfifo "$BASE_DIR/periodic_update_finished"
+            trap "periodic_update_finish" EXIT
+            # run periodic update as separate process to keep sending updates in output to server
+            periodic_update &
+            # redirect the input into output
+            tee -a "$BASE_DIR/output"
+            periodic_update_finish
+            # ensure the periodic update finished before we return
+            wait_for_pipe "$BASE_DIR/periodic_update_finished"
+        else
+            update
+        fi
+        ;;
+    # mark the script to be in manual mode: this means the script author needs to use `update` and `finish`
+    # commands to send output to the remote execution job or mark it as finished.
+    manual-mode)
+        touch "$BASE_DIR/manual_mode"
+        ;;
+    *)
+        echo "Unknown action $ACTION"
+        exit 1
+        ;;
+esac

data/lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+
+if ! pgrep --help 2>/dev/null >/dev/null; then
+    echo DONE 1
+    echo "pgrep is required" >&2
+    exit 1
+fi
+
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+# load the data required for generating the callback
+. "$BASE_DIR/env.sh"
+URL_PREFIX="$CALLBACK_HOST/dynflow/tasks/$TASK_ID"
+AUTH="$TASK_ID:$OTP"
+CURL="curl --silent --show-error --fail --max-time 10"
+
+MY_LOCK_FILE="$BASE_DIR/retrieve_lock.$$"
+MY_PID=$$
+echo $MY_PID >"$MY_LOCK_FILE"
+LOCK_FILE="$BASE_DIR/retrieve_lock"
+TMP_OUTPUT_FILE="$BASE_DIR/tmp_output"
+
+RUN_TIMEOUT=30 # for how long can the script hold the lock
+WAIT_TIMEOUT=60 # for how long the script is trying to acquire the lock
+START_TIME=$(date +%s)
+
+fail() {
+    echo RUNNING
+    echo "$1"
+    exit 1
+}
+
+acquire_lock() {
+    # try to acquire lock by creating the file (ln should be atomic an fail in case
+    # another process succeeded first). We also check the content of the lock file,
+    # in case our process won when competing over the lock while invalidating
+    # the lock on timeout.
+    ln "$MY_LOCK_FILE" "$LOCK_FILE" 2>/dev/null || [ "$(head -n1 "$LOCK_FILE")" = "$MY_PID" ]
+    return $?
+}
+
+# acquiring the lock before proceeding, to ensure only one instance of the script is running
+while ! acquire_lock; do
+    # we failed to create retrieve_lock - assuming there is already another retrieve script running
+    current_pid=$(head -n1 "$LOCK_FILE")
+    if [ -z "$current_pid" ]; then
+        continue
+    fi
+    # check whether the lock is not too old (compared to $RUN_TIMEOUT) and try to kill
+    # if it is, so that we don't have a stalled processes here
+    lock_lines_count=$(wc -l < "$LOCK_FILE")
+    current_lock_time=$(stat --format "%Y" "$LOCK_FILE")
+    current_time=$(date +%s)
+
+    if [ "$(( current_time - START_TIME ))" -gt "$WAIT_TIMEOUT" ]; then
+        # We were waiting for the lock for too long - just give up
+        fail "Wait time exceeded $WAIT_TIMEOUT"
+    elif [ "$(( current_time - current_lock_time ))" -gt "$RUN_TIMEOUT" ]; then
+        # The previous lock it hold for too long - re-acquiring procedure
+        if [ "$lock_lines_count" -gt 1 ]; then
+           # there were multiple processes waiting for lock without resolution
+           # longer than the $RUN_TIMEOUT - we reset the lock file and let processes
+           # to compete
+           echo "RETRY" > "$LOCK_FILE"
+        fi
+        if [ "$current_pid" != "RETRY" ]; then
+            # try to kill the currently stalled process
+            kill -9 "$current_pid" 2>/dev/null
+        fi
+        # try to add our process as one candidate
+        echo $MY_PID >> "$LOCK_FILE"
+        if [ "$( head -n2 "$LOCK_FILE" | tail -n1 )" = "$MY_PID" ]; then
+            # our process won the competition for the new lock: it is the first pid
+            # after the original one in the lock file - take ownership of the lock
+            # next iteration only this process will get through
+            echo $MY_PID >"$LOCK_FILE"
+        fi
+    else
+        # still waiting for the original owner to finish
+        sleep 1
+    fi
+done
+
+release_lock() {
+    rm "$MY_LOCK_FILE"
+    rm "$LOCK_FILE"
+}
+# ensure the release the lock at exit
+trap "release_lock" EXIT
+
+# make sure we clear previous tmp output file
+if [ -e "$TMP_OUTPUT_FILE" ]; then
+    rm "$TMP_OUTPUT_FILE"
+fi
+
+pid=$(cat "$BASE_DIR/pid")
+[ -f "$BASE_DIR/position" ] || echo 1 > "$BASE_DIR/position"
+position=$(cat "$BASE_DIR/position")
+
+prepare_output() {
+    if [ -e "$BASE_DIR/manual_mode" ] || ([ -n "$pid" ] && pgrep -P "$pid" >/dev/null 2>&1); then
+        echo RUNNING
+    else
+        echo "DONE $(cat "$BASE_DIR/exit_code" 2>/dev/null)"
+    fi
+    [ -f "$BASE_DIR/output" ] || exit 0
+    tail --bytes "+${position}" "$BASE_DIR/output" > "$TMP_OUTPUT_FILE"
+    cat "$TMP_OUTPUT_FILE"
+}
+
+# prepare the callback payload
+payload() {
+    if [ -n "$1" ]; then
+        exit_code="$1"
+    else
+        exit_code=null
+    fi
+
+    if [ -e "$BASE_DIR/manual_mode" ]; then
+        manual_mode=true
+        output=$(prepare_output | base64 -w0)
+    else
+        manual_mode=false
+    fi
+
+    echo "{ \"exit_code\": $exit_code,"\
+         "  \"step_id\": \"$STEP_ID\","\
+         "  \"manual_mode\": $manual_mode,"\
+         "  \"output\": \"$output\" }"
+}
+
+if [ "$1" = "push_update" ]; then
+    if [ -e "$BASE_DIR/exit_code" ]; then
+        exit_code="$(cat "$BASE_DIR/exit_code")"
+        action="done"
+    else
+        exit_code=""
+        action="update"
+    fi
+    $CURL -X POST -d "$(payload $exit_code)" -u "$AUTH" "$URL_PREFIX"/$action 2>>"$BASE_DIR/curl_stderr"
+    success=$?
+else
+    prepare_output
+    success=$?
+fi
+
+if [ "$success" = 0 ] && [ -e "$TMP_OUTPUT_FILE" ]; then
+    # in case the retrieval was successful, move the position of the cursor to be read next time
+    bytes=$(wc --bytes < "$TMP_OUTPUT_FILE")
+    expr "${position}" + "${bytes}" > "$BASE_DIR/position"
+fi

data/lib/smart_proxy_remote_execution_ssh/cockpit.rb

@@ -0,0 +1,269 @@
+require 'net/ssh'
+require 'forwardable'
+
+module Proxy::RemoteExecution
+  module Cockpit
+    # A wrapper class around different kind of sockets to comply with Net::SSH event loop
+    class BufferedSocket
+      include Net::SSH::BufferedIo
+      extend Forwardable
+
+      # The list of methods taken from OpenSSL::SSL::SocketForwarder for the object to act like a socket
+      def_delegators(:@socket, :to_io, :addr, :peeraddr, :setsockopt,
+                     :getsockopt, :fcntl, :close, :closed?, :do_not_reverse_lookup=)
+
+      def initialize(socket)
+        @socket = socket
+        initialize_buffered_io
+      end
+
+      def recv
+        raise NotImplementedError
+      end
+
+      def send
+        raise NotImplementedError
+      end
+
+      def self.applies_for?(socket)
+        raise NotImplementedError
+      end
+
+      def self.build(socket)
+        klass = [OpenSSLBufferedSocket, MiniSSLBufferedSocket, StandardBufferedSocket].find do |potential_class|
+          potential_class.applies_for?(socket)
+        end
+        raise "No suitable implementation of buffered socket available for #{socket.inspect}" unless klass
+        klass.new(socket)
+      end
+    end
+
+    class StandardBufferedSocket < BufferedSocket
+      def_delegators(:@socket, :send, :recv)
+
+      def self.applies_for?(socket)
+        socket.respond_to?(:send) && socket.respond_to?(:recv)
+      end
+    end
+
+    class OpenSSLBufferedSocket < BufferedSocket
+      def self.applies_for?(socket)
+        socket.is_a? ::OpenSSL::SSL::SSLSocket
+      end
+      def_delegators(:@socket, :read_nonblock, :write_nonblock, :close)
+
+      def recv(n)
+        res = ""
+        begin
+          # To drain a SSLSocket before we can go back to the event
+          # loop, we need to repeatedly call read_nonblock; a single
+          # call is not enough.
+          loop do
+            res += @socket.read_nonblock(n)
+          end
+        rescue IO::WaitReadable
+          # Sometimes there is no payload after reading everything
+          # from the underlying socket, but a empty string is treated
+          # as EOF by Net::SSH. So we block a bit until we have
+          # something to return.
+          if res == ""
+            IO.select([@socket.to_io])
+            retry
+          else
+            res
+          end
+        rescue IO::WaitWritable
+          # A renegotiation is happening, let it proceed.
+          IO.select(nil, [@socket.to_io])
+          retry
+        end
+      end
+
+      def send(mesg, flags)
+        @socket.write_nonblock(mesg)
+      rescue IO::WaitWritable
+        0
+      rescue IO::WaitReadable
+        IO.select([@socket.to_io])
+        retry
+      end
+    end
+
+    class MiniSSLBufferedSocket < BufferedSocket
+      def self.applies_for?(socket)
+        socket.is_a? ::Puma::MiniSSL::Socket
+      end
+      def_delegators(:@socket, :read_nonblock, :write_nonblock, :close)
+
+      def recv(n)
+        @socket.read_nonblock(n)
+      end
+
+      def send(mesg, flags)
+        @socket.write_nonblock(mesg)
+      end
+
+      def closed?
+        @socket.to_io.closed?
+      end
+    end
+
+    class Session
+      include ::Proxy::Log
+
+      def initialize(env)
+        @env = env
+      end
+
+      def valid?
+        @env["HTTP_CONNECTION"] == "upgrade" && @env["HTTP_UPGRADE"].to_s.split(',').any? { |part| part.strip == "raw" }
+      end
+
+      def hijack!
+        @socket = nil
+        if @env['ext.hijack!']
+          @socket = @env['ext.hijack!'].call
+        elsif @env['rack.hijack?']
+          begin
+            @env['rack.hijack'].call
+          rescue NotImplementedError
+          end
+          @socket = @env['rack.hijack_io']
+        end
+        raise 'Internal error: request hijacking not available' unless @socket
+        ssh_on_socket
+      end
+
+      private
+
+      def ssh_on_socket
+        with_error_handling { start_ssh_loop }
+      end
+
+      def with_error_handling
+        yield
+      rescue Net::SSH::AuthenticationFailed => e
+        send_error(401, e.message)
+      rescue Errno::EHOSTUNREACH
+        send_error(400, "No route to #{host}")
+      rescue SystemCallError => e
+        send_error(400, e.message)
+      rescue SocketError => e
+        send_error(400, e.message)
+      rescue Exception => e
+        logger.error e.message
+        logger.debug e.backtrace.join("\n")
+        send_error(500, "Internal error") unless @started
+      ensure
+        unless buf_socket.closed?
+          buf_socket.wait_for_pending_sends
+          buf_socket.close
+        end
+      end
+
+      def start_ssh_loop
+        err_buf = ""
+
+        Net::SSH.start(host, ssh_user, ssh_options) do |ssh|
+          channel = ssh.open_channel do |ch|
+            ch.exec(command) do |ch, success|
+              raise "could not execute command" unless success
+
+              ssh.listen_to(buf_socket)
+
+              ch.on_process do
+                if buf_socket.available.positive?
+                  ch.send_data(buf_socket.read_available)
+                end
+                if buf_socket.closed?
+                  ch.close
+                end
+              end
+
+              ch.on_data do |ch2, data|
+                send_start
+                buf_socket.enqueue(data)
+              end
+
+              ch.on_request('exit-status') do |ch, data|
+                code = data.read_long
+                send_start if code.zero?
+                err_buf += "Process exited with code #{code}.\r\n"
+                ch.close
+              end
+
+              ch.on_request('exit-signal') do |ch, data|
+                err_buf += "Process was terminated with signal #{data.read_string}.\r\n"
+                ch.close
+              end
+
+              ch.on_extended_data do |ch2, type, data|
+                err_buf += data
+              end
+            end
+          end
+
+          channel.wait
+          send_error(400, err_buf) unless @started
+        end
+      end
+
+      def send_start
+        unless @started
+          @started = true
+          buf_socket.enqueue("Status: 101\r\n")
+          buf_socket.enqueue("Connection: upgrade\r\n")
+          buf_socket.enqueue("Upgrade: raw\r\n")
+          buf_socket.enqueue("\r\n")
+        end
+      end
+
+      def send_error(code, msg)
+        buf_socket.enqueue("Status: #{code}\r\n")
+        buf_socket.enqueue("Connection: close\r\n")
+        buf_socket.enqueue("\r\n")
+        buf_socket.enqueue(msg)
+      end
+
+      def params
+        @params ||= MultiJson.load(@env["rack.input"].read)
+      end
+
+      def key_file
+        @key_file ||= Proxy::RemoteExecution::Ssh.private_key_file
+      end
+
+      def buf_socket
+        @buffered_socket ||= BufferedSocket.build(@socket)
+      end
+
+      def command
+        params["command"]
+      end
+
+      def ssh_user
+        params["ssh_user"]
+      end
+
+      def host
+        params["hostname"]
+      end
+
+      def ssh_options
+        auth_methods = %w[publickey]
+        auth_methods.unshift('password') if params["ssh_password"]
+
+        ret = {}
+        ret[:port] = params["ssh_port"] if params["ssh_port"]
+        ret[:keys] = [key_file] if key_file
+        ret[:password] = params["ssh_password"] if params["ssh_password"]
+        ret[:passphrase] = params["ssh_key_passphrase"] if params["ssh_key_passphrase"]
+        ret[:keys_only] = true
+        ret[:auth_methods] = auth_methods
+        ret[:verify_host_key] = true
+        ret[:number_of_password_prompts] = 1
+        ret
+      end
+    end
+  end
+end