smart_proxy_remote_execution_ssh 0.1.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f1fa864f93c540a4c0c0e1cb837c4ed96962bc5d
-  data.tar.gz: 8e25bfe21c97d5b15039c6396eb18d86ee3d0eb8
+SHA256:
+  metadata.gz: 750910e916f0d4ad411cf868636075e597573dd8cca720e414156bcee55331dc
+  data.tar.gz: 4003e71f358abc47847fb9a2e4cf3c211189bc915b6b4196dd02d6d21633d2b8
 SHA512:
-  metadata.gz: f525bf0fcd9c1ebd907bd18c18cda1619641f3284b8e908273f950d9b8e5bd2ddddfcee1ab72c1a8f0aab12433670effea419bf7df5c5cb69a8761b135f4349c
-  data.tar.gz: eee8395b05faed6a1b0889dc46d3c4a1a73807187701d8c505b4bc8650573d3507aeda0e9de92401cefaac2442a46c0d68dd53a88b9ed882b779a1b23d41f8ad
+  metadata.gz: efa2a87ce6a6125f7701979305a0c5fcc1fb3ad2703d5aef140505943789b45648311e46f892791a919a88d757f019485ff45209efc22423cc6543a298224a03
+  data.tar.gz: 4411e680ca841903d47b295090c4a194f92dfe91cc58796272d42b9c370ad6a3e6a8bc34973f0d7a85d93fe604788993428d944277b05426194893ecff0a9d51

data/README.md

@@ -51,7 +51,7 @@ The simplest thing one can do is just to trigger a command:
 ```
 curl http://my-proxy.example.com:9292/dynflow/tasks \
       -X POST -H 'Content-Type: application/json'\
-      -d '{"action_name":  "Proxy::RemoteExecution::Ssh::CommandAction",
+      -d '{"action_name":  "ForemanRemoteExecutionCore::Actions::RunScript",
            "action_input": {"task_id" : "1234'$RANDOM'",
                             "script": "/usr/bin/ls",
                             "hostname": "localhost",

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -1,5 +1,7 @@
+require 'foreman_tasks_core'
 require 'smart_proxy_remote_execution_ssh/version'
 require 'smart_proxy_dynflow'
+require 'smart_proxy_remote_execution_ssh/webrick_ext'
 require 'smart_proxy_remote_execution_ssh/plugin'
 
 module Proxy::RemoteExecution
@@ -18,6 +20,8 @@ module Proxy::RemoteExecution
         unless File.exist?(public_key_file)
           raise "Ssh public key file #{public_key_file} doesn't exist"
         end
+
+        validate_ssh_log_level!
       end
 
       def private_key_file
@@ -27,6 +31,30 @@ module Proxy::RemoteExecution
       def public_key_file
         File.expand_path("#{private_key_file}.pub")
       end
+
+      def validate_ssh_log_level!
+        wanted_level = Plugin.settings.ssh_log_level.to_s
+        levels = Plugin::SSH_LOG_LEVELS
+        unless levels.include? wanted_level
+          raise "Wrong value '#{Plugin.settings.ssh_log_level}' for ssh_log_level, must be one of #{levels.join(', ')}"
+        end
+
+        current = ::Proxy::SETTINGS.log_level.to_s.downcase
+
+        # regular log levels correspond to upcased ssh logger levels
+        ssh, regular = [wanted_level, current].map do |wanted|
+          levels.each_with_index.find { |value, _index| value == wanted }.last
+        end
+
+        if ssh < regular
+          raise 'ssh_log_level cannot be more verbose than regular log level'
+        end
+
+        Plugin.settings.ssh_log_level = Plugin.settings.ssh_log_level.to_sym
+      end
     end
+
+    require 'smart_proxy_dynflow_core/task_launcher_registry'
+    SmartProxyDynflowCore::TaskLauncherRegistry.register('ssh', ForemanTasksCore::TaskLauncher::Batch)
   end
 end

data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb

@@ -0,0 +1,20 @@
+require 'foreman_tasks_core/shareable_action'
+
+module Proxy::RemoteExecution::Ssh
+  module Actions
+    class RunScript < ForemanTasksCore::Runner::Action
+      def initiate_runner
+        additional_options = {
+          :step_id => run_step_id,
+          :uuid => execution_plan_id,
+        }
+        Proxy::RemoteExecution::Ssh::Plugin.runner_class.build(input.merge(additional_options),
+                                                               suspended_action: suspended_action)
+      end
+
+      def runner_dispatcher
+        Dispatcher.instance
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -1,9 +1,42 @@
+require 'net/ssh'
+require 'base64'
+
 module Proxy::RemoteExecution
   module Ssh
+
     class Api < ::Sinatra::Base
+      include Sinatra::Authorization::Helpers
+
       get "/pubkey" do
         File.read(Ssh.public_key_file)
       end
+
+      post "/session" do
+        do_authorize_any
+        session = Cockpit::Session.new(env)
+        unless session.valid?
+          return [ 400, "Invalid request: /ssh/session requires connection upgrade to 'raw'" ]
+        end
+        session.hijack!
+        101
+      end
+
+      delete '/known_hosts/:name' do |name|
+        do_authorize_any
+        keys = Net::SSH::KnownHosts.search_for(name)
+        return [204] if keys.empty?
+        ssh_keys = keys.map { |key| Base64.strict_encode64 key.to_blob }
+        Net::SSH::KnownHosts.hostfiles({}, :user)
+          .map { |file| File.expand_path file }
+          .select { |file| File.readable?(file) && File.writable?(file) }
+          .each do |host_file|
+            lines = File.foreach(host_file).reject do |line|
+              ssh_keys.any? { |key| line.end_with? "#{key}\n" }
+            end
+            File.open(host_file, 'w') { |f| f.write lines.join }
+          end
+        204
+      end
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+#
+# Control script for the remote execution jobs.
+#
+# The initial script calls `$CONTROL_SCRIPT init-script-finish` once the original script exits.
+# In automatic mode, the exit code is sent back to the proxy on `init-script-finish`.
+#
+# What the script provides is also a manual mode, where the author of the rex script can take
+# full control of the job lifecycle. This allows keeping the marked as running even when
+# the initial script finishes.
+#
+# The manual mode is turned on by calling `$CONTROL_SCRIPT manual-control`. After calling this,
+# one can call `echo message | $CONTROL_SCRIPT update` to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code) to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code)
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+if ! command -v curl >/dev/null; then
+    echo 'curl is required' >&2
+    exit 1
+fi
+
+# send the callback data to proxy
+update() {
+    "$BASE_DIR/retrieve.sh" push_update
+}
+
+# wait for named pipe $1 to retrieve data. If $2 is provided, it serves as timeout
+# in seconds on how long to wait when reading.
+wait_for_pipe() {
+    pipe_path=$1
+    if [ -n "$2" ]; then
+        timeout="-t $2"
+    fi
+    if read $timeout <>"$pipe_path"; then
+        rm "$pipe_path"
+        return 0
+    else
+        return 1
+    fi
+}
+
+# function run in background, when receiving update data via STDIN.
+periodic_update() {
+    interval=1
+    # reading some data from periodic_update_control signals we're done
+    while ! wait_for_pipe "$BASE_DIR/periodic_update_control" "$interval"; do
+        update
+    done
+    # one more update before we finish
+    update
+    # signal the main process that we are finished
+    echo > "$BASE_DIR/periodic_update_finished"
+}
+
+# signal the periodic_update process that the main process is finishing
+periodic_update_finish() {
+    if [ -e "$BASE_DIR/periodic_update_control" ]; then
+       echo > "$BASE_DIR/periodic_update_control"
+    fi
+}
+
+ACTION=${1:-finish}
+
+case "$ACTION" in
+    init-script-finish)
+        if ! [ -e "$BASE_DIR/manual_mode" ]; then
+            # make the exit code of initialization script the exit code of the whole job
+            cp init_exit_code exit_code
+            update
+        fi
+        ;;
+    finish)
+        # take exit code passed via the command line, with fallback
+        # to the exit code of the initialization script
+        exit_code=${2:-$(cat "$BASE_DIR/init_exit_code")}
+        echo $exit_code > "$BASE_DIR/exit_code"
+        update
+        if [ -e "$BASE_DIR/manual_mode" ]; then
+            rm "$BASE_DIR/manual_mode"
+        fi
+        ;;
+    update)
+        # read data from input when redirected though a pipe
+        if ! [ -t 0 ]; then
+            # couple of named pipes to coordinate the main process with the periodic_update
+            mkfifo "$BASE_DIR/periodic_update_control"
+            mkfifo "$BASE_DIR/periodic_update_finished"
+            trap "periodic_update_finish" EXIT
+            # run periodic update as separate process to keep sending updates in output to server
+            periodic_update &
+            # redirect the input into output
+            tee -a "$BASE_DIR/output"
+            periodic_update_finish
+            # ensure the periodic update finished before we return
+            wait_for_pipe "$BASE_DIR/periodic_update_finished"
+        else
+            update
+        fi
+        ;;
+    # mark the script to be in manual mode: this means the script author needs to use `update` and `finish`
+    # commands to send output to the remote execution job or mark it as finished.
+    manual-mode)
+        touch "$BASE_DIR/manual_mode"
+        ;;
+    *)
+        echo "Unknown action $ACTION"
+        exit 1
+        ;;
+esac

data/lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+
+if ! pgrep --help 2>/dev/null >/dev/null; then
+    echo DONE 1
+    echo "pgrep is required" >&2
+    exit 1
+fi
+
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+# load the data required for generating the callback
+. "$BASE_DIR/env.sh"
+URL_PREFIX="$CALLBACK_HOST/dynflow/tasks/$TASK_ID"
+AUTH="$TASK_ID:$OTP"
+CURL="curl --silent --show-error --fail --max-time 10"
+
+MY_LOCK_FILE="$BASE_DIR/retrieve_lock.$$"
+MY_PID=$$
+echo $MY_PID >"$MY_LOCK_FILE"
+LOCK_FILE="$BASE_DIR/retrieve_lock"
+TMP_OUTPUT_FILE="$BASE_DIR/tmp_output"
+
+RUN_TIMEOUT=30 # for how long can the script hold the lock
+WAIT_TIMEOUT=60 # for how long the script is trying to acquire the lock
+START_TIME=$(date +%s)
+
+fail() {
+    echo RUNNING
+    echo "$1"
+    exit 1
+}
+
+acquire_lock() {
+    # try to acquire lock by creating the file (ln should be atomic an fail in case
+    # another process succeeded first). We also check the content of the lock file,
+    # in case our process won when competing over the lock while invalidating
+    # the lock on timeout.
+    ln "$MY_LOCK_FILE" "$LOCK_FILE" 2>/dev/null || [ "$(head -n1 "$LOCK_FILE")" = "$MY_PID" ]
+    return $?
+}
+
+# acquiring the lock before proceeding, to ensure only one instance of the script is running
+while ! acquire_lock; do
+    # we failed to create retrieve_lock - assuming there is already another retrieve script running
+    current_pid=$(head -n1 "$LOCK_FILE")
+    if [ -z "$current_pid" ]; then
+        continue
+    fi
+    # check whether the lock is not too old (compared to $RUN_TIMEOUT) and try to kill
+    # if it is, so that we don't have a stalled processes here
+    lock_lines_count=$(wc -l < "$LOCK_FILE")
+    current_lock_time=$(stat --format "%Y" "$LOCK_FILE")
+    current_time=$(date +%s)
+
+    if [ "$(( current_time - START_TIME ))" -gt "$WAIT_TIMEOUT" ]; then
+        # We were waiting for the lock for too long - just give up
+        fail "Wait time exceeded $WAIT_TIMEOUT"
+    elif [ "$(( current_time - current_lock_time ))" -gt "$RUN_TIMEOUT" ]; then
+        # The previous lock it hold for too long - re-acquiring procedure
+        if [ "$lock_lines_count" -gt 1 ]; then
+           # there were multiple processes waiting for lock without resolution
+           # longer than the $RUN_TIMEOUT - we reset the lock file and let processes
+           # to compete
+           echo "RETRY" > "$LOCK_FILE"
+        fi
+        if [ "$current_pid" != "RETRY" ]; then
+            # try to kill the currently stalled process
+            kill -9 "$current_pid" 2>/dev/null
+        fi
+        # try to add our process as one candidate
+        echo $MY_PID >> "$LOCK_FILE"
+        if [ "$( head -n2 "$LOCK_FILE" | tail -n1 )" = "$MY_PID" ]; then
+            # our process won the competition for the new lock: it is the first pid
+            # after the original one in the lock file - take ownership of the lock
+            # next iteration only this process will get through
+            echo $MY_PID >"$LOCK_FILE"
+        fi
+    else
+        # still waiting for the original owner to finish
+        sleep 1
+    fi
+done
+
+release_lock() {
+    rm "$MY_LOCK_FILE"
+    rm "$LOCK_FILE"
+}
+# ensure the release the lock at exit
+trap "release_lock" EXIT
+
+# make sure we clear previous tmp output file
+if [ -e "$TMP_OUTPUT_FILE" ]; then
+    rm "$TMP_OUTPUT_FILE"
+fi
+
+pid=$(cat "$BASE_DIR/pid")
+[ -f "$BASE_DIR/position" ] || echo 1 > "$BASE_DIR/position"
+position=$(cat "$BASE_DIR/position")
+
+prepare_output() {
+    if [ -e "$BASE_DIR/manual_mode" ] || ([ -n "$pid" ] && pgrep -P "$pid" >/dev/null 2>&1); then
+        echo RUNNING
+    else
+        echo "DONE $(cat "$BASE_DIR/exit_code" 2>/dev/null)"
+    fi
+    [ -f "$BASE_DIR/output" ] || exit 0
+    tail --bytes "+${position}" "$BASE_DIR/output" > "$TMP_OUTPUT_FILE"
+    cat "$TMP_OUTPUT_FILE"
+}
+
+# prepare the callback payload
+payload() {
+    if [ -n "$1" ]; then
+        exit_code="$1"
+    else
+        exit_code=null
+    fi
+
+    if [ -e "$BASE_DIR/manual_mode" ]; then
+        manual_mode=true
+        output=$(prepare_output | base64 -w0)
+    else
+        manual_mode=false
+    fi
+
+    echo "{ \"exit_code\": $exit_code,"\
+         "  \"step_id\": \"$STEP_ID\","\
+         "  \"manual_mode\": $manual_mode,"\
+         "  \"output\": \"$output\" }"
+}
+
+if [ "$1" = "push_update" ]; then
+    if [ -e "$BASE_DIR/exit_code" ]; then
+        exit_code="$(cat "$BASE_DIR/exit_code")"
+        action="done"
+    else
+        exit_code=""
+        action="update"
+    fi
+    $CURL -X POST -d "$(payload $exit_code)" -u "$AUTH" "$URL_PREFIX"/$action 2>>"$BASE_DIR/curl_stderr"
+    success=$?
+else
+    prepare_output
+    success=$?
+fi
+
+if [ "$success" = 0 ] && [ -e "$TMP_OUTPUT_FILE" ]; then
+    # in case the retrieval was successful, move the position of the cursor to be read next time
+    bytes=$(wc --bytes < "$TMP_OUTPUT_FILE")
+    expr "${position}" + "${bytes}" > "$BASE_DIR/position"
+fi

data/lib/smart_proxy_remote_execution_ssh/cockpit.rb

@@ -0,0 +1,269 @@
+require 'net/ssh'
+require 'forwardable'
+
+module Proxy::RemoteExecution
+  module Cockpit
+    # A wrapper class around different kind of sockets to comply with Net::SSH event loop
+    class BufferedSocket
+      include Net::SSH::BufferedIo
+      extend Forwardable
+
+      # The list of methods taken from OpenSSL::SSL::SocketForwarder for the object to act like a socket
+      def_delegators(:@socket, :to_io, :addr, :peeraddr, :setsockopt,
+                     :getsockopt, :fcntl, :close, :closed?, :do_not_reverse_lookup=)
+
+      def initialize(socket)
+        @socket = socket
+        initialize_buffered_io
+      end
+
+      def recv
+        raise NotImplementedError
+      end
+
+      def send
+        raise NotImplementedError
+      end
+
+      def self.applies_for?(socket)
+        raise NotImplementedError
+      end
+
+      def self.build(socket)
+        klass = [OpenSSLBufferedSocket, MiniSSLBufferedSocket, StandardBufferedSocket].find do |potential_class|
+          potential_class.applies_for?(socket)
+        end
+        raise "No suitable implementation of buffered socket available for #{socket.inspect}" unless klass
+        klass.new(socket)
+      end
+    end
+
+    class StandardBufferedSocket < BufferedSocket
+      def_delegators(:@socket, :send, :recv)
+
+      def self.applies_for?(socket)
+        socket.respond_to?(:send) && socket.respond_to?(:recv)
+      end
+    end
+
+    class OpenSSLBufferedSocket < BufferedSocket
+      def self.applies_for?(socket)
+        socket.is_a? ::OpenSSL::SSL::SSLSocket
+      end
+      def_delegators(:@socket, :read_nonblock, :write_nonblock, :close)
+
+      def recv(n)
+        res = ""
+        begin
+          # To drain a SSLSocket before we can go back to the event
+          # loop, we need to repeatedly call read_nonblock; a single
+          # call is not enough.
+          loop do
+            res += @socket.read_nonblock(n)
+          end
+        rescue IO::WaitReadable
+          # Sometimes there is no payload after reading everything
+          # from the underlying socket, but a empty string is treated
+          # as EOF by Net::SSH. So we block a bit until we have
+          # something to return.
+          if res == ""
+            IO.select([@socket.to_io])
+            retry
+          else
+            res
+          end
+        rescue IO::WaitWritable
+          # A renegotiation is happening, let it proceed.
+          IO.select(nil, [@socket.to_io])
+          retry
+        end
+      end
+
+      def send(mesg, flags)
+        @socket.write_nonblock(mesg)
+      rescue IO::WaitWritable
+        0
+      rescue IO::WaitReadable
+        IO.select([@socket.to_io])
+        retry
+      end
+    end
+
+    class MiniSSLBufferedSocket < BufferedSocket
+      def self.applies_for?(socket)
+        socket.is_a? ::Puma::MiniSSL::Socket
+      end
+      def_delegators(:@socket, :read_nonblock, :write_nonblock, :close)
+
+      def recv(n)
+        @socket.read_nonblock(n)
+      end
+
+      def send(mesg, flags)
+        @socket.write_nonblock(mesg)
+      end
+
+      def closed?
+        @socket.to_io.closed?
+      end
+    end
+
+    class Session
+      include ::Proxy::Log
+
+      def initialize(env)
+        @env = env
+      end
+
+      def valid?
+        @env["HTTP_CONNECTION"] == "upgrade" && @env["HTTP_UPGRADE"].to_s.split(',').any? { |part| part.strip == "raw" }
+      end
+
+      def hijack!
+        @socket = nil
+        if @env['ext.hijack!']
+          @socket = @env['ext.hijack!'].call
+        elsif @env['rack.hijack?']
+          begin
+            @env['rack.hijack'].call
+          rescue NotImplementedError
+          end
+          @socket = @env['rack.hijack_io']
+        end
+        raise 'Internal error: request hijacking not available' unless @socket
+        ssh_on_socket
+      end
+
+      private
+
+      def ssh_on_socket
+        with_error_handling { start_ssh_loop }
+      end
+
+      def with_error_handling
+        yield
+      rescue Net::SSH::AuthenticationFailed => e
+        send_error(401, e.message)
+      rescue Errno::EHOSTUNREACH
+        send_error(400, "No route to #{host}")
+      rescue SystemCallError => e
+        send_error(400, e.message)
+      rescue SocketError => e
+        send_error(400, e.message)
+      rescue Exception => e
+        logger.error e.message
+        logger.debug e.backtrace.join("\n")
+        send_error(500, "Internal error") unless @started
+      ensure
+        unless buf_socket.closed?
+          buf_socket.wait_for_pending_sends
+          buf_socket.close
+        end
+      end
+
+      def start_ssh_loop
+        err_buf = ""
+
+        Net::SSH.start(host, ssh_user, ssh_options) do |ssh|
+          channel = ssh.open_channel do |ch|
+            ch.exec(command) do |ch, success|
+              raise "could not execute command" unless success
+
+              ssh.listen_to(buf_socket)
+
+              ch.on_process do
+                if buf_socket.available.positive?
+                  ch.send_data(buf_socket.read_available)
+                end
+                if buf_socket.closed?
+                  ch.close
+                end
+              end
+
+              ch.on_data do |ch2, data|
+                send_start
+                buf_socket.enqueue(data)
+              end
+
+              ch.on_request('exit-status') do |ch, data|
+                code = data.read_long
+                send_start if code.zero?
+                err_buf += "Process exited with code #{code}.\r\n"
+                ch.close
+              end
+
+              ch.on_request('exit-signal') do |ch, data|
+                err_buf += "Process was terminated with signal #{data.read_string}.\r\n"
+                ch.close
+              end
+
+              ch.on_extended_data do |ch2, type, data|
+                err_buf += data
+              end
+            end
+          end
+
+          channel.wait
+          send_error(400, err_buf) unless @started
+        end
+      end
+
+      def send_start
+        unless @started
+          @started = true
+          buf_socket.enqueue("Status: 101\r\n")
+          buf_socket.enqueue("Connection: upgrade\r\n")
+          buf_socket.enqueue("Upgrade: raw\r\n")
+          buf_socket.enqueue("\r\n")
+        end
+      end
+
+      def send_error(code, msg)
+        buf_socket.enqueue("Status: #{code}\r\n")
+        buf_socket.enqueue("Connection: close\r\n")
+        buf_socket.enqueue("\r\n")
+        buf_socket.enqueue(msg)
+      end
+
+      def params
+        @params ||= MultiJson.load(@env["rack.input"].read)
+      end
+
+      def key_file
+        @key_file ||= Proxy::RemoteExecution::Ssh.private_key_file
+      end
+
+      def buf_socket
+        @buffered_socket ||= BufferedSocket.build(@socket)
+      end
+
+      def command
+        params["command"]
+      end
+
+      def ssh_user
+        params["ssh_user"]
+      end
+
+      def host
+        params["hostname"]
+      end
+
+      def ssh_options
+        auth_methods = %w[publickey]
+        auth_methods.unshift('password') if params["ssh_password"]
+
+        ret = {}
+        ret[:port] = params["ssh_port"] if params["ssh_port"]
+        ret[:keys] = [key_file] if key_file
+        ret[:password] = params["ssh_password"] if params["ssh_password"]
+        ret[:passphrase] = params[:ssh_key_passphrase] if params[:ssh_key_passphrase]
+        ret[:keys_only] = true
+        ret[:auth_methods] = auth_methods
+        ret[:verify_host_key] = true
+        ret[:number_of_password_prompts] = 1
+        ret
+      end
+    end
+  end
+end