smart_proxy_remote_execution_ssh 0.3.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9532552fe00ed7bf26c3755d710ad2ea87b4ac9255a12851a2a2f20cd6e3afbf
-  data.tar.gz: 5b2d309cc8bdec3ed2c20d2b0a944c6bc1863a3fd728e7febc5e020dc3f88239
+  metadata.gz: e2f91ec259ca105bcc18a5cecf4342019f4023d44cbff5695998b5e57c25e325
+  data.tar.gz: 51ccf299324d4adf10c07475dca2749d885c258913c8d9e7593d8b8149e6d18d
 SHA512:
-  metadata.gz: a2359e2706bb4d465fbb98285e48099d0ff3c6f58ab7aae0e4972d101ec2b5663bea62e965be074c784dea19cda7de551d1f27f26aba6c4502d4d96c9151bd7d
-  data.tar.gz: f7bce64d724947712ee9c2d394802b07f64df2c581d92d36cb3ac4732b9a8609b2c1935e4189a39a0712692c2946111e426eb2ea5452f379906eb3ff379142ee
+  metadata.gz: 74313e91e51ff5d58fce0bc880131b12984a750110a2d76c88875a6b824eec40a2ba9d7336dfda34df7c11d465ded62c14d4d33ba10cf4bed1b2e0309aff9a2b
+  data.tar.gz: 92f85e971f10d9e45418e75d0325f3e548e65954160be9f86cb154adde6bc3a512b4e3a5429d162291704b49a50a5570fcdeb4b129f735b8f8f737938a0194d0

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -0,0 +1,110 @@
+require 'mqtt'
+require 'json'
+
+module Proxy::RemoteExecution::Ssh::Actions
+  class PullScript < Proxy::Dynflow::Action::Runner
+    JobDelivered = Class.new
+
+    execution_plan_hooks.use :cleanup, :on => :stopped
+
+    def plan(action_input, mqtt: false)
+      super(action_input)
+      input[:with_mqtt] = mqtt
+    end
+
+    def run(event = nil)
+      if event == JobDelivered
+        output[:state] = :delivered
+        suspend
+      else
+        super
+      end
+    end
+
+    def init_run
+      otp_password = if input[:with_mqtt]
+                       ::Proxy::Dynflow::OtpManager.generate_otp(execution_plan_id)
+                     end
+      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script])
+      output[:state] = :ready_for_pickup
+      output[:result] = []
+      mqtt_start(otp_password) if input[:with_mqtt]
+      suspend
+    end
+
+    def cleanup(_plan = nil)
+      job_storage.drop_job(execution_plan_id, run_step_id)
+      Proxy::Dynflow::OtpManager.passwords.delete(execution_plan_id)
+    end
+
+    def process_external_event(event)
+      output[:state] = :running
+      data = event.data
+      continuous_output = Proxy::Dynflow::ContinuousOutput.new
+      Array(data['output']).each { |line| continuous_output.add_output(line, 'stdout') } if data.key?('output')
+      exit_code = data['exit_code'].to_i if data['exit_code']
+      process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
+    end
+
+    def kill_run
+      case output[:state]
+      when :ready_for_pickup
+        # If the job is not running yet on the client, wipe it from storage
+        cleanup
+        # TODO: Stop the action
+      when :notified, :running
+        # Client was notified or is already running, dealing with this situation
+        # is only supported if mqtt is available
+        # Otherwise we have to wait it out
+        # TODO
+        # if input[:with_mqtt]
+      end
+      suspend
+    end
+
+    def mqtt_start(otp_password)
+      payload = {
+        type: 'data',
+        message_id: SecureRandom.uuid,
+        version: 1,
+        sent: DateTime.now.iso8601,
+        directive: 'foreman',
+        metadata: {
+          'job_uuid': input[:job_uuid],
+          'username': execution_plan_id,
+          'password': otp_password,
+          'return_url': "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}/update",
+        },
+        content: "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}",
+      }
+      mqtt_notify payload
+      output[:state] = :notified
+    end
+
+    def mqtt_notify(payload)
+      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port) do |c|
+        c.publish(mqtt_topic, JSON.dump(payload), false, 1)
+      end
+    end
+
+    def host_name
+      alternative_names = input.fetch(:alternative_names, {})
+
+      alternative_names[:consumer_uuid] ||
+        alternative_names[:fqdn] ||
+        input[:hostname]
+    end
+
+    def mqtt_topic
+      "yggdrasil/#{host_name}/data/in"
+    end
+
+    def settings
+      Proxy::RemoteExecution::Ssh::Plugin.settings
+    end
+
+    def job_storage
+      Proxy::RemoteExecution::Ssh.job_storage
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb

@@ -0,0 +1,34 @@
+require 'smart_proxy_dynflow/action/shareable'
+require 'smart_proxy_dynflow/action/runner'
+
+module Proxy::RemoteExecution::Ssh
+  module Actions
+    class RunScript < ::Dynflow::Action
+      def plan(*args)
+        mode = Proxy::RemoteExecution::Ssh::Plugin.settings.mode
+        case mode
+        when :ssh, :'ssh-async'
+          plan_action(ScriptRunner, *args)
+        when :pull, :'pull-mqtt'
+          plan_action(PullScript, *args,
+                      mqtt: mode == :'pull-mqtt')
+        end
+      end
+    end
+
+    class ScriptRunner < Proxy::Dynflow::Action::Runner
+      def initiate_runner
+        additional_options = {
+          :step_id => run_step_id,
+          :uuid => execution_plan_id,
+        }
+        Proxy::RemoteExecution::Ssh::Plugin.runner_class.build(input.merge(additional_options),
+                                                               suspended_action: suspended_action)
+      end
+
+      def runner_dispatcher
+        Dispatcher.instance
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/actions.rb

@@ -0,0 +1,6 @@
+module Proxy::RemoteExecution::Ssh
+  module Actions
+    require 'smart_proxy_remote_execution_ssh/actions/run_script'
+    require 'smart_proxy_remote_execution_ssh/actions/pull_script'
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -1,11 +1,13 @@
 require 'net/ssh'
 require 'base64'
+require 'smart_proxy_dynflow/runner'
 
 module Proxy::RemoteExecution
   module Ssh
 
     class Api < ::Sinatra::Base
       include Sinatra::Authorization::Helpers
+      include Proxy::Dynflow::Helpers
 
       get "/pubkey" do
         File.read(Ssh.public_key_file)
@@ -37,6 +39,53 @@ module Proxy::RemoteExecution
           end
         204
       end
+
+      # Payload is a hash where
+      # exit_code: Integer | NilClass
+      # output: String
+      post '/jobs/:job_uuid/update' do |job_uuid|
+        do_authorize_with_ssl_client
+
+        with_authorized_job(job_uuid) do |job_record|
+          data = MultiJson.load(request.body.read)
+          notify_job(job_record, ::Proxy::Dynflow::Runner::ExternalEvent.new(data))
+        end
+      end
+
+      get '/jobs' do
+        do_authorize_with_ssl_client
+
+        MultiJson.dump(Proxy::RemoteExecution::Ssh.job_storage.job_uuids_for_host(https_cert_cn))
+      end
+
+      get "/jobs/:job_uuid" do |job_uuid|
+        do_authorize_with_ssl_client
+
+        with_authorized_job(job_uuid) do |job_record|
+          notify_job(job_record, Actions::PullScript::JobDelivered)
+          job_record[:job]
+        end
+      end
+
+      private
+
+      def notify_job(job_record, event)
+        world.event(job_record[:execution_plan_uuid], job_record[:run_step_id], event)
+      end
+
+      def with_authorized_job(uuid)
+        if (job = authorized_job(uuid))
+          yield job
+        else
+          halt 404
+        end
+      end
+
+      def authorized_job(uuid)
+        job_record = Proxy::RemoteExecution::Ssh.job_storage.find_job(uuid) || {}
+        return job_record if authorize_with_token(clear: false, task_id: job_record[:execution_plan_uuid]) ||
+                             job_record[:hostname] == https_cert_cn
+      end
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+#
+# Control script for the remote execution jobs.
+#
+# The initial script calls `$CONTROL_SCRIPT init-script-finish` once the original script exits.
+# In automatic mode, the exit code is sent back to the proxy on `init-script-finish`.
+#
+# What the script provides is also a manual mode, where the author of the rex script can take
+# full control of the job lifecycle. This allows keeping the marked as running even when
+# the initial script finishes.
+#
+# The manual mode is turned on by calling `$CONTROL_SCRIPT manual-control`. After calling this,
+# one can call `echo message | $CONTROL_SCRIPT update` to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code) to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code)
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+if ! command -v curl >/dev/null; then
+    echo 'curl is required' >&2
+    exit 1
+fi
+
+# send the callback data to proxy
+update() {
+    "$BASE_DIR/retrieve.sh" push_update
+}
+
+# wait for named pipe $1 to retrieve data. If $2 is provided, it serves as timeout
+# in seconds on how long to wait when reading.
+wait_for_pipe() {
+    pipe_path=$1
+    if [ -n "$2" ]; then
+        timeout="-t $2"
+    fi
+    if read $timeout <>"$pipe_path"; then
+        rm "$pipe_path"
+        return 0
+    else
+        return 1
+    fi
+}
+
+# function run in background, when receiving update data via STDIN.
+periodic_update() {
+    interval=1
+    # reading some data from periodic_update_control signals we're done
+    while ! wait_for_pipe "$BASE_DIR/periodic_update_control" "$interval"; do
+        update
+    done
+    # one more update before we finish
+    update
+    # signal the main process that we are finished
+    echo > "$BASE_DIR/periodic_update_finished"
+}
+
+# signal the periodic_update process that the main process is finishing
+periodic_update_finish() {
+    if [ -e "$BASE_DIR/periodic_update_control" ]; then
+       echo > "$BASE_DIR/periodic_update_control"
+    fi
+}
+
+ACTION=${1:-finish}
+
+case "$ACTION" in
+    init-script-finish)
+        if ! [ -e "$BASE_DIR/manual_mode" ]; then
+            # make the exit code of initialization script the exit code of the whole job
+            cp init_exit_code exit_code
+            update
+        fi
+        ;;
+    finish)
+        # take exit code passed via the command line, with fallback
+        # to the exit code of the initialization script
+        exit_code=${2:-$(cat "$BASE_DIR/init_exit_code")}
+        echo $exit_code > "$BASE_DIR/exit_code"
+        update
+        if [ -e "$BASE_DIR/manual_mode" ]; then
+            rm "$BASE_DIR/manual_mode"
+        fi
+        ;;
+    update)
+        # read data from input when redirected though a pipe
+        if ! [ -t 0 ]; then
+            # couple of named pipes to coordinate the main process with the periodic_update
+            mkfifo "$BASE_DIR/periodic_update_control"
+            mkfifo "$BASE_DIR/periodic_update_finished"
+            trap "periodic_update_finish" EXIT
+            # run periodic update as separate process to keep sending updates in output to server
+            periodic_update &
+            # redirect the input into output
+            tee -a "$BASE_DIR/output"
+            periodic_update_finish
+            # ensure the periodic update finished before we return
+            wait_for_pipe "$BASE_DIR/periodic_update_finished"
+        else
+            update
+        fi
+        ;;
+    # mark the script to be in manual mode: this means the script author needs to use `update` and `finish`
+    # commands to send output to the remote execution job or mark it as finished.
+    manual-mode)
+        touch "$BASE_DIR/manual_mode"
+        ;;
+    *)
+        echo "Unknown action $ACTION"
+        exit 1
+        ;;
+esac

data/lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+
+if ! pgrep --help 2>/dev/null >/dev/null; then
+    echo DONE 1
+    echo "pgrep is required" >&2
+    exit 1
+fi
+
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+# load the data required for generating the callback
+. "$BASE_DIR/env.sh"
+URL_PREFIX="$CALLBACK_HOST/dynflow/tasks/$TASK_ID"
+AUTH="$TASK_ID:$OTP"
+CURL="curl --silent --show-error --fail --max-time 10"
+
+MY_LOCK_FILE="$BASE_DIR/retrieve_lock.$$"
+MY_PID=$$
+echo $MY_PID >"$MY_LOCK_FILE"
+LOCK_FILE="$BASE_DIR/retrieve_lock"
+TMP_OUTPUT_FILE="$BASE_DIR/tmp_output"
+
+RUN_TIMEOUT=30 # for how long can the script hold the lock
+WAIT_TIMEOUT=60 # for how long the script is trying to acquire the lock
+START_TIME=$(date +%s)
+
+fail() {
+    echo RUNNING
+    echo "$1"
+    exit 1
+}
+
+acquire_lock() {
+    # try to acquire lock by creating the file (ln should be atomic an fail in case
+    # another process succeeded first). We also check the content of the lock file,
+    # in case our process won when competing over the lock while invalidating
+    # the lock on timeout.
+    ln "$MY_LOCK_FILE" "$LOCK_FILE" 2>/dev/null || [ "$(head -n1 "$LOCK_FILE")" = "$MY_PID" ]
+    return $?
+}
+
+# acquiring the lock before proceeding, to ensure only one instance of the script is running
+while ! acquire_lock; do
+    # we failed to create retrieve_lock - assuming there is already another retrieve script running
+    current_pid=$(head -n1 "$LOCK_FILE")
+    if [ -z "$current_pid" ]; then
+        continue
+    fi
+    # check whether the lock is not too old (compared to $RUN_TIMEOUT) and try to kill
+    # if it is, so that we don't have a stalled processes here
+    lock_lines_count=$(wc -l < "$LOCK_FILE")
+    current_lock_time=$(stat --format "%Y" "$LOCK_FILE")
+    current_time=$(date +%s)
+
+    if [ "$(( current_time - START_TIME ))" -gt "$WAIT_TIMEOUT" ]; then
+        # We were waiting for the lock for too long - just give up
+        fail "Wait time exceeded $WAIT_TIMEOUT"
+    elif [ "$(( current_time - current_lock_time ))" -gt "$RUN_TIMEOUT" ]; then
+        # The previous lock it hold for too long - re-acquiring procedure
+        if [ "$lock_lines_count" -gt 1 ]; then
+           # there were multiple processes waiting for lock without resolution
+           # longer than the $RUN_TIMEOUT - we reset the lock file and let processes
+           # to compete
+           echo "RETRY" > "$LOCK_FILE"
+        fi
+        if [ "$current_pid" != "RETRY" ]; then
+            # try to kill the currently stalled process
+            kill -9 "$current_pid" 2>/dev/null
+        fi
+        # try to add our process as one candidate
+        echo $MY_PID >> "$LOCK_FILE"
+        if [ "$( head -n2 "$LOCK_FILE" | tail -n1 )" = "$MY_PID" ]; then
+            # our process won the competition for the new lock: it is the first pid
+            # after the original one in the lock file - take ownership of the lock
+            # next iteration only this process will get through
+            echo $MY_PID >"$LOCK_FILE"
+        fi
+    else
+        # still waiting for the original owner to finish
+        sleep 1
+    fi
+done
+
+release_lock() {
+    rm "$MY_LOCK_FILE"
+    rm "$LOCK_FILE"
+}
+# ensure the release the lock at exit
+trap "release_lock" EXIT
+
+# make sure we clear previous tmp output file
+if [ -e "$TMP_OUTPUT_FILE" ]; then
+    rm "$TMP_OUTPUT_FILE"
+fi
+
+pid=$(cat "$BASE_DIR/pid")
+[ -f "$BASE_DIR/position" ] || echo 1 > "$BASE_DIR/position"
+position=$(cat "$BASE_DIR/position")
+
+prepare_output() {
+    if [ -e "$BASE_DIR/manual_mode" ] || ([ -n "$pid" ] && pgrep -P "$pid" >/dev/null 2>&1); then
+        echo RUNNING
+    else
+        echo "DONE $(cat "$BASE_DIR/exit_code" 2>/dev/null)"
+    fi
+    [ -f "$BASE_DIR/output" ] || exit 0
+    tail --bytes "+${position}" "$BASE_DIR/output" > "$TMP_OUTPUT_FILE"
+    cat "$TMP_OUTPUT_FILE"
+}
+
+# prepare the callback payload
+payload() {
+    if [ -n "$1" ]; then
+        exit_code="$1"
+    else
+        exit_code=null
+    fi
+
+    if [ -e "$BASE_DIR/manual_mode" ]; then
+        manual_mode=true
+        output=$(prepare_output | base64 -w0)
+    else
+        manual_mode=false
+    fi
+
+    echo "{ \"exit_code\": $exit_code,"\
+         "  \"step_id\": \"$STEP_ID\","\
+         "  \"manual_mode\": $manual_mode,"\
+         "  \"output\": \"$output\" }"
+}
+
+if [ "$1" = "push_update" ]; then
+    if [ -e "$BASE_DIR/exit_code" ]; then
+        exit_code="$(cat "$BASE_DIR/exit_code")"
+        action="done"
+    else
+        exit_code=""
+        action="update"
+    fi
+    $CURL -X POST -d "$(payload $exit_code)" -u "$AUTH" "$URL_PREFIX"/$action 2>>"$BASE_DIR/curl_stderr"
+    success=$?
+else
+    prepare_output
+    success=$?
+fi
+
+if [ "$success" = 0 ] && [ -e "$TMP_OUTPUT_FILE" ]; then
+    # in case the retrieval was successful, move the position of the cursor to be read next time
+    bytes=$(wc --bytes < "$TMP_OUTPUT_FILE")
+    expr "${position}" + "${bytes}" > "$BASE_DIR/position"
+fi