smart_proxy_dynflow 0.2.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 23e6132a792409de0a63851c335a1cbbbbe6c539
-  data.tar.gz: d9eeb7b70c68915fc84b36008e043942cbbcddd9
+SHA256:
+  metadata.gz: be2013a41681b30175177e9bcdb2bdc24186e6b6edccf3dcbdadcb5ca006f49b
+  data.tar.gz: 7c54a27ae6b0055cdd5b1189a8c89a1c644626fbe0386a9e48e3a48da3ebf71e
 SHA512:
-  metadata.gz: 38be1713aabfeea11b229752bd65fe925b30bf05f1bf15a9105bc70066f3181bdedef45a966c377874efae42e8fa01abb2aefcc1c6caa93b1ba00ca6365bccca
-  data.tar.gz: 740e626094df5c0f64580bb94159a03dc1b075409f5b602be0dd8ed1490c6a3efe9e79b439c497056bfc4a6ba3998137a7d7edaf8d54a830126e20fd917f01ae
+  metadata.gz: 6fb8325c06bdac95fbc2b8b3e8b59cb9cf75f43ba71a944dc1cf423a36ea7cfe7a02be1b928f6f2f21728c7de90011cc5c24867dcce58ba90e66bda6396922cc
+  data.tar.gz: 913260f459727549e5559938558f09f398dce1d78e0c7cd42ad596e9da58394d92b2f9ba9dc5dd421371cffa71ae9f4c2038db15d0ce1e6ca4290627910a61ef

data/Gemfile

@@ -10,29 +10,14 @@ group :test do
   gem 'smart_proxy', :git => "https://github.com/theforeman/smart-proxy", :branch => "develop"
   gem 'smart_proxy_dynflow', :path => '.'
 
-  if RUBY_VERSION < '2.1'
-    gem 'public_suffix', '< 3'
-    gem 'rainbow', '< 3'
-    gem 'rubocop', '< 0.51.0'
-  else
-    gem 'public_suffix'
-    gem 'rubocop', '~> 0.52.1'
-  end
-
-  if RUBY_VERSION < '2.2'
-    gem 'rack-test', '< 0.8'
-  else
-    gem 'rack-test'
-  end
+  gem 'public_suffix'
+  gem 'rack-test'
+  gem 'rubocop', '~> 0.52.1'
 end
 
-if RUBY_VERSION < '2.2'
-  gem 'rack', '>= 1.1', '< 2.0.0'
-  gem 'sinatra', '< 2'
-else
-  gem 'rack', '>= 1.1'
-  gem 'sinatra'
-end
+gem 'logging-journald', '~> 2.0', :platforms => [:ruby], :require => false
+gem 'rack', '>= 1.1'
+gem 'sinatra'
 
 # load bundler.d
 Dir["#{File.dirname(__FILE__)}/bundler.d/*.rb"].each do |bundle|

data/lib/smart_proxy_dynflow.rb

@@ -1,5 +1,20 @@
+require 'dynflow'
+
+require 'smart_proxy_dynflow/task_launcher_registry'
+require 'smart_proxy_dynflow/middleware/keep_current_request_id'
+
+require 'smart_proxy_dynflow/log'
+require 'smart_proxy_dynflow/settings'
+require 'smart_proxy_dynflow/ticker'
+require 'smart_proxy_dynflow/core'
+require 'smart_proxy_dynflow/callback'
+
 require 'smart_proxy_dynflow/version'
 require 'smart_proxy_dynflow/plugin'
-require 'smart_proxy_dynflow/callback'
 require 'smart_proxy_dynflow/helpers'
 require 'smart_proxy_dynflow/api'
+
+module Proxy
+  module Dynflow
+  end
+end

data/lib/smart_proxy_dynflow/action.rb

@@ -0,0 +1,12 @@
+module Proxy::Dynflow
+  module Action
+  end
+end
+
+require 'smart_proxy_dynflow/action/batch'
+require 'smart_proxy_dynflow/action/batch_callback'
+require 'smart_proxy_dynflow/action/batch_runner'
+require 'smart_proxy_dynflow/action/output_collector'
+require 'smart_proxy_dynflow/action/shareable'
+require 'smart_proxy_dynflow/action/runner'
+require 'smart_proxy_dynflow/action/single_runner_batch'

data/lib/smart_proxy_dynflow/action/batch.rb

@@ -0,0 +1,21 @@
+module Proxy::Dynflow::Action
+  class Batch < ::Dynflow::Action
+    include Dynflow::Action::WithSubPlans
+    include Dynflow::Action::WithPollingSubPlans
+
+    # { task_id => { :action_class => Klass, :input => input } }
+    def plan(launcher, input_hash)
+      launcher.launch_children(self, input_hash)
+      plan_self
+    end
+
+    def initiate
+      ping suspended_action
+      wait_for_sub_plans sub_plans
+    end
+
+    def rescue_strategy
+      Dynflow::Action::Rescue::Fail
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/batch_callback.rb

@@ -0,0 +1,20 @@
+module Proxy::Dynflow::Action
+  class BatchCallback < ::Dynflow::Action
+    def plan(input_hash, results)
+      plan_self :targets => input_hash, :results => results
+    end
+
+    def run
+      payload = format_payload(input['targets'], input['results'])
+      SmartProxyDynflowCore::Callback::Request.new.callback({ :callbacks => payload }.to_json)
+    end
+
+    private
+
+    def format_payload(input_hash, results)
+      input_hash.map do |task_id, callback|
+        { :callback => callback, :data => results[task_id] }
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/batch_runner.rb

@@ -0,0 +1,14 @@
+require 'smart_proxy_dynflow/action/runner'
+
+module Proxy::Dynflow::Action
+  class BatchRunner < ::Proxy::Dynflow::Action::Runner
+    def plan(launcher, input)
+      plan_self :targets => launcher.runner_input(input), :operation => launcher.operation
+    end
+
+    def initiate_runner
+      launcher = SmartProxyDynflowCore::TaskLauncherRegistry.fetch(input[:operation])
+      launcher.runner_class.new(input[:targets], suspended_action: suspended_action)
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/output_collector.rb

@@ -0,0 +1,8 @@
+module Proxy::Dynflow::Action
+  class OutputCollector < ::Proxy::Dynflow::Runner::Action
+    def init_run
+      output[:result] = []
+      suspend
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/runner.rb

@@ -0,0 +1,76 @@
+require 'smart_proxy_dynflow/action/shareable'
+module Proxy::Dynflow
+  module Action
+    class Runner < Shareable
+      include ::Dynflow::Action::Cancellable
+
+      def run(event = nil)
+        case event
+        when nil
+          init_run
+        when Proxy::Dynflow::Runner::Update
+          process_update(event)
+        when Proxy::Dynflow::Runner::ExternalEvent
+          process_external_event(event)
+        when ::Dynflow::Action::Cancellable::Cancel
+          kill_run
+        else
+          raise "Unexpected event #{event.inspect}"
+        end
+      rescue => e
+        action_logger.error(e)
+        process_update(Proxy::Dynflow::Runner::Update.encode_exception('Proxy error', e))
+      end
+
+      def finalize
+        # To mark the task as a whole as failed
+        error! 'Script execution failed' if on_proxy? && failed_run?
+      end
+
+      def rescue_strategy_for_self
+        ::Dynflow::Action::Rescue::Fail
+      end
+
+      def initiate_runner
+        raise NotImplementedError
+      end
+
+      def init_run
+        output[:result] = []
+        output[:runner_id] = runner_dispatcher.start(suspended_action, initiate_runner)
+        suspend
+      end
+
+      def runner_dispatcher
+        Proxy::Dynflow::Runner::Dispatcher.instance
+      end
+
+      def kill_run
+        runner_dispatcher.kill(output[:runner_id])
+        suspend
+      end
+
+      def finish_run(update)
+        output[:exit_status] = update.exit_status
+      end
+
+      def process_external_event(event)
+        runner_dispatcher.external_event(output[:runner_id], event)
+        suspend
+      end
+
+      def process_update(update)
+        output[:result].concat(update.continuous_output.raw_outputs)
+        if update.exit_status
+          finish_run(update)
+        else
+          suspend
+        end
+      end
+
+      def failed_run?
+        output[:exit_status] != 0
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/shareable.rb

@@ -0,0 +1,25 @@
+module Proxy::Dynflow::Action
+  class Shareable < ::Dynflow::Action
+    def plan(input)
+      input = input.dup
+      callback = input.delete('callback')
+      if callback
+        input[:task_id] = callback['task_id']
+      else
+        input[:task_id] ||= SecureRandom.uuid
+      end
+
+      planned_action = plan_self(input)
+      # code only applicable, when run with SmartProxyDynflowCore in place
+      if on_proxy? && callback
+        plan_action(SmartProxyDynflowCore::Callback::Action, callback, planned_action.output)
+      end
+    end
+
+    private
+
+    def on_proxy?
+      defined?(SmartProxyDynflowCore::Callback)
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/single_runner_batch.rb

@@ -0,0 +1,39 @@
+module Proxy::Dynflow::Action
+  class SingleRunnerBatch < Batch
+    def plan(launcher, input_hash)
+      launcher.launch_children(self, input_hash)
+      sequence do
+        results = plan_self
+        plan_action BatchCallback, launcher.prepare_batch(input_hash), results.output[:results]
+      end
+    end
+
+    def run(event = nil)
+      super unless event == Dynflow::Action::Skip
+    end
+
+    def initiate
+      ping suspended_action
+      wait_for_sub_plans sub_plans
+    end
+
+    def check_for_errors!(optional = true)
+      super unless optional
+    end
+
+    def on_finish
+      output[:results] = sub_plans.map(&:entry_action).reduce({}) do |acc, cur|
+        acc.merge(cur.execution_plan_id => cur.output)
+      end
+    end
+
+    def finalize
+      output.delete(:results)
+      check_for_errors!
+    end
+
+    def rescue_strategy_for_self
+      Dynflow::Action::Rescue::Skip
+    end
+  end
+end

data/lib/smart_proxy_dynflow/api.rb

@@ -3,63 +3,86 @@ require 'proxy/helpers'
 require 'sinatra/authorization'
 
 module Proxy
-  class Dynflow
+  module Dynflow
     class Api < ::Sinatra::Base
       helpers ::Proxy::Helpers
       helpers ::Proxy::Log
       helpers ::Proxy::Dynflow::Helpers
 
+      include ::Sinatra::Authorization::Helpers
+
+      TASK_UPDATE_REGEXP_PATH = %r{/tasks/(\S+)/(update|done)}
+
       before do
-        content_type :json
-        if request.env['HTTP_AUTHORIZATION'] && request.env['PATH_INFO'].end_with?('/done')
-          # Halt running before callbacks if a token is provided and the request is notifying about task being done
-          return
+        if match = request.path_info.match(TASK_UPDATE_REGEXP_PATH)
+          task_id = match[1]
+          action = match[2]
+          authorize_with_token(task_id: task_id, clear: action == 'done')
         else
-          do_authorize_with_ssl_client
-          do_authorize_with_trusted_hosts
+          do_authorize_any
         end
+        content_type :json
       end
 
-      # TODO: move this to foreman-proxy to reduce code duplicities
-      def do_authorize_with_trusted_hosts
-        # When :trusted_hosts is given, we check the client against the list
-        # HTTPS: test the certificate CN
-        # HTTP: test the reverse DNS entry of the remote IP
-        trusted_hosts = Proxy::SETTINGS.trusted_hosts
-        if trusted_hosts
-          if ['yes', 'on', 1].include? request.env['HTTPS'].to_s
-            fqdn = https_cert_cn
-            source = 'SSL_CLIENT_CERT'
-          else
-            fqdn = remote_fqdn(Proxy::SETTINGS.forward_verify)
-            source = 'REMOTE_ADDR'
-          end
-          fqdn = fqdn.downcase
-          logger.debug "verifying remote client #{fqdn} (based on #{source}) against trusted_hosts #{trusted_hosts}"
-
-          unless Proxy::SETTINGS.trusted_hosts.include?(fqdn)
-            log_halt 403, "Untrusted client #{fqdn} attempted " \
-                          "to access #{request.path_info}. Check :trusted_hosts: in settings.yml"
-          end
+      post "/tasks/status" do
+        params = MultiJson.load(request.body.read)
+        ids = params.fetch('task_ids', [])
+        result = world.persistence
+                      .find_execution_plans(:filters => { :uuid => ids }).reduce({}) do |acc, plan|
+          acc.update(plan.id => { 'state' => plan.state, 'result' => plan.result })
         end
+        MultiJson.dump(result)
       end
 
-      def do_authorize_with_ssl_client
-        if %w[yes on 1].include? request.env['HTTPS'].to_s
-          if request.env['SSL_CLIENT_CERT'].to_s.empty?
-            log_halt 403, "No client SSL certificate supplied"
-          end
-        else
-          logger.debug('require_ssl_client_verification: skipping, non-HTTPS request')
-        end
+      post "/tasks/launch/?" do
+        params = MultiJson.load(request.body.read)
+        launcher = launcher_class(params).new(world, callback_host(params, request), params.fetch('options', {}))
+        launcher.launch!(params['input'])
+        launcher.results.to_json
+      end
+
+      post "/tasks/?" do
+        params = MultiJson.load(request.body.read)
+        trigger_task(::Dynflow::Utils.constantize(params['action_name']),
+                     params['action_input'].merge(:callback_host => callback_host(params, request))).to_json
+      end
+
+      post "/tasks/:task_id/cancel" do |task_id|
+        cancel_task(task_id).to_json
       end
 
-      post "/*" do
-        relay_request
+      get "/tasks/:task_id/status" do |task_id|
+        task_status(task_id).to_json
       end
 
-      get "/*" do
-        relay_request
+      get "/tasks/count" do
+        tasks_count(params['state']).to_json
+      end
+
+      # capturing post "/tasks/:task_id/(update|done)"
+      post TASK_UPDATE_REGEXP_PATH do |task_id, _action|
+        data = MultiJson.load(request.body.read)
+        dispatch_external_event(task_id, data)
+      end
+
+      get "/tasks/operations" do
+        TaskLauncherRegistry.operations.to_json
+      end
+
+      private
+
+      def callback_host(params, request)
+        params.fetch('action_input', {})['proxy_url'] ||
+          request.env.values_at('HTTP_X_FORWARDED_FOR', 'HTTP_HOST').compact.first
+      end
+
+      def launcher_class(params)
+        operation = params.fetch('operation')
+        if TaskLauncherRegistry.key?(operation)
+          TaskLauncherRegistry.fetch(operation)
+        else
+          halt 404, MultiJson.dump(:error => "Unknown operation '#{operation}' requested.")
+        end
       end
     end
   end

data/lib/smart_proxy_dynflow/callback.rb

@@ -1,33 +1,77 @@
-require 'proxy/request'
-
-module Proxy
-  class Dynflow
-    module Callback
-      class Core < Proxy::HttpRequest::ForemanRequest
-        def uri
-          @uri ||= URI.parse Proxy::Dynflow::Plugin.settings.core_url
+require 'rest-client'
+
+module Proxy::Dynflow
+  module Callback
+    class Request
+      class << self
+        def send_to_foreman_tasks(callback_info, data)
+          self.new.callback(prepare_payload(callback_info, data))
+        end
+
+        def ssl_options
+          return @ssl_options if defined? @ssl_options
+          @ssl_options = {}
+          settings = Proxy::SETTINGS
+          return @ssl_options unless URI.parse(settings.foreman_url).scheme == 'https'
+
+          @ssl_options[:verify_ssl] = OpenSSL::SSL::VERIFY_PEER
+
+          private_key_file = settings.foreman_ssl_key || settings.ssl_private_key
+          if private_key_file
+            private_key = File.read(private_key_file)
+            @ssl_options[:ssl_client_key] = OpenSSL::PKey::RSA.new(private_key)
+          end
+          certificate_file = settings.foreman_ssl_cert || settings.ssl_certificate
+          if certificate_file
+            certificate = File.read(certificate_file)
+            @ssl_options[:ssl_client_cert] = OpenSSL::X509::Certificate.new(certificate)
+          end
+          ca_file = settings.foreman_ssl_ca || settings.ssl_ca_file
+          @ssl_options[:ssl_ca_file] = ca_file if ca_file
+          @ssl_options
         end
+        # rubocop:enable Metrics/PerceivedComplexity
 
-        def relay(request, from, to)
-          path = request.path.gsub(from, to)
-          message = "Proxy request from #{request.host_with_port}#{request.path} to #{uri}#{path}"
-          Proxy::LogBuffer::Decorator.instance.debug message
-          req = case request.env['REQUEST_METHOD']
-                when 'GET'
-                  request_factory.create_get path, request.env['rack.request.query_hash']
-                when 'POST'
-                  request_factory.create_post path, request.body.read
-                end
-          req['X-Forwarded-For'] = request.env['HTTP_HOST']
-          req['AUTHORIZATION'] = request.env['HTTP_AUTHORIZATION']
-          response = send_request req
-          Proxy::LogBuffer::Decorator.instance.debug "Proxy request status #{response.code} - #{response}"
-          response
+        private
+
+        def prepare_payload(callback, data)
+          { :callback => callback, :data => data }.to_json
         end
+      end
 
-        def self.relay(request, from, to)
-          self.new.relay request, from, to
+      def callback(payload)
+        response = callback_resource.post(payload, :content_type => :json)
+        if response.code.to_s != "200"
+          raise "Failed performing callback to Foreman server: #{response.code} #{response.body}"
         end
+        response
+      end
+
+      private
+
+      def callback_resource
+        @resource ||= RestClient::Resource.new(Proxy::SETTINGS.foreman_url + '/foreman_tasks/api/tasks/callback',
+                                               self.class.ssl_options)
+      end
+    end
+
+    class Action < ::Dynflow::Action
+      def plan(callback, data)
+        plan_self(:callback => callback, :data => data)
+      end
+
+      def run
+        Callback::Request.send_to_foreman_tasks(input[:callback], input[:data])
+      end
+    end
+
+    module PlanHelper
+      def plan_with_callback(input)
+        input = input.dup
+        callback = input.delete('callback')
+
+        planned_action = plan_self(input)
+        plan_action(Callback::Action, callback, planned_action.output) if callback
       end
     end
   end