smart_proxy_dynflow 0.2.4 → 0.5.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +6 -25
  3. data/{bundler.plugins.d → bundler.d}/dynflow.rb +0 -0
  4. data/lib/smart_proxy_dynflow.rb +16 -1
  5. data/lib/smart_proxy_dynflow/action.rb +12 -0
  6. data/lib/smart_proxy_dynflow/action/batch.rb +21 -0
  7. data/lib/smart_proxy_dynflow/action/batch_callback.rb +20 -0
  8. data/lib/smart_proxy_dynflow/action/batch_runner.rb +14 -0
  9. data/lib/smart_proxy_dynflow/action/output_collector.rb +8 -0
  10. data/lib/smart_proxy_dynflow/action/runner.rb +76 -0
  11. data/lib/smart_proxy_dynflow/action/shareable.rb +25 -0
  12. data/lib/smart_proxy_dynflow/action/single_runner_batch.rb +39 -0
  13. data/lib/smart_proxy_dynflow/api.rb +63 -40
  14. data/lib/smart_proxy_dynflow/callback.rb +69 -25
  15. data/lib/smart_proxy_dynflow/continuous_output.rb +50 -0
  16. data/lib/smart_proxy_dynflow/core.rb +121 -0
  17. data/lib/smart_proxy_dynflow/helpers.rb +52 -6
  18. data/lib/smart_proxy_dynflow/http_config.ru +6 -16
  19. data/lib/smart_proxy_dynflow/log.rb +52 -0
  20. data/lib/smart_proxy_dynflow/middleware/keep_current_request_id.rb +59 -0
  21. data/lib/smart_proxy_dynflow/otp_manager.rb +36 -0
  22. data/lib/smart_proxy_dynflow/plugin.rb +9 -14
  23. data/lib/smart_proxy_dynflow/proxy_adapter.rb +1 -1
  24. data/lib/smart_proxy_dynflow/runner.rb +10 -0
  25. data/lib/smart_proxy_dynflow/runner/base.rb +98 -0
  26. data/lib/smart_proxy_dynflow/runner/command.rb +40 -0
  27. data/lib/smart_proxy_dynflow/runner/command_runner.rb +11 -0
  28. data/lib/smart_proxy_dynflow/runner/dispatcher.rb +191 -0
  29. data/lib/smart_proxy_dynflow/runner/parent.rb +57 -0
  30. data/lib/smart_proxy_dynflow/runner/update.rb +28 -0
  31. data/lib/smart_proxy_dynflow/settings.rb +9 -0
  32. data/lib/smart_proxy_dynflow/settings_loader.rb +53 -0
  33. data/lib/smart_proxy_dynflow/task_launcher.rb +9 -0
  34. data/lib/smart_proxy_dynflow/task_launcher/abstract.rb +44 -0
  35. data/lib/smart_proxy_dynflow/task_launcher/batch.rb +37 -0
  36. data/lib/smart_proxy_dynflow/task_launcher/group.rb +48 -0
  37. data/lib/smart_proxy_dynflow/task_launcher/single.rb +17 -0
  38. data/lib/smart_proxy_dynflow/task_launcher_registry.rb +31 -0
  39. data/lib/smart_proxy_dynflow/testing.rb +24 -0
  40. data/lib/smart_proxy_dynflow/ticker.rb +47 -0
  41. data/lib/smart_proxy_dynflow/version.rb +2 -2
  42. data/settings.d/dynflow.yml.example +6 -5
  43. metadata +83 -11
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8a7cdbcdb7278d4915da1d29d0f23cb8f31a1f5be7d76dc2def3cf0a12554024
4
- data.tar.gz: ea8e11e0af60f83760e05994a6270362ea0ecbd3c0d2bd4285b72c8d9acc8872
3
+ metadata.gz: ad7f3e5a2de559c96dab48e0da3b064a2339a902af0bb23935f2c5eb90a4f61a
4
+ data.tar.gz: '08b7073408bef24c3b0d6d358e1b5b23da954056c9fdb30740c376b05c7dc088'
5
5
  SHA512:
6
- metadata.gz: b61851ddd7b0a4d22449dae1f940f14edb1af423ac091ed9ba4751ab34ac88db4f321097c24d61afaf0a7be4feb194fa642162d6b7fe7dc36747edecb3854504
7
- data.tar.gz: c44d24f2b06a88f4799a1092bcdf414b909fb6c7a5b8ca6dcf880485d95426aae90478d4a3b97c527fe7da49580b4a8d7e684e7134dc1fd346695da04677dfbe
6
+ metadata.gz: 8bd8529c02290846b47bd745920293fda07efe134b410455dee796f8e99a94d9a8a56478d313824a0659dbaee6e208f6d0dbea6392000b1f81a30a3fd10a4075
7
+ data.tar.gz: 231132ad10b549c2e2c5b7262f261726ec4b4d4780fa73390b18a17d1715243e50709a4416a9590edd32375ae23c5a13e41c454e8fe9524cc093714e58ea9a12
data/Gemfile CHANGED
@@ -10,33 +10,14 @@ group :test do
10
10
  gem 'smart_proxy', :git => "https://github.com/theforeman/smart-proxy", :branch => "develop"
11
11
  gem 'smart_proxy_dynflow', :path => '.'
12
12
 
13
- if RUBY_VERSION < '2.1'
14
- gem 'public_suffix', '< 3'
15
- gem 'rainbow', '< 3'
16
- else
17
- gem 'public_suffix'
18
- gem 'rubocop', '~> 0.52.1'
19
- end
20
-
21
- if RUBY_VERSION < '2.2'
22
- gem 'rack-test', '< 0.8'
23
- else
24
- gem 'rack-test'
25
- end
26
- end
27
-
28
- if RUBY_VERSION < '2.2'
29
- gem 'rack', '>= 1.1', '< 2.0.0'
30
- gem 'sinatra', '< 2'
31
- else
32
- gem 'rack', '>= 1.1'
33
- gem 'sinatra'
13
+ gem 'public_suffix'
14
+ gem 'rack-test'
15
+ gem 'rubocop', '~> 0.52.1'
34
16
  end
35
17
 
36
- # load bundler.d
37
- Dir["#{File.dirname(__FILE__)}/bundler.d/*.rb"].each do |bundle|
38
- self.instance_eval(Bundler.read_file(bundle))
39
- end
18
+ gem 'logging-journald', '~> 2.0', :platforms => [:ruby], :require => false
19
+ gem 'rack', '>= 1.1'
20
+ gem 'sinatra'
40
21
 
41
22
  # load local gemfile
42
23
  local_gemfile = File.join(File.dirname(__FILE__), 'Gemfile.local.rb')
File without changes
@@ -1,5 +1,20 @@
1
+ require 'dynflow'
2
+
3
+ require 'smart_proxy_dynflow/task_launcher_registry'
4
+ require 'smart_proxy_dynflow/middleware/keep_current_request_id'
5
+
6
+ require 'smart_proxy_dynflow/log'
7
+ require 'smart_proxy_dynflow/settings'
8
+ require 'smart_proxy_dynflow/ticker'
9
+ require 'smart_proxy_dynflow/core'
10
+ require 'smart_proxy_dynflow/callback'
11
+
1
12
  require 'smart_proxy_dynflow/version'
2
13
  require 'smart_proxy_dynflow/plugin'
3
- require 'smart_proxy_dynflow/callback'
4
14
  require 'smart_proxy_dynflow/helpers'
5
15
  require 'smart_proxy_dynflow/api'
16
+
17
+ module Proxy
18
+ module Dynflow
19
+ end
20
+ end
@@ -0,0 +1,12 @@
1
+ module Proxy::Dynflow
2
+ module Action
3
+ end
4
+ end
5
+
6
+ require 'smart_proxy_dynflow/action/batch'
7
+ require 'smart_proxy_dynflow/action/batch_callback'
8
+ require 'smart_proxy_dynflow/action/batch_runner'
9
+ require 'smart_proxy_dynflow/action/output_collector'
10
+ require 'smart_proxy_dynflow/action/shareable'
11
+ require 'smart_proxy_dynflow/action/runner'
12
+ require 'smart_proxy_dynflow/action/single_runner_batch'
@@ -0,0 +1,21 @@
1
+ module Proxy::Dynflow::Action
2
+ class Batch < ::Dynflow::Action
3
+ include Dynflow::Action::WithSubPlans
4
+ include Dynflow::Action::WithPollingSubPlans
5
+
6
+ # { task_id => { :action_class => Klass, :input => input } }
7
+ def plan(launcher, input_hash)
8
+ launcher.launch_children(self, input_hash)
9
+ plan_self
10
+ end
11
+
12
+ def initiate
13
+ ping suspended_action
14
+ wait_for_sub_plans sub_plans
15
+ end
16
+
17
+ def rescue_strategy
18
+ Dynflow::Action::Rescue::Fail
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,20 @@
1
+ module Proxy::Dynflow::Action
2
+ class BatchCallback < ::Dynflow::Action
3
+ def plan(input_hash, results)
4
+ plan_self :targets => input_hash, :results => results
5
+ end
6
+
7
+ def run
8
+ payload = format_payload(input['targets'], input['results'])
9
+ Proxy::Dynflow::Callback::Request.new.callback({ :callbacks => payload }.to_json)
10
+ end
11
+
12
+ private
13
+
14
+ def format_payload(input_hash, results)
15
+ input_hash.map do |task_id, callback|
16
+ { :callback => callback, :data => results[task_id] }
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,14 @@
1
+ require 'smart_proxy_dynflow/action/runner'
2
+
3
+ module Proxy::Dynflow::Action
4
+ class BatchRunner < ::Proxy::Dynflow::Action::Runner
5
+ def plan(launcher, input)
6
+ plan_self :targets => launcher.runner_input(input), :operation => launcher.operation
7
+ end
8
+
9
+ def initiate_runner
10
+ launcher = Proxy::Dynflow::TaskLauncherRegistry.fetch(input[:operation])
11
+ launcher.runner_class.new(input[:targets], suspended_action: suspended_action)
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,8 @@
1
+ module Proxy::Dynflow::Action
2
+ class OutputCollector < ::Proxy::Dynflow::Action::Runner
3
+ def init_run
4
+ output[:result] = []
5
+ suspend
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,76 @@
1
+ require 'smart_proxy_dynflow/action/shareable'
2
+ module Proxy::Dynflow
3
+ module Action
4
+ class Runner < Shareable
5
+ include ::Dynflow::Action::Cancellable
6
+
7
+ def run(event = nil)
8
+ case event
9
+ when nil
10
+ init_run
11
+ when Proxy::Dynflow::Runner::Update
12
+ process_update(event)
13
+ when Proxy::Dynflow::Runner::ExternalEvent
14
+ process_external_event(event)
15
+ when ::Dynflow::Action::Cancellable::Cancel
16
+ kill_run
17
+ else
18
+ raise "Unexpected event #{event.inspect}"
19
+ end
20
+ rescue => e
21
+ action_logger.error(e)
22
+ process_update(Proxy::Dynflow::Runner::Update.encode_exception('Proxy error', e))
23
+ end
24
+
25
+ def finalize
26
+ # To mark the task as a whole as failed
27
+ error! 'Script execution failed' if on_proxy? && failed_run?
28
+ end
29
+
30
+ def rescue_strategy_for_self
31
+ ::Dynflow::Action::Rescue::Fail
32
+ end
33
+
34
+ def initiate_runner
35
+ raise NotImplementedError
36
+ end
37
+
38
+ def init_run
39
+ output[:result] = []
40
+ output[:runner_id] = runner_dispatcher.start(suspended_action, initiate_runner)
41
+ suspend
42
+ end
43
+
44
+ def runner_dispatcher
45
+ Proxy::Dynflow::Runner::Dispatcher.instance
46
+ end
47
+
48
+ def kill_run
49
+ runner_dispatcher.kill(output[:runner_id])
50
+ suspend
51
+ end
52
+
53
+ def finish_run(update)
54
+ output[:exit_status] = update.exit_status
55
+ end
56
+
57
+ def process_external_event(event)
58
+ runner_dispatcher.external_event(output[:runner_id], event)
59
+ suspend
60
+ end
61
+
62
+ def process_update(update)
63
+ output[:result].concat(update.continuous_output.raw_outputs)
64
+ if update.exit_status
65
+ finish_run(update)
66
+ else
67
+ suspend
68
+ end
69
+ end
70
+
71
+ def failed_run?
72
+ output[:exit_status] != 0
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,25 @@
1
+ module Proxy::Dynflow::Action
2
+ class Shareable < ::Dynflow::Action
3
+ def plan(input)
4
+ input = input.dup
5
+ callback = input.delete('callback')
6
+ if callback
7
+ input[:task_id] = callback['task_id']
8
+ else
9
+ input[:task_id] ||= SecureRandom.uuid
10
+ end
11
+
12
+ planned_action = plan_self(input)
13
+ # code only applicable, when run with SmartProxyDynflowCore in place
14
+ if on_proxy? && callback
15
+ plan_action(Proxy::Dynflow::Callback::Action, callback, planned_action.output)
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ def on_proxy?
22
+ true
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,39 @@
1
+ module Proxy::Dynflow::Action
2
+ class SingleRunnerBatch < Batch
3
+ def plan(launcher, input_hash)
4
+ launcher.launch_children(self, input_hash)
5
+ sequence do
6
+ results = plan_self
7
+ plan_action BatchCallback, launcher.prepare_batch(input_hash), results.output[:results]
8
+ end
9
+ end
10
+
11
+ def run(event = nil)
12
+ super unless event == Dynflow::Action::Skip
13
+ end
14
+
15
+ def initiate
16
+ ping suspended_action
17
+ wait_for_sub_plans sub_plans
18
+ end
19
+
20
+ def check_for_errors!(optional = true)
21
+ super unless optional
22
+ end
23
+
24
+ def on_finish
25
+ output[:results] = sub_plans.map(&:entry_action).reduce({}) do |acc, cur|
26
+ acc.merge(cur.execution_plan_id => cur.output)
27
+ end
28
+ end
29
+
30
+ def finalize
31
+ output.delete(:results)
32
+ check_for_errors!
33
+ end
34
+
35
+ def rescue_strategy_for_self
36
+ Dynflow::Action::Rescue::Skip
37
+ end
38
+ end
39
+ end
@@ -3,63 +3,86 @@ require 'proxy/helpers'
3
3
  require 'sinatra/authorization'
4
4
 
5
5
  module Proxy
6
- class Dynflow
6
+ module Dynflow
7
7
  class Api < ::Sinatra::Base
8
8
  helpers ::Proxy::Helpers
9
9
  helpers ::Proxy::Log
10
10
  helpers ::Proxy::Dynflow::Helpers
11
11
 
12
+ include ::Sinatra::Authorization::Helpers
13
+
14
+ TASK_UPDATE_REGEXP_PATH = %r{/tasks/(\S+)/(update|done)}
15
+
12
16
  before do
13
- content_type :json
14
- if request.env['HTTP_AUTHORIZATION'] && request.path_info =~ %r{/tasks/.*/(update|done)}
15
- # Halt running before callbacks if a token is provided and the request is notifying about task being done
16
- return
17
+ if match = request.path_info.match(TASK_UPDATE_REGEXP_PATH)
18
+ task_id = match[1]
19
+ action = match[2]
20
+ authorize_with_token(task_id: task_id, clear: action == 'done')
17
21
  else
18
- do_authorize_with_ssl_client
19
- do_authorize_with_trusted_hosts
22
+ do_authorize_any
20
23
  end
24
+ content_type :json
21
25
  end
22
26
 
23
- # TODO: move this to foreman-proxy to reduce code duplicities
24
- def do_authorize_with_trusted_hosts
25
- # When :trusted_hosts is given, we check the client against the list
26
- # HTTPS: test the certificate CN
27
- # HTTP: test the reverse DNS entry of the remote IP
28
- trusted_hosts = Proxy::SETTINGS.trusted_hosts
29
- if trusted_hosts
30
- if ['yes', 'on', 1].include? request.env['HTTPS'].to_s
31
- fqdn = https_cert_cn
32
- source = 'SSL_CLIENT_CERT'
33
- else
34
- fqdn = remote_fqdn(Proxy::SETTINGS.forward_verify)
35
- source = 'REMOTE_ADDR'
36
- end
37
- fqdn = fqdn.downcase
38
- logger.debug "verifying remote client #{fqdn} (based on #{source}) against trusted_hosts #{trusted_hosts}"
39
-
40
- unless Proxy::SETTINGS.trusted_hosts.include?(fqdn)
41
- log_halt 403, "Untrusted client #{fqdn} attempted " \
42
- "to access #{request.path_info}. Check :trusted_hosts: in settings.yml"
43
- end
27
+ post "/tasks/status" do
28
+ params = MultiJson.load(request.body.read)
29
+ ids = params.fetch('task_ids', [])
30
+ result = world.persistence
31
+ .find_execution_plans(:filters => { :uuid => ids }).reduce({}) do |acc, plan|
32
+ acc.update(plan.id => { 'state' => plan.state, 'result' => plan.result })
44
33
  end
34
+ MultiJson.dump(result)
45
35
  end
46
36
 
47
- def do_authorize_with_ssl_client
48
- if %w[yes on 1].include? request.env['HTTPS'].to_s
49
- if request.env['SSL_CLIENT_CERT'].to_s.empty?
50
- log_halt 403, "No client SSL certificate supplied"
51
- end
52
- else
53
- logger.debug('require_ssl_client_verification: skipping, non-HTTPS request')
54
- end
37
+ post "/tasks/launch/?" do
38
+ params = MultiJson.load(request.body.read)
39
+ launcher = launcher_class(params).new(world, callback_host(params, request), params.fetch('options', {}))
40
+ launcher.launch!(params['input'])
41
+ launcher.results.to_json
42
+ end
43
+
44
+ post "/tasks/?" do
45
+ params = MultiJson.load(request.body.read)
46
+ trigger_task(::Dynflow::Utils.constantize(params['action_name']),
47
+ params['action_input'].merge(:callback_host => callback_host(params, request))).to_json
48
+ end
49
+
50
+ post "/tasks/:task_id/cancel" do |task_id|
51
+ cancel_task(task_id).to_json
55
52
  end
56
53
 
57
- post "/*" do
58
- relay_request
54
+ get "/tasks/:task_id/status" do |task_id|
55
+ task_status(task_id).to_json
59
56
  end
60
57
 
61
- get "/*" do
62
- relay_request
58
+ get "/tasks/count" do
59
+ tasks_count(params['state']).to_json
60
+ end
61
+
62
+ # capturing post "/tasks/:task_id/(update|done)"
63
+ post TASK_UPDATE_REGEXP_PATH do |task_id, _action|
64
+ data = MultiJson.load(request.body.read)
65
+ dispatch_external_event(task_id, data)
66
+ end
67
+
68
+ get "/tasks/operations" do
69
+ TaskLauncherRegistry.operations.to_json
70
+ end
71
+
72
+ private
73
+
74
+ def callback_host(params, request)
75
+ params.fetch('action_input', {})['proxy_url'] ||
76
+ request.env.values_at('HTTP_X_FORWARDED_FOR', 'HTTP_HOST').compact.first
77
+ end
78
+
79
+ def launcher_class(params)
80
+ operation = params.fetch('operation')
81
+ if TaskLauncherRegistry.key?(operation)
82
+ TaskLauncherRegistry.fetch(operation)
83
+ else
84
+ halt 404, MultiJson.dump(:error => "Unknown operation '#{operation}' requested.")
85
+ end
63
86
  end
64
87
  end
65
88
  end
@@ -1,33 +1,77 @@
1
- require 'proxy/request'
2
-
3
- module Proxy
4
- class Dynflow
5
- module Callback
6
- class Core < Proxy::HttpRequest::ForemanRequest
7
- def uri
8
- @uri ||= URI.parse Proxy::Dynflow::Plugin.settings.core_url
1
+ require 'rest-client'
2
+
3
+ module Proxy::Dynflow
4
+ module Callback
5
+ class Request
6
+ class << self
7
+ def send_to_foreman_tasks(callback_info, data)
8
+ self.new.callback(prepare_payload(callback_info, data))
9
+ end
10
+
11
+ def ssl_options
12
+ return @ssl_options if defined? @ssl_options
13
+ @ssl_options = {}
14
+ settings = Proxy::SETTINGS
15
+ return @ssl_options unless URI.parse(settings.foreman_url).scheme == 'https'
16
+
17
+ @ssl_options[:verify_ssl] = OpenSSL::SSL::VERIFY_PEER
18
+
19
+ private_key_file = settings.foreman_ssl_key || settings.ssl_private_key
20
+ if private_key_file
21
+ private_key = File.read(private_key_file)
22
+ @ssl_options[:ssl_client_key] = OpenSSL::PKey::RSA.new(private_key)
23
+ end
24
+ certificate_file = settings.foreman_ssl_cert || settings.ssl_certificate
25
+ if certificate_file
26
+ certificate = File.read(certificate_file)
27
+ @ssl_options[:ssl_client_cert] = OpenSSL::X509::Certificate.new(certificate)
28
+ end
29
+ ca_file = settings.foreman_ssl_ca || settings.ssl_ca_file
30
+ @ssl_options[:ssl_ca_file] = ca_file if ca_file
31
+ @ssl_options
9
32
  end
33
+ # rubocop:enable Metrics/PerceivedComplexity
10
34
 
11
- def relay(request, from, to)
12
- path = request.path.gsub(from, to)
13
- message = "Proxy request from #{request.host_with_port}#{request.path} to #{uri}#{path}"
14
- Proxy::LogBuffer::Decorator.instance.debug message
15
- req = case request.env['REQUEST_METHOD']
16
- when 'GET'
17
- request_factory.create_get path, request.env['rack.request.query_hash']
18
- when 'POST'
19
- request_factory.create_post path, request.body.read
20
- end
21
- req['X-Forwarded-For'] = request.env['HTTP_HOST']
22
- req['AUTHORIZATION'] = request.env['HTTP_AUTHORIZATION']
23
- response = send_request req
24
- Proxy::LogBuffer::Decorator.instance.debug "Proxy request status #{response.code} - #{response}"
25
- response
35
+ private
36
+
37
+ def prepare_payload(callback, data)
38
+ { :callback => callback, :data => data }.to_json
26
39
  end
40
+ end
27
41
 
28
- def self.relay(request, from, to)
29
- self.new.relay request, from, to
42
+ def callback(payload)
43
+ response = callback_resource.post(payload, :content_type => :json)
44
+ if response.code.to_s != "200"
45
+ raise "Failed performing callback to Foreman server: #{response.code} #{response.body}"
30
46
  end
47
+ response
48
+ end
49
+
50
+ private
51
+
52
+ def callback_resource
53
+ @resource ||= RestClient::Resource.new(Proxy::SETTINGS.foreman_url + '/foreman_tasks/api/tasks/callback',
54
+ self.class.ssl_options)
55
+ end
56
+ end
57
+
58
+ class Action < ::Dynflow::Action
59
+ def plan(callback, data)
60
+ plan_self(:callback => callback, :data => data)
61
+ end
62
+
63
+ def run
64
+ Callback::Request.send_to_foreman_tasks(input[:callback], input[:data])
65
+ end
66
+ end
67
+
68
+ module PlanHelper
69
+ def plan_with_callback(input)
70
+ input = input.dup
71
+ callback = input.delete('callback')
72
+
73
+ planned_action = plan_self(input)
74
+ plan_action(Callback::Action, callback, planned_action.output) if callback
31
75
  end
32
76
  end
33
77
  end