smart_proxy_dynflow 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3c381a5358391ea92bacf8c59c3a279b49a50a49369bd962c0ff94e7c4d459e
-  data.tar.gz: 3d83db86f0ce2a76b5d1e1f4ca5afd57eb82a5b327ba1ad2f561c77217e4ae0f
+  metadata.gz: b904806b1f4b0084f948d91f800d255ecf7bca92b1ed6d78fee49761a29affc9
+  data.tar.gz: 1eb7b63daaac6fbc3bdcb24d2938a93d2bd5728bcdb934f8384f41709a072cbe
 SHA512:
-  metadata.gz: da9ce875255e93fcc491fc438eb6cbe3fb0b803b6524539a71575f47b29e0a65bcffb1378a6bacc3eee85e4c5194a3d0e3a9f77faff3fe5a82211f4242f671ea
-  data.tar.gz: 55b4f92fdf94ca2b0cde9282f02be12d167674bd358729bd26a766d320a45de0dcc2f8535d94dc31b164963691e076b6b3740c60f385b4f9ca18b2d06c8abf82
+  metadata.gz: 887d7788a14a93d693c3ec2a7ddc110fc5fd6fe62619da946f6e0d68f2ec8542b69670769d0cf9e7eafccee3af04b9bf89fef3565346d69bfa1eb4f71492b119
+  data.tar.gz: e1c2f822a02805f2136ba363d13028699dea85e05ed41ba988853c5aadf3358d5f59f65006e5404321c087aea7a96699582ba8dd1fbc8f1a67a5e09943509d2d

data/Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gemspec :name => 'smart_proxy_dynflow_core'
+gemspec :name => 'smart_proxy_dynflow'
 
 group :development do
   gem 'pry'
@@ -8,10 +8,12 @@ end
 
 group :test do
   gem 'smart_proxy', :git => "https://github.com/theforeman/smart-proxy", :branch => "develop"
-  gem 'smart_proxy_dynflow', :path => '.'
 
+  gem 'minitest'
+  gem 'mocha'
   gem 'public_suffix'
   gem 'rack-test'
+  gem 'rake'
   gem 'rubocop', '~> 0.52.1'
 end
 
@@ -19,11 +21,6 @@ gem 'logging-journald', '~> 2.0', :platforms => [:ruby], :require => false
 gem 'rack', '>= 1.1'
 gem 'sinatra'
 
-# load bundler.d
-Dir["#{File.dirname(__FILE__)}/bundler.d/*.rb"].each do |bundle|
-  self.instance_eval(Bundler.read_file(bundle))
-end
-
 # load local gemfile
 local_gemfile = File.join(File.dirname(__FILE__), 'Gemfile.local.rb')
 self.instance_eval(Bundler.read_file(local_gemfile)) if File.exist?(local_gemfile)

data/lib/smart_proxy_dynflow/action/batch.rb

@@ -0,0 +1,21 @@
+module Proxy::Dynflow::Action
+  class Batch < ::Dynflow::Action
+    include Dynflow::Action::WithSubPlans
+    include Dynflow::Action::WithPollingSubPlans
+
+    # { task_id => { :action_class => Klass, :input => input } }
+    def plan(launcher, input_hash)
+      launcher.launch_children(self, input_hash)
+      plan_self
+    end
+
+    def initiate
+      ping suspended_action
+      wait_for_sub_plans sub_plans
+    end
+
+    def rescue_strategy
+      Dynflow::Action::Rescue::Fail
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/batch_callback.rb

@@ -0,0 +1,20 @@
+module Proxy::Dynflow::Action
+  class BatchCallback < ::Dynflow::Action
+    def plan(input_hash, results)
+      plan_self :targets => input_hash, :results => results
+    end
+
+    def run
+      payload = format_payload(input['targets'], input['results'])
+      Proxy::Dynflow::Callback::Request.new.callback({ :callbacks => payload }.to_json)
+    end
+
+    private
+
+    def format_payload(input_hash, results)
+      input_hash.map do |task_id, callback|
+        { :callback => callback, :data => results[task_id] }
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/batch_runner.rb

@@ -0,0 +1,14 @@
+require 'smart_proxy_dynflow/action/runner'
+
+module Proxy::Dynflow::Action
+  class BatchRunner < ::Proxy::Dynflow::Action::Runner
+    def plan(launcher, input)
+      plan_self :targets => launcher.runner_input(input), :operation => launcher.operation
+    end
+
+    def initiate_runner
+      launcher = Proxy::Dynflow::TaskLauncherRegistry.fetch(input[:operation])
+      launcher.runner_class.new(input[:targets], suspended_action: suspended_action)
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/output_collector.rb

@@ -0,0 +1,8 @@
+module Proxy::Dynflow::Action
+  class OutputCollector < ::Proxy::Dynflow::Action::Runner
+    def init_run
+      output[:result] = []
+      suspend
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/runner.rb

@@ -0,0 +1,81 @@
+require 'smart_proxy_dynflow/action/shareable'
+module Proxy::Dynflow
+  module Action
+    class Runner < Shareable
+      include ::Dynflow::Action::Cancellable
+
+      def run(event = nil)
+        case event
+        when nil
+          init_run
+        when Proxy::Dynflow::Runner::Update
+          process_update(event)
+        when Proxy::Dynflow::Runner::ExternalEvent
+          process_external_event(event)
+        when ::Dynflow::Action::Cancellable::Cancel
+          kill_run
+        else
+          raise "Unexpected event #{event.inspect}"
+        end
+      rescue => e
+        action_logger.error(e)
+        process_update(Proxy::Dynflow::Runner::Update.encode_exception('Proxy error', e))
+      end
+
+      def finalize
+        # To mark the task as a whole as failed
+        error! 'Script execution failed' if on_proxy? && failed_run?
+      end
+
+      def rescue_strategy_for_self
+        ::Dynflow::Action::Rescue::Fail
+      end
+
+      def initiate_runner
+        raise NotImplementedError
+      end
+
+      def init_run
+        output[:result] = []
+        output[:runner_id] = runner_dispatcher.start(suspended_action, initiate_runner)
+        suspend
+      end
+
+      def runner_dispatcher
+        Proxy::Dynflow::Runner::Dispatcher.instance
+      end
+
+      def kill_run
+        runner_dispatcher.kill(output[:runner_id])
+        suspend
+      end
+
+      def finish_run(update)
+        output[:exit_status] = update.exit_status
+        output[:result] = output_result
+      end
+
+      def process_external_event(event)
+        runner_dispatcher.external_event(output[:runner_id], event)
+        suspend
+      end
+
+      def process_update(update)
+        output_chunk(update.continuous_output.raw_outputs) unless update.continuous_output.raw_outputs.empty?
+        if update.exit_status
+          finish_run(update)
+        else
+          suspend
+        end
+      end
+
+      def failed_run?
+        output[:exit_status] != 0
+      end
+
+      def output_result
+        stored_output_chunks.map { |c| c[:chunk] }.reduce(&:concat)
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/shareable.rb

@@ -0,0 +1,25 @@
+module Proxy::Dynflow::Action
+  class Shareable < ::Dynflow::Action
+    def plan(input)
+      input = input.dup
+      callback = input.delete('callback')
+      if callback
+        input[:task_id] = callback['task_id']
+      else
+        input[:task_id] ||= SecureRandom.uuid
+      end
+
+      planned_action = plan_self(input)
+      # code only applicable, when run with SmartProxyDynflowCore in place
+      if on_proxy? && callback
+        plan_action(Proxy::Dynflow::Callback::Action, callback, planned_action.output)
+      end
+    end
+
+    private
+
+    def on_proxy?
+      true
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action/single_runner_batch.rb

@@ -0,0 +1,39 @@
+module Proxy::Dynflow::Action
+  class SingleRunnerBatch < Batch
+    def plan(launcher, input_hash)
+      launcher.launch_children(self, input_hash)
+      sequence do
+        results = plan_self
+        plan_action BatchCallback, launcher.prepare_batch(input_hash), results.output[:results]
+      end
+    end
+
+    def run(event = nil)
+      super unless event == Dynflow::Action::Skip
+    end
+
+    def initiate
+      ping suspended_action
+      wait_for_sub_plans sub_plans
+    end
+
+    def check_for_errors!(optional = true)
+      super unless optional
+    end
+
+    def on_finish
+      output[:results] = sub_plans.map(&:entry_action).reduce({}) do |acc, cur|
+        acc.merge(cur.execution_plan_id => cur.output)
+      end
+    end
+
+    def finalize
+      output.delete(:results)
+      check_for_errors!
+    end
+
+    def rescue_strategy_for_self
+      Dynflow::Action::Rescue::Skip
+    end
+  end
+end

data/lib/smart_proxy_dynflow/action.rb

@@ -0,0 +1,12 @@
+module Proxy::Dynflow
+  module Action
+  end
+end
+
+require 'smart_proxy_dynflow/action/batch'
+require 'smart_proxy_dynflow/action/batch_callback'
+require 'smart_proxy_dynflow/action/batch_runner'
+require 'smart_proxy_dynflow/action/output_collector'
+require 'smart_proxy_dynflow/action/shareable'
+require 'smart_proxy_dynflow/action/runner'
+require 'smart_proxy_dynflow/action/single_runner_batch'

data/lib/smart_proxy_dynflow/api.rb

@@ -3,7 +3,7 @@ require 'proxy/helpers'
 require 'sinatra/authorization'
 
 module Proxy
-  class Dynflow
+  module Dynflow
     class Api < ::Sinatra::Base
       helpers ::Proxy::Helpers
       helpers ::Proxy::Log

data/lib/smart_proxy_dynflow/callback.rb

@@ -1,58 +1,22 @@
 require 'rest-client'
 
-class Proxy::Dynflow
+module Proxy::Dynflow
   module Callback
-    class Request
-      class << self
-        def send_to_foreman_tasks(callback_info, data)
-          self.new.callback(prepare_payload(callback_info, data))
-        end
-
-        def ssl_options
-          return @ssl_options if defined? @ssl_options
-          @ssl_options = {}
-          settings = Proxy::SETTINGS
-          return @ssl_options unless URI.parse(settings.foreman_url).scheme == 'https'
-
-          @ssl_options[:verify_ssl] = OpenSSL::SSL::VERIFY_PEER
-
-          private_key_file = settings.foreman_ssl_key || settings.ssl_private_key
-          if private_key_file
-            private_key = File.read(private_key_file)
-            @ssl_options[:ssl_client_key] = OpenSSL::PKey::RSA.new(private_key)
-          end
-          certificate_file = settings.foreman_ssl_cert || settings.ssl_certificate
-          if certificate_file
-            certificate = File.read(certificate_file)
-            @ssl_options[:ssl_client_cert] = OpenSSL::X509::Certificate.new(certificate)
-          end
-          ca_file = settings.foreman_ssl_ca || settings.ssl_ca_file
-          @ssl_options[:ssl_ca_file] = ca_file if ca_file
-          @ssl_options
-        end
-        # rubocop:enable Metrics/PerceivedComplexity
-
-        private
-
-        def prepare_payload(callback, data)
-          { :callback => callback, :data => data }.to_json
-        end
+    class Request < ::Proxy::HttpRequest::ForemanRequest
+      def self.send_to_foreman_tasks(callback_info, data)
+        self.new.callback({ :callback => callback_info, :data => data }.to_json)
       end
 
       def callback(payload)
-        response = callback_resource.post(payload, :content_type => :json)
+        request = request_factory.create_post '/foreman_tasks/api/tasks/callback',
+                                              payload
+        response = send_request(request)
+
         if response.code.to_s != "200"
           raise "Failed performing callback to Foreman server: #{response.code} #{response.body}"
         end
         response
       end
-
-      private
-
-      def callback_resource
-        @resource ||= RestClient::Resource.new(Proxy::SETTINGS.foreman_url + '/foreman_tasks/api/tasks/callback',
-                                               self.class.ssl_options)
-      end
     end
 
     class Action < ::Dynflow::Action

data/lib/smart_proxy_dynflow/continuous_output.rb

@@ -0,0 +1,50 @@
+module Proxy::Dynflow
+  class ContinuousOutput
+    attr_accessor :raw_outputs
+
+    def initialize(raw_outputs = [])
+      @raw_outputs = []
+      raw_outputs.each { |raw_output| add_raw_output(raw_output) }
+    end
+
+    def add_raw_output(raw_output)
+      missing_args = %w[output_type output timestamp] - raw_output.keys
+      unless missing_args.empty?
+        raise ArgumentError, "Missing args for raw output: #{missing_args.inspect}"
+      end
+      @raw_outputs << raw_output
+    end
+
+    def empty?
+      @raw_outputs.empty?
+    end
+
+    def last_timestamp
+      return if @raw_outputs.empty?
+      @raw_outputs.last.fetch('timestamp')
+    end
+
+    def sort!
+      @raw_outputs.sort_by! { |record| record['timestamp'].to_f }
+    end
+
+    def humanize
+      sort!
+      raw_outputs.map { |output| output['output'] }.join("\n")
+    end
+
+    def add_exception(context, exception, timestamp = Time.now.getlocal)
+      add_output(context + ": #{exception.class} - #{exception.message}", 'debug', timestamp)
+    end
+
+    def add_output(*args)
+      add_raw_output(self.class.format_output(*args))
+    end
+
+    def self.format_output(message, type = 'debug', timestamp = Time.now.getlocal)
+      { 'output_type' => type,
+        'output' => message,
+        'timestamp' => timestamp.to_f }
+    end
+  end
+end

data/lib/smart_proxy_dynflow/core.rb

@@ -1,4 +1,4 @@
-class Proxy::Dynflow
+module Proxy::Dynflow
   class Core
     attr_accessor :world, :accepted_cert_serial
 
@@ -75,7 +75,7 @@ class Proxy::Dynflow
       end
 
       def silencer_matchers
-        @matchers ||= []
+        @matchers ||= [::Dynflow::DeadLetterSilencer::Matcher.new(Ticker)]
       end
 
       def register_silencer_matchers(matchers)

data/lib/smart_proxy_dynflow/helpers.rb

@@ -1,5 +1,5 @@
 module Proxy
-  class Dynflow
+  module Dynflow
     module Helpers
       def world
         Proxy::Dynflow::Core.world
@@ -7,12 +7,12 @@ module Proxy
 
       def authorize_with_token(task_id:, clear: true)
         if request.env.key? 'HTTP_AUTHORIZATION'
-          if defined?(::ForemanTasksCore)
+          if defined?(::Proxy::Dynflow)
             auth = request.env['HTTP_AUTHORIZATION']
             basic_prefix = /\ABasic /
             if !auth.to_s.empty? && auth =~ basic_prefix &&
-               ForemanTasksCore::OtpManager.authenticate(auth.gsub(basic_prefix, ''),
-                                                         expected_user: task_id, clear: clear)
+               Proxy::Dynflow::OtpManager.authenticate(auth.gsub(basic_prefix, ''),
+                                                       expected_user: task_id, clear: clear)
               Log.instance.debug('authorized with token')
               return true
             end
@@ -35,7 +35,12 @@ module Proxy
 
       def task_status(task_id)
         ep = world.persistence.load_execution_plan(task_id)
-        ep.to_hash.merge(:actions => ep.actions.map(&:to_hash))
+        actions = ep.actions.map do |action|
+          hash = action.to_hash
+          hash[:output][:result] = action.output_result if action.is_a?(Proxy::Dynflow::Action::Runner)
+          hash
+        end
+        ep.to_hash.merge(:actions => actions)
       rescue KeyError => _e
         status 404
         {}
@@ -51,7 +56,7 @@ module Proxy
       def dispatch_external_event(task_id, params)
         world.event(task_id,
                     params['step_id'].to_i,
-                    ::ForemanTasksCore::Runner::ExternalEvent.new(params))
+                    ::Proxy::Dynflow::Runner::ExternalEvent.new(params))
       end
     end
   end

data/lib/smart_proxy_dynflow/log.rb

@@ -1,6 +1,6 @@
 require 'logging'
 
-class Proxy::Dynflow
+module Proxy::Dynflow
   class Log
     LOGGER_NAME = 'dynflow-core'.freeze
 

data/lib/smart_proxy_dynflow/otp_manager.rb

@@ -0,0 +1,36 @@
+require 'base64'
+require 'securerandom'
+
+module Proxy::Dynflow
+  class OtpManager
+    class << self
+      def generate_otp(username)
+        otp = SecureRandom.hex
+        passwords[username] = otp.to_s
+      end
+
+      def drop_otp(username, password)
+        passwords.delete(username) if passwords[username] == password
+      end
+
+      def passwords
+        @password ||= {}
+      end
+
+      def authenticate(hash, expected_user: nil, clear: true)
+        plain = Base64.decode64(hash)
+        username, otp = plain.split(':', 2)
+        if expected_user
+          return false unless expected_user == username
+        end
+        password_matches = passwords[username] == otp
+        passwords.delete(username) if clear && password_matches
+        password_matches
+      end
+
+      def tokenize(username, password)
+        Base64.strict_encode64("#{username}:#{password}")
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/plugin.rb

@@ -2,7 +2,7 @@ require 'proxy/log'
 require 'proxy/pluggable'
 require 'proxy/plugin'
 
-class Proxy::Dynflow
+module Proxy::Dynflow
   class Plugin < Proxy::Plugin
     rackup_path = File.expand_path('http_config.ru', __dir__)
     http_rackup_path rackup_path
@@ -15,6 +15,14 @@ class Proxy::Dynflow
     plugin :dynflow, Proxy::Dynflow::VERSION
 
     after_activation do
+      require 'smart_proxy_dynflow/settings_loader'
+      require 'smart_proxy_dynflow/otp_manager'
+      require 'smart_proxy_dynflow/action'
+      require 'smart_proxy_dynflow/task_launcher'
+
+      Proxy::Dynflow::TaskLauncherRegistry.register('single',
+                                                    Proxy::Dynflow::TaskLauncher::Single)
+
       Proxy::Dynflow::Core.ensure_initialized
     end
   end

data/lib/smart_proxy_dynflow/proxy_adapter.rb

@@ -1,5 +1,5 @@
 module Proxy
-  class Dynflow
+  module Dynflow
     class ProxyAdapter < ::Dynflow::LoggerAdapters::Simple
       def initialize(logger, level = Logger::DEBUG, formatters = [Formatters::Exception])
         super(nil, level, formatters)

data/lib/smart_proxy_dynflow/runner/base.rb

@@ -0,0 +1,98 @@
+module Proxy::Dynflow
+  module Runner
+    # Runner is an object that is able to initiate some action and
+    # provide update data on refresh call.
+    class Base
+      attr_reader :id
+      attr_writer :logger
+
+      def initialize(*_args, suspended_action: nil)
+        @suspended_action = suspended_action
+        @id = SecureRandom.uuid
+        initialize_continuous_outputs
+      end
+
+      def logger
+        @logger ||= Logger.new(STDERR)
+      end
+
+      def run_refresh
+        logger.debug('refreshing runner')
+        refresh
+        generate_updates
+      end
+
+      # by default, external event just causes the refresh to be triggered: this allows the descendants
+      # of the Base to add custom logic to process the external events.
+      # Similarly as `run_refresh`, it's expected to return updates to be dispatched.
+      def external_event(_event)
+        run_refresh
+      end
+
+      def start
+        raise NotImplementedError
+      end
+
+      def refresh
+        raise NotImplementedError
+      end
+
+      def kill
+        # Override when you can kill the runner in the middle
+      end
+
+      def close
+        # if cleanup is needed
+      end
+
+      def timeout
+        # Override when timeouts and regular kills should be handled differently
+        publish_data('Timeout for execution passed, trying to stop the job', 'debug')
+        kill
+      end
+
+      def timeout_interval
+        # A number of seconds after which the runner should receive a #timeout
+        #   or nil for no timeout
+      end
+
+      def publish_data(data, type)
+        @continuous_output.add_output(data, type)
+      end
+
+      def publish_exception(context, exception, fatal = true)
+        logger.error("#{context} - #{exception.class} #{exception.message}:\n" + \
+                     exception.backtrace.join("\n"))
+        dispatch_exception context, exception
+        publish_exit_status('EXCEPTION') if fatal
+      end
+
+      def publish_exit_status(status)
+        @exit_status = status
+      end
+
+      def dispatch_exception(context, exception)
+        @continuous_output.add_exception(context, exception)
+      end
+
+      def generate_updates
+        return no_update if @continuous_output.empty? && @exit_status.nil?
+        new_data = @continuous_output
+        @continuous_output = Proxy::Dynflow::ContinuousOutput.new
+        new_update(new_data, @exit_status)
+      end
+
+      def no_update
+        {}
+      end
+
+      def new_update(data, exit_status)
+        { @suspended_action => Runner::Update.new(data, exit_status) }
+      end
+
+      def initialize_continuous_outputs
+        @continuous_output = ::Proxy::Dynflow::ContinuousOutput.new
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/command.rb

@@ -0,0 +1,40 @@
+module Proxy::Dynflow
+  module Runner
+    module Command
+      def initialize_command(*command)
+        @command_out, @command_in, @command_pid = PTY.spawn(*command)
+      rescue Errno::ENOENT => e
+        publish_exception("Error running command '#{command.join(' ')}'", e)
+      end
+
+      def refresh
+        return if @command_out.nil?
+        ready_outputs, * = IO.select([@command_out], nil, nil, 0.1)
+        if ready_outputs
+          if @command_out.nread.positive?
+            lines = @command_out.read_nonblock(@command_out.nread)
+          else
+            close_io
+            Process.wait(@command_pid)
+            publish_exit_status($CHILD_STATUS.exitstatus)
+          end
+          publish_data(lines, 'stdout') if lines && !lines.empty?
+        end
+      end
+
+      def close
+        close_io
+      end
+
+      private
+
+      def close_io
+        @command_out.close if @command_out && !@command_out.closed?
+        @command_out = nil
+
+        @command_in.close if @command_in && !@command_in.closed?
+        @command_in = nil
+      end
+    end
+  end
+end