smart_proxy_dynflow 0.2.3 → 0.5.1

data/lib/smart_proxy_dynflow/continuous_output.rb

@@ -0,0 +1,50 @@
+module Proxy::Dynflow
+  class ContinuousOutput
+    attr_accessor :raw_outputs
+
+    def initialize(raw_outputs = [])
+      @raw_outputs = []
+      raw_outputs.each { |raw_output| add_raw_output(raw_output) }
+    end
+
+    def add_raw_output(raw_output)
+      missing_args = %w[output_type output timestamp] - raw_output.keys
+      unless missing_args.empty?
+        raise ArgumentError, "Missing args for raw output: #{missing_args.inspect}"
+      end
+      @raw_outputs << raw_output
+    end
+
+    def empty?
+      @raw_outputs.empty?
+    end
+
+    def last_timestamp
+      return if @raw_outputs.empty?
+      @raw_outputs.last.fetch('timestamp')
+    end
+
+    def sort!
+      @raw_outputs.sort_by! { |record| record['timestamp'].to_f }
+    end
+
+    def humanize
+      sort!
+      raw_outputs.map { |output| output['output'] }.join("\n")
+    end
+
+    def add_exception(context, exception, timestamp = Time.now.getlocal)
+      add_output(context + ": #{exception.class} - #{exception.message}", 'debug', timestamp)
+    end
+
+    def add_output(*args)
+      add_raw_output(self.class.format_output(*args))
+    end
+
+    def self.format_output(message, type = 'debug', timestamp = Time.now.getlocal)
+      { 'output_type' => type,
+        'output' => message,
+        'timestamp' => timestamp.to_f }
+    end
+  end
+end

data/lib/smart_proxy_dynflow/core.rb

@@ -0,0 +1,121 @@
+module Proxy::Dynflow
+  class Core
+    attr_accessor :world, :accepted_cert_serial
+
+    def initialize
+      @world = create_world
+      cert_file = Proxy::SETTINGS.foreman_ssl_cert || Proxy::SETTINGS.ssl_certificate
+      if cert_file
+        client_cert = File.read(cert_file)
+        # we trust only requests using the same certificate as we are
+        # (in other words the local proxy only)
+        @accepted_cert_serial = OpenSSL::X509::Certificate.new(client_cert).serial
+      end
+    end
+
+    def create_world(&block)
+      config = default_world_config(&block)
+      world = ::Dynflow::World.new(config)
+      world.middleware.use ::Actions::Middleware::KeepCurrentRequestID
+      world
+    end
+
+    def persistence_conn_string
+      return ENV['DYNFLOW_DB_CONN_STRING'] if ENV.key? 'DYNFLOW_DB_CONN_STRING'
+      db_conn_string = 'sqlite:/'
+
+      db_file = Settings.instance.database
+      if db_file.nil? || db_file.empty?
+        Log.instance.warn "Could not open DB for dynflow at '#{db_file}', " \
+                          "will keep data in memory. Restart will drop all dynflow data."
+      else
+        db_conn_string += "/#{db_file}"
+      end
+
+      db_conn_string
+    end
+
+    def persistence_adapter
+      ::Dynflow::PersistenceAdapters::Sequel.new persistence_conn_string
+    end
+
+    def default_world_config
+      ::Dynflow::Config.new.tap do |config|
+        config.auto_rescue = true
+        config.logger_adapter = logger_adapter
+        config.persistence_adapter = persistence_adapter
+        config.execution_plan_cleaner = execution_plan_cleaner
+        # TODO: There has to be a better way
+        matchers = config.silent_dead_letter_matchers.call.concat(self.class.silencer_matchers)
+        config.silent_dead_letter_matchers = matchers
+        yield config if block_given?
+      end
+    end
+
+    def logger_adapter
+      Log::ProxyAdapter.new(Proxy::LogBuffer::Decorator.instance, Log.instance.level)
+    end
+
+    def execution_plan_cleaner
+      proc do |world|
+        age = Settings.instance.execution_plan_cleaner_age
+        options = { :poll_interval => age, :max_age => age }
+        ::Dynflow::Actors::ExecutionPlanCleaner.new(world, options)
+      end
+    end
+
+    class << self
+      attr_reader :instance
+
+      def ensure_initialized
+        return @instance if @instance
+        @instance = Core.new
+        after_initialize_blocks.each { |block| block.call(@instance) }
+        @instance
+      end
+
+      def silencer_matchers
+        @matchers ||= [::Dynflow::DeadLetterSilencer::Matcher.new(Ticker)]
+      end
+
+      def register_silencer_matchers(matchers)
+        silencer_matchers.concat matchers
+      end
+
+      def web_console
+        require 'dynflow/web'
+        dynflow_console = ::Dynflow::Web.setup do
+          # we can't use the proxy's after_activation hook, as
+          # it happens before the Daemon forks the process (including
+          # closing opened file descriptors)
+          # TODO: extend smart proxy to enable hooks that happen after
+          # the forking
+          helpers Helpers
+          include ::Sinatra::Authorization::Helpers
+
+          before do
+            do_authorize_with_ssl_client if Settings.instance.console_auth
+          end
+
+          Core.ensure_initialized
+          set :world, Core.world
+        end
+        dynflow_console
+      end
+
+      def world
+        instance.world
+      end
+
+      def after_initialize(&block)
+        after_initialize_blocks << block
+      end
+
+      private
+
+      def after_initialize_blocks
+        @after_initialize_blocks ||= []
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/helpers.rb

@@ -1,11 +1,57 @@
 module Proxy
-  class Dynflow
+  module Dynflow
     module Helpers
-      def relay_request(from = %r{^/dynflow}, to = '')
-        response = Proxy::Dynflow::Callback::Core.relay(request, from, to)
-        content_type response.content_type
-        status response.code
-        body response.body
+      def world
+        Proxy::Dynflow::Core.world
+      end
+
+      def authorize_with_token(task_id:, clear: true)
+        if request.env.key? 'HTTP_AUTHORIZATION'
+          if defined?(::Proxy::Dynflow)
+            auth = request.env['HTTP_AUTHORIZATION']
+            basic_prefix = /\ABasic /
+            if !auth.to_s.empty? && auth =~ basic_prefix &&
+               Proxy::Dynflow::OtpManager.authenticate(auth.gsub(basic_prefix, ''),
+                                                       expected_user: task_id, clear: clear)
+              Log.instance.debug('authorized with token')
+              return true
+            end
+          end
+          halt 403, MultiJson.dump(:error => 'Invalid username or password supplied')
+        end
+        false
+      end
+
+      def trigger_task(*args)
+        triggered = world.trigger(*args)
+        { :task_id => triggered.id }
+      end
+
+      def cancel_task(task_id)
+        execution_plan = world.persistence.load_execution_plan(task_id)
+        cancel_events = execution_plan.cancel
+        { :task_id => task_id, :canceled_steps_count => cancel_events.size }
+      end
+
+      def task_status(task_id)
+        ep = world.persistence.load_execution_plan(task_id)
+        ep.to_hash.merge(:actions => ep.actions.map(&:to_hash))
+      rescue KeyError => _e
+        status 404
+        {}
+      end
+
+      def tasks_count(state)
+        state ||= 'all'
+        filter = state != 'all' ? { :filters => { :state => [state] } } : {}
+        tasks = world.persistence.find_execution_plans(filter)
+        { :count => tasks.count, :state => state }
+      end
+
+      def dispatch_external_event(task_id, params)
+        world.event(task_id,
+                    params['step_id'].to_i,
+                    ::Proxy::Dynflow::Runner::ExternalEvent.new(params))
       end
     end
   end

data/lib/smart_proxy_dynflow/http_config.ru

@@ -1,21 +1,11 @@
-# Internal core will be used if external core is either disabled or unset
-# and the core gem can be loaded
+require 'smart_proxy_dynflow/api'
 
-if !::Proxy::Dynflow::Plugin.settings.external_core && Proxy::Dynflow::Plugin.internal_core_available?
-  require 'smart_proxy_dynflow_core/api'
-  require 'smart_proxy_dynflow_core/launcher'
-
-  SmartProxyDynflowCore::Settings.load_from_proxy(p)
-
-  map "/dynflow" do
-    SmartProxyDynflowCore::Launcher.route_mapping(self)
+map "/dynflow" do
+  map '/console' do
+    run Proxy::Dynflow::Core.web_console
   end
-else
-  require 'smart_proxy_dynflow/api'
 
-  map "/dynflow" do
-    map '/' do
-      run Proxy::Dynflow::Api
-    end
+  map '/' do
+    run Proxy::Dynflow::Api
   end
 end

data/lib/smart_proxy_dynflow/log.rb

@@ -0,0 +1,52 @@
+require 'logging'
+
+module Proxy::Dynflow
+  class Log
+    LOGGER_NAME = 'dynflow-core'.freeze
+
+    begin
+      require 'syslog/logger'
+      @syslog_available = true
+    rescue LoadError
+      @syslog_available = false
+    end
+
+    class << self
+      def reload!
+        Logging.logger[LOGGER_NAME].appenders.each(&:close)
+        Logging.logger[LOGGER_NAME].clear_appenders
+        @logger = nil
+        instance
+      end
+
+      def instance
+        ::Proxy::LogBuffer::Decorator.instance
+      end
+    end
+
+    class ProxyStructuredFormater < ::Dynflow::LoggerAdapters::Formatters::Abstract
+      def format(message)
+        if message.is_a?(Exception)
+          subject = "#{message.message} (#{message.class})"
+          if @base.respond_to?(:exception)
+            @base.exception("Error details", message)
+            subject
+          else
+            "#{subject}\n#{message.backtrace.join("\n")}"
+          end
+        else
+          @original_formatter.call(severity, datetime, prog_name, message)
+        end
+      end
+    end
+
+    class ProxyAdapter < ::Dynflow::LoggerAdapters::Simple
+      def initialize(logger, level = Logger::DEBUG, _formatters = [])
+        @logger           = logger
+        @logger.level     = level
+        @action_logger    = apply_formatters(ProgNameWrapper.new(@logger, ' action'), [])
+        @dynflow_logger   = apply_formatters(ProgNameWrapper.new(@logger, 'dynflow'), [])
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/middleware/keep_current_request_id.rb

@@ -0,0 +1,59 @@
+module Actions
+  module Middleware
+    class KeepCurrentRequestID < Dynflow::Middleware
+      def delay(*args)
+        pass(*args).tap { store_current_request_id }
+      end
+
+      def plan(*args)
+        with_current_request_id do
+          pass(*args).tap { store_current_request_id }
+        end
+      end
+
+      def run(*args)
+        restore_current_request_id { pass(*args) }
+      end
+
+      def finalize
+        restore_current_request_id { pass }
+      end
+
+      # Run all execution plan lifecycle hooks as the original request_id
+      def hook(*args)
+        restore_current_request_id { pass(*args) }
+      end
+
+      private
+
+      def with_current_request_id
+        if action.input[:current_request_id].nil?
+          yield
+        else
+          restore_current_request_id { yield }
+        end
+      end
+
+      def store_current_request_id
+        action.input[:current_request_id] = ::Logging.mdc['request']
+      end
+
+      def restore_current_request_id
+        unless (restored_id = action.input[:current_request_id]).nil?
+          old_id = ::Logging.mdc['request']
+          if !old_id.nil? && old_id != restored_id
+            action.action_logger.warn('Changing request id %{request_id} to saved id %{saved_id}' % { :saved_id => restored_id, :request_id => old_id })
+          end
+          ::Logging.mdc['request'] = restored_id
+        end
+        yield
+      ensure
+        # Reset to original request id only when not nil
+        # Otherwise, keep the id until it's cleaned in  Dynflow's run_user_code block
+        # so that it will stay valid for the rest of the processing of the current step
+        # (even outside of the middleware lifecycle)
+        ::Logging.mdc['request'] = old_id unless old_id.nil?
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/otp_manager.rb

@@ -0,0 +1,36 @@
+require 'base64'
+require 'securerandom'
+
+module Proxy::Dynflow
+  class OtpManager
+    class << self
+      def generate_otp(username)
+        otp = SecureRandom.hex
+        passwords[username] = otp.to_s
+      end
+
+      def drop_otp(username, password)
+        passwords.delete(username) if passwords[username] == password
+      end
+
+      def passwords
+        @password ||= {}
+      end
+
+      def authenticate(hash, expected_user: nil, clear: true)
+        plain = Base64.decode64(hash)
+        username, otp = plain.split(':', 2)
+        if expected_user
+          return false unless expected_user == username
+        end
+        password_matches = passwords[username] == otp
+        passwords.delete(username) if clear && password_matches
+        password_matches
+      end
+
+      def tokenize(username, password)
+        Base64.strict_encode64("#{username}:#{password}")
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/plugin.rb

@@ -2,30 +2,25 @@ require 'proxy/log'
 require 'proxy/pluggable'
 require 'proxy/plugin'
 
-class Proxy::Dynflow
+module Proxy::Dynflow
   class Plugin < Proxy::Plugin
     rackup_path = File.expand_path('http_config.ru', __dir__)
     http_rackup_path rackup_path
     https_rackup_path rackup_path
 
     settings_file "dynflow.yml"
-    requires :foreman_proxy, ">= 1.12.0"
-    default_settings :core_url => 'http://localhost:8008'
+    requires :foreman_proxy, ">= 1.16.0"
+    default_settings :console_auth => true,
+                     :execution_plan_cleaner_age => 60 * 60 * 24
     plugin :dynflow, Proxy::Dynflow::VERSION
 
     after_activation do
-      # Ensure the core gem is loaded, if configure NOT to use the external core
-      if Proxy::Dynflow::Plugin.settings.external_core == false && !internal_core_available?
-        raise "'smart_proxy_dynflow_core' gem is required, but not available"
-      end
-    end
+      require 'smart_proxy_dynflow/settings_loader'
+      require 'smart_proxy_dynflow/otp_manager'
+      require 'smart_proxy_dynflow/action'
+      require 'smart_proxy_dynflow/task_launcher'
 
-    def self.internal_core_available?
-      @core_available ||= begin
-                            require 'smart_proxy_dynflow_core'
-                            true
-                          rescue LoadError # rubocop:disable Lint/HandleExceptions
-                          end
+      Proxy::Dynflow::Core.ensure_initialized
     end
   end
 end