dynflow 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ee35eec200e14b25add8941b4b4637e994012053a271898c4abc4a70234942e
-  data.tar.gz: 720fc9161e5aadff8f165c12f6bb278cfa65435ef7a1d96d93df8614d21a1de8
+  metadata.gz: '0678aaa84d4b56ba08bbd68a6e034076f22b9f44bf85110f34a9150403490243'
+  data.tar.gz: c91ee787f316b332f8e802557ed31aa6490db2b62d82b595e3397549f5596aef
 SHA512:
-  metadata.gz: 1753d21be5307643a16704a27b0e343fe55a7e74330469d1eece503d58cfa9fd04be1da57918e6e09e73f7e2822f2be2bd517e9eea58b89432bee7226d51ab93
-  data.tar.gz: ac12ce027be3e289227db4a2b5311328342de364c0f5d51f5badc27e7ba484a6f137488ce681d319e050b1a04ef55ac4821d0d87a06d6d70019f3b2c6e889fca
+  metadata.gz: '003039069ca213958db31c0dcb6d158c8ab675182881ac4f4edfbafbbec826ca36fe51ce606bbd2621226851ac72bde934995d4aae36a6da43410ee573917e27'
+  data.tar.gz: 5321596fe890b8398f3e207c39991a72febf76f6238bc4c5c91df1634e66ded7149574624846fead73d52270a089a429928bbd8b3d4f6f33318ad6c880180f95

data/.rubocop.yml

@@ -20,3 +20,7 @@ Style/MultilineOperationIndentation:
 # Cop supports --auto-correct.
 Style/EmptyLines:
   Enabled: true
+
+Metrics/ModuleLength:
+  Exclude:
+  - test/**/*

data/Gemfile

@@ -41,3 +41,7 @@ group :rails do
   gem 'rails', '>= 4.2.9'
   gem 'logging'
 end
+
+group :telemetry do
+  gem 'statsd-instrument'
+end

data/examples/example_helper.rb

@@ -17,6 +17,7 @@ class ExampleHelper
       config.persistence_adapter = persistence_adapter
       config.logger_adapter      = logger_adapter
       config.auto_rescue         = false
+      config.telemetry_adapter   = telemetry_adapter
       yield config if block_given?
       Dynflow::World.new(config).tap do |world|
         puts "World #{world.id} started..."
@@ -27,6 +28,14 @@ class ExampleHelper
       ENV['DB_CONN_STRING'] || 'sqlite:/'
     end
 
+    def telemetry_adapter
+      if (host = ENV['TELEMETRY_STATSD_HOST'])
+        Dynflow::TelemetryAdapters::StatsD.new host
+      else
+        Dynflow::TelemetryAdapters::Dummy.new
+      end
+    end
+
     def persistence_adapter
       Dynflow::PersistenceAdapters::Sequel.new persistence_conn_string
     end

data/extras/statsd_mapping.conf

@@ -0,0 +1,65 @@
+---
+mappings:
+- name: dynflow_active_execution_plans
+  match: dynflow_active_execution_plans.*.*.*
+  labels:
+    action: "$1"
+    world: "$2"
+    state: "$3"
+  help: The number of active execution plans
+- name: dynflow_active_workers
+  match: dynflow_active_workers.*.*
+  labels:
+    queue: "$1"
+    world: "$2"
+  help: The number of currently busy workers
+- name: dynflow_queue_size
+  match: dynflow_queue_size.*.*
+  labels:
+    queue: "$1"
+    world: "$2"
+  help: The number of events in the queue
+- name: dynflow_connector_envelopes
+  match: dynflow_connector_envelopes.*.*
+  labels:
+    world: "$1"
+    direction: "$2"
+  help: The number of envelopes handled by a connector
+- name: dynflow_finished_execution_plans
+  match: dynflow_finished_execution_plans.*.*.*
+  labels:
+    action: "$1"
+    world: "$2"
+    result: "$3"
+  help: The number of execution plans
+- name: dynflow_step_execution_time
+  match: dynflow_step_execution_time.*.*
+  labels:
+    action: "$1"
+    phase: "$2"
+  help: The time spent executing a step
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600, 1200]
+  timer_type: histogram
+- name: dynflow_step_real_time
+  match: dynflow_step_real_time.*.*
+  labels:
+    action: "$1"
+    phase: "$2"
+  help: The time between the start end end of the step
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600, 1200]
+  timer_type: histogram
+- name: dynflow_worker_events
+  match: dynflow_worker_events.*.*.*
+  labels:
+    queue: "$1"
+    world: "$2"
+    worker: "$3"
+  help: The number of processed events
+- name: dynflow_persistence
+  match: dynflow_persistence.*.*
+  labels:
+    world: "$1"
+    method: "$2"
+  help: The time spent communicating with the database
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600, 1200]
+  timer_type: histogram

data/lib/dynflow.rb

@@ -50,6 +50,7 @@ module Dynflow
   require 'dynflow/delayed_executors'
   require 'dynflow/semaphores'
   require 'dynflow/throttle_limiter'
+  require 'dynflow/telemetry'
   require 'dynflow/config'
 
   if defined? ::ActiveJob

data/lib/dynflow/action.rb

@@ -511,6 +511,7 @@ module Dynflow
       end
     end
 
+    # TODO: This is getting out of hand, refactoring needed
     def execute_run(event)
       phase! Run
       @world.logger.debug format('%13s %s:%2d got event %s',
@@ -544,7 +545,6 @@ module Dynflow
 
           check_serializable :output
         end
-
       else
         raise "wrong state #{state} when event:#{event}"
       end

data/lib/dynflow/active_job/queue_adapter.rb

@@ -45,6 +45,10 @@ module Dynflow
         def label
           input[:job_class]
         end
+
+        def rescue_strategy
+          Action::Rescue::Skip
+        end
       end
     end
   end

data/lib/dynflow/config.rb

@@ -182,6 +182,10 @@ module Dynflow
       './backup'
     end
 
+    config_attr :telemetry_adapter, ::Dynflow::TelemetryAdapters::Abstract do |world|
+      ::Dynflow::TelemetryAdapters::Dummy.new
+    end
+
     def validate(config_for_world)
       if defined? ::ActiveRecord::Base
         begin

data/lib/dynflow/connectors/abstract.rb

@@ -8,7 +8,11 @@ module Dynflow
         raise NotImplementedError
       end
 
-      def stop_listening(world)
+      def stop_receiving_new_work(_, timeout = nil)
+        raise NotImplementedError
+      end
+
+      def stop_listening(world, timeout = nil)
         raise NotImplementedError
       end
 
@@ -24,6 +28,7 @@ module Dynflow
       # between words: we need to know the one to send the message to
       def receive(world, envelope)
         Type! envelope, Dispatcher::Envelope
+        Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => world.id, :direction => 'incoming') }
         match(envelope.message,
               (on Dispatcher::Ping do
                  response_envelope = envelope.build_response_envelope(Dispatcher::Pong, world)

data/lib/dynflow/connectors/database.rb

@@ -159,15 +159,16 @@ module Dynflow
         @core.ask([:start_listening, world])
       end
 
-      def stop_receiving_new_work(_)
-        @core.ask(:stop_receiving_new_work).wait
+      def stop_receiving_new_work(_, timeout = nil)
+        @core.ask(:stop_receiving_new_work).wait(timeout)
       end
 
-      def stop_listening(_)
-        @core.ask(:stop_listening).then { @core.ask(:terminate!) }.wait
+      def stop_listening(_, timeout = nil)
+        @core.ask(:stop_listening).then { @core.ask(:terminate!) }.wait(timeout)
       end
 
       def send(envelope)
+        Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id, :direction => 'outgoing') }
         @core.ask([:handle_envelope, envelope])
       end
     end

data/lib/dynflow/connectors/direct.rb

@@ -55,15 +55,16 @@ module Dynflow
         @core.ask([:start_listening, world])
       end
 
-      def stop_receiving_new_work(world)
-        @core.ask([:stop_receiving_new_work, world]).wait
+      def stop_receiving_new_work(world, timeout = nil)
+        @core.ask([:stop_receiving_new_work, world]).wait(timeout)
       end
 
-      def stop_listening(world)
-        @core.ask([:stop_listening, world]).wait
+      def stop_listening(world, timeout = nil)
+        @core.ask([:stop_listening, world]).wait(timeout)
       end
 
       def send(envelope)
+        Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => envelope.sender_id) }
         @core.ask([:handle_envelope, envelope])
       end
     end

data/lib/dynflow/coordinator_adapters/sequel.rb

@@ -9,8 +9,12 @@ module Dynflow
 
       def create_record(record)
         @sequel_adapter.insert_coordinator_record(record.to_hash)
-      rescue ::Sequel::UniqueConstraintViolation
-        raise Coordinator::DuplicateRecordError.new(record)
+      rescue Errors::PersistenceError => e
+        if e.cause.is_a? ::Sequel::UniqueConstraintViolation
+          raise Coordinator::DuplicateRecordError.new(record)
+        else
+          raise e
+        end
       end
 
       def update_record(record)

data/lib/dynflow/debug/telemetry/persistence.rb

@@ -0,0 +1,42 @@
+module Dynflow
+  module Debug
+    module Telemetry
+      module Persistence
+        methods = [
+          :load_action,
+          :load_actions,
+          :load_action_for_presentation,
+          :load_action,
+          :load_actions,
+          :load_action_for_presentation,
+          :load_actions_attributes,
+          :save_action,
+          :find_execution_plans,
+          :find_execution_plan_counts,
+          :delete_execution_plans,
+          :load_execution_plan,
+          :save_execution_plan,
+          :find_old_execution_plans,
+          :find_past_delayed_plans,
+          :delete_delayed_plans,
+          :save_delayed_plan,
+          :set_delayed_plan_frozen,
+          :load_delayed_plan,
+          :load_step,
+          :load_steps,
+          :save_step,
+          :push_envelope,
+          :pull_envelopes
+        ]
+
+        methods.each do |name|
+          define_method(name) do |*args|
+            Dynflow::Telemetry.measure(:dynflow_persistence, :method => name, :world => @world.id) { super *args }
+          end
+        end
+      end
+    end
+  end
+end
+
+::Dynflow::Persistence.send(:prepend, ::Dynflow::Debug::Persistence)

data/lib/dynflow/director.rb

@@ -84,6 +84,10 @@ module Dynflow
       @rescued_steps = {}
     end
 
+    def current_execution_plan_ids
+      @execution_plan_managers.keys
+    end
+
     def start_execution(execution_plan_id, finished)
       manager = track_execution_plan(execution_plan_id, finished)
       return [] unless manager
@@ -108,6 +112,17 @@ module Dynflow
       unless_done(manager, manager.what_is_next(work))
     end
 
+    # called when there was an unhandled exception during the execution
+    # of the work (such as persistence issue) - in this case we just clean up the
+    # runtime from the execution plan and let it go (common cause for this is the execution
+    # plan being removed from database by external user)
+    def work_failed(work)
+      if (manager = @execution_plan_managers[work.execution_plan_id])
+        manager.terminate
+        finish_manager(manager)
+      end
+    end
+
     def terminate
       unless @execution_plan_managers.empty?
         logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."

data/lib/dynflow/errors.rb

@@ -34,6 +34,7 @@ module Dynflow
     class DataConsistencyError < Dynflow::Error
     end
 
+    # any persistence errors
     class PersistenceError < Dynflow::Error
       def self.delegate(original_exception)
         self.new("caused by #{original_exception.class}: #{original_exception.message}").tap do |e|
@@ -41,5 +42,9 @@ module Dynflow
         end
       end
     end
+
+    # persistence errors that can't be recovered from, such as continuous connection issues
+    class FatalPersistenceError < PersistenceError
+    end
   end
 end

data/lib/dynflow/execution_plan.rb

@@ -120,7 +120,12 @@ module Dynflow
         @ended_at       = Time.now
         @real_time      = @ended_at - @started_at unless @started_at.nil?
         @execution_time = compute_execution_time
-        hooks_to_run << (failure? ? :failure : :success)
+        key = failure? ? :failure : :success
+        Dynflow::Telemetry.with_instance do |t|
+          t.increment_counter(:dynflow_finished_execution_plans, 1,
+                              telemetry_common_options.merge(:result => key.to_s))
+        end
+        hooks_to_run << key
         unlock_all_singleton_locks!
       when :paused
         unlock_all_singleton_locks!
@@ -130,6 +135,8 @@ module Dynflow
       logger.debug format('%13s %s    %9s >> %9s',
                           'ExecutionPlan', id, original, state)
       self.save
+      toggle_telemetry_state original == :pending ? nil : original.to_s,
+                             self.state == :stopped ? nil : self.state.to_s
       hooks_to_run.each { |kind| run_hooks kind }
     end
 
@@ -548,6 +555,21 @@ module Dynflow
       end
     end
 
+    def toggle_telemetry_state(original, new)
+      return if original == new
+      @label = root_plan_step.action_class if @label.nil?
+      Dynflow::Telemetry.with_instance do |t|
+        t.set_gauge(:dynflow_active_execution_plans, '-1',
+                    telemetry_common_options.merge(:state => original)) unless original.nil?
+        t.set_gauge(:dynflow_active_execution_plans, '+1',
+                    telemetry_common_options.merge(:state => new)) unless new.nil?
+      end
+    end
+
+    def telemetry_common_options
+      { :world => @world.id, :action => @label }
+    end
+
     private_class_method :steps_from_hash
   end
   # rubocop:enable Metrics/ClassLength

data/lib/dynflow/execution_plan/steps/abstract.rb

@@ -162,9 +162,11 @@ module Dynflow
         block.call
       ensure
         calculate_progress(action)
-        @ended_at        = Time.now
-        @execution_time += @ended_at - start
+        @ended_at = Time.now
+        current_execution_time = @ended_at - start
+        @execution_time += current_execution_time
         @real_time       = @ended_at - @started_at
+        update_step_telemetry(current_execution_time)
       end
 
       def calculate_progress(action)
@@ -174,6 +176,19 @@ module Dynflow
           @progress_done = 0
         end
       end
+
+      def update_step_telemetry(current_execution_time)
+        Dynflow::Telemetry.with_instance do |t|
+          if [:success, :skipped].include?(state)
+            t.observe_histogram(:dynflow_step_real_time,
+                                real_time * 1000,
+                                :action => action_class.to_s, :phase => phase.to_s_humanized)
+          end
+          t.observe_histogram(:dynflow_step_execution_time,
+                              current_execution_time * 1000,
+                              :action => action_class.to_s, :phase => phase.to_s_humanized)
+        end
+      end
     end
   end
 end

data/lib/dynflow/executors/parallel/core.rb

@@ -21,8 +21,8 @@ module Dynflow
           default_pool_size = @queues_options[:default][:pool_size]
           @queues_options.each do |(queue_name, queue_options)|
             queue_pool_size = queue_options.fetch(:pool_size, default_pool_size)
-            @pools[queue_name] = Pool.spawn("pool #{queue_name}", reference,
-                                            queue_name, queue_pool_size,
+            @pools[queue_name] = Pool.spawn("pool #{queue_name}", @world,
+                                            reference, queue_name, queue_pool_size,
                                             @world.transaction_adapter)
           end
         end
@@ -49,10 +49,14 @@ module Dynflow
           feed_pool(@director.work_finished(work))
         end
 
-        def handle_persistence_error(error)
-          logger.fatal "PersistenceError in executor: terminating"
-          logger.fatal error
-          @world.terminate
+        def handle_persistence_error(error, work = nil)
+          logger.error "PersistenceError in executor"
+          logger.error error
+          @director.work_failed(work) if work
+          if error.is_a? Errors::FatalPersistenceError
+            logger.fatal "Terminating"
+            @world.terminate
+          end
         end
 
         def start_termination(*args)
@@ -66,7 +70,7 @@ module Dynflow
           # we expect this message from all worker pools
           return unless @pools.empty?
           @director.terminate
-          logger.error '... core terminated.'
+          logger.info '... Dynflow core terminated.'
           super()
         end
 

data/lib/dynflow/executors/parallel/pool.rb

@@ -19,6 +19,10 @@ module Dynflow
             @jobs[execution_plan_id].shift.tap { delete execution_plan_id if @jobs[execution_plan_id].empty? }
           end
 
+          def queue_size
+            execution_status.values.reduce(0, :+)
+          end
+
           def empty?
             @jobs.empty?
           end
@@ -46,27 +50,35 @@ module Dynflow
           end
         end
 
-        def initialize(core, name, pool_size, transaction_adapter)
+        def initialize(world, core, name, pool_size, transaction_adapter)
+          @world = world
           @name = name
           @executor_core = core
           @pool_size     = pool_size
-          @free_workers  = Array.new(pool_size) { |i| Worker.spawn("worker-#{i}", reference, transaction_adapter) }
           @jobs          = JobStorage.new
+          @free_workers  = Array.new(pool_size) do |i|
+            name = "worker-#{i}"
+            Worker.spawn(name, reference, transaction_adapter, telemetry_options.merge(:worker => name))
+          end
         end
 
         def schedule_work(work)
           @jobs.add work
           distribute_jobs
+          update_telemetry
         end
 
         def worker_done(worker, work)
           @executor_core.tell([:work_finished, work])
           @free_workers << worker
+          Dynflow::Telemetry.with_instance { |t| t.set_gauge(:dynflow_active_workers, -1, telemetry_options) }
           distribute_jobs
         end
 
-        def handle_persistence_error(error)
-          @executor_core.tell([:handle_persistence_error, error])
+        def handle_persistence_error(worker, error, work = nil)
+          @executor_core.tell([:handle_persistence_error, error, work])
+          @free_workers << worker
+          distribute_jobs
         end
 
         def start_termination(*args)
@@ -92,7 +104,19 @@ module Dynflow
 
         def distribute_jobs
           try_to_terminate
-          @free_workers.pop << @jobs.pop until @free_workers.empty? || @jobs.empty?
+          until @free_workers.empty? || @jobs.empty?
+            Dynflow::Telemetry.with_instance { |t| t.set_gauge(:dynflow_active_workers, '+1', telemetry_options) }
+            @free_workers.pop << @jobs.pop
+            update_telemetry
+          end
+        end
+
+        def telemetry_options
+          { :queue => @name.to_s, :world => @world.id }
+        end
+
+        def update_telemetry
+          Dynflow::Telemetry.with_instance { |t| t.set_gauge(:dynflow_queue_size, @jobs.queue_size, telemetry_options) }
         end
       end
     end

data/lib/dynflow/executors/parallel/worker.rb

@@ -2,19 +2,23 @@ module Dynflow
   module Executors
     class Parallel < Abstract
       class Worker < Actor
-        def initialize(pool, transaction_adapter)
+        def initialize(pool, transaction_adapter, telemetry_options = {})
           @pool                = Type! pool, Concurrent::Actor::Reference
           @transaction_adapter = Type! transaction_adapter, TransactionAdapters::Abstract
+          @telemetry_options   = telemetry_options
         end
 
         def on_message(work_item)
+          already_responded = false
           Executors.run_user_code do
             work_item.execute
           end
         rescue Errors::PersistenceError => e
-          @pool.tell([:handle_persistence_error, e])
+          @pool.tell([:handle_persistence_error, reference, e, work_item])
+          already_responded = true
         ensure
-          @pool.tell([:worker_done, reference, work_item])
+          Dynflow::Telemetry.with_instance { |t| t.increment_counter(:dynflow_worker_events, 1, @telemetry_options) }
+          @pool.tell([:worker_done, reference, work_item]) unless already_responded
         end
       end
     end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -435,19 +435,19 @@ module Dynflow
         attempts = 0
         begin
           yield
-        rescue ::Sequel::UniqueConstraintViolation => e
-          raise e
-        rescue Exception => e
+        rescue ::Sequel::DatabaseConnectionError, ::Sequel::DatabaseDisconnectError => e
           attempts += 1
           log(:error, e)
           if attempts > MAX_RETRIES
             log(:error, "The number of MAX_RETRIES exceeded")
-            raise Errors::PersistenceError.delegate(e)
+            raise Errors::FatalPersistenceError.delegate(e)
           else
             log(:error, "Persistence retry no. #{attempts}")
             sleep RETRY_DELAY
             retry
           end
+        rescue Exception => e
+          raise Errors::PersistenceError.delegate(e)
         end
       end
 

data/lib/dynflow/rails.rb

@@ -36,10 +36,10 @@ module Dynflow
       end
       init_world.tap do |world|
         @world = world
-
+        config.run_on_init_hooks(false, world)
         unless config.remote?
           config.increase_db_pool_size(world)
-          config.run_on_init_hooks(world)
+          config.run_on_init_hooks(true, world)
           # leave this just for long-running executors
           unless config.rake_task_with_executor?
             invalidated_worlds = world.perform_validity_checks

data/lib/dynflow/rails/configuration.rb

@@ -39,7 +39,8 @@ module Dynflow
         self.lazy_initialization      = !::Rails.env.production?
         self.rake_tasks_with_executor = %w(db:migrate db:seed)
 
-        @on_init = []
+        @on_init           = []
+        @on_executor_init  = []
       end
 
       # Action related info such as exceptions raised inside the actions' methods
@@ -54,12 +55,14 @@ module Dynflow
         ::Rails.logger
       end
 
-      def on_init(&block)
-        @on_init << block
+      def on_init(executor = true, &block)
+        destination = executor ? @on_executor_init : @on_init
+        destination << block
       end
 
-      def run_on_init_hooks(world)
-        @on_init.each { |init| init.call(world) }
+      def run_on_init_hooks(executor, world)
+        source = executor ? @on_executor_init : @on_init
+        source.each { |init| init.call(world) }
       end
 
       def initialize_world(world_class = ::Dynflow::World)
@@ -114,7 +117,7 @@ module Dynflow
         @world_config ||= ::Dynflow::Config.new.tap do |config|
           config.auto_rescue         = true
           config.logger_adapter      = ::Dynflow::LoggerAdapters::Delegator.new(action_logger, dynflow_logger)
-          config.pool_size           = 5
+          config.pool_size           = self.pool_size
           config.persistence_adapter = ->(world, _) { initialize_persistence(world) }
           config.transaction_adapter = transaction_adapter
           config.executor            = ->(world, _) { initialize_executor(world) }

data/lib/dynflow/telemetry.rb

@@ -0,0 +1,65 @@
+require 'dynflow/telemetry_adapters/abstract'
+require 'dynflow/telemetry_adapters/dummy'
+require 'dynflow/telemetry_adapters/statsd'
+
+module Dynflow
+  class Telemetry
+    class << self
+      attr_reader :instance
+
+      # Configures the adapter to use for telemetry
+      #
+      # @param [TelemetryAdapters::Abstract] adapter the adapter to use
+      def set_adapter(adapter)
+        @instance = adapter
+      end
+
+      # Passes the block into the current telemetry adapter's
+      # {TelemetryAdapters::Abstract#with_instance} method
+      def with_instance(&block)
+        @instance.with_instance &block
+      end
+
+      def measure(name, tags = {}, &block)
+        @instance.measure name, tags, &block
+      end
+
+      # Registers the metrics to be collected
+      # @return [void]
+      def register_metrics!
+        return if @registered
+        @registered = true
+        with_instance do |t|
+          # Worker related
+          t.add_gauge   :dynflow_active_workers, 'The number of currently busy workers',
+                        [:queue, :world]
+          t.add_counter :dynflow_worker_events, 'The number of processed events',
+                        [:queue, :world, :worker]
+
+          # Execution plan related
+          t.add_gauge   :dynflow_active_execution_plans, 'The number of active execution plans',
+                        [:action, :world, :state]
+          t.add_gauge   :dynflow_queue_size, 'Number of items in queue',
+                        [:queue, :world]
+          t.add_counter :dynflow_finished_execution_plans, 'The number of execution plans',
+                        [:action, :world, :result]
+
+          # Step related
+          # TODO: Configure buckets in a sane manner
+          t.add_histogram :dynflow_step_real_time, 'The time between the start end end of the step',
+                          [:action, :phase]
+          t.add_histogram :dynflow_step_execution_time, 'The time spent executing a step',
+                          [:action, :phase]
+
+          # Connector related
+          t.add_counter :dynflow_connector_envelopes, 'The number of envelopes handled by a connector',
+                        [:world, :direction]
+
+          # Persistence related
+          t.add_histogram :dynflow_persistence, 'The time spent communicating with the database',
+                          [:world, :method]
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/telemetry_adapters/abstract.rb

@@ -0,0 +1,80 @@
+module Dynflow
+  module TelemetryAdapters
+    class Abstract
+      # Default buckets to use when defining a histogram
+      DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600, 1200].freeze
+
+      # Configures a counter to be collected
+      #
+      # @param [String] name Name of the counter
+      # @param [String] description Human-readable description of the counter
+      # @param [Array<String>] instance_labels Labels which will be assigned to the collected data
+      # @return [void]
+      def add_counter(name, description, instance_labels = [])
+      end
+
+      # Configures a gauge to be collected
+      #
+      # @param [String] name Name of the gauge
+      # @param [String] description Human-readable description of the gauge
+      # @param [Array<String>] instance_labels Labels which will be assigned to the collected data
+      # @return [void]
+      def add_gauge(name, description, instance_labels = [])
+      end
+
+      # Configures a histogram to be collected
+      #
+      # @param [String] name Name of the histogram
+      # @param [String] description Human-readable description of the histogram
+      # @param [Array<String>] instance_labels Labels which will be assigned to the collected data
+      # @param [Array<Integer>] buckest Buckets to fit the value into
+      # @return [void]
+      def add_histogram(name, description, instance_labels = [], buckets = DEFAULT_BUCKETS)
+      end
+
+      # Increments a counter
+      #
+      # @param [String,Symbol] name Name of the counter to increment
+      # @param [Integer] value Step to increment by
+      # @param [Hash{Symbol=>String}] tags Tags to apply to this record
+      # @return [void]
+      def increment_counter(name, value = 1, tags = {})
+      end
+
+      # Modifies a gauge
+      #
+      # @param [String,Symbol] name Name of the gauge to increment
+      # @param [String,Integer] value Step to change by
+      # @param [Hash{Symbol=>String}] tags Tags to apply to this record
+      # @return [void]
+      def set_gauge(name, value, tags = {})
+      end
+
+      # Records a histogram entry
+      #
+      # @param [String,Symbol] name Name of the histogram
+      # @param [String,Integer] value Value to record
+      # @param [Hash{Symbol=>String}] tags Tags to apply to this record
+      # @return [void]
+      def observe_histogram(name, value, tags = {})
+      end
+
+      # Passes self into the block and evaulates it
+      #
+      # @yieldparam [Abstract] adapter the current telemetry adapter
+      # @return [void]
+      def with_instance
+        yield self if block_given?
+      end
+
+      def measure(name, tags = {})
+        before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        yield
+      ensure
+        after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        duration = (after - before) * 1000 # In miliseconds
+        observe_histogram(name, duration, tags)
+      end
+    end
+  end
+end

data/lib/dynflow/telemetry_adapters/dummy.rb

@@ -0,0 +1,18 @@
+module Dynflow
+  module TelemetryAdapters
+    # Telemetry adapter which does not evaluate blocks passed to {#with_instance}.
+    class Dummy < Abstract
+      # Does nothing with the block passed to it
+      #
+      # @return void
+      def with_instance
+        # Do nothing
+      end
+
+      def measure(_name, _tags = {})
+        # Just call the block
+        yield
+      end
+    end
+  end
+end

data/lib/dynflow/telemetry_adapters/statsd.rb

@@ -0,0 +1,48 @@
+module Dynflow
+  module TelemetryAdapters
+    class StatsD < Abstract
+      def initialize(host = '127.0.0.1:8125')
+        require 'statsd-instrument'
+
+        @instances = {}
+        @host = host
+        ::StatsD.backend = ::StatsD::Instrument::Backends::UDPBackend.new(host, :statsd)
+      end
+
+      def add_counter(name, description, instance_labels)
+        raise "Metric already registered: #{name}" if @instances[name]
+        @instances[name] = instance_labels
+      end
+
+      def add_gauge(name, description, instance_labels)
+        raise "Metric already registered: #{name}" if @instances[name]
+        @instances[name] = instance_labels
+      end
+
+      def add_histogram(name, description, instance_labels, buckets = DEFAULT_BUCKETS)
+        raise "Metric already registered: #{name}" if @instances[name]
+        @instances[name] = instance_labels
+      end
+
+      def increment_counter(name, value, tags)
+        ::StatsD.increment(name_tag_mapping(name, tags), value)
+      end
+
+      def set_gauge(name, value, tags)
+        ::StatsD.gauge(name_tag_mapping(name, tags), value)
+      end
+
+      def observe_histogram(name, value, tags)
+        ::StatsD.measure(name_tag_mapping(name, tags), value)
+      end
+
+      private
+
+      def name_tag_mapping(name, tags)
+        instances = @instances[name]
+        return name if instances.nil? || instances.empty?
+        (name.to_s + '.' + instances.map {|x| tags[x]}.compact.join('.')).tr('-:/ ', '____')
+      end
+    end
+  end
+end

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '1.1.0'.freeze
+  VERSION = '1.1.1'.freeze
 end

data/lib/dynflow/world.rb

@@ -14,9 +14,14 @@ module Dynflow
                 :termination_timeout, :terminated, :dead_letter_handler, :execution_plan_cleaner
 
     def initialize(config)
+      @config = Config::ForWorld.new(config, self)
+
+      # Set the telemetry instance as soon as possible
+      Dynflow::Telemetry.set_adapter @config.telemetry_adapter
+      Dynflow::Telemetry.register_metrics!
+
       @id                     = SecureRandom.uuid
       @clock                  = spawn_and_wait(Clock, 'clock')
-      @config                 = Config::ForWorld.new(config, self)
       @logger_adapter         = @config.logger_adapter
       @config.validate
       @transaction_adapter    = @config.transaction_adapter
@@ -238,8 +243,49 @@ module Dynflow
     end
 
     def terminate(future = Concurrent.future)
+      start_termination.tangle(future)
+      future
+    end
+
+    def terminating?
+      defined?(@terminating)
+    end
+
+    # 24119 - ensure delayed executor is preserved after invalidation
+    # executes plans that are planned/paused and haven't reported any error yet (usually when no executor
+    # was available by the time of planning or terminating)
+    def auto_execute
+      coordinator.acquire(Coordinator::AutoExecuteLock.new(self)) do
+        planned_execution_plans =
+            self.persistence.find_execution_plans filters: { 'state' => %w(planned paused), 'result' => (ExecutionPlan.results - [:error]).map(&:to_s) }
+        planned_execution_plans.map do |ep|
+          if coordinator.find_locks(Dynflow::Coordinator::ExecutionLock.unique_filter(ep.id)).empty?
+            execute(ep.id)
+          end
+        end.compact
+      end
+    rescue Coordinator::LockError => e
+      logger.info "auto-executor lock already aquired: #{e.message}"
+      []
+    end
+
+    def try_spawn(what, lock_class = nil)
+      object = nil
+      return nil if !executor || (object = @config.public_send(what)).nil?
+
+      coordinator.acquire(lock_class.new(self)) if lock_class
+      object.spawn.wait
+      object
+    rescue Coordinator::LockError => e
+      nil
+    end
+
+    private
+
+    def start_termination
       @termination_barrier.synchronize do
-        @terminating ||= Concurrent.future do
+        return @terminating if @terminating
+        termination_future ||= Concurrent.future do
           begin
             run_before_termination_hooks
 
@@ -252,7 +298,7 @@ module Dynflow
             throttle_limiter.terminate.wait(termination_timeout)
 
             if executor
-              connector.stop_receiving_new_work(self)
+              connector.stop_receiving_new_work(self, termination_timeout)
 
               logger.info "start terminating executor..."
               executor.terminate.wait(termination_timeout)
@@ -269,7 +315,7 @@ module Dynflow
             client_dispatcher_terminated.wait(termination_timeout)
 
             logger.info "stop listening for new events..."
-            connector.stop_listening(self)
+            connector.stop_listening(self, termination_timeout)
 
             if @clock
               logger.info "start terminating clock..."
@@ -282,49 +328,15 @@ module Dynflow
           rescue => e
             logger.fatal(e)
           end
+        end
+        @terminating = Concurrent.future do
+          termination_future.wait(termination_timeout)
         end.on_completion do
           Thread.new { Kernel.exit } if @exit_on_terminate.true?
         end
       end
-
-      @terminating.tangle(future)
-      future
-    end
-
-    def terminating?
-      defined?(@terminating)
-    end
-
-    # 24119 - ensure delayed executor is preserved after invalidation
-    # executes plans that are planned/paused and haven't reported any error yet (usually when no executor
-    # was available by the time of planning or terminating)
-    def auto_execute
-      coordinator.acquire(Coordinator::AutoExecuteLock.new(self)) do
-        planned_execution_plans =
-            self.persistence.find_execution_plans filters: { 'state' => %w(planned paused), 'result' => (ExecutionPlan.results - [:error]).map(&:to_s) }
-        planned_execution_plans.map do |ep|
-          if coordinator.find_locks(Dynflow::Coordinator::ExecutionLock.unique_filter(ep.id)).empty?
-            execute(ep.id)
-          end
-        end.compact
-      end
-    rescue Coordinator::LockError => e
-      logger.info "auto-executor lock already aquired: #{e.message}"
-      []
-    end
-
-    def try_spawn(what, lock_class = nil)
-      object = nil
-      return nil if !executor || (object = @config.public_send(what)).nil?
-
-      coordinator.acquire(lock_class.new(self)) if lock_class
-      object.spawn.wait
-      object
-    rescue Coordinator::LockError => e
-      nil
     end
 
-    private
     def calculate_subscription_index
       @subscription_index =
           action_classes.each_with_object(Hash.new { |h, k| h[k] = [] }) do |klass, index|
@@ -337,11 +349,14 @@ module Dynflow
 
     def run_before_termination_hooks
       until @before_termination_hooks.empty?
-        begin
-          @before_termination_hooks.pop.call
-        rescue => e
-          logger.error e
+        hook_run = Concurrent.future do
+          begin
+            @before_termination_hooks.pop.call
+          rescue => e
+            logger.error e
+          end
         end
+        logger.error "timeout running before_termination_hook" unless hook_run.wait(termination_timeout)
       end
     end
 

data/test/executor_test.rb

@@ -94,6 +94,7 @@ module Dynflow
         end
 
         describe "when being executed" do
+          include TestHelpers
 
           let :execution_plan do
             world.plan(Support::CodeWorkflowExample::IncomingIssue, { 'text' => 'get a break' })
@@ -128,6 +129,16 @@ module Dynflow
               assert_raises(Dynflow::Error) { world.execute(execution_plan.id).value! }
             end
           end
+
+          it "handles when the execution plan is deleted" do
+            TestPause.when_paused do
+              world.persistence.delete_execution_plans(uuid: [execution_plan.id])
+            end
+            director = get_director(world)
+            wait_for('execution plan removed from executor') do
+              !director.current_execution_plan_ids.include?(execution_plan.id)
+            end
+          end
         end
       end
 

data/test/test_helper.rb

@@ -162,15 +162,21 @@ module TestHelpers
     end
   end
 
+  # get director for deeper investigation of the current execution state
+  def get_director(world)
+    core_context = world.executor.instance_variable_get('@core').instance_variable_get('@core').context
+    core_context.instance_variable_get('@director')
+  end
+
   # waits for the passed block to return non-nil value and reiterates it while getting false
   # (till some reasonable timeout). Useful for forcing the tests for some event to occur
-  def wait_for
+  def wait_for(waiting_message = 'something to happen')
     30.times do
       ret = yield
       return ret if ret
       sleep 0.3
     end
-    raise 'waiting for something to happen was not successful'
+    raise "waiting for #{waiting_message} was not successful"
   end
 
   def executor_id_for_plan(execution_plan_id)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-09 00:00:00.000000000 Z
+date: 2018-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -379,6 +379,7 @@ files:
 - examples/singletons.rb
 - examples/sub_plan_concurrency_control.rb
 - examples/sub_plans.rb
+- extras/statsd_mapping.conf
 - lib/dynflow.rb
 - lib/dynflow/action.rb
 - lib/dynflow/action/cancellable.rb
@@ -408,6 +409,7 @@ files:
 - lib/dynflow/coordinator_adapters/abstract.rb
 - lib/dynflow/coordinator_adapters/sequel.rb
 - lib/dynflow/dead_letter_silencer.rb
+- lib/dynflow/debug/telemetry/persistence.rb
 - lib/dynflow/delayed_executors.rb
 - lib/dynflow/delayed_executors/abstract.rb
 - lib/dynflow/delayed_executors/abstract_core.rb
@@ -500,6 +502,9 @@ files:
 - lib/dynflow/serializers/noop.rb
 - lib/dynflow/stateful.rb
 - lib/dynflow/telemetry.rb
+- lib/dynflow/telemetry_adapters/abstract.rb
+- lib/dynflow/telemetry_adapters/dummy.rb
+- lib/dynflow/telemetry_adapters/statsd.rb
 - lib/dynflow/testing.rb
 - lib/dynflow/testing/assertions.rb
 - lib/dynflow/testing/dummy_execution_plan.rb