dynflow 0.8.21 → 0.8.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25cceba5d864e15547744e5a0038c3aa39da72c1
-  data.tar.gz: 6743618a3dec29ce3221c79840983343b9e4371d
+  metadata.gz: 2dca125fdba138da52e0ffaff7b17a979945be2b
+  data.tar.gz: de40a2fcd90aa602ba2db2cc12e28bfc435c750c
 SHA512:
-  metadata.gz: aa59cc73158b14f8dee9dd6a8ac5388559973dd53396608b4f57687eb92263890016ebc0cfd1cc0d3227cc9879de6bbb594fc458f5ae7988eb3cefe8b4ba4fdb
-  data.tar.gz: d2c95326b522da13d3107e517bf7ff276f83dd2904da94175d621b68e9300b976998c3ef8c1a2841cccfa3aa8d064aa20351def789516af4cc0c5fb48ec57e26
+  metadata.gz: 760ace645251b0b91e0454c2794b4546a93504f542fb2f07452fe76eaed1e75326d1aeabcc9394391b12a410064a08e35413a813f88f9acf0ffcef3227309369
+  data.tar.gz: 4b69a6c1590bfff3c96b04dc5da8c76119680482375f8341331984b3a8deb1bdbc08a06b194f56bacc28fc323a6a55a7e8ff6f0d60b0edc7121ae29f101fa4ab

data/.travis.yml

@@ -7,6 +7,8 @@ rvm:
   - "2.0.0"
   - "2.1.5"
   - "2.2.0"
+  - "2.3.1"
+  - "2.4.0"
 
 env:
   global:

data/Gemfile

@@ -8,6 +8,7 @@ end
 
 group :pry do
   gem 'pry'
+  gem 'pry-byebug'
 end
 
 group :postgresql do
@@ -29,3 +30,7 @@ end
 group :lint do
   gem 'rubocop', '0.39.0'
 end
+
+group :memory_watcher do
+  gem 'get_process_mem'
+end

data/examples/memory_limit_watcher.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require_relative 'example_helper'
+
+example_description = <<DESC
+  Memory limit watcher Example
+  ===========================
+
+In this example we are setting a watcher that will terminate our world object
+when process memory consumption exceeds a limit that will be set.
+
+
+DESC
+
+module MemorylimiterExample
+  class SampleAction < Dynflow::Action
+    def plan(memory_to_use)
+      plan_self(number: memory_to_use)
+    end
+
+    def run
+      array = Array.new(input[:number].to_i)
+      puts "[action] allocated #{input[:number]} cells"
+    end
+  end
+end
+
+if $0 == __FILE__
+  puts example_description
+
+  world = ExampleHelper.create_world do |config|
+    config.exit_on_terminate = false
+  end
+
+  world.terminated.on_completion do
+    puts '[world] The world has been terminated'
+  end
+
+  require 'get_process_mem'
+  memory_info_provider = GetProcessMem.new
+  puts '[info] Preparing memory watcher: '
+  require 'dynflow/watchers/memory_consumption_watcher'
+  puts "[info] now the process consumes #{memory_info_provider.bytes} bytes."
+  limit = memory_info_provider.bytes + 500_000
+  puts "[info] Setting memory limit to #{limit} bytes"
+  watcher = Dynflow::Watchers::MemoryConsumptionWatcher.new(world, limit, polling_interval: 1)
+  puts '[info] Small action: '
+  world.trigger(MemorylimiterExample::SampleAction, 10)
+  sleep 2
+  puts "[info] now the process consumes #{memory_info_provider.bytes} bytes."
+  puts '[info] Big action: '
+  world.trigger(MemorylimiterExample::SampleAction, 500_000)
+  sleep 2
+  puts "[info] now the process consumes #{memory_info_provider.bytes} bytes."
+  puts '[info] Small action again - will not execute, the world is not accepting requests'
+  world.trigger(MemorylimiterExample::SampleAction, 500_000)
+  sleep 2
+  puts 'Done'
+end

data/lib/dynflow/action.rb

@@ -1,4 +1,5 @@
 module Dynflow
+  # rubocop:disable Metrics/ClassLength
   class Action < Serializable
 
     OutputReference = ExecutionPlan::OutputReference
@@ -21,6 +22,7 @@ module Dynflow
     require 'dynflow/action/polling'
     require 'dynflow/action/cancellable'
     require 'dynflow/action/with_sub_plans'
+    require 'dynflow/action/with_bulk_sub_plans'
 
     def self.all_children
       children.values.inject(children.values) do |children, child|
@@ -122,6 +124,10 @@ module Dynflow
         raise TypeError, "Wrong phase #{phase}, required #{phases}"
     end
 
+    def label
+      self.class.name
+    end
+
     def input=(hash)
       Type! hash, Hash
       phase! Plan
@@ -543,4 +549,5 @@ module Dynflow
       @trigger.nil?
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/dynflow/action/with_bulk_sub_plans.rb

@@ -0,0 +1,88 @@
+module Dynflow
+  module Action::WithBulkSubPlans
+    include Dynflow::Action::Cancellable
+
+    DEFAULT_BATCH_SIZE = 100
+
+    # Should return a slice of size items starting from item with index from
+    def batch(from, size)
+      raise NotImplementedError
+    end
+
+    PlanNextBatch = Algebrick.atom
+
+    def run(event = nil)
+      if event === PlanNextBatch
+        spawn_plans if can_spawn_next_batch?
+        suspend
+      else
+        super
+      end
+    end
+
+    def initiate
+      output[:planned_count] = 0
+      output[:total_count]   = total_count
+      super
+    end
+
+    def increase_counts(planned, failed)
+      super(planned, failed, false)
+      output[:planned_count] += planned
+    end
+
+    # Should return the expected total count of tasks
+    def total_count
+      raise NotImplementedError
+    end
+
+    # Returns the items in the current batch
+    def current_batch
+      start_position = output[:planned_count]
+      size = start_position + batch_size > total_count ? total_count - start_position : batch_size
+      batch(start_position, size)
+    end
+
+    def batch_size
+      DEFAULT_BATCH_SIZE
+    end
+
+    # The same logic as in Action::WithSubPlans, but calculated using the expected total count
+    def run_progress
+      if counts_set?
+        (output[:success_count] + output[:failed_count]).to_f / total_count
+      else
+        0.1
+      end
+    end
+
+    def spawn_plans
+      super
+    ensure
+      suspended_action << PlanNextBatch
+    end
+
+    def cancel!
+      # Count the not-yet-planned tasks as failed
+      output[:failed_count] += total_count - output[:planned_count]
+      if uses_concurrency_control
+        # Tell the throttle limiter to cancel the tasks its managing
+        world.throttle_limiter.cancel!(execution_plan_id)
+      else
+        # Just stop the tasks which were not started yet
+        sub_plans(:state => 'planned').each { |sub_plan| sub_plan.update_state(:stopped) }
+      end
+      running = sub_plans(:state => 'running')
+      # Pass the cancel event to running sub plans if they can be cancelled
+      running.each { |sub_plan| sub_plan.cancel! if sub_plan.cancellable? }
+      suspend
+    end
+
+    private
+
+    def can_spawn_next_batch?
+      total_count - output[:success_count] - output[:pending_count] - output[:failed_count] > 0
+    end
+
+  end
+end

data/lib/dynflow/action/with_sub_plans.rb

@@ -26,16 +26,16 @@ module Dynflow
     end
 
     def initiate
-      sub_plans = create_sub_plans
-      sub_plans = Array[sub_plans] unless sub_plans.is_a? Array
       if uses_concurrency_control
-        planned, failed = sub_plans.partition { |plan| plan.state == :planned }
-        calculate_time_distribution sub_plans.count
-        sub_plans = world.throttle_limiter.handle_plans!(execution_plan_id,
-                                                         planned.map(&:id),
-                                                         failed.map(&:id),
-                                                         input[:concurrency_control])
+        calculate_time_distribution
+        world.throttle_limiter.initialize_plan(execution_plan_id, input[:concurrency_control])
       end
+      spawn_plans
+    end
+
+    def spawn_plans
+      sub_plans = create_sub_plans
+      sub_plans = Array[sub_plans] unless sub_plans.is_a? Array
       wait_for_sub_plans sub_plans
     end
 
@@ -71,19 +71,26 @@ module Dynflow
     # Helper for creating sub plans
     def trigger(*args)
       if uses_concurrency_control
-        world.plan_with_caller(self, *args)
+        trigger_with_concurrency_control(*args)
       else
         world.trigger { world.plan_with_caller(self, *args) }
       end
     end
 
+    def trigger_with_concurrency_control(*args)
+      record = world.plan_with_caller(self, *args)
+      records = [[record.id], []]
+      records.reverse! unless record.state == :planned
+      @world.throttle_limiter.handle_plans!(execution_plan_id, *records).first
+    end
+
     def limit_concurrency_level(level)
       input[:concurrency_control] ||= {}
       input[:concurrency_control][:level] = ::Dynflow::Semaphores::Stateful.new(level).to_hash
     end
 
-    def calculate_time_distribution(count)
-      time = input[:concurrency_control][:time]
+    def calculate_time_distribution
+      time, count = input[:concurrency_control][:time]
       unless time.nil? || time.is_a?(Hash)
         # Assume concurrency level 1 unless stated otherwise
         level = input[:concurrency_control].fetch(:level, {}).fetch(:free, 1)
@@ -94,25 +101,14 @@ module Dynflow
       end
     end
 
-    def distribute_over_time(time_span)
+    def distribute_over_time(time_span, count)
       input[:concurrency_control] ||= {}
-      input[:concurrency_control][:time] = time_span
+      input[:concurrency_control][:time] = [time_span, count]
     end
 
     def wait_for_sub_plans(sub_plans)
-      output.update(total_count: 0,
-                    failed_count: 0,
-                    success_count: 0,
-                    pending_count: 0)
-
       planned, failed = sub_plans.partition(&:planned?)
-
-      sub_plan_ids = (planned + failed).map(&:execution_plan_id)
-
-      output[:total_count] = sub_plan_ids.size
-      output[:failed_count] = failed.size
-      output[:pending_count] = planned.size
-
+      increase_counts(planned.count, failed.count)
       if planned.any?
         notify_on_finish(planned)
       else
@@ -120,8 +116,16 @@ module Dynflow
       end
     end
 
+    def increase_counts(planned, failed, track_total = true)
+      output[:total_count]   = output.fetch(:total_count, 0) + planned + failed if track_total
+      output[:failed_count]  = output.fetch(:failed_count, 0) + failed
+      output[:pending_count] = output.fetch(:pending_count, 0) + planned
+      output[:success_count] ||= 0
+    end
+
     def try_to_finish
       if done?
+        world.throttle_limiter.finish(execution_plan_id)
         check_for_errors!
         on_finish
         return true
@@ -132,6 +136,8 @@ module Dynflow
 
     def resume
       if sub_plans.all? { |sub_plan| sub_plan.error_in_plan? }
+        # We're starting over and need to reset the counts
+        %w(total failed pending success).each { |key| output.delete("#{key}_count".to_sym) }
         initiate
       else
         recalculate_counts

data/lib/dynflow/active_job/queue_adapter.rb

@@ -28,6 +28,10 @@ module Dynflow
         def run
           input[:job_class].constantize.perform_now(*input[:job_arguments])
         end
+
+        def label
+          input[:job_class]
+        end
       end
     end
   end

data/lib/dynflow/execution_plan.rb

@@ -12,7 +12,8 @@ module Dynflow
     require 'dynflow/execution_plan/output_reference'
     require 'dynflow/execution_plan/dependency_graph'
 
-    attr_reader :id, :world, :root_plan_step, :steps, :run_flow, :finalize_flow,
+    attr_reader :id, :world, :label,
+                :root_plan_step, :steps, :run_flow, :finalize_flow,
                 :started_at, :ended_at, :execution_time, :real_time, :execution_history
 
     def self.states
@@ -36,6 +37,7 @@ module Dynflow
     # all params with default values are part of *private* api
     def initialize(world,
                    id                = SecureRandom.uuid,
+                   label             = nil,
                    state             = :pending,
                    root_plan_step    = nil,
                    run_flow          = Flows::Concurrence.new([]),
@@ -49,6 +51,7 @@ module Dynflow
 
       @id                = Type! id, String
       @world             = Type! world, World
+      @label             = Type! label, String, NilClass
       self.state         = state
       @run_flow          = Type! run_flow, Flows::Abstract
       @finalize_flow     = Type! finalize_flow, Flows::Abstract
@@ -203,7 +206,8 @@ module Dynflow
       update_state(:planning)
       world.middleware.execute(:plan_phase, root_plan_step.action_class, self) do
         with_planning_scope do
-          root_plan_step.execute(self, nil, false, *args)
+          root_action = root_plan_step.execute(self, nil, false, *args)
+          @label = root_action.label
 
           if @dependency_graph.unresolved?
             raise "Some dependencies were not resolved: #{@dependency_graph.inspect}"
@@ -336,9 +340,10 @@ module Dynflow
     end
 
     def to_hash
-      recursive_to_hash id:                self.id,
+      recursive_to_hash id:                id,
                         class:             self.class.to_s,
-                        state:             self.state,
+                        label:             label,
+                        state:             state,
                         result:            result,
                         root_plan_step_id: root_plan_step && root_plan_step.id,
                         run_flow:          run_flow,
@@ -361,6 +366,7 @@ module Dynflow
       steps             = steps_from_hash(hash[:step_ids], execution_plan_id, world)
       self.new(world,
                execution_plan_id,
+               hash[:label],
                hash[:state],
                steps[hash[:root_plan_step_id]],
                Flows::Abstract.from_hash(hash[:run_flow]),

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -27,7 +27,7 @@ module Dynflow
         META_DATA.fetch :execution_plan
       end
 
-      META_DATA = { execution_plan:      %w(state result started_at ended_at real_time execution_time),
+      META_DATA = { execution_plan:      %w(label state result started_at ended_at real_time execution_time),
                     action:              %w(caller_execution_plan_id caller_action_id),
                     step:                %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight),
                     envelope:            %w(receiver_id),
@@ -309,10 +309,11 @@ module Dynflow
       def filter(what, data_set, filters)
         Type! filters, NilClass, Hash
         return data_set if filters.nil?
+        filters = filters.each.with_object({}) { |(k, v), hash| hash[k.to_s] = v }
 
-        unknown = filters.keys.map(&:to_s) - META_DATA.fetch(what)
+        unknown = filters.keys - META_DATA.fetch(what)
         if what == :execution_plan
-          unknown -= %w[uuid caller_execution_plan_id caller_action_id]
+          unknown -= %w[uuid caller_execution_plan_id caller_action_id delayed]
 
           if filters.key?('caller_action_id') && !filters.key?('caller_execution_plan_id')
             raise ArgumentError, "caller_action_id given but caller_execution_plan_id missing"
@@ -322,6 +323,11 @@ module Dynflow
             data_set = data_set.join_table(:inner, TABLES[:action], :execution_plan_uuid => :uuid).
                 select_all(TABLES[:execution_plan]).distinct
           end
+          if filters.key?('delayed')
+            filters.delete('delayed')
+            data_set = data_set.join_table(:inner, TABLES[:delayed], :execution_plan_uuid => :uuid).
+              select_all(TABLES[:execution_plan]).distinct
+          end
         end
 
         unless unknown.empty?

data/lib/dynflow/persistence_adapters/sequel_migrations/010_add_execution_plans_label.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:dynflow_execution_plans) do
+      add_column :label, String
+    end
+  end
+end

data/lib/dynflow/testing/in_thread_world.rb

@@ -52,6 +52,7 @@ module Dynflow
         @executor.terminate
         coordinator.delete_world(registered_world)
         future.success true
+        @terminated.complete
       rescue => e
         future.fail e
       end

data/lib/dynflow/throttle_limiter.rb

@@ -8,6 +8,14 @@ module Dynflow
       spawn
     end
 
+    def initialize_plan(plan_id, semaphores_hash)
+      core.tell([:initialize_plan, plan_id, semaphores_hash])
+    end
+
+    def finish(plan_id)
+      core.tell([:finish, plan_id])
+    end
+
     def handle_plans!(*args)
       core.ask!([:handle_plans, *args])
     end
@@ -44,10 +52,12 @@ module Dynflow
         @semaphores = {}
       end
 
-      def handle_plans(parent_id, planned_ids, failed_ids, semaphores_hash)
-        @semaphores[parent_id] = create_semaphores(semaphores_hash)
-        set_up_clock_for(parent_id, true)
+      def initialize_plan(plan_id, semaphores_hash)
+        @semaphores[plan_id] = create_semaphores(semaphores_hash)
+        set_up_clock_for(plan_id, true)
+      end
 
+      def handle_plans(parent_id, planned_ids, failed_ids)
         failed = failed_ids.map do |plan_id|
           ::Dynflow::World::Triggered[plan_id, Concurrent.future].tap do |triggered|
             execute_triggered(triggered)
@@ -82,7 +92,6 @@ module Dynflow
         if semaphore.has_waiting? && semaphore.get == 1
           execute_triggered(semaphore.get_waiting)
         end
-        @semaphores.delete(plan_id) unless semaphore.has_waiting?
       end
 
       def cancel(parent_id, reason = nil)
@@ -92,10 +101,14 @@ module Dynflow
             cancel_plan_id(triggered.execution_plan_id, reason)
             triggered.future.fail(reason)
           end
-          @semaphores.delete(parent_id)
+          finish(parent_id)
         end
       end
 
+      def finish(parent_id)
+        @semaphores.delete(parent_id)
+      end
+
       private
 
       def cancel_plan_id(plan_id, reason)

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.21'
+  VERSION = '0.8.22'
 end

data/lib/dynflow/watchers/memory_consumption_watcher.rb

@@ -0,0 +1,32 @@
+require 'get_process_mem'
+
+module Dynflow
+  module Watchers
+    class MemoryConsumptionWatcher
+
+      attr_reader :memory_limit, :world
+
+      def initialize(world, memory_limit, options)
+        @memory_limit = memory_limit
+        @world = world
+        @polling_interval = options[:polling_interval] || 60
+        @memory_info_provider = options[:memory_info_provider] || GetProcessMem.new
+        set_timer options[:initial_wait] || @polling_interval
+      end
+
+      def check_memory_state
+        if @memory_info_provider.bytes > @memory_limit
+          # terminate the world and stop polling
+          world.terminate
+        else
+          # memory is under the limit - keep waiting
+          set_timer
+        end
+      end
+
+      def set_timer(interval = @polling_interval)
+        @world.clock.ping(self, interval, nil, :check_memory_state)
+      end
+    end
+  end
+end

data/lib/dynflow/world.rb

@@ -7,7 +7,8 @@ module Dynflow
     attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
                 :transaction_adapter, :logger_adapter, :coordinator,
                 :persistence, :action_classes, :subscription_index,
-                :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter
+                :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter,
+                :terminated
 
     def initialize(config)
       @id                     = SecureRandom.uuid
@@ -30,6 +31,7 @@ module Dynflow
       @auto_validity_check    = config_for_world.auto_validity_check
       @validity_check_timeout = config_for_world.validity_check_timeout
       @throttle_limiter       = config_for_world.throttle_limiter
+      @terminated             = Concurrent.event
       calculate_subscription_index
 
       if executor
@@ -204,7 +206,7 @@ module Dynflow
 
     def terminate(future = Concurrent.future)
       @termination_barrier.synchronize do
-        @terminated ||= Concurrent.future do
+        @terminating ||= Concurrent.future do
           begin
             run_before_termination_hooks
 
@@ -242,6 +244,7 @@ module Dynflow
             end
 
             coordinator.delete_world(registered_world)
+            @terminated.complete
             true
           rescue => e
             logger.fatal(e)
@@ -251,12 +254,12 @@ module Dynflow
         end
       end
 
-      @terminated.tangle(future)
+      @terminating.tangle(future)
       future
     end
 
     def terminating?
-      defined?(@terminated)
+      defined?(@terminating)
     end
 
     # Invalidate another world, that left some data in the runtime,

data/test/batch_sub_tasks_test.rb

@@ -0,0 +1,98 @@
+require_relative 'test_helper'
+
+module Dynflow
+  module BatchSubTaskTest
+    describe 'Batch sub-tasks' do
+      include PlanAssertions
+      include Dynflow::Testing::Assertions
+      include Dynflow::Testing::Factories
+      include TestHelpers
+
+      class FailureSimulator
+        def self.should_fail?
+          @should_fail
+        end
+
+        def self.should_fail!
+          @should_fail = true
+        end
+
+        def self.wont_fail!
+          @should_fail = false
+        end
+      end
+
+      let(:world) { WorldFactory.create_world }
+
+      class ChildAction < ::Dynflow::Action
+        def plan(should_fail = false)
+          raise "Simulated failure" if FailureSimulator.should_fail?
+          plan_self
+        end
+
+        def run
+          output[:run] = true
+        end
+      end
+
+      class ParentAction < ::Dynflow::Action
+        include Dynflow::Action::WithSubPlans
+        include Dynflow::Action::WithBulkSubPlans
+
+        def plan(count, concurrency_level = nil, time_span = nil)
+          limit_concurrency_level(concurrency_level) unless concurrency_level.nil?
+          distribute_over_time(time_span, count) unless time_span.nil?
+          plan_self :count => count
+        end
+
+        def create_sub_plans
+          output[:batch_count] ||= 0
+          output[:batch_count] += 1
+          current_batch.map { |i| trigger(ChildAction) }
+        end
+
+        def batch(from, size)
+          (1..total_count).to_a.slice(from, size)
+        end
+
+        def batch_size
+          5
+        end
+
+        def total_count
+          input[:count]
+        end
+      end
+
+      it 'starts tasks in batches' do
+        FailureSimulator.wont_fail!
+        plan = world.plan(ParentAction, 20)
+        future = world.execute plan.id
+        wait_for { future.completed? }
+        action = plan.entry_action
+
+        action.output[:batch_count].must_equal action.total_count / action.batch_size
+      end
+
+      it 'can resume tasks' do
+        FailureSimulator.should_fail!
+        plan = world.plan(ParentAction, 20)
+        future = world.execute plan.id
+        wait_for { future.completed? }
+        action = plan.entry_action
+        action.output[:batch_count].must_equal 1
+        future.value.state.must_equal :paused
+
+        FailureSimulator.wont_fail!
+        future = world.execute plan.id
+        wait_for { future.completed? }
+        action = future.value.entry_action
+        future.value.state.must_equal :stopped
+        action.output[:batch_count].must_equal (action.total_count / action.batch_size) + 1
+        action.output[:total_count].must_equal action.total_count
+        action.output[:success_count].must_equal action.total_count
+      end
+
+    end
+  end
+end

data/test/concurrency_control_test.rb

@@ -63,7 +63,7 @@ module Dynflow
 
         def plan(count, concurrency_level = nil, time_span = nil, should_sleep = nil)
           limit_concurrency_level(concurrency_level) unless concurrency_level.nil?
-          distribute_over_time(time_span) unless time_span.nil?
+          distribute_over_time(time_span, count) unless time_span.nil?
           plan_self :count => count, :should_sleep => should_sleep
         end
 

data/test/execution_plan_test.rb

@@ -29,6 +29,7 @@ module Dynflow
 
           it 'restores the plan properly' do
             deserialized_execution_plan.id.must_equal execution_plan.id
+            deserialized_execution_plan.label.must_equal execution_plan.label
 
             assert_steps_equal execution_plan.root_plan_step, deserialized_execution_plan.root_plan_step
             assert_equal execution_plan.steps.keys, deserialized_execution_plan.steps.keys
@@ -44,6 +45,20 @@ module Dynflow
 
       end
 
+      describe '#label' do
+        let :execution_plan do
+          world.plan(Support::CodeWorkflowExample::FastCommit, 'sha' => 'abc123')
+        end
+
+        let :dummy_execution_plan do
+          world.plan(Support::CodeWorkflowExample::Dummy)
+        end
+
+        it 'is determined by the action#label method of entry action' do
+          execution_plan.label.must_equal 'Support::CodeWorkflowExample::FastCommit'
+          dummy_execution_plan.label.must_equal 'dummy_action'
+        end
+      end
       describe '#result' do
 
         let :execution_plan do

data/test/memory_cosumption_watcher_test.rb

@@ -0,0 +1,87 @@
+require_relative 'test_helper'
+require 'fileutils'
+require 'dynflow/watchers/memory_consumption_watcher'
+
+module Dynflow
+  module MemoryConsumptionWatcherTest
+    describe ::Dynflow::Watchers::MemoryConsumptionWatcher do
+      let(:world) { Minitest::Mock.new('world') }
+      describe 'initialization' do
+        it 'starts a timer on the world' do
+          clock = Minitest::Mock.new('clock')
+          world.expect(:clock, clock)
+          init_interval = 1000
+          clock.expect(:ping, true) do |clock_who, clock_when, _|
+            clock_when.must_equal init_interval
+          end
+
+          Dynflow::Watchers::MemoryConsumptionWatcher.new world, 1, initial_wait: init_interval
+
+          clock.verify
+        end
+      end
+
+      describe 'polling' do
+        let(:memory_info_provider) { Minitest::Mock.new('memory_info_provider') }
+        it 'continues to poll, if memory limit is not exceeded' do
+          clock = Minitest::Mock.new('clock')
+          # define method clock
+          world.expect(:clock, clock)
+          init_interval = 1000
+          polling_interval = 2000
+          clock.expect(:ping, true) do |clock_who, clock_when, _|
+            clock_when.must_equal init_interval
+            true
+          end
+          clock.expect(:ping, true) do |clock_who, clock_when, _|
+            clock_when.must_equal polling_interval
+            true
+          end
+          memory_info_provider.expect(:bytes, 0)
+
+          # stub the clock method to always return our mock clock
+          world.stub(:clock, clock) do
+            watcher = Dynflow::Watchers::MemoryConsumptionWatcher.new(
+              world,
+              1,
+              initial_wait: init_interval,
+              memory_info_provider: memory_info_provider,
+              polling_interval: polling_interval
+            )
+            watcher.check_memory_state
+          end
+
+          clock.verify
+          memory_info_provider.verify
+        end
+
+        it 'terminates the world, if memory limit reached' do
+          clock = Minitest::Mock.new('clock')
+          # define method clock
+          world.expect(:clock, clock)
+          world.expect(:terminate, true)
+
+          init_interval = 1000
+          clock.expect(:ping, true) do |clock_who, clock_when, _|
+            clock_when.must_equal init_interval
+            true
+          end
+          memory_info_provider.expect(:bytes, 10)
+
+          # stub the clock method to always return our mock clock
+          watcher = Dynflow::Watchers::MemoryConsumptionWatcher.new(
+            world,
+            1,
+            initial_wait: init_interval,
+            memory_info_provider: memory_info_provider
+          )
+          watcher.check_memory_state
+
+          clock.verify
+          memory_info_provider.verify
+          world.verify
+        end
+      end
+    end
+  end
+end

data/test/persistence_test.rb

@@ -1,15 +1,14 @@
 require_relative 'test_helper'
-require 'fileutils'
 
 module Dynflow
   module PersistenceTest
     describe 'persistence adapters' do
 
       let :execution_plans_data do
-        [{ id: 'plan1', state: 'paused' },
-         { id: 'plan2', state: 'stopped' },
-         { id: 'plan3', state: 'paused' },
-         { id: 'plan4', state: 'paused' }]
+        [{ id: 'plan1', :label => 'test1', state: 'paused' },
+         { id: 'plan2', :label => 'test2', state: 'stopped' },
+         { id: 'plan3', :label => 'test3', state: 'paused' },
+         { id: 'plan4', :label => 'test4', state: 'paused' }]
       end
 
       let :action_data do
@@ -37,6 +36,10 @@ module Dynflow
         end
       end
 
+      def format_time(time)
+        time.strftime('%Y-%m-%d %H:%M:%S')
+      end
+
       def prepare_action(plan)
         adapter.save_action(plan, action_data[:id], action_data)
       end
@@ -57,6 +60,7 @@ module Dynflow
         end
       end
 
+      # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       def self.it_acts_as_persistence_adapter
         before do
           # the tests expect clean field
@@ -76,20 +80,22 @@ module Dynflow
 
           it 'supports ordering' do
             prepare_plans
-            if adapter.ordering_by.include?(:state)
+            if adapter.ordering_by.include?('state')
               loaded_plans = adapter.find_execution_plans(order_by: 'state')
-              loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan3', 'plan2']
+              loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan3 plan4 plan2)
 
               loaded_plans = adapter.find_execution_plans(order_by: 'state', desc: true)
-              loaded_plans.map { |h| h[:id] }.must_equal ['plan2', 'plan3', 'plan1']
+              loaded_plans.map { |h| h[:id] }.must_equal %w(plan2 plan1 plan3 plan4)
             end
           end
 
           it 'supports filtering' do
             prepare_plans
-            if adapter.ordering_by.include?(:state)
+            if adapter.ordering_by.include?('state')
+              loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'] })
+              loaded_plans.map { |h| h[:id] }.must_equal ['plan1']
               loaded_plans = adapter.find_execution_plans(filters: { state: ['paused'] })
-              loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan3']
+              loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan3', 'plan4']
 
               loaded_plans = adapter.find_execution_plans(filters: { state: ['stopped'] })
               loaded_plans.map { |h| h[:id] }.must_equal ['plan2']
@@ -98,10 +104,20 @@ module Dynflow
               loaded_plans.map { |h| h[:id] }.must_equal []
 
               loaded_plans = adapter.find_execution_plans(filters: { state: ['stopped', 'paused'] })
-              loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan2', 'plan3']
+              loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan2 plan3 plan4)
 
               loaded_plans = adapter.find_execution_plans(filters: { 'state' => ['stopped', 'paused'] })
-              loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan2', 'plan3']
+              loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan2 plan3 plan4)
+
+              loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'], :delayed => true })
+              loaded_plans.must_be_empty
+
+              adapter.save_delayed_plan('plan1',
+                                        :execution_plan_uuid => 'plan1',
+                                        :start_at => format_time(Time.now + 60),
+                                        :start_before => format_time(Time.now - 60))
+              loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'], :delayed => true })
+              loaded_plans.map { |h| h[:id] }.must_equal ['plan1']
             end
           end
         end
@@ -112,7 +128,7 @@ module Dynflow
             prepare_plans
             adapter.load_execution_plan('plan1')[:id].must_equal 'plan1'
             adapter.load_execution_plan('plan1')['id'].must_equal 'plan1'
-            adapter.load_execution_plan('plan1').keys.size.must_equal 7
+            adapter.load_execution_plan('plan1').keys.size.must_equal 8
 
             adapter.save_execution_plan('plan1', nil)
             -> { adapter.load_execution_plan('plan1') }.must_raise KeyError
@@ -173,11 +189,12 @@ module Dynflow
           it 'finds plans with start_before in past' do
             start_time = Time.now
             prepare_plans
-            fmt =->(time) { time.strftime('%Y-%m-%d %H:%M:%S') }
-            adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :start_at => fmt.call(start_time + 60), :start_before => fmt.call(start_time - 60))
-            adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :start_at => fmt.call(start_time - 60))
-            adapter.save_delayed_plan('plan3', :execution_plan_uuid => 'plan3', :start_at => fmt.call(start_time + 60))
-            adapter.save_delayed_plan('plan4', :execution_plan_uuid => 'plan4', :start_at => fmt.call(start_time - 60), :start_before => fmt.call(start_time - 60))
+            adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :start_at => format_time(start_time + 60),
+                                      :start_before => format_time(start_time - 60))
+            adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :start_at => format_time(start_time - 60))
+            adapter.save_delayed_plan('plan3', :execution_plan_uuid => 'plan3', :start_at => format_time(start_time + 60))
+            adapter.save_delayed_plan('plan4', :execution_plan_uuid => 'plan4', :start_at => format_time(start_time - 60),
+                                      :start_before => format_time(start_time - 60))
             plans = adapter.find_past_delayed_plans(start_time)
             plans.length.must_equal 3
             plans.map { |plan| plan[:execution_plan_uuid] }.must_equal %w(plan2 plan4 plan1)

data/test/support/code_workflow_example.rb

@@ -214,6 +214,9 @@ module Support
     end
 
     class Dummy < Dynflow::Action
+      def label
+        'dummy_action'
+      end
     end
 
     class DummyWithFinalize < Dynflow::Action

data/test/testing_test.rb

@@ -168,7 +168,7 @@ module Dynflow
 
     describe "in thread executor" do
       let :world do
-        Dynflow::Testing::InThreadWorld.instance
+        WorldFactory.create_world(Dynflow::Testing::InThreadWorld)
       end
 
       let :issues_data do

data/test/world_test.rb

@@ -18,6 +18,18 @@ module Dynflow
           registered_world.meta.must_equal('fast' => true)
         end
       end
+
+      describe '#terminate' do
+        it 'fires an event after termination' do
+          terminated_event = world.terminated
+          terminated_event.completed?.must_equal false
+          world.terminate
+          # wait for termination process to finish, but don't block
+          # the test from running.
+          terminated_event.wait(10)
+          terminated_event.completed?.must_equal true
+        end
+      end
     end
   end
 end

data/web/views/index.erb

@@ -9,7 +9,7 @@
   <thead>
     <tr>
       <th><%= order_link(:id, "Id") %></th>
-      <th><%= order_link(:action, "Action") %></th>
+      <th><%= order_link(:label, "Label") %></th>
       <th><%= order_link(:state, "State") %></th>
       <th><%= order_link(:result, "Result") %></th>
       <th><%= order_link(:started_at, "Started at") %></th>
@@ -20,7 +20,7 @@
 <% @plans.each do |plan| %>
   <tr>
     <td><%= h(plan.id) %></td>
-    <td><%= h(plan.root_plan_step.action_class.name) %></td>
+    <td><%= h(plan.label || plan.root_plan_step.action_class.name) %></td>
     <th><%= h(plan.state) %></th>
     <th><%= h(plan.result) %></th>
     <th><%= h(plan.started_at) %></th>

data/web/views/show.erb

@@ -1,3 +1,11 @@
+<p>
+  <b>Id:</b>
+  <%= h(@plan.id) %>
+</p>
+<p>
+  <b>Label:</b>
+  <%= h(@plan.label) %>
+</p>
 <p>
   <b>Status:</b>
   <%= h(@plan.state) %>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 0.8.21
+  version: 0.8.22
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-05 00:00:00.000000000 Z
+date: 2017-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -365,6 +365,7 @@ files:
 - dynflow.gemspec
 - examples/example_helper.rb
 - examples/future_execution.rb
+- examples/memory_limit_watcher.rb
 - examples/orchestrate.rb
 - examples/orchestrate_evented.rb
 - examples/remote_executor.rb
@@ -380,6 +381,7 @@ files:
 - lib/dynflow/action/rescue.rb
 - lib/dynflow/action/suspended.rb
 - lib/dynflow/action/timeouts.rb
+- lib/dynflow/action/with_bulk_sub_plans.rb
 - lib/dynflow/action/with_sub_plans.rb
 - lib/dynflow/active_job/queue_adapter.rb
 - lib/dynflow/actor.rb
@@ -459,6 +461,7 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/010_add_execution_plans_label.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -493,6 +496,7 @@ files:
 - lib/dynflow/transaction_adapters/none.rb
 - lib/dynflow/utils.rb
 - lib/dynflow/version.rb
+- lib/dynflow/watchers/memory_consumption_watcher.rb
 - lib/dynflow/web.rb
 - lib/dynflow/web/console.rb
 - lib/dynflow/web/console_helpers.rb
@@ -503,6 +507,7 @@ files:
 - test/abnormal_states_recovery_test.rb
 - test/action_test.rb
 - test/activejob_adapter.rb
+- test/batch_sub_tasks_test.rb
 - test/clock_test.rb
 - test/concurrency_control_test.rb
 - test/coordinator_test.rb
@@ -510,6 +515,7 @@ files:
 - test/execution_plan_test.rb
 - test/executor_test.rb
 - test/future_execution_test.rb
+- test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
 - test/prepare_travis_env.sh
@@ -581,6 +587,7 @@ test_files:
 - test/abnormal_states_recovery_test.rb
 - test/action_test.rb
 - test/activejob_adapter.rb
+- test/batch_sub_tasks_test.rb
 - test/clock_test.rb
 - test/concurrency_control_test.rb
 - test/coordinator_test.rb
@@ -588,6 +595,7 @@ test_files:
 - test/execution_plan_test.rb
 - test/executor_test.rb
 - test/future_execution_test.rb
+- test/memory_cosumption_watcher_test.rb
 - test/middleware_test.rb
 - test/persistence_test.rb
 - test/prepare_travis_env.sh