dynflow 0.1.0 → 0.2.0

data/lib/dynflow/world.rb

@@ -0,0 +1,168 @@
+module Dynflow
+  class World
+    include Algebrick::TypeCheck
+
+    attr_reader :executor, :persistence, :transaction_adapter, :action_classes, :subscription_index,
+                :logger_adapter, :options
+
+    def initialize(options_hash = {})
+      @options             = default_options.merge options_hash
+      @logger_adapter      = Type! option_val(:logger_adapter), LoggerAdapters::Abstract
+      @transaction_adapter = Type! option_val(:transaction_adapter), TransactionAdapters::Abstract
+      persistence_adapter  = Type! option_val(:persistence_adapter), PersistenceAdapters::Abstract
+      @persistence         = Persistence.new(self, persistence_adapter)
+      @executor            = Type! option_val(:executor), Executors::Abstract
+      @action_classes      = option_val(:action_classes)
+      calculate_subscription_index
+
+      executor.initialized.wait
+      @termination_barrier = Mutex.new
+
+      transaction_adapter.check self
+    end
+
+    def default_options
+      @default_options ||=
+          { action_classes: Action.all_children,
+            logger_adapter: LoggerAdapters::Simple.new,
+            executor:       -> world { Executors::Parallel.new(world, options[:pool_size]) } }
+    end
+
+    def clock
+      @clock ||= Clock.new(logger)
+    end
+
+    def logger
+      logger_adapter.dynflow_logger
+    end
+
+    def action_logger
+      logger_adapter.action_logger
+    end
+
+    def subscribed_actions(action_class)
+      @subscription_index.has_key?(action_class) ? @subscription_index[action_class] : []
+    end
+
+    # reload actions classes, intended only for devel
+    def reload!
+      @action_classes.map! { |klass| klass.to_s.constantize }
+      calculate_subscription_index
+    end
+
+    class TriggerResult
+      include Algebrick::TypeCheck
+
+      attr_reader :execution_plan_id, :planned, :finished
+      alias_method :id, :execution_plan_id
+      alias_method :planned?, :planned
+
+      def initialize(execution_plan_id, planned, finished)
+        @execution_plan_id = Type! execution_plan_id, String
+        @planned           = Type! planned, TrueClass, FalseClass
+        @finished          = Type! finished, Future
+      end
+
+      def to_a
+        [execution_plan_id, planned, finished]
+      end
+    end
+
+    # @return [TriggerResult]
+    # blocks until action_class is planned
+    def trigger(action_class, *args)
+      execution_plan = plan(action_class, *args)
+      planned        = execution_plan.state == :planned
+      finished       = if planned
+                         execute(execution_plan.id)
+                       else
+                         Future.new.resolve(execution_plan)
+                       end
+      return TriggerResult.new(execution_plan.id, planned, finished)
+    end
+
+    def event(execution_plan_id, step_id, event, future = Future.new)
+      executor.event execution_plan_id, step_id, event, future
+    end
+
+    def plan(action_class, *args)
+      ExecutionPlan.new(self).tap do |execution_plan|
+        execution_plan.prepare(action_class)
+        execution_plan.plan(*args)
+      end
+    end
+
+    # @return [Future] containing execution_plan when finished
+    # raises when ExecutionPlan is not accepted for execution
+    def execute(execution_plan_id, finished = Future.new)
+      executor.execute execution_plan_id, finished
+    end
+
+    def terminate(future = Future.new)
+      @termination_barrier.synchronize do
+        if @executor_terminated.nil?
+          @executor_terminated = Future.new
+          @clock_terminated    = Future.new
+          executor.terminate(@executor_terminated).
+              do_then { clock.ask(MicroActor::Terminate, @clock_terminated) }
+        end
+      end
+      Future.join([@executor_terminated, @clock_terminated], future)
+    end
+
+    # Detects execution plans that are marked as running but no executor
+    # handles them (probably result of non-standard executor termination)
+    #
+    # The current implementation expects no execution_plan being actually run
+    # by the executor.
+    #
+    # TODO: persist the running executors in the system, so that we can detect
+    # the orphaned execution plans. The register should be managable by the
+    # console, so that the administrator can unregister dead executors when needed.
+    # After the executor is unregistered, the consistency check should be performed
+    # to fix the orphaned plans as well.
+    def consistency_check
+      abnormal_execution_plans = self.persistence.find_execution_plans filters: { 'state' => %w(running planning) }
+      if abnormal_execution_plans.empty?
+        logger.info 'Clean start.'
+      else
+        format_str = '%36s %10s %10s'
+        message    = ['Abnormal execution plans, process was probably killed.',
+                      'Following ExecutionPlans will be set to paused, admin has to fix them manually.',
+                      (format format_str, 'ExecutionPlan', 'state', 'result'),
+                      *(abnormal_execution_plans.map { |ep| format format_str, ep.id, ep.state, ep.result })]
+
+        logger.error message.join("\n")
+
+        abnormal_execution_plans.each do |ep|
+          ep.update_state case ep.state
+                          when :planning
+                            :stopped
+                          when :running
+                            :paused
+                          else
+                            raise
+                          end
+        end
+      end
+    end
+
+    private
+
+    def calculate_subscription_index
+      @subscription_index = action_classes.each_with_object(Hash.new { |h, k| h[k] = [] }) do |klass, index|
+        next unless klass.subscribe
+        Array(klass.subscribe).each { |subscribed_class| index[subscribed_class.to_s.constantize] << klass }
+      end.tap { |o| o.freeze }
+    end
+
+    def option_val(key)
+      val = options.fetch(key)
+      if val.is_a? Proc
+        options[key] = val.call(self)
+      else
+        val
+      end
+    end
+  end
+end

data/test/action_test.rb

@@ -1,25 +1,103 @@
-require 'test_helper'
+require_relative 'test_helper'
+require_relative 'code_workflow_example'
 
 module Dynflow
-  class ActionTest < Action
 
-    output_format do
-      param :id, String
+
+  describe Action::Missing do
+    include WorldInstance
+
+    let :action_data do
+      { class:             'RenamedAction',
+        id:                123,
+        input:             {},
+        execution_plan_id: 123 }
     end
 
-    def run
-      output['id'] = input['name']
+    subject do
+      state_holder = ExecutionPlan::Steps::Abstract.allocate
+      state_holder.set_state :success, true
+      Action.from_hash(action_data, :run_phase, state_holder, world)
     end
 
+    specify { subject.action_class.name.must_equal 'RenamedAction' }
+    specify { assert subject.is_a? Action }
   end
 
-  describe 'running an action' do
+  describe "extending action phase" do
+
+    module TestExtending
+
+      module Extension
+        def new_method
+        end
+      end
 
-    it 'executed the run method storing results to output attribute'do
-      action = ActionTest.new('name' => 'zoo')
-      action.run
-      action.output.must_equal('id' => "zoo")
+      class ExtendedAction < Dynflow::Action
+        def self.phase_modules
+          super.merge(run_phase: [Extension]) { |key, old, new| old + new }.freeze
+        end
+      end
     end
 
+    it "is possible to extend the action just for some phase" do
+      refute TestExtending::ExtendedAction.plan_phase.instance_methods.include?(:new_method)
+      refute Dynflow::Action.run_phase.instance_methods.include?(:new_method)
+      assert TestExtending::ExtendedAction.run_phase.instance_methods.include?(:new_method)
+    end
+  end
+
+
+  describe 'children' do
+    include WorldInstance
+
+    smart_action_class   = Class.new(Dynflow::Action)
+    smarter_action_class = Class.new(smart_action_class)
+
+    specify { refute smart_action_class.phase? }
+    specify { refute smarter_action_class.phase? }
+    specify { assert smarter_action_class.plan_phase.phase? }
+
+    specify { smart_action_class.all_children.must_include smarter_action_class }
+    specify { smart_action_class.all_children.size.must_equal 1 }
+    specify { smart_action_class.all_children.wont_include smarter_action_class.plan_phase }
+    specify { smart_action_class.all_children.wont_include smarter_action_class.run_phase }
+    specify { smart_action_class.all_children.wont_include smarter_action_class.finalize_phase }
+
+    describe 'World#subscribed_actions' do
+      event_action_class      = CodeWorkflowExample::Triage
+      subscribed_action_class = CodeWorkflowExample::NotifyAssignee
+
+      specify { subscribed_action_class.subscribe.must_equal event_action_class }
+      specify { world.subscribed_actions(event_action_class).must_include subscribed_action_class }
+      specify { world.subscribed_actions(event_action_class).size.must_equal 1 }
+    end
+  end
+
+  describe Action::Presenter do
+    include WorldInstance
+
+    let :execution_plan do
+      id, planned, finished = *world.trigger(CodeWorkflowExample::IncomingIssues, issues_data)
+      raise unless planned
+      finished.value
+    end
+
+    let :issues_data do
+      [{ 'author' => 'Peter Smith', 'text' => 'Failing test' },
+       { 'author' => 'John Doe', 'text' => 'Internal server error' }]
+    end
+
+    let :presenter do
+      execution_plan.actions.find do |action|
+        action.is_a? CodeWorkflowExample::IncomingIssues
+      end
+    end
+
+    specify { presenter.action_class.must_equal CodeWorkflowExample::IncomingIssues }
+
+    it 'allows aggregating data from other actions' do
+      presenter.summary.must_equal(assignees: ["John Doe"])
+    end
   end
 end

data/test/clock_test.rb

@@ -0,0 +1,59 @@
+require_relative 'test_helper'
+require 'logger'
+
+clock_class = Dynflow::Clock
+
+describe clock_class do
+
+  let(:clock) { clock_class.new Logger.new($stderr) }
+
+  it 'refuses who without #<< method' do
+    clock.initialized.wait
+    -> { clock.ping Object.new, 0.1, :pong }.must_raise TypeError
+    clock.ping [], 0.1, :pong
+  end
+
+
+  it 'pongs' do
+    q = Queue.new
+    clock.initialized.wait
+    start = Time.now
+
+    clock.ping q, 0.1, o = Object.new
+    assert_equal o, q.pop
+    finish = Time.now
+    assert_in_delta 0.1, finish - start, 0.02
+  end
+
+  it 'pongs on expected times' do
+    q = Queue.new
+    clock.initialized.wait
+    start = Time.now
+
+    clock.ping q, 0.3, :a
+    clock.ping q, 0.1, :b
+    clock.ping q, 0.2, :c
+
+    assert_equal :b, q.pop
+    assert_in_delta 0.1, Time.now - start, 0.02
+    assert_equal :c, q.pop
+    assert_in_delta 0.2, Time.now - start, 0.02
+    assert_equal :a, q.pop
+    assert_in_delta 0.3, Time.now - start, 0.02
+  end
+
+  it 'works under stress' do
+    clock.initialized.wait
+    threads = Array.new(4) do
+      Thread.new do
+        q     = Queue.new
+        times = 20
+        times.times { |i| clock.ping q, rand, i }
+        assert_equal (0...times).to_a, Array.new(times) { q.pop }.sort
+      end
+    end
+    threads.each &:join
+  end
+
+
+end

data/test/code_workflow_example.rb

@@ -0,0 +1,382 @@
+require 'logger'
+
+module Dynflow
+  module CodeWorkflowExample
+
+    class IncomingIssues < Action
+
+      def plan(issues)
+        issues.each do |issue|
+          plan_action(IncomingIssue, issue)
+        end
+        plan_self('issues' => issues)
+      end
+
+      input_format do
+        param :issues, Array do
+          param :author, String
+          param :text, String
+        end
+      end
+
+      def finalize
+        TestExecutionLog.finalize << self
+      end
+
+      def summary
+        triages   = all_actions.find_all do |action|
+          action.is_a? Dynflow::CodeWorkflowExample::Triage
+        end
+        assignees = triages.map do |triage|
+          triage.output[:classification] &&
+              triage.output[:classification][:assignee]
+        end.compact.uniq
+        { assignees: assignees }
+      end
+
+    end
+
+    class Slow < Action
+      def plan(seconds)
+        plan_self interval: seconds
+      end
+
+      def run
+        sleep input[:interval]
+        action_logger.debug 'done with sleeping'
+        $slow_actions_done ||= 0
+        $slow_actions_done +=1
+      end
+    end
+
+    class IncomingIssue < Action
+
+      def plan(issue)
+        plan_self(issue)
+        plan_action(Triage, issue)
+      end
+
+      input_format do
+        param :author, String
+        param :text, String
+      end
+
+    end
+
+    class Triage < Action
+
+      def plan(issue)
+        triage = plan_self(issue)
+        plan_action(UpdateIssue,
+                    author:   triage.input[:author],
+                    text:     triage.input[:text],
+                    assignee: triage.output[:classification][:assignee],
+                    severity: triage.output[:classification][:severity])
+      end
+
+      input_format do
+        param :author, String
+        param :text, String
+      end
+
+      output_format do
+        param :classification, Hash do
+          param :assignee, String
+          param :severity, %w[low medium high]
+        end
+      end
+
+      def run
+        TestExecutionLog.run << self
+        TestPause.pause if input[:text].include? 'get a break'
+        error! 'Trolling detected' if input[:text] == "trolling"
+        self.output[:classification] = { assignee: 'John Doe', severity: 'medium' }
+      end
+
+      def finalize
+        error! 'Trolling detected' if input[:text] == "trolling in finalize"
+        TestExecutionLog.finalize << self
+      end
+
+    end
+
+    class UpdateIssue < Action
+
+      input_format do
+        param :author, String
+        param :text, String
+        param :assignee, String
+        param :severity, %w[low medium high]
+      end
+
+      def run
+      end
+    end
+
+    class NotifyAssignee < Action
+
+      def self.subscribe
+        Triage
+      end
+
+      input_format do
+        param :triage, Triage.output_format
+      end
+
+      def plan(*args)
+        plan_self(:triage => trigger.output)
+      end
+
+      def run
+      end
+
+      def finalize
+        TestExecutionLog.finalize << self
+      end
+    end
+
+    class Commit < Action
+      input_format do
+        param :sha, String
+      end
+
+      def plan(commit, reviews = { 'Morfeus' => true, 'Neo' => true })
+        sequence do
+          ci, review_actions = concurrence do
+            [plan_action(Ci, :commit => commit),
+             reviews.map do |name, result|
+               plan_action(Review, commit, name, result)
+             end]
+          end
+
+          plan_action(Merge,
+                      commit:         commit,
+                      ci_result:      ci.output[:passed],
+                      review_results: review_actions.map { |ra| ra.output[:passed] })
+        end
+      end
+    end
+
+    class FastCommit < Action
+
+      def plan(commit)
+        sequence do
+          ci, review = concurrence do
+            [plan_action(Ci, commit: commit),
+             plan_action(Review, commit, 'Morfeus', true)]
+          end
+
+          plan_action(Merge,
+                      commit:         commit,
+                      ci_result:      ci.output[:passed],
+                      review_results: [review.output[:passed]])
+        end
+      end
+
+      input_format do
+        param :sha, String
+      end
+
+    end
+
+    class Ci < Action
+
+      input_format do
+        param :commit, Commit.input_format
+      end
+
+      output_format do
+        param :passed, :boolean
+      end
+
+      def run
+        output.update passed: true
+      end
+    end
+
+    class Review < Action
+
+      input_format do
+        param :reviewer, String
+        param :commit, Commit.input_format
+      end
+
+      output_format do
+        param :passed, :boolean
+      end
+
+      def plan(commit, reviewer, result = true)
+        plan_self commit: commit, reviewer: reviewer, result: result
+      end
+
+      def run
+        output.update passed: input[:result]
+      end
+    end
+
+    class Merge < Action
+
+      input_format do
+        param :commit, Commit.input_format
+        param :ci_result, Ci.output_format
+        param :review_results, array_of(Review.output_format)
+      end
+
+      def run
+        output.update passed: (input.fetch(:ci_result) && input.fetch(:review_results).all?)
+      end
+    end
+
+    class Dummy < Action
+    end
+
+    class DummyWithFinalize < Action
+      def finalize
+        TestExecutionLog.finalize << self
+      end
+    end
+
+    class DummyTrigger < Action
+    end
+
+    class DummyAnotherTrigger < Action
+    end
+
+    class DummySubscribe < Action
+
+      def self.subscribe
+        DummyTrigger
+      end
+
+      def run
+      end
+
+    end
+
+    class DummyMultiSubscribe < Action
+
+      def self.subscribe
+        [DummyTrigger, DummyAnotherTrigger]
+      end
+
+      def run
+      end
+
+    end
+
+    class CancelableSuspended < Action
+      include Dynflow::Action::CancellablePolling
+
+      Cancel = Dynflow::Action::CancellablePolling::Cancel
+
+      def invoke_external_task
+        { progress: 0 }
+      end
+
+      def poll_external_task
+        progress = external_task[:progress] + 10
+        if progress > 25 && input[:text] =~ /cancel/
+          world.event execution_plan_id, run_step_id, Cancel
+        end
+        { progress: progress }
+      end
+
+      def cancel_external_task
+        if input[:text] !~ /cancel fail/
+          { cancelled: true }
+        else
+          error! 'action cancelled'
+        end
+      end
+
+      def external_task=(external_task_data)
+        self.output.update external_task_data
+      end
+
+      def external_task
+        output
+      end
+
+      def done?
+        external_task[:progress] >= 100
+      end
+
+      def poll_interval
+        0.1
+      end
+
+      def run_progress
+        output[:progress].to_f / 100
+      end
+    end
+
+    class DummySuspended < Action
+      include Action::Polling
+
+      def invoke_external_task
+        error! 'Trolling detected' if input[:text] == 'troll setup'
+        { progress: 0, done: false }
+      end
+
+      def external_task=(external_task_data)
+        self.output.update external_task_data
+      end
+
+      def external_task
+        output
+      end
+
+      def poll_external_task
+        if input[:text] == 'troll progress' && !output[:trolled]
+          output[:trolled] = true
+          error! 'Trolling detected'
+        end
+
+        if input[:text] =~ /pause in progress (\d+)/
+          TestPause.pause if output[:progress] == $1.to_i
+        end
+
+        progress = output[:progress] + 10
+        { progress: progress, done: progress >= 100 }
+      end
+
+      def done?
+        external_task[:progress] >= 100
+      end
+
+      def poll_interval
+        0.001
+      end
+
+      def run_progress
+        output[:progress].to_f / 100
+      end
+    end
+
+    class DummyHeavyProgress < Action
+
+      def plan(input)
+        sequence do
+          plan_self(input)
+          plan_action(DummySuspended, input)
+        end
+      end
+
+      def run
+      end
+
+      def finalize
+        $dummy_heavy_progress = 'dummy_heavy_progress'
+      end
+
+      def run_progress_weight
+        4
+      end
+
+      def finalize_progress_weight
+        5
+      end
+    end
+
+  end
+end