dynflow 0.7.6 → 0.7.7

data/doc/pages/source/images/logo-square.svg

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="156.38499"
+   height="156.38499"
+   viewBox="0 0 156.385 156.38499"
+   id="svg3634"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="logo-square.svg"
+   inkscape:export-filename="/Users/pitr/Workspace/redhat/dynflow/pages/source/images/logo-square.png"
+   inkscape:export-xdpi="73.660004"
+   inkscape:export-ydpi="73.660004">
+  <defs
+     id="defs3636" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="-118.07845"
+     inkscape:cy="174.77927"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:window-width="1920"
+     inkscape:window-height="977"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata3639">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Vrstva 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-895.97718)">
+    <g
+       id="g3650"
+       transform="matrix(0.96003386,0,0,0.96003386,-198.84571,240.90468)">
+      <path
+         inkscape:connector-curvature="0"
+         id="path4180"
+         d="m 273.38806,813.90518 c 10.67131,0 21.34262,0 32.01393,0 2.80651,-14.44298 5.61302,-28.88596 8.41953,-43.32896 14.0093,0 28.0186,0 42.0279,0 1.32345,-6.63958 2.64689,-13.27917 3.97034,-19.91875 -14.03211,0 -28.06422,0 -42.09632,0 1.2318,-6.34315 2.46359,-12.6863 3.69539,-19.02944 14.89948,0 29.79898,0 44.69847,0 1.30064,-6.63959 2.60127,-13.27917 3.90192,-19.91876 -21.72018,0.008 -43.44037,0.0166 -65.16055,0.025 -13.50291,13.09022 -17.93615,33.58442 -13.92119,51.63274 4.1749,18.53456 -3.50427,38.1567 -17.54942,50.53818 z"
+         style="font-style:italic;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:199.62593079px;line-height:521.00000381%;font-family:Sans;-inkscape-font-specification:'Sans Bold Italic';letter-spacing:-15.00960922;word-spacing:0px;fill:#f78a29;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <path
+         inkscape:connector-curvature="0"
+         id="path4182"
+         d="m 302.88929,711.73558 c -8.97951,0.003 -17.95902,0.007 -26.93853,0.0105 -2.53902,13.13095 -5.07803,26.26191 -7.61704,39.39286 -3.94471,-7.6785 -12.21396,-12.70145 -20.86255,-12.52779 -13.12339,-0.65457 -24.86575,7.77854 -31.69186,18.45936 -7.81528,12.04757 -10.96616,27.6448 -6.82946,41.55316 2.90262,9.5693 12.03429,16.63718 22.02555,17.062 9.83511,1.18439 19.6167,-3.36795 25.89418,-10.86051 2.40839,-3.54988 0.50699,1.60166 0.39438,3.15516 -0.40232,1.97496 -0.80464,3.9499 -1.20696,5.92485 4.97717,0 9.95433,0 14.9315,0 6.23473,-4.38616 10.69735,-10.95265 14.26689,-17.58865 5.72598,-10.85952 6.75245,-23.72144 3.82667,-35.55166 -3.03798,-17.2946 1.49379,-36.20142 13.80723,-49.02929 z m -50.96838,43.93282 c 5.88513,-0.46419 11.40435,4.14517 11.87951,10.04989 0.83507,10.60817 -1.53732,22.75724 -10.22572,29.76296 -5.55537,4.22384 -15.26841,4.98105 -19.7752,-1.15819 -3.69118,-6.08712 -2.40972,-13.72962 -1.09341,-20.34581 2.18961,-8.98104 8.97423,-18.5654 19.21482,-18.30885 z"
+         style="font-style:italic;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:199.62593079px;line-height:521.00000381%;font-family:Sans;-inkscape-font-specification:'Sans Bold Italic';letter-spacing:-15.00960922;word-spacing:0px;fill:#289fd5;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    </g>
+  </g>
+</svg>

data/doc/pages/source/index.md

@@ -0,0 +1,64 @@
+---
+layout: page
+title: About
+skip-title: true
+sharing: true
+---
+
+<div id="big_logo" class="well"><img src="/images/logo-long.png"></div>
+
+Dynflow (**DYN**amic work**FLOW**) is a workflow engine
+written in Ruby that allows to:
+
+-   Keep track of the progress of running processes
+-   Run the code asynchronously
+-   When something goes wrong, pause the process, optionally let user interact,
+    resume the process, skip some steps when needed
+-   Detect independent parts and run them concurrently
+-   Compose simple actions into more complex scenarios
+-   Extend the workflows from third-party libraries
+-   Keep consistency between local transactional database and
+    external services
+-   Suspend the long-running steps, not blocking the thread pool
+-   Cancel steps when possible
+-   Extend the actions behavior with middlewares
+-   Pick different adapters to provide: storage backend, transactions, or executor implementation
+
+Dynflow has been developed to be able to support orchestration of services in the
+[Katello](http://katello.org) and [Foreman](http://theforeman.org/) projects.
+
+### Planned features
+
+-   Define the input/output interface between the building blocks
+-   Define rollback for the workflow
+-   Have multiple workers for distributing the load (in progress)
+-   Migration to [concurrent-ruby](http://concurrent-ruby.com) (in progress)
+
+## Getting started
+
+### Requirements
+
+-   Ruby MRI 1.9.3, 2.0, or 2.1.
+-   JRuby and Rubinius support is on the way.
+
+### Installation
+
+`gem install dynflow`
+
+*TODO*
+
+## Links
+
+-   [Github](https://github.com/dynflow/dynflow)
+
+## Current status
+
+![Build](https://img.shields.io/travis/Dynflow/dynflow/master.svg?style=flat)
+![Issues](https://img.shields.io/github/issues/Dynflow/dynflow.svg?style=flat)
+![Gem version](https://img.shields.io/gem/v/dynflow.svg?style=flat)
+![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)
+
+## Authors
+
+-   [Ivan Nečas](https://github.com/iNecas)
+-   [Petr Chalupa](http://blog.pitr.ch)

data/doc/pages/source/media/index.md

@@ -0,0 +1,20 @@
+---
+layout: page
+title: "Media"
+comments: false
+sharing: false
+footer: true
+---
+
+## Presentations
+
+*TODO add* 
+
+## Deep-dives
+
+-   [Foreman deep dive: Rewriting orchestration to Dynflow](https://www.youtube.com/watch?v=fXKgEvoAQpg)
+-   [Foreman deep dive: Rewriting orchestration to Dynflow (part 2)](https://www.youtube.com/watch?v=hOouaJRk3U4)
+-   [MCOflow - MCollective over Dynflow](https://www.youtube.com/watch?v=UyO4R7VUFZ4)
+-   [Katello DynFlow Walkthrough](https://www.youtube.com/watch?v=CEIdXFA5CU0)
+-   [Foreman deep dive: Dynflow tools and concepts (part 1 of 2)](https://www.youtube.com/watch?v=5dprXA7gbSI)
+-   [Foreman deep dive: Dynflow executors (part 2 of 2)](https://www.youtube.com/watch?v=drBY8Z1CcKs)

data/doc/pages/source/projects/index.md

@@ -0,0 +1,32 @@
+---
+layout: page
+title: "Related projects"
+comments: false
+sharing: false
+footer: true
+---
+
+## Used by
+
+-   [Foreman](http://theforeman.org) - lifecycle management tool for
+    physical and virtual servers.
+-   [Katello](http://katello.org) - content management plugin for
+    Foreman: integrates couple of REST services for managing the
+    software updates in the infrastructure.
+-   [Foreman-tasks](https://github.com/iNecas/foreman-tasks) - Foreman
+    plugin providing the tasks management with Dynflow on the back-end.
+-   [Dyntask](https://github.com/iNecas/dyntask) - generic Rails engine
+    providing the tasks management features with Dynflow on the back-end.
+-   [Sysflow](https://github.com/iNecas/sysflow) - set of reusable tools
+    for running system tasks with Dynflow, comes with simple Web-UI for
+    testing it.
+
+## Using
+
+-   [Algebrick](https://github.com/pitr-ch/algebrick) - Typed structs on steroids based on
+    algebraic types and pattern matching.
+-   [concurrent-ruby](http://www.concurrent-ruby.com/) - Modern concurrency tools including agents,
+    futures, promises, thread pools, supervisors, and more. Inspired by Erlang, Clojure, Scala,
+    Go, Java, JavaScript, and classic concurrency patterns.
+
+

data/dynflow.gemspec

@@ -5,12 +5,11 @@ require "dynflow/version"
 Gem::Specification.new do |s|
   s.name        = "dynflow"
   s.version     = Dynflow::VERSION
-  s.authors     = ["Ivan Necas"]
+  s.authors     = ["Ivan Necas", "Petr Chalupa"]
   s.email       = ["inecas@redhat.com"]
   s.homepage    = "http://github.com/Dynflow/dynflow"
   s.summary     = "DYNamic workFLOW engine"
-  s.description = "Generate and executed workflows dynamically based "+
-      "on input data and leave it open for others to jump into it as well"
+  s.description = "Ruby workflow/orchestration engine"
   s.license     = "MIT"
 
   s.files         = `git ls-files`.split("\n")

data/examples/sub_plans.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+example_description = <<DESC
+  Sub Plans Example
+  ===================
+
+  This example shows, how to trigger the execution plans from within a
+  run method of some action and waing for them to finish.
+
+  This is useful, when doing bulk actions, where having one big
+  execution plan would not be effective, or in case all the data are
+  not available by the time of original action planning.
+
+DESC
+
+require_relative 'example_helper'
+require_relative 'orchestrate_evented'
+
+
+class SubPlansExample < Dynflow::Action
+  include Dynflow::Action::WithSubPlans
+
+  def create_sub_plans
+    10.times.map { |i| trigger(OrchestrateEvented::CreateMachine, "host-#{i}", 'web_server') }
+  end
+end
+
+if $0 == __FILE__
+  triggered = ExampleHelper.world.trigger(SubPlansExample)
+  puts example_description
+  puts <<-MSG.gsub(/^.*\|/, '')
+    |  Execution plan #{triggered.id} with sub plans triggered
+    |  You can see the details at http://localhost:4567/#{triggered.id}
+  MSG
+
+  ExampleHelper.run_web_console
+end

data/lib/dynflow/action.rb

@@ -22,6 +22,7 @@ module Dynflow
 
     require 'dynflow/action/polling'
     require 'dynflow/action/cancellable'
+    require 'dynflow/action/with_sub_plans'
 
     def self.all_children
       children.values.inject(children.values) do |children, child|
@@ -83,7 +84,8 @@ module Dynflow
     end
 
     attr_reader :world, :phase, :execution_plan_id, :id, :input,
-                :plan_step_id, :run_step_id, :finalize_step_id
+                :plan_step_id, :run_step_id, :finalize_step_id,
+                :caller_execution_plan_id, :caller_action_id
 
     middleware.use Action::Progress::Calculate
 
@@ -102,6 +104,9 @@ module Dynflow
 
       @execution_plan    = Type!(attributes.fetch(:execution_plan), ExecutionPlan) if phase? Present
 
+      @caller_execution_plan_id  = Type!(attributes.fetch(:caller_execution_plan_id, nil), String, NilClass)
+      @caller_action_id          = Type!(attributes.fetch(:caller_action_id, nil), Integer, NilClass)
+
       getter =-> key, required do
         required ? attributes.fetch(key) : attributes.fetch(key, {})
       end
@@ -140,6 +145,19 @@ module Dynflow
       end
     end
 
+    def caller_action
+      plase! Present
+      return nil if @caller_action_id
+      return @caller_action if @caller_action
+
+      caller_execution_plan = if @caller_execution_plan_id == execution_plan.id
+                                execution_plan
+                              else
+                                world.persistence.load_execution_plan(@caller_execution_plan_id)
+                              end
+      @caller_action = world.persistence.load_action_for_presentation(caller_execution_plan, @caller_action_id)
+    end
+
     def set_plan_context(execution_plan, trigger, from_subscription)
       phase! Plan
       @execution_plan    = Type! execution_plan, ExecutionPlan
@@ -209,13 +227,15 @@ module Dynflow
 
     def to_hash
       recursive_to_hash(
-          { class:             self.class.name,
-            execution_plan_id: execution_plan_id,
-            id:                id,
-            plan_step_id:      plan_step_id,
-            run_step_id:       run_step_id,
-            finalize_step_id:  finalize_step_id,
-            input:             input },
+          { class:                     self.class.name,
+            execution_plan_id:         execution_plan_id,
+            id:                        id,
+            plan_step_id:              plan_step_id,
+            run_step_id:               run_step_id,
+            finalize_step_id:          finalize_step_id,
+            caller_execution_plan_id:  caller_execution_plan_id,
+            caller_action_id:          caller_action_id,
+            input:                     input },
           if phase? Run, Finalize, Present
             { output: output }
           end)

data/lib/dynflow/action/polling.rb

@@ -16,6 +16,7 @@ module Dynflow
       else
         raise "unrecognized event #{event}"
       end
+      done? ? on_finish : suspend_and_ping
     end
 
     def done?
@@ -30,6 +31,9 @@ module Dynflow
       raise NotImplementedError
     end
 
+    def on_finish
+    end
+
     # External task data. It should return nil when the task has not
     # been triggered yet.
     def external_task
@@ -68,7 +72,6 @@ module Dynflow
 
     def initiate_external_action
       self.external_task = invoke_external_task
-      suspend_and_ping unless done?
     end
 
     def resume_external_action
@@ -85,21 +88,19 @@ module Dynflow
       poll_attempts[:total] += 1
       self.external_task = poll_external_task
       poll_attempts[:failed] = 0
-      suspend_and_ping unless done?
     rescue => error
       poll_attempts[:failed] += 1
-      rescue_poll_external_task(error)
+      rescue_external_task(error)
     end
 
     def poll_attempts
       output[:poll_attempts] ||= { total: 0, failed: 0 }
     end
 
-    def rescue_poll_external_task(error)
+    def rescue_external_task(error)
       if poll_attempts[:failed] < poll_max_retries
         action_logger.warn("Polling failed, attempt no. #{poll_attempts[:failed]}, retrying in #{poll_interval}")
         action_logger.warn(error)
-        suspend_and_ping
       else
         raise error
       end

data/lib/dynflow/action/with_sub_plans.rb

@@ -0,0 +1,162 @@
+module Dynflow
+  module Action::WithSubPlans
+    SubPlanFinished = Algebrick.type do
+      fields! :execution_plan_id => String,
+              :success           => type { variants TrueClass, FalseClass }
+    end
+
+    def run(event = nil)
+      match event,
+            (on nil do
+               if output[:total_count]
+                 resume
+               else
+                 initiate
+               end
+             end),
+            (on SubPlanFinished do
+              mark_as_done(event.execution_plan_id, event.success)
+              try_to_finish or suspend
+             end)
+    end
+
+    def initiate
+      sub_plans = create_sub_plans
+      sub_plans = Array[sub_plans] unless sub_plans.is_a? Array
+      wait_for_sub_plans(sub_plans)
+    end
+
+    # @abstract when the logic for the initiation of the subtasks
+    #      is different from the default one.
+    # @returns a triggered task or array of triggered tasks
+    # @example
+    #
+    #        def create_sub_plans
+    #          trigger(MyAction, "Hello")
+    #        end
+    #
+    # @example
+    #
+    #        def create_sub_plans
+    #          [trigger(MyAction, "Hello 1"), trigger(MyAction, "Hello 2")]
+    #        end
+    #
+    def create_sub_plans
+      raise NotImplementedError
+    end
+
+    # @api method to be called after all the sub tasks finished
+    def on_finish
+    end
+
+    # Helper for creating sub plans
+    def trigger(*args)
+      world.trigger do
+        world.plan_with_caller(self, *args)
+      end
+    end
+
+    def wait_for_sub_plans(sub_plans)
+      output.update(total_count: 0,
+                    failed_count: 0,
+                    success_count: 0)
+
+      planned, failed = sub_plans.partition(&:planned?)
+
+      sub_plan_ids = (planned + failed).map(&:execution_plan_id)
+
+      output[:total_count] = sub_plan_ids.size
+      output[:failed_count] = failed.size
+
+      if planned.any?
+        notify_on_finish(planned)
+      else
+        check_for_errors!
+      end
+    end
+
+    def try_to_finish
+      if done?
+        check_for_errors!
+        on_finish
+        return true
+      else
+        return false
+      end
+    end
+
+    def resume
+      if sub_plans.all? { |sub_plan| sub_plan.error_in_plan? }
+        initiate
+      else
+        recalculate_counts
+        try_to_finish or fail "Some sub plans are still not finished"
+      end
+    end
+
+    def sub_plans
+      @sub_plans ||= world.persistence.find_execution_plans(filters: { 'caller_execution_plan_id' => execution_plan_id,
+                                                                       'caller_action_id' => self.id } )
+    end
+
+    def notify_on_finish(plans)
+      suspend do |suspended_action|
+        plans.each do |plan|
+          plan.finished.do_then do |value|
+            suspended_action << SubPlanFinished[plan.execution_plan_id,
+                                                value.result == :success]
+          end
+        end
+      end
+    end
+
+    def mark_as_done(plan_id, success)
+      if success
+        output[:success_count] += 1
+      else
+        output[:failed_count] += 1
+      end
+    end
+
+    def done?
+      if counts_set?
+        output[:total_count] - output[:success_count] - output[:failed_count] <= 0
+      else
+        false
+      end
+    end
+
+    def run_progress
+      if counts_set? && output[:total_count] > 0
+        (output[:success_count] + output[:failed_count]).to_f / output[:total_count]
+      else
+        0.1
+      end
+    end
+
+    def recalculate_counts
+      output.update(total_count: 0,
+                    failed_count: 0,
+                    success_count: 0)
+      sub_plans.each do |sub_plan|
+        output[:total_count] += 1
+        if sub_plan.state == :stopped
+          if sub_plan.error?
+            output[:failed_count] += 1
+          else
+            output[:success_count] += 1
+          end
+        end
+      end
+    end
+
+    def counts_set?
+      output[:total_count] && output[:success_count] && output[:failed_count]
+    end
+
+    def check_for_errors!
+      fail "A sub task failed" if output[:failed_count] > 0
+    end
+
+  end
+end