dynflow 0.7.5 → 0.7.6

data/README.md

@@ -1,6 +1,8 @@
 ![Dynflow](doc/images/logo.png)
 =======
 
+![Build](https://travis-ci.org/Dynflow/dynflow.svg?branch=master)
+
 Dynflow [DYN(amic work)FLOW] is a workflow engine
 written in Ruby that allows to:
 

data/dynflow.gemspec

@@ -7,20 +7,22 @@ Gem::Specification.new do |s|
   s.version     = Dynflow::VERSION
   s.authors     = ["Ivan Necas"]
   s.email       = ["inecas@redhat.com"]
-  s.homepage    = "http://github.com/iNecas/dynflow"
+  s.homepage    = "http://github.com/Dynflow/dynflow"
   s.summary     = "DYNamic workFLOW engine"
   s.description = "Generate and executed workflows dynamically based "+
       "on input data and leave it open for others to jump into it as well"
+  s.license     = "MIT"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
 
+  s.required_ruby_version = '>= 1.9.3'
+
   s.add_dependency "activesupport"
   s.add_dependency "multi_json"
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.4.0'
-  s.add_dependency "uuidtools"
 
   s.add_development_dependency "rack-test"
   s.add_development_dependency "minitest"

data/examples/example_helper.rb

@@ -13,8 +13,17 @@ class ExampleHelper
       Dynflow::SimpleWorld.new(options)
     end
 
+    def persistence_conn_string
+      ENV['DB_CONN_STRING'] || 'sqlite:/'
+    end
+
+    def persistence_adapter
+      Dynflow::PersistenceAdapters::Sequel.new persistence_conn_string
+    end
+
     def default_world_options
-      { logger_adapter: logger_adapter }
+      { logger_adapter: logger_adapter,
+        persistence_adapter: persistence_adapter }
     end
 
     def logger_adapter

data/lib/dynflow/action/suspended.rb

@@ -12,7 +12,7 @@ module Dynflow
       @world.event execution_plan_id, step_id, event, future
     end
 
-    def <<(event)
+    def <<(event = nil)
       event event
     end
 

data/lib/dynflow/clock.rb

@@ -43,7 +43,7 @@ module Dynflow
     def ping(who, time, with_what = nil, where = :<<)
       Type! time, Time, Numeric
       time  = Time.now + time if time.is_a? Numeric
-      timer = Timer[who, time, with_what.nil? ? None : Some[Object][with_what], where]
+      timer = Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
       if terminated?
         Thread.new do
           sleep [timer.when - Time.now, 0].max

data/lib/dynflow/errors.rb

@@ -24,5 +24,13 @@ module Dynflow
         "#{self.class.inspect}: #{message}"
       end
     end
+
+    class PersistenceError < StandardError
+      def self.delegate(original_exception)
+        self.new("caused by #{original_exception.class}: #{original_exception.message}").tap do |e|
+          e.set_backtrace original_exception.backtrace
+        end
+      end
+    end
   end
 end

data/lib/dynflow/execution_plan.rb

@@ -1,4 +1,4 @@
-require 'uuidtools'
+require 'securerandom'
 
 module Dynflow
 
@@ -30,7 +30,7 @@ module Dynflow
 
     # all params with default values are part of *private* api
     def initialize(world,
-        id = UUIDTools::UUID.random_create.to_s,
+        id = SecureRandom.uuid,
         state = :pending,
         root_plan_step = nil,
         run_flow = Flows::Concurrence.new([]),

data/lib/dynflow/executors/parallel.rb

@@ -36,8 +36,9 @@ module Dynflow
         variants Work::Step, Work::Event, Work::Finalize
       end
 
-      PoolDone   = Algebrick.type { fields! work: Work }
-      WorkerDone = Algebrick.type { fields! work: Work, worker: Worker }
+      PoolDone       = Algebrick.type { fields! work: Work }
+      PoolTerminated = Algebrick.atom
+      WorkerDone     = Algebrick.type { fields! work: Work, worker: Worker }
 
       def initialize(world, pool_size = 10)
         super(world)

data/lib/dynflow/executors/parallel/core.rb

@@ -25,15 +25,25 @@ module Dynflow
                 end),
                 (on ~Parallel::Event do |event|
                   event(event)
-                end),
+                 end),
+                (on Parallel::PoolTerminated do
+                   finish_termination
+                 end),
                 (on PoolDone.(~any) do |step|
                   update_manager(step)
-                end)
+                 end),
+                (on ~Errors::PersistenceError.to_m do |error|
+                   logger.fatal "PersistenceError in executor: terminating"
+                   logger.fatal error
+                   @world.terminate
+                 end)
+        rescue Errors::PersistenceError => e
+          self << e
         end
 
         def termination
           logger.info 'shutting down Core ...'
-          try_to_terminate
+          @pool << MicroActor::Terminate
         end
 
         # @return false on problem
@@ -85,6 +95,7 @@ module Dynflow
         end
 
         def rescue?(manager)
+          return false if terminating?
           @world.auto_rescue && manager.execution_plan.state == :paused &&
               !@plan_ids_in_rescue.include?(manager.execution_plan.id)
         end
@@ -121,30 +132,43 @@ module Dynflow
         def set_future(manager)
           @plan_ids_in_rescue.delete(manager.execution_plan.id)
           manager.future.resolve manager.execution_plan
-          try_to_terminate
         end
 
 
         def event(event)
           Type! event, Parallel::Event
+          if terminating?
+            raise Dynflow::Error,
+                  "cannot accept event: #{event} core is terminating"
+          end
           execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
           if execution_plan_manager
             feed_pool execution_plan_manager.event(event)
             true
           else
-            logger.warn format('dropping event %s - no manager for %s:%s',
-                               event, event.execution_plan_id, event.step_id)
-            event.result.fail UnprocessableEvent.new(
-                                  "no manager for #{event.execution_plan_id}:#{event.step_id}")
+            raise Dynflow::Error, "no manager for #{event.execution_plan_id}:#{event.step_id}"
           end
+        rescue Dynflow::Error => e
+          event.result.fail e.message
+          raise e
         end
 
-        def try_to_terminate
-          if terminating? && @execution_plan_managers.empty?
-            @pool.ask(Terminate).wait
-            logger.info '... Core terminated.'
-            terminate!
+        def finish_termination
+          unless @execution_plan_managers.empty?
+            logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
+            begin
+              @execution_plan_managers.values.each do |manager|
+                manager.terminate
+              end
+            rescue Errors::PersistenceError
+              logger.error "could not to clean the data properly"
+            end
+            @execution_plan_managers.values.each do |manager|
+              finish_plan(manager.execution_plan.id)
+            end
           end
+          logger.error '... core terminated.'
+          terminate!
         end
       end
     end

data/lib/dynflow/executors/parallel/execution_plan_manager.rb

@@ -71,6 +71,13 @@ module Dynflow
           (!@run_manager || @run_manager.done?) && (!@finalize_manager || @finalize_manager.done?)
         end
 
+        def terminate
+          @running_steps_manager.terminate
+          unless @execution_plan.state == :paused
+            @execution_plan.update_state(:paused)
+          end
+        end
+
         private
 
         def no_work

data/lib/dynflow/executors/parallel/pool.rb

@@ -76,24 +76,34 @@ module Dynflow
 
         def on_message(message)
           match message,
-                ~Work >-> work do
+                (on ~Work do |work|
                   @jobs.add work
                   distribute_jobs
-                end,
-                WorkerDone.(~any, ~any) >-> step, worker do
+                 end),
+                (on WorkerDone.(~any, ~any) do |step, worker|
                   @core << PoolDone[step]
                   @free_workers << worker
                   distribute_jobs
-                end
+                 end),
+                (on Errors::PersistenceError do
+                   @core << message
+                 end)
         end
 
         def termination
-          raise unless @free_workers.size == @pool_size
-          @free_workers.map { |worker| worker.ask(Terminate) }.each(&:wait)
-          super
+          try_to_terminate
+        end
+
+        def try_to_terminate
+          if terminating? && @free_workers.size == @pool_size
+            @free_workers.map { |worker| worker.ask(Terminate) }.each(&:wait)
+            @core << PoolTerminated
+            terminate!
+          end
         end
 
         def distribute_jobs
+          try_to_terminate
           @free_workers.pop << @jobs.pop until @free_workers.empty? || @jobs.empty?
         end
       end

data/lib/dynflow/executors/parallel/running_steps_manager.rb

@@ -13,6 +13,15 @@ module Dynflow
           @events        = WorkQueue.new(Integer, Work)
         end
 
+        def terminate
+          pending_work = @events.clear.values.flatten
+          pending_work.each do |w|
+            if Work::Event === w
+              w.event.result.fail UnprocessableEvent.new("dropping due to termination")
+            end
+          end
+        end
+
         def add(step, work)
           Type! step, ExecutionPlan::Steps::RunStep
           @running_steps[step.id] = step

data/lib/dynflow/executors/parallel/work_queue.rb

@@ -29,6 +29,12 @@ module Dynflow
           !present?(key)
         end
 
+        def clear
+          ret = @stash.dup
+          @stash.clear
+          ret
+        end
+
         def size(key)
           return 0 if empty?(key)
           @stash[key].size

data/lib/dynflow/executors/parallel/worker.rb

@@ -21,9 +21,11 @@ module Dynflow
                 end),
                 (on Work::Finalize.(~any, any) do |sequential_manager|
                   sequential_manager.finalize
-                end)
-          @pool << WorkerDone[work: message, worker: self]
+                 end)
+        rescue Errors::PersistenceError => e
+          @pool << e
         ensure
+          @pool << WorkerDone[work: message, worker: self]
           @transaction_adapter.cleanup
         end
       end

data/lib/dynflow/persistence.rb

@@ -9,6 +9,7 @@ module Dynflow
     def initialize(world, persistence_adapter)
       @world   = world
       @adapter = persistence_adapter
+      @adapter.register_world(world)
     end
 
     def load_action(step)

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -1,6 +1,19 @@
 module Dynflow
   module PersistenceAdapters
     class Abstract
+
+      # The logger is set by the world when used inside it
+      attr_accessor :logger
+
+      def register_world(world)
+        @worlds ||= Set.new
+        @worlds << world
+      end
+
+      def log(level, message)
+        (@worlds.first && @worlds.first.logger).send(level, message)
+      end
+
       def pagination?
         false
       end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -9,6 +9,9 @@ module Dynflow
     class Sequel < Abstract
       include Algebrick::TypeCheck
 
+      MAX_RETRIES = 10
+      RETRY_DELAY = 1
+
       attr_reader :db
 
       def pagination?
@@ -27,8 +30,8 @@ module Dynflow
                     action:         [],
                     step:           %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight) }
 
-      def initialize(db_path)
-        @db = initialize_db db_path
+      def initialize(config)
+        @db = initialize_db config
         migrate_db
       end
 
@@ -94,7 +97,7 @@ module Dynflow
 
       def save(what, condition, value)
         table           = table(what)
-        existing_record = table.first condition
+        existing_record = with_retry { table.first condition }
 
         if value
           value         = value.with_indifferent_access
@@ -105,20 +108,20 @@ module Dynflow
           record.each { |k, v| record[k] = v.to_s if v.is_a? Symbol }
 
           if existing_record
-            table.where(condition).update(record)
+            with_retry { table.where(condition).update(record) }
           else
-            table.insert record
+            with_retry { table.insert record }
           end
 
         else
-          existing_record and table.where(condition).delete
+          existing_record and with_retry { table.where(condition).delete }
         end
         value
       end
 
       def load(what, condition)
         table = table(what)
-        if (record = table.first(condition.symbolize_keys))
+        if (record = with_retry { table.first(condition.symbolize_keys) } )
           HashWithIndifferentAccess.new MultiJson.load(record[:data])
         else
           raise KeyError, "searching: #{what} by: #{condition.inspect}"
@@ -155,6 +158,24 @@ module Dynflow
 
         data_set.where filters.symbolize_keys
       end
+
+      def with_retry
+        attempts = 0
+        begin
+          yield
+        rescue Exception => e
+          attempts += 1
+          log(:error, e)
+          if attempts > MAX_RETRIES
+            log(:error, "The number of MAX_RETRIES exceeded")
+            raise Errors::PersistenceError.delegate(e)
+          else
+            log(:error, "Persistence retry no. #{attempts}")
+            sleep RETRY_DELAY
+            retry
+          end
+        end
+      end
     end
   end
 end

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.7.5'
+  VERSION = '0.7.6'
 end

data/lib/dynflow/world.rb

@@ -14,25 +14,28 @@ module Dynflow
       @executor            = Type! option_val(:executor), Executors::Abstract
       @action_classes      = option_val(:action_classes)
       @auto_rescue         = option_val(:auto_rescue)
+      @exit_on_terminate   = option_val(:exit_on_terminate)
       @middleware          = Middleware::World.new
       calculate_subscription_index
 
       executor.initialized.wait
       @termination_barrier = Mutex.new
+      @clock_barrier       = Mutex.new
 
       transaction_adapter.check self
     end
 
     def default_options
       @default_options ||=
-          { action_classes: Action.all_children,
-            logger_adapter: LoggerAdapters::Simple.new,
-            executor:       -> world { Executors::Parallel.new(world, options[:pool_size]) },
-            auto_rescue:    true }
+          { action_classes:    Action.all_children,
+            logger_adapter:    LoggerAdapters::Simple.new,
+            executor:          -> world { Executors::Parallel.new(world, options[:pool_size]) },
+            exit_on_terminate: true,
+            auto_rescue:       true }
     end
 
     def clock
-      @clock ||= Clock.new(logger)
+      @clock_barrier.synchronize { @clock ||= Clock.new(logger) }
     end
 
     def logger
@@ -127,6 +130,9 @@ module Dynflow
           @clock_terminated    = Future.new
           executor.terminate(@executor_terminated).
               do_then { clock.ask(MicroActor::Terminate, @clock_terminated) }
+          if @exit_on_terminate
+            future.do_then { Kernel.exit }
+          end
         end
       end
       Future.join([@executor_terminated, @clock_terminated], future)

data/test/remote_via_socket_test.rb

@@ -16,6 +16,7 @@ describe 'remote communication' do
     def create_world
       Dynflow::SimpleWorld.new logger_adapter:      logger_adapter,
                                auto_terminate:      false,
+                               exit_on_terminate:   false,
                                persistence_adapter: persistence_adapter
     end
 
@@ -24,6 +25,7 @@ describe 'remote communication' do
           logger_adapter:      logger_adapter,
           auto_terminate:      false,
           persistence_adapter: persistence_adapter,
+          exit_on_terminate:   false,
           executor:            -> remote_world do
             Dynflow::Executors::RemoteViaSocket.new(remote_world, socket_path)
           end)

data/test/test_helper.rb

@@ -116,6 +116,7 @@ module WorldInstance
     options = { pool_size: 5,
                 persistence_adapter: Dynflow::PersistenceAdapters::Sequel.new('sqlite:/'),
                 transaction_adapter: Dynflow::TransactionAdapters::None.new,
+                exit_on_terminate: false,
                 logger_adapter: logger_adapter,
                 auto_rescue: false }.merge(options)
     Dynflow::World.new(options)
@@ -128,6 +129,7 @@ module WorldInstance
     world       = Dynflow::World.new(
         logger_adapter:      logger_adapter,
         auto_terminate:      false,
+        exit_on_terminate:   false,
         persistence_adapter: -> remote_world { world.persistence.adapter },
         transaction_adapter: Dynflow::TransactionAdapters::None.new,
         executor:            -> remote_world do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-10 00:00:00.000000000 Z
+date: 2015-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -75,22 +75,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.4.0
-- !ruby/object:Gem::Dependency
-  name: uuidtools
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
@@ -345,8 +329,9 @@ files:
 - web/views/layout.erb
 - web/views/plan_step.erb
 - web/views/show.erb
-homepage: http://github.com/iNecas/dynflow
-licenses: []
+homepage: http://github.com/Dynflow/dynflow
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -356,7 +341,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: