dynflow 0.8.1 → 0.8.2

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -55,6 +55,22 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def find_past_scheduled_plans(options = {})
+        raise NotImplementedError
+      end
+
+      def delete_scheduled_plans(filters, batch_size = 1000)
+        raise NotImplementedError
+      end
+
+      def load_scheduled_plan(execution_plan_id)
+        raise NotImplementedError
+      end
+
+      def save_scheduled_plan(execution_plan_id, value)
+        raise NotImplementedError
+      end
+
       def load_step(execution_plan_id, step_id)
         raise NotImplementedError
       end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -31,7 +31,8 @@ module Dynflow
                     action:              %w(caller_execution_plan_id caller_action_id),
                     step:                %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight),
                     envelope:            %w(receiver_id),
-                    coordinator_record:  %w(id owner_id class) }
+                    coordinator_record:  %w(id owner_id class),
+                    scheduled:           %w(execution_plan_uuid start_at start_before args_serializer)}
 
       def initialize(config)
         config = config.dup
@@ -54,7 +55,6 @@ module Dynflow
                                 paginate(table(:execution_plan), options),
                                 options),
                           options[:filters])
-
         data_set.map { |record| load_data(record) }
       end
 
@@ -79,6 +79,33 @@ module Dynflow
         save :execution_plan, { uuid: execution_plan_id }, value
       end
 
+      def delete_scheduled_plans(filters, batch_size = 1000)
+        count = 0
+        filter(:scheduled, table(:scheduled), filters).each_slice(batch_size) do |plans|
+          uuids = plans.map { |p| p.fetch(:execution_plan_uuid) }
+          @db.transaction do
+            count += table(:scheduled).where(execution_plan_uuid: uuids).delete
+          end
+        end
+        count
+      end
+
+      def find_past_scheduled_plans(time)
+        table(:scheduled)
+          .where('start_at <= ?', time)
+          .order_by(:start_at)
+          .all
+          .map { |plan| load_data(plan) }
+      end
+
+      def load_scheduled_plan(execution_plan_id)
+        load :scheduled, execution_plan_uuid: execution_plan_id
+      end
+
+      def save_scheduled_plan(execution_plan_id, value)
+        save :scheduled, { execution_plan_uuid: execution_plan_id }, value
+      end
+
       def load_step(execution_plan_id, step_id)
         load :step, execution_plan_uuid: execution_plan_id, id: step_id
       end
@@ -164,7 +191,8 @@ module Dynflow
                  action:              :dynflow_actions,
                  step:                :dynflow_steps,
                  envelope:            :dynflow_envelopes,
-                 coordinator_record:  :dynflow_coordinator_records }
+                 coordinator_record:  :dynflow_coordinator_records,
+                 scheduled:           :dynflow_scheduled_plans }
 
       def table(which)
         db[TABLES.fetch(which)]

data/lib/dynflow/persistence_adapters/sequel_migrations/006_fix_data_length.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  up do
+    alter_table(:dynflow_steps) do
+      if @db.database_type == :mysql
+        set_column_type :data, :mediumtext
+      end
+    end
+  end
+
+  down do
+    alter_table(:dynflow_steps) do
+      if @db.database_type == :mysql
+        set_column_type :data, :text
+      end
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  change do
+    create_table(:dynflow_scheduled_plans) do
+      foreign_key :execution_plan_uuid, :dynflow_execution_plans, type: String, size: 36, fixed: true
+      index :execution_plan_uuid
+      column :start_at, Time
+      index :start_at
+      column :start_before, Time
+      column :data, String, text: true
+      column :args_serializer, String
+    end
+  end
+end

data/lib/dynflow/scheduled_plan.rb

@@ -0,0 +1,65 @@
+module Dynflow
+  class ScheduledPlan < Serializable
+
+    include Algebrick::TypeCheck
+
+    attr_reader :execution_plan_uuid, :start_at, :start_before, :args
+
+    def initialize(world, execution_plan_uuid, start_at, start_before, args = [], args_serializer = nil)
+      @world               = Type! world, World
+      @execution_plan_uuid = Type! execution_plan_uuid, String
+      @start_at            = Type! start_at, Time
+      @start_before        = Type! start_before, Time, NilClass
+      @args                = Type! args, Array
+      @args_serializer     = Type! args_serializer, Serializers::Abstract
+    end
+
+    def execution_plan
+      @execution_plan ||= @world.persistence.load_execution_plan(@execution_plan_uuid)
+    end
+
+    def plan
+      execution_plan.root_plan_step.load_action
+      execution_plan.generate_action_id
+      execution_plan.generate_step_id
+      execution_plan.plan(*@args)
+    end
+
+    def timeout
+      error("Execution plan could not be started before set time (#{@start_before})", 'timeout')
+    end
+
+    def error(message, history_entry = nil)
+      execution_plan.root_plan_step.state = :error
+      execution_plan.root_plan_step.error = ::Dynflow::ExecutionPlan::Steps::Error.new(message)
+      execution_plan.root_plan_step.save
+      execution_plan.execution_history.add history_entry, @world.id unless history_entry.nil?
+      execution_plan.update_state :stopped
+    end
+
+    def execute
+      @world.execute(execution_plan.id)
+    end
+
+    def to_hash
+       recursive_to_hash :execution_plan_uuid => @execution_plan_uuid,
+                         :start_at            => time_to_str(@start_at),
+                         :start_before        => time_to_str(@start_before),
+                         :args                => @args_serializer.serialize(*@args),
+                         :args_serializer     => @args_serializer.class.name
+    end
+
+    # @api private
+    def self.new_from_hash(world, hash, *args)
+      serializer = hash[:args_serializer].constantize.new
+      self.new(world,
+               hash[:execution_plan_uuid],
+               string_to_time(hash[:start_at]),
+               string_to_time(hash[:start_before]),
+               serializer.deserialize(hash[:args]),
+               serializer)
+    rescue NameError => e
+      error(e.message)
+    end
+  end
+end

data/lib/dynflow/schedulers.rb

@@ -0,0 +1,9 @@
+module Dynflow
+  module Schedulers
+
+    require 'dynflow/schedulers/abstract'
+    require 'dynflow/schedulers/abstract_core'
+    require 'dynflow/schedulers/polling'
+
+  end
+end

data/lib/dynflow/schedulers/abstract.rb

@@ -0,0 +1,37 @@
+module Dynflow
+  module Schedulers
+    class Abstract
+
+      attr_reader :core
+
+      def initialize(world, options = {})
+        @world = world
+        @options = options
+        spawn
+      end
+
+      def start
+        @core.ask(:start)
+      end
+
+      def terminate
+        @core.ask(:terminate!)
+      end
+
+      private
+
+      def core_class
+        raise NotImplementedError
+      end
+
+      def spawn
+        Concurrent.future.tap do |initialized|
+          @core = core_class.spawn name: 'scheduler',
+                                   args: [@world, @options],
+                                   initialized: initialized
+        end
+      end
+
+    end
+  end
+end

data/lib/dynflow/schedulers/abstract_core.rb

@@ -0,0 +1,65 @@
+module Dynflow
+  module Schedulers
+    class AbstractCore < Actor
+
+      include Algebrick::TypeCheck
+      attr_reader :world, :logger
+
+      def initialize(world, options = {})
+        @world = Type! world, World
+        @logger = world.logger
+        configure(options)
+      end
+
+      def start
+        raise NotImplementedError
+      end
+
+      def configure(options)
+        @time_source = options.fetch(:time_source, -> { Time.now.utc })
+      end
+
+      def check_schedule
+        raise NotImplementedError
+      end
+
+      private
+
+      def time
+        @time_source.call()
+      end
+
+      def scheduled_execution_plans(time)
+        with_error_handling([]) do
+          world.persistence.find_past_scheduled_plans(time)
+        end
+      end
+
+      def with_error_handling(error_retval = nil, &block)
+        block.call
+      rescue Exception => e
+        @logger.fatal e.backtrace.join("\n")
+        error_retval
+      end
+
+      def process(scheduled_plans, check_time)
+        processed_plan_uuids = []
+        scheduled_plans.each do |plan|
+          with_error_handling do
+            if !plan.start_before.nil? && plan.start_before < check_time
+              @logger.debug "Failing plan #{plan.execution_plan_uuid}"
+              plan.timeout
+            else
+              @logger.debug "Executing plan #{plan.execution_plan_uuid}"
+              plan.plan
+              plan.execute
+            end
+            processed_plan_uuids << plan.execution_plan_uuid
+          end
+        end
+        world.persistence.delete_scheduled_plans(:execution_plan_uuid => processed_plan_uuids) unless processed_plan_uuids.empty?
+      end
+
+    end
+  end
+end

data/lib/dynflow/schedulers/polling.rb

@@ -0,0 +1,32 @@
+module Dynflow
+  module Schedulers
+    class Polling < Abstract
+
+      def core_class
+        Dynflow::Schedulers::PollingCore
+      end
+
+    end
+
+    class PollingCore < AbstractCore
+      attr_reader :poll_interval
+
+      def configure(options)
+        super(options)
+        @poll_interval = options[:poll_interval]
+      end
+
+      def start
+        check_schedule
+      end
+
+      def check_schedule
+        check_time = time
+        plans = scheduled_execution_plans(check_time)
+        process plans, check_time
+
+        world.clock.ping(self, poll_interval, :check_schedule)
+      end
+    end
+  end
+end

data/lib/dynflow/serializers.rb

@@ -0,0 +1,8 @@
+module Dynflow
+  module Serializers
+
+    require 'dynflow/serializers/abstract'
+    require 'dynflow/serializers/noop'
+
+  end
+end

data/lib/dynflow/serializers/abstract.rb

@@ -0,0 +1,15 @@
+module Dynflow
+  module Serializers
+    class Abstract
+
+      def serialize(*args)
+        raise NotImplementedError
+      end
+
+      def deserialize(serialized_args)
+        raise NotImplementedError
+      end
+
+    end
+  end
+end

data/lib/dynflow/serializers/noop.rb

@@ -0,0 +1,15 @@
+module Dynflow
+  module Serializers
+    class Noop < Abstract
+
+      def serialize(*args)
+        args
+      end
+
+      def deserialize(serialized_args)
+        serialized_args
+      end
+
+    end
+  end
+end

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.1'
+  VERSION = '0.8.2'
 end

data/lib/dynflow/web/console.rb

@@ -37,31 +37,16 @@ module Dynflow
         erb :worlds
       end
 
-      post('/worlds/:id/ping') do |id|
-        timeout = 5
-        ping_response = world.ping(id, timeout).wait
-        if ping_response.failed?
-          response = "failed: #{ping_response.reason.message}"
-          inactive_world_id = id
-        else
-          response = 'pong'
-        end
-        redirect(url "/worlds?notice=#{url_encode(response)}&inactive_world_id=#{inactive_world_id}")
+      post('/worlds/check') do
+        @worlds = world.coordinator.find_worlds
+        @validation_results = world.worlds_validity_check(params[:invalidate])
+        erb :worlds
       end
 
-      post('/worlds/:id/invalidate') do |id|
-        invalidated_world = world.coordinator.find_worlds(false, id: id).first
-        unless invalidated_world
-          response = "World #{id} not found"
-        else
-          begin
-            world.invalidate(invalidated_world)
-            response = "World #{invalidated_world.id} invalidated"
-          rescue => e
-            response = "World invalidation failed: #{e.message}"
-          end
-        end
-        redirect(url "/worlds?notice=#{url_encode(response)}")
+      post('/worlds/:id/check') do |id|
+        @worlds = world.coordinator.find_worlds
+        @validation_results = world.worlds_validity_check(params[:invalidate], id: params[:id])
+        erb :worlds
       end
 
       get('/:id') do |id|

data/lib/dynflow/web/console_helpers.rb

@@ -1,6 +1,14 @@
 module Dynflow
   module Web
     module ConsoleHelpers
+      def validation_result_css_class(result)
+        if result == :valid
+          "success"
+        else
+          "danger"
+        end
+      end
+
       def prettify_value(value)
         YAML.dump(value)
       end
@@ -107,6 +115,8 @@ module Dynflow
           "success"
         when :error
           "important"
+        else
+          "default"
         end
       end