dynflow 1.3.0 → 1.4.0

data/lib/dynflow/executors/sidekiq/worker_jobs.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+module Dynflow
+  module Executors
+    module Sidekiq
+      module WorkerJobs
+        class PerformWork < InternalJobBase
+          def perform(work_item)
+            with_telemetry(work_item) do
+              Executors.run_user_code do
+                work_item.world = Dynflow.process_world
+                work_item.execute
+              end
+            end
+          rescue Errors::PersistenceError => e
+            OrchestratorJobs::HandlePersistenceError.perform_async(e, work_item)
+          ensure
+            step = work_item.step if work_item.is_a?(Director::StepWorkItem)
+            OrchestratorJobs::WorkerDone.perform_async(work_item, step && step.delayed_events)
+          end
+
+          private
+
+          def with_telemetry(work_item)
+            Dynflow::Telemetry.with_instance { |t| t.set_gauge(:dynflow_active_workers, +1, telemetry_options(work_item)) }
+            yield
+          ensure
+            Dynflow::Telemetry.with_instance do |t|
+              t.increment_counter(:dynflow_worker_events, 1, telemetry_options(work_item))
+              t.set_gauge(:dynflow_active_workers, -1, telemetry_options(work_item))
+            end
+          end
+        end
+
+        class DrainMarker < InternalJobBase
+          def perform(world_id)
+            OrchestratorJobs::StartupComplete.perform_async(world_id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/flows.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'forwardable'
 
 module Dynflow

data/lib/dynflow/flows/abstract.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Flows
 

data/lib/dynflow/flows/abstract_composed.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Flows
     class AbstractComposed < Abstract

data/lib/dynflow/flows/atom.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Flows
     class Atom < Abstract

data/lib/dynflow/flows/concurrence.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Flows
     class Concurrence < AbstractComposed

data/lib/dynflow/flows/sequence.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Flows
     class Sequence < AbstractComposed

data/lib/dynflow/logger_adapters.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module LoggerAdapters
     require 'dynflow/logger_adapters/formatters'

data/lib/dynflow/logger_adapters/abstract.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module LoggerAdapters
     class Abstract

data/lib/dynflow/logger_adapters/delegator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module LoggerAdapters
     class Delegator < Abstract

data/lib/dynflow/logger_adapters/formatters.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module LoggerAdapters
     module Formatters

data/lib/dynflow/logger_adapters/formatters/abstract.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module LoggerAdapters
     module Formatters

data/lib/dynflow/logger_adapters/formatters/exception.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module LoggerAdapters
     module Formatters

data/lib/dynflow/logger_adapters/simple.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'English'
 
 module Dynflow

data/lib/dynflow/middleware.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   class Middleware
     require 'dynflow/middleware/register'

data/lib/dynflow/middleware/common/singleton.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Middleware::Common
     class Singleton < Middleware

data/lib/dynflow/middleware/common/transaction.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Middleware::Common
     class Transaction < Middleware

data/lib/dynflow/middleware/register.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   class Middleware::Register
     include Algebrick::TypeCheck

data/lib/dynflow/middleware/resolver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'tsort'
 
 module Dynflow

data/lib/dynflow/middleware/stack.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   class Middleware::Stack
     include Algebrick::TypeCheck

data/lib/dynflow/middleware/world.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   class Middleware::World
 

data/lib/dynflow/persistence.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'dynflow/persistence_adapters'
 
 module Dynflow
@@ -117,8 +118,8 @@ module Dynflow
       end
     end
 
-    def save_step(step)
-      adapter.save_step(step.execution_plan_id, step.id, step.to_hash)
+    def save_step(step, conditions = {})
+      adapter.save_step(step.execution_plan_id, step.id, step.to_hash, conditions)
     end
 
     def push_envelope(envelope)

data/lib/dynflow/persistence_adapters.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module PersistenceAdapters
     require 'dynflow/persistence_adapters/abstract'

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module PersistenceAdapters
     class Abstract

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sequel'
 require 'multi_json'
 require 'fileutils'
@@ -100,7 +101,7 @@ module Dynflow
       end
 
       def save_execution_plan(execution_plan_id, value)
-        save :execution_plan, { uuid: execution_plan_id }, value, false
+        save :execution_plan, { uuid: execution_plan_id }, value, with_data: false
       end
 
       def delete_delayed_plans(filters, batch_size = 1000)
@@ -138,7 +139,7 @@ module Dynflow
       end
 
       def save_delayed_plan(execution_plan_id, value)
-        save :delayed, { execution_plan_uuid: execution_plan_id }, value, false
+        save :delayed, { execution_plan_uuid: execution_plan_id }, value, with_data: false
       end
 
       def load_step(execution_plan_id, step_id)
@@ -149,8 +150,9 @@ module Dynflow
         load_records :step, execution_plan_uuid: execution_plan_id
       end
 
-      def save_step(execution_plan_id, step_id, value)
-        save :step, { execution_plan_uuid: execution_plan_id, id: step_id }, value, false
+      def save_step(execution_plan_id, step_id, value, update_conditions = {})
+        save :step, { execution_plan_uuid: execution_plan_id, id: step_id }, value,
+             with_data: false, update_conditions: update_conditions
       end
 
       def load_action(execution_plan_id, action_id)
@@ -166,7 +168,7 @@ module Dynflow
       end
 
       def save_action(execution_plan_id, action_id, value)
-        save :action, { execution_plan_uuid: execution_plan_id, id: action_id }, value, false
+        save :action, { execution_plan_uuid: execution_plan_id, id: action_id }, value, with_data: false
       end
 
       def connector_feature!
@@ -286,14 +288,15 @@ module Dynflow
         end
       end
 
-      def save(what, condition, value, with_data = true)
+      def save(what, condition, value, with_data: true, update_conditions: {})
         table           = table(what)
         existing_record = with_retry { table.first condition } unless condition.empty?
 
         if value
           record = prepare_record(what, value, (existing_record || condition), with_data)
           if existing_record
-            with_retry { table.where(condition).update(record) }
+            condition = update_conditions.merge(condition)
+            return with_retry { table.where(condition).update(record) }
           else
             with_retry { table.insert record }
           end

data/lib/dynflow/persistence_adapters/sequel_migrations/001_initial.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   up do
     create_table(:dynflow_execution_plans) do

data/lib/dynflow/persistence_adapters/sequel_migrations/002_incremental_progress.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_steps) do

data/lib/dynflow/persistence_adapters/sequel_migrations/003_parent_action.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_actions) do

data/lib/dynflow/persistence_adapters/sequel_migrations/004_coordinator_records.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     create_table(:dynflow_coordinator_records) do

data/lib/dynflow/persistence_adapters/sequel_migrations/005_envelopes.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     create_table(:dynflow_envelopes) do

data/lib/dynflow/persistence_adapters/sequel_migrations/006_fix_data_length.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   up do
     alter_table(:dynflow_steps) do

data/lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     create_table(:dynflow_scheduled_plans) do

data/lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     rename_table(:dynflow_scheduled_plans, :dynflow_delayed_plans)

data/lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   affected_tables = [:dynflow_actions, :dynflow_coordinator_records, :dynflow_delayed_plans,
                      :dynflow_envelopes, :dynflow_execution_plans]

data/lib/dynflow/persistence_adapters/sequel_migrations/010_add_execution_plans_label.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_execution_plans) do

data/lib/dynflow/persistence_adapters/sequel_migrations/011_placeholder.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   # Placeholder for 011_add_uuid_column.rb - it was readded in
   # 018_add_uuid_column.rb with fixed check for postgresql

data/lib/dynflow/persistence_adapters/sequel_migrations/012_add_delayed_plans_serialized_args.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_delayed_plans) do

data/lib/dynflow/persistence_adapters/sequel_migrations/013_add_action_columns.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_actions) do

data/lib/dynflow/persistence_adapters/sequel_migrations/014_add_step_columns.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_steps) do

data/lib/dynflow/persistence_adapters/sequel_migrations/015_add_execution_plan_columns.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_execution_plans) do

data/lib/dynflow/persistence_adapters/sequel_migrations/016_add_step_queue.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_steps) do

data/lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 Sequel.migration do
   change do
     alter_table(:dynflow_delayed_plans) do

data/lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 def to_uuid(table_name, column_name)
   set_column_type(table_name, column_name, :uuid, :using => "#{column_name}::uuid")
 end

data/lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+Sequel.migration do
+  up do
+    alter_table(:dynflow_execution_plans) do
+      if @db.database_type == :mysql
+        set_column_type :started_at, 'datetime(3)'
+        set_column_type :ended_at, 'datetime(3)'
+      end
+    end
+
+    alter_table(:dynflow_steps) do
+      if @db.database_type == :mysql
+        set_column_type :started_at, 'datetime(3)'
+        set_column_type :ended_at, 'datetime(3)'
+      end
+    end
+
+    alter_table(:dynflow_delayed_plans) do
+      if @db.database_type == :mysql
+        set_column_type :start_at, 'datetime(3)'
+        set_column_type :start_before, 'datetime(3)'
+      end
+    end
+  end
+
+  down do
+    alter_table(:dynflow_steps) do
+      if @db.database_type == :mysql
+        set_column_type :started_at, Time
+        set_column_type :ended_at, Time
+      end
+    end
+
+    alter_table(:dynflow_steps) do
+      if @db.database_type == :mysql
+        set_column_type :started_at, Time
+        set_column_type :ended_at, Time
+      end
+    end
+
+    alter_table(:dynflow_delayed_plans) do
+      if @db.database_type == :mysql
+        set_column_type :start_at, Time
+        set_column_type :start_before, Time
+      end
+    end
+  end
+end

data/lib/dynflow/rails.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# frozen_string_literal: true
 module Dynflow
   # Class for configuring and preparing the Dynflow runtime environment.
   class Rails

data/lib/dynflow/rails/configuration.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'rails'
 require 'active_record'
 
@@ -167,9 +168,11 @@ module Dynflow
         if remote?
           false
         else
-          ::Dynflow::Executors::Parallel.new(world,
-                                             world.config.executor_heartbeat_interval,
-                                             world.config.queues)
+          if defined?(::Sidekiq) && !Sidekiq.options[:dynflow_world].nil?
+            ::Dynflow::Executors::Sidekiq::Core
+          else
+            ::Dynflow::Executors::Parallel::Core
+          end
         end
       end
 

data/lib/dynflow/rails/daemon.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'fileutils'
 require 'get_process_mem'
 require 'dynflow/watchers/memory_consumption_watcher'

data/lib/dynflow/round_robin.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   # A simple round-robin scheduling implementation used at various
   # places in Dynflow