dynflow 0.8.35 → 0.8.36

data/lib/dynflow/execution_plan/steps/abstract.rb

@@ -75,19 +75,19 @@ module Dynflow
       end
 
       def to_hash
-        recursive_to_hash execution_plan_id: execution_plan_id,
-                          id:                id,
-                          state:             state,
-                          class:             self.class.to_s,
-                          action_class:      action_class.to_s,
-                          action_id:         action_id,
-                          error:             error,
-                          started_at:        time_to_str(started_at),
-                          ended_at:          time_to_str(ended_at),
-                          execution_time:    execution_time,
-                          real_time:         real_time,
-                          progress_done:     progress_done,
-                          progress_weight:   progress_weight
+        recursive_to_hash execution_plan_uuid: execution_plan_id,
+                          id:                  id,
+                          state:               state,
+                          class:               self.class.to_s,
+                          action_class:        action_class.to_s,
+                          action_id:           action_id,
+                          error:               error,
+                          started_at:          started_at,
+                          ended_at:            ended_at,
+                          execution_time:      execution_time,
+                          real_time:           real_time,
+                          progress_done:       progress_done,
+                          progress_weight:     progress_weight
       end
 
       def progress_done

data/lib/dynflow/persistence.rb

@@ -23,6 +23,13 @@ module Dynflow
       return Action.from_hash(attributes, step.world)
     end
 
+    def load_actions(execution_plan, action_ids)
+      attributes = adapter.load_actions(execution_plan.id, action_ids)
+      attributes.map do |action|
+        Action.from_hash(action.merge(phase: Action::Present, execution_plan: execution_plan), @world)
+      end
+    end
+
     def load_action_for_presentation(execution_plan, action_id, step = nil)
       attributes = adapter.load_action(execution_plan.id, action_id)
       Action.from_hash(attributes.update(phase: Action::Present, execution_plan: execution_plan, step: step), @world).tap do |present_action|
@@ -30,6 +37,10 @@ module Dynflow
       end
     end
 
+    def load_actions_attributes(execution_plan_id, attributes)
+      adapter.load_actions_attributes(execution_plan_id, attributes).reject(&:empty?)
+    end
+
     def save_action(execution_plan_id, action)
       adapter.save_action(execution_plan_id, action.id, action.to_hash)
     end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -91,6 +91,14 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def load_actions_attributes(execution_plan_id, attributes)
+        raise NotImplementedError
+      end
+
+      def load_actions(execution_plan_id, action_ids)
+        raise NotImplementedError
+      end
+
       def save_action(execution_plan_id, action_id, value)
         raise NotImplementedError
       end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -7,6 +7,7 @@ module Dynflow
   module PersistenceAdapters
 
     Sequel.extension :migration
+    Sequel.database_timezone = :utc
 
     class Sequel < Abstract
       include Algebrick::TypeCheck
@@ -29,13 +30,19 @@ module Dynflow
         META_DATA.fetch :execution_plan
       end
 
-      META_DATA = { execution_plan:      %w(label state result started_at ended_at real_time execution_time),
-                    action:              %w(caller_execution_plan_id caller_action_id),
-                    step:                %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight),
+      META_DATA = { execution_plan:      %w(label state result started_at ended_at real_time execution_time root_plan_step_id class),
+                    action:              %w(caller_execution_plan_id caller_action_id class plan_step_id run_step_id finalize_step_id),
+                    step:                %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight
+                                            class action_class execution_plan_uuid),
                     envelope:            %w(receiver_id),
                     coordinator_record:  %w(id owner_id class),
                     delayed:             %w(execution_plan_uuid start_at start_before args_serializer)}
 
+      SERIALIZABLE_COLUMNS = { action:  %w(input output),
+                               delayed: %w(serialized_args),
+                               execution_plan: %w(run_flow finalize_flow execution_history step_ids),
+                               step:    %w(error children) }
+
       def initialize(config)
         config = config.dup
         @additional_responsibilities = { coordinator: true, connector: true }
@@ -51,13 +58,14 @@ module Dynflow
       end
 
       def find_execution_plans(options = {})
+        table_name = :execution_plan
         options[:order_by] ||= :started_at
-        data_set = filter(:execution_plan,
-                          order(:execution_plan,
-                                paginate(table(:execution_plan), options),
+        data_set = filter(table_name,
+                          order(table_name,
+                                paginate(table(table_name), options),
                                 options),
                           options[:filters])
-        data_set.all.map { |record| load_data(record) }
+        data_set.all.map { |record| execution_plan_column_map(load_data(record, table_name)) }
       end
 
       def find_execution_plan_counts(options = {})
@@ -88,11 +96,11 @@ module Dynflow
       end
 
       def load_execution_plan(execution_plan_id)
-        load :execution_plan, uuid: execution_plan_id
+        execution_plan_column_map(load :execution_plan, uuid: execution_plan_id)
       end
 
       def save_execution_plan(execution_plan_id, value)
-        save :execution_plan, { uuid: execution_plan_id }, value
+        save :execution_plan, { uuid: execution_plan_id }, value, false
       end
 
       def delete_delayed_plans(filters, batch_size = 1000)
@@ -107,17 +115,19 @@ module Dynflow
       end
 
       def find_old_execution_plans(age)
-        table(:execution_plan)
+        table_name = :execution_plan
+        table(table_name)
           .where(::Sequel.lit('ended_at <= ? AND state = ?', age, 'stopped'))
-          .all.map { |plan| load_data plan }
+          .all.map { |plan| execution_plan_column_map(load_data plan, table_name) }
       end
 
       def find_past_delayed_plans(time)
-        table(:delayed)
+        table_name = :delayed
+        table(table_name)
           .where(::Sequel.lit('start_at <= ? OR (start_before IS NOT NULL AND start_before <= ?)', time, time))
           .order_by(:start_at)
           .all
-          .map { |plan| load_data(plan) }
+          .map { |plan| load_data(plan, table_name) }
       end
 
       def load_delayed_plan(execution_plan_id)
@@ -127,7 +137,7 @@ module Dynflow
       end
 
       def save_delayed_plan(execution_plan_id, value)
-        save :delayed, { execution_plan_uuid: execution_plan_id }, value
+        save :delayed, { execution_plan_uuid: execution_plan_id }, value, false
       end
 
       def load_step(execution_plan_id, step_id)
@@ -139,15 +149,23 @@ module Dynflow
       end
 
       def save_step(execution_plan_id, step_id, value)
-        save :step, { execution_plan_uuid: execution_plan_id, id: step_id }, value
+        save :step, { execution_plan_uuid: execution_plan_id, id: step_id }, value, false
       end
 
       def load_action(execution_plan_id, action_id)
         load :action, execution_plan_uuid: execution_plan_id, id: action_id
       end
 
+      def load_actions(execution_plan_id, action_ids)
+        load_records :action, { execution_plan_uuid: execution_plan_id, id: action_ids }
+      end
+
+      def load_actions_attributes(execution_plan_id, attributes)
+        load_records :action, { execution_plan_uuid: execution_plan_id }, attributes
+      end
+
       def save_action(execution_plan_id, action_id, value)
-        save :action, { execution_plan_uuid: execution_plan_id, id: action_id }, value
+        save :action, { execution_plan_uuid: execution_plan_id, id: action_id }, value, false
       end
 
       def connector_feature!
@@ -242,22 +260,37 @@ module Dynflow
         ::Sequel::Migrator.run(db, self.class.migrations_path, table: 'dynflow_schema_info')
       end
 
-      def prepare_record(table_name, value, base = {})
+      def prepare_record(table_name, value, base = {}, with_data = true)
         record = base.dup
-        if table(table_name).columns.include?(:data)
+        if with_data && table(table_name).columns.include?(:data)
           record[:data] = dump_data(value)
+        else
+          record[:data] = nil
+          record.merge! serialize_columns(table_name, value)
         end
+
         record.merge! extract_metadata(table_name, value)
         record.each { |k, v| record[k] = v.to_s if v.is_a? Symbol }
+
         record
       end
 
-      def save(what, condition, value)
+      def serialize_columns(table_name, record)
+        record.reduce({}) do |acc, (key, value)|
+          if SERIALIZABLE_COLUMNS.fetch(table_name, []).include?(key.to_s)
+            acc.merge(key.to_sym => dump_data(value))
+          else
+            acc
+          end
+        end
+      end
+
+      def save(what, condition, value, with_data = true)
         table           = table(what)
         existing_record = with_retry { table.first condition } unless condition.empty?
 
         if value
-          record = prepare_record(what, value, (existing_record || condition))
+          record = prepare_record(what, value, (existing_record || condition), with_data)
           if existing_record
             with_retry { table.where(condition).update(record) }
           else
@@ -273,7 +306,7 @@ module Dynflow
       def load_record(what, condition)
         table = table(what)
         if (record = with_retry { table.first(Utils.symbolize_keys(condition)) } )
-          load_data(record)
+          load_data(record, what)
         else
           raise KeyError, "searching: #{what} by: #{condition.inspect}"
         end
@@ -281,14 +314,34 @@ module Dynflow
 
       alias_method :load, :load_record
 
-      def load_records(what, condition)
+      def load_records(what, condition, keys = nil)
         table = table(what)
-        records = with_retry { table.filter(Utils.symbolize_keys(condition)).all }
-        records.map { |record| load_data(record) }
+        records = with_retry do
+          filtered = table.filter(Utils.symbolize_keys(condition))
+          # Filter out requested columns which the table doesn't have, load data just in case
+          filtered = filtered.select(:data, *(table.columns & keys)) unless keys.nil?
+          filtered.all
+        end
+        records = records.map { |record| load_data(record, what) }
+        return records if keys.nil?
+        records.map do |record|
+          keys.reduce({}) do |acc, key|
+            acc.merge(key => record[key])
+          end
+        end
       end
 
-      def load_data(record)
-        Utils.indifferent_hash(MultiJson.load(record[:data]))
+      def load_data(record, what = nil)
+        hash = if record[:data].nil?
+                 SERIALIZABLE_COLUMNS.fetch(what, []).each do |key|
+                   key = key.to_sym
+                   record[key] = MultiJson.load(record[key]) unless record[key].nil?
+                 end
+                 record
+               else
+                 MultiJson.load(record[:data])
+               end
+        Utils.indifferent_hash(hash)
       end
 
       def ensure_backup_dir(backup_dir)
@@ -314,13 +367,14 @@ module Dynflow
       end
 
       def extract_metadata(what, value)
-        meta_keys = META_DATA.fetch(what)
+        meta_keys = META_DATA.fetch(what) - SERIALIZABLE_COLUMNS.fetch(what, [])
         value     = Utils.indifferent_hash(value)
         meta_keys.inject({}) { |h, k| h.update k.to_sym => value[k] }
       end
 
       def dump_data(value)
-        MultiJson.dump Type!(value, Hash)
+        return if value.nil?
+        MultiJson.dump Type!(value, Hash, Array)
       end
 
       def paginate(data_set, options)
@@ -395,6 +449,11 @@ module Dynflow
           end
         end
       end
+
+      def execution_plan_column_map(plan)
+        plan[:id] = plan[:uuid] unless plan[:uuid].nil?
+        plan
+      end
     end
   end
 end

data/lib/dynflow/persistence_adapters/sequel_migrations/011_add_uuid_column.rb

@@ -0,0 +1,61 @@
+def to_uuid(table_name, column_name)
+  set_column_type(table_name, column_name, :uuid, :using => "#{column_name}::uuid")
+end
+
+def from_uuid(table_name, column_name)
+  set_column_type table_name, column_name, String, primary_key: true, size: 36, fixed: true
+end
+
+def with_foreign_key_recreation(&block)
+  # Drop the foreign key constraints so we can change the column type
+  alter_table :dynflow_actions do
+    drop_foreign_key [:execution_plan_uuid]
+  end
+  alter_table :dynflow_steps do
+    drop_foreign_key [:execution_plan_uuid]
+    drop_foreign_key [:execution_plan_uuid, :action_id], :name => :dynflow_steps_execution_plan_uuid_fkey1
+  end
+  alter_table :dynflow_delayed_plans do
+    drop_foreign_key [:execution_plan_uuid]
+  end
+
+  block.call
+
+  # Recreat the foreign key constraints as they were before
+  alter_table :dynflow_actions do
+    add_foreign_key [:execution_plan_uuid], :dynflow_execution_plans
+  end
+  alter_table :dynflow_steps do
+    add_foreign_key [:execution_plan_uuid], :dynflow_execution_plans
+    add_foreign_key [:execution_plan_uuid, :action_id], :dynflow_actions,
+                    :name => :dynflow_steps_execution_plan_uuid_fkey1
+  end
+  alter_table :dynflow_delayed_plans do
+    add_foreign_key [:execution_plan_uuid], :dynflow_execution_plans,
+                    :name => :dynflow_scheduled_plans_execution_plan_uuid_fkey
+  end
+end
+
+Sequel.migration do
+  up do
+    if database_type == :postgresql
+      with_foreign_key_recreation do
+        to_uuid :dynflow_execution_plans, :uuid
+        to_uuid :dynflow_actions,         :execution_plan_uuid
+        to_uuid :dynflow_steps,           :execution_plan_uuid
+        to_uuid :dynflow_delayed_plans,   :execution_plan_uuid
+      end
+    end
+  end
+
+  down do
+    if database_type == :postgresql
+      with_foreign_key_recreation do
+        from_uuid :dynflow_execution_plans, :uuid
+        from_uuid :dynflow_actions,         :execution_plan_uuid
+        from_uuid :dynflow_steps,           :execution_plan_uuid
+        from_uuid :dynflow_delayed_plans,   :execution_plan_uuid
+      end
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/012_add_delayed_plans_serialized_args.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:dynflow_delayed_plans) do
+      long_text_type = @db.database_type == :mysql ? :mediumtext : String
+      add_column :serialized_args, long_text_type
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/013_add_action_columns.rb

@@ -0,0 +1,16 @@
+Sequel.migration do
+  change do
+    alter_table(:dynflow_actions) do
+      long_text_type = @db.database_type == :mysql ? :mediumtext : String
+      add_column :class, String
+      add_column :input, long_text_type
+      add_column :output, long_text_type
+
+      # These could be removed in the future because an action can have at most one of each
+      #   and each belongs to an action
+      add_column :plan_step_id, Integer
+      add_column :run_step_id, Integer
+      add_column :finalize_step_id, Integer
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/014_add_step_columns.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  change do
+    alter_table(:dynflow_steps) do
+      add_column :class, String
+      add_column :error, @db.database_type == :mysql ? :mediumtext : String
+
+      # These could be removed in the future because an action can have at most one of each
+      #   and each belongs to an action
+      add_column :action_class, String
+      add_column :children, String
+    end
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/015_add_execution_plan_columns.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    alter_table(:dynflow_execution_plans) do
+      long_text_type = @db.database_type == :mysql ? :mediumtext : String
+
+      add_column :class, String
+
+      add_column :run_flow, long_text_type
+      add_column :finalize_flow, long_text_type
+      add_column :execution_history, long_text_type
+
+      # These could be removed in the future because an action can have at most one of each
+      #   and each belongs to an action
+      add_column :root_plan_step_id, Integer
+      add_column :step_ids, long_text_type
+    end
+  end
+end

data/lib/dynflow/serializable.rb

@@ -40,7 +40,7 @@ module Dynflow
       if values.size == 1
         value = values.first
         case value
-        when String, Numeric, Symbol, TrueClass, FalseClass, NilClass
+        when String, Numeric, Symbol, TrueClass, FalseClass, NilClass, Time
           value
         when Hash
           value.inject({}) { |h, (k, v)| h.update k => recursive_to_hash(v) }
@@ -57,6 +57,7 @@ module Dynflow
 
     def self.string_to_time(string)
       return if string.nil?
+      return string if string.is_a?(Time)
       _, year, month, day, hour, min, sec =
           */(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/.match(string)
       Time.utc(year.to_i, month.to_i, day.to_i, hour.to_i, min.to_i, sec.to_i)