cloud-toaster 1.1.2

data/lib/toaster/model/automation_run.rb

@@ -0,0 +1,104 @@
+
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "active_record"
+require "toaster/model/automation"
+require "toaster/model/task_execution"
+require "toaster/model/run_attribute"
+require "toaster/state/system_state"
+require "toaster/util/timestamp"
+require "toaster/util/util"
+
+module Toaster
+  class AutomationRun < ActiveRecord::Base
+
+    @@current_run = nil
+
+    belongs_to :user
+    belongs_to :automation
+    has_many :task_executions, nil, {:autosave => true, :dependent => :delete_all}
+    has_many :run_attributes, nil, {:autosave => true, :dependent => :delete_all}
+
+    # FIELDS: 
+    #  :uuid, :machine_id, :automation, :start_time, 
+    #  :end_time, :success, :error_details, :attributes
+
+    def initialize(attr_hash)
+      if !attr_hash[:uuid]
+        attr_hash[:uuid] = Util.generate_short_uid()
+      end
+      if !attr_hash[:start_time]
+        attr_hash[:start_time] = TimeStamp.now.to_i
+      end
+      if !attr_hash[:machine_id]
+        attr_hash[:machine_id] = Util.get_machine_id()
+      end
+      super(attr_hash)
+    end
+  
+    # used in the context of a currently active run
+    def self.get_current
+      return @@current_run
+    end
+    # used in the context of a currently active run
+    def self.set_current(run)
+      run.save
+      @@current_run = run
+    end
+
+    def get_flat_attributes()
+      KeyValuePair.get_as_hash(run_attributes)
+    end
+
+    def get_executed_tasks()
+      result = []
+      task_executions.each do |exec|
+        result << exec.task
+      end
+      return result
+    end
+
+    def get_num_task_executions()
+      return task_executions.size
+    end
+
+    def get_task_execution(task)
+      task_executions().each do |e|
+        if e.task.uuid == task.uuid
+          return e
+        end
+      end
+    end
+
+    def task_execution_index(task_exec)
+      task_executions().each_with_index do |e,idx|
+        if e.uuid == task_exec.uuid
+          return idx
+        end
+      end
+      return nil
+    end
+
+    def duration
+      return nil if !end_time || !start_time
+      return end_time - start_time
+    end
+
+    def self.load(id, automation = nil)
+      id = DB.instance.wrap_db_id(id)
+      preset_fields = {}
+      preset_fields["automation"] = automation if automation
+      runs = find({"_id" => id}, preset_fields)
+      return nil if !runs || runs.empty?
+      return runs[0]
+    end
+
+    def self.find(criteria={})
+      DB.find_activerecord(AutomationRun, criteria)
+    end
+
+  end
+end

data/lib/toaster/model/ignore_property.rb

@@ -0,0 +1,11 @@
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/model/key_value_pair"
+
+module Toaster
+  class IgnoreProperty < KeyValuePair
+
+  end
+end

data/lib/toaster/model/key_value_pair.rb

@@ -0,0 +1,59 @@
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "active_record"
+
+module Toaster
+  class KeyValuePair < ActiveRecord::Base
+    self.inheritance_column = :type
+
+    serialize :value, JSON
+
+    def initialize(hash)
+      if !hash[:type]
+        type = IgnoreProperty.to_s
+      end
+      super(hash)
+      @attributes_cache = {} if !@attributes_cache # fixes bug in active_record v4.1
+    end
+
+    def self.get_as_hash(list)
+      attrs = {}
+      list.each do |a|
+        attrs[a.key] = a.value
+      end
+      return attrs
+    end
+
+    def self.from_hash(hash, clazz=KeyValuePair)
+      result = []
+      return result if !hash
+      hash.each { |key,value|
+        result << clazz.new(
+          :key => key,
+          :value => value
+        )
+      }
+      return result
+    end
+    
+    def to_s
+      return "#{self.class}(#{key}=#{value})"
+    end
+
+    def hash
+      begin
+        return id if id
+      rescue
+      end
+      h = 0
+      h += key.hash rescue 0
+      h += value.hash rescue 0
+      h += type.hash rescue 0
+      h += data_type.hash rescue 0
+      return h
+    end
+    
+  end
+end

data/lib/toaster/model/run_attribute.rb

@@ -0,0 +1,17 @@
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/model/key_value_pair"
+
+module Toaster
+  class RunAttribute < KeyValuePair
+
+    belongs_to :automation_run
+
+    def self.from_hash(hash)
+      super(hash, RunAttribute)
+    end
+
+  end
+end

data/lib/toaster/model/state.rb

@@ -0,0 +1,17 @@
+
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+module Toaster
+
+  class State
+    attr_accessor :properties
+  end
+
+  class StateProperty
+    attr_accessor :key, :value
+  end
+
+end

data/lib/toaster/model/state_change.rb

@@ -0,0 +1,79 @@
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "active_record"
+
+module Toaster
+
+  class StateChange < ActiveRecord::Base
+    # attr_accessor :property, :action, :value, :old_value, :task_execution
+
+    ACTION_INSERT = "insert"
+    ACTION_MODIFY = "modify"
+    ACTION_DELETE = "delete"
+    ACTION_UNKNOWN = "unknown"
+    VALUE_UNKNOWN = nil
+
+    belongs_to :task_execution
+
+    def initialize1(property=nil, action=nil, value=nil, old_value=nil)
+      @property = property
+      @action = action
+      @value = value
+      @old_value = old_value
+      @task_execution = nil
+    end
+
+    def to_s
+      if action == ACTION_MODIFY
+        return "Modification of property '#{property}': '#{old_value}' --> '#{value}'"
+      elsif action == ACTION_DELETE
+        return "Deletion of property '#{property}'"
+      elsif action == ACTION_INSERT
+        return "Insertion of property '#{property}': '#{value}'"
+      end
+      return "Action '#{@action}' on property '#{property}', value='#{value}', old_value='#{old_value}'"
+    end
+
+    def is_delete?()
+      return action == ACTION_DELETE
+    end
+    def is_modify?()
+      return action == ACTION_MODIFY
+    end
+    def is_insert?()
+      return action == ACTION_INSERT
+    end
+
+    def self.from_hash_array(prop_change_array)
+      prop_changes = []
+      prop_change_array.each do |pc|
+        prop_changes << from_hash(pc)
+      end
+      return prop_changes
+    end
+
+    def ==(obj)
+      return eql?(obj)
+    end
+
+    def eql?(obj)
+      return obj.kind_of?(StateChange) && 
+        obj.property == @property && obj.action == @action && 
+        obj.value == @value && obj.old_value == @old_value
+    end
+
+    def hash
+      h = 0
+      h += @property.hash if @property
+      h += @value.hash if @value
+      h += @action.hash if @action
+      h += @old_value.hash if @old_value
+      return h
+    end
+
+  end
+
+end

data/lib/toaster/model/task.rb

@@ -0,0 +1,292 @@
+
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "active_record"
+require "toaster/util/util"
+require "toaster/model/task_execution"
+require "toaster/model/task_parameter"
+require "toaster/state/state_transition"
+require "toaster/state/system_state"
+require "toaster/chef/resource_inspector"
+
+module Toaster
+  class Task < ActiveRecord::Base
+
+    has_many :task_parameters
+    has_many :task_executions
+    belongs_to :automation
+
+    attr_accessor :resource_obj
+
+    def initialize(attr_hash)
+      if !attr_hash[:uuid]
+        attr_hash[:uuid] = Util.generate_short_uid()
+      end
+      super(attr_hash)
+    end
+  
+    def initialize1(resource, action, sourcecode, uuid = nil)
+      @db_type = "automation_task"
+      @resource = resource ? resource.to_s : nil
+      @resource_obj = resource
+      @action = action ? action.to_s : nil
+      @uuid = uuid ? uuid : Util.generate_short_uid()
+      @sourcecode = sourcecode
+      @sourcehash = sourcecode ? Util.md5(sourcecode) : nil
+      @sourcefile = nil
+      @sourceline = 0
+      @parameters = []
+      @executions_cache = []
+    end
+
+    #
+    # Return an array of TaskExecution instances for this task.
+    #
+    def global_executions(criteria={})
+      criteria[:task] = self
+      return TaskExecution.find(criteria)
+    end
+
+    def global_num_executions()
+      return global_executions().size
+    end
+    def global_successful_executions()
+      return global_executions(:success => true)
+    end
+    def global_num_successful_executions()
+      return global_successful_executions().size
+    end
+    def global_success_rate()
+      return global_num_successful_executions().to_f / global_num_executions().to_f
+    end
+    def global_success_percentage()
+      return global_num_successful_executions().to_f / global_num_executions().to_f * 100.0
+    end
+
+    def set_executions(execs)
+      @executions_cache = execs
+    end
+
+    #
+    # Return an array of lists of StateChange.
+    # Length of the array is the number of global executions of this task.
+    # For each task execution, the array contains a list with all state
+    # changes caused by this execution.
+    # 
+    def global_state_prop_changes(global_execs = nil)
+      return StateChange.joins(:task_execution => :task).where(
+        "task_executions.task_id" => self.id)
+    end
+    def global_num_state_prop_changes(global_executions = nil)
+      return global_state_prop_changes(global_executions).size
+    end
+
+    #
+    # Return a map StateChange -> Integer .
+    # This method maps each state property change to the number
+    # of occurrences, summed up for all executions of this task.
+    #
+    def global_state_prop_changes_map(global_execs = nil)
+      result = {}
+      global_state_prop_changes(global_execs).each do |c|
+        result[c] = 0 if !result[c]
+        result[c] += 1
+      end
+      return result
+    end
+
+    #
+    # Return a set of StateTransition objects for this task. 
+    # A StateTransition basically represents a triple (pre-state, parameters, post-state).
+    # * pre-state and post-state are maps of state property values {prop_key=>value}.
+    # * parameters is a map of input parameter values {param_name=>value}.
+    #
+    # The result set contains all state transitions that were recorded in
+    # any of the executions of this task.
+    # 
+    # This method is the basis for construction of the state transition graph.
+    # 
+    def global_state_transitions(global_execs=nil, insert_nil_prestate_prop_for_insertions=false)
+      result = Set.new
+      global_execs = global_executions() if !global_execs
+      global_execs.each do |exe|
+        changes = exe.state_changes
+        prestate = {}
+        changes.each { |c|
+          if c.is_delete?
+            prestate[c.property] = c.value
+          elsif c.is_modify?
+            prestate[c.property] = c.old_value
+          elsif c.is_insert?
+            if insert_nil_prestate_prop_for_insertions
+              prestate[c.property] = nil
+            end
+          end
+        }
+        poststate = {}
+        changes.each { |c| 
+          if c.is_delete?
+            poststate[c.property] = nil
+          elsif c.is_modify? || c.is_insert?
+            poststate[c.property] = c.value
+          end
+        }
+        params = exe.get_used_parameters()
+        trans = StateTransition.new(prestate, params, poststate)
+        result << trans
+      end
+      return result
+    end
+
+    #
+    # Return a set of state property mappings {prop_key => value} .
+    #
+    # The result set contains all states which were observed either as pre-state 
+    # or as post-state of any of the executions of this task. The states are
+    # reduced to their "relevant" parts, i.e., only those state properties are 
+    # included which are inserted/deleted/modified by the task, AND only those 
+    # state properties which are not in the list of ignored properties of this 
+    # task's automation.
+    #
+    def global_states_reduced(state_transitions = nil, include_empty_states = true, 
+          add_prestates = true, add_poststates = true)
+      ignore_props = automation ? automation.ignore_properties.to_a.dup : []
+      # add global ignore properties
+      ignore_props.concat(SystemState.read_ignore_properties())
+      puts "WARN: No automation found for task UUID #{@uuid}!" if !automation
+      result = Set.new
+      state_transitions = global_state_transitions() if !state_transitions
+      state_transitions.each do |t|
+        if add_prestates
+          SystemState.remove_ignore_props!(t.pre_state, ignore_props)
+          if (include_empty_states || !t.pre_state.empty?)
+            result << t.pre_state
+          end
+        end
+        if add_poststates
+          SystemState.remove_ignore_props!(t.post_state, ignore_props)
+          if (include_empty_states || !t.post_state.empty?)
+            result << t.post_state
+          end
+        end
+      end
+      return result
+    end
+    def global_pre_states_reduced(state_transitions = nil, include_empty_states = true)
+      return global_states_reduced(state_transitions, include_empty_states, true, false)
+    end
+    def global_post_states_reduced(state_transitions = nil, include_empty_states = true)
+      return global_states_reduced(state_transitions, include_empty_states, false, true)
+    end
+
+    def global_pre_states(global_execs = nil)
+      result = []
+      global_execs = global_executions() if !global_execs
+      # set list of ignored properties
+      ignore_props = automation ? automation.ignore_properties.to_a.dup : []
+      ignore_props.concat(SystemState.read_ignore_properties())
+      global_execs.each do |exe|
+        state = exe.state_before
+        # elimination of map entries is required for removing ignore props!
+        MarkupUtil.eliminate_inserted_map_entries!(state)
+        SystemState.remove_ignore_props!(state, ignore_props) 
+        result << state
+      end
+      return result
+    end
+
+    def global_post_states(global_execs = nil)
+      result = []
+      global_execs = global_executions() if !global_execs
+      # set list of ignored properties
+      ignore_props = automation ? automation.ignore_properties.to_a.dup : []
+      ignore_props.concat(SystemState.read_ignore_properties())
+      global_execs.each do |exe|
+        state = exe.state_after
+        # elimination of map entries is required for removing ignore props!
+        MarkupUtil.eliminate_inserted_map_entries!(state)
+        SystemState.remove_ignore_props!(state, ignore_props)
+        result << state
+      end
+      return result
+    end
+
+    def global_pre_states_flat(global_execs = nil)
+      return Task.properties_flat(global_pre_states(global_execs))
+    end
+
+    def global_post_states_flat(global_execs = nil)
+      return Task.properties_flat(global_post_states(global_execs))
+    end
+
+    def self.properties_flat(states_array)
+      result = []
+      states_array.each do |s|
+        MarkupUtil.eliminate_inserted_map_entries!(s)
+        s = SystemState.get_flat_attributes(s)
+        s.each do |k,v|
+          entry = [k,v]
+          if !result.include?(entry)
+            result << entry
+          end
+        end
+      end
+      return result
+    end
+
+    def toaster_testing_task?()
+      return sourcefile == "toaster/recipes/testing.rb"
+    end
+
+    #
+    # Returns a hash which maps identifier=>configurations, indicating which types 
+    # of state changes this task, upon execution, is *potentially* going to 
+    # perform. For instance, if the task starts/stops a system service,
+    # the identifier "ports" will be in the hash keys. If the task modifies 
+    # some files, the key will contain the identifier "files", and possibly a list of
+    # potential files that may be edited. 
+    # This helps us to develop tailor-made state capturing tools (e.g., implemented 
+    # as ohai plugins) for different types of tasks.
+    # 
+    def get_config_for_potential_state_changes()
+      return ResourceInspector.get_config_for_potential_state_changes(self)
+    end
+
+    def guess_potential_state_changes()
+      return ResourceInspector.guess_potential_state_changes(self)
+    end
+
+    def name
+      return "#{resource}::#{action}"
+    end
+
+    def self.find(criteria={})
+      DB.find_activerecord(Task, criteria)
+    end
+
+    def self.load_from_chef_source(resource, action, sourcecode, sourcefile, sourceline)
+      sourcecode.strip! if sourcecode.kind_of?(String)
+      params = {
+        :resource => resource.to_s,
+        :action => action,
+        :sourcecode => sourcecode,
+        :sourcefile => sourcefile,
+        :sourceline => sourceline
+      }
+      task = find_by(params)
+      if !task
+        task = Task.new(params)
+      end
+      task.resource_obj = resource
+      return task
+    end
+
+    def eql?(other)
+      return other.kind_of?(Task) && !uuid.nil? && uuid == other.uuid
+    end
+
+  end
+end

data/lib/toaster/model/task_execution.rb

@@ -0,0 +1,76 @@
+
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "active_record"
+require "toaster/model/task"
+require "toaster/model/state"
+require "toaster/model/state_change"
+require "toaster/model/automation_run"
+require "toaster/util/timestamp"
+
+module Toaster
+  class TaskExecution < ActiveRecord::Base
+
+    belongs_to :task
+    belongs_to :automation_run
+    has_many :state_changes
+    serialize :state_before, JSON
+    serialize :state_after, JSON
+
+    def initialize(attr_hash)
+      if !attr_hash[:uuid]
+        attr_hash[:uuid] = Util.generate_short_uid()
+      end
+      if !attr_hash[:start_time]
+        attr_hash[:start_time] = TimeStamp.now.to_i
+      end
+      super(attr_hash)
+    end
+
+    #
+    # Return a map {param_name => value} of parameter values that 
+    # were used for this task execution.
+    #
+    def get_used_parameters()
+      result = {}
+      run_attributes = automation_run.get_flat_attributes()
+      task.task_parameters.each do |p|
+        if p
+          value = run_attributes[p.key]
+          result[p.key] = value
+        end
+      end
+      return result
+    end
+
+    def relevant_state_changes()
+      result = state_changes.to_a.dup
+      ignore_props = SystemState.read_ignore_properties()
+      # remove ignored properties from state changes
+      SystemState.remove_ignore_props!(result, ignore_props)
+      return result
+    end
+
+    def reduce_and_set_state_after(state)
+      self.state_after = state
+      # limit the number of state properties which are 
+      # saved for the pre-state and the post-state.
+      states_new = SystemState.reduce_state_size(state_before, state_after)
+      #puts "states before/after #{self}: #{state_after} #{states_new[1]}"
+      self.state_before = states_new[0]
+      self.state_after = states_new[1]
+    end
+
+    def self.find(criteria={})
+      DB.find_activerecord(TaskExecution, criteria)
+    end
+
+    def self.load_all_for_automation(auto)
+      return joins(:automation_run).where(:automation_run => {:automation_id => auto.id})
+    end
+
+  end
+end