dtk-action-agent 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 59abb9f8d4c53e5d82e17b93033b27601585ff70
-  data.tar.gz: b3885713293250f5beb51480f31669bfc723054f
+  metadata.gz: 91b2a5186f6e4ccaba5687ce9ffac7b2baa1c603
+  data.tar.gz: 7b2176431344e9e2078fe2372616cff51b705627
 SHA512:
-  metadata.gz: ee0795deb8734cac59bfad5fb537602f6947c7c1e9e6dd6f674798950941752fd18f09e5c53e857b8facffcf9ee9c8c9b57c6c003bcc075dfe3ab2234cf4abaf
-  data.tar.gz: 8391a8ebaaf3a2efbc5c14a5b273e7e18977555ad5d036040ae1cf098756a1927111fe68f4527cdc39ca5ab7156346a2d903b8105df5a21cc65373b2b1a3f53a
+  metadata.gz: a0e490c27b693136932285d1e1632270725b4a2381eef22f0e00c59cdcf281f5fcf704faad40a3ca7d021f2fdf37cdd935845c63fce90f6306fdab5049e9fedd
+  data.tar.gz: ac498ebb4fd65f5f289e61a198d8e2dcd2de13f2b7f74a1d01db6f25ba5d7c710ed9f32195b5df30363e1d6f4bacde956b70c3f9aef3d13cd4985be3f265675c

data/dtk-action-agent.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = "#{DTK::ActionAgent::VERSION}.#{ARGV[3]}".chomp(".")
 
-  gem.add_dependency 'posix-spawn','0.3.8'
   gem.add_dependency 'awesome_print', '1.1.0'
   gem.add_dependency 'json', '1.8.2'
   gem.add_dependency 'git', '1.2.9.1'
+  gem.add_dependency 'after_do', '0.3.1'
 end

data/lib/arbiter.rb

@@ -1,5 +1,4 @@
 require 'ap'
-require 'posix-spawn'
 
 module DTK
   module Agent
@@ -12,6 +11,11 @@ module DTK
         @process_pool     = []
         @execution_list   = @received_message['execution_list']||[]
 
+        @top_task_id = @received_message['top_task_id']
+        @task_id = @received_message['task_id']
+        @module_name = @received_message['module_name']
+        @action_name = @received_message['action_name']
+
         # no need to run other commands if there is no execution list
         if @execution_list.empty?
           Log.error "Execution list is not provided or empty, DTK Action Agent has nothing to run"
@@ -27,8 +31,14 @@ module DTK
         # start commander runnes
         @commander.run()
 
+        results = @commander.results()
+
+        if @top_task_id && @task_id
+          Log.log_results(@execution_list, results, @module_name, @action_name, @top_task_id, @task_id)
+        end
+
         # return results
-        { :results => @commander.results(), :errors => Log.execution_errors }
+        { :results => results, :errors => Log.execution_errors }
       end
 
     end

data/lib/command.rb

@@ -1,3 +1,6 @@
+require 'open3'
+require 'timeout'
+
 module DTK
   module Agent
 
@@ -6,7 +9,7 @@ module DTK
 
     class Command
 
-      attr_accessor :command_type, :command, :if_success, :if_fail, :process, :child_task
+      attr_accessor :command_type, :command, :process, :backtrace
 
       ##
       # command         - string to be run on system, e.g. ifconfig
@@ -14,7 +17,6 @@ module DTK
       # if              - callback to be run if exit status is  = 0
       # unless          - callback to be run if exit status is != 0
       # stdout_redirect - redirect all output to stdout
-      # child_task      - if it is spawned by another main task
       #
 
       STDOUT_REDIRECT = ' 2>&1'
@@ -23,15 +25,15 @@ module DTK
         @command_type    = value_hash['type']
         @command         = value_hash['command']
         @stdout_redirect = !!value_hash['stdout_redirect']
-        @if_success      = value_hash['if']
-        @if_fail         = value_hash['unless']
-        @spawned         = false
-        @child_task      = value_hash['child_task'] || false
+
+        @if              = value_hash['if']
+        @unless          = value_hash['unless']
+
         @timeout         = (value_hash['timeout'] || 0).to_i
 
         @env_vars        = value_hash['env_vars']
 
-        if @if_success && @if_fail
+        if @if && @unless
           Log.warn "Unexpected case, both if/unless conditions have been set for command #{@command}(#{@command_type})"
         end
       end
@@ -40,15 +42,20 @@ module DTK
       # Creates Posix Spawn of given process
       #
       def start_task
-
         begin
           Commander.set_environment_variables(@env_vars)
-          @process = POSIX::Spawn::Child.new(formulate_command, :timeout => @timeout)
-          Log.debug("Command started: '#{self.to_s}'")
-        rescue POSIX::Spawn::TimeoutExceeded => e
+
+          Timeout.timeout(@timeout) do
+            Log.debug("Command started: '#{self.to_s}'")
+            @out, @err, @process_status = Open3.capture3(formulate_command)
+          end
+
+        rescue Timeout::Error
           @error_message = "Timeout (#{@timeout} sec) for this action has been exceeded"
         rescue Exception => e
           @error_message = e.message
+          @backtrace = e.backtrace
+          Log.error(@error_message, @backtrace)
         ensure
           Commander.clear_environment_variables(@env_vars)
         end
@@ -58,50 +65,61 @@ module DTK
       # Checks if there is callaback present, callback beeing if/unless command
       #
       def callback_pending?
-        return false if @spawned
-        command_to_run = (self.exitstatus.to_i == 0) ? @if_success : @if_fail
-        !!command_to_run
+        @if || @unless
       end
 
       def is_positioning?
         'file'.eql?(@command_type)
       end
 
-
       ##
-      # Creates Command object for callback, first check 'if' than 'unless'. There should be no both set so priority is given
-      # to 'if' callback in case there are two
+      # Returns true/false based on condition data and result of process
       #
-      def spawn_callback_task
-        callback_command = (self.exitstatus.to_i == 0) ? @if_success : @if_fail
-        new_command = Command.new('type' => @command_type, 'command' => callback_command, 'child_task' => true)
-        @spawned = true
-        new_command
+      def run_condition_task
+        condition_command   = @if
+        condition_command ||= @unless
+
+        # this is needed since Timeout block will not allow initialization of new variables
+        condition_process_status = nil
+
+        begin
+          Timeout.timeout(@timeout) do
+            _out, _err, condition_process_status = Open3.capture3(condition_command)
+          end
+        rescue Exception => e
+          # do not log error in cases it was expected. Meaning that 'unless' condition was set.
+          Log.warn("Condition command '#{condition_command}' ran into an exception, message: #{e.message}") unless @unless
+          # return true if unless condition was used
+          return @unless ? true : false
+        end
+
+        return condition_process_status.exitstatus > 0 ? false : true if @if
+        return condition_process_status.exitstatus > 0 ? true  : false if @unless
       end
 
       def exited?
         return true if @error_message
-        self.process.status.exited?
+        @process_status.exited?
+      end
+
+      def started?
+        return true if @error_message
+        !!@process_status
       end
 
       def exitstatus
         return 1 if @error_message
-        self.process.status.exitstatus
+        @process_status.exitstatus
       end
 
       def out
         return '' if @error_message
-        self.process.out
+        @out.encode('UTF-8', :invalid => :replace, :undef => :replace, :replace => '')
       end
 
       def err
         return @error_message if @error_message
-        self.process.err
-      end
-
-      def started?
-        return true if @error_message
-        !!self.process
+        @err.encode!('UTF-8', :invalid => :replace, :undef => :replace, :replace => '')
       end
 
       def to_s
@@ -114,7 +132,7 @@ module DTK
       # Based on stdout-redirect flag
       #
       def formulate_command
-        @stdout_redirect ? "#{@command} #{STDOUT_REDIRECT}" : @command
+        @command
       end
 
     end

data/lib/commander.rb

@@ -28,8 +28,13 @@ module DTK
 
       def sequential_run
         @command_tasks.each do |command_task|
-          command_task.start_task
 
+          if command_task.callback_pending? && !command_task.run_condition_task
+            Log.info("Skipping command task #{command_task}, conditions have not been met")
+            next
+          end
+
+          command_task.start_task
           loop do
             if command_task.exited?
               Log.debug("Command '#{command_task}' finished, with status #{command_task.exitstatus}")
@@ -37,9 +42,6 @@ module DTK
               # exit if there is an error
               return nil if (command_task.exitstatus.to_i > 0)
 
-              # if there is a callback start it
-              spawn_callback_task(command_task) if command_task.callback_pending?
-
               break
             end
 
@@ -51,6 +53,10 @@ module DTK
 
       def parallel_run
         @command_tasks.each do |command_task|
+          if command_task.callback_pending? && !command_task.run_condition_task
+            Log.info("Skipping command task #{command_task}, conditions have not been met")
+            next
+          end
           command_task.start_task
         end
 
@@ -64,13 +70,6 @@ module DTK
             # is task finished
             if command_task.exited?
               Log.debug("Command '#{command_task}' finished, with status #{command_task.exitstatus}")
-
-              # if there is a callback start it
-              if command_task.callback_pending?
-                spawn_callback_task(command_task, true)
-                # new task added we need to check again
-                all_finished = false
-              end
             else
               # we are not ready yet, some tasks need to finish
               all_finished = false
@@ -81,13 +80,6 @@ module DTK
         end
       end
 
-      def spawn_callback_task(command_task, start_task = false)
-        new_command_task = command_task.spawn_callback_task
-        new_command_task.start_task if start_task
-        @command_tasks << new_command_task
-        Log.debug("Command '#{new_command_task}' spawned as callback")
-      end
-
       def results
         res = @command_tasks.collect do |command_task|
           next unless command_task.started?
@@ -96,7 +88,7 @@ module DTK
             :stdout      => command_task.out,
             :stderr      => command_task.err,
             :description => command_task.to_s,
-            :child_task  => command_task.child_task
+            :backtrace   => command_task.backtrace
           }
         end
 

data/lib/dtk-action-agent/version.rb

@@ -1,5 +1,5 @@
 module DTK
   module ActionAgent
-    VERSION = "0.0.6"
+    VERSION = "0.0.7"
   end
 end

data/lib/logger.rb

@@ -1,20 +1,29 @@
 require 'logger'
 require 'singleton'
+require 'after_do'
+require 'fileutils'
+require 'json'
 
 module DTK
   module Agent
     class Log
       include Singleton
 
-      attr_accessor :logger, :error_msgs
+      attr_accessor :logger, :error_msgs, :all_msgs
+
+      self.singleton_class.extend AfterDo
 
       LOG_TO_CONSOLE = false
       LOG_TO_FILE    = '/var/log/action-agent.log'
+      LOGS_DIR       = '/var/log/dtk/action-agent'
       # LOG_TO_FILE    = '/Users/haris/test.log'
 
       def initialize
         # @logger = Logger.new(File.new(LOG_TO_FILE,'w'))
+        @all_msgs   =[]
         @error_msgs =[]
+
+        FileUtils.mkdir_p LOGS_DIR unless File.directory?(LOGS_DIR)
       end
 
       def self.execution_errors()
@@ -31,16 +40,35 @@ module DTK
         ap "info: #{msg}" if LOG_TO_CONSOLE
       end
 
-      def self.warn(msg)
+      def self.warn(msg, backtrace = nil)
         # self.instance.logger.warn(msg)
         ap "warn: #{msg}" if LOG_TO_CONSOLE
-        self.instance.error_msgs << msg
+        self.instance.error_msgs <<  { :message => msg, :backtrace => backtrace }
       end
 
-      def self.error(msg)
+      def self.error(msg, backtrace = nil)
         # self.instance.logger.error(msg)
         ap "error: #{msg}" if LOG_TO_CONSOLE
-        self.instance.error_msgs << msg
+        self.instance.error_msgs << { :message => msg, :backtrace => backtrace }
+      end
+
+      def self.log_results(params_in, results, component_name, action_name, top_task_id, task_id)
+        component_dir = File.join(LOGS_DIR, "#{component_name}_#{top_task_id}")
+        FileUtils.mkdir_p(component_dir) unless File.directory?(component_dir)
+
+        filename = File.join(component_dir, "#{task_id}_#{action_name}.log")
+        File.open(filename, 'w') do |file|
+          file.puts('Input data: ')
+          file.puts JSON.pretty_generate(params_in)
+          file.puts self.instance.all_msgs.join("\n")
+          file.puts('Results: ')
+          file.puts JSON.pretty_generate(results)
+        end
+      end
+
+
+      self.singleton_class.after :debug, :info, :warn, :error do |msg, _backtrace|
+        self.instance.all_msgs << msg
       end
 
     end

data/lib/position.rb

@@ -1,33 +1,48 @@
 require 'thread'
 require 'git'
 require 'fileutils'
+require File.expand_path('../utils/permission', __FILE__)
+
 
 module DTK
   module Agent
     class Position
 
-      attr_accessor :position_file_info, :exitstatus, :started, :out, :err, :child_task
+      attr_accessor :position_file_info, :exitstatus, :started, :out, :err, :backtrace, :owner, :mode
 
       def initialize(command_hash)
         source_info, target_info = command_hash['source'], command_hash['target']
 
+        @exited     = false
+        @started    = false
+        @exitstatus = 0
+
         @type    = source_info['type'].to_sym
         @git_url = source_info['url']
         @branch  = source_info['ref'] || 'master'
         @content = source_info['content']
 
+        @owner  = command_hash['owner']
+
+        if command_hash['mode']
+          if DTK::Utils::Permission.check(command_hash['mode'])
+            @mode   = command_hash['mode'].to_s.oct
+          else
+            trigger_error("Permissions '#{command_hash['mode']}' are not valid, aborting operation")
+          end
+        end
+
         @env_vars = command_hash['env_vars']
 
         @target_path = target_info['path']
-
-        @exited     = false
-        @started    = false
-        @exitstatus = 0
-        @child_task = false
       end
 
       def start_task()
         @started = true
+
+        # for cases when there was an error
+        return if @exited
+
         prepare_path()
 
         Commander.set_environment_variables(@env_vars)
@@ -41,7 +56,7 @@ module DTK
           end
         rescue Exception => e
           cleanup_path()
-          raise e
+          trigger_error(e.message, 1, e.backtrace)
         ensure
           Commander.clear_environment_variables(@env_vars)
         end
@@ -59,10 +74,6 @@ module DTK
         @started
       end
 
-      def spawn_callback_task
-        raise "Callback task is not supported for positioner"
-      end
-
       def callback_pending?
         # not supported at the moment
         false
@@ -74,6 +85,14 @@ module DTK
 
     private
 
+      def trigger_error(error_message, err_status = 1, error_backtrace = nil)
+        @err = error_message
+        Log.error(error_message, error_backtrace)
+        @exitstatus = err_status
+        @started = true
+        @exited  = true
+      end
+
       def position_git()
         unless File.directory?(@target_path)
           begin
@@ -81,9 +100,7 @@ module DTK
             Log.info("Positioner successfully cloned git repository '#{@git_url}@#{@branch}' to location '#{@target_path}'")
           rescue Exception => e
             cleanup_path()
-            @exitstatus = 1
-            Log.error("Positioner unable to clone #{@git_url}")
-            Log.error(e.message)
+            trigger_error("Positioner unable to clone provided url #{@git_url}. Reasone: #{e.message}", 1, e.backtrace)
           end
         else
           Log.warn("Positioner detected folder '#{@target_path}' skipping git clone")
@@ -92,10 +109,29 @@ module DTK
         @exited = true
       end
 
-      def position_in_payload(position_info)
+      def position_in_payload()
         # write to file
-        File.open(@target_path, 'w') { |file| file.write(@content) }
+        file = File.open(@target_path, 'w')
+        file.write(@content)
+
+        if @owner
+          begin
+            FileUtils.chown(@owner, nil, file.path)
+          rescue Exception => e
+            Log.warn("Not able to set owner '#{@owner}', reason: " + e.message)
+          end
+        end
+
+        if @mode
+          begin
+            FileUtils.chmod(@mode, file.path)
+          rescue Exception => e
+            Log.warn("Not able to set chmod permissions '#{@mode}', reason: " + e.message)
+          end
+        end
+
         Log.info("Positioner successfully created 'IN_PAYLOAD' file '#{@target_path}'")
+        file.close
         @exited = true
       end
 

data/lib/utils/permission.rb

@@ -0,0 +1,13 @@
+module DTK
+  module Utils
+    class Permission
+
+      PERMISSION_CHECKER = /^[01]?[0-7]{3}$/
+
+      def self.check(permission_value)
+        !permission_value.to_s.match(PERMISSION_CHECKER).nil?
+      end
+
+    end
+  end
+end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: dtk-action-agent
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Rich PELAVIN
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-20 00:00:00.000000000 Z
+date: 2015-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: posix-spawn
+  name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.8
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.8
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.8.2
 - !ruby/object:Gem::Dependency
-  name: json
+  name: git
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.2
+        version: 1.2.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.2
+        version: 1.2.9.1
 - !ruby/object:Gem::Dependency
-  name: git
+  name: after_do
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.9.1
+        version: 0.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.9.1
+        version: 0.3.1
 description: The DTK Action Agent is designed to run commands on remote machine.
 email:
 - rich@reactor8.com
@@ -83,6 +83,7 @@ files:
 - lib/dtk-action-agent/version.rb
 - lib/logger.rb
 - lib/position.rb
+- lib/utils/permission.rb
 homepage: ''
 licenses:
 - GPL-3.0