naf 1.1.4 → 2.0.0

data/app/models/logical/naf/log_parser/job.rb

@@ -0,0 +1,65 @@
+require 'yajl'
+
+module Logical::Naf
+  module LogParser
+    class Job < Base
+
+      def initialize(params)
+        super(params)
+      end
+
+      def logs
+        retrieve_logs
+      end
+
+      private
+
+      def insert_log_line(elem)
+        "&nbsp;&nbsp;<span>#{elem['line_number']} #{elem['output_time']}: #{elem['message']}</br></span>"
+      end
+
+      def sort_jsons
+        # Sort log lines based on timestamp
+        @jsons = jsons.sort { |x, y| x['line_number'] <=> y['line_number'] }
+      end
+
+      def parse_newest_log
+        if newest_log.scan(/\d+ \d{4}-\d{2}-\d{2}/).present?
+          newest_log.slice!(newest_log.split(/ /).first + ' ')
+        end
+
+        if newest_log.scan(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}:/).present?
+          date = newest_log.slice!(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}:/)[0..-2]
+          @newest_log = date + newest_log
+        end
+        newest_log
+      end
+
+      def retrieve_log_files_from_s3
+        s3_log_reader.retrieve_job_files(record_id)
+      end
+
+      def get_files
+        if log_type == 'old' && read_from_s3 == 'true'
+          get_s3_files do
+            @s3_log_reader = ::Logical::Naf::LogReader.new
+            return retrieve_log_files_from_s3
+          end
+        else
+          files = Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{record_id}/*"]
+          if files.present?
+            # Sort log files based on time
+            return files.sort { |x, y| Time.parse(y.scan(DATE_REGEX)[0][0]) <=> Time.parse(x.scan(DATE_REGEX)[0][0]) }
+          else
+            get_s3_files do
+              @read_from_s3 = 'true'
+              @s3_log_reader = ::Logical::Naf::LogReader.new
+              return retrieve_log_files_from_s3
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/app/models/logical/naf/log_parser/machine.rb

@@ -0,0 +1,64 @@
+require 'yajl'
+
+module Logical::Naf
+  module LogParser
+    class Machine < Base
+
+      def initialize(params)
+        super(params)
+      end
+
+      def logs
+        retrieve_logs
+      end
+
+      private
+
+      def insert_log_line(elem)
+        "&nbsp;&nbsp;<span>#{elem['output_time']} <font color='333399'><b>jid(#{elem['job_id']}):</b></font> #{elem['message']}</br></span>"
+      end
+
+      def sort_jsons
+        # Sort log lines based on timestamp
+        @jsons = jsons.sort { |x, y| Time.parse(x['output_time']) <=> Time.parse(y['output_time']) }
+      end
+
+      def parse_newest_log
+        if newest_log.scan(/jid\(\d*\)\: /).present?
+          newest_log.slice!(/jid\(\d*\)\: /)
+        end
+        newest_log
+      end
+
+      def retrieve_log_files_from_s3
+        s3_log_reader.log_files
+      end
+
+      def get_files
+        if log_type == 'old' && read_from_s3 == 'true'
+          get_s3_files do
+            @s3_log_reader = ::Logical::Naf::LogReader.new
+            return retrieve_log_files_from_s3
+          end
+        else
+          files = Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/*/*"]
+          if files.present?
+            # Sort log files based on time
+            return files.sort { |x, y| Time.parse(y.scan(DATE_REGEX)[0][0]) <=> Time.parse(x.scan(DATE_REGEX)[0][0]) }
+          else
+            get_s3_files do
+              @read_from_s3 = 'true'
+              @s3_log_reader = ::Logical::Naf::LogReader.new
+              return retrieve_log_files_from_s3
+            end
+          end
+        end
+      end
+
+      def get_job_id(file)
+        file.scan(/\d+_\d{8}/).first.split('_').first
+      end
+
+    end
+  end
+end

data/app/models/logical/naf/log_parser/runner.rb

@@ -0,0 +1,72 @@
+require 'yajl'
+
+module Logical::Naf
+  module LogParser
+    class Runner < Base
+
+      attr_accessor :invocations_ids
+
+      def initialize(params)
+        super(params)
+        @invocations_ids = {}
+      end
+
+      def logs
+        retrieve_logs
+      end
+
+      private
+
+      def insert_log_line(elem)
+        "&nbsp;&nbsp;<span>#{elem['output_time']} #{invocation_link(elem['id'])}: #{elem['message']}</br></span>"
+      end
+
+      def invocation_link(id)
+        "<a href=\"\/job_system\/machine_runner_invocations\/#{id}\" style=\"font-weight:bold; color: #333399\">invocation(#{id})</a>"
+      end
+
+      def sort_jsons
+        # Sort log lines based on timestamp
+        @jsons = jsons.sort { |x, y| Time.parse(x['output_time']) <=> Time.parse(y['output_time']) }
+      end
+
+      def parse_newest_log
+        "#{newest_log.scan(/\d{4}.*\.\d{3}/).first} #{newest_log.scan(/Process.*/).first.try(:split, '<br>').try(:first)}"
+      end
+
+      def retrieve_log_files_from_s3
+        s3_log_reader.runner_log_files(record_id)
+      end
+
+      def get_invocation_id(uuid)
+        if invocations_ids[uuid].blank?
+          @invocations_ids[uuid] = ::Naf::MachineRunnerInvocation.find_by_uuid(uuid).id
+        end
+
+        invocations_ids[uuid]
+      end
+
+      def get_files
+        if log_type == 'old' && read_from_s3 == 'true'
+          get_s3_files do
+            @s3_log_reader = ::Logical::Naf::LogReader.new
+            return s3_log_reader.runner_log_files(record_id)
+          end
+        else
+          files = Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/*/*"]
+          if files.present?
+            # Sort log files based on time
+            return files.sort { |x, y| Time.parse(y.scan(DATE_REGEX)[0][0]) <=> Time.parse(x.scan(DATE_REGEX)[0][0]) }
+          else
+            get_s3_files do
+              @read_from_s3 = 'true'
+              @s3_log_reader = ::Logical::Naf::LogReader.new
+              return retrieve_log_files_from_s3
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/app/models/logical/naf/log_reader.rb

@@ -0,0 +1,85 @@
+require 'aws'
+
+module Logical
+  module Naf
+    class LogReader
+
+      DATE_REGEX = /((\d){4}-(\d){2}-(\d){2} (\d){2}:(\d){2}:(\d){2} UTC)/
+
+      def log_files
+        tree = bucket.objects.with_prefix(prefix).as_tree
+        directories = tree.children.select(&:branch?).collect(&:prefix).uniq
+
+        files = []
+        directories.each do |directory|
+          tree = bucket.objects.with_prefix(directory).as_tree
+          tree.children.select(&:leaf?).collect(&:key).each do |file|
+            files << file
+          end
+        end
+        return sort_files(files)
+      end
+
+      def runner_log_files(runner_id)
+        @runner_id = runner_id
+        tree = bucket.objects.with_prefix(prefix).as_tree
+        directories = tree.children.select(&:branch?).collect(&:prefix).uniq
+
+        files = []
+        directories.each do |directory|
+          tree = bucket.objects.with_prefix(directory).as_tree
+          tree.children.select(&:leaf?).collect(&:key).each do |file|
+            files << file
+          end
+        end
+        return sort_files(files)
+      end
+
+
+      def retrieve_file(file)
+        bucket.objects[file].read
+      end
+
+      def retrieve_job_files(job_id)
+        tree = bucket.objects.with_prefix(prefix + "#{job_id}").as_tree
+        sort_files(tree.children.select(&:leaf?).collect(&:key))
+      end
+
+      private
+
+      def s3
+        # Use AWS credentials to access S3
+        @s3 ||= AWS::S3.new(access_key_id: AWS_ID,
+                            secret_access_key: AWS_KEY,
+                            ssl_verify_peer: false)
+      end
+
+      def bucket
+        @bucket ||= s3.buckets[NAF_BUCKET]
+      end
+
+      def prefix
+        if @runner_id.present?
+          "naf/#{project_name}/#{Rails.env}/#{creation_time}/#{::Naf::NAF_DATABASE_HOSTNAME}/#{::Naf::NAF_DATABASE}/#{::Naf.schema_name}/runners/#{@runner_id}/invocations/"
+        else
+          "naf/#{project_name}/#{Rails.env}/#{creation_time}/#{::Naf::NAF_DATABASE_HOSTNAME}/#{::Naf::NAF_DATABASE}/#{::Naf.schema_name}/jobs/"
+        end
+      end
+
+      def sort_files(files)
+        files.sort do |x, y|
+          -Time.parse(x.scan(/\d{8}_\d{4}/).last).to_i <=> -Time.parse(y.scan(/\d{8}_\d{4}/).last).to_i
+        end
+      end
+
+    	def project_name
+    		(`git remote -v`).slice(/\/\S+/).sub('.git','')[1..-1]
+    	end
+
+      def creation_time
+      	::Naf::ApplicationType.first.created_at.strftime("%Y%m%d_%H%M%S")
+      end
+
+    end
+  end
+end

data/app/models/logical/naf/machine.rb

@@ -61,7 +61,11 @@ module Logical
       def affinities
         @machine.machine_affinity_slots.map do |slot|
           if slot.affinity_short_name
-            slot.affinity_short_name
+            if slot.affinity_parameter.present? && slot.affinity_parameter > 0
+              slot.affinity_short_name + "(#{slot.affinity_parameter})"
+            else
+              slot.affinity_short_name
+            end
           else
             name = slot.affinity_classification_name + '_' + slot.affinity_name
             name = name + '_required' if slot.required
@@ -80,6 +84,40 @@ module Logical
         end
       end
 
+      def status
+        runner_down = true
+        @machine.machine_runners.each do |runner|
+          if runner.machine_runner_invocations.where(wind_down_at: nil, dead_at: nil).count > 0
+            runner_down = false
+            break;
+          end
+        end
+
+        status = 'Good'
+        if runner_down
+          notes = 'Runner down'
+          status = 'Bad'
+        else
+          notes = ''
+        end
+
+        { server_name: name,
+          status: status,
+          notes: notes }
+      end
+
+      def runner
+        if @machine.server_name.present?
+          @machine.server_name.to_s
+        else
+          if Rails.env == 'development'
+            "localhost:#{Rails::Server.new.options[:Port]}"
+          else
+            @machine.server_address
+          end
+        end
+      end
+
     end
   end
 end

data/app/models/naf/affinity.rb

@@ -51,6 +51,24 @@ module Naf
       where(selectable: true)
     end
 
+    def self.names_list
+      selectable.map do |a|
+        classification = a.affinity_classification
+        if classification.affinity_classification_name == 'machine'
+          if a.affinity_short_name.present?
+            [a.affinity_short_name, a.id]
+          elsif ::Naf::Machine.find_by_id(Integer(a.affinity_name)).present?
+            machine = ::Naf::Machine.find_by_id(Integer(a.affinity_name))
+            [machine.hostname, a.id]
+          else
+            ['Bad affinity: ' + classification.affinity_classification_name + ', ' + a.affinity_name, a.id]
+          end
+        else
+          [classification.affinity_classification_name + ', ' + a.affinity_name, a.id]
+        end
+      end
+    end
+
     def self.pre_unpickle(unpickler)
       unpickler.references.merge!(Hash[::Naf::Affinity.all.map do |m|
                                     [{ association_model_name: ::Naf::Affinity.name,

data/app/models/naf/application_schedule_affinity_tab.rb

@@ -28,6 +28,7 @@ module Naf
     #++++++++++++++++++++
 
     delegate :affinity_name,
+             :affinity_short_name,
              :affinity_classification_name, to: :affinity
 
 

data/app/models/naf/application_type.rb

@@ -33,7 +33,8 @@ module Naf
     end
 
     def invoke(job, command)
-      Process.spawn({ "NAF_JOB_ID" => job.id.to_s }, command)
+      ENV['NAF_JOB_ID'] = job.id.to_s
+      Open4::popen4(command)
     end
 
     def rails_invocator(job)

data/app/models/naf/historical_job.rb

@@ -30,7 +30,6 @@ module Naf
                     :request_to_terminate,
                     :marked_dead_by_machine_id,
                     :log_level,
-                    :tags,
                     :machine_runner_invocation_id
 
     JOB_STALE_TIME = 1.week
@@ -59,6 +58,12 @@ module Naf
       class_name: "::Naf::ApplicationRunGroupRestriction"
     belongs_to :machine_runner_invocation,
       class_name: "::Naf::MachineRunnerInvocation"
+    has_one :running_job,
+      class_name: "::Naf::RunningJob",
+      foreign_key: :id
+    has_one :queued_job,
+      class_name: "::Naf::QueuedJob",
+      foreign_key: :id
     # Must access instance methods job_prerequisites through helper methods so we can use partitioning sql
     has_many :historical_job_prerequisites,
       class_name: "::Naf::HistoricalJobPrerequisite",
@@ -99,6 +104,9 @@ module Naf
 
     delegate :application_run_group_restriction_name, to: :application_run_group_restriction
     delegate :script_type_name, to: :application_type
+    delegate :affinity_name,
+             :affinity_classification_name,
+             :affinity_short_name, to: :affinity
 
     #------------------
     # *** Partition ***
@@ -302,33 +310,5 @@ module Naf
       application_type.spawn(self)
     end
 
-    def add_tags(tags_to_add)
-      tags_array = nil
-      if self.tags.present?
-        tags_array = self.tags.gsub(/[{}]/,'').split(',')
-        new_tags = '{' + (tags_array | tags_to_add).join(',') + '}'
-      else
-        new_tags = '{' + tags_to_add.join(',') + '}'
-      end
-
-      self.tags = new_tags
-      self.save!
-    end
-
-    def remove_tags(tags_to_remove)
-      if self.tags.present?
-        tags_array = self.tags.gsub(/[{}]/,'').split(',')
-        new_tags = '{' + (tags_array - tags_to_remove).join(',') + '}'
-
-        self.tags = new_tags
-        self.save!
-      end
-    end
-
-    def remove_all_tags
-      self.tags = '{}'
-      self.save!
-    end
-
   end
 end