naf 1.1.4 → 2.0.0

data/app/models/logical/naf/job_creator.rb

@@ -14,17 +14,8 @@ module Logical
         # jobs (running/queued) is equal to or greater than the application
         # run group limit, or if enqueue_backlogs is set to false. If so,
         # do not add the job to the queue
-        running_jobs = ::Naf::RunningJob.
-          select('application_run_group_limit, MAX(created_at) AS created_at, count(*)').
-          where('command = ? AND application_run_group_name = ?',
-            application.command, application_run_group_name).
-          group('application_run_group_name, application_run_group_limit').first
-
-        queued_jobs = ::Naf::QueuedJob.
-          select('application_run_group_limit, MAX(created_at) AS created_at, count(*)').
-          where('command = ? AND application_run_group_name = ?',
-            application.command, application_run_group_name).
-          group('application_run_group_name, application_run_group_limit').first
+        running_jobs = retrieve_jobs(::Naf::RunningJob, application.command, application_run_group_name)
+        queued_jobs = retrieve_jobs(::Naf::QueuedJob, application.command, application_run_group_name)
 
         if enqueue == false && (running_jobs.present? || queued_jobs.present?)
           group_limit = running_jobs.try(:application_run_group_limit).to_i + queued_jobs.try(:application_run_group_limit).to_i
@@ -42,7 +33,6 @@ module Logical
                                                         application_run_group_limit: application_run_group_limit,
                                                         priority: priority,
                                                         log_level: application.log_level)
-          historical_job.add_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:pre_work]])
 
           # Create historical job affinity tabs for each affinity associated with the historical job
           affinities.each do |affinity|
@@ -55,12 +45,7 @@ module Logical
                                                    affinity_parameter: affinity_parameter)
           end
 
-          historical_job.verify_prerequisites(prerequisites)
-          # Create historical job prerequisites for each prerequisite associated with the historical job
-          prerequisites.each do |prerequisite|
-            ::Naf::HistoricalJobPrerequisite.create(historical_job_id: historical_job.id,
-                                                    prerequisite_historical_job_id: prerequisite.id)
-          end
+          verify_and_create_prerequisites(historical_job, prerequisites)
 
           create_queue_job(historical_job)
 
@@ -68,6 +53,12 @@ module Logical
         end
       end
 
+      def retrieve_jobs(klass, command, application_run_group_name)
+        klass.select('application_run_group_limit, MAX(created_at) AS created_at, count(*)').
+          where('command = ? AND application_run_group_name = ?', command, application_run_group_name).
+          group('application_run_group_name, application_run_group_limit').first
+      end
+
       def queue_application_schedule(application_schedule, schedules_queued_already = [])
         prerequisite_jobs = []
 
@@ -114,11 +105,7 @@ module Logical
             ::Naf::HistoricalJobAffinityTab.create(historical_job_id: historical_job.id, affinity_id: affinity.id)
           end
 
-          historical_job.verify_prerequisites(prerequisites)
-          prerequisites.each do |prerequisite|
-            ::Naf::HistoricalJobPrerequisite.create(historical_job_id: historical_job.id,
-                                                    prerequisite_historical_job_id: prerequisite.id)
-          end
+          verify_and_create_prerequisites(historical_job, prerequisites)
 
           create_queue_job(historical_job)
 
@@ -126,6 +113,15 @@ module Logical
         end
       end
 
+      def verify_and_create_prerequisites(job, prerequisites)
+        job.verify_prerequisites(prerequisites)
+        # Create historical job prerequisites for each prerequisite associated with the historical job
+        prerequisites.each do |prerequisite|
+          ::Naf::HistoricalJobPrerequisite.create(historical_job_id: job.id,
+                                                  prerequisite_historical_job_id: prerequisite.id)
+        end
+      end
+
       def create_queue_job(historical_job)
         queued_job = ::Naf::QueuedJob.new(application_id: historical_job.application_id,
                                           application_type_id: historical_job.application_type_id,

data/app/models/logical/naf/job_fetcher.rb

@@ -18,8 +18,8 @@ module Logical
 
           if running_job.present?
             # Update tags
-            running_job.historical_job.remove_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:pre_work]])
-            running_job.historical_job.add_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:work]])
+            running_job.remove_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:pre_work]])
+            running_job.add_tags([::Naf::HistoricalJob::SYSTEM_TAGS[:work]])
 
             # found a job
             parse_log_level(running_job)
@@ -142,8 +142,7 @@ module Logical
         log_levels = {}
         unless machine.log_level.blank?
           begin
-            log_level_hash = JSON.parse(machine.log_level)
-            log_levels.merge!(log_level_hash)
+            log_levels.merge!(retrieve_log_levels(machine.log_level))
           rescue StandardError => e
             logger.error "couldn't parse machine.log_level: #{machine.log_level}: (#{e.message})"
           end
@@ -151,17 +150,48 @@ module Logical
 
         unless running_job.application.nil? || running_job.application.log_level.blank?
           begin
-            log_level_hash = JSON.parse(running_job.application.log_level)
-            log_levels.merge!(log_level_hash)
+            log_levels.merge!(retrieve_log_levels(running_job.application.log_level, running_job.application.command))
           rescue StandardError => e
             logger.error "couldn't parse running_job.application.log_level: #{running_job.application.log_level}: (#{e.message})"
           end
         end
 
         running_job.log_level = log_levels.to_json
+        running_job.historical_job.log_level = log_levels.to_json
+        running_job.historical_job.save!
         running_job.save!
       end
 
+      def retrieve_log_levels(log_level, command = nil)
+        log_level_hash = {}
+
+        if log_level[0] == '{'
+          # String is in JSON format
+          log_level_hash = JSON.parse(log_level)
+        else
+          if log_level.include?('=')
+            # Pairs of logger_name=logger_level were given
+            log_level.split(',').each do |elem|
+              name, level = elem.split('=')
+              log_level_hash[name.strip.to_sym] = level.strip
+            end
+          else
+            # Only log level threshold was given
+            if command.present?
+              key = command.split('.').first
+              if key[0..1] == '::'
+                key = key[2..-1]
+              end
+              log_level_hash[key] = log_level
+            else
+              log_level_hash[:default] = log_level
+            end
+          end
+        end
+
+        log_level_hash
+      end
+
     end
   end
 end

data/app/models/logical/naf/log_file.rb

@@ -0,0 +1,70 @@
+module Logical
+  module Naf
+    class LogFile
+      LOG_MAX_SIZE = 50_000
+
+      attr_reader :file_line_number,
+                    :line_number,
+                    :file,
+                    :log_area,
+                    :lines_cache
+
+      def initialize(log_area)
+        @file_line_number = @line_number = 1
+        @file = nil
+        @log_area = log_area
+        @lines_cache = ''
+      end
+
+      def <<(message)
+        log_line = JSON.pretty_generate({
+          line_number: line_number,
+          output_time: Time.zone.now.strftime("%Y-%m-%d %H:%M:%S.%L"),
+          message: message
+         })
+        @lines_cache << log_line
+        @line_number += 1
+      end
+
+      def write
+        check_file_size
+        output = lines_cache
+        unless output.blank?
+          file.write(output)
+          @lines_cache = ''
+          flush
+        end
+      end
+
+      def flush
+        file.flush
+      end
+
+      def open
+        # Create the directory path if it doesn't exist
+        FileUtils.mkdir_p(log_area)
+
+        filename = Dir[log_area + "/#{file_line_number}_*"].first
+        if filename.blank?
+          @file = File.open(log_area + "/#{file_line_number}_#{Time.zone.now.strftime('%Y%m%d_%H%M%S')}.json", 'wb')
+        else
+          @file = File.open(filename, 'wb')
+        end
+      end
+
+      def close
+        @file.try(:close)
+        @file = nil
+      end
+
+      def check_file_size
+        if file.size > LOG_MAX_SIZE
+          @file_line_number = line_number
+          close
+          @file = File.open(log_area + "/#{file_line_number}_#{Time.zone.now.strftime('%Y%m%d_%H%M%S')}.json", 'wb')
+        end
+      end
+    end
+
+  end
+end

data/app/models/logical/naf/log_parser/base.rb

@@ -0,0 +1,272 @@
+require 'yajl'
+
+module Logical::Naf
+  module LogParser
+    class Base
+
+      REGEX_OPTIONS = {
+        'i' => Regexp::IGNORECASE,
+        'x' => Regexp::EXTENDED,
+        'm' => Regexp::MULTILINE
+      }
+      DATE_REGEX = /((\d){8}_(\d){6})/
+      UUID_REGEX = /[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}/
+      LOG_SIZE_CHUNKS = 500
+
+      attr_accessor  :search_params,
+                     :regex_options,
+                     :grep,
+                     :search_from_time,
+                     :search_to_time,
+                     :jsons,
+                     :logs_size,
+                     :log_type,
+                     :newest_log,
+                     :record_id,
+                     :read_from_s3,
+                     :s3_log_reader,
+                     :last_file_checked,
+                     :newest_file_checked
+
+      def initialize(params)
+        @search_params = params['search_params'].nil? ? '' : params['search_params']
+        @regex_options = get_option_value(params['regex_options'])
+        @grep = params['grep']
+        @search_from_time = params['from_time']
+        @search_to_time = params['to_time']
+        @jsons = []
+        @logs_size = 0
+        @log_type = params['log_type']
+        @newest_log = params['newest_log']
+        @record_id = params['record_id']
+        @read_from_s3 = params['read_from_s3']
+        @last_file_checked = params['last_file_checked']
+        @newest_file_checked = params['newest_file_checked']
+      end
+
+      def retrieve_logs
+        parse_files
+
+        check_repeated_logs
+
+        output = ''
+        jsons.reverse_each do |elem|
+          output.insert(0, insert_log_line(elem))
+        end
+
+        return {
+          logs: output.html_safe,
+          read_from_s3: read_from_s3,
+          last_file_checked: last_file_checked,
+          newest_file_checked: newest_file_checked,
+          newest_log: newest_log
+        }
+      end
+
+      def check_repeated_logs
+        if log_type == 'new' && newest_log.present?
+          @jsons.each_with_index do |elem, index|
+            if parse_log(elem) == parse_newest_log
+              @jsons = @jsons[(index + 1)..-1]
+              return
+            end
+          end
+        end
+      end
+
+      def parse_log(log)
+        if log['message'].scan(/\t/).present?
+          message = log['message'].clone
+          message.slice!('<top (required)>')
+          "#{log['output_time']} #{message}"
+        else
+          "#{log['output_time']} #{log['message']}"
+        end
+      end
+
+      def parse_files
+        files = filter_files
+
+        files.each do |file|
+          # Use Yajl JSON library to parse the log files, as they contain multiple JSON blocks
+          parser = Yajl::Parser.new
+          json = get_json_from_log_file(file)
+          parser.parse(json) do |log|
+            if self.class.to_s == 'Logical::Naf::LogParser::Runner'
+              log['id'] = get_invocation_id(file.scan(UUID_REGEX).first)
+            elsif self.class.to_s == 'Logical::Naf::LogParser::Machine'
+              log['job_id'] = get_job_id(file)
+            end
+            filter_log_messages(log)
+          end
+
+          sort_jsons
+
+          if logs_size >= LOG_SIZE_CHUNKS
+            update_last_file_checked(file.scan(/\d+_\d{8}_\d{6}.*/).first)
+            break
+          end
+        end
+
+        if logs_size < LOG_SIZE_CHUNKS && files.present?
+           update_last_file_checked(files.last.scan(/\d+_\d{8}_\d{6}.*/).first)
+        end
+      end
+
+      def update_last_file_checked(file)
+        if file.present? && last_file_checked.present? && last_file_checked != 'null'
+          if Time.parse(file.scan(/\d{8}_\d{6}/).first) < Time.parse(last_file_checked.scan(/\d{8}_\d{6}/).first)
+            @last_file_checked = file
+          end
+        elsif file.present?
+          @last_file_checked = file
+        end
+      end
+
+      def get_json_from_log_file(file)
+        if read_from_s3 == 'true' && s3_log_reader.present?
+          s3_log_reader.retrieve_file(file)
+        else
+          File.new(file, 'r')
+        end
+      end
+
+      def filter_files
+        files = get_files
+        original_size = files.size
+
+        files.each_with_index do |file, index|
+          filename = file.scan(/\d+_\d{8}_\d{6}.*/).first
+
+          if log_type == 'old'
+            if filename == last_file_checked
+              if files.size == 1
+                files = []
+              else
+                files = files[(index + 1)..-1]
+              end
+            end
+
+            if files.size == 0 && read_from_s3 != 'true'
+              get_s3_files do
+                @read_from_s3 = 'true'
+                @s3_log_reader = ::Logical::Naf::LogReader.new
+                return retrieve_log_files_from_s3
+              end
+            end
+          elsif log_type == 'new'
+            if filename == newest_file_checked
+              if files.size == 1
+                files = []
+              else
+                files = files[0..(index - 1)]
+              end
+            end
+          end
+
+          break if original_size != files.size
+        end
+
+        if files.present?
+          if newest_file_checked.blank? || newest_file_checked == 'null'
+            @newest_file_checked = files[0].scan(/\d+_\d{8}_\d{6}.*/).first
+          else
+            if Time.parse(newest_file_checked.scan(DATE_REGEX)[0][0]) < Time.parse(files[0].scan(DATE_REGEX)[0][0])
+              @newest_file_checked = files[0].scan(/\d+_\d{8}_\d{6}.*/).first
+            end
+          end
+        end
+
+        return files
+      end
+
+      def filter_log_messages(log)
+        # Check that the message matches the search query. Highlight the matching results
+        if search_params.present?
+          log['message'].scan(Regexp.new(search_params, regex_options)).each do |match|
+            log['message'].gsub!(match, "<span style='background-color:yellow;'>#{match}</span>")
+          end
+        end
+
+        # Check that the log happened within the time range specified
+        if log_within_time_range(log['output_time'])
+          # If grep is selected, only show log messages that match the
+          # search query. Otherwise, show all log messages.
+          if grep == 'false' || log['message'] =~ Regexp.new(search_params, regex_options)
+            @jsons << log
+            @logs_size += 1
+          end
+        end
+      end
+
+      def log_within_time_range(log_time)
+        return true if (search_from_time.join('').blank? && search_to_time.join('').blank?) || log_time.blank?
+
+        if search_from_time.join('').present? && search_to_time.join('').present?
+          Time.parse(log_time) <= Time.parse(build_time_string(search_to_time)) &&
+          Time.parse(log_time) >= Time.parse(build_time_string(search_from_time))
+        elsif search_from_time.join('').present?
+          Time.parse(log_time) >= Time.parse(build_time_string(search_from_time))
+        elsif search_to_time.join('').present?
+          Time.parse(log_time) <= Time.parse(build_time_string(search_to_time))
+        end
+      end
+
+      def build_time_string(search_time)
+        # Year
+        search_built_time = search_time[0] + "-"
+        # Month
+        if search_time[1].to_i < 10
+          search_built_time << '0' + search_time[1] + '-'
+        else
+          search_built_time << search_time[1] + '-'
+        end
+        # Day
+        if search_time[2].to_i < 10
+          search_built_time << '0' + search_time[2] + ' '
+        else
+          search_built_time << search_time[2] + ' '
+        end
+        # Hour
+        search_built_time << search_time[3] + ':'
+        # Minute
+        if search_time[4].to_i < 10
+          search_built_time << '0' + search_time[4]
+        else
+          search_built_time << search_time[4]
+        end
+        # Second
+        search_built_time << ':00 -0500'
+
+        search_built_time
+      end
+
+      def get_option_value(options)
+        return 0 if options.blank?
+
+        options = options.split(//)
+        result = 0
+        options.each do |opt|
+          result |= REGEX_OPTIONS[opt]
+        end
+
+        result
+      end
+
+      def get_s3_files
+        begin
+          yield
+        rescue
+          @jsons << {
+            'line_number' => 0,
+            'output_time' => Time.zone.now.strftime("%Y-%m-%d %H:%M:%S.%L"),
+            'message' => 'AWS S3 Access Denied. Please check your permissions.'
+          }
+
+          return []
+        end
+      end
+
+    end
+  end
+end