naf 2.0.4 → 2.1.0

data/app/models/naf/queued_job.rb

@@ -2,6 +2,7 @@ module Naf
   class QueuedJob < NafBase
     # Protect from mass-assignment issue
     attr_accessible :application_id,
+                    :application_schedule_id,
                     :application_type_id,
                     :command,
                     :application_run_group_restriction_id,
@@ -19,6 +20,8 @@ module Naf
       foreign_key: :id
     belongs_to :application,
       class_name: "::Naf::Application"
+    belongs_to :application_schedule,
+      class_name: '::Naf::ApplicationSchedule'
     belongs_to :application_type,
       class_name: '::Naf::ApplicationType'
     belongs_to :application_run_group_restriction,

data/app/models/naf/run_interval_style.rb

@@ -0,0 +1,20 @@
+module Naf
+  class RunIntervalStyle < NafBase
+    # Protect from mass-assignment issue
+    attr_accessible :name
+
+    #---------------------
+    # *** Associations ***
+    #+++++++++++++++++++++
+
+    has_many :application_schedules,
+      class_name: '::Naf::ApplicationSchedule'
+
+    #--------------------
+    # *** Validations ***
+    #++++++++++++++++++++
+
+    validates :name, presence: true
+
+  end
+end

data/app/models/naf/running_job.rb

@@ -2,6 +2,7 @@ module Naf
   class RunningJob < NafBase
     # Protect from mass-assignment issue
     attr_accessible :application_id,
+                    :application_schedule_id,
                     :application_type_id,
                     :command,
                     :application_run_group_restriction_id,
@@ -24,6 +25,8 @@ module Naf
       foreign_key: :id
     belongs_to :application,
       class_name: "::Naf::Application"
+    belongs_to :application_schedule,
+      class_name: '::Naf::ApplicationSchedule'
     belongs_to :application_type,
       class_name: '::Naf::ApplicationType'
     belongs_to :application_run_group_restriction,

data/app/models/process/naf/data_migration/backfill_application_schedule_run_interval.rb

@@ -0,0 +1,85 @@
+module Process::Naf::DataMigration
+  class BackfillApplicationScheduleRunInterval < ::Process::Naf::Application
+
+    opt :update, 'update run interval styles id'
+    opt :rollback, 'rollback changes'
+    opt_select :update_or_rollback, one_of: [:update, :rollback]
+
+    attr_reader :run_interval_styles
+
+    def work
+      logger.info 'Starting to update application schedules...'
+
+      if @update.present?
+        setup
+        update_run_interval_styles_ids
+        logger.info 'Finished updating application schedules'
+      elsif @rollback.present?
+        if validate_applications
+          rollback_changes
+          logger.info 'Finished updating application schedules'
+        end
+      end
+    end
+
+    private
+
+    def setup
+      @run_interval_styles = {}
+      ::Naf::RunIntervalStyle.all.each do |ris|
+        @run_interval_styles[ris.name] = ris.id
+      end
+    end
+
+    def update_run_interval_styles_ids
+      ::Naf::ApplicationSchedule.all.each do |schedule|
+        logger.info "Updating application schedule: #{schedule.id}"
+
+        if schedule.run_start_minute.present?
+          schedule.run_interval = schedule.run_start_minute
+          schedule.run_interval_style_id = run_interval_styles['at beginning of day']
+        elsif schedule.run_interval.present?
+          schedule.run_interval_style_id = run_interval_styles['after previous run']
+        else
+          schedule.run_interval = 0
+          schedule.run_interval_style_id = run_interval_styles['at beginning of day']
+        end
+
+        schedule.save!
+      end
+    end
+
+    def validate_applications
+      applications = []
+      ::Naf::Application.all.each do |application|
+        if application.application_schedules.count > 1
+          applications << application.title
+        end
+      end
+
+      if applications.present?
+        logger.warn "The following applications have more than one schedule: #{applications.join(', ')}. " +
+        "In order to rollback successfully, an application can have at most one schedule."
+        return false
+      else
+        return true
+      end
+    end
+
+    def rollback_changes
+      ::Naf::ApplicationSchedule.all.each do |schedule|
+        logger.info "Updating application schedule: #{schedule.id}"
+
+        if schedule.run_interval_style.name == 'at beginning of day'
+          schedule.run_start_minute = schedule.run_interval
+          schedule.run_interval = nil
+        else
+          schedule.run_start_minute = nil
+        end
+
+        schedule.save!
+      end
+    end
+
+  end
+end

data/app/models/process/naf/log_archiver.rb

@@ -4,7 +4,7 @@ module Process::Naf
   class LogArchiver < ::Process::Naf::Application
 
     NAF_JOBS_LOG_PATH = "#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/"
-    NAF_RUNNERS_LOG_PATH = "#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/*/*"
+    NAF_RUNNERS_LOG_PATH = "#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/"
     DATE_REGEX = /\d{8}_\d{6}/
     LOG_RETENTION = 1
 

data/app/models/process/naf/log_reader.rb

@@ -0,0 +1,141 @@
+require 'yajl'
+
+module Process::Naf
+  class LogReader < ::Process::Naf::Application
+
+    opt :job_id, type: :int
+    opt :runner_id, type: :int
+    opt_select :job_id_or_runner_id, one_of: [:job_id, :runner_id]
+
+    UUID_REGEX = /[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}/
+
+    attr_accessor :read_from_s3,
+                  :s3_log_reader,
+                  :jsons
+
+  	def work
+      @jsons = []
+  		if @job_id.present?
+        job_log_files.each do |file|
+          # Use Yajl JSON library to parse the log files, as they contain multiple JSON blocks
+          parser = Yajl::Parser.new
+          json = get_json_from_log_file(file)
+          parser.parse(json) do |log|
+            @jsons << log
+          end
+
+          sort_jsons
+        end
+
+        if jsons.present?
+          jsons.each do |elem|
+            puts insert_log_line(elem)
+          end
+        else
+          puts 'No logs found'
+        end
+  		elsif @runner_id.present?
+        runner_log_files.each do |file|
+          # Use Yajl JSON library to parse the log files, as they contain multiple JSON blocks
+          parser = Yajl::Parser.new
+          json = get_json_from_log_file(file)
+          parser.parse(json) do |log|
+            log['id'] = get_invocation_id(file.scan(UUID_REGEX).first)
+            @jsons << log
+          end
+
+          sort_jsons
+        end
+
+        if jsons.present?
+          jsons.each do |elem|
+            puts insert_log_line(elem)
+          end
+        else
+          puts 'No logs found'
+        end
+  		end
+  	end
+
+  	private
+
+    def job_log_files
+      files = Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{@job_id}/*"]
+      if files.present?
+        @read_from_s3 = false
+        # Sort log files based on time
+        return files.sort { |x, y| Time.parse(y.scan(::Logical::Naf::LogParser::Base::DATE_REGEX)[0][0]) <=> Time.parse(x.scan(::Logical::Naf::LogParser::Base::DATE_REGEX)[0][0]) }
+      else
+        get_s3_files do
+          @read_from_s3 = true
+          @s3_log_reader = ::Logical::Naf::LogReader.new
+          return s3_log_reader.retrieve_job_files(@job_id)
+        end
+      end
+    end
+
+    def runner_log_files
+      invocation_uuids = ::Naf::MachineRunnerInvocation.where(machine_runner_id: @runner_id).map(&:uuid)
+      files = []
+      invocation_uuids.each do |uuid|
+        if Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/#{uuid}/*"].present?
+          files += Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/runners/#{uuid}/*"]
+        end
+      end
+
+      if files.present?
+        @read_from_s3 = false
+        # Sort log files based on time
+        return files.sort { |x, y| Time.parse(y.scan(::Logical::Naf::LogParser::Base::DATE_REGEX)[0][0]) <=> Time.parse(x.scan(::Logical::Naf::LogParser::Base::DATE_REGEX)[0][0]) }
+      else
+        get_s3_files do
+          @read_from_s3 = true
+          @s3_log_reader = ::Logical::Naf::LogReader.new
+          s3_log_reader.runner_log_files.each do |file|
+            if invocation_uuids.include?(file.scan(UUID_REGEX).first)
+              files << file
+            end
+          end
+
+          return files
+        end
+      end
+    end
+
+    def get_s3_files
+      begin
+        yield
+      rescue
+        logger.info 'AWS S3 Access Denied. Please check your permissions.'
+        return []
+      end
+    end
+
+    def get_json_from_log_file(file)
+      if read_from_s3 == true
+        @s3_log_reader.retrieve_file(file)
+      else
+        File.new(file, 'r')
+      end
+    end
+
+    def sort_jsons
+      # Sort log lines based on timestamp
+      @jsons = jsons.sort { |x, y| x['line_number'] <=> y['line_number'] }
+    end
+
+    def insert_log_line(elem)
+      if @job_id
+        "#{elem['line_number']} #{elem['output_time']}: #{elem['message']}"
+      elsif @runner_id
+        "#{elem['output_time']} invocation(#{elem['id']}): #{elem['message']}"
+      end
+    end
+
+    def get_invocation_id(uuid)
+      ::Naf::MachineRunnerInvocation.find_by_uuid(uuid).id
+    end
+
+
+  end
+end

data/app/models/process/naf/runner.rb

@@ -18,9 +18,6 @@ module Process::Naf
         "time between checking schedules",
         argument_note: "MINUTES",
         default: 1
-    opt :schedule_fudge_scale,
-        "amount of time to look back in schedule for run_start_minute schedules (scaled to --check-schedule-period)",
-        default: 5
     opt :runner_stale_period,
         "amount of time to consider a machine out of touch if it hasn't updated its machine entry",
         argument_note: "MINUTES",
@@ -253,12 +250,12 @@ module Process::Naf
           ::Naf::ApplicationSchedule.unlock_schedules
 
           # check scheduled tasks
-          should_be_queued.each do |application_schedule|
+          ::Naf::ApplicationSchedule.should_be_queued.each do |application_schedule|
             logger.info escape_html("scheduled application: #{application_schedule}")
             begin
               naf_boss = ::Logical::Naf::ConstructionZone::Boss.new
               # this doesn't work very well for run_group_limits in the thousands
-              Range.new(0, application_schedule.application_run_group_limit || 1, true).each do
+              Range.new(0, application_schedule.application_run_group_quantum || 1, true).each do
                 naf_boss.enqueue_application_schedule(application_schedule)
               end
             rescue ::Naf::HistoricalJob::JobPrerequisiteLoop => jpl
@@ -608,48 +605,6 @@ module Process::Naf
       end
     end
 
-    def should_be_queued
-      not_finished_applications = ::Naf::HistoricalJob.
-        queued_between(Time.zone.now - Naf::HistoricalJob::JOB_STALE_TIME, Time.zone.now).
-        where("finished_at IS NULL AND request_to_terminate = false").
-        find_all{ |job| job.application_id.present? }.
-        index_by{ |job| job.application_id }
-
-      application_last_runs = ::Naf::HistoricalJob.application_last_runs.
-        index_by{ |job| job.application_id }
-
-      # find the run_interval based schedules that should be queued
-      # select anything that isn't currently running and completed
-      # running more than run_interval minutes ago
-      relative_schedules_what_need_queuin = ::Naf::ApplicationSchedule.where(enabled: true).relative_schedules.select do |schedule|
-        (not_finished_applications[schedule.application_id].nil? &&
-          (application_last_runs[schedule.application_id].nil? ||
-            (Time.zone.now - application_last_runs[schedule.application_id].finished_at) > (schedule.run_interval.minutes)))
-      end
-
-      # find the run_start_minute based schedules
-      # select anything that
-      #  isn't currently running (or queued) AND
-      #  hasn't run since run_start_time AND
-      #  should have been run by now AND
-      #  that should have run within fudge period AND
-      exact_schedules_what_need_queuin = ::Naf::ApplicationSchedule.where(enabled: true).exact_schedules.select do |schedule|
-        (not_finished_applications[schedule.application_id].nil? &&
-          (application_last_runs[schedule.application_id].nil? ||
-            ((Time.zone.now.to_date + schedule.run_start_minute.minutes) >= application_last_runs[schedule.application_id].finished_at)) &&
-          (Time.zone.now - (Time.zone.now.to_date + schedule.run_start_minute.minutes)) >= 0.seconds &&
-          ((Time.zone.now - (Time.zone.now.to_date + schedule.run_start_minute.minutes)) <= (@check_schedules_period * @schedule_fudge_scale).minutes)
-        )
-      end
-
-      foreman = ::Logical::Naf::ConstructionZone::Foreman.new()
-      return (relative_schedules_what_need_queuin + exact_schedules_what_need_queuin).select do |schedule|
-        schedule.enqueue_backlogs || !foreman.limited_by_run_group?(schedule.application_run_group_restriction,
-                                                                    schedule.application_run_group_name,
-                                                                    schedule.application_run_group_limit)
-      end
-    end
-
     def memory_available_to_spawn?
       Facter.clear
       memory_size = Facter.memorysize_mb.to_f
@@ -657,10 +612,12 @@ module Process::Naf
       memory_free_percentage = (memory_free / memory_size) * 100.0
 
       if (memory_free_percentage >= @minimum_memory_free)
-        logger.detail "memory available: #{memory_free_percentage}% (free) >= #{@minimum_memory_free}% (min percent)"
+        logger.detail "memory available: #{memory_free_percentage}% (free) >= " +
+          "#{@minimum_memory_free}% (min percent)"
         return true
       end
-      logger.alarm "#{Facter.hostname}.#{Facter.domain}: not enough memory to spawn: #{memory_free_percentage}% (free) < #{@minimum_memory_free}% (min percent)"
+      logger.alarm "#{Facter.hostname}.#{Facter.domain}: not enough memory to spawn: " +
+        "#{memory_free_percentage}% (free) < #{@minimum_memory_free}% (min percent)"
 
       return false
     end

data/app/views/naf/application_schedule_affinity_tabs/_form.html.erb

@@ -1,4 +1,4 @@
-<%= form_for([@application, @application_schedule, @tab]) do |f| %>
+<%= form_for([@application_schedule, @tab]) do |f| %>
 
   <% if @tab.errors.any? %>
   <div id="errorExplanation">

data/app/views/naf/{applications → application_schedules}/_application_schedule_prerequisites.html.erb

@@ -7,7 +7,7 @@
     <div class="field">
       <%= f.label :prerequisite_application_schedule_id, "Prerequisite application schedule*" %>
       <br/>
-      <% schedules = ::Naf::ApplicationSchedule.all.map { |a| [a.title, a.id] unless a.id == @application.application_schedule.id }.compact %>
+      <% schedules = ::Naf::ApplicationSchedule.all.map { |a| [a.to_s, a.id] unless @application_schedule.id == a.id }.compact %>
       <%= f.select(:prerequisite_application_schedule_id, schedules, include_blank: true) %>
     </div>
     <br/>

data/app/views/naf/application_schedules/_form.html.erb

@@ -0,0 +1,120 @@
+<%= form_for @application_schedule do |f| %>
+  <% if @application_schedule.errors.any? %>
+    <div id='errorExplanation'>
+      <h2><%= pluralize(@application_schedule.errors.count, 'error') %> prohibited this application schedule from being saved:</h2>
+    </div>
+  <% end %>
+
+  <%= f.hidden_field :application_id, value: @application_schedule.application_id %>
+
+  <div class="field">
+    <%= f.label :application_run_group_restriction_id, "Application run group restriction*" %>
+    <%= f.select(:application_run_group_restriction_id,
+                 ::Naf::ApplicationRunGroupRestriction.all.map { |a| [a.application_run_group_restriction_name, a.id] },
+                 include_blank: true) %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :application_run_group_name %>
+    <%= select_tag(:run_group_name_type, options_for_select([["custom", "custom"],
+                                                             ["command", "command"],
+                                                             ["not set", "not set"]],
+                                                             @run_group_name_type)) %><br/>
+    <%= f.text_area :application_run_group_name, rows: 4, cols: 32, placeholder: "Please input Application Run Group Name" %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :run_interval, "Run Interval (minutes)" %>
+    <%= f.text_field :run_interval %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :run_interval_style_id, "Run Interval Style*" %>
+    <%= f.select(:run_interval_style_id,
+                 ::Naf::RunIntervalStyle.all.map { |s| [s.name, s.id] },
+                 include_blank: true) %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :priority %>
+    <%= f.text_field :priority %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :application_run_group_limit %>
+    <%= f.text_field :application_run_group_limit %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :application_run_group_quantum%>
+    <%= f.text_field :application_run_group_quantum %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :enabled %>
+    <%= f.check_box :enabled %>
+  </div>
+  </br/>
+
+  <div class="field">
+    <%= f.label :visible %>
+    <%= f.check_box :visible %>
+  </div>
+  <br/>
+
+  <div class="field">
+    <%= f.label :enqueue_backlogs %>
+    <%= f.check_box :enqueue_backlogs %>
+  </div>
+  <br/>
+
+  <% if @show_prerequisite %>
+    <%= f.fields_for :application_schedule_prerequisites do |builder| %>
+      <%= render "application_schedule_prerequisites", f: builder %>
+    <% end %>
+  <% else %>
+    <% f.fields_for :application_schedule_prerequisites do |builder| %>
+      <%= render "application_schedule_prerequisites", f: builder %>
+    <% end %>
+  <% end %>
+
+  <%= naf_link_to_add_fields "Add Schedule Prerequisite", f, :application_schedule_prerequisites %>
+
+  <div class='actions'>
+    <br/>
+    <%= f.submit %>
+  </div>
+<% end %>
+
+<% content_for :javascripts do %>
+  <script type='text/javascript'>
+    jQuery(document).ready(function() {
+      var run_group = jQuery("#application_schedule_application_run_group_name");
+
+      if(jQuery("#run_group_name_type").val() != 'custom') {
+        run_group.hide();
+        run_group.val(null);
+      }
+
+      jQuery("#run_group_name_type").change(function () {
+        var type = jQuery("#run_group_name_type").val();
+        if(type == 'command') {
+          run_group.hide();
+          run_group.val(null);
+        } else if(type == 'not set') {
+          run_group.hide();
+          run_group.val(null);
+        } else {
+          run_group.slideToggle("slow");
+        }
+      });
+    });
+  </script>
+<% end %>