naf 2.1.12 → 2.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/Gemfile +4 -3
  2. data/RELEASE_NOTES.rdoc +17 -4
  3. data/app/assets/images/download.png +0 -0
  4. data/app/assets/javascripts/dataTablesTemplates/jobs.js +37 -0
  5. data/app/controllers/naf/historical_jobs_controller.rb +30 -0
  6. data/app/controllers/naf/log_parsers_controller.rb +9 -0
  7. data/app/helpers/naf/application_helper.rb +1 -1
  8. data/app/models/logical/naf/application_schedule.rb +7 -3
  9. data/app/models/logical/naf/log_parser/base.rb +1 -0
  10. data/app/models/logical/naf/log_parser/job.rb +4 -3
  11. data/app/models/logical/naf/log_parser/job_downloader.rb +156 -0
  12. data/app/models/logical/naf/log_parser/runner.rb +4 -3
  13. data/app/models/logical/naf/metric_sender.rb +62 -0
  14. data/app/models/process/naf/database_models_cleanup.rb +91 -0
  15. data/app/models/process/naf/runner.rb +52 -35
  16. data/app/views/naf/historical_jobs/_button_control.html.erb +64 -0
  17. data/app/views/naf/historical_jobs/index.json.erb +26 -5
  18. data/app/views/naf/historical_jobs/show.html.erb +20 -29
  19. data/app/views/naf/log_viewer/_job_log_download_button.html.erb +11 -0
  20. data/app/views/naf/log_viewer/_job_logs.html.erb +3 -0
  21. data/app/views/naf/log_viewer/_log_display.html.erb +4 -4
  22. data/app/views/naf/log_viewer/_log_layout.html.erb +5 -0
  23. data/config/routes.rb +4 -0
  24. data/lib/naf.rb +8 -0
  25. data/lib/naf/configuration.rb +5 -1
  26. data/lib/naf/version.rb +1 -1
  27. data/naf.gemspec +5 -2
  28. data/spec/controllers/naf/log_parsers_controller_spec.rb +35 -0
  29. data/spec/models/logical/naf/application_schedule_spec.rb +41 -0
  30. data/spec/models/logical/naf/construction_zone/boss_spec.rb +5 -0
  31. data/spec/models/logical/naf/construction_zone/foreman_spec.rb +6 -3
  32. data/spec/models/logical/naf/job_downloader_spec.rb +72 -0
  33. data/spec/models/logical/naf/job_statuses/errored_spec.rb +33 -0
  34. data/spec/models/logical/naf/job_statuses/finished_less_minute_spec.rb +33 -0
  35. data/spec/models/logical/naf/job_statuses/finished_spec.rb +33 -0
  36. data/spec/models/logical/naf/job_statuses/queued_spec.rb +34 -0
  37. data/spec/models/logical/naf/job_statuses/running_spec.rb +37 -0
  38. data/spec/models/logical/naf/job_statuses/terminated_spec.rb +33 -0
  39. data/spec/models/logical/naf/job_statuses/waiting_spec.rb +33 -0
  40. metadata +80 -6
data/Gemfile CHANGED
@@ -7,12 +7,13 @@ group :assets do
7
7
  gem 'coffee-rails', '~> 3.2.1'
8
8
  gem 'uglifier', '>= 1.0.3'
9
9
  end
10
+
10
11
  gem 'jquery-ui-rails'
11
- gem 'awesome_print'
12
12
  gem 'will_paginate'
13
13
  gem 'facter', '1.7.5'
14
14
  gem 'shoulda-matchers', '2.0.0'
15
15
  gem 'timecop', '0.4.5'
16
- gem 'yajl-ruby'
17
- gem 'aws-sdk'
16
+ gem 'yajl-ruby', '>= 1.1.0'
17
+ gem 'aws-sdk', '>= 1.1.0'
18
18
  gem 'fiksu-af'
19
+ gem 'dogstatsd-ruby', '>= 1.2.0'
@@ -1,5 +1,18 @@
1
1
  = Release Notes
2
2
 
3
+ === Version 2.1.13
4
+ Bug fixes:
5
+ * Large run interval for application schedule causes argument out of range error
6
+
7
+ Features/Changes:
8
+ * Cleanup script to remove data from the Naf schema
9
+ * aws-sdk and yajl-ruby gems added in naf.gemspec
10
+ * Html escape characters in log display instead of log files
11
+ * Scripts tab renamed to Applications
12
+ * Ability to download job logs
13
+ * Ability to re-enqueue finished/errored/terminated job
14
+ * Send Datadog metrics on runners
15
+
3
16
  === Version 2.1.12
4
17
  Bug fixes:
5
18
  * Logical::Naf::ConstructionZone#enqueue_application missing application_schedule parameter
@@ -10,7 +23,7 @@ Bug fixes:
10
23
  * Adding application schedule prerequisites works correctly
11
24
  * Log display outputs custom message when record id is not present
12
25
 
13
- Changes:
26
+ Features/Changes:
14
27
  * Only show affinities associated with machines that are not deleted
15
28
  * Runner will cleanup other runners after they are dead
16
29
  * Machine manager updates machines row if server address or server name match
@@ -23,7 +36,7 @@ Bug fixes:
23
36
  * Custom validation for machine marked as enabled and deleted
24
37
  * UI links behave correctly if the engine's mount path is different than '/job_system'
25
38
 
26
- Changes:
39
+ Features/Changes:
27
40
  * Improved memory management
28
41
  * Removed machine log display
29
42
 
@@ -46,7 +59,7 @@ Bug fixes:
46
59
  * Deleted application still runs if it has an enabled schedule
47
60
  * LogArchiver did not delete empty folders
48
61
 
49
- Changes:
62
+ Features/Changes:
50
63
  * Created a log archival queuer script
51
64
  * Created Api Authenticator for logs
52
65
 
@@ -56,7 +69,7 @@ Bug fixes:
56
69
  * Pagination of application schedules on the index page was not properly working.
57
70
  * When mouse hovered over the application schedule on the applicaiton show page, it did not show a detailed information about the schedule.
58
71
 
59
- Changes:
72
+ Features/Changes:
60
73
  * Included application name when editing an application schedule
61
74
 
62
75
  === Version 2.1.5
@@ -75,6 +75,43 @@ jQuery(document).ready(function() {
75
75
  }
76
76
  });
77
77
  });
78
+
79
+ jQuery(document).delegate('.re-enqueue', "click", function(){
80
+ var url = jQuery(this).attr('content');
81
+ var new_params = { data: jQuery(this).attr('data') };
82
+ new_params['job_id'] = jQuery(this).attr('id');
83
+
84
+ if (jQuery(this).attr('app_id')) {
85
+ new_params['app_id'] = jQuery(this).attr('app_id');
86
+ }
87
+
88
+ if (jQuery(this).attr('link')) {
89
+ new_params['link'] = jQuery(this).attr('link');
90
+ }
91
+
92
+ if (jQuery(this).attr('title_name')) {
93
+ new_params['title_name'] = jQuery(this).attr('title_name');
94
+ }
95
+
96
+ var answer = confirm("Would you like to enqueue this job?");
97
+
98
+ if (!answer) {
99
+ return false;
100
+ }
101
+ jQuery.post(url, new_params, function (data) {
102
+ if (data.success) {
103
+ jQuery("<p id='notice'>Congratulations, a Job " + data.title + " was added!</p>").
104
+ appendTo('#flash_message').slideDown().delay(5000).slideUp();
105
+ setTimeout('window.location.reload()', 5600);
106
+ }
107
+ else {
108
+ jQuery("<div class='error'>Sorry, \'" + data.title +
109
+ "\' cannot add a Job to the queue right now!</div>").
110
+ appendTo('#flash_message').slideDown().delay(5000).slideUp();
111
+ jQuery('#datatable').dataTable().fnDraw();
112
+ }
113
+ });
114
+ });
78
115
  });
79
116
 
80
117
  function addLinkToJob(nRow, aData) {
@@ -44,6 +44,36 @@ module Naf
44
44
  @historical_job = Naf::HistoricalJob.new
45
45
  end
46
46
 
47
+ # If there is an application id specified, then the controller enqueues that application
48
+ def reenqueue
49
+ job = Naf::HistoricalJob.find(params[:job_id].to_i)
50
+ success = false
51
+ if params[:app_id].present?
52
+ app = Naf::Application.find(params[:app_id].to_i)
53
+ title = app.title
54
+ @historical_job = ::Logical::Naf::ConstructionZone::Boss.new.enqueue_application(
55
+ app,
56
+ job.application_run_group_restriction,
57
+ job.application_run_group_name,
58
+ job.application_run_group_limit,
59
+ job.priority,
60
+ job.job_affinities,
61
+ job.prerequisites,
62
+ false,
63
+ job.application_schedule)
64
+ if @historical_job.present?
65
+ success = true
66
+ end
67
+ else
68
+ title = job.command
69
+ @historical_job = ::Logical::Naf::ConstructionZone::Boss.new.reenqueue(job)
70
+ if @historical_job.present?
71
+ success = true
72
+ end
73
+ end
74
+ render json: { success: success, title: title }.to_json
75
+ end
76
+
47
77
  def create
48
78
  @historical_job = Naf::HistoricalJob.new(params[:historical_job])
49
79
  if params[:historical_job][:application_id] &&
@@ -24,5 +24,14 @@ module Naf
24
24
  end
25
25
  end
26
26
 
27
+ def download
28
+ job_log_downloader = Logical::Naf::LogParser::JobDownloader.new({
29
+ 'record_id' => params[:record_id]
30
+ })
31
+ logs = job_log_downloader.logs_for_download + "\n"
32
+ send_data logs, filename: "job_#{params[:record_id]}_log.txt",
33
+ type: "text/plain", disposition: 'attachment'
34
+ end
35
+
27
36
  end
28
37
  end
@@ -168,7 +168,7 @@ module Naf
168
168
  when "runners"
169
169
  link_to "Runners", naf.machine_runners_path
170
170
  when "scripts"
171
- link_to "Scripts", naf.applications_path
171
+ link_to "Applications", naf.applications_path
172
172
  when "janitorial_assignments"
173
173
  link_to "Janitorial Assignments", naf.janitorial_archive_assignments_path
174
174
  when "janitorial_archive_assignments"
@@ -128,7 +128,7 @@ module Logical
128
128
  output = ''
129
129
  time = schedule.run_interval
130
130
  if schedule.run_interval_style.name == 'at beginning of day'
131
- output = exact_time_of_day(time)
131
+ output = exact_time_of_day
132
132
  else
133
133
  output = interval_time(time)
134
134
  end
@@ -136,11 +136,15 @@ module Logical
136
136
  output
137
137
  end
138
138
 
139
- def exact_time_of_day(time)
139
+ def exact_time_of_day
140
140
  output = ''
141
141
  minutes = schedule.run_interval % 60
142
142
  hours = schedule.run_interval / 60
143
- output << hours.to_s + ':'
143
+ if hours >= 24
144
+ output << (hours % 24).to_s + ':'
145
+ else
146
+ output << hours.to_s + ':'
147
+ end
144
148
  output << '%02d' % minutes
145
149
  output = Time.parse(output).strftime('%I:%M %p')
146
150
 
@@ -95,6 +95,7 @@ module Logical::Naf
95
95
  if self.class.to_s == 'Logical::Naf::LogParser::Runner'
96
96
  log['id'] = get_invocation_id(file.scan(UUID_REGEX).first)
97
97
  end
98
+ log['message'] = CGI::escapeHTML(log['message'])
98
99
  filter_log_messages(log)
99
100
  end
100
101
 
@@ -1,5 +1,3 @@
1
- require 'yajl'
2
-
3
1
  module Logical::Naf
4
2
  module LogParser
5
3
  class Job < Base
@@ -15,7 +13,10 @@ module Logical::Naf
15
13
  private
16
14
 
17
15
  def insert_log_line(elem)
18
- "&nbsp;&nbsp;<span>#{elem['line_number']} #{elem['output_time']}: #{elem['message']}</br></span>"
16
+ output_line = "<span><pre style='display: inline; word-wrap: break-word;'>"
17
+ output_line += "#{elem['line_number']} #{elem['output_time']}: #{elem['message']}"
18
+ output_line += "</pre></br></span>"
19
+ output_line
19
20
  end
20
21
 
21
22
  def sort_jsons
@@ -0,0 +1,156 @@
1
+ module Logical::Naf
2
+ module LogParser
3
+ class JobDownloader < Base
4
+
5
+ ############################################################################
6
+ # Description
7
+ # -----------
8
+ # Logical model to return the contents of all log files
9
+ # associated with a specific job ID.
10
+ #
11
+ # Interface
12
+ # ---------
13
+ # initialize: Pass intitialize a hash of only one parameter ('record_id')
14
+ # Initializes the JobDownloader by linking it to a record_id
15
+ #
16
+ # logs_for_download: Takes no arguments
17
+ # Returns a string containing all parsed log messages (plain text)
18
+ # from all local and S3 files associated with the JobDownloader's
19
+ # record_id.
20
+ #
21
+ ############################################################################
22
+
23
+ # Description: Initializes instance variables for the class
24
+ # params: Params must include 'record_id' key/value
25
+ # Note: Only uses record_id from params
26
+ def initialize(params)
27
+ @jsons = []
28
+ @record_id = params['record_id']
29
+ @read_from_s3 = true;
30
+ end
31
+
32
+ # Description: Public method used to return a string of all logs
33
+ # The string contains all parsed json elements from all
34
+ # accessible files (local and s3) for @record_id
35
+ # Returns: String (of logs)
36
+ def logs_for_download
37
+ retrieve_logs_for_download
38
+ end
39
+
40
+ ###################################################
41
+ private
42
+ ###################################################
43
+
44
+ # Description: Returns a string containing all parsed json elements form all
45
+ # accessible files (local and s3) for @record_id
46
+ # Returns: String (of logs)
47
+ def retrieve_logs_for_download
48
+ parse_files_for_download
49
+
50
+ output = ''
51
+ jsons.reverse_each do |elem|
52
+ output.insert(0, insert_log_line_for_download(elem))
53
+ end
54
+
55
+ output
56
+ end
57
+
58
+ # Description: Formats a single log line
59
+ # elem: Takes in a hash of a single log entry,
60
+ # (originating from Yajl::Parser parsing json)
61
+ # Returns: String (single log line, formatted as plain text, without any added html)
62
+ def insert_log_line_for_download(elem)
63
+ if elem['message'].include? "AWS S3 Access Denied. Please check your permissions"
64
+ # If it cannot access AWS S3, the JobDownlaoder will still return all the logs stored locally.
65
+ output_line = ""
66
+ else
67
+ output_line = "#{elem['line_number']} #{elem['output_time']}: #{elem['message']}\n"
68
+ end
69
+
70
+ return output_line
71
+ end
72
+
73
+ # Acts on instance variable @jsons (sorts them)
74
+ def sort_jsons
75
+ # Sort log lines based on timestamp
76
+ @jsons = jsons.sort { |x, y| x['line_number'] <=> y['line_number'] }
77
+ end
78
+
79
+ # Calls the Logical::Naf::LogReader retrieve_job_files method for record_id
80
+ # Returns: Array of file names on S3 corresponding to record_id
81
+ def retrieve_log_files_from_s3
82
+ s3_log_reader.retrieve_job_files(record_id)
83
+ end
84
+
85
+ # Description: Finds file names (across local and S3) associated with record_id
86
+ # Returns: Hash (file_name => is_on_s3_bool)
87
+ def get_files_for_download
88
+ files_to_download = {}
89
+ s3_files = []
90
+
91
+ # S3
92
+ get_s3_files do
93
+ @s3_log_reader = ::Logical::Naf::LogReader.new
94
+ s3_files = retrieve_log_files_from_s3
95
+ end
96
+
97
+ # Add S3 files to the hash, mapping them to true (need to read from s3)
98
+ s3_files.each do |file_to_add|
99
+ files_to_download[file_to_add] = true
100
+ end
101
+
102
+ return files_to_download unless record_id.present?
103
+
104
+ # Non-S3
105
+ if File.directory?("#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{record_id}")
106
+ files = Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{record_id}/*"]
107
+ else
108
+ return files_to_download
109
+ end
110
+
111
+ if files.present?
112
+ # Sort log files based on time and add them the hash (mapped to false)
113
+ files.sort { |x, y| Time.parse(y.scan(DATE_REGEX)[0][0]) <=> Time.parse(x.scan(DATE_REGEX)[0][0]) }.each do |file_to_add|
114
+ files_to_download[file_to_add] = false
115
+ end
116
+ end
117
+
118
+ return files_to_download
119
+ end
120
+
121
+ # Either read local file or retrieve contents from s3 as needed
122
+ # Returns contents of file
123
+ def get_json_from_log_file_for_download(file)
124
+ if @read_from_s3 == true
125
+ if s3_log_reader.present?
126
+ s3_log_reader.retrieve_file(file)
127
+ end
128
+ else
129
+ File.new(file, 'r')
130
+ end
131
+ end
132
+
133
+ # Retrieves file_names and iterates through them, using yajl to parse the json
134
+ # Adds all jsons to the instance variable @jsons
135
+ def parse_files_for_download
136
+ files = get_files_for_download # now a hash
137
+
138
+ unless files.present?
139
+ return ""
140
+ else
141
+ files.each_pair do |file, s3_needed|
142
+ # Use Yajl JSON library to parse the log files, as they contain multiple JSON blocks
143
+ parser = Yajl::Parser.new
144
+ @read_from_s3 = s3_needed
145
+ json = get_json_from_log_file_for_download(file)
146
+ parser.parse(json) do |log|
147
+ @jsons << log
148
+ end
149
+ sort_jsons
150
+ end
151
+ end
152
+ end
153
+
154
+ end
155
+ end
156
+ end
@@ -1,5 +1,3 @@
1
- require 'yajl'
2
-
3
1
  module Logical::Naf
4
2
  module LogParser
5
3
  class Runner < Base
@@ -18,7 +16,10 @@ module Logical::Naf
18
16
  private
19
17
 
20
18
  def insert_log_line(elem)
21
- "&nbsp;&nbsp;<span>#{elem['output_time']} #{invocation_link(elem['id'])}: #{elem['message']}</br></span>"
19
+ output_line = "<span><pre style='display: inline; word-wrap: break-word;'>"
20
+ output_line += CGI::unescapeHTML("#{elem['output_time']} #{invocation_link(elem['id'])}:")
21
+ output_line += " #{elem['message']}</pre></br></span>"
22
+ output_line
22
23
  end
23
24
 
24
25
  def invocation_link(id)
@@ -0,0 +1,62 @@
1
+ require 'statsd'
2
+
3
+ module Logical
4
+ module Naf
5
+ class MetricSender
6
+
7
+ attr_reader :statsd,
8
+ :machine,
9
+ :metric_send_delay
10
+
11
+ attr_accessor :last_sent_metrics
12
+
13
+ def initialize(metric_send_delay, machine)
14
+ @metric_send_delay = metric_send_delay
15
+ @statsd = Statsd.new
16
+ @last_sent_metrics = nil
17
+ @machine = machine
18
+ end
19
+
20
+ # Instance methods
21
+
22
+ def send_metrics
23
+ if last_sent_metrics.nil? || (Time.zone.now - last_sent_metrics) > metric_send_delay.seconds
24
+ running_job_count = ::Naf::HistoricalJob.where(
25
+ "(started_at IS NOT NULL AND finished_at IS NULL AND " +
26
+ "started_on_machine_id = ?)",
27
+ machine.id).count +
28
+ ::Naf::HistoricalJob.where(
29
+ "(started_at IS NOT NULL AND finished_at > ? AND " +
30
+ "started_on_machine_id = ?)",
31
+ Time.zone.now - metric_send_delay.seconds, machine.id).count
32
+ terminating_job_count = ::Naf::HistoricalJob.where(
33
+ "(finished_at IS NULL AND request_to_terminate = true AND " +
34
+ "started_on_machine_id = ?)",
35
+ machine.id).count
36
+ long_terminating_job_count = ::Naf::HistoricalJob.where(
37
+ "(finished_at IS NULL AND request_to_terminate = true AND " +
38
+ "updated_at < ? AND started_on_machine_id = ?)",
39
+ Time.zone.now - 30.minutes, machine.id).count
40
+ recent_errored_job_count = ::Naf::HistoricalJob.where(
41
+ "(finished_at IS NOT NULL AND exit_status > 0 AND " +
42
+ "finished_at > ? AND request_to_terminate = false " +
43
+ "AND started_on_machine_id = ?)",
44
+ Time.zone.now - metric_send_delay.seconds, machine.id).count
45
+
46
+ statsd.gauge("naf.runner.alive",
47
+ 1, tags: ::Naf.configuration.metric_tags)
48
+ statsd.gauge("naf.jobs.running",
49
+ running_job_count, tags: ::Naf.configuration.metric_tags)
50
+ statsd.gauge("naf.jobs.terminating",
51
+ terminating_job_count, tags: ::Naf.configuration.metric_tags)
52
+ statsd.gauge("naf.jobs.terminating-long",
53
+ long_terminating_job_count, tags: ::Naf.configuration.metric_tags)
54
+ statsd.gauge("naf.jobs.recent-errored",
55
+ recent_errored_job_count, tags: ::Naf.configuration.metric_tags)
56
+ last_sent_metrics = Time.zone.now
57
+ end
58
+ end
59
+
60
+ end
61
+ end
62
+ end