naf 2.1.12 → 2.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -3
- data/RELEASE_NOTES.rdoc +17 -4
- data/app/assets/images/download.png +0 -0
- data/app/assets/javascripts/dataTablesTemplates/jobs.js +37 -0
- data/app/controllers/naf/historical_jobs_controller.rb +30 -0
- data/app/controllers/naf/log_parsers_controller.rb +9 -0
- data/app/helpers/naf/application_helper.rb +1 -1
- data/app/models/logical/naf/application_schedule.rb +7 -3
- data/app/models/logical/naf/log_parser/base.rb +1 -0
- data/app/models/logical/naf/log_parser/job.rb +4 -3
- data/app/models/logical/naf/log_parser/job_downloader.rb +156 -0
- data/app/models/logical/naf/log_parser/runner.rb +4 -3
- data/app/models/logical/naf/metric_sender.rb +62 -0
- data/app/models/process/naf/database_models_cleanup.rb +91 -0
- data/app/models/process/naf/runner.rb +52 -35
- data/app/views/naf/historical_jobs/_button_control.html.erb +64 -0
- data/app/views/naf/historical_jobs/index.json.erb +26 -5
- data/app/views/naf/historical_jobs/show.html.erb +20 -29
- data/app/views/naf/log_viewer/_job_log_download_button.html.erb +11 -0
- data/app/views/naf/log_viewer/_job_logs.html.erb +3 -0
- data/app/views/naf/log_viewer/_log_display.html.erb +4 -4
- data/app/views/naf/log_viewer/_log_layout.html.erb +5 -0
- data/config/routes.rb +4 -0
- data/lib/naf.rb +8 -0
- data/lib/naf/configuration.rb +5 -1
- data/lib/naf/version.rb +1 -1
- data/naf.gemspec +5 -2
- data/spec/controllers/naf/log_parsers_controller_spec.rb +35 -0
- data/spec/models/logical/naf/application_schedule_spec.rb +41 -0
- data/spec/models/logical/naf/construction_zone/boss_spec.rb +5 -0
- data/spec/models/logical/naf/construction_zone/foreman_spec.rb +6 -3
- data/spec/models/logical/naf/job_downloader_spec.rb +72 -0
- data/spec/models/logical/naf/job_statuses/errored_spec.rb +33 -0
- data/spec/models/logical/naf/job_statuses/finished_less_minute_spec.rb +33 -0
- data/spec/models/logical/naf/job_statuses/finished_spec.rb +33 -0
- data/spec/models/logical/naf/job_statuses/queued_spec.rb +34 -0
- data/spec/models/logical/naf/job_statuses/running_spec.rb +37 -0
- data/spec/models/logical/naf/job_statuses/terminated_spec.rb +33 -0
- data/spec/models/logical/naf/job_statuses/waiting_spec.rb +33 -0
- metadata +80 -6
data/Gemfile
CHANGED
@@ -7,12 +7,13 @@ group :assets do
|
|
7
7
|
gem 'coffee-rails', '~> 3.2.1'
|
8
8
|
gem 'uglifier', '>= 1.0.3'
|
9
9
|
end
|
10
|
+
|
10
11
|
gem 'jquery-ui-rails'
|
11
|
-
gem 'awesome_print'
|
12
12
|
gem 'will_paginate'
|
13
13
|
gem 'facter', '1.7.5'
|
14
14
|
gem 'shoulda-matchers', '2.0.0'
|
15
15
|
gem 'timecop', '0.4.5'
|
16
|
-
gem 'yajl-ruby'
|
17
|
-
gem 'aws-sdk'
|
16
|
+
gem 'yajl-ruby', '>= 1.1.0'
|
17
|
+
gem 'aws-sdk', '>= 1.1.0'
|
18
18
|
gem 'fiksu-af'
|
19
|
+
gem 'dogstatsd-ruby', '>= 1.2.0'
|
data/RELEASE_NOTES.rdoc
CHANGED
@@ -1,5 +1,18 @@
|
|
1
1
|
= Release Notes
|
2
2
|
|
3
|
+
=== Version 2.1.13
|
4
|
+
Bug fixes:
|
5
|
+
* Large run interval for application schedule causes argument out of range error
|
6
|
+
|
7
|
+
Features/Changes:
|
8
|
+
* Cleanup script to remove data from the Naf schema
|
9
|
+
* aws-sdk and yajl-ruby gems added in naf.gemspec
|
10
|
+
* Html escape characters in log display instead of log files
|
11
|
+
* Scripts tab renamed to Applications
|
12
|
+
* Ability to download job logs
|
13
|
+
* Ability to re-enqueue finished/errored/terminated job
|
14
|
+
* Send Datadog metrics on runners
|
15
|
+
|
3
16
|
=== Version 2.1.12
|
4
17
|
Bug fixes:
|
5
18
|
* Logical::Naf::ConstructionZone#enqueue_application missing application_schedule parameter
|
@@ -10,7 +23,7 @@ Bug fixes:
|
|
10
23
|
* Adding application schedule prerequisites works correctly
|
11
24
|
* Log display outputs custom message when record id is not present
|
12
25
|
|
13
|
-
Changes:
|
26
|
+
Features/Changes:
|
14
27
|
* Only show affinities associated with machines that are not deleted
|
15
28
|
* Runner will cleanup other runners after they are dead
|
16
29
|
* Machine manager updates machines row if server address or server name match
|
@@ -23,7 +36,7 @@ Bug fixes:
|
|
23
36
|
* Custom validation for machine marked as enabled and deleted
|
24
37
|
* UI links behave correctly if the engine's mount path is different than '/job_system'
|
25
38
|
|
26
|
-
Changes:
|
39
|
+
Features/Changes:
|
27
40
|
* Improved memory management
|
28
41
|
* Removed machine log display
|
29
42
|
|
@@ -46,7 +59,7 @@ Bug fixes:
|
|
46
59
|
* Deleted application still runs if it has an enabled schedule
|
47
60
|
* LogArchiver did not delete empty folders
|
48
61
|
|
49
|
-
Changes:
|
62
|
+
Features/Changes:
|
50
63
|
* Created a log archival queuer script
|
51
64
|
* Created Api Authenticator for logs
|
52
65
|
|
@@ -56,7 +69,7 @@ Bug fixes:
|
|
56
69
|
* Pagination of application schedules on the index page was not properly working.
|
57
70
|
* When mouse hovered over the application schedule on the applicaiton show page, it did not show a detailed information about the schedule.
|
58
71
|
|
59
|
-
Changes:
|
72
|
+
Features/Changes:
|
60
73
|
* Included application name when editing an application schedule
|
61
74
|
|
62
75
|
=== Version 2.1.5
|
Binary file
|
@@ -75,6 +75,43 @@ jQuery(document).ready(function() {
|
|
75
75
|
}
|
76
76
|
});
|
77
77
|
});
|
78
|
+
|
79
|
+
jQuery(document).delegate('.re-enqueue', "click", function(){
|
80
|
+
var url = jQuery(this).attr('content');
|
81
|
+
var new_params = { data: jQuery(this).attr('data') };
|
82
|
+
new_params['job_id'] = jQuery(this).attr('id');
|
83
|
+
|
84
|
+
if (jQuery(this).attr('app_id')) {
|
85
|
+
new_params['app_id'] = jQuery(this).attr('app_id');
|
86
|
+
}
|
87
|
+
|
88
|
+
if (jQuery(this).attr('link')) {
|
89
|
+
new_params['link'] = jQuery(this).attr('link');
|
90
|
+
}
|
91
|
+
|
92
|
+
if (jQuery(this).attr('title_name')) {
|
93
|
+
new_params['title_name'] = jQuery(this).attr('title_name');
|
94
|
+
}
|
95
|
+
|
96
|
+
var answer = confirm("Would you like to enqueue this job?");
|
97
|
+
|
98
|
+
if (!answer) {
|
99
|
+
return false;
|
100
|
+
}
|
101
|
+
jQuery.post(url, new_params, function (data) {
|
102
|
+
if (data.success) {
|
103
|
+
jQuery("<p id='notice'>Congratulations, a Job " + data.title + " was added!</p>").
|
104
|
+
appendTo('#flash_message').slideDown().delay(5000).slideUp();
|
105
|
+
setTimeout('window.location.reload()', 5600);
|
106
|
+
}
|
107
|
+
else {
|
108
|
+
jQuery("<div class='error'>Sorry, \'" + data.title +
|
109
|
+
"\' cannot add a Job to the queue right now!</div>").
|
110
|
+
appendTo('#flash_message').slideDown().delay(5000).slideUp();
|
111
|
+
jQuery('#datatable').dataTable().fnDraw();
|
112
|
+
}
|
113
|
+
});
|
114
|
+
});
|
78
115
|
});
|
79
116
|
|
80
117
|
function addLinkToJob(nRow, aData) {
|
@@ -44,6 +44,36 @@ module Naf
|
|
44
44
|
@historical_job = Naf::HistoricalJob.new
|
45
45
|
end
|
46
46
|
|
47
|
+
# If there is an application id specified, then the controller enqueues that application
|
48
|
+
def reenqueue
|
49
|
+
job = Naf::HistoricalJob.find(params[:job_id].to_i)
|
50
|
+
success = false
|
51
|
+
if params[:app_id].present?
|
52
|
+
app = Naf::Application.find(params[:app_id].to_i)
|
53
|
+
title = app.title
|
54
|
+
@historical_job = ::Logical::Naf::ConstructionZone::Boss.new.enqueue_application(
|
55
|
+
app,
|
56
|
+
job.application_run_group_restriction,
|
57
|
+
job.application_run_group_name,
|
58
|
+
job.application_run_group_limit,
|
59
|
+
job.priority,
|
60
|
+
job.job_affinities,
|
61
|
+
job.prerequisites,
|
62
|
+
false,
|
63
|
+
job.application_schedule)
|
64
|
+
if @historical_job.present?
|
65
|
+
success = true
|
66
|
+
end
|
67
|
+
else
|
68
|
+
title = job.command
|
69
|
+
@historical_job = ::Logical::Naf::ConstructionZone::Boss.new.reenqueue(job)
|
70
|
+
if @historical_job.present?
|
71
|
+
success = true
|
72
|
+
end
|
73
|
+
end
|
74
|
+
render json: { success: success, title: title }.to_json
|
75
|
+
end
|
76
|
+
|
47
77
|
def create
|
48
78
|
@historical_job = Naf::HistoricalJob.new(params[:historical_job])
|
49
79
|
if params[:historical_job][:application_id] &&
|
@@ -24,5 +24,14 @@ module Naf
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
+
def download
|
28
|
+
job_log_downloader = Logical::Naf::LogParser::JobDownloader.new({
|
29
|
+
'record_id' => params[:record_id]
|
30
|
+
})
|
31
|
+
logs = job_log_downloader.logs_for_download + "\n"
|
32
|
+
send_data logs, filename: "job_#{params[:record_id]}_log.txt",
|
33
|
+
type: "text/plain", disposition: 'attachment'
|
34
|
+
end
|
35
|
+
|
27
36
|
end
|
28
37
|
end
|
@@ -168,7 +168,7 @@ module Naf
|
|
168
168
|
when "runners"
|
169
169
|
link_to "Runners", naf.machine_runners_path
|
170
170
|
when "scripts"
|
171
|
-
link_to "
|
171
|
+
link_to "Applications", naf.applications_path
|
172
172
|
when "janitorial_assignments"
|
173
173
|
link_to "Janitorial Assignments", naf.janitorial_archive_assignments_path
|
174
174
|
when "janitorial_archive_assignments"
|
@@ -128,7 +128,7 @@ module Logical
|
|
128
128
|
output = ''
|
129
129
|
time = schedule.run_interval
|
130
130
|
if schedule.run_interval_style.name == 'at beginning of day'
|
131
|
-
output = exact_time_of_day
|
131
|
+
output = exact_time_of_day
|
132
132
|
else
|
133
133
|
output = interval_time(time)
|
134
134
|
end
|
@@ -136,11 +136,15 @@ module Logical
|
|
136
136
|
output
|
137
137
|
end
|
138
138
|
|
139
|
-
def exact_time_of_day
|
139
|
+
def exact_time_of_day
|
140
140
|
output = ''
|
141
141
|
minutes = schedule.run_interval % 60
|
142
142
|
hours = schedule.run_interval / 60
|
143
|
-
|
143
|
+
if hours >= 24
|
144
|
+
output << (hours % 24).to_s + ':'
|
145
|
+
else
|
146
|
+
output << hours.to_s + ':'
|
147
|
+
end
|
144
148
|
output << '%02d' % minutes
|
145
149
|
output = Time.parse(output).strftime('%I:%M %p')
|
146
150
|
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'yajl'
|
2
|
-
|
3
1
|
module Logical::Naf
|
4
2
|
module LogParser
|
5
3
|
class Job < Base
|
@@ -15,7 +13,10 @@ module Logical::Naf
|
|
15
13
|
private
|
16
14
|
|
17
15
|
def insert_log_line(elem)
|
18
|
-
"
|
16
|
+
output_line = "<span><pre style='display: inline; word-wrap: break-word;'>"
|
17
|
+
output_line += "#{elem['line_number']} #{elem['output_time']}: #{elem['message']}"
|
18
|
+
output_line += "</pre></br></span>"
|
19
|
+
output_line
|
19
20
|
end
|
20
21
|
|
21
22
|
def sort_jsons
|
@@ -0,0 +1,156 @@
|
|
1
|
+
module Logical::Naf
|
2
|
+
module LogParser
|
3
|
+
class JobDownloader < Base
|
4
|
+
|
5
|
+
############################################################################
|
6
|
+
# Description
|
7
|
+
# -----------
|
8
|
+
# Logical model to return the contents of all log files
|
9
|
+
# associated with a specific job ID.
|
10
|
+
#
|
11
|
+
# Interface
|
12
|
+
# ---------
|
13
|
+
# initialize: Pass intitialize a hash of only one parameter ('record_id')
|
14
|
+
# Initializes the JobDownloader by linking it to a record_id
|
15
|
+
#
|
16
|
+
# logs_for_download: Takes no arguments
|
17
|
+
# Returns a string containing all parsed log messages (plain text)
|
18
|
+
# from all local and S3 files associated with the JobDownloader's
|
19
|
+
# record_id.
|
20
|
+
#
|
21
|
+
############################################################################
|
22
|
+
|
23
|
+
# Description: Initializes instance variables for the class
|
24
|
+
# params: Params must include 'record_id' key/value
|
25
|
+
# Note: Only uses record_id from params
|
26
|
+
def initialize(params)
|
27
|
+
@jsons = []
|
28
|
+
@record_id = params['record_id']
|
29
|
+
@read_from_s3 = true;
|
30
|
+
end
|
31
|
+
|
32
|
+
# Description: Public method used to return a string of all logs
|
33
|
+
# The string contains all parsed json elements from all
|
34
|
+
# accessible files (local and s3) for @record_id
|
35
|
+
# Returns: String (of logs)
|
36
|
+
def logs_for_download
|
37
|
+
retrieve_logs_for_download
|
38
|
+
end
|
39
|
+
|
40
|
+
###################################################
|
41
|
+
private
|
42
|
+
###################################################
|
43
|
+
|
44
|
+
# Description: Returns a string containing all parsed json elements form all
|
45
|
+
# accessible files (local and s3) for @record_id
|
46
|
+
# Returns: String (of logs)
|
47
|
+
def retrieve_logs_for_download
|
48
|
+
parse_files_for_download
|
49
|
+
|
50
|
+
output = ''
|
51
|
+
jsons.reverse_each do |elem|
|
52
|
+
output.insert(0, insert_log_line_for_download(elem))
|
53
|
+
end
|
54
|
+
|
55
|
+
output
|
56
|
+
end
|
57
|
+
|
58
|
+
# Description: Formats a single log line
|
59
|
+
# elem: Takes in a hash of a single log entry,
|
60
|
+
# (originating from Yajl::Parser parsing json)
|
61
|
+
# Returns: String (single log line, formatted as plain text, without any added html)
|
62
|
+
def insert_log_line_for_download(elem)
|
63
|
+
if elem['message'].include? "AWS S3 Access Denied. Please check your permissions"
|
64
|
+
# If it cannot access AWS S3, the JobDownlaoder will still return all the logs stored locally.
|
65
|
+
output_line = ""
|
66
|
+
else
|
67
|
+
output_line = "#{elem['line_number']} #{elem['output_time']}: #{elem['message']}\n"
|
68
|
+
end
|
69
|
+
|
70
|
+
return output_line
|
71
|
+
end
|
72
|
+
|
73
|
+
# Acts on instance variable @jsons (sorts them)
|
74
|
+
def sort_jsons
|
75
|
+
# Sort log lines based on timestamp
|
76
|
+
@jsons = jsons.sort { |x, y| x['line_number'] <=> y['line_number'] }
|
77
|
+
end
|
78
|
+
|
79
|
+
# Calls the Logical::Naf::LogReader retrieve_job_files method for record_id
|
80
|
+
# Returns: Array of file names on S3 corresponding to record_id
|
81
|
+
def retrieve_log_files_from_s3
|
82
|
+
s3_log_reader.retrieve_job_files(record_id)
|
83
|
+
end
|
84
|
+
|
85
|
+
# Description: Finds file names (across local and S3) associated with record_id
|
86
|
+
# Returns: Hash (file_name => is_on_s3_bool)
|
87
|
+
def get_files_for_download
|
88
|
+
files_to_download = {}
|
89
|
+
s3_files = []
|
90
|
+
|
91
|
+
# S3
|
92
|
+
get_s3_files do
|
93
|
+
@s3_log_reader = ::Logical::Naf::LogReader.new
|
94
|
+
s3_files = retrieve_log_files_from_s3
|
95
|
+
end
|
96
|
+
|
97
|
+
# Add S3 files to the hash, mapping them to true (need to read from s3)
|
98
|
+
s3_files.each do |file_to_add|
|
99
|
+
files_to_download[file_to_add] = true
|
100
|
+
end
|
101
|
+
|
102
|
+
return files_to_download unless record_id.present?
|
103
|
+
|
104
|
+
# Non-S3
|
105
|
+
if File.directory?("#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{record_id}")
|
106
|
+
files = Dir["#{::Naf::PREFIX_PATH}/#{::Naf.schema_name}/jobs/#{record_id}/*"]
|
107
|
+
else
|
108
|
+
return files_to_download
|
109
|
+
end
|
110
|
+
|
111
|
+
if files.present?
|
112
|
+
# Sort log files based on time and add them the hash (mapped to false)
|
113
|
+
files.sort { |x, y| Time.parse(y.scan(DATE_REGEX)[0][0]) <=> Time.parse(x.scan(DATE_REGEX)[0][0]) }.each do |file_to_add|
|
114
|
+
files_to_download[file_to_add] = false
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
return files_to_download
|
119
|
+
end
|
120
|
+
|
121
|
+
# Either read local file or retrieve contents from s3 as needed
|
122
|
+
# Returns contents of file
|
123
|
+
def get_json_from_log_file_for_download(file)
|
124
|
+
if @read_from_s3 == true
|
125
|
+
if s3_log_reader.present?
|
126
|
+
s3_log_reader.retrieve_file(file)
|
127
|
+
end
|
128
|
+
else
|
129
|
+
File.new(file, 'r')
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# Retrieves file_names and iterates through them, using yajl to parse the json
|
134
|
+
# Adds all jsons to the instance variable @jsons
|
135
|
+
def parse_files_for_download
|
136
|
+
files = get_files_for_download # now a hash
|
137
|
+
|
138
|
+
unless files.present?
|
139
|
+
return ""
|
140
|
+
else
|
141
|
+
files.each_pair do |file, s3_needed|
|
142
|
+
# Use Yajl JSON library to parse the log files, as they contain multiple JSON blocks
|
143
|
+
parser = Yajl::Parser.new
|
144
|
+
@read_from_s3 = s3_needed
|
145
|
+
json = get_json_from_log_file_for_download(file)
|
146
|
+
parser.parse(json) do |log|
|
147
|
+
@jsons << log
|
148
|
+
end
|
149
|
+
sort_jsons
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'yajl'
|
2
|
-
|
3
1
|
module Logical::Naf
|
4
2
|
module LogParser
|
5
3
|
class Runner < Base
|
@@ -18,7 +16,10 @@ module Logical::Naf
|
|
18
16
|
private
|
19
17
|
|
20
18
|
def insert_log_line(elem)
|
21
|
-
"
|
19
|
+
output_line = "<span><pre style='display: inline; word-wrap: break-word;'>"
|
20
|
+
output_line += CGI::unescapeHTML("#{elem['output_time']} #{invocation_link(elem['id'])}:")
|
21
|
+
output_line += " #{elem['message']}</pre></br></span>"
|
22
|
+
output_line
|
22
23
|
end
|
23
24
|
|
24
25
|
def invocation_link(id)
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'statsd'
|
2
|
+
|
3
|
+
module Logical
|
4
|
+
module Naf
|
5
|
+
class MetricSender
|
6
|
+
|
7
|
+
attr_reader :statsd,
|
8
|
+
:machine,
|
9
|
+
:metric_send_delay
|
10
|
+
|
11
|
+
attr_accessor :last_sent_metrics
|
12
|
+
|
13
|
+
def initialize(metric_send_delay, machine)
|
14
|
+
@metric_send_delay = metric_send_delay
|
15
|
+
@statsd = Statsd.new
|
16
|
+
@last_sent_metrics = nil
|
17
|
+
@machine = machine
|
18
|
+
end
|
19
|
+
|
20
|
+
# Instance methods
|
21
|
+
|
22
|
+
def send_metrics
|
23
|
+
if last_sent_metrics.nil? || (Time.zone.now - last_sent_metrics) > metric_send_delay.seconds
|
24
|
+
running_job_count = ::Naf::HistoricalJob.where(
|
25
|
+
"(started_at IS NOT NULL AND finished_at IS NULL AND " +
|
26
|
+
"started_on_machine_id = ?)",
|
27
|
+
machine.id).count +
|
28
|
+
::Naf::HistoricalJob.where(
|
29
|
+
"(started_at IS NOT NULL AND finished_at > ? AND " +
|
30
|
+
"started_on_machine_id = ?)",
|
31
|
+
Time.zone.now - metric_send_delay.seconds, machine.id).count
|
32
|
+
terminating_job_count = ::Naf::HistoricalJob.where(
|
33
|
+
"(finished_at IS NULL AND request_to_terminate = true AND " +
|
34
|
+
"started_on_machine_id = ?)",
|
35
|
+
machine.id).count
|
36
|
+
long_terminating_job_count = ::Naf::HistoricalJob.where(
|
37
|
+
"(finished_at IS NULL AND request_to_terminate = true AND " +
|
38
|
+
"updated_at < ? AND started_on_machine_id = ?)",
|
39
|
+
Time.zone.now - 30.minutes, machine.id).count
|
40
|
+
recent_errored_job_count = ::Naf::HistoricalJob.where(
|
41
|
+
"(finished_at IS NOT NULL AND exit_status > 0 AND " +
|
42
|
+
"finished_at > ? AND request_to_terminate = false " +
|
43
|
+
"AND started_on_machine_id = ?)",
|
44
|
+
Time.zone.now - metric_send_delay.seconds, machine.id).count
|
45
|
+
|
46
|
+
statsd.gauge("naf.runner.alive",
|
47
|
+
1, tags: ::Naf.configuration.metric_tags)
|
48
|
+
statsd.gauge("naf.jobs.running",
|
49
|
+
running_job_count, tags: ::Naf.configuration.metric_tags)
|
50
|
+
statsd.gauge("naf.jobs.terminating",
|
51
|
+
terminating_job_count, tags: ::Naf.configuration.metric_tags)
|
52
|
+
statsd.gauge("naf.jobs.terminating-long",
|
53
|
+
long_terminating_job_count, tags: ::Naf.configuration.metric_tags)
|
54
|
+
statsd.gauge("naf.jobs.recent-errored",
|
55
|
+
recent_errored_job_count, tags: ::Naf.configuration.metric_tags)
|
56
|
+
last_sent_metrics = Time.zone.now
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|