bookie_accounting 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -0
- data/Rakefile +4 -7
- data/TODO +1 -0
- data/bin/bookie-create-tables +1 -1
- data/bin/bookie-send +10 -1
- data/bookie_accounting.gemspec +6 -3
- data/lib/bookie/database.rb +241 -175
- data/lib/bookie/extensions.rb +72 -0
- data/lib/bookie/formatter.rb +15 -7
- data/lib/bookie/formatters/comma_dump.rb +9 -3
- data/lib/bookie/formatters/stdout.rb +1 -1
- data/lib/bookie/sender.rb +76 -16
- data/lib/bookie/senders/standalone.rb +2 -2
- data/lib/bookie/version.rb +1 -1
- data/rpm/bookie_accounting.spec.erb +2 -0
- data/spec/comma_dump_formatter_spec.rb +21 -12
- data/spec/database_spec.rb +199 -103
- data/spec/extensions_spec.rb +25 -0
- data/spec/formatter_spec.rb +37 -32
- data/spec/sender_spec.rb +127 -68
- data/spec/spec_helper.rb +15 -1
- data/spec/spreadsheet_formatter_spec.rb +16 -14
- data/spec/stdout_formatter_spec.rb +10 -8
- metadata +57 -11
@@ -0,0 +1,72 @@
|
|
1
|
+
|
2
|
+
##
|
3
|
+
#Reopened to add some useful methods
|
4
|
+
class Range
|
5
|
+
##
|
6
|
+
#If end < begin, returns an empty range (begin ... begin)
|
7
|
+
#Otherwise, returns the original range
|
8
|
+
def normalized
|
9
|
+
return self.begin ... self.begin if self.end < self.begin
|
10
|
+
self
|
11
|
+
end
|
12
|
+
|
13
|
+
##
|
14
|
+
#Returns the empty status of the range
|
15
|
+
#
|
16
|
+
#A range is empty if end < begin or if begin == end and exclude_end? is true.
|
17
|
+
def empty?
|
18
|
+
(self.end < self.begin) || (exclude_end? && (self.begin == self.end))
|
19
|
+
end
|
20
|
+
|
21
|
+
#This code probably works, but we're not using it anywhere.
|
22
|
+
# def intersection(other)
|
23
|
+
# self_n = self.normalized
|
24
|
+
# other = other.normalized
|
25
|
+
#
|
26
|
+
# new_begin, new_end, exclude_end = nil
|
27
|
+
#
|
28
|
+
# if self_n.cover?(other.begin)
|
29
|
+
# new_first = other.begin
|
30
|
+
# elsif other.cover?(self_n.begin)
|
31
|
+
# new_first = self_n.begin
|
32
|
+
# end
|
33
|
+
#
|
34
|
+
# return self_n.begin ... self_n.begin unless new_first
|
35
|
+
#
|
36
|
+
# if self_n.cover?(other.end)
|
37
|
+
# unless other.exclude_end? && other.end == self_n.begin
|
38
|
+
# new_end = other.end
|
39
|
+
# exclude_end = other.exclude_end?
|
40
|
+
# end
|
41
|
+
# elsif other.cover?(self_n.end)
|
42
|
+
# unless self_n.exclude_end? && self_n.end == other.begin
|
43
|
+
# new_end = self_n.end
|
44
|
+
# exclude_end = self_n.exclude_end?
|
45
|
+
# end
|
46
|
+
# end
|
47
|
+
#
|
48
|
+
# #If we still haven't found new_end, try one more case:
|
49
|
+
# unless new_end
|
50
|
+
# if self_n.end == other.end
|
51
|
+
# #We'll only get here if both ranges exclude their ends and have the same end.
|
52
|
+
# new_end = self_n.end
|
53
|
+
# exclude_end = true
|
54
|
+
# end
|
55
|
+
# end
|
56
|
+
#
|
57
|
+
# return self_n.begin ... self_n.begin unless new_end
|
58
|
+
#
|
59
|
+
# Range.new(new_begin, new_end, exclude_end)
|
60
|
+
# end
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
#Reopened to add some useful methods
|
65
|
+
class Date
|
66
|
+
##
|
67
|
+
#Converts the Date to a Time, using UTC as the time zone
|
68
|
+
def to_utc_time
|
69
|
+
Time.utc(year, month, day)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
data/lib/bookie/formatter.rb
CHANGED
@@ -120,7 +120,7 @@ module Bookie
|
|
120
120
|
job.system.system_type.name,
|
121
121
|
job.start_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
|
122
122
|
job.end_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
|
123
|
-
Formatter.format_duration(job.end_time - job.start_time),
|
123
|
+
Formatter.format_duration(job.end_time.to_i - job.start_time.to_i),
|
124
124
|
Formatter.format_duration(job.cpu_time),
|
125
125
|
"#{job.memory}kb#{memory_stat_type}",
|
126
126
|
job.command_name,
|
@@ -131,15 +131,23 @@ module Bookie
|
|
131
131
|
protected :fields_for_each_job
|
132
132
|
|
133
133
|
##
|
134
|
-
#Formats a duration in
|
134
|
+
#Formats a duration in a human-readable format
|
135
135
|
#
|
136
|
-
#<tt>dur</tt> should be
|
136
|
+
#<tt>dur</tt> should be an integer representing the number of seconds.
|
137
137
|
def self.format_duration(dur)
|
138
|
-
dur =
|
138
|
+
dur = dur.to_i
|
139
|
+
days = dur / (3600 * 24)
|
140
|
+
dur -= days * (3600 * 24)
|
139
141
|
hours = dur / 3600
|
140
|
-
|
141
|
-
|
142
|
-
|
142
|
+
dur -= hours * 3600
|
143
|
+
minutes = dur / 60
|
144
|
+
dur -= minutes * 60
|
145
|
+
seconds = dur
|
146
|
+
|
147
|
+
weeks = days / 7
|
148
|
+
days = days % 7
|
149
|
+
|
150
|
+
"%i week%s, %i day%s, %02i:%02i:%02i" % [weeks, weeks == 1 ? '' : 's', days, days == 1 ? '' : 's', hours, minutes, seconds]
|
143
151
|
end
|
144
152
|
end
|
145
153
|
|
@@ -9,16 +9,22 @@ module Bookie
|
|
9
9
|
|
10
10
|
def do_print_summary(field_values)
|
11
11
|
Formatter::SUMMARY_FIELD_LABELS.zip(field_values) do |label, value|
|
12
|
-
@file.puts "#{label}, #{value}"
|
12
|
+
@file.puts "#{CommaDump.quote(label)}, #{CommaDump.quote(value)}"
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
16
|
def do_print_jobs(jobs)
|
17
17
|
@file.puts Formatter::DETAILS_FIELD_LABELS.join(', ')
|
18
18
|
fields_for_each_job(jobs) do |fields|
|
19
|
-
@file.puts fields.join(', ')
|
19
|
+
@file.puts fields.map{ |s| CommaDump.quote(s) }.join(', ')
|
20
20
|
end
|
21
21
|
end
|
22
|
+
|
23
|
+
##
|
24
|
+
#Quotes a value for use as a CSV element
|
25
|
+
def self.quote(val)
|
26
|
+
%{"#{val.to_s.gsub('"', '""')}"}
|
27
|
+
end
|
22
28
|
end
|
23
29
|
end
|
24
|
-
end
|
30
|
+
end
|
@@ -19,7 +19,7 @@ module Bookie
|
|
19
19
|
|
20
20
|
def do_print_jobs(jobs)
|
21
21
|
#To consider: optimize by moving out of the function?
|
22
|
-
format_string = "%-15.15s %-15.15s %-20.20s %-20.20s %-26.
|
22
|
+
format_string = "%-15.15s %-15.15s %-20.20s %-20.20s %-26.26s %-26.26s %-30.30s %-30.30s %-20.20s %-20.20s %-11.11s\n"
|
23
23
|
heading = sprintf(format_string, *Formatter::DETAILS_FIELD_LABELS)
|
24
24
|
@io.write heading
|
25
25
|
@io.puts '-' * (heading.length - 1)
|
data/lib/bookie/sender.rb
CHANGED
@@ -34,20 +34,16 @@ module Bookie
|
|
34
34
|
#Grab data from the first job:
|
35
35
|
each_job(filename) do |job|
|
36
36
|
next if filtered?(job)
|
37
|
-
|
38
|
-
|
39
|
-
duplicate = system.jobs.find_by_end_time(end_time)
|
40
|
-
raise "Jobs already exist in the database for '#{filename}'." if duplicate
|
37
|
+
system = Bookie::Database::System.find_current(self, job.end_time)
|
38
|
+
raise "Jobs already exist in the database for '#{filename}'." if duplicate(job, system)
|
41
39
|
time_min = job.start_time
|
42
|
-
time_max = end_time
|
40
|
+
time_max = job.end_time
|
43
41
|
break
|
44
42
|
end
|
45
43
|
|
46
44
|
#If there are no jobs, return.
|
47
45
|
return unless time_min
|
48
46
|
|
49
|
-
#To do: add an option to resume an interrupted send.
|
50
|
-
|
51
47
|
#Send the job data:
|
52
48
|
each_job(filename) do |job|
|
53
49
|
next if filtered?(job)
|
@@ -55,6 +51,8 @@ module Bookie
|
|
55
51
|
time_min = (model.start_time < time_min) ? model.start_time : time_min
|
56
52
|
time_max = (model.end_time > time_max) ? model.end_time : time_max
|
57
53
|
#To consider: handle files that don't have jobs sorted by end time?
|
54
|
+
#To consider: this should rarely happen in real life. Remove test?
|
55
|
+
#(This situation can only arise if log files from different versions of the system are concatenated before sending.)
|
58
56
|
if system.end_time && model.end_time > system.end_time
|
59
57
|
system = Database::System.find_current(self, model.end_time)
|
60
58
|
end
|
@@ -69,17 +67,52 @@ module Bookie
|
|
69
67
|
end
|
70
68
|
|
71
69
|
#Clear out the summaries that would have been affected by the new data:
|
72
|
-
|
73
|
-
|
70
|
+
clear_summaries(time_min.to_date, time_max.to_date)
|
71
|
+
end
|
72
|
+
|
73
|
+
##
|
74
|
+
#Undoes a previous send operation
|
75
|
+
def undo_send(filename)
|
76
|
+
raise IOError.new("File '#{filename}' does not exist.") unless File.exists?(filename)
|
77
|
+
|
78
|
+
system = nil
|
79
|
+
|
80
|
+
time_min, time_max = nil
|
81
|
+
|
82
|
+
#Grab data from the first job:
|
83
|
+
each_job(filename) do |job|
|
84
|
+
next if filtered?(job)
|
85
|
+
system = Bookie::Database::System.find_current(self, job.end_time)
|
86
|
+
time_min = job.start_time
|
87
|
+
time_max = job.end_time
|
88
|
+
break
|
89
|
+
end
|
90
|
+
|
91
|
+
return unless time_min
|
74
92
|
|
75
|
-
|
93
|
+
each_job(filename) do |job|
|
94
|
+
next if filtered?(job)
|
95
|
+
if system.end_time && job.end_time > system.end_time
|
96
|
+
system = Database::System.find_current(self, job.end_time)
|
97
|
+
end
|
98
|
+
#To consider: optimize this query?
|
99
|
+
#(It should be possible to delete all of the jobs with end times between those of the first and last jobs of the file (exclusive),
|
100
|
+
#but jobs with end times matching those of the first/last jobs in the file might be from an earlier or later file, not this one.
|
101
|
+
#This assumes that the files all have jobs sorted by end time.
|
102
|
+
model = duplicate(job, system)
|
103
|
+
break unless model
|
104
|
+
time_min = (model.start_time < time_min) ? model.start_time : time_min
|
105
|
+
time_max = (model.end_time > time_max) ? model.end_time : time_max
|
106
|
+
model.delete
|
107
|
+
end
|
108
|
+
|
109
|
+
clear_summaries(time_min.to_date, time_max.to_date)
|
76
110
|
end
|
77
111
|
|
78
112
|
##
|
79
113
|
#The name of the Bookie::Database::SystemType that systems using this sender will have
|
80
114
|
def system_type
|
81
|
-
|
82
|
-
Bookie::Database::SystemType.find_or_create!(system_type_name, memory_stat_type)
|
115
|
+
@system_type ||= Bookie::Database::SystemType.find_or_create!(system_type_name, memory_stat_type)
|
83
116
|
end
|
84
117
|
|
85
118
|
##
|
@@ -88,6 +121,28 @@ module Bookie
|
|
88
121
|
def filtered?(job)
|
89
122
|
@config.excluded_users.include?job.user_name
|
90
123
|
end
|
124
|
+
|
125
|
+
##
|
126
|
+
#Finds the first job that is a duplicate of the provided job
|
127
|
+
def duplicate(job, system)
|
128
|
+
system.jobs.where({
|
129
|
+
:start_time => job.start_time,
|
130
|
+
:wall_time => job.wall_time,
|
131
|
+
:command_name => job.command_name,
|
132
|
+
:cpu_time => job.cpu_time,
|
133
|
+
:memory => job.memory,
|
134
|
+
:exit_code => job.exit_code
|
135
|
+
}).by_user_name(job.user_name).by_group_name(job.group_name).first
|
136
|
+
end
|
137
|
+
|
138
|
+
#Used internally by #send_data and #undo_send
|
139
|
+
def clear_summaries(date_min, date_max)
|
140
|
+
#Since joins don't mix with DELETE statements, we have to do this the hard way.
|
141
|
+
systems = Database::System.by_name(@config.hostname).all
|
142
|
+
systems.map!{ |sys| sys.id }
|
143
|
+
Database::JobSummary.where('job_summaries.system_id in (?)', systems).where('date >= ? AND date <= ?', date_min, date_max).delete_all
|
144
|
+
end
|
145
|
+
private :clear_summaries
|
91
146
|
end
|
92
147
|
|
93
148
|
##
|
@@ -99,17 +154,22 @@ module Bookie
|
|
99
154
|
job = Bookie::Database::Job.new
|
100
155
|
job.command_name = self.command_name
|
101
156
|
job.start_time = self.start_time
|
102
|
-
job.end_time = self.start_time + self.wall_time
|
103
157
|
job.wall_time = self.wall_time
|
104
158
|
job.cpu_time = self.cpu_time
|
105
159
|
job.memory = self.memory
|
106
160
|
job.exit_code = self.exit_code
|
107
|
-
|
161
|
+
job
|
162
|
+
end
|
163
|
+
|
164
|
+
##
|
165
|
+
#Returns the end time
|
166
|
+
def end_time
|
167
|
+
start_time + wall_time
|
108
168
|
end
|
109
169
|
end
|
110
|
-
|
170
|
+
|
111
171
|
#Contains all sender plugins
|
112
172
|
module Senders
|
113
173
|
|
114
174
|
end
|
115
|
-
end
|
175
|
+
end
|
@@ -30,8 +30,8 @@ end
|
|
30
30
|
#Originates from the <tt>pacct</tt> gem
|
31
31
|
module Pacct
|
32
32
|
##
|
33
|
-
#Originates from the <tt>pacct</tt> gem;
|
33
|
+
#Originates from the <tt>pacct</tt> gem; reopened here to include Bookie::Sender::ModelHelpers
|
34
34
|
class Entry
|
35
35
|
include Bookie::ModelHelpers
|
36
36
|
end
|
37
|
-
end
|
37
|
+
end
|
data/lib/bookie/version.rb
CHANGED
@@ -32,25 +32,34 @@ describe Bookie::Formatters::CommaDump do
|
|
32
32
|
end
|
33
33
|
|
34
34
|
it "correctly formats jobs" do
|
35
|
-
|
36
|
-
|
35
|
+
with_utc do
|
36
|
+
@formatter.print_jobs(@jobs.order(:start_time).limit(2).all)
|
37
|
+
@m.buf.should eql <<-eos
|
37
38
|
User, Group, System, System type, Start time, End time, Wall time, CPU time, Memory usage, Command, Exit code
|
38
|
-
root, root, test1, Standalone, 2012-01-01 00:00:00, 2012-01-01 01:00:00, 01:00:00, 00:01:40, 200kb (avg), vi, 0
|
39
|
-
test, default, test1, Standalone, 2012-01-01 01:00:00, 2012-01-01 02:00:00, 01:00:00, 00:01:40, 200kb (avg), emacs, 1
|
39
|
+
"root", "root", "test1", "Standalone", "2012-01-01 00:00:00", "2012-01-01 01:00:00", "0 weeks, 0 days, 01:00:00", "0 weeks, 0 days, 00:01:40", "200kb (avg)", "vi", "0"
|
40
|
+
"test", "default", "test1", "Standalone", "2012-01-01 01:00:00", "2012-01-01 02:00:00", "0 weeks, 0 days, 01:00:00", "0 weeks, 0 days, 00:01:40", "200kb (avg)", "emacs", "1"
|
40
41
|
eos
|
42
|
+
end
|
41
43
|
end
|
42
44
|
|
43
45
|
it "correctly formats summaries" do
|
44
|
-
Time.expects(:now).returns(
|
46
|
+
Time.expects(:now).returns(base_time + 40.hours).at_least_once
|
45
47
|
@formatter.print_summary(@jobs, @summaries, Bookie::Database::System)
|
46
48
|
@m.buf.should eql <<-eos
|
47
|
-
Number of jobs, 40
|
48
|
-
Total CPU time, 01:06:40
|
49
|
-
Successful, 50.0000%
|
50
|
-
Available CPU time,
|
51
|
-
CPU time used, 0.7937%
|
52
|
-
Available memory (average), 1750000 kb
|
53
|
-
Memory used (average), 0.0114%
|
49
|
+
"Number of jobs", "40"
|
50
|
+
"Total CPU time", "0 weeks, 0 days, 01:06:40"
|
51
|
+
"Successful", "50.0000%"
|
52
|
+
"Available CPU time", "0 weeks, 5 days, 20:00:00"
|
53
|
+
"CPU time used", "0.7937%"
|
54
|
+
"Available memory (average)", "1750000 kb"
|
55
|
+
"Memory used (average)", "0.0114%"
|
54
56
|
eos
|
55
57
|
end
|
58
|
+
|
59
|
+
it "correctly quotes values" do
|
60
|
+
Formatter = Bookie::Formatters::CommaDump
|
61
|
+
Formatter.quote("test").should eql '"test"'
|
62
|
+
Formatter.quote('"test"').should eql '"""test"""'
|
63
|
+
Formatter.quote(0).should eql '"0"'
|
64
|
+
end
|
56
65
|
end
|