bookie_accounting 1.2.3 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +4 -3
- data/README.md +4 -24
- data/Rakefile +9 -116
- data/bin/bookie-data +48 -7
- data/bin/bookie-send +6 -14
- data/bookie_accounting.gemspec +4 -3
- data/lib/bookie/database/group.rb +33 -0
- data/lib/bookie/database/job.rb +201 -0
- data/lib/bookie/database/job_summary.rb +268 -0
- data/lib/bookie/database/lock.rb +36 -0
- data/lib/bookie/database/system.rb +166 -0
- data/lib/bookie/database/system_type.rb +80 -0
- data/lib/bookie/database/user.rb +54 -0
- data/lib/bookie/database.rb +7 -805
- data/lib/bookie/extensions.rb +23 -44
- data/lib/bookie/formatter.rb +8 -4
- data/lib/bookie/sender.rb +12 -14
- data/lib/bookie/version.rb +1 -1
- data/snapshot/test_config.json +2 -2
- data/spec/config_spec.rb +2 -2
- data/spec/database/group_spec.rb +36 -0
- data/spec/database/job_spec.rb +308 -0
- data/spec/database/job_summary_spec.rb +302 -0
- data/spec/database/lock_spec.rb +41 -0
- data/spec/database/migration_spec.rb +44 -0
- data/spec/database/system_spec.rb +232 -0
- data/spec/database/system_type_spec.rb +68 -0
- data/spec/database/user_spec.rb +69 -0
- data/spec/formatter_spec.rb +44 -37
- data/spec/{comma_dump_formatter_spec.rb → formatters/comma_dump_spec.rb} +16 -30
- data/spec/formatters/spreadsheet_spec.rb +98 -0
- data/spec/{stdout_formatter_spec.rb → formatters/stdout_spec.rb} +15 -29
- data/spec/sender_spec.rb +92 -66
- data/spec/{standalone_sender_spec.rb → senders/standalone_spec.rb} +10 -9
- data/spec/{torque_cluster_sender_spec.rb → senders/torque_cluster_spec.rb} +9 -13
- data/spec/spec_helper.rb +111 -57
- data/todo.txt +13 -0
- metadata +38 -23
- data/rpm/activesupport.erb +0 -151
- data/rpm/bundle.erb +0 -71
- data/rpm/default.erb +0 -147
- data/rpm/mysql2.erb +0 -149
- data/rpm/pacct.erb +0 -147
- data/rpm/rspec-core.erb +0 -149
- data/rpm/sqlite3.erb +0 -147
- data/spec/database_spec.rb +0 -1078
- data/spec/spreadsheet_formatter_spec.rb +0 -114
data/lib/bookie/database.rb
CHANGED
@@ -2,750 +2,18 @@ require 'bookie/config'
|
|
2
2
|
require 'bookie/extensions'
|
3
3
|
|
4
4
|
require 'active_record'
|
5
|
-
#
|
5
|
+
#TODO: remove when code is updated.
|
6
6
|
require 'protected_attributes'
|
7
7
|
|
8
|
+
require 'bookie/database/job'
|
9
|
+
require 'bookie/database/job_summary'
|
10
|
+
require 'bookie/database/user.rb'
|
11
|
+
require 'bookie/database/system.rb'
|
12
|
+
|
8
13
|
module Bookie
|
9
14
|
##
|
10
15
|
#Contains database-related code and models
|
11
16
|
module Database
|
12
|
-
|
13
|
-
##
|
14
|
-
#Represents a lock on a table
|
15
|
-
#
|
16
|
-
#Based on http://kseebaldt.blogspot.com/2007/11/synchronizing-using-active-record.html
|
17
|
-
#
|
18
|
-
#This should probably not be called within a transaction block; the lock might not be released
|
19
|
-
#until the outer transaction completes, and even if it is released before then, there might be
|
20
|
-
#concurrency issues.
|
21
|
-
class Lock < ActiveRecord::Base
|
22
|
-
##
|
23
|
-
#Acquires the lock, runs the given block, and releases the lock when finished
|
24
|
-
def synchronize
|
25
|
-
transaction do
|
26
|
-
#Lock this record to be inaccessible to others until this transaction is completed.
|
27
|
-
self.class.lock.find(id)
|
28
|
-
yield
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
@locks = {}
|
33
|
-
|
34
|
-
##
|
35
|
-
#Returns a lock by name
|
36
|
-
def self.[](name)
|
37
|
-
@locks[name.to_sym] ||= find_by_name(name.to_s) or raise "Unable to find lock '#{name}'"
|
38
|
-
end
|
39
|
-
|
40
|
-
validates_presence_of :name
|
41
|
-
end
|
42
|
-
|
43
|
-
##
|
44
|
-
#A reported job
|
45
|
-
#
|
46
|
-
#The various filter methods can be chained to produce more complex queries.
|
47
|
-
#
|
48
|
-
#===Examples
|
49
|
-
# Bookie::Database::Job.by_user_name('root').by_system_name('localhost').find_each do |job|
|
50
|
-
# puts job.inspect
|
51
|
-
# end
|
52
|
-
#
|
53
|
-
class Job < ActiveRecord::Base
|
54
|
-
belongs_to :user
|
55
|
-
belongs_to :system
|
56
|
-
|
57
|
-
##
|
58
|
-
#The time at which the job ended
|
59
|
-
def end_time
|
60
|
-
return start_time + wall_time
|
61
|
-
end
|
62
|
-
|
63
|
-
#To consider: disable #end_time= ?
|
64
|
-
|
65
|
-
def self.by_user(user)
|
66
|
-
where('jobs.user_id = ?', user.id)
|
67
|
-
end
|
68
|
-
|
69
|
-
##
|
70
|
-
#Filters by user name
|
71
|
-
def self.by_user_name(user_name)
|
72
|
-
joins(:user).where('users.name = ?', user_name)
|
73
|
-
end
|
74
|
-
|
75
|
-
def self.by_system(system)
|
76
|
-
where('jobs.system_id = ?', system.id)
|
77
|
-
end
|
78
|
-
|
79
|
-
##
|
80
|
-
#Filters by system name
|
81
|
-
def self.by_system_name(system_name)
|
82
|
-
joins(:system).where('systems.name = ?', system_name)
|
83
|
-
end
|
84
|
-
|
85
|
-
##
|
86
|
-
#Filters by group name
|
87
|
-
def self.by_group_name(group_name)
|
88
|
-
group = Group.find_by_name(group_name)
|
89
|
-
return joins(:user).where('users.group_id = ?', group.id) if group
|
90
|
-
self.none
|
91
|
-
end
|
92
|
-
|
93
|
-
##
|
94
|
-
#Filters by system type
|
95
|
-
def self.by_system_type(system_type)
|
96
|
-
joins(:system).where('systems.system_type_id = ?', system_type.id)
|
97
|
-
end
|
98
|
-
|
99
|
-
##
|
100
|
-
#Filters by command name
|
101
|
-
def self.by_command_name(c_name)
|
102
|
-
where('jobs.command_name = ?', c_name)
|
103
|
-
end
|
104
|
-
|
105
|
-
##
|
106
|
-
#Filters by a range of start times
|
107
|
-
def self.by_start_time_range(time_range)
|
108
|
-
where('? <= jobs.start_time AND jobs.start_time < ?', time_range.first, time_range.last)
|
109
|
-
end
|
110
|
-
|
111
|
-
##
|
112
|
-
#Filters by a range of end times
|
113
|
-
def self.by_end_time_range(time_range)
|
114
|
-
where('? <= jobs.end_time AND jobs.end_time < ?', time_range.first, time_range.last)
|
115
|
-
end
|
116
|
-
|
117
|
-
##
|
118
|
-
#Finds all jobs whose running intervals overlap the given time range
|
119
|
-
def self.by_time_range_inclusive(time_range)
|
120
|
-
if time_range.empty?
|
121
|
-
self.none
|
122
|
-
elsif time_range.exclude_end?
|
123
|
-
where('? <= jobs.end_time AND jobs.start_time < ?', time_range.first, time_range.last)
|
124
|
-
else
|
125
|
-
where('? <= jobs.end_time AND jobs.start_time <= ?', time_range.first, time_range.last)
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
##
|
130
|
-
#Produces a summary of the jobs in the given time interval
|
131
|
-
#
|
132
|
-
#Returns a hash with the following fields:
|
133
|
-
#- <tt>:jobs</tt>: an array of all jobs in the interval
|
134
|
-
#- <tt>:cpu_time</tt>: the total CPU time used
|
135
|
-
#- <tt>:memory_time</tt>: the sum of memory * wall_time for all jobs in the interval
|
136
|
-
#- <tt>:successful</tt>: the number of jobs that have completed successfully
|
137
|
-
#
|
138
|
-
#This method should probably not be chained with other queries that filter by start/end time.
|
139
|
-
#
|
140
|
-
#To consider: filter out jobs with 0 CPU time?
|
141
|
-
def self.summary(time_range = nil)
|
142
|
-
time_range = time_range.normalized if time_range
|
143
|
-
jobs = self
|
144
|
-
jobs = jobs.by_time_range_inclusive(time_range) if time_range
|
145
|
-
jobs = jobs.all_with_relations
|
146
|
-
cpu_time = 0
|
147
|
-
successful_jobs = 0
|
148
|
-
memory_time = 0
|
149
|
-
#To consider: job.end_time should be <= Time.now, but it might be good to check for that.
|
150
|
-
#Maybe in a database consistency checker tool?
|
151
|
-
#What if the system clock is off?
|
152
|
-
#Also consider a check for system start times.
|
153
|
-
jobs.each do |job|
|
154
|
-
job_start_time = job.start_time
|
155
|
-
job_end_time = job.end_time
|
156
|
-
if time_range
|
157
|
-
job_start_time = [job_start_time, time_range.first].max
|
158
|
-
job_end_time = [job_end_time, time_range.last].min
|
159
|
-
end
|
160
|
-
clipped_wall_time = job_end_time.to_i - job_start_time.to_i
|
161
|
-
if job.wall_time != 0
|
162
|
-
cpu_time += job.cpu_time * clipped_wall_time / job.wall_time
|
163
|
-
#To consider: what should I do about jobs that only report a max memory value?
|
164
|
-
memory_time += job.memory * clipped_wall_time
|
165
|
-
end
|
166
|
-
successful_jobs += 1 if job.exit_code == 0
|
167
|
-
end
|
168
|
-
|
169
|
-
return {
|
170
|
-
:jobs => jobs,
|
171
|
-
:cpu_time => cpu_time,
|
172
|
-
:memory_time => memory_time,
|
173
|
-
:successful => successful_jobs,
|
174
|
-
}
|
175
|
-
end
|
176
|
-
|
177
|
-
##
|
178
|
-
#Returns an array of all jobs, pre-loading relations to reduce the need for extra queries
|
179
|
-
#
|
180
|
-
#Relations are not cached between calls.
|
181
|
-
#
|
182
|
-
#To do: use ActiveRecord's #includes instead of this scheme?
|
183
|
-
def self.all_with_relations
|
184
|
-
jobs = self.all.to_a
|
185
|
-
users = {}
|
186
|
-
groups = {}
|
187
|
-
systems = {}
|
188
|
-
system_types = {}
|
189
|
-
jobs.each do |job|
|
190
|
-
system = systems[job.system_id]
|
191
|
-
if system
|
192
|
-
job.system = system
|
193
|
-
else
|
194
|
-
system = job.system
|
195
|
-
systems[system.id] = system
|
196
|
-
end
|
197
|
-
system_type = system_types[system.system_type_id]
|
198
|
-
if system_type
|
199
|
-
system.system_type = system_type
|
200
|
-
else
|
201
|
-
system_type = system.system_type
|
202
|
-
system_types[system_type.id] = system_type
|
203
|
-
end
|
204
|
-
user = users[job.user_id]
|
205
|
-
if user
|
206
|
-
job.user = user
|
207
|
-
else
|
208
|
-
user = job.user
|
209
|
-
users[user.id] = user
|
210
|
-
end
|
211
|
-
group = groups[user.group_id]
|
212
|
-
if group
|
213
|
-
user.group = group
|
214
|
-
else
|
215
|
-
group = user.group
|
216
|
-
groups[group.id] = group
|
217
|
-
end
|
218
|
-
end
|
219
|
-
|
220
|
-
jobs
|
221
|
-
end
|
222
|
-
|
223
|
-
before_save do
|
224
|
-
write_attribute(:end_time, end_time)
|
225
|
-
end
|
226
|
-
|
227
|
-
before_update do
|
228
|
-
write_attribute(:end_time, end_time)
|
229
|
-
end
|
230
|
-
|
231
|
-
validates_presence_of :user, :system, :cpu_time,
|
232
|
-
:start_time, :wall_time, :memory, :exit_code
|
233
|
-
|
234
|
-
validates_each :command_name do |record, attr, value|
|
235
|
-
record.errors.add(attr, 'must not be nil') if value == nil
|
236
|
-
end
|
237
|
-
|
238
|
-
validates_each :cpu_time, :wall_time, :memory do |record, attr, value|
|
239
|
-
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
240
|
-
end
|
241
|
-
end
|
242
|
-
|
243
|
-
##
|
244
|
-
#A cached summary of Jobs in the database
|
245
|
-
#
|
246
|
-
#Most summary operations should be performed through this class to improve efficiency.
|
247
|
-
class JobSummary < ActiveRecord::Base
|
248
|
-
self.table_name = :job_summaries
|
249
|
-
|
250
|
-
belongs_to :user
|
251
|
-
belongs_to :system
|
252
|
-
|
253
|
-
attr_accessible :date, :user, :user_id, :system, :system_id, :command_name, :cpu_time, :memory_time
|
254
|
-
|
255
|
-
##
|
256
|
-
#Filters by date
|
257
|
-
def self.by_date(date)
|
258
|
-
where('job_summaries.date = ?', date)
|
259
|
-
end
|
260
|
-
|
261
|
-
##
|
262
|
-
#Filters by a date range
|
263
|
-
def self.by_date_range(range)
|
264
|
-
range = range.normalized
|
265
|
-
if range.exclude_end?
|
266
|
-
where('? <= job_summaries.date AND job_summaries.date < ?', range.begin, range.end)
|
267
|
-
else
|
268
|
-
where('? <= job_summaries.date AND job_summaries.date <= ?', range.begin, range.end)
|
269
|
-
end
|
270
|
-
end
|
271
|
-
|
272
|
-
##
|
273
|
-
#Filters by user
|
274
|
-
def self.by_user(user)
|
275
|
-
where('job_summaries.user_id = ?', user.id)
|
276
|
-
end
|
277
|
-
|
278
|
-
##
|
279
|
-
#Filters by user name
|
280
|
-
def self.by_user_name(name)
|
281
|
-
joins(:user).where('users.name = ?', name)
|
282
|
-
end
|
283
|
-
|
284
|
-
##
|
285
|
-
#Filters by group
|
286
|
-
def self.by_group(group)
|
287
|
-
joins(:user).where('users.group_id = ?', group.id)
|
288
|
-
end
|
289
|
-
|
290
|
-
##
|
291
|
-
#Filters by group name
|
292
|
-
def self.by_group_name(name)
|
293
|
-
group = Group.find_by_name(name)
|
294
|
-
return by_group(group) if group
|
295
|
-
self.none
|
296
|
-
end
|
297
|
-
|
298
|
-
##
|
299
|
-
#Filters by system
|
300
|
-
def self.by_system(system)
|
301
|
-
where('job_summaries.system_id = ?', system.id)
|
302
|
-
end
|
303
|
-
|
304
|
-
##
|
305
|
-
#Filters by system name
|
306
|
-
def self.by_system_name(name)
|
307
|
-
joins(:system).where('systems.name = ?', name)
|
308
|
-
end
|
309
|
-
|
310
|
-
##
|
311
|
-
#Filters by system type
|
312
|
-
def self.by_system_type(type)
|
313
|
-
joins(:system).where('systems.system_type_id = ?', type.id)
|
314
|
-
end
|
315
|
-
|
316
|
-
##
|
317
|
-
#Filters by command name
|
318
|
-
def self.by_command_name(cmd)
|
319
|
-
where('job_summaries.command_name = ?', cmd)
|
320
|
-
end
|
321
|
-
|
322
|
-
##
|
323
|
-
#Attempts to find a JobSummary with the given date, user_id, system_id, and command_name
|
324
|
-
#
|
325
|
-
#If one does not exist, a new JobSummary will be instantiated (but not saved to the database).
|
326
|
-
def self.find_or_new(date, user_id, system_id, command_name)
|
327
|
-
str = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).to_sql
|
328
|
-
summary = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).first
|
329
|
-
summary ||= new(
|
330
|
-
:date => date,
|
331
|
-
:user_id => user_id,
|
332
|
-
:system_id => system_id,
|
333
|
-
:command_name => command_name
|
334
|
-
)
|
335
|
-
summary
|
336
|
-
end
|
337
|
-
|
338
|
-
##
|
339
|
-
#Create cached summaries for the given date
|
340
|
-
#
|
341
|
-
#The date is interpreted as being in UTC.
|
342
|
-
#
|
343
|
-
#If there is nothing to summarize, a dummy summary will be created.
|
344
|
-
#
|
345
|
-
#Uses Lock::synchronize internally; should not be used in transaction blocks
|
346
|
-
def self.summarize(date)
|
347
|
-
jobs = Job
|
348
|
-
unscoped = self.unscoped
|
349
|
-
time_min = date.to_utc_time
|
350
|
-
time_range = time_min ... time_min + 1.days
|
351
|
-
day_jobs = jobs.by_time_range_inclusive(time_range)
|
352
|
-
|
353
|
-
#Find the sets of unique values.
|
354
|
-
value_sets = day_jobs.select('user_id, system_id, command_name').uniq
|
355
|
-
if value_sets.empty?
|
356
|
-
user = User.select(:id).first
|
357
|
-
system = System.select(:id).first
|
358
|
-
#If there are no users or no systems, we can't create the dummy summary, so just return.
|
359
|
-
return unless user && system
|
360
|
-
#Create a dummy summary so summary() doesn't keep trying to create one.
|
361
|
-
Lock[:job_summaries].synchronize do
|
362
|
-
sum = unscoped.find_or_new(date, user.id, system.id, '')
|
363
|
-
sum.cpu_time = 0
|
364
|
-
sum.memory_time = 0
|
365
|
-
sum.save!
|
366
|
-
end
|
367
|
-
else
|
368
|
-
value_sets.each do |set|
|
369
|
-
summary_jobs = jobs.where(:user_id => set.user_id).where(:system_id => set.system_id).by_command_name(set.command_name)
|
370
|
-
summary = summary_jobs.summary(time_range)
|
371
|
-
Lock[:job_summaries].synchronize do
|
372
|
-
sum = unscoped.find_or_new(date, set.user_id, set.system_id, set.command_name)
|
373
|
-
sum.cpu_time = summary[:cpu_time]
|
374
|
-
sum.memory_time = summary[:memory_time]
|
375
|
-
sum.save!
|
376
|
-
end
|
377
|
-
end
|
378
|
-
end
|
379
|
-
end
|
380
|
-
|
381
|
-
##
|
382
|
-
#Returns a summary of jobs in the database
|
383
|
-
#
|
384
|
-
#The following options are supported:
|
385
|
-
#- [<tt>:range</tt>] restricts the summary to a specific time interval (specified as a Range of Time objects)
|
386
|
-
#- [<tt>:jobs</tt>] the jobs on which the summary should operate
|
387
|
-
#
|
388
|
-
#Internally, this may call JobSummary::summary, which uses Lock#synchronize, so this should not be used inside a transaction block.
|
389
|
-
#
|
390
|
-
#When filtering, the same filters must be applied to both the Jobs and the JobSummaries. For example:
|
391
|
-
# jobs = Bookie::Database::Job.by_user_name('root')
|
392
|
-
# summaries = Bookie::Database::Job.by_user_name('root')
|
393
|
-
# puts summaries.summary(:jobs => jobs)
|
394
|
-
def self.summary(opts = {})
|
395
|
-
jobs = opts[:jobs] || Job
|
396
|
-
range = opts[:range]
|
397
|
-
unless range
|
398
|
-
end_time = nil
|
399
|
-
if System.active_systems.any?
|
400
|
-
end_time = Time.now
|
401
|
-
else
|
402
|
-
last_ended_system = System.order('end_time DESC').first
|
403
|
-
end_time = last_ended_system.end_time if last_ended_system
|
404
|
-
end
|
405
|
-
if end_time
|
406
|
-
first_started_system = System.order(:start_time).first
|
407
|
-
range = first_started_system.start_time ... end_time
|
408
|
-
else
|
409
|
-
range = Time.new ... Time.new
|
410
|
-
end
|
411
|
-
end
|
412
|
-
range = range.normalized
|
413
|
-
|
414
|
-
num_jobs = 0
|
415
|
-
cpu_time = 0
|
416
|
-
memory_time = 0
|
417
|
-
successful = 0
|
418
|
-
|
419
|
-
#Is the beginning somewhere between days?
|
420
|
-
date_begin = range.begin.utc.to_date
|
421
|
-
unless date_begin.to_utc_time == range.begin
|
422
|
-
date_begin += 1
|
423
|
-
time_before_max = [date_begin.to_utc_time, range.end].min
|
424
|
-
time_before_min = range.begin
|
425
|
-
summary = jobs.summary(time_before_min ... time_before_max)
|
426
|
-
cpu_time += summary[:cpu_time]
|
427
|
-
memory_time += summary[:memory_time]
|
428
|
-
end
|
429
|
-
|
430
|
-
#Is the end somewhere between days?
|
431
|
-
date_end = range.end.utc.to_date
|
432
|
-
time_after_min = date_end.to_utc_time
|
433
|
-
unless time_after_min <= range.begin
|
434
|
-
time_after_max = range.end
|
435
|
-
time_after_range = Range.new(time_after_min, time_after_max, range.exclude_end?)
|
436
|
-
unless time_after_range.empty?
|
437
|
-
summary = jobs.summary(time_after_range)
|
438
|
-
cpu_time += summary[:cpu_time]
|
439
|
-
memory_time += summary[:memory_time]
|
440
|
-
end
|
441
|
-
end
|
442
|
-
|
443
|
-
date_range = date_begin ... date_end
|
444
|
-
|
445
|
-
unscoped = self.unscoped
|
446
|
-
summaries = by_date_range(date_range).order(:date).to_a
|
447
|
-
index = 0
|
448
|
-
date_range.each do |date|
|
449
|
-
new_index = index
|
450
|
-
sum = summaries[new_index]
|
451
|
-
while sum && sum.date == date do
|
452
|
-
cpu_time += sum.cpu_time
|
453
|
-
memory_time += sum.memory_time
|
454
|
-
new_index += 1
|
455
|
-
sum = summaries[new_index]
|
456
|
-
end
|
457
|
-
#Did we actually process any summaries?
|
458
|
-
if new_index == index
|
459
|
-
#Nope. Create the summaries.
|
460
|
-
#To consider: optimize out the query?
|
461
|
-
unscoped.summarize(date)
|
462
|
-
sums = by_date(date)
|
463
|
-
sums.each do |sum|
|
464
|
-
cpu_time += sum.cpu_time
|
465
|
-
memory_time += sum.memory_time
|
466
|
-
end
|
467
|
-
end
|
468
|
-
end
|
469
|
-
|
470
|
-
if range && range.empty?
|
471
|
-
num_jobs = 0
|
472
|
-
successful = 0
|
473
|
-
else
|
474
|
-
jobs = jobs.by_time_range_inclusive(range)
|
475
|
-
num_jobs = jobs.count
|
476
|
-
successful = jobs.where('jobs.exit_code = 0').count
|
477
|
-
end
|
478
|
-
|
479
|
-
{
|
480
|
-
:num_jobs => num_jobs,
|
481
|
-
:cpu_time => cpu_time,
|
482
|
-
:memory_time => memory_time,
|
483
|
-
:successful => successful,
|
484
|
-
}
|
485
|
-
end
|
486
|
-
|
487
|
-
validates_presence_of :user_id, :system_id, :date, :cpu_time, :memory_time
|
488
|
-
|
489
|
-
validates_each :command_name do |record, attr, value|
|
490
|
-
record.errors.add(attr, 'must not be nil') if value == nil
|
491
|
-
end
|
492
|
-
|
493
|
-
validates_each :cpu_time, :memory_time do |record, attr, value|
|
494
|
-
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
495
|
-
end
|
496
|
-
end
|
497
|
-
|
498
|
-
##
|
499
|
-
#A group of users
|
500
|
-
class Group < ActiveRecord::Base
|
501
|
-
has_many :users
|
502
|
-
|
503
|
-
##
|
504
|
-
#Finds a group by name, creating it if it doesn't exist
|
505
|
-
#
|
506
|
-
#If <tt>known_groups</tt> is provided, it will be used as a cache to reduce the number of database lookups needed.
|
507
|
-
#
|
508
|
-
#This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
|
509
|
-
def self.find_or_create!(name, known_groups = nil)
|
510
|
-
group = known_groups[name] if known_groups
|
511
|
-
unless group
|
512
|
-
Lock[:groups].synchronize do
|
513
|
-
group = find_by_name(name)
|
514
|
-
group ||= create!(:name => name)
|
515
|
-
end
|
516
|
-
known_groups[name] = group if known_groups
|
517
|
-
end
|
518
|
-
group
|
519
|
-
end
|
520
|
-
|
521
|
-
validates_presence_of :name
|
522
|
-
end
|
523
|
-
|
524
|
-
#ActiveRecord structure for a user
|
525
|
-
class User < ActiveRecord::Base
|
526
|
-
belongs_to :group
|
527
|
-
|
528
|
-
def self.by_name(name)
|
529
|
-
where('users.name = ?', name)
|
530
|
-
end
|
531
|
-
|
532
|
-
def self.by_group(group)
|
533
|
-
return where('users.group_id = ?', group.id)
|
534
|
-
end
|
535
|
-
|
536
|
-
def self.by_group_name(name)
|
537
|
-
group = Group.find_by_name(name)
|
538
|
-
return by_group(group) if group
|
539
|
-
self.none
|
540
|
-
end
|
541
|
-
|
542
|
-
##
|
543
|
-
#Finds a user by name and group, creating it if it doesn't exist
|
544
|
-
#
|
545
|
-
#If <tt>known_users</tt> is provided, it will be used as a cache to reduce the number of database lookups needed.
|
546
|
-
#
|
547
|
-
#This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
|
548
|
-
def self.find_or_create!(name, group, known_users = nil)
|
549
|
-
#Determine if the user/group pair must be added to/retrieved from the database.
|
550
|
-
user = known_users[[name, group]] if known_users
|
551
|
-
unless user
|
552
|
-
Lock[:users].synchronize do
|
553
|
-
#Does the user already exist?
|
554
|
-
user = Bookie::Database::User.find_by_name_and_group_id(name, group.id)
|
555
|
-
user ||= Bookie::Database::User.create!(
|
556
|
-
:name => name,
|
557
|
-
:group => group
|
558
|
-
)
|
559
|
-
end
|
560
|
-
known_users[[name, group]] = user if known_users
|
561
|
-
end
|
562
|
-
user
|
563
|
-
end
|
564
|
-
|
565
|
-
validates_presence_of :group, :name
|
566
|
-
end
|
567
|
-
|
568
|
-
##
|
569
|
-
#A system on the network
|
570
|
-
class System < ActiveRecord::Base
|
571
|
-
##
|
572
|
-
#Raised when a system's specifications are different from those of the active system in the database
|
573
|
-
SystemConflictError = Class.new(RuntimeError)
|
574
|
-
|
575
|
-
has_many :jobs
|
576
|
-
belongs_to :system_type
|
577
|
-
|
578
|
-
##
|
579
|
-
#Finds all systems that are active (i.e. all systems with NULL values for end_time)
|
580
|
-
def self.active_systems
|
581
|
-
where('systems.end_time IS NULL')
|
582
|
-
end
|
583
|
-
|
584
|
-
##
|
585
|
-
#Filters by name
|
586
|
-
def self.by_name(name)
|
587
|
-
where('systems.name = ?', name)
|
588
|
-
end
|
589
|
-
|
590
|
-
##
|
591
|
-
#Filters by system type
|
592
|
-
def self.by_system_type(sys_type)
|
593
|
-
where('systems.system_type_id = ?', sys_type.id)
|
594
|
-
end
|
595
|
-
|
596
|
-
##
|
597
|
-
#Finds all systems whose running intervals overlap the given time range
|
598
|
-
#
|
599
|
-
#To do: unit test.
|
600
|
-
def self.by_time_range_inclusive(time_range)
|
601
|
-
if time_range.empty?
|
602
|
-
self.none
|
603
|
-
elsif time_range.exclude_end?
|
604
|
-
where('(? <= systems.end_time OR systems.end_time IS NULL) AND systems.start_time < ?', time_range.first, time_range.last)
|
605
|
-
else
|
606
|
-
where('(? <= systems.end_time OR systems.end_time IS NULL) AND systems.start_time <= ?', time_range.first, time_range.last)
|
607
|
-
end
|
608
|
-
end
|
609
|
-
|
610
|
-
##
|
611
|
-
#Finds the current system for a given sender and time
|
612
|
-
#
|
613
|
-
#This method also checks that this system's specifications are the same as those in the database and raises an error if they are different.
|
614
|
-
#
|
615
|
-
#This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
|
616
|
-
def self.find_current(sender, time = nil)
|
617
|
-
time ||= Time.now
|
618
|
-
config = sender.config
|
619
|
-
system = nil
|
620
|
-
name = config.hostname
|
621
|
-
Lock[:systems].synchronize do
|
622
|
-
system = by_name(config.hostname).where('systems.start_time <= :time AND (:time <= systems.end_time OR systems.end_time IS NULL)', :time => time).first
|
623
|
-
if system
|
624
|
-
mismatch = !(system.cores == config.cores && system.memory == config.memory)
|
625
|
-
mismatch ||= sender.system_type != system.system_type
|
626
|
-
if mismatch
|
627
|
-
raise SystemConflictError.new("The specifications on record for '#{name}' do not match this system's specifications.
|
628
|
-
Please make sure that all previous systems with this hostname have been marked as decommissioned.")
|
629
|
-
end
|
630
|
-
else
|
631
|
-
raise "There is no system with hostname '#{config.hostname}' in the database at #{time}."
|
632
|
-
end
|
633
|
-
end
|
634
|
-
system
|
635
|
-
end
|
636
|
-
|
637
|
-
##
|
638
|
-
#Returns an array of all systems, pre-loading relations to reduce the need for extra queries
|
639
|
-
#
|
640
|
-
#Relations are not cached between calls.
|
641
|
-
def self.all_with_relations
|
642
|
-
systems = self.all.to_a
|
643
|
-
system_types = {}
|
644
|
-
systems.each do |system|
|
645
|
-
system_type = system_types[system.system_type_id]
|
646
|
-
if system_type
|
647
|
-
system.system_type = system_type
|
648
|
-
else
|
649
|
-
system_type = system.system_type
|
650
|
-
system_types[system_type.id] = system_type
|
651
|
-
end
|
652
|
-
end
|
653
|
-
systems
|
654
|
-
end
|
655
|
-
|
656
|
-
##
|
657
|
-
#Produces a summary of all the systems for the given time interval
|
658
|
-
#
|
659
|
-
#Returns a hash with the following fields:
|
660
|
-
#- [<tt>:systems</tt>] an array containing all systems that are active in the interval
|
661
|
-
#- [<tt>:avail_cpu_time</tt>] the total CPU time available for the interval
|
662
|
-
#- [<tt>:avail_memory_time</tt>] the total amount of memory-time available (in kilobyte-seconds)
|
663
|
-
#- [<tt>:avail_memory_avg</tt>] the average amount of memory available (in kilobytes)
|
664
|
-
#
|
665
|
-
#To consider: include the start/end times for the summary (especially if they aren't provided as arguments)?
|
666
|
-
#
|
667
|
-
#Notes:
|
668
|
-
#
|
669
|
-
#Results may be slightly off when an inclusive range is used.
|
670
|
-
#To consider: is this worth fixing?
|
671
|
-
def self.summary(time_range = nil)
|
672
|
-
#To consider: how to handle time zones with Rails apps?
|
673
|
-
current_time = Time.now
|
674
|
-
#Sums that are actually returned
|
675
|
-
avail_cpu_time = 0
|
676
|
-
avail_memory_time = 0
|
677
|
-
#Find all the systems within the time range.
|
678
|
-
systems = System
|
679
|
-
if time_range
|
680
|
-
time_range = time_range.normalized
|
681
|
-
#To do: unit test.
|
682
|
-
systems = systems.by_time_range_inclusive(time_range)
|
683
|
-
end
|
684
|
-
|
685
|
-
all_systems = systems.all_with_relations
|
686
|
-
|
687
|
-
all_systems.each do |system|
|
688
|
-
system_start_time = system.start_time
|
689
|
-
system_end_time = system.end_time
|
690
|
-
#Is there a time range constraint?
|
691
|
-
if time_range
|
692
|
-
system_start_time = [system_start_time, time_range.first].max
|
693
|
-
system_end_time = [system_end_time, time_range.last].min if system.end_time
|
694
|
-
system_end_time ||= time_range.last
|
695
|
-
else
|
696
|
-
system_end_time ||= current_time
|
697
|
-
end
|
698
|
-
system_wall_time = system_end_time.to_i - system_start_time.to_i
|
699
|
-
avail_cpu_time += system.cores * system_wall_time
|
700
|
-
avail_memory_time += system.memory * system_wall_time
|
701
|
-
end
|
702
|
-
|
703
|
-
wall_time_range = 0
|
704
|
-
if time_range
|
705
|
-
wall_time_range = time_range.last - time_range.first
|
706
|
-
else
|
707
|
-
first_started_system = systems.order(:start_time).first
|
708
|
-
if first_started_system
|
709
|
-
#Is there a system still active?
|
710
|
-
last_ended_system = systems.where('systems.end_time IS NULL').first
|
711
|
-
if last_ended_system
|
712
|
-
wall_time_range = current_time.to_i - first_started_system.start_time.to_i
|
713
|
-
else
|
714
|
-
#No; find the system that was brought down last.
|
715
|
-
last_ended_system = systems.order('end_time DESC').first
|
716
|
-
wall_time_range = last_ended_system.end_time.to_i - first_started_system.start_time.to_i
|
717
|
-
end
|
718
|
-
end
|
719
|
-
end
|
720
|
-
|
721
|
-
{
|
722
|
-
:systems => all_systems,
|
723
|
-
:avail_cpu_time => avail_cpu_time,
|
724
|
-
:avail_memory_time => avail_memory_time,
|
725
|
-
:avail_memory_avg => if wall_time_range == 0 then 0.0 else Float(avail_memory_time) / wall_time_range end,
|
726
|
-
}
|
727
|
-
end
|
728
|
-
|
729
|
-
##
|
730
|
-
#Decommissions the given system, setting its end time to <tt>end_time</tt>
|
731
|
-
#
|
732
|
-
#This should be called any time a system is brought down or its specifications are changed.
|
733
|
-
def decommission(end_time)
|
734
|
-
self.end_time = end_time
|
735
|
-
self.save!
|
736
|
-
end
|
737
|
-
|
738
|
-
validates_presence_of :name, :cores, :memory, :system_type, :start_time
|
739
|
-
|
740
|
-
validates_each :cores, :memory do |record, attr, value|
|
741
|
-
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
742
|
-
end
|
743
|
-
|
744
|
-
validates_each :end_time do |record, attr, value|
|
745
|
-
record.errors.add(attr, 'must be at or after start time') if value && value < record.start_time
|
746
|
-
end
|
747
|
-
end
|
748
|
-
|
749
17
|
##
|
750
18
|
#A hash mapping memory stat type names to their database codes
|
751
19
|
#
|
@@ -753,73 +21,7 @@ Please make sure that all previous systems with this hostname have been marked a
|
|
753
21
|
#- <tt>:avg => 1</tt>
|
754
22
|
#- <tt>:max => 2</tt>
|
755
23
|
#
|
756
|
-
MEMORY_STAT_TYPE = {:unknown => 0, :avg => 1, :max => 2}
|
757
|
-
|
758
|
-
##
|
759
|
-
#The inverse of MEMORY_STAT_TYPE
|
760
|
-
MEMORY_STAT_TYPE_INVERSE = MEMORY_STAT_TYPE.invert
|
761
|
-
|
762
|
-
#A system type
|
763
|
-
class SystemType < ActiveRecord::Base
|
764
|
-
has_many :systems
|
765
|
-
|
766
|
-
validates_presence_of :name, :memory_stat_type
|
767
|
-
|
768
|
-
##
|
769
|
-
#Finds a system type by name and memory stat type, creating it if it doesn't exist
|
770
|
-
#
|
771
|
-
#It is an error to attempt to create two types with the same name but different memory stat types.
|
772
|
-
#
|
773
|
-
#This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
|
774
|
-
def self.find_or_create!(name, memory_stat_type)
|
775
|
-
sys_type = nil
|
776
|
-
Lock[:system_types].synchronize do
|
777
|
-
sys_type = SystemType.find_by_name(name)
|
778
|
-
if sys_type
|
779
|
-
unless sys_type.memory_stat_type == memory_stat_type
|
780
|
-
type_code = MEMORY_STAT_TYPE[memory_stat_type]
|
781
|
-
if type_code == nil
|
782
|
-
raise "Unrecognized memory stat type '#{memory_stat_type}'"
|
783
|
-
else
|
784
|
-
raise "The recorded memory stat type for system type '#{name}' does not match the required type of #{type_code}"
|
785
|
-
end
|
786
|
-
end
|
787
|
-
else
|
788
|
-
sys_type = create!(
|
789
|
-
:name => name,
|
790
|
-
:memory_stat_type => memory_stat_type
|
791
|
-
)
|
792
|
-
end
|
793
|
-
end
|
794
|
-
sys_type
|
795
|
-
end
|
796
|
-
|
797
|
-
##
|
798
|
-
#Returns the memory stat type as a symbol
|
799
|
-
#
|
800
|
-
#See Bookie::Database::MEMORY_STAT_TYPE for possible values.
|
801
|
-
#
|
802
|
-
#Based on http://www.kensodev.com/2012/05/08/the-simplest-enum-you-will-ever-find-for-your-activerecord-models/
|
803
|
-
def memory_stat_type
|
804
|
-
type_code = read_attribute(:memory_stat_type)
|
805
|
-
raise 'Memory stat type must not be nil' if type_code == nil
|
806
|
-
type = MEMORY_STAT_TYPE_INVERSE[type_code]
|
807
|
-
raise "Unrecognized memory stat type code #{type_code}" unless type
|
808
|
-
type
|
809
|
-
end
|
810
24
|
|
811
|
-
##
|
812
|
-
#Sets the memory stat type
|
813
|
-
#
|
814
|
-
#<tt>type</tt> should be a symbol.
|
815
|
-
def memory_stat_type=(type)
|
816
|
-
raise 'Memory stat type must not be nil' if type == nil
|
817
|
-
type_code = MEMORY_STAT_TYPE[type]
|
818
|
-
raise "Unrecognized memory stat type '#{type}'" unless type_code
|
819
|
-
write_attribute(:memory_stat_type, type_code)
|
820
|
-
end
|
821
|
-
end
|
822
|
-
|
823
25
|
##
|
824
26
|
#Database migrations
|
825
27
|
module Migration
|
@@ -906,7 +108,7 @@ Please make sure that all previous systems with this hostname have been marked a
|
|
906
108
|
t.integer :memory, :null => false
|
907
109
|
t.integer :exit_code, :null => false
|
908
110
|
end
|
909
|
-
#
|
111
|
+
#TODO: more indices?
|
910
112
|
change_table :jobs do |t|
|
911
113
|
t.index :user_id
|
912
114
|
t.index :system_id
|