bookie_accounting 1.2.3 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +4 -3
  3. data/README.md +4 -24
  4. data/Rakefile +9 -116
  5. data/bin/bookie-data +48 -7
  6. data/bin/bookie-send +6 -14
  7. data/bookie_accounting.gemspec +4 -3
  8. data/lib/bookie/database/group.rb +33 -0
  9. data/lib/bookie/database/job.rb +201 -0
  10. data/lib/bookie/database/job_summary.rb +268 -0
  11. data/lib/bookie/database/lock.rb +36 -0
  12. data/lib/bookie/database/system.rb +166 -0
  13. data/lib/bookie/database/system_type.rb +80 -0
  14. data/lib/bookie/database/user.rb +54 -0
  15. data/lib/bookie/database.rb +7 -805
  16. data/lib/bookie/extensions.rb +23 -44
  17. data/lib/bookie/formatter.rb +8 -4
  18. data/lib/bookie/sender.rb +12 -14
  19. data/lib/bookie/version.rb +1 -1
  20. data/snapshot/test_config.json +2 -2
  21. data/spec/config_spec.rb +2 -2
  22. data/spec/database/group_spec.rb +36 -0
  23. data/spec/database/job_spec.rb +308 -0
  24. data/spec/database/job_summary_spec.rb +302 -0
  25. data/spec/database/lock_spec.rb +41 -0
  26. data/spec/database/migration_spec.rb +44 -0
  27. data/spec/database/system_spec.rb +232 -0
  28. data/spec/database/system_type_spec.rb +68 -0
  29. data/spec/database/user_spec.rb +69 -0
  30. data/spec/formatter_spec.rb +44 -37
  31. data/spec/{comma_dump_formatter_spec.rb → formatters/comma_dump_spec.rb} +16 -30
  32. data/spec/formatters/spreadsheet_spec.rb +98 -0
  33. data/spec/{stdout_formatter_spec.rb → formatters/stdout_spec.rb} +15 -29
  34. data/spec/sender_spec.rb +92 -66
  35. data/spec/{standalone_sender_spec.rb → senders/standalone_spec.rb} +10 -9
  36. data/spec/{torque_cluster_sender_spec.rb → senders/torque_cluster_spec.rb} +9 -13
  37. data/spec/spec_helper.rb +111 -57
  38. data/todo.txt +13 -0
  39. metadata +38 -23
  40. data/rpm/activesupport.erb +0 -151
  41. data/rpm/bundle.erb +0 -71
  42. data/rpm/default.erb +0 -147
  43. data/rpm/mysql2.erb +0 -149
  44. data/rpm/pacct.erb +0 -147
  45. data/rpm/rspec-core.erb +0 -149
  46. data/rpm/sqlite3.erb +0 -147
  47. data/spec/database_spec.rb +0 -1078
  48. data/spec/spreadsheet_formatter_spec.rb +0 -114
@@ -2,750 +2,18 @@ require 'bookie/config'
2
2
  require 'bookie/extensions'
3
3
 
4
4
  require 'active_record'
5
- #To do: remove when code is updated.
5
+ #TODO: remove when code is updated.
6
6
  require 'protected_attributes'
7
7
 
8
+ require 'bookie/database/job'
9
+ require 'bookie/database/job_summary'
10
+ require 'bookie/database/user.rb'
11
+ require 'bookie/database/system.rb'
12
+
8
13
  module Bookie
9
14
  ##
10
15
  #Contains database-related code and models
11
16
  module Database
12
-
13
- ##
14
- #Represents a lock on a table
15
- #
16
- #Based on http://kseebaldt.blogspot.com/2007/11/synchronizing-using-active-record.html
17
- #
18
- #This should probably not be called within a transaction block; the lock might not be released
19
- #until the outer transaction completes, and even if it is released before then, there might be
20
- #concurrency issues.
21
- class Lock < ActiveRecord::Base
22
- ##
23
- #Acquires the lock, runs the given block, and releases the lock when finished
24
- def synchronize
25
- transaction do
26
- #Lock this record to be inaccessible to others until this transaction is completed.
27
- self.class.lock.find(id)
28
- yield
29
- end
30
- end
31
-
32
- @locks = {}
33
-
34
- ##
35
- #Returns a lock by name
36
- def self.[](name)
37
- @locks[name.to_sym] ||= find_by_name(name.to_s) or raise "Unable to find lock '#{name}'"
38
- end
39
-
40
- validates_presence_of :name
41
- end
42
-
43
- ##
44
- #A reported job
45
- #
46
- #The various filter methods can be chained to produce more complex queries.
47
- #
48
- #===Examples
49
- # Bookie::Database::Job.by_user_name('root').by_system_name('localhost').find_each do |job|
50
- # puts job.inspect
51
- # end
52
- #
53
- class Job < ActiveRecord::Base
54
- belongs_to :user
55
- belongs_to :system
56
-
57
- ##
58
- #The time at which the job ended
59
- def end_time
60
- return start_time + wall_time
61
- end
62
-
63
- #To consider: disable #end_time= ?
64
-
65
- def self.by_user(user)
66
- where('jobs.user_id = ?', user.id)
67
- end
68
-
69
- ##
70
- #Filters by user name
71
- def self.by_user_name(user_name)
72
- joins(:user).where('users.name = ?', user_name)
73
- end
74
-
75
- def self.by_system(system)
76
- where('jobs.system_id = ?', system.id)
77
- end
78
-
79
- ##
80
- #Filters by system name
81
- def self.by_system_name(system_name)
82
- joins(:system).where('systems.name = ?', system_name)
83
- end
84
-
85
- ##
86
- #Filters by group name
87
- def self.by_group_name(group_name)
88
- group = Group.find_by_name(group_name)
89
- return joins(:user).where('users.group_id = ?', group.id) if group
90
- self.none
91
- end
92
-
93
- ##
94
- #Filters by system type
95
- def self.by_system_type(system_type)
96
- joins(:system).where('systems.system_type_id = ?', system_type.id)
97
- end
98
-
99
- ##
100
- #Filters by command name
101
- def self.by_command_name(c_name)
102
- where('jobs.command_name = ?', c_name)
103
- end
104
-
105
- ##
106
- #Filters by a range of start times
107
- def self.by_start_time_range(time_range)
108
- where('? <= jobs.start_time AND jobs.start_time < ?', time_range.first, time_range.last)
109
- end
110
-
111
- ##
112
- #Filters by a range of end times
113
- def self.by_end_time_range(time_range)
114
- where('? <= jobs.end_time AND jobs.end_time < ?', time_range.first, time_range.last)
115
- end
116
-
117
- ##
118
- #Finds all jobs whose running intervals overlap the given time range
119
- def self.by_time_range_inclusive(time_range)
120
- if time_range.empty?
121
- self.none
122
- elsif time_range.exclude_end?
123
- where('? <= jobs.end_time AND jobs.start_time < ?', time_range.first, time_range.last)
124
- else
125
- where('? <= jobs.end_time AND jobs.start_time <= ?', time_range.first, time_range.last)
126
- end
127
- end
128
-
129
- ##
130
- #Produces a summary of the jobs in the given time interval
131
- #
132
- #Returns a hash with the following fields:
133
- #- <tt>:jobs</tt>: an array of all jobs in the interval
134
- #- <tt>:cpu_time</tt>: the total CPU time used
135
- #- <tt>:memory_time</tt>: the sum of memory * wall_time for all jobs in the interval
136
- #- <tt>:successful</tt>: the number of jobs that have completed successfully
137
- #
138
- #This method should probably not be chained with other queries that filter by start/end time.
139
- #
140
- #To consider: filter out jobs with 0 CPU time?
141
- def self.summary(time_range = nil)
142
- time_range = time_range.normalized if time_range
143
- jobs = self
144
- jobs = jobs.by_time_range_inclusive(time_range) if time_range
145
- jobs = jobs.all_with_relations
146
- cpu_time = 0
147
- successful_jobs = 0
148
- memory_time = 0
149
- #To consider: job.end_time should be <= Time.now, but it might be good to check for that.
150
- #Maybe in a database consistency checker tool?
151
- #What if the system clock is off?
152
- #Also consider a check for system start times.
153
- jobs.each do |job|
154
- job_start_time = job.start_time
155
- job_end_time = job.end_time
156
- if time_range
157
- job_start_time = [job_start_time, time_range.first].max
158
- job_end_time = [job_end_time, time_range.last].min
159
- end
160
- clipped_wall_time = job_end_time.to_i - job_start_time.to_i
161
- if job.wall_time != 0
162
- cpu_time += job.cpu_time * clipped_wall_time / job.wall_time
163
- #To consider: what should I do about jobs that only report a max memory value?
164
- memory_time += job.memory * clipped_wall_time
165
- end
166
- successful_jobs += 1 if job.exit_code == 0
167
- end
168
-
169
- return {
170
- :jobs => jobs,
171
- :cpu_time => cpu_time,
172
- :memory_time => memory_time,
173
- :successful => successful_jobs,
174
- }
175
- end
176
-
177
- ##
178
- #Returns an array of all jobs, pre-loading relations to reduce the need for extra queries
179
- #
180
- #Relations are not cached between calls.
181
- #
182
- #To do: use ActiveRecord's #includes instead of this scheme?
183
- def self.all_with_relations
184
- jobs = self.all.to_a
185
- users = {}
186
- groups = {}
187
- systems = {}
188
- system_types = {}
189
- jobs.each do |job|
190
- system = systems[job.system_id]
191
- if system
192
- job.system = system
193
- else
194
- system = job.system
195
- systems[system.id] = system
196
- end
197
- system_type = system_types[system.system_type_id]
198
- if system_type
199
- system.system_type = system_type
200
- else
201
- system_type = system.system_type
202
- system_types[system_type.id] = system_type
203
- end
204
- user = users[job.user_id]
205
- if user
206
- job.user = user
207
- else
208
- user = job.user
209
- users[user.id] = user
210
- end
211
- group = groups[user.group_id]
212
- if group
213
- user.group = group
214
- else
215
- group = user.group
216
- groups[group.id] = group
217
- end
218
- end
219
-
220
- jobs
221
- end
222
-
223
- before_save do
224
- write_attribute(:end_time, end_time)
225
- end
226
-
227
- before_update do
228
- write_attribute(:end_time, end_time)
229
- end
230
-
231
- validates_presence_of :user, :system, :cpu_time,
232
- :start_time, :wall_time, :memory, :exit_code
233
-
234
- validates_each :command_name do |record, attr, value|
235
- record.errors.add(attr, 'must not be nil') if value == nil
236
- end
237
-
238
- validates_each :cpu_time, :wall_time, :memory do |record, attr, value|
239
- record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
240
- end
241
- end
242
-
243
- ##
244
- #A cached summary of Jobs in the database
245
- #
246
- #Most summary operations should be performed through this class to improve efficiency.
247
- class JobSummary < ActiveRecord::Base
248
- self.table_name = :job_summaries
249
-
250
- belongs_to :user
251
- belongs_to :system
252
-
253
- attr_accessible :date, :user, :user_id, :system, :system_id, :command_name, :cpu_time, :memory_time
254
-
255
- ##
256
- #Filters by date
257
- def self.by_date(date)
258
- where('job_summaries.date = ?', date)
259
- end
260
-
261
- ##
262
- #Filters by a date range
263
- def self.by_date_range(range)
264
- range = range.normalized
265
- if range.exclude_end?
266
- where('? <= job_summaries.date AND job_summaries.date < ?', range.begin, range.end)
267
- else
268
- where('? <= job_summaries.date AND job_summaries.date <= ?', range.begin, range.end)
269
- end
270
- end
271
-
272
- ##
273
- #Filters by user
274
- def self.by_user(user)
275
- where('job_summaries.user_id = ?', user.id)
276
- end
277
-
278
- ##
279
- #Filters by user name
280
- def self.by_user_name(name)
281
- joins(:user).where('users.name = ?', name)
282
- end
283
-
284
- ##
285
- #Filters by group
286
- def self.by_group(group)
287
- joins(:user).where('users.group_id = ?', group.id)
288
- end
289
-
290
- ##
291
- #Filters by group name
292
- def self.by_group_name(name)
293
- group = Group.find_by_name(name)
294
- return by_group(group) if group
295
- self.none
296
- end
297
-
298
- ##
299
- #Filters by system
300
- def self.by_system(system)
301
- where('job_summaries.system_id = ?', system.id)
302
- end
303
-
304
- ##
305
- #Filters by system name
306
- def self.by_system_name(name)
307
- joins(:system).where('systems.name = ?', name)
308
- end
309
-
310
- ##
311
- #Filters by system type
312
- def self.by_system_type(type)
313
- joins(:system).where('systems.system_type_id = ?', type.id)
314
- end
315
-
316
- ##
317
- #Filters by command name
318
- def self.by_command_name(cmd)
319
- where('job_summaries.command_name = ?', cmd)
320
- end
321
-
322
- ##
323
- #Attempts to find a JobSummary with the given date, user_id, system_id, and command_name
324
- #
325
- #If one does not exist, a new JobSummary will be instantiated (but not saved to the database).
326
- def self.find_or_new(date, user_id, system_id, command_name)
327
- str = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).to_sql
328
- summary = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).first
329
- summary ||= new(
330
- :date => date,
331
- :user_id => user_id,
332
- :system_id => system_id,
333
- :command_name => command_name
334
- )
335
- summary
336
- end
337
-
338
- ##
339
- #Create cached summaries for the given date
340
- #
341
- #The date is interpreted as being in UTC.
342
- #
343
- #If there is nothing to summarize, a dummy summary will be created.
344
- #
345
- #Uses Lock::synchronize internally; should not be used in transaction blocks
346
- def self.summarize(date)
347
- jobs = Job
348
- unscoped = self.unscoped
349
- time_min = date.to_utc_time
350
- time_range = time_min ... time_min + 1.days
351
- day_jobs = jobs.by_time_range_inclusive(time_range)
352
-
353
- #Find the sets of unique values.
354
- value_sets = day_jobs.select('user_id, system_id, command_name').uniq
355
- if value_sets.empty?
356
- user = User.select(:id).first
357
- system = System.select(:id).first
358
- #If there are no users or no systems, we can't create the dummy summary, so just return.
359
- return unless user && system
360
- #Create a dummy summary so summary() doesn't keep trying to create one.
361
- Lock[:job_summaries].synchronize do
362
- sum = unscoped.find_or_new(date, user.id, system.id, '')
363
- sum.cpu_time = 0
364
- sum.memory_time = 0
365
- sum.save!
366
- end
367
- else
368
- value_sets.each do |set|
369
- summary_jobs = jobs.where(:user_id => set.user_id).where(:system_id => set.system_id).by_command_name(set.command_name)
370
- summary = summary_jobs.summary(time_range)
371
- Lock[:job_summaries].synchronize do
372
- sum = unscoped.find_or_new(date, set.user_id, set.system_id, set.command_name)
373
- sum.cpu_time = summary[:cpu_time]
374
- sum.memory_time = summary[:memory_time]
375
- sum.save!
376
- end
377
- end
378
- end
379
- end
380
-
381
- ##
382
- #Returns a summary of jobs in the database
383
- #
384
- #The following options are supported:
385
- #- [<tt>:range</tt>] restricts the summary to a specific time interval (specified as a Range of Time objects)
386
- #- [<tt>:jobs</tt>] the jobs on which the summary should operate
387
- #
388
- #Internally, this may call JobSummary::summary, which uses Lock#synchronize, so this should not be used inside a transaction block.
389
- #
390
- #When filtering, the same filters must be applied to both the Jobs and the JobSummaries. For example:
391
- # jobs = Bookie::Database::Job.by_user_name('root')
392
- # summaries = Bookie::Database::Job.by_user_name('root')
393
- # puts summaries.summary(:jobs => jobs)
394
- def self.summary(opts = {})
395
- jobs = opts[:jobs] || Job
396
- range = opts[:range]
397
- unless range
398
- end_time = nil
399
- if System.active_systems.any?
400
- end_time = Time.now
401
- else
402
- last_ended_system = System.order('end_time DESC').first
403
- end_time = last_ended_system.end_time if last_ended_system
404
- end
405
- if end_time
406
- first_started_system = System.order(:start_time).first
407
- range = first_started_system.start_time ... end_time
408
- else
409
- range = Time.new ... Time.new
410
- end
411
- end
412
- range = range.normalized
413
-
414
- num_jobs = 0
415
- cpu_time = 0
416
- memory_time = 0
417
- successful = 0
418
-
419
- #Is the beginning somewhere between days?
420
- date_begin = range.begin.utc.to_date
421
- unless date_begin.to_utc_time == range.begin
422
- date_begin += 1
423
- time_before_max = [date_begin.to_utc_time, range.end].min
424
- time_before_min = range.begin
425
- summary = jobs.summary(time_before_min ... time_before_max)
426
- cpu_time += summary[:cpu_time]
427
- memory_time += summary[:memory_time]
428
- end
429
-
430
- #Is the end somewhere between days?
431
- date_end = range.end.utc.to_date
432
- time_after_min = date_end.to_utc_time
433
- unless time_after_min <= range.begin
434
- time_after_max = range.end
435
- time_after_range = Range.new(time_after_min, time_after_max, range.exclude_end?)
436
- unless time_after_range.empty?
437
- summary = jobs.summary(time_after_range)
438
- cpu_time += summary[:cpu_time]
439
- memory_time += summary[:memory_time]
440
- end
441
- end
442
-
443
- date_range = date_begin ... date_end
444
-
445
- unscoped = self.unscoped
446
- summaries = by_date_range(date_range).order(:date).to_a
447
- index = 0
448
- date_range.each do |date|
449
- new_index = index
450
- sum = summaries[new_index]
451
- while sum && sum.date == date do
452
- cpu_time += sum.cpu_time
453
- memory_time += sum.memory_time
454
- new_index += 1
455
- sum = summaries[new_index]
456
- end
457
- #Did we actually process any summaries?
458
- if new_index == index
459
- #Nope. Create the summaries.
460
- #To consider: optimize out the query?
461
- unscoped.summarize(date)
462
- sums = by_date(date)
463
- sums.each do |sum|
464
- cpu_time += sum.cpu_time
465
- memory_time += sum.memory_time
466
- end
467
- end
468
- end
469
-
470
- if range && range.empty?
471
- num_jobs = 0
472
- successful = 0
473
- else
474
- jobs = jobs.by_time_range_inclusive(range)
475
- num_jobs = jobs.count
476
- successful = jobs.where('jobs.exit_code = 0').count
477
- end
478
-
479
- {
480
- :num_jobs => num_jobs,
481
- :cpu_time => cpu_time,
482
- :memory_time => memory_time,
483
- :successful => successful,
484
- }
485
- end
486
-
487
- validates_presence_of :user_id, :system_id, :date, :cpu_time, :memory_time
488
-
489
- validates_each :command_name do |record, attr, value|
490
- record.errors.add(attr, 'must not be nil') if value == nil
491
- end
492
-
493
- validates_each :cpu_time, :memory_time do |record, attr, value|
494
- record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
495
- end
496
- end
497
-
498
- ##
499
- #A group of users
500
- class Group < ActiveRecord::Base
501
- has_many :users
502
-
503
- ##
504
- #Finds a group by name, creating it if it doesn't exist
505
- #
506
- #If <tt>known_groups</tt> is provided, it will be used as a cache to reduce the number of database lookups needed.
507
- #
508
- #This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
509
- def self.find_or_create!(name, known_groups = nil)
510
- group = known_groups[name] if known_groups
511
- unless group
512
- Lock[:groups].synchronize do
513
- group = find_by_name(name)
514
- group ||= create!(:name => name)
515
- end
516
- known_groups[name] = group if known_groups
517
- end
518
- group
519
- end
520
-
521
- validates_presence_of :name
522
- end
523
-
524
- #ActiveRecord structure for a user
525
- class User < ActiveRecord::Base
526
- belongs_to :group
527
-
528
- def self.by_name(name)
529
- where('users.name = ?', name)
530
- end
531
-
532
- def self.by_group(group)
533
- return where('users.group_id = ?', group.id)
534
- end
535
-
536
- def self.by_group_name(name)
537
- group = Group.find_by_name(name)
538
- return by_group(group) if group
539
- self.none
540
- end
541
-
542
- ##
543
- #Finds a user by name and group, creating it if it doesn't exist
544
- #
545
- #If <tt>known_users</tt> is provided, it will be used as a cache to reduce the number of database lookups needed.
546
- #
547
- #This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
548
- def self.find_or_create!(name, group, known_users = nil)
549
- #Determine if the user/group pair must be added to/retrieved from the database.
550
- user = known_users[[name, group]] if known_users
551
- unless user
552
- Lock[:users].synchronize do
553
- #Does the user already exist?
554
- user = Bookie::Database::User.find_by_name_and_group_id(name, group.id)
555
- user ||= Bookie::Database::User.create!(
556
- :name => name,
557
- :group => group
558
- )
559
- end
560
- known_users[[name, group]] = user if known_users
561
- end
562
- user
563
- end
564
-
565
- validates_presence_of :group, :name
566
- end
567
-
568
- ##
569
- #A system on the network
570
- class System < ActiveRecord::Base
571
- ##
572
- #Raised when a system's specifications are different from those of the active system in the database
573
- SystemConflictError = Class.new(RuntimeError)
574
-
575
- has_many :jobs
576
- belongs_to :system_type
577
-
578
- ##
579
- #Finds all systems that are active (i.e. all systems with NULL values for end_time)
580
- def self.active_systems
581
- where('systems.end_time IS NULL')
582
- end
583
-
584
- ##
585
- #Filters by name
586
- def self.by_name(name)
587
- where('systems.name = ?', name)
588
- end
589
-
590
- ##
591
- #Filters by system type
592
- def self.by_system_type(sys_type)
593
- where('systems.system_type_id = ?', sys_type.id)
594
- end
595
-
596
- ##
597
- #Finds all systems whose running intervals overlap the given time range
598
- #
599
- #To do: unit test.
600
- def self.by_time_range_inclusive(time_range)
601
- if time_range.empty?
602
- self.none
603
- elsif time_range.exclude_end?
604
- where('(? <= systems.end_time OR systems.end_time IS NULL) AND systems.start_time < ?', time_range.first, time_range.last)
605
- else
606
- where('(? <= systems.end_time OR systems.end_time IS NULL) AND systems.start_time <= ?', time_range.first, time_range.last)
607
- end
608
- end
609
-
610
- ##
611
- #Finds the current system for a given sender and time
612
- #
613
- #This method also checks that this system's specifications are the same as those in the database and raises an error if they are different.
614
- #
615
- #This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
616
- def self.find_current(sender, time = nil)
617
- time ||= Time.now
618
- config = sender.config
619
- system = nil
620
- name = config.hostname
621
- Lock[:systems].synchronize do
622
- system = by_name(config.hostname).where('systems.start_time <= :time AND (:time <= systems.end_time OR systems.end_time IS NULL)', :time => time).first
623
- if system
624
- mismatch = !(system.cores == config.cores && system.memory == config.memory)
625
- mismatch ||= sender.system_type != system.system_type
626
- if mismatch
627
- raise SystemConflictError.new("The specifications on record for '#{name}' do not match this system's specifications.
628
- Please make sure that all previous systems with this hostname have been marked as decommissioned.")
629
- end
630
- else
631
- raise "There is no system with hostname '#{config.hostname}' in the database at #{time}."
632
- end
633
- end
634
- system
635
- end
636
-
637
- ##
638
- #Returns an array of all systems, pre-loading relations to reduce the need for extra queries
639
- #
640
- #Relations are not cached between calls.
641
- def self.all_with_relations
642
- systems = self.all.to_a
643
- system_types = {}
644
- systems.each do |system|
645
- system_type = system_types[system.system_type_id]
646
- if system_type
647
- system.system_type = system_type
648
- else
649
- system_type = system.system_type
650
- system_types[system_type.id] = system_type
651
- end
652
- end
653
- systems
654
- end
655
-
656
- ##
657
- #Produces a summary of all the systems for the given time interval
658
- #
659
- #Returns a hash with the following fields:
660
- #- [<tt>:systems</tt>] an array containing all systems that are active in the interval
661
- #- [<tt>:avail_cpu_time</tt>] the total CPU time available for the interval
662
- #- [<tt>:avail_memory_time</tt>] the total amount of memory-time available (in kilobyte-seconds)
663
- #- [<tt>:avail_memory_avg</tt>] the average amount of memory available (in kilobytes)
664
- #
665
- #To consider: include the start/end times for the summary (especially if they aren't provided as arguments)?
666
- #
667
- #Notes:
668
- #
669
- #Results may be slightly off when an inclusive range is used.
670
- #To consider: is this worth fixing?
671
- def self.summary(time_range = nil)
672
- #To consider: how to handle time zones with Rails apps?
673
- current_time = Time.now
674
- #Sums that are actually returned
675
- avail_cpu_time = 0
676
- avail_memory_time = 0
677
- #Find all the systems within the time range.
678
- systems = System
679
- if time_range
680
- time_range = time_range.normalized
681
- #To do: unit test.
682
- systems = systems.by_time_range_inclusive(time_range)
683
- end
684
-
685
- all_systems = systems.all_with_relations
686
-
687
- all_systems.each do |system|
688
- system_start_time = system.start_time
689
- system_end_time = system.end_time
690
- #Is there a time range constraint?
691
- if time_range
692
- system_start_time = [system_start_time, time_range.first].max
693
- system_end_time = [system_end_time, time_range.last].min if system.end_time
694
- system_end_time ||= time_range.last
695
- else
696
- system_end_time ||= current_time
697
- end
698
- system_wall_time = system_end_time.to_i - system_start_time.to_i
699
- avail_cpu_time += system.cores * system_wall_time
700
- avail_memory_time += system.memory * system_wall_time
701
- end
702
-
703
- wall_time_range = 0
704
- if time_range
705
- wall_time_range = time_range.last - time_range.first
706
- else
707
- first_started_system = systems.order(:start_time).first
708
- if first_started_system
709
- #Is there a system still active?
710
- last_ended_system = systems.where('systems.end_time IS NULL').first
711
- if last_ended_system
712
- wall_time_range = current_time.to_i - first_started_system.start_time.to_i
713
- else
714
- #No; find the system that was brought down last.
715
- last_ended_system = systems.order('end_time DESC').first
716
- wall_time_range = last_ended_system.end_time.to_i - first_started_system.start_time.to_i
717
- end
718
- end
719
- end
720
-
721
- {
722
- :systems => all_systems,
723
- :avail_cpu_time => avail_cpu_time,
724
- :avail_memory_time => avail_memory_time,
725
- :avail_memory_avg => if wall_time_range == 0 then 0.0 else Float(avail_memory_time) / wall_time_range end,
726
- }
727
- end
728
-
729
- ##
730
- #Decommissions the given system, setting its end time to <tt>end_time</tt>
731
- #
732
- #This should be called any time a system is brought down or its specifications are changed.
733
- def decommission(end_time)
734
- self.end_time = end_time
735
- self.save!
736
- end
737
-
738
- validates_presence_of :name, :cores, :memory, :system_type, :start_time
739
-
740
- validates_each :cores, :memory do |record, attr, value|
741
- record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
742
- end
743
-
744
- validates_each :end_time do |record, attr, value|
745
- record.errors.add(attr, 'must be at or after start time') if value && value < record.start_time
746
- end
747
- end
748
-
749
17
  ##
750
18
  #A hash mapping memory stat type names to their database codes
751
19
  #
@@ -753,73 +21,7 @@ Please make sure that all previous systems with this hostname have been marked a
753
21
  #- <tt>:avg => 1</tt>
754
22
  #- <tt>:max => 2</tt>
755
23
  #
756
- MEMORY_STAT_TYPE = {:unknown => 0, :avg => 1, :max => 2}
757
-
758
- ##
759
- #The inverse of MEMORY_STAT_TYPE
760
- MEMORY_STAT_TYPE_INVERSE = MEMORY_STAT_TYPE.invert
761
-
762
- #A system type
763
- class SystemType < ActiveRecord::Base
764
- has_many :systems
765
-
766
- validates_presence_of :name, :memory_stat_type
767
-
768
- ##
769
- #Finds a system type by name and memory stat type, creating it if it doesn't exist
770
- #
771
- #It is an error to attempt to create two types with the same name but different memory stat types.
772
- #
773
- #This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
774
- def self.find_or_create!(name, memory_stat_type)
775
- sys_type = nil
776
- Lock[:system_types].synchronize do
777
- sys_type = SystemType.find_by_name(name)
778
- if sys_type
779
- unless sys_type.memory_stat_type == memory_stat_type
780
- type_code = MEMORY_STAT_TYPE[memory_stat_type]
781
- if type_code == nil
782
- raise "Unrecognized memory stat type '#{memory_stat_type}'"
783
- else
784
- raise "The recorded memory stat type for system type '#{name}' does not match the required type of #{type_code}"
785
- end
786
- end
787
- else
788
- sys_type = create!(
789
- :name => name,
790
- :memory_stat_type => memory_stat_type
791
- )
792
- end
793
- end
794
- sys_type
795
- end
796
-
797
- ##
798
- #Returns the memory stat type as a symbol
799
- #
800
- #See Bookie::Database::MEMORY_STAT_TYPE for possible values.
801
- #
802
- #Based on http://www.kensodev.com/2012/05/08/the-simplest-enum-you-will-ever-find-for-your-activerecord-models/
803
- def memory_stat_type
804
- type_code = read_attribute(:memory_stat_type)
805
- raise 'Memory stat type must not be nil' if type_code == nil
806
- type = MEMORY_STAT_TYPE_INVERSE[type_code]
807
- raise "Unrecognized memory stat type code #{type_code}" unless type
808
- type
809
- end
810
24
 
811
- ##
812
- #Sets the memory stat type
813
- #
814
- #<tt>type</tt> should be a symbol.
815
- def memory_stat_type=(type)
816
- raise 'Memory stat type must not be nil' if type == nil
817
- type_code = MEMORY_STAT_TYPE[type]
818
- raise "Unrecognized memory stat type '#{type}'" unless type_code
819
- write_attribute(:memory_stat_type, type_code)
820
- end
821
- end
822
-
823
25
  ##
824
26
  #Database migrations
825
27
  module Migration
@@ -906,7 +108,7 @@ Please make sure that all previous systems with this hostname have been marked a
906
108
  t.integer :memory, :null => false
907
109
  t.integer :exit_code, :null => false
908
110
  end
909
- #To do: more indices?
111
+ #TODO: more indices?
910
112
  change_table :jobs do |t|
911
113
  t.index :user_id
912
114
  t.index :system_id