bookie_accounting 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +133 -0
- data/bin/bookie-data +14 -4
- data/bookie_accounting.gemspec +2 -2
- data/lib/bookie/database.rb +324 -61
- data/lib/bookie/formatter.rb +10 -10
- data/lib/bookie/sender.rb +35 -28
- data/lib/bookie/version.rb +1 -1
- data/rpm/bookie_accounting.spec.erb +81 -0
- data/rpm/spec_template.erb +77 -0
- data/spec/comma_dump_formatter_spec.rb +8 -8
- data/spec/database_spec.rb +441 -94
- data/spec/formatter_spec.rb +7 -6
- data/spec/sender_spec.rb +103 -18
- data/spec/spreadsheet_formatter_spec.rb +9 -9
- data/spec/stdout_formatter_spec.rb +7 -7
- metadata +116 -139
data/Rakefile
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
#!/usr/bin/env rake
|
2
|
+
require 'bundler'
|
2
3
|
require "bundler/gem_tasks"
|
3
4
|
require "rspec/core/rake_task"
|
4
5
|
|
@@ -13,3 +14,135 @@ end
|
|
13
14
|
task :docs do
|
14
15
|
system("rdoc rdoc lib")
|
15
16
|
end
|
17
|
+
|
18
|
+
desc "Build RPM and dependencies (intended for IBEST internal use)"
|
19
|
+
task :rpm_deps do
|
20
|
+
require 'erb'
|
21
|
+
require 'yaml'
|
22
|
+
|
23
|
+
build_rpms({
|
24
|
+
'bookie_accounting' => {
|
25
|
+
:is_app => true,
|
26
|
+
:has_binaries => true,
|
27
|
+
:gem_file => "bookie_accounting-#{Bookie::VERSION}.gem",
|
28
|
+
:license => 'MIT',
|
29
|
+
:skip_deps => Set.new(['sqlite3']),
|
30
|
+
},
|
31
|
+
'bundler' => {
|
32
|
+
|
33
|
+
},
|
34
|
+
'spreadsheet' => {
|
35
|
+
:has_binaries => true,
|
36
|
+
:license => 'GPL3',
|
37
|
+
},
|
38
|
+
'ruby-ole' => {
|
39
|
+
:license => 'MIT',
|
40
|
+
},
|
41
|
+
'pacct' => {
|
42
|
+
:license => 'MIT',
|
43
|
+
},
|
44
|
+
})
|
45
|
+
end
|
46
|
+
|
47
|
+
def find_gem_file(gem_name)
|
48
|
+
Dir.glob("#{gem_name}-*.gem")[0]
|
49
|
+
end
|
50
|
+
|
51
|
+
def build_rpms(modules_to_build)
|
52
|
+
this_dir = Dir.pwd
|
53
|
+
pkg_dir = File.join(this_dir, 'pkg')
|
54
|
+
|
55
|
+
home_dir = Etc.getpwuid.dir
|
56
|
+
|
57
|
+
rpmbuild_dir = File.join(home_dir, 'rpmbuild')
|
58
|
+
spec_dir = File.join(rpmbuild_dir, 'SPECS')
|
59
|
+
src_dir = File.join(rpmbuild_dir, 'SOURCES')
|
60
|
+
rpm_dir = File.join(rpmbuild_dir, 'RPMS')
|
61
|
+
|
62
|
+
[spec_dir, src_dir].each do |dir|
|
63
|
+
FileUtils.mkdir_p(dir)
|
64
|
+
end
|
65
|
+
|
66
|
+
#To do: packaging for Ruby versions other than 1.8
|
67
|
+
template = ERB.new(File.read('rpm/spec_template.erb'))
|
68
|
+
lockfile = Bundler::LockfileParser.new(Bundler.read_file("Gemfile.lock"))
|
69
|
+
lockfile.specs.each do |spec|
|
70
|
+
extra_data = modules_to_build[spec.name]
|
71
|
+
next unless extra_data
|
72
|
+
|
73
|
+
gem_name = spec.name
|
74
|
+
version = spec.version
|
75
|
+
|
76
|
+
puts gem_name
|
77
|
+
|
78
|
+
#Has the RPM already been built?
|
79
|
+
next if Dir.glob(File.join(rpm_dir, "*/(rubygem-)?#{gem_name}-#{version}-*.rpm")).length > 0
|
80
|
+
|
81
|
+
ruby_version = RUBY_VERSION.split('.')[0 .. 1].join('.')
|
82
|
+
|
83
|
+
#Get the gem.
|
84
|
+
Dir.chdir(src_dir)
|
85
|
+
gem_file = extra_data[:gem_file]
|
86
|
+
if gem_file
|
87
|
+
FileUtils.cp(File.join(pkg_dir, gem_file), '.')
|
88
|
+
else
|
89
|
+
gem_file = find_gem_file(gem_name)
|
90
|
+
#To do: handle old versions of gems existing in src_dir
|
91
|
+
system("gem fetch #{gem_name} -v #{version}") unless gem_file && File.exists?(gem_file)
|
92
|
+
gem_file ||= find_gem_file(gem_name)
|
93
|
+
end
|
94
|
+
fail "Unable to find gem file for #{gem_name}" unless gem_file
|
95
|
+
gem = File.basename(gem_file, '.gem')
|
96
|
+
s = YAML.load(`gem spec #{gem_file}`)
|
97
|
+
|
98
|
+
#Build the RPM spec.
|
99
|
+
Dir.chdir(spec_dir)
|
100
|
+
if s.licenses.length > 0
|
101
|
+
license = s.licenses.join(' AND ')
|
102
|
+
else
|
103
|
+
license = extra_data[:license]
|
104
|
+
end
|
105
|
+
url = s.homepage
|
106
|
+
summary = s.summary
|
107
|
+
description = s.description
|
108
|
+
requires = []
|
109
|
+
build_requires = []
|
110
|
+
s.dependencies.each do |dep|
|
111
|
+
case dep.type
|
112
|
+
when :development
|
113
|
+
build_requires.push(dep)
|
114
|
+
else
|
115
|
+
requires.push(dep)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
requires.map!{ |r| "rubygem-#{r.name}" }
|
119
|
+
|
120
|
+
is_app = extra_data[:is_app]
|
121
|
+
has_binaries = extra_data[:has_binaries]
|
122
|
+
|
123
|
+
if is_app
|
124
|
+
spec_filename = "#{gem_name}.spec"
|
125
|
+
else
|
126
|
+
spec_filename = "rubygem-#{gem_name}.spec"
|
127
|
+
end
|
128
|
+
spec_filename = File.join(spec_dir, spec_filename)
|
129
|
+
|
130
|
+
File.open(spec_filename, "w") do |file|
|
131
|
+
template_filename = File.join(this_dir, "rpm/#{gem_name}.spec.erb")
|
132
|
+
if File.exists?(template_filename)
|
133
|
+
file.write(ERB.new(File.read(template_filename)).result(binding))
|
134
|
+
else
|
135
|
+
file.write(template.result(binding))
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
msg = `rpmbuild -bb #{spec_filename}`
|
140
|
+
unless $?.success?
|
141
|
+
puts msg
|
142
|
+
exit 1
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
Dir.chdir(this_dir)
|
147
|
+
end
|
148
|
+
|
data/bin/bookie-data
CHANGED
@@ -8,6 +8,7 @@ require 'optparse'
|
|
8
8
|
require 'bookie/formatter'
|
9
9
|
|
10
10
|
jobs = Bookie::Database::Job
|
11
|
+
summaries = Bookie::Database::JobSummary
|
11
12
|
systems = Bookie::Database::System
|
12
13
|
|
13
14
|
config_filename = '/etc/bookie/config.json'
|
@@ -37,7 +38,7 @@ opts = OptionParser.new do |opts|
|
|
37
38
|
opts.banner = "Usage: bookie-data [options]"
|
38
39
|
|
39
40
|
opts.on('-c', '--config FILE', String, "use the given configuration file") do |file|
|
40
|
-
#This is just here so it shows up in the message.
|
41
|
+
#This is just here for validation and so it shows up in the usage message.
|
41
42
|
end
|
42
43
|
|
43
44
|
opts.on('-d', '--details', "include full details") do
|
@@ -46,14 +47,22 @@ opts = OptionParser.new do |opts|
|
|
46
47
|
|
47
48
|
opts.on('-u', '--user NAME', "filter by username") do |name|
|
48
49
|
jobs = jobs.by_user_name(name)
|
50
|
+
summaries = summaries.by_user_name(name)
|
49
51
|
end
|
50
52
|
|
51
|
-
opts.on('-g', '--group NAME' "filter by group") do |name|
|
53
|
+
opts.on('-g', '--group NAME', "filter by group") do |name|
|
52
54
|
jobs = jobs.by_group_name(name)
|
55
|
+
summaries = summaries.by_group_name(name)
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on('-m', '--command', "filter by command") do |cmd|
|
59
|
+
jobs = jobs.by_command_name(cmd)
|
60
|
+
summaries = summaries.by_command_name(cmd)
|
53
61
|
end
|
54
62
|
|
55
63
|
opts.on('-s', '--system HOSTNAME', "filter by system") do |hostname|
|
56
64
|
jobs = jobs.by_system_name(hostname)
|
65
|
+
summaries = summaries.by_system_name(hostname)
|
57
66
|
systems = systems.by_name(hostname)
|
58
67
|
end
|
59
68
|
|
@@ -64,6 +73,7 @@ opts = OptionParser.new do |opts|
|
|
64
73
|
exit 1
|
65
74
|
end
|
66
75
|
jobs = jobs.by_system_type(t)
|
76
|
+
summaries = summaries.by_system_type(t)
|
67
77
|
systems = systems.by_system_type(t)
|
68
78
|
end
|
69
79
|
|
@@ -96,7 +106,7 @@ end
|
|
96
106
|
|
97
107
|
formatter = Bookie::Formatter.new(output_type, filename)
|
98
108
|
|
99
|
-
jobs_summary, systems_summary = formatter.print_summary(jobs, systems, t_min, t_max)
|
109
|
+
jobs_summary, systems_summary = formatter.print_summary(jobs, summaries, systems, t_min, t_max)
|
100
110
|
jobs = jobs.by_time_range_inclusive(t_min, t_max) if t_min
|
101
|
-
formatter.print_jobs(
|
111
|
+
formatter.print_jobs(jobs.all) if include_details
|
102
112
|
formatter.flush
|
data/bookie_accounting.gemspec
CHANGED
@@ -4,6 +4,7 @@ require File.expand_path('../lib/bookie/version', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.authors = ["Ben Merritt"]
|
6
6
|
gem.email = ["blm768@gmail.com"]
|
7
|
+
gem.license = "MIT"
|
7
8
|
gem.description = %q{A simple system to record and query process accounting records}
|
8
9
|
gem.summary = %q{A simple system to record and query process accounting records}
|
9
10
|
gem.homepage = "https://github.com/blm768/bookie/"
|
@@ -15,11 +16,10 @@ Gem::Specification.new do |gem|
|
|
15
16
|
gem.require_paths = ["lib"]
|
16
17
|
gem.version = Bookie::VERSION
|
17
18
|
|
18
|
-
gem.add_dependency('json')
|
19
19
|
gem.add_dependency('activerecord')
|
20
20
|
#For some reason, this is needed for Ruby 1.8.7 using RVM on CentOS.
|
21
21
|
#To do: remove when no longer needed
|
22
|
-
gem.add_dependency('mysql2')
|
22
|
+
#gem.add_dependency('mysql2')
|
23
23
|
gem.add_dependency('pacct')
|
24
24
|
gem.add_dependency('spreadsheet')
|
25
25
|
gem.add_development_dependency('mocha')
|
data/lib/bookie/database.rb
CHANGED
@@ -58,11 +58,19 @@ module Bookie
|
|
58
58
|
return start_time + wall_time
|
59
59
|
end
|
60
60
|
|
61
|
+
def self.by_user(user)
|
62
|
+
where('jobs.user_id = ?', user.id)
|
63
|
+
end
|
64
|
+
|
61
65
|
##
|
62
66
|
#Filters by user name
|
63
67
|
def self.by_user_name(user_name)
|
64
68
|
joins(:user).where('users.name = ?', user_name)
|
65
69
|
end
|
70
|
+
|
71
|
+
def self.by_system(system)
|
72
|
+
where('jobs.system_id = ?', system.id)
|
73
|
+
end
|
66
74
|
|
67
75
|
##
|
68
76
|
#Filters by system name
|
@@ -92,21 +100,20 @@ module Bookie
|
|
92
100
|
|
93
101
|
##
|
94
102
|
#Filters by a range of start times
|
95
|
-
def self.by_start_time_range(
|
96
|
-
where('? <= jobs.start_time AND jobs.start_time < ?',
|
103
|
+
def self.by_start_time_range(time_range)
|
104
|
+
where('? <= jobs.start_time AND jobs.start_time < ?', time_range.first, time_range.last)
|
97
105
|
end
|
98
106
|
|
99
107
|
##
|
100
108
|
#Filters by a range of end times
|
101
|
-
def self.by_end_time_range(
|
102
|
-
where('? <= jobs.end_time AND jobs.end_time < ?',
|
109
|
+
def self.by_end_time_range(time_range)
|
110
|
+
where('? <= jobs.end_time AND jobs.end_time < ?', time_range.first, time_range.last)
|
103
111
|
end
|
104
112
|
|
105
113
|
##
|
106
114
|
#Finds all jobs whose running intervals overlap the given time range
|
107
|
-
def self.by_time_range_inclusive(
|
108
|
-
|
109
|
-
where('jobs.start_time < ? AND jobs.end_time > ?', max_time, min_time)
|
115
|
+
def self.by_time_range_inclusive(time_range)
|
116
|
+
where('? <= jobs.end_time AND jobs.start_time < ?', time_range.first, time_range.last)
|
110
117
|
end
|
111
118
|
|
112
119
|
##
|
@@ -114,20 +121,16 @@ module Bookie
|
|
114
121
|
#
|
115
122
|
#Returns a hash with the following fields:
|
116
123
|
#- <tt>:jobs</tt>: an array of all jobs in the interval
|
117
|
-
#- <tt>:wall_time</tt>: the sum of all the jobs' wall times
|
118
124
|
#- <tt>:cpu_time</tt>: the total CPU time used
|
119
125
|
#- <tt>:memory_time</tt>: the sum of memory * wall_time for all jobs in the interval
|
120
|
-
#- <tt>:successful</tt>: the
|
126
|
+
#- <tt>:successful</tt>: the number of jobs that have completed successfully
|
121
127
|
#
|
122
128
|
#This method should probably not be used with other queries that filter by start/end time.
|
123
|
-
def self.summary(
|
129
|
+
def self.summary(time_range = nil)
|
130
|
+
time_range = time_range.normalized if time_range
|
124
131
|
jobs = self
|
125
|
-
if
|
126
|
-
raise ArgumentError.new('Max time must be specified with min time') unless max_time
|
127
|
-
jobs = jobs.by_time_range_inclusive(min_time, max_time)
|
128
|
-
end
|
132
|
+
jobs = jobs.by_time_range_inclusive(time_range) if time_range
|
129
133
|
jobs = jobs.where('jobs.cpu_time > 0').all_with_relations
|
130
|
-
wall_time = 0
|
131
134
|
cpu_time = 0
|
132
135
|
successful_jobs = 0
|
133
136
|
memory_time = 0
|
@@ -138,28 +141,27 @@ module Bookie
|
|
138
141
|
jobs.each do |job|
|
139
142
|
job_start_time = job.start_time
|
140
143
|
job_end_time = job.end_time
|
141
|
-
if
|
142
|
-
job_start_time = [job_start_time,
|
143
|
-
job_end_time = [job_end_time,
|
144
|
+
if time_range
|
145
|
+
job_start_time = [job_start_time, time_range.first].max
|
146
|
+
job_end_time = [job_end_time, time_range.last].min
|
144
147
|
end
|
145
148
|
clipped_wall_time = job_end_time.to_i - job_start_time.to_i
|
146
|
-
wall_time += clipped_wall_time
|
147
149
|
if job.wall_time != 0
|
148
150
|
cpu_time += job.cpu_time * clipped_wall_time / job.wall_time
|
149
151
|
#To consider: what should I do about jobs that only report a max memory value?
|
150
152
|
memory_time += job.memory * clipped_wall_time
|
151
153
|
end
|
152
|
-
|
154
|
+
#Only count the job as successful if it's actually finished by the end of the summary.
|
155
|
+
if job.exit_code == 0 && (!time_range || job.end_time < time_range.end) then
|
156
|
+
successful_jobs += 1
|
157
|
+
end
|
153
158
|
end
|
154
159
|
|
155
160
|
return {
|
156
161
|
:jobs => jobs,
|
157
|
-
#To consider: is this field even useful? It's really in job-seconds, not just seconds.
|
158
|
-
#What about one in just seconds (that considers gaps in activity)?
|
159
|
-
:wall_time => wall_time,
|
160
162
|
:cpu_time => cpu_time,
|
161
163
|
:memory_time => memory_time,
|
162
|
-
:successful =>
|
164
|
+
:successful => successful_jobs,
|
163
165
|
}
|
164
166
|
end
|
165
167
|
|
@@ -219,16 +221,183 @@ module Bookie
|
|
219
221
|
|
220
222
|
validates_presence_of :user, :system, :cpu_time,
|
221
223
|
:start_time, :wall_time, :memory, :exit_code
|
224
|
+
|
225
|
+
validates_each :command_name do |record, attr, value|
|
226
|
+
record.errors.add(attr, 'must not be nil') if value == nil
|
227
|
+
end
|
222
228
|
|
223
229
|
validates_each :cpu_time, :wall_time, :memory do |record, attr, value|
|
224
230
|
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
225
231
|
end
|
232
|
+
end
|
233
|
+
|
234
|
+
class JobSummary < ActiveRecord::Base
|
235
|
+
self.table_name = :job_summaries
|
236
|
+
|
237
|
+
belongs_to :user
|
238
|
+
belongs_to :system
|
239
|
+
|
240
|
+
def self.by_date(date)
|
241
|
+
where('job_summaries.date = ?', date)
|
242
|
+
end
|
243
|
+
|
244
|
+
def self.by_user(user)
|
245
|
+
where('job_summaries.user_id = ?', user.id)
|
246
|
+
end
|
247
|
+
|
248
|
+
def self.by_user_name(name)
|
249
|
+
joins(:user).where('users.name = ?', name)
|
250
|
+
end
|
251
|
+
|
252
|
+
def self.by_group(group)
|
253
|
+
joins(:user).where('users.group_id = ?', group.id)
|
254
|
+
end
|
255
|
+
|
256
|
+
def self.by_group_name(name)
|
257
|
+
group = Group.find_by_name(name)
|
258
|
+
return by_group(group) if group
|
259
|
+
limit(0)
|
260
|
+
end
|
261
|
+
|
262
|
+
def self.by_system(system)
|
263
|
+
where('job_summaries.system_id = ?', system.id)
|
264
|
+
end
|
265
|
+
|
266
|
+
def self.by_system_name(name)
|
267
|
+
joins(:system).where('systems.name = ?', name)
|
268
|
+
end
|
269
|
+
|
270
|
+
def self.by_system_type(type)
|
271
|
+
joins(:system).where('systems.system_type_id = ?', type.id)
|
272
|
+
end
|
273
|
+
|
274
|
+
def self.by_command_name(cmd)
|
275
|
+
where('job_summaries.command_name = ?', cmd)
|
276
|
+
end
|
277
|
+
|
278
|
+
def self.find_or_new(date, user_id, system_id, command_name)
|
279
|
+
str = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).to_sql
|
280
|
+
summary = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).first
|
281
|
+
summary ||= new(
|
282
|
+
:date => date,
|
283
|
+
:user_id => user_id,
|
284
|
+
:system_id => system_id,
|
285
|
+
:command_name => command_name
|
286
|
+
)
|
287
|
+
summary
|
288
|
+
end
|
289
|
+
|
290
|
+
def self.summarize(date)
|
291
|
+
jobs = Job
|
292
|
+
unscoped = self.unscoped
|
293
|
+
time_range = date.to_time ... (date + 1).to_time
|
294
|
+
day_jobs = jobs.by_time_range_inclusive(time_range)
|
295
|
+
value_sets = day_jobs.select('user_id, system_id, command_name').uniq
|
296
|
+
value_sets.each do |set|
|
297
|
+
summary_jobs = jobs.where(:user_id => set.user_id).where(:system_id => set.system_id).by_command_name(set.command_name)
|
298
|
+
summary = summary_jobs.summary(time_range)
|
299
|
+
Lock[:job_summaries].synchronize do
|
300
|
+
sum = unscoped.find_or_new(date, set.user_id, set.system_id, set.command_name)
|
301
|
+
#To consider: do we even need this field? (Time-range summaries don't use it because of overlap issues.)
|
302
|
+
sum.num_jobs = summary[:jobs].length
|
303
|
+
sum.cpu_time = summary[:cpu_time]
|
304
|
+
sum.memory_time = summary[:memory_time]
|
305
|
+
sum.successful = summary[:successful]
|
306
|
+
sum.save!
|
307
|
+
end
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
def self.summary(opts = {})
|
312
|
+
jobs = opts[:jobs] || Job
|
313
|
+
range = opts[:range]
|
314
|
+
unless range
|
315
|
+
end_time = nil
|
316
|
+
if System.active_systems.any?
|
317
|
+
end_time = Time.now
|
318
|
+
else
|
319
|
+
last_ended_system = System.order('end_time DESC').first
|
320
|
+
end_time = last_ended_system.end_time if last_ended_system
|
321
|
+
end
|
322
|
+
if end_time
|
323
|
+
first_started_system = System.order(:start_time).first
|
324
|
+
range = first_started_system.start_time ... end_time
|
325
|
+
else
|
326
|
+
range = Date.new ... Date.new
|
327
|
+
end
|
328
|
+
end
|
329
|
+
range = range.normalized
|
330
|
+
|
331
|
+
num_jobs = 0
|
332
|
+
cpu_time = 0
|
333
|
+
memory_time = 0
|
334
|
+
successful = 0
|
335
|
+
|
336
|
+
#Could there be partial days at the beginning/end?
|
337
|
+
date_range = range
|
338
|
+
unless range.begin.kind_of?(Date) && range.end.kind_of?(Date)
|
339
|
+
date_begin = range.begin.to_date
|
340
|
+
unless date_begin.to_time == range.begin
|
341
|
+
date_begin += 1
|
342
|
+
time_before_max = [date_begin.to_time, range.end].min
|
343
|
+
time_before_min = range.begin
|
344
|
+
summary = jobs.summary(time_before_min ... time_before_max)
|
345
|
+
cpu_time += summary[:cpu_time]
|
346
|
+
memory_time += summary[:memory_time]
|
347
|
+
successful += summary[:successful]
|
348
|
+
end
|
349
|
+
|
350
|
+
date_end = range.end.to_date
|
351
|
+
time_after_min = date_end.to_time
|
352
|
+
unless time_after_min <= range.begin
|
353
|
+
time_after_max = range.end
|
354
|
+
time_after_range = Range.new(time_after_min, time_after_max, range.exclude_end?)
|
355
|
+
unless time_after_range.empty?
|
356
|
+
summary = jobs.summary(time_after_range)
|
357
|
+
cpu_time += summary[:cpu_time]
|
358
|
+
memory_time += summary[:memory_time]
|
359
|
+
successful += summary[:successful]
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
date_range = date_begin ... date_end
|
364
|
+
end
|
365
|
+
|
366
|
+
unscoped = self.unscoped
|
367
|
+
date = date_range.begin
|
368
|
+
while date_range.cover?(date) do
|
369
|
+
#To do: what if there aren't any summaries to be made? Will we just run summarize() each time?
|
370
|
+
summarize(date) if unscoped.by_date(date).empty?
|
371
|
+
summaries = by_date(date)
|
372
|
+
summaries.all.each do |summary|
|
373
|
+
cpu_time += summary.cpu_time
|
374
|
+
memory_time += summary.memory_time
|
375
|
+
successful += summary.successful
|
376
|
+
end
|
377
|
+
date += 1
|
378
|
+
end
|
379
|
+
|
380
|
+
time_range = Range.new(range.begin.to_time, range.end.to_time, range.exclude_end?)
|
381
|
+
jobs = jobs.by_time_range_inclusive(time_range)
|
382
|
+
num_jobs = (range && range.empty?) ? 0 : jobs.count
|
383
|
+
|
384
|
+
{
|
385
|
+
:num_jobs => num_jobs,
|
386
|
+
:cpu_time => cpu_time,
|
387
|
+
:memory_time => memory_time,
|
388
|
+
:successful => successful,
|
389
|
+
}
|
390
|
+
end
|
391
|
+
|
392
|
+
validates_presence_of :user_id, :system_id, :date, :num_jobs, :cpu_time, :memory_time, :successful
|
226
393
|
|
227
|
-
validates_each :
|
228
|
-
|
229
|
-
record.errors.add(attr, 'must be a time object') unless value.is_a?(Time)
|
394
|
+
validates_each :command_name do |record, attr, value|
|
395
|
+
record.errors.add(attr, 'must not be nil') if value == nil
|
230
396
|
end
|
231
397
|
|
398
|
+
validates_each :num_jobs, :cpu_time, :memory_time, :successful do |record, attr, value|
|
399
|
+
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
400
|
+
end
|
232
401
|
end
|
233
402
|
|
234
403
|
##
|
@@ -261,6 +430,10 @@ module Bookie
|
|
261
430
|
class User < ActiveRecord::Base
|
262
431
|
belongs_to :group
|
263
432
|
|
433
|
+
def self.by_name(name)
|
434
|
+
where('users.name = ?', name)
|
435
|
+
end
|
436
|
+
|
264
437
|
##
|
265
438
|
#Finds a user by name and group, creating it if it doesn't exist
|
266
439
|
#
|
@@ -276,7 +449,8 @@ module Bookie
|
|
276
449
|
user = Bookie::Database::User.find_by_name_and_group_id(name, group.id)
|
277
450
|
user ||= Bookie::Database::User.create!(
|
278
451
|
:name => name,
|
279
|
-
:group => group
|
452
|
+
:group => group
|
453
|
+
)
|
280
454
|
end
|
281
455
|
known_users[[name, group]] = user if known_users
|
282
456
|
end
|
@@ -315,28 +489,28 @@ module Bookie
|
|
315
489
|
end
|
316
490
|
|
317
491
|
##
|
318
|
-
#Finds the
|
319
|
-
#
|
320
|
-
#<tt>values</tt> should contain a list of fields, including the name, in the format that would normally be passed to System.create!.
|
492
|
+
#Finds the current system for a given sender and time
|
321
493
|
#
|
322
494
|
#This method also checks that this system's specifications are the same as those in the database and raises an error if they are different.
|
323
495
|
#
|
324
496
|
#This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
|
325
|
-
|
497
|
+
#
|
498
|
+
def self.find_current(sender, time = nil)
|
499
|
+
time ||= Time.now
|
500
|
+
config = sender.config
|
326
501
|
system = nil
|
327
|
-
name =
|
502
|
+
name = config.hostname
|
328
503
|
Lock[:systems].synchronize do
|
329
|
-
system =
|
504
|
+
system = by_name(config.hostname).where('systems.start_time <= :time AND (:time <= systems.end_time OR systems.end_time IS NULL)', :time => time).first
|
330
505
|
if system
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
end
|
506
|
+
mismatch = !(system.cores == config.cores && system.memory == config.memory)
|
507
|
+
mismatch ||= sender.system_type != system.system_type
|
508
|
+
if mismatch
|
509
|
+
raise SystemConflictError.new("The specifications on record for '#{name}' do not match this system's specifications.
|
510
|
+
Please make sure that all previous systems with this hostname have been marked as decommissioned.")
|
337
511
|
end
|
338
512
|
else
|
339
|
-
raise "There is no
|
513
|
+
raise "There is no system with hostname '#{config.hostname}' in the database at #{time}."
|
340
514
|
end
|
341
515
|
end
|
342
516
|
system
|
@@ -349,31 +523,35 @@ module Bookie
|
|
349
523
|
#- <tt>:avail_cpu_time</tt>: the total CPU time available for the interval
|
350
524
|
#- <tt>:avail_memory_time</tt>: the total amount of memory-time available (in kilobyte-seconds)
|
351
525
|
#- <tt>:avail_memory_avg</tt>: the average amount of memory available (in kilobytes)
|
352
|
-
|
526
|
+
#
|
527
|
+
#To consider: include the start/end times for the summary (especially if they aren't provided as arguments)?
|
528
|
+
#
|
529
|
+
#To do: make this and other summaries operate differently on inclusive and exclusive ranges.
|
530
|
+
#(Current behavior is as if the range were always exclusive.)
|
531
|
+
def self.summary(time_range = nil)
|
353
532
|
current_time = Time.now
|
354
533
|
#Sums that are actually returned
|
355
534
|
avail_cpu_time = 0
|
356
535
|
avail_memory_time = 0
|
357
536
|
#Find all the systems within the time range.
|
358
537
|
systems = System
|
359
|
-
if
|
360
|
-
|
361
|
-
raise ArgumentError.new('Max time must be greater than or equal to min time') if max_time < min_time
|
538
|
+
if time_range
|
539
|
+
time_range = time_range.normalized
|
362
540
|
#To consider: optimize as union of queries?
|
363
541
|
systems = systems.where(
|
364
542
|
'systems.start_time < ? AND (systems.end_time IS NULL OR systems.end_time > ?)',
|
365
|
-
|
366
|
-
|
543
|
+
time_range.last,
|
544
|
+
time_range.first)
|
367
545
|
end
|
368
546
|
|
369
547
|
systems.all.each do |system|
|
370
548
|
system_start_time = system.start_time
|
371
549
|
system_end_time = system.end_time
|
372
550
|
#Is there a time range constraint?
|
373
|
-
if
|
374
|
-
system_start_time = [system_start_time,
|
375
|
-
system_end_time = [system_end_time,
|
376
|
-
system_end_time ||=
|
551
|
+
if time_range
|
552
|
+
system_start_time = [system_start_time, time_range.first].max
|
553
|
+
system_end_time = [system_end_time, time_range.last].min if system.end_time
|
554
|
+
system_end_time ||= time_range.last
|
377
555
|
else
|
378
556
|
system_end_time ||= current_time
|
379
557
|
end
|
@@ -383,8 +561,8 @@ module Bookie
|
|
383
561
|
end
|
384
562
|
|
385
563
|
wall_time_range = 0
|
386
|
-
if
|
387
|
-
wall_time_range =
|
564
|
+
if time_range
|
565
|
+
wall_time_range = time_range.last - time_range.first
|
388
566
|
else
|
389
567
|
first_started_system = systems.order(:start_time).first
|
390
568
|
if first_started_system
|
@@ -422,11 +600,6 @@ module Bookie
|
|
422
600
|
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
423
601
|
end
|
424
602
|
|
425
|
-
validates_each :start_time do |record, attr, value|
|
426
|
-
value = value.to_time if value.respond_to?(:to_time)
|
427
|
-
record.errors.add(attr, 'must be a time object') unless value.is_a?(Time)
|
428
|
-
end
|
429
|
-
|
430
603
|
validates_each :end_time do |record, attr, value|
|
431
604
|
record.errors.add(attr, 'must be at or after start time') if value && value < record.start_time
|
432
605
|
end
|
@@ -508,7 +681,7 @@ module Bookie
|
|
508
681
|
|
509
682
|
##
|
510
683
|
#Database migrations
|
511
|
-
module Migration
|
684
|
+
module Migration
|
512
685
|
class CreateUsers < ActiveRecord::Migration
|
513
686
|
def up
|
514
687
|
create_table :users do |t|
|
@@ -585,7 +758,7 @@ module Bookie
|
|
585
758
|
create_table :jobs do |t|
|
586
759
|
t.references :user, :null => false
|
587
760
|
t.references :system, :null => false
|
588
|
-
t.string :command_name, :limit => 24
|
761
|
+
t.string :command_name, :limit => 24, :null => false
|
589
762
|
t.datetime :start_time, :null => false
|
590
763
|
t.datetime :end_time, :null => false
|
591
764
|
t.integer :wall_time, :null => false
|
@@ -607,6 +780,31 @@ module Bookie
|
|
607
780
|
end
|
608
781
|
end
|
609
782
|
|
783
|
+
class CreateJobSummaries < ActiveRecord::Migration
|
784
|
+
def up
|
785
|
+
create_table :job_summaries do |t|
|
786
|
+
t.references :user, :null => false
|
787
|
+
t.references :system, :null => false
|
788
|
+
t.date :date, :null => false
|
789
|
+
t.string :command_name, :null => false
|
790
|
+
t.integer :num_jobs, :null => false
|
791
|
+
t.integer :cpu_time, :null => false
|
792
|
+
t.integer :memory_time, :null => false
|
793
|
+
t.integer :successful, :null => false
|
794
|
+
end
|
795
|
+
change_table :job_summaries do |t|
|
796
|
+
#To consider: reorder for optimum efficiency?
|
797
|
+
t.index [:date, :user_id, :system_id, :command_name], :unique => true, :name => 'identity'
|
798
|
+
t.index :command_name
|
799
|
+
t.index :date
|
800
|
+
end
|
801
|
+
end
|
802
|
+
|
803
|
+
def down
|
804
|
+
drop_table :job_summaries
|
805
|
+
end
|
806
|
+
end
|
807
|
+
|
610
808
|
class CreateLocks < ActiveRecord::Migration
|
611
809
|
def up
|
612
810
|
create_table :locks do |t|
|
@@ -616,7 +814,7 @@ module Bookie
|
|
616
814
|
t.index :name, :unique => true
|
617
815
|
end
|
618
816
|
|
619
|
-
['users', 'groups', 'systems', 'system_types'].each do |name|
|
817
|
+
['users', 'groups', 'systems', 'system_types', 'job_summaries'].each do |name|
|
620
818
|
Lock.create!(:name => name)
|
621
819
|
end
|
622
820
|
end
|
@@ -630,11 +828,13 @@ module Bookie
|
|
630
828
|
##
|
631
829
|
#Brings up all migrations
|
632
830
|
def up
|
831
|
+
ActiveRecord::Migration.verbose = false
|
633
832
|
CreateUsers.new.up
|
634
833
|
CreateGroups.new.up
|
635
834
|
CreateSystems.new.up
|
636
835
|
CreateSystemTypes.new.up
|
637
836
|
CreateJobs.new.up
|
837
|
+
CreateJobSummaries.new.up
|
638
838
|
CreateLocks.new.up
|
639
839
|
end
|
640
840
|
|
@@ -643,14 +843,77 @@ module Bookie
|
|
643
843
|
#
|
644
844
|
#Warning: this will destroy all data!
|
645
845
|
def down
|
846
|
+
ActiveRecord::Migration.verbose = false
|
646
847
|
CreateUsers.new.down
|
647
848
|
CreateGroups.new.down
|
648
849
|
CreateSystems.new.down
|
649
850
|
CreateSystemTypes.new.down
|
650
851
|
CreateJobs.new.down
|
852
|
+
CreateJobSummaries.new.down
|
651
853
|
CreateLocks.new.down
|
652
854
|
end
|
653
855
|
end
|
654
856
|
end
|
655
857
|
end
|
656
858
|
end
|
859
|
+
|
860
|
+
##
|
861
|
+
#Reopened to add some useful methods
|
862
|
+
class Range
|
863
|
+
##
|
864
|
+
#If end < begin, returns an empty range (begin ... begin)
|
865
|
+
#Otherwise, returns the original range
|
866
|
+
def normalized
|
867
|
+
return self.begin ... self.begin if self.end < self.begin
|
868
|
+
self
|
869
|
+
end
|
870
|
+
|
871
|
+
##
|
872
|
+
#Returns the empty status of the range
|
873
|
+
#
|
874
|
+
#A range is empty if end < begin or if begin == end and exclude_end? is true.
|
875
|
+
def empty?
|
876
|
+
(self.end < self.begin) || (exclude_end? && (self.begin == self.end))
|
877
|
+
end
|
878
|
+
|
879
|
+
#This code probably works, but we're not using it anywhere.
|
880
|
+
# def intersection(other)
|
881
|
+
# self_n = self.normalized
|
882
|
+
# other = other.normalized
|
883
|
+
#
|
884
|
+
# new_begin, new_end, exclude_end = nil
|
885
|
+
#
|
886
|
+
# if self_n.cover?(other.begin)
|
887
|
+
# new_first = other.begin
|
888
|
+
# elsif other.cover?(self_n.begin)
|
889
|
+
# new_first = self_n.begin
|
890
|
+
# end
|
891
|
+
#
|
892
|
+
# return self_n.begin ... self_n.begin unless new_first
|
893
|
+
#
|
894
|
+
# if self_n.cover?(other.end)
|
895
|
+
# unless other.exclude_end? && other.end == self_n.begin
|
896
|
+
# new_end = other.end
|
897
|
+
# exclude_end = other.exclude_end?
|
898
|
+
# end
|
899
|
+
# elsif other.cover?(self_n.end)
|
900
|
+
# unless self_n.exclude_end? && self_n.end == other.begin
|
901
|
+
# new_end = self_n.end
|
902
|
+
# exclude_end = self_n.exclude_end?
|
903
|
+
# end
|
904
|
+
# end
|
905
|
+
#
|
906
|
+
# #If we still haven't found new_end, try one more case:
|
907
|
+
# unless new_end
|
908
|
+
# if self_n.end == other.end
|
909
|
+
# #We'll only get here if both ranges exclude their ends and have the same end.
|
910
|
+
# new_end = self_n.end
|
911
|
+
# exclude_end = true
|
912
|
+
# end
|
913
|
+
# end
|
914
|
+
#
|
915
|
+
# return self_n.begin ... self_n.begin unless new_end
|
916
|
+
#
|
917
|
+
# Range.new(new_begin, new_end, exclude_end)
|
918
|
+
# end
|
919
|
+
end
|