bookie_accounting 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +133 -0
- data/bin/bookie-data +14 -4
- data/bookie_accounting.gemspec +2 -2
- data/lib/bookie/database.rb +324 -61
- data/lib/bookie/formatter.rb +10 -10
- data/lib/bookie/sender.rb +35 -28
- data/lib/bookie/version.rb +1 -1
- data/rpm/bookie_accounting.spec.erb +81 -0
- data/rpm/spec_template.erb +77 -0
- data/spec/comma_dump_formatter_spec.rb +8 -8
- data/spec/database_spec.rb +441 -94
- data/spec/formatter_spec.rb +7 -6
- data/spec/sender_spec.rb +103 -18
- data/spec/spreadsheet_formatter_spec.rb +9 -9
- data/spec/stdout_formatter_spec.rb +7 -7
- metadata +116 -139
data/Rakefile
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env rake
|
|
2
|
+
require 'bundler'
|
|
2
3
|
require "bundler/gem_tasks"
|
|
3
4
|
require "rspec/core/rake_task"
|
|
4
5
|
|
|
@@ -13,3 +14,135 @@ end
|
|
|
13
14
|
task :docs do
|
|
14
15
|
system("rdoc rdoc lib")
|
|
15
16
|
end
|
|
17
|
+
|
|
18
|
+
desc "Build RPM and dependencies (intended for IBEST internal use)"
|
|
19
|
+
task :rpm_deps do
|
|
20
|
+
require 'erb'
|
|
21
|
+
require 'yaml'
|
|
22
|
+
|
|
23
|
+
build_rpms({
|
|
24
|
+
'bookie_accounting' => {
|
|
25
|
+
:is_app => true,
|
|
26
|
+
:has_binaries => true,
|
|
27
|
+
:gem_file => "bookie_accounting-#{Bookie::VERSION}.gem",
|
|
28
|
+
:license => 'MIT',
|
|
29
|
+
:skip_deps => Set.new(['sqlite3']),
|
|
30
|
+
},
|
|
31
|
+
'bundler' => {
|
|
32
|
+
|
|
33
|
+
},
|
|
34
|
+
'spreadsheet' => {
|
|
35
|
+
:has_binaries => true,
|
|
36
|
+
:license => 'GPL3',
|
|
37
|
+
},
|
|
38
|
+
'ruby-ole' => {
|
|
39
|
+
:license => 'MIT',
|
|
40
|
+
},
|
|
41
|
+
'pacct' => {
|
|
42
|
+
:license => 'MIT',
|
|
43
|
+
},
|
|
44
|
+
})
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def find_gem_file(gem_name)
|
|
48
|
+
Dir.glob("#{gem_name}-*.gem")[0]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def build_rpms(modules_to_build)
|
|
52
|
+
this_dir = Dir.pwd
|
|
53
|
+
pkg_dir = File.join(this_dir, 'pkg')
|
|
54
|
+
|
|
55
|
+
home_dir = Etc.getpwuid.dir
|
|
56
|
+
|
|
57
|
+
rpmbuild_dir = File.join(home_dir, 'rpmbuild')
|
|
58
|
+
spec_dir = File.join(rpmbuild_dir, 'SPECS')
|
|
59
|
+
src_dir = File.join(rpmbuild_dir, 'SOURCES')
|
|
60
|
+
rpm_dir = File.join(rpmbuild_dir, 'RPMS')
|
|
61
|
+
|
|
62
|
+
[spec_dir, src_dir].each do |dir|
|
|
63
|
+
FileUtils.mkdir_p(dir)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
#To do: packaging for Ruby versions other than 1.8
|
|
67
|
+
template = ERB.new(File.read('rpm/spec_template.erb'))
|
|
68
|
+
lockfile = Bundler::LockfileParser.new(Bundler.read_file("Gemfile.lock"))
|
|
69
|
+
lockfile.specs.each do |spec|
|
|
70
|
+
extra_data = modules_to_build[spec.name]
|
|
71
|
+
next unless extra_data
|
|
72
|
+
|
|
73
|
+
gem_name = spec.name
|
|
74
|
+
version = spec.version
|
|
75
|
+
|
|
76
|
+
puts gem_name
|
|
77
|
+
|
|
78
|
+
#Has the RPM already been built?
|
|
79
|
+
next if Dir.glob(File.join(rpm_dir, "*/(rubygem-)?#{gem_name}-#{version}-*.rpm")).length > 0
|
|
80
|
+
|
|
81
|
+
ruby_version = RUBY_VERSION.split('.')[0 .. 1].join('.')
|
|
82
|
+
|
|
83
|
+
#Get the gem.
|
|
84
|
+
Dir.chdir(src_dir)
|
|
85
|
+
gem_file = extra_data[:gem_file]
|
|
86
|
+
if gem_file
|
|
87
|
+
FileUtils.cp(File.join(pkg_dir, gem_file), '.')
|
|
88
|
+
else
|
|
89
|
+
gem_file = find_gem_file(gem_name)
|
|
90
|
+
#To do: handle old versions of gems existing in src_dir
|
|
91
|
+
system("gem fetch #{gem_name} -v #{version}") unless gem_file && File.exists?(gem_file)
|
|
92
|
+
gem_file ||= find_gem_file(gem_name)
|
|
93
|
+
end
|
|
94
|
+
fail "Unable to find gem file for #{gem_name}" unless gem_file
|
|
95
|
+
gem = File.basename(gem_file, '.gem')
|
|
96
|
+
s = YAML.load(`gem spec #{gem_file}`)
|
|
97
|
+
|
|
98
|
+
#Build the RPM spec.
|
|
99
|
+
Dir.chdir(spec_dir)
|
|
100
|
+
if s.licenses.length > 0
|
|
101
|
+
license = s.licenses.join(' AND ')
|
|
102
|
+
else
|
|
103
|
+
license = extra_data[:license]
|
|
104
|
+
end
|
|
105
|
+
url = s.homepage
|
|
106
|
+
summary = s.summary
|
|
107
|
+
description = s.description
|
|
108
|
+
requires = []
|
|
109
|
+
build_requires = []
|
|
110
|
+
s.dependencies.each do |dep|
|
|
111
|
+
case dep.type
|
|
112
|
+
when :development
|
|
113
|
+
build_requires.push(dep)
|
|
114
|
+
else
|
|
115
|
+
requires.push(dep)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
requires.map!{ |r| "rubygem-#{r.name}" }
|
|
119
|
+
|
|
120
|
+
is_app = extra_data[:is_app]
|
|
121
|
+
has_binaries = extra_data[:has_binaries]
|
|
122
|
+
|
|
123
|
+
if is_app
|
|
124
|
+
spec_filename = "#{gem_name}.spec"
|
|
125
|
+
else
|
|
126
|
+
spec_filename = "rubygem-#{gem_name}.spec"
|
|
127
|
+
end
|
|
128
|
+
spec_filename = File.join(spec_dir, spec_filename)
|
|
129
|
+
|
|
130
|
+
File.open(spec_filename, "w") do |file|
|
|
131
|
+
template_filename = File.join(this_dir, "rpm/#{gem_name}.spec.erb")
|
|
132
|
+
if File.exists?(template_filename)
|
|
133
|
+
file.write(ERB.new(File.read(template_filename)).result(binding))
|
|
134
|
+
else
|
|
135
|
+
file.write(template.result(binding))
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
msg = `rpmbuild -bb #{spec_filename}`
|
|
140
|
+
unless $?.success?
|
|
141
|
+
puts msg
|
|
142
|
+
exit 1
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
Dir.chdir(this_dir)
|
|
147
|
+
end
|
|
148
|
+
|
data/bin/bookie-data
CHANGED
|
@@ -8,6 +8,7 @@ require 'optparse'
|
|
|
8
8
|
require 'bookie/formatter'
|
|
9
9
|
|
|
10
10
|
jobs = Bookie::Database::Job
|
|
11
|
+
summaries = Bookie::Database::JobSummary
|
|
11
12
|
systems = Bookie::Database::System
|
|
12
13
|
|
|
13
14
|
config_filename = '/etc/bookie/config.json'
|
|
@@ -37,7 +38,7 @@ opts = OptionParser.new do |opts|
|
|
|
37
38
|
opts.banner = "Usage: bookie-data [options]"
|
|
38
39
|
|
|
39
40
|
opts.on('-c', '--config FILE', String, "use the given configuration file") do |file|
|
|
40
|
-
#This is just here so it shows up in the message.
|
|
41
|
+
#This is just here for validation and so it shows up in the usage message.
|
|
41
42
|
end
|
|
42
43
|
|
|
43
44
|
opts.on('-d', '--details', "include full details") do
|
|
@@ -46,14 +47,22 @@ opts = OptionParser.new do |opts|
|
|
|
46
47
|
|
|
47
48
|
opts.on('-u', '--user NAME', "filter by username") do |name|
|
|
48
49
|
jobs = jobs.by_user_name(name)
|
|
50
|
+
summaries = summaries.by_user_name(name)
|
|
49
51
|
end
|
|
50
52
|
|
|
51
|
-
opts.on('-g', '--group NAME' "filter by group") do |name|
|
|
53
|
+
opts.on('-g', '--group NAME', "filter by group") do |name|
|
|
52
54
|
jobs = jobs.by_group_name(name)
|
|
55
|
+
summaries = summaries.by_group_name(name)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
opts.on('-m', '--command', "filter by command") do |cmd|
|
|
59
|
+
jobs = jobs.by_command_name(cmd)
|
|
60
|
+
summaries = summaries.by_command_name(cmd)
|
|
53
61
|
end
|
|
54
62
|
|
|
55
63
|
opts.on('-s', '--system HOSTNAME', "filter by system") do |hostname|
|
|
56
64
|
jobs = jobs.by_system_name(hostname)
|
|
65
|
+
summaries = summaries.by_system_name(hostname)
|
|
57
66
|
systems = systems.by_name(hostname)
|
|
58
67
|
end
|
|
59
68
|
|
|
@@ -64,6 +73,7 @@ opts = OptionParser.new do |opts|
|
|
|
64
73
|
exit 1
|
|
65
74
|
end
|
|
66
75
|
jobs = jobs.by_system_type(t)
|
|
76
|
+
summaries = summaries.by_system_type(t)
|
|
67
77
|
systems = systems.by_system_type(t)
|
|
68
78
|
end
|
|
69
79
|
|
|
@@ -96,7 +106,7 @@ end
|
|
|
96
106
|
|
|
97
107
|
formatter = Bookie::Formatter.new(output_type, filename)
|
|
98
108
|
|
|
99
|
-
jobs_summary, systems_summary = formatter.print_summary(jobs, systems, t_min, t_max)
|
|
109
|
+
jobs_summary, systems_summary = formatter.print_summary(jobs, summaries, systems, t_min, t_max)
|
|
100
110
|
jobs = jobs.by_time_range_inclusive(t_min, t_max) if t_min
|
|
101
|
-
formatter.print_jobs(
|
|
111
|
+
formatter.print_jobs(jobs.all) if include_details
|
|
102
112
|
formatter.flush
|
data/bookie_accounting.gemspec
CHANGED
|
@@ -4,6 +4,7 @@ require File.expand_path('../lib/bookie/version', __FILE__)
|
|
|
4
4
|
Gem::Specification.new do |gem|
|
|
5
5
|
gem.authors = ["Ben Merritt"]
|
|
6
6
|
gem.email = ["blm768@gmail.com"]
|
|
7
|
+
gem.license = "MIT"
|
|
7
8
|
gem.description = %q{A simple system to record and query process accounting records}
|
|
8
9
|
gem.summary = %q{A simple system to record and query process accounting records}
|
|
9
10
|
gem.homepage = "https://github.com/blm768/bookie/"
|
|
@@ -15,11 +16,10 @@ Gem::Specification.new do |gem|
|
|
|
15
16
|
gem.require_paths = ["lib"]
|
|
16
17
|
gem.version = Bookie::VERSION
|
|
17
18
|
|
|
18
|
-
gem.add_dependency('json')
|
|
19
19
|
gem.add_dependency('activerecord')
|
|
20
20
|
#For some reason, this is needed for Ruby 1.8.7 using RVM on CentOS.
|
|
21
21
|
#To do: remove when no longer needed
|
|
22
|
-
gem.add_dependency('mysql2')
|
|
22
|
+
#gem.add_dependency('mysql2')
|
|
23
23
|
gem.add_dependency('pacct')
|
|
24
24
|
gem.add_dependency('spreadsheet')
|
|
25
25
|
gem.add_development_dependency('mocha')
|
data/lib/bookie/database.rb
CHANGED
|
@@ -58,11 +58,19 @@ module Bookie
|
|
|
58
58
|
return start_time + wall_time
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
+
def self.by_user(user)
|
|
62
|
+
where('jobs.user_id = ?', user.id)
|
|
63
|
+
end
|
|
64
|
+
|
|
61
65
|
##
|
|
62
66
|
#Filters by user name
|
|
63
67
|
def self.by_user_name(user_name)
|
|
64
68
|
joins(:user).where('users.name = ?', user_name)
|
|
65
69
|
end
|
|
70
|
+
|
|
71
|
+
def self.by_system(system)
|
|
72
|
+
where('jobs.system_id = ?', system.id)
|
|
73
|
+
end
|
|
66
74
|
|
|
67
75
|
##
|
|
68
76
|
#Filters by system name
|
|
@@ -92,21 +100,20 @@ module Bookie
|
|
|
92
100
|
|
|
93
101
|
##
|
|
94
102
|
#Filters by a range of start times
|
|
95
|
-
def self.by_start_time_range(
|
|
96
|
-
where('? <= jobs.start_time AND jobs.start_time < ?',
|
|
103
|
+
def self.by_start_time_range(time_range)
|
|
104
|
+
where('? <= jobs.start_time AND jobs.start_time < ?', time_range.first, time_range.last)
|
|
97
105
|
end
|
|
98
106
|
|
|
99
107
|
##
|
|
100
108
|
#Filters by a range of end times
|
|
101
|
-
def self.by_end_time_range(
|
|
102
|
-
where('? <= jobs.end_time AND jobs.end_time < ?',
|
|
109
|
+
def self.by_end_time_range(time_range)
|
|
110
|
+
where('? <= jobs.end_time AND jobs.end_time < ?', time_range.first, time_range.last)
|
|
103
111
|
end
|
|
104
112
|
|
|
105
113
|
##
|
|
106
114
|
#Finds all jobs whose running intervals overlap the given time range
|
|
107
|
-
def self.by_time_range_inclusive(
|
|
108
|
-
|
|
109
|
-
where('jobs.start_time < ? AND jobs.end_time > ?', max_time, min_time)
|
|
115
|
+
def self.by_time_range_inclusive(time_range)
|
|
116
|
+
where('? <= jobs.end_time AND jobs.start_time < ?', time_range.first, time_range.last)
|
|
110
117
|
end
|
|
111
118
|
|
|
112
119
|
##
|
|
@@ -114,20 +121,16 @@ module Bookie
|
|
|
114
121
|
#
|
|
115
122
|
#Returns a hash with the following fields:
|
|
116
123
|
#- <tt>:jobs</tt>: an array of all jobs in the interval
|
|
117
|
-
#- <tt>:wall_time</tt>: the sum of all the jobs' wall times
|
|
118
124
|
#- <tt>:cpu_time</tt>: the total CPU time used
|
|
119
125
|
#- <tt>:memory_time</tt>: the sum of memory * wall_time for all jobs in the interval
|
|
120
|
-
#- <tt>:successful</tt>: the
|
|
126
|
+
#- <tt>:successful</tt>: the number of jobs that have completed successfully
|
|
121
127
|
#
|
|
122
128
|
#This method should probably not be used with other queries that filter by start/end time.
|
|
123
|
-
def self.summary(
|
|
129
|
+
def self.summary(time_range = nil)
|
|
130
|
+
time_range = time_range.normalized if time_range
|
|
124
131
|
jobs = self
|
|
125
|
-
if
|
|
126
|
-
raise ArgumentError.new('Max time must be specified with min time') unless max_time
|
|
127
|
-
jobs = jobs.by_time_range_inclusive(min_time, max_time)
|
|
128
|
-
end
|
|
132
|
+
jobs = jobs.by_time_range_inclusive(time_range) if time_range
|
|
129
133
|
jobs = jobs.where('jobs.cpu_time > 0').all_with_relations
|
|
130
|
-
wall_time = 0
|
|
131
134
|
cpu_time = 0
|
|
132
135
|
successful_jobs = 0
|
|
133
136
|
memory_time = 0
|
|
@@ -138,28 +141,27 @@ module Bookie
|
|
|
138
141
|
jobs.each do |job|
|
|
139
142
|
job_start_time = job.start_time
|
|
140
143
|
job_end_time = job.end_time
|
|
141
|
-
if
|
|
142
|
-
job_start_time = [job_start_time,
|
|
143
|
-
job_end_time = [job_end_time,
|
|
144
|
+
if time_range
|
|
145
|
+
job_start_time = [job_start_time, time_range.first].max
|
|
146
|
+
job_end_time = [job_end_time, time_range.last].min
|
|
144
147
|
end
|
|
145
148
|
clipped_wall_time = job_end_time.to_i - job_start_time.to_i
|
|
146
|
-
wall_time += clipped_wall_time
|
|
147
149
|
if job.wall_time != 0
|
|
148
150
|
cpu_time += job.cpu_time * clipped_wall_time / job.wall_time
|
|
149
151
|
#To consider: what should I do about jobs that only report a max memory value?
|
|
150
152
|
memory_time += job.memory * clipped_wall_time
|
|
151
153
|
end
|
|
152
|
-
|
|
154
|
+
#Only count the job as successful if it's actually finished by the end of the summary.
|
|
155
|
+
if job.exit_code == 0 && (!time_range || job.end_time < time_range.end) then
|
|
156
|
+
successful_jobs += 1
|
|
157
|
+
end
|
|
153
158
|
end
|
|
154
159
|
|
|
155
160
|
return {
|
|
156
161
|
:jobs => jobs,
|
|
157
|
-
#To consider: is this field even useful? It's really in job-seconds, not just seconds.
|
|
158
|
-
#What about one in just seconds (that considers gaps in activity)?
|
|
159
|
-
:wall_time => wall_time,
|
|
160
162
|
:cpu_time => cpu_time,
|
|
161
163
|
:memory_time => memory_time,
|
|
162
|
-
:successful =>
|
|
164
|
+
:successful => successful_jobs,
|
|
163
165
|
}
|
|
164
166
|
end
|
|
165
167
|
|
|
@@ -219,16 +221,183 @@ module Bookie
|
|
|
219
221
|
|
|
220
222
|
validates_presence_of :user, :system, :cpu_time,
|
|
221
223
|
:start_time, :wall_time, :memory, :exit_code
|
|
224
|
+
|
|
225
|
+
validates_each :command_name do |record, attr, value|
|
|
226
|
+
record.errors.add(attr, 'must not be nil') if value == nil
|
|
227
|
+
end
|
|
222
228
|
|
|
223
229
|
validates_each :cpu_time, :wall_time, :memory do |record, attr, value|
|
|
224
230
|
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
|
225
231
|
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
class JobSummary < ActiveRecord::Base
|
|
235
|
+
self.table_name = :job_summaries
|
|
236
|
+
|
|
237
|
+
belongs_to :user
|
|
238
|
+
belongs_to :system
|
|
239
|
+
|
|
240
|
+
def self.by_date(date)
|
|
241
|
+
where('job_summaries.date = ?', date)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def self.by_user(user)
|
|
245
|
+
where('job_summaries.user_id = ?', user.id)
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def self.by_user_name(name)
|
|
249
|
+
joins(:user).where('users.name = ?', name)
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def self.by_group(group)
|
|
253
|
+
joins(:user).where('users.group_id = ?', group.id)
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def self.by_group_name(name)
|
|
257
|
+
group = Group.find_by_name(name)
|
|
258
|
+
return by_group(group) if group
|
|
259
|
+
limit(0)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def self.by_system(system)
|
|
263
|
+
where('job_summaries.system_id = ?', system.id)
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def self.by_system_name(name)
|
|
267
|
+
joins(:system).where('systems.name = ?', name)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def self.by_system_type(type)
|
|
271
|
+
joins(:system).where('systems.system_type_id = ?', type.id)
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def self.by_command_name(cmd)
|
|
275
|
+
where('job_summaries.command_name = ?', cmd)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def self.find_or_new(date, user_id, system_id, command_name)
|
|
279
|
+
str = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).to_sql
|
|
280
|
+
summary = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).first
|
|
281
|
+
summary ||= new(
|
|
282
|
+
:date => date,
|
|
283
|
+
:user_id => user_id,
|
|
284
|
+
:system_id => system_id,
|
|
285
|
+
:command_name => command_name
|
|
286
|
+
)
|
|
287
|
+
summary
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def self.summarize(date)
|
|
291
|
+
jobs = Job
|
|
292
|
+
unscoped = self.unscoped
|
|
293
|
+
time_range = date.to_time ... (date + 1).to_time
|
|
294
|
+
day_jobs = jobs.by_time_range_inclusive(time_range)
|
|
295
|
+
value_sets = day_jobs.select('user_id, system_id, command_name').uniq
|
|
296
|
+
value_sets.each do |set|
|
|
297
|
+
summary_jobs = jobs.where(:user_id => set.user_id).where(:system_id => set.system_id).by_command_name(set.command_name)
|
|
298
|
+
summary = summary_jobs.summary(time_range)
|
|
299
|
+
Lock[:job_summaries].synchronize do
|
|
300
|
+
sum = unscoped.find_or_new(date, set.user_id, set.system_id, set.command_name)
|
|
301
|
+
#To consider: do we even need this field? (Time-range summaries don't use it because of overlap issues.)
|
|
302
|
+
sum.num_jobs = summary[:jobs].length
|
|
303
|
+
sum.cpu_time = summary[:cpu_time]
|
|
304
|
+
sum.memory_time = summary[:memory_time]
|
|
305
|
+
sum.successful = summary[:successful]
|
|
306
|
+
sum.save!
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def self.summary(opts = {})
|
|
312
|
+
jobs = opts[:jobs] || Job
|
|
313
|
+
range = opts[:range]
|
|
314
|
+
unless range
|
|
315
|
+
end_time = nil
|
|
316
|
+
if System.active_systems.any?
|
|
317
|
+
end_time = Time.now
|
|
318
|
+
else
|
|
319
|
+
last_ended_system = System.order('end_time DESC').first
|
|
320
|
+
end_time = last_ended_system.end_time if last_ended_system
|
|
321
|
+
end
|
|
322
|
+
if end_time
|
|
323
|
+
first_started_system = System.order(:start_time).first
|
|
324
|
+
range = first_started_system.start_time ... end_time
|
|
325
|
+
else
|
|
326
|
+
range = Date.new ... Date.new
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
range = range.normalized
|
|
330
|
+
|
|
331
|
+
num_jobs = 0
|
|
332
|
+
cpu_time = 0
|
|
333
|
+
memory_time = 0
|
|
334
|
+
successful = 0
|
|
335
|
+
|
|
336
|
+
#Could there be partial days at the beginning/end?
|
|
337
|
+
date_range = range
|
|
338
|
+
unless range.begin.kind_of?(Date) && range.end.kind_of?(Date)
|
|
339
|
+
date_begin = range.begin.to_date
|
|
340
|
+
unless date_begin.to_time == range.begin
|
|
341
|
+
date_begin += 1
|
|
342
|
+
time_before_max = [date_begin.to_time, range.end].min
|
|
343
|
+
time_before_min = range.begin
|
|
344
|
+
summary = jobs.summary(time_before_min ... time_before_max)
|
|
345
|
+
cpu_time += summary[:cpu_time]
|
|
346
|
+
memory_time += summary[:memory_time]
|
|
347
|
+
successful += summary[:successful]
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
date_end = range.end.to_date
|
|
351
|
+
time_after_min = date_end.to_time
|
|
352
|
+
unless time_after_min <= range.begin
|
|
353
|
+
time_after_max = range.end
|
|
354
|
+
time_after_range = Range.new(time_after_min, time_after_max, range.exclude_end?)
|
|
355
|
+
unless time_after_range.empty?
|
|
356
|
+
summary = jobs.summary(time_after_range)
|
|
357
|
+
cpu_time += summary[:cpu_time]
|
|
358
|
+
memory_time += summary[:memory_time]
|
|
359
|
+
successful += summary[:successful]
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
date_range = date_begin ... date_end
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
unscoped = self.unscoped
|
|
367
|
+
date = date_range.begin
|
|
368
|
+
while date_range.cover?(date) do
|
|
369
|
+
#To do: what if there aren't any summaries to be made? Will we just run summarize() each time?
|
|
370
|
+
summarize(date) if unscoped.by_date(date).empty?
|
|
371
|
+
summaries = by_date(date)
|
|
372
|
+
summaries.all.each do |summary|
|
|
373
|
+
cpu_time += summary.cpu_time
|
|
374
|
+
memory_time += summary.memory_time
|
|
375
|
+
successful += summary.successful
|
|
376
|
+
end
|
|
377
|
+
date += 1
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
time_range = Range.new(range.begin.to_time, range.end.to_time, range.exclude_end?)
|
|
381
|
+
jobs = jobs.by_time_range_inclusive(time_range)
|
|
382
|
+
num_jobs = (range && range.empty?) ? 0 : jobs.count
|
|
383
|
+
|
|
384
|
+
{
|
|
385
|
+
:num_jobs => num_jobs,
|
|
386
|
+
:cpu_time => cpu_time,
|
|
387
|
+
:memory_time => memory_time,
|
|
388
|
+
:successful => successful,
|
|
389
|
+
}
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
validates_presence_of :user_id, :system_id, :date, :num_jobs, :cpu_time, :memory_time, :successful
|
|
226
393
|
|
|
227
|
-
validates_each :
|
|
228
|
-
|
|
229
|
-
record.errors.add(attr, 'must be a time object') unless value.is_a?(Time)
|
|
394
|
+
validates_each :command_name do |record, attr, value|
|
|
395
|
+
record.errors.add(attr, 'must not be nil') if value == nil
|
|
230
396
|
end
|
|
231
397
|
|
|
398
|
+
validates_each :num_jobs, :cpu_time, :memory_time, :successful do |record, attr, value|
|
|
399
|
+
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
|
400
|
+
end
|
|
232
401
|
end
|
|
233
402
|
|
|
234
403
|
##
|
|
@@ -261,6 +430,10 @@ module Bookie
|
|
|
261
430
|
class User < ActiveRecord::Base
|
|
262
431
|
belongs_to :group
|
|
263
432
|
|
|
433
|
+
def self.by_name(name)
|
|
434
|
+
where('users.name = ?', name)
|
|
435
|
+
end
|
|
436
|
+
|
|
264
437
|
##
|
|
265
438
|
#Finds a user by name and group, creating it if it doesn't exist
|
|
266
439
|
#
|
|
@@ -276,7 +449,8 @@ module Bookie
|
|
|
276
449
|
user = Bookie::Database::User.find_by_name_and_group_id(name, group.id)
|
|
277
450
|
user ||= Bookie::Database::User.create!(
|
|
278
451
|
:name => name,
|
|
279
|
-
:group => group
|
|
452
|
+
:group => group
|
|
453
|
+
)
|
|
280
454
|
end
|
|
281
455
|
known_users[[name, group]] = user if known_users
|
|
282
456
|
end
|
|
@@ -315,28 +489,28 @@ module Bookie
|
|
|
315
489
|
end
|
|
316
490
|
|
|
317
491
|
##
|
|
318
|
-
#Finds the
|
|
319
|
-
#
|
|
320
|
-
#<tt>values</tt> should contain a list of fields, including the name, in the format that would normally be passed to System.create!.
|
|
492
|
+
#Finds the current system for a given sender and time
|
|
321
493
|
#
|
|
322
494
|
#This method also checks that this system's specifications are the same as those in the database and raises an error if they are different.
|
|
323
495
|
#
|
|
324
496
|
#This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
|
|
325
|
-
|
|
497
|
+
#
|
|
498
|
+
def self.find_current(sender, time = nil)
|
|
499
|
+
time ||= Time.now
|
|
500
|
+
config = sender.config
|
|
326
501
|
system = nil
|
|
327
|
-
name =
|
|
502
|
+
name = config.hostname
|
|
328
503
|
Lock[:systems].synchronize do
|
|
329
|
-
system =
|
|
504
|
+
system = by_name(config.hostname).where('systems.start_time <= :time AND (:time <= systems.end_time OR systems.end_time IS NULL)', :time => time).first
|
|
330
505
|
if system
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
end
|
|
506
|
+
mismatch = !(system.cores == config.cores && system.memory == config.memory)
|
|
507
|
+
mismatch ||= sender.system_type != system.system_type
|
|
508
|
+
if mismatch
|
|
509
|
+
raise SystemConflictError.new("The specifications on record for '#{name}' do not match this system's specifications.
|
|
510
|
+
Please make sure that all previous systems with this hostname have been marked as decommissioned.")
|
|
337
511
|
end
|
|
338
512
|
else
|
|
339
|
-
raise "There is no
|
|
513
|
+
raise "There is no system with hostname '#{config.hostname}' in the database at #{time}."
|
|
340
514
|
end
|
|
341
515
|
end
|
|
342
516
|
system
|
|
@@ -349,31 +523,35 @@ module Bookie
|
|
|
349
523
|
#- <tt>:avail_cpu_time</tt>: the total CPU time available for the interval
|
|
350
524
|
#- <tt>:avail_memory_time</tt>: the total amount of memory-time available (in kilobyte-seconds)
|
|
351
525
|
#- <tt>:avail_memory_avg</tt>: the average amount of memory available (in kilobytes)
|
|
352
|
-
|
|
526
|
+
#
|
|
527
|
+
#To consider: include the start/end times for the summary (especially if they aren't provided as arguments)?
|
|
528
|
+
#
|
|
529
|
+
#To do: make this and other summaries operate differently on inclusive and exclusive ranges.
|
|
530
|
+
#(Current behavior is as if the range were always exclusive.)
|
|
531
|
+
def self.summary(time_range = nil)
|
|
353
532
|
current_time = Time.now
|
|
354
533
|
#Sums that are actually returned
|
|
355
534
|
avail_cpu_time = 0
|
|
356
535
|
avail_memory_time = 0
|
|
357
536
|
#Find all the systems within the time range.
|
|
358
537
|
systems = System
|
|
359
|
-
if
|
|
360
|
-
|
|
361
|
-
raise ArgumentError.new('Max time must be greater than or equal to min time') if max_time < min_time
|
|
538
|
+
if time_range
|
|
539
|
+
time_range = time_range.normalized
|
|
362
540
|
#To consider: optimize as union of queries?
|
|
363
541
|
systems = systems.where(
|
|
364
542
|
'systems.start_time < ? AND (systems.end_time IS NULL OR systems.end_time > ?)',
|
|
365
|
-
|
|
366
|
-
|
|
543
|
+
time_range.last,
|
|
544
|
+
time_range.first)
|
|
367
545
|
end
|
|
368
546
|
|
|
369
547
|
systems.all.each do |system|
|
|
370
548
|
system_start_time = system.start_time
|
|
371
549
|
system_end_time = system.end_time
|
|
372
550
|
#Is there a time range constraint?
|
|
373
|
-
if
|
|
374
|
-
system_start_time = [system_start_time,
|
|
375
|
-
system_end_time = [system_end_time,
|
|
376
|
-
system_end_time ||=
|
|
551
|
+
if time_range
|
|
552
|
+
system_start_time = [system_start_time, time_range.first].max
|
|
553
|
+
system_end_time = [system_end_time, time_range.last].min if system.end_time
|
|
554
|
+
system_end_time ||= time_range.last
|
|
377
555
|
else
|
|
378
556
|
system_end_time ||= current_time
|
|
379
557
|
end
|
|
@@ -383,8 +561,8 @@ module Bookie
|
|
|
383
561
|
end
|
|
384
562
|
|
|
385
563
|
wall_time_range = 0
|
|
386
|
-
if
|
|
387
|
-
wall_time_range =
|
|
564
|
+
if time_range
|
|
565
|
+
wall_time_range = time_range.last - time_range.first
|
|
388
566
|
else
|
|
389
567
|
first_started_system = systems.order(:start_time).first
|
|
390
568
|
if first_started_system
|
|
@@ -422,11 +600,6 @@ module Bookie
|
|
|
422
600
|
record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
|
|
423
601
|
end
|
|
424
602
|
|
|
425
|
-
validates_each :start_time do |record, attr, value|
|
|
426
|
-
value = value.to_time if value.respond_to?(:to_time)
|
|
427
|
-
record.errors.add(attr, 'must be a time object') unless value.is_a?(Time)
|
|
428
|
-
end
|
|
429
|
-
|
|
430
603
|
validates_each :end_time do |record, attr, value|
|
|
431
604
|
record.errors.add(attr, 'must be at or after start time') if value && value < record.start_time
|
|
432
605
|
end
|
|
@@ -508,7 +681,7 @@ module Bookie
|
|
|
508
681
|
|
|
509
682
|
##
|
|
510
683
|
#Database migrations
|
|
511
|
-
module Migration
|
|
684
|
+
module Migration
|
|
512
685
|
class CreateUsers < ActiveRecord::Migration
|
|
513
686
|
def up
|
|
514
687
|
create_table :users do |t|
|
|
@@ -585,7 +758,7 @@ module Bookie
|
|
|
585
758
|
create_table :jobs do |t|
|
|
586
759
|
t.references :user, :null => false
|
|
587
760
|
t.references :system, :null => false
|
|
588
|
-
t.string :command_name, :limit => 24
|
|
761
|
+
t.string :command_name, :limit => 24, :null => false
|
|
589
762
|
t.datetime :start_time, :null => false
|
|
590
763
|
t.datetime :end_time, :null => false
|
|
591
764
|
t.integer :wall_time, :null => false
|
|
@@ -607,6 +780,31 @@ module Bookie
|
|
|
607
780
|
end
|
|
608
781
|
end
|
|
609
782
|
|
|
783
|
+
class CreateJobSummaries < ActiveRecord::Migration
|
|
784
|
+
def up
|
|
785
|
+
create_table :job_summaries do |t|
|
|
786
|
+
t.references :user, :null => false
|
|
787
|
+
t.references :system, :null => false
|
|
788
|
+
t.date :date, :null => false
|
|
789
|
+
t.string :command_name, :null => false
|
|
790
|
+
t.integer :num_jobs, :null => false
|
|
791
|
+
t.integer :cpu_time, :null => false
|
|
792
|
+
t.integer :memory_time, :null => false
|
|
793
|
+
t.integer :successful, :null => false
|
|
794
|
+
end
|
|
795
|
+
change_table :job_summaries do |t|
|
|
796
|
+
#To consider: reorder for optimum efficiency?
|
|
797
|
+
t.index [:date, :user_id, :system_id, :command_name], :unique => true, :name => 'identity'
|
|
798
|
+
t.index :command_name
|
|
799
|
+
t.index :date
|
|
800
|
+
end
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
def down
|
|
804
|
+
drop_table :job_summaries
|
|
805
|
+
end
|
|
806
|
+
end
|
|
807
|
+
|
|
610
808
|
class CreateLocks < ActiveRecord::Migration
|
|
611
809
|
def up
|
|
612
810
|
create_table :locks do |t|
|
|
@@ -616,7 +814,7 @@ module Bookie
|
|
|
616
814
|
t.index :name, :unique => true
|
|
617
815
|
end
|
|
618
816
|
|
|
619
|
-
['users', 'groups', 'systems', 'system_types'].each do |name|
|
|
817
|
+
['users', 'groups', 'systems', 'system_types', 'job_summaries'].each do |name|
|
|
620
818
|
Lock.create!(:name => name)
|
|
621
819
|
end
|
|
622
820
|
end
|
|
@@ -630,11 +828,13 @@ module Bookie
|
|
|
630
828
|
##
|
|
631
829
|
#Brings up all migrations
|
|
632
830
|
def up
|
|
831
|
+
ActiveRecord::Migration.verbose = false
|
|
633
832
|
CreateUsers.new.up
|
|
634
833
|
CreateGroups.new.up
|
|
635
834
|
CreateSystems.new.up
|
|
636
835
|
CreateSystemTypes.new.up
|
|
637
836
|
CreateJobs.new.up
|
|
837
|
+
CreateJobSummaries.new.up
|
|
638
838
|
CreateLocks.new.up
|
|
639
839
|
end
|
|
640
840
|
|
|
@@ -643,14 +843,77 @@ module Bookie
|
|
|
643
843
|
#
|
|
644
844
|
#Warning: this will destroy all data!
|
|
645
845
|
def down
|
|
846
|
+
ActiveRecord::Migration.verbose = false
|
|
646
847
|
CreateUsers.new.down
|
|
647
848
|
CreateGroups.new.down
|
|
648
849
|
CreateSystems.new.down
|
|
649
850
|
CreateSystemTypes.new.down
|
|
650
851
|
CreateJobs.new.down
|
|
852
|
+
CreateJobSummaries.new.down
|
|
651
853
|
CreateLocks.new.down
|
|
652
854
|
end
|
|
653
855
|
end
|
|
654
856
|
end
|
|
655
857
|
end
|
|
656
858
|
end
|
|
859
|
+
|
|
860
|
+
##
|
|
861
|
+
#Reopened to add some useful methods
|
|
862
|
+
class Range
|
|
863
|
+
##
|
|
864
|
+
#If end < begin, returns an empty range (begin ... begin)
|
|
865
|
+
#Otherwise, returns the original range
|
|
866
|
+
def normalized
|
|
867
|
+
return self.begin ... self.begin if self.end < self.begin
|
|
868
|
+
self
|
|
869
|
+
end
|
|
870
|
+
|
|
871
|
+
##
|
|
872
|
+
#Returns the empty status of the range
|
|
873
|
+
#
|
|
874
|
+
#A range is empty if end < begin or if begin == end and exclude_end? is true.
|
|
875
|
+
def empty?
|
|
876
|
+
(self.end < self.begin) || (exclude_end? && (self.begin == self.end))
|
|
877
|
+
end
|
|
878
|
+
|
|
879
|
+
#This code probably works, but we're not using it anywhere.
|
|
880
|
+
# def intersection(other)
|
|
881
|
+
# self_n = self.normalized
|
|
882
|
+
# other = other.normalized
|
|
883
|
+
#
|
|
884
|
+
# new_begin, new_end, exclude_end = nil
|
|
885
|
+
#
|
|
886
|
+
# if self_n.cover?(other.begin)
|
|
887
|
+
# new_first = other.begin
|
|
888
|
+
# elsif other.cover?(self_n.begin)
|
|
889
|
+
# new_first = self_n.begin
|
|
890
|
+
# end
|
|
891
|
+
#
|
|
892
|
+
# return self_n.begin ... self_n.begin unless new_first
|
|
893
|
+
#
|
|
894
|
+
# if self_n.cover?(other.end)
|
|
895
|
+
# unless other.exclude_end? && other.end == self_n.begin
|
|
896
|
+
# new_end = other.end
|
|
897
|
+
# exclude_end = other.exclude_end?
|
|
898
|
+
# end
|
|
899
|
+
# elsif other.cover?(self_n.end)
|
|
900
|
+
# unless self_n.exclude_end? && self_n.end == other.begin
|
|
901
|
+
# new_end = self_n.end
|
|
902
|
+
# exclude_end = self_n.exclude_end?
|
|
903
|
+
# end
|
|
904
|
+
# end
|
|
905
|
+
#
|
|
906
|
+
# #If we still haven't found new_end, try one more case:
|
|
907
|
+
# unless new_end
|
|
908
|
+
# if self_n.end == other.end
|
|
909
|
+
# #We'll only get here if both ranges exclude their ends and have the same end.
|
|
910
|
+
# new_end = self_n.end
|
|
911
|
+
# exclude_end = true
|
|
912
|
+
# end
|
|
913
|
+
# end
|
|
914
|
+
#
|
|
915
|
+
# return self_n.begin ... self_n.begin unless new_end
|
|
916
|
+
#
|
|
917
|
+
# Range.new(new_begin, new_end, exclude_end)
|
|
918
|
+
# end
|
|
919
|
+
end
|