bookie_accounting 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +19 -0
- data/Gemfile +3 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +15 -0
- data/bin/bookie-create-tables +52 -0
- data/bin/bookie-data +102 -0
- data/bin/bookie-send +110 -0
- data/bookie_accounting.gemspec +28 -0
- data/lib/bookie.rb +11 -0
- data/lib/bookie/config.rb +101 -0
- data/lib/bookie/database.rb +656 -0
- data/lib/bookie/formatter.rb +149 -0
- data/lib/bookie/formatters/comma_dump.rb +24 -0
- data/lib/bookie/formatters/spreadsheet.rb +45 -0
- data/lib/bookie/formatters/stdout.rb +32 -0
- data/lib/bookie/sender.rb +108 -0
- data/lib/bookie/senders/standalone.rb +37 -0
- data/lib/bookie/senders/torque_cluster.rb +166 -0
- data/lib/bookie/version.rb +4 -0
- data/snapshot/config.json +12 -0
- data/snapshot/default.json +11 -0
- data/snapshot/pacct +0 -0
- data/snapshot/pacct_large +0 -0
- data/snapshot/pacct_test_config.json +14 -0
- data/snapshot/test_config.json +13 -0
- data/snapshot/torque +3 -0
- data/snapshot/torque_invalid_lines +5 -0
- data/snapshot/torque_invalid_lines_2 +4 -0
- data/snapshot/torque_invalid_lines_3 +3 -0
- data/snapshot/torque_large +100 -0
- data/spec/comma_dump_formatter_spec.rb +56 -0
- data/spec/config_spec.rb +55 -0
- data/spec/database_spec.rb +625 -0
- data/spec/formatter_spec.rb +93 -0
- data/spec/sender_spec.rb +104 -0
- data/spec/spec_helper.rb +121 -0
- data/spec/spreadsheet_formatter_spec.rb +112 -0
- data/spec/standalone_sender_spec.rb +40 -0
- data/spec/stdout_formatter_spec.rb +66 -0
- data/spec/torque_cluster_sender_spec.rb +111 -0
- metadata +227 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Ben Merritt
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Bookie
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'bookie'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install bookie
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require "rspec/core/rake_task"
|
4
|
+
|
5
|
+
task :default => :spec
|
6
|
+
|
7
|
+
desc "Run specs"
|
8
|
+
RSpec::Core::RakeTask.new(:spec) do |task|
|
9
|
+
task.rspec_opts =%w{--color --format progress}
|
10
|
+
task.pattern = 'spec/*_spec.rb'
|
11
|
+
end
|
12
|
+
|
13
|
+
task :docs do
|
14
|
+
system("rdoc rdoc lib")
|
15
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
#For development
|
4
|
+
#$LOAD_PATH << 'lib'
|
5
|
+
|
6
|
+
require 'optparse'
|
7
|
+
|
8
|
+
require 'bookie/database'
|
9
|
+
|
10
|
+
config_file = '/etc/bookie/config.json'
|
11
|
+
drop = false
|
12
|
+
|
13
|
+
opts = OptionParser.new do |opts|
|
14
|
+
opts.banner = "bookie-create-tables [options]"
|
15
|
+
|
16
|
+
opts.on('-c', '--config FILE', String, "use the given configuration file") do |file|
|
17
|
+
config_file = file
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on("--drop", "drop all tables") do
|
21
|
+
drop = true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
begin
|
25
|
+
opts.parse!(ARGV)
|
26
|
+
rescue OptionParser::ParseError => e
|
27
|
+
puts e.message
|
28
|
+
puts opts
|
29
|
+
exit 1
|
30
|
+
end
|
31
|
+
|
32
|
+
config = Bookie::Config.new(config_file)
|
33
|
+
config.connect
|
34
|
+
|
35
|
+
if drop
|
36
|
+
STDOUT.write "Are you sure? This will destroy all tables in the database. "
|
37
|
+
response = nil
|
38
|
+
until response
|
39
|
+
response = STDIN.gets.chomp
|
40
|
+
case response
|
41
|
+
when "YES"
|
42
|
+
Bookie::Database::Migration.down
|
43
|
+
when /^NO$/i
|
44
|
+
exit 0
|
45
|
+
else
|
46
|
+
$stdout.write "Please answer 'YES' or 'NO'. "
|
47
|
+
response = nil
|
48
|
+
end
|
49
|
+
end
|
50
|
+
else
|
51
|
+
Bookie::Database::Migration.up
|
52
|
+
end
|
data/bin/bookie-data
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
#For development
|
6
|
+
#$LOAD_PATH << 'lib'
|
7
|
+
|
8
|
+
require 'bookie/formatter'
|
9
|
+
|
10
|
+
jobs = Bookie::Database::Job
|
11
|
+
systems = Bookie::Database::System
|
12
|
+
|
13
|
+
config_filename = '/etc/bookie/config.json'
|
14
|
+
include_details = false
|
15
|
+
|
16
|
+
output_type = :stdout
|
17
|
+
filename = nil
|
18
|
+
|
19
|
+
t_min = nil
|
20
|
+
t_max = nil
|
21
|
+
|
22
|
+
#Process arguments
|
23
|
+
|
24
|
+
#The first run only gets the configuration filename
|
25
|
+
ARGV.each_with_index do |value, i|
|
26
|
+
if value == '-c' || value == '--config'
|
27
|
+
v = ARGV[i + 1]
|
28
|
+
#If the argument is missing, ignore it; OptionParser will catch it later.
|
29
|
+
config_filename = v if v
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
config = Bookie::Config.new(config_filename)
|
34
|
+
config.connect
|
35
|
+
|
36
|
+
opts = OptionParser.new do |opts|
|
37
|
+
opts.banner = "Usage: bookie-data [options]"
|
38
|
+
|
39
|
+
opts.on('-c', '--config FILE', String, "use the given configuration file") do |file|
|
40
|
+
#This is just here so it shows up in the message.
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on('-d', '--details', "include full details") do
|
44
|
+
include_details = true
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on('-u', '--user NAME', "filter by username") do |name|
|
48
|
+
jobs = jobs.by_user_name(name)
|
49
|
+
end
|
50
|
+
|
51
|
+
opts.on('-g', '--group NAME' "filter by group") do |name|
|
52
|
+
jobs = jobs.by_group_name(name)
|
53
|
+
end
|
54
|
+
|
55
|
+
opts.on('-s', '--system HOSTNAME', "filter by system") do |hostname|
|
56
|
+
jobs = jobs.by_system_name(hostname)
|
57
|
+
systems = systems.by_name(hostname)
|
58
|
+
end
|
59
|
+
|
60
|
+
opts.on('-t', '--type TYPE', "filter by system type") do |type|
|
61
|
+
t = Bookie::Database::SystemType.find_by_name(type)
|
62
|
+
unless t
|
63
|
+
STDERR.puts "Unknown system type '#{type}'"
|
64
|
+
exit 1
|
65
|
+
end
|
66
|
+
jobs = jobs.by_system_type(t)
|
67
|
+
systems = systems.by_system_type(t)
|
68
|
+
end
|
69
|
+
|
70
|
+
opts.on('-r', '--time BEGIN,END', Array, "filter by a time range") do |t|
|
71
|
+
t_min = Time.parse(t[0])
|
72
|
+
t_max = Time.parse(t[1])
|
73
|
+
end
|
74
|
+
|
75
|
+
opts.on('-o', '--output-file FILENAME', "send formatted output to FILENAME",
|
76
|
+
"Output format is inferred from the filename extension.") do |output_filename|
|
77
|
+
filename = output_filename
|
78
|
+
case filename
|
79
|
+
when /\.xls$/
|
80
|
+
output_type = :spreadsheet
|
81
|
+
when /\.csv$/
|
82
|
+
output_type = :comma_dump
|
83
|
+
else
|
84
|
+
$stderr.puts "Unrecognized output file extension"
|
85
|
+
exit 1
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
begin
|
90
|
+
opts.parse!(ARGV)
|
91
|
+
rescue OptionParser::ParseError => e
|
92
|
+
STDERR.puts e.message
|
93
|
+
STDERR.puts opts
|
94
|
+
exit 1
|
95
|
+
end
|
96
|
+
|
97
|
+
formatter = Bookie::Formatter.new(output_type, filename)
|
98
|
+
|
99
|
+
jobs_summary, systems_summary = formatter.print_summary(jobs, systems, t_min, t_max)
|
100
|
+
jobs = jobs.by_time_range_inclusive(t_min, t_max) if t_min
|
101
|
+
formatter.print_jobs(jobs_summary[:jobs]) if include_details
|
102
|
+
formatter.flush
|
data/bin/bookie-send
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
#For development:
|
6
|
+
#$LOAD_PATH << 'lib'
|
7
|
+
|
8
|
+
#To consider: restore for production?
|
9
|
+
=begin
|
10
|
+
unless Process.uid == 0
|
11
|
+
$stderr.puts "This command must be run as root."
|
12
|
+
exit 1
|
13
|
+
end
|
14
|
+
=end
|
15
|
+
|
16
|
+
require 'bookie/sender'
|
17
|
+
|
18
|
+
config_file = '/etc/bookie/config.json'
|
19
|
+
will_create = false
|
20
|
+
will_decommission = false
|
21
|
+
system_start_time = nil
|
22
|
+
system_hostname = nil
|
23
|
+
system_end_time = nil
|
24
|
+
|
25
|
+
opts = OptionParser.new do |opts|
|
26
|
+
opts.banner = "Usage: bookie-send [data-file] [options]"
|
27
|
+
|
28
|
+
opts.on('-c', '--config FILE', String, "use the given configuration file") do |file|
|
29
|
+
config_file = file
|
30
|
+
end
|
31
|
+
|
32
|
+
opts.on("--create [TIME]",
|
33
|
+
"Create an entry for this system, recording the given time as its start time") do |time|
|
34
|
+
will_create = true
|
35
|
+
system_start_time = Time.parse(time) if time
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on("--decommission [HOSTNAME [TIME]]",
|
39
|
+
String, String,
|
40
|
+
"Decommission the system with the given hostname, recording the given time as its end time") do |hostname, time|
|
41
|
+
will_decommission = true
|
42
|
+
system_hostname = hostname if hostname
|
43
|
+
system_end_time = Time.parse(time) if time
|
44
|
+
end
|
45
|
+
end
|
46
|
+
begin
|
47
|
+
opts.parse!(ARGV)
|
48
|
+
rescue OptionParser::ParseError => e
|
49
|
+
puts e.message
|
50
|
+
puts opts
|
51
|
+
exit 1
|
52
|
+
end
|
53
|
+
|
54
|
+
config = Bookie::Config.new(config_file)
|
55
|
+
config.connect
|
56
|
+
|
57
|
+
filename = ARGV[0]
|
58
|
+
fail("No operation specified") unless filename || will_create || will_decommission
|
59
|
+
|
60
|
+
if filename
|
61
|
+
sender = Bookie::Sender.new(config)
|
62
|
+
sender.send_data(filename)
|
63
|
+
end
|
64
|
+
|
65
|
+
if will_decommission
|
66
|
+
system_hostname ||= config.hostname
|
67
|
+
system_end_time ||= Time.now
|
68
|
+
Bookie::Database::Lock[:systems].synchronize do
|
69
|
+
system = Bookie::Database::System.active_systems.find_by_name(system_hostname)
|
70
|
+
if system
|
71
|
+
puts "Note: make sure that all of this system's jobs have been recorded in the database before decommissioning it."
|
72
|
+
STDOUT.write "Decommission this system? "
|
73
|
+
response = nil
|
74
|
+
until response
|
75
|
+
response = STDIN.gets.chomp.downcase
|
76
|
+
case response
|
77
|
+
when "yes"
|
78
|
+
system.decommission(system_end_time)
|
79
|
+
when "no"
|
80
|
+
exit 0
|
81
|
+
else
|
82
|
+
STDOUT.write("Please answer 'yes' or 'no'.")
|
83
|
+
response = nil
|
84
|
+
end
|
85
|
+
end
|
86
|
+
else
|
87
|
+
stderr.puts "No active system with hostname #{system_hostname}"
|
88
|
+
exit 1
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
if will_create
|
94
|
+
system_start_time ||= Time.now
|
95
|
+
Bookie::Database::Lock[:systems].synchronize do
|
96
|
+
system = Bookie::Database::System.active_systems.find_by_name(config.hostname)
|
97
|
+
if system
|
98
|
+
stderr.puts "An active system is already in the database with hostname '#{config.hostname}'."
|
99
|
+
exit 1
|
100
|
+
end
|
101
|
+
end
|
102
|
+
Bookie::Database::System.create!(
|
103
|
+
:name => config.hostname,
|
104
|
+
:system_type => Bookie::Sender.new(config).system_type,
|
105
|
+
:start_time => system_start_time,
|
106
|
+
:end_time => nil,
|
107
|
+
:cores => config.cores,
|
108
|
+
:memory => config.memory
|
109
|
+
)
|
110
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/bookie/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Ben Merritt"]
|
6
|
+
gem.email = ["blm768@gmail.com"]
|
7
|
+
gem.description = %q{A simple system to record and query process accounting records}
|
8
|
+
gem.summary = %q{A simple system to record and query process accounting records}
|
9
|
+
gem.homepage = "https://github.com/blm768/bookie/"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(spec)/})
|
14
|
+
gem.name = "bookie_accounting"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = Bookie::VERSION
|
17
|
+
|
18
|
+
gem.add_dependency('json')
|
19
|
+
gem.add_dependency('activerecord')
|
20
|
+
#For some reason, this is needed for Ruby 1.8.7 using RVM on CentOS.
|
21
|
+
#To do: remove when no longer needed
|
22
|
+
gem.add_dependency('mysql2')
|
23
|
+
gem.add_dependency('pacct')
|
24
|
+
gem.add_dependency('spreadsheet')
|
25
|
+
gem.add_development_dependency('mocha')
|
26
|
+
gem.add_development_dependency('rspec')
|
27
|
+
gem.add_development_dependency('sqlite3')
|
28
|
+
end
|
data/lib/bookie.rb
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'json'
|
3
|
+
require 'logger'
|
4
|
+
require 'set'
|
5
|
+
|
6
|
+
module Bookie
|
7
|
+
##
|
8
|
+
#Holds database configuration, etc. for Bookie components
|
9
|
+
class Config
|
10
|
+
#The database type
|
11
|
+
#
|
12
|
+
#Corresponds to ActiveRecord database adapter name
|
13
|
+
attr_accessor :db_type
|
14
|
+
#The database server's hostname
|
15
|
+
attr_accessor :server
|
16
|
+
#The database server's port
|
17
|
+
#
|
18
|
+
#If nil, use the default port.
|
19
|
+
attr_accessor :port
|
20
|
+
#The name of the database to use
|
21
|
+
attr_accessor :database
|
22
|
+
#The username for the database
|
23
|
+
attr_accessor :username
|
24
|
+
#The password for the database
|
25
|
+
attr_accessor :password
|
26
|
+
#A set containing the names of users to be excluded
|
27
|
+
attr_accessor :excluded_users
|
28
|
+
#The system type
|
29
|
+
attr_accessor :system_type
|
30
|
+
#The system's hostname
|
31
|
+
attr_accessor :hostname
|
32
|
+
#The number of cores on the system
|
33
|
+
attr_accessor :cores
|
34
|
+
#The RAM (in KB) in the system
|
35
|
+
attr_accessor :memory
|
36
|
+
|
37
|
+
##
|
38
|
+
#Creates a new Config object using values from the provided JSON file
|
39
|
+
def initialize(filename)
|
40
|
+
file = File.open(filename)
|
41
|
+
data = JSON::parse(file.read)
|
42
|
+
file.close
|
43
|
+
|
44
|
+
@db_type = data['Database type']
|
45
|
+
verify_type(@db_type, 'Database type', String)
|
46
|
+
|
47
|
+
@server = data['Server']
|
48
|
+
verify_type(@server, 'Server', String)
|
49
|
+
@port = data['Port']
|
50
|
+
verify_type(@port, 'Port', Integer) unless @port == nil
|
51
|
+
|
52
|
+
@database = data['Database']
|
53
|
+
verify_type(@database, 'Database', String)
|
54
|
+
@username = data['Username']
|
55
|
+
verify_type(@username, 'Username', String)
|
56
|
+
@password = data['Password']
|
57
|
+
verify_type(@password, 'Password', String)
|
58
|
+
|
59
|
+
excluded_users_array = data['Excluded users'] || []
|
60
|
+
verify_type(excluded_users_array, 'Excluded users', Array)
|
61
|
+
@excluded_users = Set.new(excluded_users_array)
|
62
|
+
|
63
|
+
@system_type = data['System type']
|
64
|
+
verify_type(@system_type, 'System type', String)
|
65
|
+
|
66
|
+
@hostname = data['Hostname']
|
67
|
+
verify_type(@hostname, 'Hostname', String)
|
68
|
+
|
69
|
+
@cores = data['Cores']
|
70
|
+
verify_type(@cores, 'Cores', Integer)
|
71
|
+
|
72
|
+
@memory = data['Memory']
|
73
|
+
verify_type(@memory, 'Memory', Integer)
|
74
|
+
end
|
75
|
+
|
76
|
+
#Verifies that a field is of the correct type, raising an error if the type does not match
|
77
|
+
def verify_type(value, name, type)
|
78
|
+
if value == nil
|
79
|
+
raise "Field \"#{name}\" must have a non-null value."
|
80
|
+
end
|
81
|
+
raise TypeError.new("Invalid data type #{value.class} for JSON field \"#{name}\": #{type} expected") unless value.class <= type
|
82
|
+
end
|
83
|
+
|
84
|
+
#Connects to the database specified in the configuration file
|
85
|
+
def connect()
|
86
|
+
#To consider: disable colorized logging?
|
87
|
+
#To consider: create config option for this?
|
88
|
+
#ActiveRecord::Base.logger = Logger.new(STDERR)
|
89
|
+
#ActiveRecord::Base.logger.level = Logger::WARN
|
90
|
+
ActiveRecord::Base.time_zone_aware_attributes = true
|
91
|
+
ActiveRecord::Base.default_timezone = :utc
|
92
|
+
ActiveRecord::Base.establish_connection(
|
93
|
+
:adapter => self.db_type,
|
94
|
+
:database => self.database,
|
95
|
+
:username => self.username,
|
96
|
+
:password => self.password,
|
97
|
+
:host => self.server,
|
98
|
+
:port => self.port)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|