bookie_accounting 1.2.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f140628ce83250fabffabda656f8f66a73cb89dc
-  data.tar.gz: b0c4ed64a1adbd44986df8856eb9c01a2aa52b61
+  metadata.gz: 8d726372976ed3f86e19d29e2a391d233c972d5d
+  data.tar.gz: 99ea6555c0e94a6be3d04c5168a4a45dde020d90
 SHA512:
-  metadata.gz: ca2c83c5150357c4cef54d129670c9ef93f530c4ec4846f333de7ea79b19a9cf88a377a69e1895459a40bffa1f254a26bab55a2684cb69e13301df3ee710ab88
-  data.tar.gz: 1ebafa8e77bf140b8942289b53bab7d9a6345250f2e2888c8ecceb417181f7eee7f6975d09bd5ce7744af940320f27ec8c3c39f55d826521a17377e484b5964c
+  metadata.gz: e123b58f99c5529014a135948ccfbbcde6c2090a2436d5b9076b49bcb839fce55d1f37636713301c7eb783b3aeffbc7444d5f4b424a757ab92c3dd30464c13b5
+  data.tar.gz: 3fad9d7b47452444935ea70b821229c066789a04f8b929d2cf33cae6e10b8573c7ff87ebe2d581e33f5bd7082b6b4efabc5faefd85825d353feb72c17eb10308

data/Gemfile

@@ -3,8 +3,9 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'bundler', :group => :development
-gem 'mocha', :group => :development
-gem 'rspec', :group => :development
-gem 'simplecov', :group => :development
+gem 'byebug', :group => :test
+gem 'mocha', :group => :test
+gem 'rspec', :group => :test
+gem 'simplecov', :group => :test
 gem 'sqlite3', :group => :development
 

data/README.md

@@ -1,29 +1,9 @@
 # Bookie
-
-TODO: Write a gem description
+A simple system to consolidate and analyze process accounting records
 
 ## Installation
+See the [Installation](https://github.com/blm768/bookie/wiki/Installation) page on the wiki.
 
-Add this line to your application's Gemfile:
-
-    gem 'bookie'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install bookie
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Contributing
+##Usage
+The wiki contains a [command reference](https://github.com/blm768/bookie/wiki/Command-line-utilities
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request

data/Rakefile

@@ -3,130 +3,23 @@ require 'bundler'
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
-require 'erb'
+require 'find'
 
 task :default => :spec
 
 desc "Run specs"
 RSpec::Core::RakeTask.new(:spec) do |task|
-  task.rspec_opts =%w{--color --format progress}
-  task.pattern = 'spec/*_spec.rb'
-end
-
-task :rdoc do
-  system("rdoc rdoc lib")
-end
-
-#Returns whether the given file is older than its dependencies (or doesn't even exist)
-def older(file, *dep_files)
-  dep_files.each do |dep_file|
-    return true if !File.exists?(file) || File.mtime(file) < File.mtime(dep_file)
-  end
-  false
-end
-
-desc "Build RPM and dependencies (designed for use on CentOS)"
-task :rpm_deps do
-  gem_filenames = {
-    "pacct" => "pacct-0.8.4-universal-linux.gem"
-  }
-
-  lockfile = Bundler::LockfileParser.new(Bundler.read_file("Gemfile.lock"))
-
-  home_dir = ENV['HOME']
-  spec_dir = File.join(home_dir, 'rpmbuild/SPECS')
-  src_dir = File.join(home_dir, 'rpmbuild/SOURCES')
-  rpm_dir = File.join(home_dir, 'rpmbuild/RPMS')
-
-  FileUtils.mkdir_p(spec_dir)
-  FileUtils.mkdir_p(src_dir)
-
-  lockfile.specs.each do |spec|
-    puts "#{spec.name} (#{spec.version})"
-
-    spec_filename = File.join(spec_dir, "rubygem-#{spec.name}.spec")
-
-    gem_filename = gem_filenames[spec.name] || "#{spec.name}-#{spec.version}.gem"
-
-    system("gem fetch #{spec.name} -v #{spec.version}") unless File.exists?(gem_filename)
-
-    template = "rpm/#{spec.name}.erb"
-    template = "rpm/default.erb" unless File.exists?(template)
-
-    system("gem2rpm #{gem_filename} -t #{template} > #{spec_filename}") if older(spec_filename, gem_filename, template)
-
-    src = File.join(src_dir, gem_filename)
-    FileUtils.cp(gem_filename, src_dir) if older(src, gem_filename)
-
-    rpm_glob = File.join(rpm_dir, "*/rubygem-#{spec.name}-#{spec.version}-*.rpm")
-    rpm_glob_results = Dir.glob(rpm_glob)
-    if rpm_glob_results.length > 1
-      puts "Multiple RPMs exist for #{spec.name} (#{spec.version}):"
-      rpm_glob_results.each{ |r| puts "    #{r}" }
-      puts "Unable to resolve dependency due to ambiguous target"
-      exit 1
-    end
-
-    if rpm_glob_results.length == 0 || older(rpm_glob_results[0], spec_filename)
-      msg = `rpmbuild -ba #{spec_filename}`
-      unless $?.success?
-        puts msg
-        exit 1
-      end
+  task.rspec_opts =%w{--color --order rand --format progress}
+  paths = []
+  Find.find('spec') do |path|
+    if File.directory?(path)
+      paths << "#{path}/*_spec.rb"
     end
   end
+  task.pattern = paths.join(" ")
 end
 
-desc "Build RPM with all dependencies bundled"
-task "rpm_bundle" do
-  gem_filenames = {
-    "pacct" => "pacct-0.8.4-universal-linux.gem"
-  }
-
-  lockfile = Bundler::LockfileParser.new(Bundler.read_file("Gemfile.lock"))
-
-  home_dir = ENV['HOME']
-  spec_dir = File.join(home_dir, 'rpmbuild/SPECS')
-  src_dir = File.join(home_dir, 'rpmbuild/SOURCES')
-  rpm_dir = File.join(home_dir, 'rpmbuild/RPMS')
-
-  FileUtils.mkdir_p(spec_dir)
-  FileUtils.mkdir_p(src_dir)
-
-  spec_filename = File.join(spec_dir, "bookie_accounting.spec")
-
-  template = ERB.new(File.read("rpm/bundle.erb"))
-
-  specs = lockfile.specs
-
-  #To do: ensure that the rubygems version matches the local version?
-  bookie_spec = Gem::Specification.load("bookie_accounting.gemspec")
-
-  sources = []
-
-  pwd = Dir.pwd
-  Dir.chdir(src_dir)
-  specs.each do |spec|
-    puts "#{spec.name} (#{spec.version})"
-
-    #bookie_spec = spec if spec.name == "bookie_accounting"
-
-    gem_filename = gem_filenames[spec.name] || "#{spec.name}-#{spec.version}.gem"
-
-    system("gem fetch #{spec.name} -v #{spec.version}") unless File.exists?(gem_filename)
-
-    sources << gem_filename
-  end
-  Dir.chdir(pwd)
-
-  File.open(spec_filename, "w") do |f|
-    f.write(template.result(binding))
-  end
-
-  msg = `rpmbuild -ba #{spec_filename}`
-  unless $?.success?
-    puts msg
-    exit 1
-  end
+task :rdoc do
+  system("rdoc rdoc lib")
 end
 

data/bin/bookie-data

@@ -11,8 +11,12 @@ jobs = Bookie::Database::Job
 summaries = Bookie::Database::JobSummary
 systems = Bookie::Database::System
 
+DEFAULT_DETAILS_PER_PAGE = 20
+
 config_filename = ENV['BOOKIE_CONFIG'] || '/etc/bookie/config.json'
 include_details = false
+details_per_page = nil
+details_page = 1
 
 output_type = :stdout
 filename = nil
@@ -21,11 +25,11 @@ time_range = nil
 
 #Process arguments
 
-#The first run only gets the configuration filename
+#The first run only gets the configuration filename.
 ARGV.each_with_index do |value, i|
   if value == '-c' || value == '--config'
     v = ARGV[i + 1]
-    #If the argument is missing, ignore it; OptionParser will catch it later.
+    #If the argument is missing or invalid, ignore the problem; OptionParser will catch it later.
     config_filename = v if v
   end
 end
@@ -43,6 +47,25 @@ opts = OptionParser.new do |opts|
   opts.on('-d', '--details', "include full details") do
     include_details = true
   end
+
+  opts.on('-p', '--page PAGE', Integer, "show only the given page of details") do |page_num|
+    if page_num < 1
+      STDERR.puts "invalid page number: #{page_num}"
+      exit 1
+    end
+    include_details = true
+    details_per_page ||= DEFAULT_DETAILS_PER_PAGE
+    details_page = page_num
+  end
+
+  opts.on('-l', '--limit COUNT', Integer, "limit the number of jobs per page") do |count|
+    if count < 1
+      STDERR.PUTS "invalid page length: #{count}"
+      exit 1
+    end
+    include_details = true
+    details_per_page = count
+  end
   
   opts.on('-u', '--user NAME', "filter by username") do |name|
     jobs = jobs.by_user_name(name)
@@ -68,7 +91,7 @@ opts = OptionParser.new do |opts|
   opts.on('-t', '--type TYPE', "filter by system type") do |type|
     t = Bookie::Database::SystemType.find_by_name(type)
     unless t
-      STDERR.puts "Unknown system type '#{type}'"
+      STDERR.puts "unknown system type '#{type}'"
       exit 1
     end
     jobs = jobs.by_system_type(t)
@@ -89,11 +112,12 @@ opts = OptionParser.new do |opts|
     when /\.csv$/
       output_type = :comma_dump
     else
-      $stderr.puts "Unrecognized output file extension"
+      STDERR.puts "unrecognized output file extension"
       exit 1
     end
   end
 end
+
 begin
   opts.parse!(ARGV)
 rescue OptionParser::ParseError => e
@@ -104,7 +128,24 @@ end
 
 formatter = Bookie::Formatter.new(output_type, filename)
 
-jobs_summary, systems_summary = formatter.print_summary(jobs, summaries, systems, time_range)
-jobs = jobs.by_time_range_inclusive(time_range) if time_range
-formatter.print_jobs(jobs.all) if include_details
+formatter.print_summary(jobs, summaries, systems, time_range)
+
+if include_details
+  #TODO: include separator or blank line.
+
+  jobs = jobs.by_time_range(time_range) if time_range
+  jobs = jobs.order(:start_time)
+
+  if details_per_page
+    page_start = details_per_page * (details_page - 1)
+    if jobs.count <= page_start
+      puts "No jobs on page #{details_page}"
+    else
+      jobs = jobs.offset(page_start).limit(details_per_page)
+    end
+  end
+  formatter.print_jobs(jobs.all_with_associations)
+end
+
 formatter.flush
+

data/bin/bookie-send

@@ -2,17 +2,6 @@
 
 require 'optparse'
 
-#For development:
-#$LOAD_PATH << 'lib'
-
-#To consider: restore for production?
-=begin
-unless Process.uid == 0
-  $stderr.puts "This command must be run as root."
-  exit 1
-end
-=end
-
 require 'bookie/sender'
 
 config_file = ENV['BOOKIE_CONFIG'] || '/etc/bookie/config.json'
@@ -60,7 +49,10 @@ config = Bookie::Config.new(config_file)
 config.connect
 
 filename = ARGV[0]
-fail("No operation specified") unless filename || will_create || will_decommission
+unless filename || will_create || will_decommission
+  STDERR.puts "No operation specified"
+  exit 1
+end
 
 if filename
   sender = Bookie::Sender.new(config)
@@ -75,7 +67,7 @@ if will_decommission
   system_hostname ||= config.hostname
   system_end_time ||= Time.now
   Bookie::Database::Lock[:systems].synchronize do
-    system = Bookie::Database::System.active_systems.find_by_name(system_hostname)
+    system = Bookie::Database::System.active_systems.where(:name => system_hostname).first
     if system
       puts "Note: make sure that all of this system's jobs have been recorded in the database before decommissioning it."
       STDOUT.write "Decommission this system? "
@@ -102,7 +94,7 @@ end
 if will_create
   system_start_time ||= Time.now
   Bookie::Database::Lock[:systems].synchronize do
-    system = Bookie::Database::System.active_systems.find_by_name(config.hostname)
+    system = Bookie::Database::System.active_systems.where(:name => config.hostname).first
     if system
       stderr.puts "An active system is already in the database with hostname '#{config.hostname}'."
       exit 1

data/bookie_accounting.gemspec

@@ -5,8 +5,8 @@ Gem::Specification.new do |gem|
   gem.authors       = ["Ben Merritt"]
   gem.email         = ["blm768@gmail.com"]
   gem.license       = "MIT"
-  gem.description   = %q{A simple system to record and query process accounting records}
-  gem.summary       = %q{A simple system to record and query process accounting records}
+  gem.description   = %q{A simple system to consolidate and analyze process accounting records}
+  gem.summary       = %q{A simple system to consolidate and analyze process accounting records}
   gem.homepage      = "https://github.com/blm768/bookie/"
 
   gem.files         = `git ls-files`.split($\)
@@ -20,11 +20,12 @@ Gem::Specification.new do |gem|
   gem.add_dependency('json')
   #We need this because Bundler has no concept of optional dependencies
   #and complains about using non-dependency gems.
-  #To do: figure out how to remove (file issue?)
+  #TODO: figure out how to remove
   gem.add_dependency('mysql2')
   gem.add_dependency('pacct')
   #Introduces the old ActiveRecord mass assignment security methods
   #(until I update the database code for the new methods)
+  #TODO: get rid of this.
   gem.add_dependency('protected_attributes')
   gem.add_dependency('spreadsheet')
 end

data/lib/bookie/database/group.rb

@@ -0,0 +1,33 @@
+require 'active_record'
+
+require 'bookie/database/lock.rb'
+
+module Bookie
+  module Database
+    ##
+    #A group of users
+    class Group < ActiveRecord::Base
+      has_many :users
+
+      ##
+      #Finds a group by name, creating it if it doesn't exist
+      #
+      #If <tt>known_groups</tt> is provided, it will be used as a cache to reduce the number of database lookups needed.
+      #
+      #This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
+      def self.find_or_create!(name, known_groups = nil)
+        group = known_groups[name] if known_groups
+        unless group
+          Lock[:groups].synchronize do
+            group = find_by_name(name)
+            group ||= create!(:name => name)
+          end
+          known_groups[name] = group if known_groups
+        end
+        group
+      end
+      
+      validates_presence_of :name
+    end
+  end
+end

data/lib/bookie/database/job.rb

@@ -0,0 +1,201 @@
+require 'active_record'
+
+require 'bookie/database/lock.rb'
+require 'bookie/database/user.rb'
+require 'bookie/database/system.rb'
+
+module Bookie
+  module Database
+    ##
+    #A reported job
+    #
+    #The various filter methods can be chained to produce more complex queries.
+    #
+    #===Examples
+    #  Bookie::Database::Job.by_user_name('root').by_system_name('localhost').find_each do |job|
+    #    puts job.inspect
+    #  end
+    #
+    class Job < ActiveRecord::Base
+      belongs_to :user
+      belongs_to :system
+      has_one :group, :through => :user
+      has_one :system_type, :through => :system
+      
+      ##
+      #The time at which the job ended
+      def end_time
+        return start_time + wall_time
+      end
+
+      def end_time=(time)
+        self.wall_time = (time - start_time)
+      end
+
+      #To consider: disable #end_time= ?
+      
+      def self.by_user(user)
+        where('jobs.user_id = ?', user.id)
+      end
+      
+      ##
+      #Filters by user name
+      def self.by_user_name(user_name)
+        joins(:user).where('users.name = ?', user_name)
+      end
+      
+      def self.by_system(system)
+        where('jobs.system_id = ?', system.id)
+      end
+
+      ##
+      #Filters by system name
+      def self.by_system_name(system_name)
+        joins(:system).where('systems.name = ?', system_name)
+      end
+      
+      ##
+      #Filters by group name
+      def self.by_group_name(group_name)
+        group = Group.find_by_name(group_name)
+        return joins(:user).where('users.group_id = ?', group.id) if group
+        self.none
+      end
+      
+      ##
+      #Filters by system type
+      def self.by_system_type(system_type)
+        joins(:system).where('systems.system_type_id = ?', system_type.id)
+      end
+      
+      ##
+      #Filters by command name
+      def self.by_command_name(c_name)
+        where('jobs.command_name = ?', c_name)
+      end
+      
+      ##
+      #Finds all jobs that were running at some point in a given time range
+      def self.by_time_range(time_range)
+        if time_range.empty?
+          self.none
+        else
+          time_range = time_range.exclusive
+          where('jobs.end_time > ? AND jobs.start_time < ?', time_range.begin, time_range.end)
+        end
+      end
+
+      ##
+      #Similar to #by_time_range, but only includes jobs that are completely contained within the
+      #time range
+      def self.within_time_range(time_range)
+        if time_range.empty?
+          self.none
+        else
+          time_range = time_range.exclusive
+          #The second "<=" operator _is_ intentional.
+          #If the job's end_time is one second past the last value in the range, it
+          #is still considered to be contained within time_range because it did not
+          #run outside time_range; it only _stopped_ outside it.
+          where('? <= jobs.start_time AND jobs.end_time <= ?', time_range.begin, time_range.end)
+        end
+      end
+      
+      ##
+      #Finds all jobs that overlap the edges of the given time range
+      def self.overlapping_edges(time_range)
+        if time_range.empty?
+          self.none
+        else
+          time_range = time_range.exclusive
+          query_str = ['begin', 'end'].map{ |edge| "(jobs.start_time < :#{edge} AND jobs.end_time > :#{edge})" }.join(" OR ")
+          where(query_str, {:begin => time_range.begin, :end => time_range.end})
+        end
+      end
+     
+      ##
+      #Produces a summary of the jobs in the given time interval
+      #
+      #Returns a hash with the following fields:
+      #- <tt>:num_jobs</tt>: the number of jobs in the interval
+      #- <tt>:successful</tt>: the number of jobs that have completed successfully
+      #- <tt>:cpu_time</tt>: the total CPU time used
+      #- <tt>:memory_time</tt>: the sum of memory * wall_time for all jobs in the interval
+      #
+      #This method should probably not be chained with other queries that filter by start/end time.
+      #It also doesn't work with the limit() method.
+      #
+      #The time_range parameter is always treated as if time_range.exclude_end? is true.
+      def self.summary(time_range = nil)
+        jobs = self
+
+        num_jobs = 0
+        successful = 0
+        cpu_time = 0.0
+        memory_time = 0
+
+        if time_range
+          unless time_range.empty?
+            time_range = time_range.exclusive
+
+            #Any jobs that are completely within the time range can
+            #be summarized as-is.
+            jobs_within = jobs.within_time_range(time_range)
+            #TODO: optimize into one query?
+            num_jobs += jobs_within.count
+            successful += jobs_within.where(:exit_code => 0).count
+            cpu_time += jobs_within.sum(:cpu_time)
+            memory_time += jobs_within.sum('jobs.memory * jobs.wall_time')
+
+            #Any jobs that overlap an edge of the time range
+            #must be clipped.
+            jobs_overlapped = jobs.overlapping_edges(time_range)
+            jobs_overlapped.find_each do |job|
+              start_time = [job.start_time, time_range.begin].max
+              end_time = [job.end_time, time_range.end].min
+              clipped_wall_time = end_time.to_i - start_time.to_i
+              if job.wall_time != 0
+                cpu_time += Float(job.cpu_time * clipped_wall_time) / job.wall_time
+                memory_time += job.memory * clipped_wall_time
+              end
+              num_jobs += 1
+              successful += 1 if job.exit_code == 0
+            end
+          end
+        else
+          #There's no time_range constraint; just summarize everything.
+          num_jobs = jobs.count
+          successful = jobs.where(:exit_code => 0).count
+          cpu_time = jobs.sum(:cpu_time)
+          memory_time = jobs.sum('jobs.memory * jobs.wall_time')
+        end
+
+        return {
+          :num_jobs => num_jobs,
+          :successful => successful,
+          :cpu_time => cpu_time.round,
+          :memory_time => memory_time,
+        }
+      end
+      
+      before_save do
+        write_attribute(:end_time, end_time)
+      end
+      
+      before_update do
+        write_attribute(:end_time, end_time)
+      end
+      
+      validates_presence_of :user, :system, :cpu_time,
+        :start_time, :wall_time, :memory, :exit_code
+        
+      validates_each :command_name do |record, attr, value|
+        record.errors.add(attr, 'must not be nil') if value == nil
+      end
+       
+      validates_each :cpu_time, :wall_time, :memory do |record, attr, value|
+        record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
+      end
+    end
+  end
+end