bookie_accounting 1.2.3 → 2.0.0

data/lib/bookie/extensions.rb

@@ -1,3 +1,4 @@
+#TODO: revise unit tests.
 
 ##
 #Reopened to add some useful methods
@@ -6,58 +7,36 @@ class Range
   #If end < begin, returns an empty range (begin ... begin)
   #Otherwise, returns the original range
   def normalized
-    return self.begin ... self.begin if self.end < self.begin
+   if self.end < self.begin
+     self.begin ... self.begin
+   else
     self
+   end
+  end
+
+  ##
+  #Converts the range to an equivalent exclusive range (one where exclude_end? is true)
+  #
+  #Only works for ranges with discrete steps between values (i.e. integers)
+  def exclusive
+    if exclude_end?
+      self
+    else
+      Range.new(self.begin, self.end + 1, true)
+    end
   end
   
   ##
-  #Returns the empty status of the range
+  #Returns whether the range is empty
   #
   #A range is empty if end < begin or if begin == end and exclude_end? is true.
   def empty?
-    (self.end < self.begin) || (exclude_end? && (self.begin == self.end))
+    if exclude_end?
+      self.end <= self.begin
+    else
+      self.end < self.begin
+    end
   end
-
-#This code probably works, but we're not using it anywhere.  
-#   def intersection(other)
-#     self_n = self.normalized
-#     other = other.normalized
-#     
-#     new_begin, new_end, exclude_end = nil
-#     
-#     if self_n.cover?(other.begin)
-#       new_first = other.begin
-#     elsif other.cover?(self_n.begin)
-#       new_first = self_n.begin
-#     end
-#     
-#     return self_n.begin ... self_n.begin unless new_first
-#     
-#     if self_n.cover?(other.end)
-#       unless other.exclude_end? && other.end == self_n.begin
-#         new_end = other.end
-#         exclude_end = other.exclude_end?
-#       end
-#     elsif other.cover?(self_n.end)
-#       unless self_n.exclude_end? && self_n.end == other.begin
-#         new_end = self_n.end
-#         exclude_end = self_n.exclude_end?
-#       end
-#     end
-#     
-#     #If we still haven't found new_end, try one more case:
-#     unless new_end
-#       if self_n.end == other.end
-#         #We'll only get here if both ranges exclude their ends and have the same end.
-#         new_end = self_n.end
-#         exclude_end = true
-#       end
-#     end
-#     
-#     return self_n.begin ... self_n.begin unless new_end
-# 
-#     Range.new(new_begin, new_end, exclude_end)
-#   end
 end
 
 ##

data/lib/bookie/formatter.rb

@@ -38,6 +38,7 @@ module Bookie
       
       ##
       #An array containing the labels for each field in a details table
+      #TODO: remove some fields?
       DETAILS_FIELD_LABELS = [
         'User', 'Group', 'System', 'System type', 'Start time', 'End time', 'Wall time',
         'CPU time', 'Memory usage', 'Command', 'Exit code'
@@ -54,6 +55,7 @@ module Bookie
     #
     #Returns the summaries for <tt>jobs</tt> and <tt>systems</tt>
     def print_summary(jobs, summaries, systems, time_range = nil)
+      jobs = jobs.includes(:user, :group, :system, :system_type)
       jobs_summary = summaries.summary(:jobs => jobs, :range => time_range)
       num_jobs = jobs_summary[:num_jobs]
       systems_summary = systems.summary(time_range)
@@ -62,6 +64,7 @@ module Bookie
       memory_time = jobs_summary[:memory_time]
       avail_memory_time = systems_summary[:avail_memory_time]
       successful = (num_jobs == 0) ? 0.0 : Float(jobs_summary[:successful]) / num_jobs
+
       field_values = [
         num_jobs,
         Formatter.format_duration(cpu_time),
@@ -72,13 +75,14 @@ module Bookie
         if avail_memory_time == 0 then '0.0000%' else '%.4f%%' % (Float(memory_time) / avail_memory_time * 100) end
       ]
       do_print_summary(field_values)
-      return jobs_summary, systems_summary
     end
     
     ##
     #Prints a table containing all details of <tt>jobs</tt>
     #
-    #<tt>jobs</tt> should be an array.
+    #<tt>jobs</tt> should be an ActiveRecord model or relation.
+    #
+    #To consider: allow any Enumerable?
     def print_jobs(jobs)
       do_print_jobs(jobs)
     end
@@ -97,7 +101,7 @@ module Bookie
     #call-seq:
     #  fields_for_each_job(jobs) { |fields| ... }
     #
-    #<tt>jobs</tt> should be an array of Bookie::Database::Job objects.
+    #<tt>jobs</tt> should be an ActiveRecord model or relation.
     #
     #===Examples
     #  formatter.fields_for_each_job(jobs) do |fields|
@@ -117,6 +121,7 @@ module Bookie
           job.user.name,
           job.user.group.name,
           job.system.name,
+          #TODO: remove this field?
           job.system.system_type.name,
           job.start_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
           job.end_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
@@ -128,7 +133,6 @@ module Bookie
         ]
       end
     end
-    protected :fields_for_each_job
     
     ##
     #Formats a duration in a human-readable format

data/lib/bookie/sender.rb

@@ -14,9 +14,9 @@ module Bookie
     #<tt>config</tt> should be an instance of Bookie::Config.
     def initialize(config)
       @config = config
-      t = @config.system_type
-      require "bookie/senders/#{t}"
-      extend Bookie::Senders.const_get(t.camelize)
+      sys_type = config.system_type
+      require "bookie/senders/#{sys_type}"
+      extend Bookie::Senders.const_get(sys_type.camelize)
     end
     
     ##
@@ -47,7 +47,7 @@ module Bookie
       #Send the job data:
       each_job(filename) do |job|
         next if filtered?(job)
-        model = job.to_model
+        model = job.to_record
         time_min = (model.start_time < time_min) ? model.start_time : time_min
         time_max = (model.end_time > time_max) ? model.end_time : time_max
         #To consider: handle files that don't have jobs sorted by end time?
@@ -95,14 +95,14 @@ module Bookie
         if system.end_time && job.end_time > system.end_time
           system = Database::System.find_current(self, job.end_time)
         end
-        #To consider: optimize this query?
+        #TODO: optimize this operation?
         #(It should be possible to delete all of the jobs with end times between those of the first and last jobs of the file (exclusive),
         #but jobs with end times matching those of the first/last jobs in the file might be from an earlier or later file, not this one.
         #This assumes that the files all have jobs sorted by end time.
         model = duplicate(job, system)
         break unless model
-        time_min = (model.start_time < time_min) ? model.start_time : time_min
-        time_max = (model.end_time > time_max) ? model.end_time : time_max
+        time_min = [model.start_time, time_min].min
+        time_max = [model.end_time, time_max].max
         model.delete
       end
       
@@ -136,11 +136,11 @@ module Bookie
     end
     
     #Used internally by #send_data and #undo_send
-    def clear_summaries(date_min, date_max)      
+    def clear_summaries(date_min, date_max)
       #Since joins don't mix with DELETE statements, we have to do this the hard way.
-      systems = Database::System.by_name(@config.hostname).to_a
-      systems.map!{ |sys| sys.id }
-      Database::JobSummary.where('job_summaries.system_id in (?)', systems).where('date >= ? AND date <= ?', date_min, date_max).delete_all
+      #To consider: prune systems by time?
+      system_ids = Database::System.by_name(@config.hostname).pluck(:id)
+      Database::JobSummary.where('job_summaries.system_id in (?)', system_ids).where('date >= ? AND date <= ?', date_min, date_max).delete_all
     end
     private :clear_summaries
   end
@@ -150,7 +150,7 @@ module Bookie
   module ModelHelpers
     ##
     #Converts the object to a Bookie::Database::Job
-    def to_model()
+    def to_record()
       job = Bookie::Database::Job.new
       job.command_name = self.command_name
       job.start_time = self.start_time
@@ -161,8 +161,6 @@ module Bookie
       job
     end
 
-    ##
-    #Returns the end time
     def end_time
       start_time + wall_time
     end

data/lib/bookie/version.rb

@@ -1,4 +1,4 @@
 module Bookie
   #The library version
-  VERSION = "1.2.3"
+  VERSION = "2.0.0"
 end

data/snapshot/test_config.json

@@ -2,7 +2,7 @@
   "Database type" : "sqlite3",
   "Server"        : "localhost",
   "Port"          : 8080,
-  "Database"      : "test.sqlite",
+  "Database"      : ":memory:",
   "Username"      : "blm768",
   "Password"      : "test",
   "Excluded users": ["root"],
@@ -10,4 +10,4 @@
   "Hostname"      : "localhost",
   "Cores"         : 8,
   "Memory"        : 8000000
-}
+}

data/spec/config_spec.rb

@@ -9,7 +9,7 @@ describe Bookie::Config do
     config.db_type.should eql "sqlite3"
     config.server.should eql "localhost"
     config.port.should eql 8080
-    config.database.should eql "test.sqlite"
+    config.database.should eql ":memory:"
     config.username.should eql "blm768"
     config.password.should eql "test"
     config.excluded_users.should eql Set.new(["root"])
@@ -52,4 +52,4 @@ describe Bookie::Config do
     ActiveRecord::Base.time_zone_aware_attributes.should eql true
     ActiveRecord::Base.default_timezone.should eql :utc
   end
-end
+end

data/spec/database/group_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Bookie::Database::Group do
+  describe "#find_or_create" do
+    it "creates the group if needed" do
+      Bookie::Database::Group.expects(:"create!")
+      Bookie::Database::Group.find_or_create!('non_root')
+    end
+    
+    it "returns the cached group if one exists" do
+      group = Bookie::Database::Group.find_by_name('root')
+      known_groups = {'root' => group}
+      Bookie::Database::Group.find_or_create!('root', known_groups).should equal group
+    end
+    
+    it "queries the database when this group is not cached" do
+      group = Bookie::Database::Group.find_by_name('root')
+      known_groups = {}
+      Bookie::Database::Group.expects(:find_by_name).returns(group).twice
+      Bookie::Database::Group.expects(:"create!").never
+      Bookie::Database::Group.find_or_create!('root', known_groups).should eql group
+      Bookie::Database::Group.find_or_create!('root', nil).should eql group
+      known_groups.should include 'root'
+    end
+  end
+  
+  it "validates the name field" do
+    group = Bookie::Database::Group.new(:name => nil)
+    group.valid?.should eql false
+    group.name = ''
+    group.valid?.should eql false
+    group.name = 'test'
+    group.valid?.should eql true
+  end
+end
+

data/spec/database/job_spec.rb

@@ -0,0 +1,308 @@
+require 'spec_helper'
+
+include Bookie::Database
+
+RSpec::Matchers.define :be_job_within_time_range do |time_range|
+  match do |job|
+    expect(time_range).to cover(job.start_time)
+    #This check is required because of a peculiarity of #within_time_range;
+    #jobs with an end_time one second beyond the last value in the range
+    #are still included (intentionally).
+    range_extended = Range.new(time_range.begin, time_range.end + 1, time_range.exclude_end?)
+    expect(range_extended).to cover(job.end_time)
+  end
+end
+
+describe Bookie::Database::Job do
+  it "correctly sets end times" do
+    Job.find_each do |job|
+      job.end_time.should eql job.start_time + job.wall_time
+      job.end_time.should eql job.read_attribute(:end_time)
+    end
+
+    #Test the update hook.
+    job = Job.first
+    job.start_time -= 1
+    job.save!
+    job.end_time.should eql job.read_attribute(:end_time)
+  end
+
+  describe "#end_time=" do
+    it "correctly adjusts time values" do
+      job = Job.first
+      old_end_time = job.end_time
+      job.end_time -= 1
+      #We use #to_i because directly subtracting Time objects produces
+      #floating-point numbers.
+      expect(job.wall_time).to eql(job.end_time.to_i - job.start_time.to_i)
+      expect(job.end_time).to eql(old_end_time - 1)
+    end
+  end
+  
+  it "correctly filters by user" do
+    user = User.by_name('test').order(:id).first
+    jobs = Job.by_user(user).to_a
+    jobs.each do |job|
+      job.user.should eql user
+    end
+    jobs.length.should eql 10 
+  end
+  
+  it "correctly filters by user name" do
+    jobs = Job.by_user_name('root').to_a
+    jobs.length.should eql 10
+    jobs[0].user.name.should eql "root"
+    jobs = Job.by_user_name('test').order(:end_time).to_a
+    jobs.length.should eql 20
+    jobs.each do |job|
+      job.user.name.should eql 'test'
+    end
+    jobs[0].user_id.should_not eql jobs[-1].user_id
+    jobs = Job.by_user_name('user').to_a
+    jobs.length.should eql 0
+  end
+
+  it "correctly filters by group name" do
+    jobs = Job.by_group_name("root").to_a
+    jobs.length.should eql 10
+    jobs.each do |job|
+      job.user.group.name.should eql "root"
+    end
+    jobs = Job.by_group_name("admin").order(:start_time).to_a
+    jobs.length.should eql 20
+    jobs[0].user.name.should_not eql jobs[1].user.name
+    jobs = Job.by_group_name("test").to_a
+    jobs.length.should eql 0
+  end
+  
+  it "correctly filters by system" do
+    sys = System.first
+    jobs = Job.by_system(sys)
+    jobs.length.should eql 10
+    jobs.each do |job|
+      job.system.should eql sys
+    end
+  end
+  
+  it "correctly filters by system name" do
+    jobs = Job.by_system_name('test1')
+    jobs.length.should eql 20
+    jobs = Job.by_system_name('test2')
+    jobs.length.should eql 10
+    jobs = Job.by_system_name('test3')
+    jobs.length.should eql 10
+    jobs = Job.by_system_name('test4')
+    jobs.length.should eql 0
+  end
+  
+  it "correctly filters by system type" do
+    sys_type = SystemType.find_by_name('Standalone')
+    jobs = Job.by_system_type(sys_type)
+    jobs.length.should eql 20
+    sys_type = SystemType.find_by_name('TORQUE cluster')
+    jobs = Job.by_system_type(sys_type)
+    jobs.length.should eql 20
+  end
+
+  it "correctly filters by command name" do
+    jobs = Job.by_command_name('vi')
+    jobs.length.should eql 20
+    jobs = Job.by_command_name('emacs')
+    jobs.length.should eql 20
+  end
+  
+  describe "#by_time_range" do
+    let(:base_start) { base_time + 1.hours }
+    let(:base_end) { base_start + 2.hours }
+
+    it "filters by inclusive time range" do
+      jobs = Job.by_time_range(base_start ... base_end + 1)
+      jobs.count.should eql 3
+      jobs = Job.by_time_range(base_start + 1 ... base_end) 
+      jobs.count.should eql 2
+      jobs = Job.by_time_range(base_start ... base_start)
+      jobs.length.should eql 0
+    end
+
+    it "filters by exclusive time range" do
+      jobs = Job.by_time_range(base_start + 1 .. base_end)
+      jobs.count.should eql 3
+      jobs = Job.by_time_range(base_start .. base_end - 1)
+      jobs.count.should eql 2
+      jobs = Job.by_time_range(base_start .. base_start)
+      jobs.count.should eql 1
+    end
+    
+    context "with an empty range" do
+      it "finds no jobs" do
+        (-1 .. 0).each do |offset|
+          jobs = Job.by_time_range(base_start ... base_start + offset)
+          jobs.count.should eql 0
+        end
+      end
+    end
+  end
+
+  describe "#within_time_range" do
+    let(:base_start) { base_time + 1.hours + 1 }
+    let(:base_end) { base_time + 5.hours - 1 }
+
+    it "finds jobs within the range" do
+      [true, false].each do |exclude_end|
+        time_range = Range.new(base_start, base_end, exclude_end)
+        jobs = Job.within_time_range(time_range)
+        expect(jobs.count).to eql (exclude_end ? 2 : 3)
+        jobs.each do |job|
+          expect(job).to be_job_within_time_range(time_range)
+        end
+      end
+    end
+
+    context "with an empty range" do
+      it "finds no jobs" do
+        jobs = Job.within_time_range(base_start ... base_start)
+        expect(jobs.count).to eql(0)
+      end
+    end
+  end
+
+  describe "overlapping_edges" do
+    let(:base_start) { base_time }
+    let(:base_end) { base_time + 3600 }
+
+    context "with exclusive ranges" do
+      it "finds jobs that overlap the edges" do
+        #Overlapping beginning
+        jobs = Job.overlapping_edges(base_start + 1 ... base_end)
+        expect(jobs.count).to eql 1
+        expect(jobs.first.start_time).to eql base_time
+        
+        #Overlapping end
+        jobs = Job.overlapping_edges(base_start ... base_end - 1)
+        expect(jobs.count).to eql 1
+        expect(jobs.first.start_time).to eql base_time
+
+        #One job overlapping both
+        jobs = Job.overlapping_edges(base_start + 1 ... base_end - 1)
+        expect(jobs.count).to eql 1
+        expect(jobs.first.start_time).to eql base_time
+
+        #Two jobs overlapping the endpoints
+        jobs = Job.overlapping_edges(base_end - 1 ... base_end + 1) 
+        expect(jobs.count).to eql 2
+        
+        #Not overlapping any endpoints
+        jobs = Job.overlapping_edges(base_start ... base_end)
+        expect(jobs.count).to eql 0
+      end
+    end
+
+    context "with inclusive ranges" do
+      it "finds jobs that overlap the edges" do
+        #This is more pared-down because inclusive and exclusive ranges mostly
+        #share the same codepath.
+        jobs = Job.overlapping_edges(base_start .. base_end)
+        expect(jobs.count).to eql 1
+
+        jobs = Job.overlapping_edges(base_start .. base_start)
+        expect(jobs.count).to eql 1
+      end
+    end
+
+    context "with empty ranges" do
+      it "finds no jobs" do
+        jobs = Job.overlapping_edges(base_start + 1 ... base_start + 1)
+        expect(jobs.count).to eql 0
+        jobs = Job.overlapping_edges(base_start + 1 .. base_start)
+        expect(jobs.count).to eql 0
+      end
+    end
+  end
+   
+  describe "#summary" do
+    let(:count) { Job.count }
+    let(:summary) { create_summaries(Job, base_time) }
+
+    #TODO: test the case where a job extends on both sides of the summary range?
+    it "produces correct summary totals" do
+      expect(summary[:all]).to eql({
+        :num_jobs => count,
+        :successful => 20,
+        :cpu_time => count * 100,
+        :memory_time => count * 200 * 1.hour,
+      })
+
+      expect(summary[:all_constrained]).to eql(summary[:all])
+      expect(summary[:wide]).to eql(summary[:all])
+
+      expect(summary[:all_filtered]).to eql({
+        :num_jobs => count / 2,
+        :successful => 20,
+        :cpu_time => count * 100 / 2,
+        :memory_time => count * 100 * 1.hour,
+      })
+
+      num_clipped_jobs = summary[:clipped][:num_jobs]
+      expect(summary[:clipped]).to eql({
+        :num_jobs => 25,
+        :cpu_time => num_clipped_jobs * 100 - 50,
+        #TODO: this seems off. Why?
+        :memory_time => num_clipped_jobs * 200 * 3600 - 100 * 3600,
+        :successful => num_clipped_jobs / 2 + 1,
+      })
+    end
+    
+    it "correctly handles summaries of empty sets" do
+      summary[:empty].should eql({
+          :num_jobs => 0,
+          :cpu_time => 0,
+          :memory_time => 0,
+          :successful => 0,
+        })
+    end
+    
+    it "correctly handles inverted ranges" do
+      Job.summary(Time.now() ... Time.now() - 1).should eql summary[:empty]
+      Job.summary(Time.now() .. Time.now() - 1).should eql summary[:empty]
+    end
+
+    it "distinguishes between inclusive and exclusive ranges" do
+      sum = Job.summary(base_time ... base_time + 3600)
+      sum[:num_jobs].should eql 1
+      sum = Job.summary(base_time .. base_time + 3600)
+      sum[:num_jobs].should eql 2
+    end
+  end
+  
+  it "validates fields" do
+    fields = {
+      :user => User.first,
+      :system => System.first,
+      :command_name => '',
+      :cpu_time => 100,
+      :start_time => base_time,
+      :wall_time => 1000,
+      :memory => 10000,
+      :exit_code => 0
+    }
+    
+    job = Job.new(fields)
+    job.valid?.should eql true
+    
+    fields.each_key do |field|
+      job = Job.new(fields)
+      job.method("#{field}=".intern).call(nil)
+      job.valid?.should eql false
+    end
+    
+    [:cpu_time, :wall_time, :memory].each do |field|
+      job = Job.new(fields)
+      m = job.method("#{field}=".intern)
+      m.call(-1)
+      job.valid?.should eql false
+      m.call(0)
+      job.valid?.should eql true
+    end
+  end
+end
+