bookie_accounting 1.0.0 → 1.1.0

data/lib/bookie/database.rb

@@ -1,11 +1,11 @@
 require 'bookie/config'
+require 'bookie/extensions'
 
 require 'active_record'
 
 module Bookie
-  #Contains ActiveRecord structures for the central database
-  #
-  #For a list of fields in the various models, see {Database Tables}[link:rdoc/database_rdoc.html]
+  ##
+  #Contains database-related code and models
   module Database
   
     ##
@@ -57,6 +57,8 @@ module Bookie
       def end_time
         return start_time + wall_time
       end
+
+      #To consider: disable #end_time= ?
       
       def self.by_user(user)
         where('jobs.user_id = ?', user.id)
@@ -83,7 +85,7 @@ module Bookie
       def self.by_group_name(group_name)
         group = Group.find_by_name(group_name)
         return joins(:user).where('users.group_id = ?', group.id) if group
-        limit(0)
+        where('1=0')
       end
       
       ##
@@ -113,7 +115,13 @@ module Bookie
       ##
       #Finds all jobs whose running intervals overlap the given time range
       def self.by_time_range_inclusive(time_range)
-        where('? <= jobs.end_time AND jobs.start_time < ?', time_range.first, time_range.last)
+        if time_range.empty?
+          where('1=0')
+        elsif time_range.exclude_end?
+          where('? <= jobs.end_time AND jobs.start_time < ?', time_range.first, time_range.last)
+        else
+          where('? <= jobs.end_time AND jobs.start_time <= ?', time_range.first, time_range.last)
+        end
       end
       
       ##
@@ -125,12 +133,14 @@ module Bookie
       #- <tt>:memory_time</tt>: the sum of memory * wall_time for all jobs in the interval
       #- <tt>:successful</tt>: the number of jobs that have completed successfully
       #
-      #This method should probably not be used with other queries that filter by start/end time.
+      #This method should probably not be chained with other queries that filter by start/end time.
+      #
+      #To consider: filter out jobs with 0 CPU time?
       def self.summary(time_range = nil)
         time_range = time_range.normalized if time_range
         jobs = self
         jobs = jobs.by_time_range_inclusive(time_range) if time_range
-        jobs = jobs.where('jobs.cpu_time > 0').all_with_relations
+        jobs = jobs.all_with_relations
         cpu_time = 0
         successful_jobs = 0
         memory_time = 0
@@ -151,10 +161,7 @@ module Bookie
             #To consider: what should I do about jobs that only report a max memory value?
             memory_time += job.memory * clipped_wall_time
           end
-          #Only count the job as successful if it's actually finished by the end of the summary.
-          if job.exit_code == 0 && (!time_range || job.end_time < time_range.end) then
-            successful_jobs += 1
-          end
+          successful_jobs += 1 if job.exit_code == 0
         end
       
         return {
@@ -171,40 +178,38 @@ module Bookie
       #Relations are not cached between calls.
       def self.all_with_relations
         jobs = all
-        transaction do
-          users = {}
-          groups = {}
-          systems = {}
-          system_types = {}
-          jobs.each do |job|
-            system = systems[job.system_id]
-            if system
-              job.system = system
-            else
-              system = job.system
-              systems[system.id] = system
-            end
-            system_type = system_types[system.system_type_id]
-            if system_type
-              system.system_type = system_type
-            else
-              system_type = system.system_type
-              system_types[system_type.id] = system_type
-            end
-            user = users[job.user_id]
-            if user
-              job.user = user
-            else
-              user = job.user
-              users[user.id] = user
-            end
-            group = groups[user.group_id]
-            if group
-              user.group = group
-            else
-              group = user.group
-              groups[group.id] = group
-            end
+        users = {}
+        groups = {}
+        systems = {}
+        system_types = {}
+        jobs.each do |job|
+          system = systems[job.system_id]
+          if system
+            job.system = system
+          else
+            system = job.system
+            systems[system.id] = system
+          end
+          system_type = system_types[system.system_type_id]
+          if system_type
+            system.system_type = system_type
+          else
+            system_type = system.system_type
+            system_types[system_type.id] = system_type
+          end
+          user = users[job.user_id]
+          if user
+            job.user = user
+          else
+            user = job.user
+            users[user.id] = user
+          end
+          group = groups[user.group_id]
+          if group
+            user.group = group
+          else
+            group = user.group
+            groups[group.id] = group
           end
         end
         
@@ -231,50 +236,89 @@ module Bookie
       end
     end
     
+    ##
+    #A cached summary of Jobs in the database
+    #
+    #Most summary operations should be performed through this class to improve efficiency.
     class JobSummary < ActiveRecord::Base
       self.table_name = :job_summaries
     
       belongs_to :user
       belongs_to :system
+
+      attr_accessible :date, :user, :user_id, :system, :system_id, :command_name, :cpu_time, :memory_time
       
+      ##
+      #Filters by date
       def self.by_date(date)
         where('job_summaries.date = ?', date)
       end
+
+      ##
+      #Filters by a date range
+      def self.by_date_range(range)
+        range = range.normalized
+        if range.exclude_end?
+          where('? <= job_summaries.date AND job_summaries.date < ?', range.begin, range.end)
+        else
+          where('? <= job_summaries.date AND job_summaries.date <= ?', range.begin, range.end)
+        end
+      end
       
+      ##
+      #Filters by user
       def self.by_user(user)
         where('job_summaries.user_id = ?', user.id)
       end
       
+      ##
+      #Filters by user name
       def self.by_user_name(name)
         joins(:user).where('users.name = ?', name)
       end
       
+      ##
+      #Filters by group
       def self.by_group(group)
         joins(:user).where('users.group_id = ?', group.id)
       end
       
+      ##
+      #Filters by group name
       def self.by_group_name(name)
         group = Group.find_by_name(name)
         return by_group(group) if group
-        limit(0)
+        where('1=0')
       end
       
+      ##
+      #Filters by system
       def self.by_system(system)
         where('job_summaries.system_id = ?', system.id)
       end
       
+      ##
+      #Filters by system name
       def self.by_system_name(name)
         joins(:system).where('systems.name = ?', name)
       end
       
+      ##
+      #Filters by system type
       def self.by_system_type(type)
         joins(:system).where('systems.system_type_id = ?', type.id)
       end
       
+      ##
+      #Filters by command name
       def self.by_command_name(cmd)
         where('job_summaries.command_name = ?', cmd)
       end
       
+      ##
+      #Attempts to find a JobSummary with the given date, user_id, system_id, and command_name
+      #
+      #If one does not exist, a new JobSummary will be instantiated (but not saved to the database).
       def self.find_or_new(date, user_id, system_id, command_name)
         str = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).to_sql
         summary = by_date(date).where(:user_id => user_id, :system_id => system_id).by_command_name(command_name).first
@@ -287,27 +331,62 @@ module Bookie
         summary
       end
       
+      ##
+      #Create cached summaries for the given date
+      #
+      #The date is interpreted as being in UTC.
+      #
+      #If there is nothing to summarize, a dummy summary will be created.
+      #
+      #Uses Lock::synchronize internally; should not be used in transaction blocks
       def self.summarize(date)
         jobs = Job
         unscoped = self.unscoped
-        time_range = date.to_time ... (date + 1).to_time
+        time_min = date.to_utc_time 
+        time_range = time_min ... time_min + 1.days
         day_jobs = jobs.by_time_range_inclusive(time_range)
+
+        #Find the sets of unique values.
         value_sets = day_jobs.select('user_id, system_id, command_name').uniq
-        value_sets.each do |set|
-          summary_jobs = jobs.where(:user_id => set.user_id).where(:system_id => set.system_id).by_command_name(set.command_name)
-          summary = summary_jobs.summary(time_range)
+        if value_sets.empty?
+          user = User.select(:id).first
+          system = System.select(:id).first
+          #If there are no users or no systems, we can't create the dummy summary, so just return.
+          return unless user && system
+          #Create a dummy summary so summary() doesn't keep trying to create one.
           Lock[:job_summaries].synchronize do
-            sum = unscoped.find_or_new(date, set.user_id, set.system_id, set.command_name)
-            #To consider: do we even need this field? (Time-range summaries don't use it because of overlap issues.)
-            sum.num_jobs = summary[:jobs].length
-            sum.cpu_time = summary[:cpu_time]
-            sum.memory_time = summary[:memory_time]
-            sum.successful = summary[:successful]
+            sum = unscoped.find_or_new(date, user.id, system.id, '')
+            sum.cpu_time = 0
+            sum.memory_time = 0
             sum.save!
           end
+        else
+          value_sets.each do |set|
+            summary_jobs = jobs.where(:user_id => set.user_id).where(:system_id => set.system_id).by_command_name(set.command_name)
+            summary = summary_jobs.summary(time_range)
+            Lock[:job_summaries].synchronize do
+              sum = unscoped.find_or_new(date, set.user_id, set.system_id, set.command_name)
+              sum.cpu_time = summary[:cpu_time]
+              sum.memory_time = summary[:memory_time]
+              sum.save!
+            end
+          end
         end
       end
       
+      ##
+      #Returns a summary of jobs in the database
+      #
+      #The following options are supported:
+      #- [<tt>:range</tt>] restricts the summary to a specific time interval (specified as a Range of Time objects)
+      #- [<tt>:jobs</tt>] the jobs on which the summary should operate
+      #
+      #Internally, this may call JobSummary::summary, which uses Lock#synchronize, so this should not be used inside a transaction block.
+      #
+      #When filtering, the same filters must be applied to both the Jobs and the JobSummaries. For example:
+      # jobs = Bookie::Database::Job.by_user_name('root')
+      # summaries = Bookie::Database::Job.by_user_name('root')
+      # puts summaries.summary(:jobs => jobs)
       def self.summary(opts = {})
         jobs = opts[:jobs] || Job
         range = opts[:range]
@@ -323,7 +402,7 @@ module Bookie
             first_started_system = System.order(:start_time).first
             range = first_started_system.start_time ... end_time
           else
-            range = Date.new ... Date.new
+            range = Time.new ... Time.new
           end
         end
         range = range.normalized
@@ -333,54 +412,66 @@ module Bookie
         memory_time = 0
         successful = 0
         
-        #Could there be partial days at the beginning/end?
-        date_range = range
-        unless range.begin.kind_of?(Date) && range.end.kind_of?(Date)
-          date_begin = range.begin.to_date
-          unless date_begin.to_time == range.begin
-            date_begin += 1
-            time_before_max = [date_begin.to_time, range.end].min
-            time_before_min = range.begin
-            summary = jobs.summary(time_before_min ... time_before_max)
+        #Is the beginning somewhere between days?
+        date_begin = range.begin.utc.to_date
+        unless date_begin.to_utc_time == range.begin
+          date_begin += 1
+          time_before_max = [date_begin.to_utc_time, range.end].min
+          time_before_min = range.begin
+          summary = jobs.summary(time_before_min ... time_before_max)
+          cpu_time += summary[:cpu_time]
+          memory_time += summary[:memory_time]
+        end
+
+        #Is the end somewhere between days?
+        date_end = range.end.utc.to_date
+        time_after_min = date_end.to_utc_time
+        unless time_after_min <= range.begin
+          time_after_max = range.end
+          time_after_range = Range.new(time_after_min, time_after_max, range.exclude_end?)
+          unless time_after_range.empty?
+            summary = jobs.summary(time_after_range)
             cpu_time += summary[:cpu_time]
             memory_time += summary[:memory_time]
-            successful += summary[:successful]
           end
-
-          date_end = range.end.to_date
-          time_after_min = date_end.to_time
-          unless time_after_min <= range.begin
-            time_after_max = range.end
-            time_after_range = Range.new(time_after_min, time_after_max, range.exclude_end?)
-            unless time_after_range.empty?
-              summary = jobs.summary(time_after_range)
-              cpu_time += summary[:cpu_time]
-              memory_time += summary[:memory_time]
-              successful += summary[:successful]
-            end
-          end
-          
-          date_range = date_begin ... date_end
         end
         
+        date_range = date_begin ... date_end
+        
         unscoped = self.unscoped
-        date = date_range.begin
-        while date_range.cover?(date) do
-          #To do: what if there aren't any summaries to be made? Will we just run summarize() each time?
-          summarize(date) if unscoped.by_date(date).empty?
-          summaries = by_date(date)
-          summaries.all.each do |summary|
-            cpu_time += summary.cpu_time
-            memory_time += summary.memory_time
-            successful += summary.successful
-          end
-          date += 1
+        summaries = by_date_range(date_range).order(:date).all
+        index = 0
+        date_range.each do |date|
+          new_index = index
+          sum = summaries[new_index]
+          while sum && sum.date == date do
+            cpu_time += sum.cpu_time
+            memory_time += sum.memory_time
+            new_index += 1
+            sum = summaries[new_index]
+          end
+          #Did we actually process any summaries?
+          if new_index == index
+            #Nope. Create the summaries.
+            #To consider: optimize out the query?
+            unscoped.summarize(date)
+            sums = by_date(date)
+            sums.each do |sum|
+              cpu_time += sum.cpu_time
+              memory_time += sum.memory_time
+            end
+          end
         end
         
-        time_range = Range.new(range.begin.to_time, range.end.to_time, range.exclude_end?)
-        jobs = jobs.by_time_range_inclusive(time_range)
-        num_jobs = (range && range.empty?) ? 0 : jobs.count
-        
+        if range && range.empty?
+          num_jobs = 0
+          successful = 0
+        else
+          jobs = jobs.by_time_range_inclusive(range)
+          num_jobs = jobs.count
+          successful = jobs.where('jobs.exit_code = 0').count
+        end
+
         {
           :num_jobs => num_jobs,
           :cpu_time => cpu_time,
@@ -389,19 +480,19 @@ module Bookie
         }
       end
       
-      validates_presence_of :user_id, :system_id, :date, :num_jobs, :cpu_time, :memory_time, :successful
+      validates_presence_of :user_id, :system_id, :date, :cpu_time, :memory_time
       
       validates_each :command_name do |record, attr, value|
         record.errors.add(attr, 'must not be nil') if value == nil
       end
       
-      validates_each :num_jobs, :cpu_time, :memory_time, :successful do |record, attr, value|
+      validates_each :cpu_time, :memory_time do |record, attr, value|
         record.errors.add(attr, 'must be a non-negative integer') unless value && value >= 0
       end
     end
     
     ##
-    #A group
+    #A group of users
     class Group < ActiveRecord::Base
       has_many :users
       
@@ -488,13 +579,26 @@ module Bookie
         where('systems.system_type_id = ?', sys_type.id)
       end
       
+      ##
+      #Finds all systems whose running intervals overlap the given time range
+      #
+      #To do: unit test.
+      def self.by_time_range_inclusive(time_range)
+        if time_range.empty?
+          where('1=0')
+        elsif time_range.exclude_end?
+          where('(? <= systems.end_time OR systems.end_time IS NULL) AND systems.start_time < ?', time_range.first, time_range.last)
+        else
+          where('(? <= systems.end_time OR systems.end_time IS NULL) AND systems.start_time <= ?', time_range.first, time_range.last)
+        end
+      end
+
       ##
       #Finds the current system for a given sender and time
       #
       #This method also checks that this system's specifications are the same as those in the database and raises an error if they are different.
       #
       #This uses Lock#synchronize internally, so it probably should not be called within a transaction block.
-      #
       def self.find_current(sender, time = nil)
         time ||= Time.now
         config = sender.config
@@ -516,19 +620,42 @@ Please make sure that all previous systems with this hostname have been marked a
         system
       end
       
+      ##
+      #Returns an array of all systems, pre-loading relations to reduce the need for extra queries
+      #
+      #Relations are not cached between calls.
+      def self.all_with_relations
+        systems = all
+        system_types = {}
+        systems.each do |system|
+          system_type = system_types[system.system_type_id]
+          if system_type
+            system.system_type = system_type
+          else
+            system_type = system.system_type
+            system_types[system_type.id] = system_type
+          end
+        end
+        systems
+      end
+ 
       ##
       #Produces a summary of all the systems for the given time interval
       #
       #Returns a hash with the following fields:
-      #- <tt>:avail_cpu_time</tt>: the total CPU time available for the interval
-      #- <tt>:avail_memory_time</tt>: the total amount of memory-time available (in kilobyte-seconds)
-      #- <tt>:avail_memory_avg</tt>: the average amount of memory available (in kilobytes)
+      #- [<tt>:systems</tt>] an array containing all systems that are active in the interval
+      #- [<tt>:avail_cpu_time</tt>] the total CPU time available for the interval
+      #- [<tt>:avail_memory_time</tt>] the total amount of memory-time available (in kilobyte-seconds)
+      #- [<tt>:avail_memory_avg</tt>] the average amount of memory available (in kilobytes)
       #
       #To consider: include the start/end times for the summary (especially if they aren't provided as arguments)?
       #
-      #To do: make this and other summaries operate differently on inclusive and exclusive ranges.
-      #(Current behavior is as if the range were always exclusive.)
+      #Notes:
+      #
+      #Results may be slightly off when an inclusive range is used.
+      #To consider: is this worth fixing?
       def self.summary(time_range = nil)
+        #To consider: how to handle time zones with Rails apps?
         current_time = Time.now
         #Sums that are actually returned
         avail_cpu_time = 0
@@ -537,14 +664,13 @@ Please make sure that all previous systems with this hostname have been marked a
         systems = System
         if time_range
           time_range = time_range.normalized
-          #To consider: optimize as union of queries?
-          systems = systems.where(
-            'systems.start_time < ? AND (systems.end_time IS NULL OR systems.end_time > ?)',
-            time_range.last,
-            time_range.first)
+          #To do: unit test.
+          systems = systems.by_time_range_inclusive(time_range)
         end
+
+        all_systems = systems.all_with_relations
         
-        systems.all.each do |system|
+        all_systems.each do |system|
           system_start_time = system.start_time
           system_end_time = system.end_time
           #Is there a time range constraint?
@@ -579,6 +705,7 @@ Please make sure that all previous systems with this hostname have been marked a
         end
           
         {
+          :systems => all_systems,
           :avail_cpu_time => avail_cpu_time,
           :avail_memory_time => avail_memory_time,
           :avail_memory_avg => if wall_time_range == 0 then 0.0 else Float(avail_memory_time) / wall_time_range end,
@@ -721,7 +848,6 @@ Please make sure that all previous systems with this hostname have been marked a
             t.datetime :start_time, :null => false
             t.datetime :end_time
             t.integer :cores, :null => false
-            #To consider: replace with a float? (more compact)
             t.integer :memory, :null => false, :limit => 8
           end
           change_table :systems do |t|
@@ -766,12 +892,14 @@ Please make sure that all previous systems with this hostname have been marked a
             t.integer :memory, :null => false
             t.integer :exit_code, :null => false
           end
+          #To do: more indices?
           change_table :jobs do |t|
             t.index :user_id
             t.index :system_id
             t.index :command_name
             t.index :start_time
             t.index :end_time
+            t.index :exit_code
           end
         end
       
@@ -787,13 +915,10 @@ Please make sure that all previous systems with this hostname have been marked a
             t.references :system, :null => false
             t.date :date, :null => false
             t.string :command_name, :null => false
-            t.integer :num_jobs, :null => false
             t.integer :cpu_time, :null => false
             t.integer :memory_time, :null => false
-            t.integer :successful, :null => false
           end
           change_table :job_summaries do |t|
-            #To consider: reorder for optimum efficiency?
             t.index [:date, :user_id, :system_id, :command_name], :unique => true, :name => 'identity'
             t.index :command_name
             t.index :date
@@ -857,63 +982,4 @@ Please make sure that all previous systems with this hostname have been marked a
   end
 end
 
-##
-#Reopened to add some useful methods
-class Range
-  ##
-  #If end < begin, returns an empty range (begin ... begin)
-  #Otherwise, returns the original range
-  def normalized
-    return self.begin ... self.begin if self.end < self.begin
-    self
-  end
-  
-  ##
-  #Returns the empty status of the range
-  #
-  #A range is empty if end < begin or if begin == end and exclude_end? is true.
-  def empty?
-    (self.end < self.begin) || (exclude_end? && (self.begin == self.end))
-  end
 
-#This code probably works, but we're not using it anywhere.  
-#   def intersection(other)
-#     self_n = self.normalized
-#     other = other.normalized
-#     
-#     new_begin, new_end, exclude_end = nil
-#     
-#     if self_n.cover?(other.begin)
-#       new_first = other.begin
-#     elsif other.cover?(self_n.begin)
-#       new_first = self_n.begin
-#     end
-#     
-#     return self_n.begin ... self_n.begin unless new_first
-#     
-#     if self_n.cover?(other.end)
-#       unless other.exclude_end? && other.end == self_n.begin
-#         new_end = other.end
-#         exclude_end = other.exclude_end?
-#       end
-#     elsif other.cover?(self_n.end)
-#       unless self_n.exclude_end? && self_n.end == other.begin
-#         new_end = self_n.end
-#         exclude_end = self_n.exclude_end?
-#       end
-#     end
-#     
-#     #If we still haven't found new_end, try one more case:
-#     unless new_end
-#       if self_n.end == other.end
-#         #We'll only get here if both ranges exclude their ends and have the same end.
-#         new_end = self_n.end
-#         exclude_end = true
-#       end
-#     end
-#     
-#     return self_n.begin ... self_n.begin unless new_end
-# 
-#     Range.new(new_begin, new_end, exclude_end)
-#   end
-end