bookie_accounting 1.0.0 → 1.1.0

data/lib/bookie/extensions.rb

@@ -0,0 +1,72 @@
+
+##
+#Reopened to add some useful methods
+class Range
+  ##
+  #If end < begin, returns an empty range (begin ... begin)
+  #Otherwise, returns the original range
+  def normalized
+    return self.begin ... self.begin if self.end < self.begin
+    self
+  end
+  
+  ##
+  #Returns the empty status of the range
+  #
+  #A range is empty if end < begin or if begin == end and exclude_end? is true.
+  def empty?
+    (self.end < self.begin) || (exclude_end? && (self.begin == self.end))
+  end
+
+#This code probably works, but we're not using it anywhere.  
+#   def intersection(other)
+#     self_n = self.normalized
+#     other = other.normalized
+#     
+#     new_begin, new_end, exclude_end = nil
+#     
+#     if self_n.cover?(other.begin)
+#       new_first = other.begin
+#     elsif other.cover?(self_n.begin)
+#       new_first = self_n.begin
+#     end
+#     
+#     return self_n.begin ... self_n.begin unless new_first
+#     
+#     if self_n.cover?(other.end)
+#       unless other.exclude_end? && other.end == self_n.begin
+#         new_end = other.end
+#         exclude_end = other.exclude_end?
+#       end
+#     elsif other.cover?(self_n.end)
+#       unless self_n.exclude_end? && self_n.end == other.begin
+#         new_end = self_n.end
+#         exclude_end = self_n.exclude_end?
+#       end
+#     end
+#     
+#     #If we still haven't found new_end, try one more case:
+#     unless new_end
+#       if self_n.end == other.end
+#         #We'll only get here if both ranges exclude their ends and have the same end.
+#         new_end = self_n.end
+#         exclude_end = true
+#       end
+#     end
+#     
+#     return self_n.begin ... self_n.begin unless new_end
+# 
+#     Range.new(new_begin, new_end, exclude_end)
+#   end
+end
+
+##
+#Reopened to add some useful methods
+class Date
+  ##
+  #Converts the Date to a Time, using UTC as the time zone
+  def to_utc_time
+    Time.utc(year, month, day)
+  end
+end
+

data/lib/bookie/formatter.rb

@@ -120,7 +120,7 @@ module Bookie
           job.system.system_type.name,
           job.start_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
           job.end_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
-          Formatter.format_duration(job.end_time - job.start_time),
+          Formatter.format_duration(job.end_time.to_i - job.start_time.to_i),
           Formatter.format_duration(job.cpu_time),
           "#{job.memory}kb#{memory_stat_type}",
           job.command_name,
@@ -131,15 +131,23 @@ module Bookie
     protected :fields_for_each_job
     
     ##
-    #Formats a duration in HH:MM:SS format
+    #Formats a duration in a human-readable format
     #
-    #<tt>dur</tt> should be a number in seconds.
+    #<tt>dur</tt> should be an integer representing the number of seconds.
     def self.format_duration(dur)
-      dur = Integer(dur)
+      dur = dur.to_i
+      days = dur / (3600 * 24)
+      dur -= days * (3600 * 24)
       hours = dur / 3600
-      minutes = (dur - hours * 3600) / 60
-      seconds = dur % 60
-      return "#{hours.to_s.rjust(2, '0')}:#{minutes.to_s.rjust(2, '0')[0 .. 1]}:#{seconds.to_s.rjust(2, '0')[0 .. 1]}"
+      dur -= hours * 3600
+      minutes = dur / 60
+      dur -= minutes * 60
+      seconds = dur
+
+      weeks = days / 7
+      days = days % 7
+
+      "%i week%s, %i day%s, %02i:%02i:%02i" % [weeks, weeks == 1 ? '' : 's', days, days == 1 ? '' : 's', hours, minutes, seconds]
     end
   end
   

data/lib/bookie/formatters/comma_dump.rb

@@ -9,16 +9,22 @@ module Bookie
       
       def do_print_summary(field_values)
         Formatter::SUMMARY_FIELD_LABELS.zip(field_values) do |label, value|
-          @file.puts "#{label}, #{value}"
+          @file.puts "#{CommaDump.quote(label)}, #{CommaDump.quote(value)}"
         end
       end
       
       def do_print_jobs(jobs)
         @file.puts Formatter::DETAILS_FIELD_LABELS.join(', ')
         fields_for_each_job(jobs) do |fields|
-          @file.puts fields.join(', ')
+          @file.puts fields.map{ |s| CommaDump.quote(s) }.join(', ')
         end
       end
+      
+      ##
+      #Quotes a value for use as a CSV element
+      def self.quote(val)
+        %{"#{val.to_s.gsub('"', '""')}"}
+      end
     end
   end
-end
+end

data/lib/bookie/formatters/stdout.rb

@@ -19,7 +19,7 @@ module Bookie
       
       def do_print_jobs(jobs)
         #To consider: optimize by moving out of the function?
-        format_string = "%-15.15s %-15.15s %-20.20s %-20.20s %-26.25s %-26.25s %-12.10s %-12.10s %-20.20s %-20.20s %-11.11s\n"
+        format_string = "%-15.15s %-15.15s %-20.20s %-20.20s %-26.26s %-26.26s %-30.30s %-30.30s %-20.20s %-20.20s %-11.11s\n"
         heading = sprintf(format_string, *Formatter::DETAILS_FIELD_LABELS)
         @io.write heading
         @io.puts '-' * (heading.length - 1)

data/lib/bookie/sender.rb

@@ -34,20 +34,16 @@ module Bookie
       #Grab data from the first job:  
       each_job(filename) do |job|
         next if filtered?(job)
-        end_time = job.start_time + job.wall_time
-        system = Bookie::Database::System.find_current(self, end_time)
-        duplicate = system.jobs.find_by_end_time(end_time)
-        raise "Jobs already exist in the database for '#{filename}'." if duplicate
+        system = Bookie::Database::System.find_current(self, job.end_time)
+        raise "Jobs already exist in the database for '#{filename}'." if duplicate(job, system)
         time_min = job.start_time
-        time_max = end_time
+        time_max = job.end_time
         break
       end
       
       #If there are no jobs, return.
       return unless time_min
       
-      #To do: add an option to resume an interrupted send.
-      
       #Send the job data:
       each_job(filename) do |job|
         next if filtered?(job)
@@ -55,6 +51,8 @@ module Bookie
         time_min = (model.start_time < time_min) ? model.start_time : time_min
         time_max = (model.end_time > time_max) ? model.end_time : time_max
         #To consider: handle files that don't have jobs sorted by end time?
+        #To consider: this should rarely happen in real life. Remove test?
+        #(This situation can only arise if log files from different versions of the system are concatenated before sending.)
         if system.end_time && model.end_time > system.end_time
           system = Database::System.find_current(self, model.end_time)
         end
@@ -69,17 +67,52 @@ module Bookie
       end
       
       #Clear out the summaries that would have been affected by the new data:
-      date_min = time_min.to_date
-      date_max = time_max.to_date
+      clear_summaries(time_min.to_date, time_max.to_date)
+    end
+    
+    ##
+    #Undoes a previous send operation
+    def undo_send(filename)
+      raise IOError.new("File '#{filename}' does not exist.") unless File.exists?(filename)
+      
+      system = nil
+      
+      time_min, time_max = nil
+      
+      #Grab data from the first job:  
+      each_job(filename) do |job|
+        next if filtered?(job)
+        system = Bookie::Database::System.find_current(self, job.end_time)
+        time_min = job.start_time
+        time_max = job.end_time
+        break
+      end
+      
+      return unless time_min
       
-      Database::JobSummary.by_system(system).where('date >= ? AND date <= ?', date_min, date_max).delete_all
+      each_job(filename) do |job|
+        next if filtered?(job)
+        if system.end_time && job.end_time > system.end_time
+          system = Database::System.find_current(self, job.end_time)
+        end
+        #To consider: optimize this query?
+        #(It should be possible to delete all of the jobs with end times between those of the first and last jobs of the file (exclusive),
+        #but jobs with end times matching those of the first/last jobs in the file might be from an earlier or later file, not this one.
+        #This assumes that the files all have jobs sorted by end time.
+        model = duplicate(job, system)
+        break unless model
+        time_min = (model.start_time < time_min) ? model.start_time : time_min
+        time_max = (model.end_time > time_max) ? model.end_time : time_max
+        model.delete
+      end
+      
+      clear_summaries(time_min.to_date, time_max.to_date)
     end
     
     ##
     #The name of the Bookie::Database::SystemType that systems using this sender will have
     def system_type
-      #To consider: cache?
-      Bookie::Database::SystemType.find_or_create!(system_type_name, memory_stat_type)
+      @system_type ||= Bookie::Database::SystemType.find_or_create!(system_type_name, memory_stat_type)
     end
     
     ##
@@ -88,6 +121,28 @@ module Bookie
     def filtered?(job)
       @config.excluded_users.include?job.user_name
     end
+    
+    ##
+    #Finds the first job that is a duplicate of the provided job
+    def duplicate(job, system)
+      system.jobs.where({
+          :start_time => job.start_time,
+          :wall_time => job.wall_time,
+          :command_name => job.command_name,
+          :cpu_time => job.cpu_time,
+          :memory => job.memory,
+          :exit_code => job.exit_code
+        }).by_user_name(job.user_name).by_group_name(job.group_name).first
+    end
+    
+    #Used internally by #send_data and #undo_send
+    def clear_summaries(date_min, date_max)      
+      #Since joins don't mix with DELETE statements, we have to do this the hard way.
+      systems = Database::System.by_name(@config.hostname).all
+      systems.map!{ |sys| sys.id }
+      Database::JobSummary.where('job_summaries.system_id in (?)', systems).where('date >= ? AND date <= ?', date_min, date_max).delete_all
+    end
+    private :clear_summaries
   end
   
   ##
@@ -99,17 +154,22 @@ module Bookie
       job = Bookie::Database::Job.new
       job.command_name = self.command_name
       job.start_time = self.start_time
-      job.end_time = self.start_time + self.wall_time
       job.wall_time = self.wall_time
       job.cpu_time = self.cpu_time
       job.memory = self.memory
       job.exit_code = self.exit_code
-      return job
+      job
+    end
+
+    ##
+    #Returns the end time
+    def end_time
+      start_time + wall_time
     end
   end
-  
+
   #Contains all sender plugins
   module Senders
     
   end
-end
+end

data/lib/bookie/senders/standalone.rb

@@ -30,8 +30,8 @@ end
 #Originates from the <tt>pacct</tt> gem
 module Pacct
   ##
-  #Originates from the <tt>pacct</tt> gem; redefined here to include Bookie::Sender::ModelHelpers
+  #Originates from the <tt>pacct</tt> gem; reopened here to include Bookie::Sender::ModelHelpers
   class Entry
     include Bookie::ModelHelpers
   end
-end
+end

data/lib/bookie/version.rb

@@ -1,4 +1,4 @@
 module Bookie
   #The library version
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/rpm/bookie_accounting.spec.erb

@@ -69,8 +69,10 @@ rm -rf %{buildroot}
 %doc %{gem_dir}/doc/%{gem}/*
 %dir %{gem_instdir}
 <% if has_binaries %>
+%attr(700,root,root)
 %{_bindir}/*
 <% end %>
+%attr(600,root,root)
 /etc/bookie/config.json
 %{gem_instdir}/*
 %{gem_instdir}/.*

data/spec/comma_dump_formatter_spec.rb

@@ -32,25 +32,34 @@ describe Bookie::Formatters::CommaDump do
   end
   
   it "correctly formats jobs" do
-    @formatter.print_jobs(@jobs.order(:start_time).limit(2).all)
-    @m.buf.should eql <<-eos
+    with_utc do
+      @formatter.print_jobs(@jobs.order(:start_time).limit(2).all)
+      @m.buf.should eql <<-eos
 User, Group, System, System type, Start time, End time, Wall time, CPU time, Memory usage, Command, Exit code
-root, root, test1, Standalone, 2012-01-01 00:00:00, 2012-01-01 01:00:00, 01:00:00, 00:01:40, 200kb (avg), vi, 0
-test, default, test1, Standalone, 2012-01-01 01:00:00, 2012-01-01 02:00:00, 01:00:00, 00:01:40, 200kb (avg), emacs, 1
+"root", "root", "test1", "Standalone", "2012-01-01 00:00:00", "2012-01-01 01:00:00", "0 weeks, 0 days, 01:00:00", "0 weeks, 0 days, 00:01:40", "200kb (avg)", "vi", "0"
+"test", "default", "test1", "Standalone", "2012-01-01 01:00:00", "2012-01-01 02:00:00", "0 weeks, 0 days, 01:00:00", "0 weeks, 0 days, 00:01:40", "200kb (avg)", "emacs", "1"
 eos
+    end
   end
   
   it "correctly formats summaries" do
-    Time.expects(:now).returns(Time.local(2012) + 36000 * 4).at_least_once
+    Time.expects(:now).returns(base_time + 40.hours).at_least_once
     @formatter.print_summary(@jobs, @summaries, Bookie::Database::System)
     @m.buf.should eql <<-eos
-Number of jobs, 40
-Total CPU time, 01:06:40
-Successful, 50.0000%
-Available CPU time, 140:00:00
-CPU time used, 0.7937%
-Available memory (average), 1750000 kb
-Memory used (average), 0.0114%
+"Number of jobs", "40"
+"Total CPU time", "0 weeks, 0 days, 01:06:40"
+"Successful", "50.0000%"
+"Available CPU time", "0 weeks, 5 days, 20:00:00"
+"CPU time used", "0.7937%"
+"Available memory (average)", "1750000 kb"
+"Memory used (average)", "0.0114%"
 eos
   end
+
+  it "correctly quotes values" do
+    Formatter = Bookie::Formatters::CommaDump
+    Formatter.quote("test").should eql '"test"'
+    Formatter.quote('"test"').should eql '"""test"""'
+    Formatter.quote(0).should eql '"0"'
+  end
 end