bookie_accounting 0.0.1

data/lib/bookie/formatter.rb

@@ -0,0 +1,149 @@
+require 'bookie/database'
+
+require 'date'
+require 'spreadsheet'
+
+module Bookie
+  ##
+  #Takes jobs from the database and creates summaries and tables in various output formats.
+  class Formatter
+    ##
+    #Creates a new Formatter object
+    #
+    #<tt>type</tt> should be a symbol that maps to one of the files in <tt>bookie/formatters</tt>.
+    #
+    #===Examples
+    #  config = Bookie::Config.new('config.json')
+    #  #Uses the spreadsheet formatter from 'bookie/formatters/spreadsheet'
+    #  formatter = Bookie::Formatter::Formatter.new(config, :spreadsheet)
+    def initialize(type, filename = nil)
+      #Needed for symbol arguments
+      type = type.to_s
+      require "bookie/formatters/#{type}"
+      extend Bookie::Formatters.const_get(type.camelize)
+      self.open(filename)
+    end
+    
+    ##
+    #An array containing the labels for each field in a summary
+    SUMMARY_FIELD_LABELS = [
+        "Number of jobs",
+        "Total wall time",
+        "Total CPU time",
+        "Successful",
+        "Available CPU time",
+        "CPU time used",
+        "Available memory (average)",
+        "Memory used (average)",
+      ]
+      
+      ##
+      #An array containing the labels for each field in a details table
+      DETAILS_FIELD_LABELS = [
+        'User', 'Group', 'System', 'System type', 'Start time', 'End time', 'Wall time',
+        'CPU time', 'Memory usage', 'Exit code'
+      ]
+    
+    ##
+    #Prints a summary of <tt>jobs</tt> and <tt>systems</tt> to <tt>io</tt>
+    #
+    #Use start_time and end_time to filter the jobs by a time range.
+    #
+    #It is probably not a good idea to apply any time-based filters to <tt>jobs</tt> or <tt>systems</tt> beforehand.
+    #
+    #Both <tt>jobs</tt> and <tt>systems</tt> should be either models or ActiveRecord::Relation objects.
+    #
+    #Returns the summaries for <tt>jobs</tt> and <tt>systems</tt>
+    def print_summary(jobs, systems, start_time = nil, end_time = nil)
+      jobs_summary = jobs.summary(start_time, end_time)
+      systems_summary = systems.summary(start_time, end_time)
+      cpu_time = jobs_summary[:cpu_time]
+      avail_cpu_time = systems_summary[:avail_cpu_time]
+      memory_time = jobs_summary[:memory_time]
+      avail_memory_time = systems_summary[:avail_memory_time]
+      field_values = [
+        jobs_summary[:jobs].length,
+        Formatter.format_duration(jobs_summary[:wall_time]),
+        Formatter.format_duration(cpu_time),
+        '%.4f%%' % (jobs_summary[:successful] * 100),
+        Formatter.format_duration(systems_summary[:avail_cpu_time]),
+        if avail_cpu_time == 0 then '0.0000%' else '%.4f%%' % (Float(cpu_time) / avail_cpu_time * 100) end,
+        "#{Integer(systems_summary[:avail_memory_avg])} kb",
+        if avail_memory_time == 0 then '0.0000%' else '%.4f%%' % (Float(memory_time) / avail_memory_time * 100) end
+      ]
+      do_print_summary(field_values)
+      return jobs_summary, systems_summary
+    end
+    
+    ##
+    #Prints a table containing all details of <tt>jobs</tt>
+    #
+    #<tt>jobs</tt> should be an array.
+    def print_jobs(jobs)
+      do_print_jobs(jobs)
+    end
+    
+    ##
+    #Flushes all output
+    #
+    #Should always be called after the desired information has been written
+    def flush()
+      do_flush() if self.respond_to?(:do_flush)
+    end
+    
+    ##
+    #For each job, yields an array containing the field values to be used when printing a table of jobs
+    #
+    #call-seq:
+    #  fields_for_each_job(jobs) { |fields| ... }
+    #
+    #<tt>jobs</tt> should be an array of Bookie::Database::Job objects.
+    #
+    #===Examples
+    #  formatter.fields_for_each_job(jobs) do |fields|
+    #    Bookie::Formatter::Formatter::DETAILS_FIELD_LABELS.zip(fields) do |label, field|
+    #      puts "#{label}: #{field}"
+    #    end
+    #  end
+    def fields_for_each_job(jobs)
+      jobs.each do |job|
+        memory_stat_type = job.system.system_type.memory_stat_type
+        if memory_stat_type == :unknown
+          memory_stat_type = ''
+        else
+          memory_stat_type = " (#{memory_stat_type})"
+        end
+        yield [
+          job.user.name,
+          job.user.group.name,
+          job.system.name,
+          job.system.system_type.name,
+          job.start_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
+          job.end_time.getlocal.strftime('%Y-%m-%d %H:%M:%S'),
+          Formatter.format_duration(job.end_time - job.start_time),
+          Formatter.format_duration(job.cpu_time),
+          "#{job.memory}kb#{memory_stat_type}",
+          job.exit_code
+        ]
+      end
+    end
+    protected :fields_for_each_job
+    
+    ##
+    #Formats a duration in HH:MM:SS format
+    #
+    #<tt>dur</tt> should be a number in seconds.
+    def self.format_duration(dur)
+      dur = Integer(dur)
+      hours = dur / 3600
+      minutes = (dur - hours * 3600) / 60
+      seconds = dur % 60
+      return "#{hours.to_s.rjust(2, '0')}:#{minutes.to_s.rjust(2, '0')[0 .. 1]}:#{seconds.to_s.rjust(2, '0')[0 .. 1]}"
+    end
+  end
+  
+  #Contains all formatter plugins
+  module Formatters
+    
+  end
+end

data/lib/bookie/formatters/comma_dump.rb

@@ -0,0 +1,24 @@
+module Bookie
+  module Formatters
+    ##
+    #Formats data as a CSV file
+    module CommaDump
+      def open(filename)
+        @file = File.open(filename)
+      end
+      
+      def do_print_summary(field_values)
+        Formatter::SUMMARY_FIELD_LABELS.zip(field_values) do |label, value|
+          @file.puts "#{label}, #{value}"
+        end
+      end
+      
+      def do_print_jobs(jobs)
+        @file.puts Formatter::DETAILS_FIELD_LABELS.join(', ')
+        fields_for_each_job(jobs) do |fields|
+          @file.puts fields.join(', ')
+        end
+      end
+    end
+  end
+end

data/lib/bookie/formatters/spreadsheet.rb

@@ -0,0 +1,45 @@
+require 'spreadsheet'
+
+module Bookie
+  module Formatters
+    ##
+    #Formats data as a Microsoft Excel spreadsheet
+    module Spreadsheet
+      def open(filename)
+        @filename = filename
+        @workbook = ::Spreadsheet::Workbook.new
+      end
+      
+      #Actual printing is delayed until object finalization due to the workings of the Spreadsheet gem.
+      def do_print_summary(field_values)
+        s = @workbook.worksheet("Summary") || @workbook.create_worksheet(:name => "Summary")
+        
+        s.column(0).width = 20
+        Formatter::SUMMARY_FIELD_LABELS.each_with_index do |value, index|
+          row = s.row(index) 
+          row[0] = value
+          row[1] = field_values[index]
+        end
+      end
+      
+      def do_print_jobs(jobs)
+        s = @workbook.worksheet("Details") || @workbook.create_worksheet(:name => "Details")
+        
+        s.row(0).concat(Formatter::DETAILS_FIELD_LABELS)
+        (0 .. (Formatter::DETAILS_FIELD_LABELS.length - 1)).step do |i|
+          s.column(i).width = 20
+        end
+        
+        index = 1
+        fields_for_each_job(jobs) do |fields|
+          s.row(index).concat(fields)
+          index += 1
+        end
+      end
+      
+      def do_flush()
+        @workbook.write(@filename)
+      end
+    end
+  end
+end

data/lib/bookie/formatters/stdout.rb

@@ -0,0 +1,32 @@
+module Bookie
+  module Formatters
+    ##
+    #Formats data in a human-readable text format intended to be send to standard output
+    module Stdout
+      def open(filename)
+        if filename
+          @io = File.open(filename)
+        else
+          @io = STDOUT
+        end
+      end
+      
+      def do_print_summary(field_values)
+        Formatter::SUMMARY_FIELD_LABELS.zip(field_values) do |label, value|
+          @io.printf("%-30.30s%s\n", "#{label}:", value)
+        end
+      end
+      
+      def do_print_jobs(jobs)
+        #To consider: optimize by moving out of the function?
+        format_string = "%-15.15s %-15.15s %-20.20s %-20.20s %-26.25s %-26.25s %-12.10s %-12.10s %-20.20s %-11.11s\n"
+        heading = sprintf(format_string, *Formatter::DETAILS_FIELD_LABELS)
+        @io.write heading
+        @io.puts '-' * (heading.length - 1)
+        fields_for_each_job(jobs) do |fields|
+          @io.printf(format_string, *fields)
+        end
+      end
+    end
+  end
+end

data/lib/bookie/sender.rb

@@ -0,0 +1,108 @@
+require 'bookie/database'
+
+require 'date'
+
+module Bookie
+  ##
+  #An object that sends data to the database
+  class Sender
+    ##
+    #Creates a new Sender
+    #
+    #<tt>config</tt> should be an instance of Bookie::Config.
+    def initialize(config)
+      @config = config
+      t = @config.system_type
+      require "bookie/senders/#{t}"
+      extend Bookie::Senders.const_get(t.camelize)
+    end
+    
+    ##
+    #Retrieves the System object with which the jobs will be associated
+    #--
+    #To consider: caching?
+    #++
+    def system
+      hostname = @config.hostname
+      system_type = self.system_type
+      Bookie::Database::System.find_active(
+        :name => hostname,
+        :system_type => system_type,
+        :start_time => Time.now,
+        :cores => @config.cores,
+        :memory => @config.memory
+      )
+    end
+    
+    ##
+    #Sends job data from the given file to the database server
+    def send_data(filename)
+      raise IOError.new("File '#{filename}' does not exist.") unless File.exists?(filename)
+    
+      system = self.system
+      
+      known_users = {}
+      known_groups = {}
+      
+      #Check the first job to see if there are entries in the database for its date from this system.
+      each_job(filename) do |job|
+        next if filtered?(job)
+        end_time = job.start_time + job.wall_time
+        duplicate = system.jobs.find_by_end_time(end_time)
+        if duplicate
+          raise "Jobs already exist in the database for the date #{end_time.strftime('%Y-%m-%d')}."
+        end
+        break
+      end
+      
+      each_job(filename) do |job|
+        next if filtered?(job)
+        db_job = job.to_model
+        #Determine if the user/group pair must be added to/retrieved from the database.
+        user = Bookie::Database::User.find_or_create!(
+          job.user_name,
+          Bookie::Database::Group.find_or_create!(job.group_name, known_groups),
+          known_users)
+        db_job.system = system
+        db_job.user = user
+        db_job.save!
+      end
+    end
+    
+    ##
+    #The name of the Bookie::Database::SystemType that systems using this sender will have
+    def system_type
+      Bookie::Database::SystemType.find_or_create!(system_type_name, memory_stat_type)
+    end
+    
+    ##
+    #Returns whether a job should be filtered from the results
+    #
+    def filtered?(job)
+      @config.excluded_users.include?job.user_name
+    end
+  end
+  
+  ##
+  #This module is mixed into various job classes used internally by senders.
+  module ModelHelpers
+    ##
+    #Converts the object to a Bookie::Database::Job
+    def to_model()
+      job = Bookie::Database::Job.new
+      job.command_name = self.command_name
+      job.start_time = self.start_time
+      job.end_time = self.start_time + self.wall_time
+      job.wall_time = self.wall_time
+      job.cpu_time = self.cpu_time
+      job.memory = self.memory
+      job.exit_code = self.exit_code
+      return job
+    end
+  end
+  
+  #Contains all sender plugins
+  module Senders
+    
+  end
+end

data/lib/bookie/senders/standalone.rb

@@ -0,0 +1,37 @@
+require 'fileutils'
+require 'pacct'
+
+module Bookie
+  module Senders
+    ##
+    #Returns data from a standalone Linux system
+    module Standalone
+      ##
+      #Yields each job in the log
+      def each_job(filename)
+        file = Pacct::Log.new(filename)
+        file.each_entry do |job|
+          yield job
+        end
+      end
+      
+      def system_type_name
+        "Standalone"
+      end
+      
+      def memory_stat_type
+        :avg
+      end
+    end
+  end
+end
+
+##
+#Originates from the <tt>pacct</tt> gem
+module Pacct
+  ##
+  #Originates from the <tt>pacct</tt> gem; redefined here to include Bookie::Sender::ModelHelpers
+  class Entry
+    include Bookie::ModelHelpers
+  end
+end

data/lib/bookie/senders/torque_cluster.rb

@@ -0,0 +1,166 @@
+
+module Bookie
+  module Senders
+    ##
+    #Returns data from a TORQUE cluster log
+    module TorqueCluster
+      #Yields each job in the log
+      def each_job(filename)
+        record = Torque::JobLog.new(filename)
+        record.each_job do |job|
+          yield job
+        end
+      end
+      
+      def system_type_name
+        return "TORQUE cluster"
+      end
+      
+      def memory_stat_type
+        return :max
+      end
+    end
+  end
+end
+
+##
+#Contains tools for working with TORQUE data
+module Torque
+  #Represents a completed job
+  class Job
+    #The name of the user who created the job
+    attr_accessor :user_name
+    #The group name of the user who created the job
+    attr_accessor :group_name
+    #The name of the command
+    attr_accessor :command_name
+    #The job's start time
+    attr_accessor :start_time
+    #The job's total wall time
+    attr_accessor :wall_time
+    #The job's total CPU time
+    attr_accessor :cpu_time
+    #The job's maximum physical memory usage in kilobytes
+    attr_accessor :physical_memory
+    #The job's maximum virtual memory usage in kilobytes
+    attr_accessor :virtual_memory
+    #The job's exit code
+    attr_accessor :exit_code
+    
+    #Returns the job's total maximum memory usage in kilobytes
+    def memory
+      physical_memory + virtual_memory
+    end
+  end
+  
+  #Represents a job record file
+  class JobLog
+    #The name of the accounting file opened
+    attr_reader :filename
+  
+    #Creates a JobRecord using the TORQUE record file for the given date
+    def initialize(filename)
+      @filename = filename
+      @file = File.open(filename)
+    end
+    
+    ##
+    #Raised when a line in the file is invalid
+    class InvalidLineError < RuntimeError
+      def initialize(filename, line_num)
+        super("Line #{line_num} of file '#{filename}' is invalid.")
+      end
+    end
+    
+    ##
+    #Yields each completed job to the given block
+    def each_job
+      @file.rewind
+      line_num = 0
+      @file.each_line do |line|
+        line_num += 1
+        next if line.strip! == ''
+        #Skip the timestamp.
+        index = line.index(';')
+        raise invalid_line_error(line_num) unless index
+        
+        #Find the event type.
+        event_type = line[index + 1]
+        old_index = index
+        index = line.index(';', index + 1)
+        raise invalid_line_error(line_num) unless index == old_index + 2
+        next unless event_type == ?E
+        
+        #Find the fields.
+        index = line.index(';', index + 1)
+        raise invalid_line_error(line_num) unless index
+        fields = line[index + 1 .. -1].split(' ')
+        
+        job = Job.new()
+        
+        #To consider: make sure all fields are present?
+        fields.each do |field|
+          key, value = *field.split('=')
+          case key
+            when "user"
+              job.user_name = value
+            when "group"
+              job.group_name = value
+            when "start"
+              job.start_time = Time.at(Integer(value))
+            when "resources_used.walltime"
+              job.wall_time = parse_duration(value)
+            when "resources_used.cput"
+              job.cpu_time = parse_duration(value)
+            when "resources_used.mem"
+              job.physical_memory = Integer(value[0 ... -2])
+            when "resources_used.vmem"
+              job.virtual_memory = Integer(value[0 ... -2])
+            when "Exit_status"
+              job.exit_code = Integer(value)
+          end
+        end
+        job.command_name = ""
+        
+        yield job
+      end
+    end
+    
+    ##
+    #Creates an InvalidLineError associated with this object's file
+    def invalid_line_error(line_num)
+      InvalidLineError.new(@filename, line_num)
+    end
+    protected :invalid_line_error
+    
+    ##
+    #Parses a duration in HH:MM:SS format, returning seconds
+    #--
+    #To consider: make class method?
+    #++
+    def parse_duration(str)
+      hours, minutes, seconds = *str.split(':').map!{ |s| Integer(s) }
+      return hours * 3600 + minutes * 60 + seconds
+    end
+    protected :parse_duration
+    
+    ##
+    #Converts a date to the name of the file holding entries for that date
+    def self.filename_for_date(date)
+      File.join(Torque::torque_root, 'server_priv', 'accounting', date.strftime("%Y%m%d"))
+    end
+  end
+  
+  class << self;
+    #The TORQUE root directory (usually the value of the environment variable TORQUEROOT)
+    attr_accessor :torque_root
+  end
+  #To consider: make class variable? Constant?
+  @torque_root = ENV['TORQUEROOT'] || '/var/spool/torque'
+end
+
+module Torque
+  class Job
+    include Bookie::ModelHelpers
+  end
+end