fiksu-af 1.1.0

data/lib/fiksu-af/application.rb

@@ -0,0 +1,361 @@
+require 'pg_advisory_locker'
+require 'pg_application_name'
+
+module Af
+  # Abstract superclass for implementing command line applications.
+  #
+  # Provides:
+  #   * Command line option parsing.
+  #   * Logging with Log4r.
+  #   * Pre and post option processes hooks.
+  #   * Proxy support access to application frame functionality from other classes
+  #
+  # Subclasses must implement:
+  #   * work
+  #
+  # Subclasses can implement:
+  #   * pre_work
+  #
+  class Application
+    include Af::OptionParser
+    include Af::Logging
+    include Af::Deprecated
+
+    #-----------------------------
+    # *** Command Line Options ***
+    #+++++++++++++++++++++++++++++
+
+    # A number of default command line switches and switch groups available to all
+    # subclasses.
+
+    opt_group :basic, "basic options", :priority => 0, :description => <<-DESCRIPTION
+      These are the standard options offered to all Af commands.
+    DESCRIPTION
+
+    opt_group :basic do
+      opt '?', "show this help (--?? for all)", :short => '?', :var => nil do
+        Helper.new(::Af::Application.singleton.af_opt_class_path).help(::Af::Application.singleton.usage)
+        exit 0
+      end
+      opt '??', "show help for all commands", :hidden => true, :var => nil do
+        Helper.new(::Af::Application.singleton.af_opt_class_path).help(::Af::Application.singleton.usage, true)
+        exit 0
+      end
+      opt :daemon, "run as daemon", :short => :d
+    end
+
+    opt_group :advanced, "advanced options", :priority => 100, :hidden => true, :description => <<-DESCRIPTION
+      These are advanced options offered to this programs.
+    DESCRIPTION
+
+    opt_group :debugging, "debugging options", :priority => 1000, :hidden => true, :description => <<-DESCRIPTION
+      These are options associated with debugging the internal workings of the Af::Application sysyem and ruby
+      in general.
+    DESCRIPTION
+
+    opt_group :debugging do
+      opt :gc_profiler, "enable the gc profiler"
+      opt :gc_profiler_interval_minutes, "number of minutes between dumping gc information", :default => 60, :argument_note => "MINUTES"
+    end
+
+    opt_group :logging, "logger options", :priority => 100, :hidden => true, :description => <<-DESCRIPTION
+      These are options associated with logging whose core is Log4r.
+      Logging files should be in yaml format and should probably define a logger for 'Af' and 'Process'.
+    DESCRIPTION
+
+    opt_group :logging, :target_container => Af::Logging::Configurator do
+      opt :log_configuration_files, "a list of yaml files for log4r to use as configurations", :type => :strings, :default => ["af.yml"]
+      opt :log_configuration_search_path, "directories to search for log4r files", :type => :strings, :default => ["."]
+      opt :log_configuration_section_names, "section names in yaml files for log4r configurations", :type => :strings,
+        :default => ["log4r_config"], :env => 'LOG_CONFIGURATION_SECTION_NAMES'
+      opt :log_dump_configuration, "show the log4r configuration"
+      opt :log_levels, "set log levels", :type => :hash
+      opt :log_stdout, "set logfile for stdout (when daemonized)", :type => :string
+      opt :log_stderr, "set logfile for stderr (when daemonized)", :type => :string
+      opt :log_console, "force logging to console"
+      opt :log_ignore_configuration, "ignore logging configuration files", :default => false
+    end
+
+    #-------------------
+    # *** Attributes ***
+    #+++++++++++++++++++
+
+    attr_accessor :has_errors
+
+    @@singleton = nil
+
+    #----------------------
+    # *** Class Methods ***
+    #++++++++++++++++++++++
+
+    # Instantiate and run the application.
+    #
+    # *Arguments*
+    #   - arguments - ????
+    def self.run(*arguments)
+      application = self.new._run(*arguments)
+      application._work
+    end
+
+    # Return the single allowable instance of this class.
+    #
+    # *Arguments*
+    #   * safe - defaults to false, instantiates instance if it doesn't exist
+    def self.singleton(safe = false)
+      if @@singleton.nil?
+        if safe
+          @@singleton = new
+        else
+          fail("Application @@singleton not initialized! Maybe you are using a Proxy before creating an instance? or use SafeProxy")
+        end
+      end
+      return @@singleton
+    end
+
+    # Run this application with the provided arguments that must adhere to
+    # configured command line switches.  It rewrites ARGV with these values.
+    #
+    # *Example*
+    #   instance._run("-v", "--file", "foo.log")
+    #
+    # *Arguments*
+    #   * arguments - list of command line option strings
+    #
+    def self._run(*arguments)
+      # this ARGV hack is here for test specs to add script arguments
+      ARGV[0..-1] = arguments if arguments.length > 0
+      self.new._run
+    end
+
+    #-------------------------
+    # *** Instance Methods ***
+    #+++++++++++++++++++++++++
+
+    # Run the application, fetching and parsing options from the command
+    # line.
+    #
+    # *Arguments*
+    #   * usage - string describing usage (optional)
+    #   * options - hash of options, containing ???
+    def _run
+      process_command_line_options(af_opt_class_path)
+      post_command_line_parsing
+      pre_work
+      return self
+    end
+
+    # Execute the actual work of the application upon execution.
+    #
+    # This method is used to wrap the actual run code with
+    # whatever specific code we are looking to maintain the
+    # execution context.
+    #
+    # one can imagine overlaoding this function with something
+    # call initiates a profiler or debugger
+    def _work
+      begin
+        work
+      rescue SystemExit => se
+        # we do nothing here
+        if se.status != 0
+          logger.error "exit called with error: #{se.message}"
+          logger.warn se
+          exit se.status
+        end
+      rescue Exception => e
+        # catching Exception cause some programs and libraries suck
+        logger.error "fatal error durring work: #{e.message}"
+        logger.warn e
+        @has_errors = true
+      end
+
+      if @gc_profiler
+        logger("GC::Profiler").info GC::Profiler.result
+      end
+
+      exit @has_errors ? 1 : 0
+    end
+
+    # Accessor for the af name set on the instance's class.
+    def af_name
+      return self.class.name
+    end
+
+    # override if you wish to include other class's opt/opt_group
+    def af_opt_class_path
+      return [self.class]
+    end
+
+    protected
+
+    def initialize
+      super
+      # sort of a singleton -- if forced to it really means "last instantiated and current"
+      @@singleton = self
+      set_connection_application_name(startup_database_application_name)
+      $stdout.sync = true
+      $stderr.sync = true
+      opt :log_configuration_search_path, :default => [".", Rails.root + "config/logging"]
+      opt :log_configuration_files, :default => ["af.yml", "#{af_name}.yml"]
+      opt :log_stdout, :default => Rails.root + "log/runner.log"
+      opt :log_stderr, :default => Rails.root + "log/runner-errors.log"
+    end
+
+    # Set the application name on the ActiveRecord connection. It is
+    # truncated to 64 characters.
+    #
+    # *Arguments*
+    #   * name - application name to set on the connection
+    def set_connection_application_name(name)
+      ActiveRecord::ConnectionAdapters::ConnectionPool.initialize_connection_application_name(name[0...63])
+    end
+
+    # Application name consisting of process PID and af name.
+    def startup_database_application_name
+      return "//pid=#{Process.pid}/#{af_name}"
+    end
+
+    # Accessor for the application name set on the ActiveRecord database connection.
+    def database_application_name
+      return self.class.startup_database_application_name
+    end
+
+    # Work performed by this application.  MUST be implemented by subclasses.
+    def work
+      raise NotImplemented.new("#{self.class.name}#work must be implemented to use the Application framework")
+    end
+
+    def post_command_line_parsing
+      # Iterate through all option_checks and option_selects in the class hierarchy.
+      # Create instance variables and accessor methods for each.
+      option_finder = OptionFinder.new(af_opt_class_path)
+
+      option_checks = option_finder.all_option_checks
+      option_selects = option_finder.all_option_selects
+
+      begin
+        option_checks.each do |option_check|
+          option_check.validate
+        end
+
+        option_selects.each do |option_select|
+          option_select.validate
+        end
+      rescue GetoptLong::Error, Error => e
+        opt_error e.message
+      end
+    end
+
+    # Overload to do any operations that need to be handled before work is called.
+    # Call exit if needed. Always call super.
+    def pre_work
+      logging_configurator.configurate
+
+      if @gc_profiler
+        logger("GC::Profiler").info "Enabling GC:Profilier"
+        logger("GC::Profiler").info "Signal USR1 will dump results"
+        logger("GC::Profiler").info "Will dump every #{@gc_profiler_interval_minutes} minutes"
+        GC::Profiler.enable
+        @last_gc_profiler_dump = Time.zone.now
+        Signal.trap("USR1") do
+          logger("GC::Profiler").info GC::Profiler.result
+        end
+      end
+
+      if @daemon
+        $stdout.reopen(Af::Logging::Configurator.log_stdout, "a")
+        $stderr.reopen(Af::Logging::Configurator.log_stderr, "a")
+        $stdout.sync = true
+        $stderr.sync = true
+        logger.info "Daemonizing"
+        pid = fork
+        if pid
+          exit 0
+        else
+          logger.info "forked"
+          Process.setsid
+          trap 'SIGHUP', 'IGNORE'
+          cleanup_after_fork
+        end
+      end
+    end
+
+    def cleanup_after_fork
+      ActiveRecord::Base.connection.reconnect!
+    end
+
+    # Returns a list of OS signals.
+    def signal_list
+      return Signal.list.keys
+    end
+
+    # Utility method to wrap code in a protective sheen
+    # use with "signal_list"
+    def protect_from_signals
+      # we are indiscriminate with the signals we block -- too bad ruby doesn't have some
+      # reasonable signal management system
+      signals = Hash[signal_list.map {|signal| [signal, Signal.trap(signal, "IGNORE")] }]
+      begin
+        yield
+      ensure
+        signals.each {|signal, saved_value| Signal.trap(signal, saved_value)}
+      end
+    end
+
+    # call this every once in a while
+    def periodic_application_checkpoint
+      if @gc_profiler
+        if (Time.zone.now - @last_gc_profiler_dump) > @gc_profiler_interval_minutes.minutes
+          @last_gc_profiler_dump = Time.zone.now
+          logger("GC::Profiler").info GC::Profiler.result
+        end
+      end
+    end
+
+    # Proxy's are used by dependant classes to reach back to the Application frame for
+    # some functionality.
+    #
+    # consider a model that wishes to use the logging functionality of Af:
+    #
+    #    class Foo < ActiveRecord::Base
+    #      include ::Af::Application::SafeProxy
+    #
+    #      after_create :do_something_after_create
+    #
+    #      def foo_logger
+    #        return af_logger(self.class.name)
+    #      end
+    #
+    #      private
+    #      def do_something_after_create
+    #        foo_logger.info "created: #{self.inspect}"
+    #      end
+    #    end
+    #
+    # The difference between Proxy and SafeProxy is simply that
+    # SafeProxy can be used in classes that may not be in an Af::Application
+    # run (ie, models that are shared with a Rails web app where Af::Application
+    # is never instantiated)
+    #
+    module Proxy
+      def af_logger(logger_name = (af_name || "Unknown"))
+        return ::Af::Application.singleton.logger(logger_name)
+      end
+
+      def af_name
+        return ::Af::Application.singleton.af_name
+      end
+    end
+
+    module SafeProxy
+      def af_logger(logger_name = (af_name || "Unknown"))
+        return ::Af::Application.singleton(true).logger(logger_name)
+      end
+
+      def af_name
+        return ::Af::Application.singleton(true).af_name
+      end
+    end
+
+  end
+end

data/lib/fiksu-af/deprecated.rb

@@ -0,0 +1,15 @@
+module Af
+  module Deprecated
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def update_opts(*x)
+      puts "don't use update_opts -- use #opt (#{caller.first})"
+      self.class.opt *x
+    end
+
+    module ClassMethods
+    end
+  end
+end

data/lib/fiksu-af/get_options.rb

@@ -0,0 +1,68 @@
+require 'getoptlong'
+
+module Af
+
+  # Subclasses Getoptlong from the Ruby standard library.
+  # Docs: http://www.ruby-doc.org/stdlib-1.9.3/libdoc/getoptlong/rdoc/GetoptLong.html.
+  # Source: https://github.com/ruby/ruby/blob/trunk/lib/getoptlong.rb.
+  class GetOptions < GetoptLong
+
+    # Local constants which map to superclass argument types.
+    ARGUMENT_FLAGS = [
+      NO_ARGUMENT = GetoptLong::NO_ARGUMENT,
+      REQUIRED_ARGUMENT = GetoptLong::REQUIRED_ARGUMENT,
+      OPTIONAL_ARGUMENT = GetoptLong::OPTIONAL_ARGUMENT
+    ]
+
+    # Instantiate a new long command line option parser with a hash of switches.
+    #
+    # *Arguments*
+    #   * switches - optional hash of command line switches, with long switch as
+    #       key to a set of options:
+    #         :short => <optional short switch>
+    #         :argument => <constant arg type>
+    #         :environment_variable => <how do these work???>
+    #         :note => <arg description>
+    def initialize(switches = {})
+      environment_variables = {} # switches that are set in the ENV
+      getopt_options = []
+
+      # Iterate through all of the switches.
+      switches.each do |long_switch, parameters|
+
+        # Set aside 
+        if parameters[:environment_variable].present?
+          environment_variables[parameters[:environment_variable]] = long_switch
+        end
+
+        # Convert hash into array, in format expected by Getoptlong#new.
+        # Example: ['--foo', '-f', 'bar']
+        options = []
+        options << long_switch
+        if (parameters[:short])
+          options << parameters[:short]
+        end
+        options << parameters[:argument]
+        getopt_options << options
+      end
+
+      # Add enviroment variables to the front of ARGV.
+      argv_additions = []
+      for environment_variable_name,value in environment_variables do
+        if ENV[environment_variable_name]
+        argv_additions << value
+        argv_additions << ENV[environment_variable_name] unless ENV[environment_variable_name].empty?
+        end
+      end
+      for arg in ARGV do
+        argv_additions << arg
+      end
+
+      # Rewrite ARGV with environment variable with the new list.
+      argv_additions.each_with_index { |v,i| ARGV[i] = v }
+
+      super(*getopt_options)
+    end
+
+  end
+end

data/lib/fiksu-af/logging/configurator.rb

@@ -0,0 +1,152 @@
+require 'log4r'
+require 'log4r/configurator'
+require 'log4r/yamlconfigurator'
+require 'log4r/outputter/consoleoutputters'
+require 'log4r_remote_syslog_outputter'
+require 'reasonable_log4r'
+
+module Af::Logging
+  class Configurator
+    @@singleton = nil
+
+    # Return the single allowable instance of this class, if the class has been instantiated
+    def self.singleton
+      return @@singleton ||= new
+    end
+
+    def initialize
+      @@singleton = self
+      Log4r::Configurator.custom_levels(:DEBUG, :DEBUG_FINE, :DEBUG_MEDIUM, :DEBUG_GROSS, :DETAIL, :INFO, :WARN, :ALARM, :ERROR, :FATAL)
+    end
+
+    # Parse and return the provided log level, which can be an integer,
+    # string integer or string constant.  Returns all loging levels if value
+    # cannot be parsed.
+    #
+    # *Arguments*
+    #   * logger_level - log level to be parsed
+    def parse_log_level(logger_level)
+      if logger_level.is_a? Integer
+        logger_level_value = logger_level
+      elsif logger_level.is_a? String
+        if logger_level[0] =~ /[0-9]/
+          logger_level_value = logger_level.to_i
+        else
+          logger_level_value = logger_level.constantize rescue nil
+          logger_level_value = "Log4r::#{logger_level.upcase}".constantize rescue nil unless logger_level_value
+        end
+      else
+        logger_level_value = Log4r::ALL
+      end
+      return logger_level_value
+    end
+
+    # Returns the logger with the provided name, instantiating it if needed.
+    #
+    # *Arguments*
+    #   * logger_name - logger to return, defaults to ":default"
+    def logger(logger_name = :default)
+      # Coerce the logger_name if needed.
+      logger_name = ::Af::Application.singleton.af_name if logger_name == :default
+      # Check with Log4r to see if there is a logger by this name.
+      # If Log4r doesn't have a logger by this name, make one with Af defaults.
+      return Log4r::Logger[logger_name] || Log4r::Logger.new(logger_name)
+    end
+
+    # Load the provided yaml Log4r configuration files.
+    #
+    # *Arguments*
+    #   * files - array of file names with full paths (??)
+    #   * yaml_sections - ???
+    def logging_load_configuration_files(files, yaml_sections)
+      begin
+        Log4r::YamlConfigurator.load_yaml_files(files, yaml_sections)
+      rescue StandardError => e
+        puts "error while parsing log configuration files: #{e.message}"
+        puts "continuing without your configuration"
+        puts e.backtrace.join("\n")
+        return false
+      end
+      return true
+    end
+
+    # Load all of the Log4r yaml configuration files.
+    def logging_load_configuration
+      files = []
+      @@log_configuration_files.each do |configuration_file|
+        @@log_configuration_search_path.each do |path|
+          pathname = Pathname.new(path) + configuration_file
+          files << pathname.to_s if pathname.file?
+        end
+      end
+      logging_load_configuration_files(files, @@log_configuration_section_names)
+    end
+
+    # TODO AK: What is purpose of this method?
+    def logging_configuration_looks_bogus
+      return Log4r::LNAMES.length == 1
+    end
+
+    # Parses and sets the provided logger levels.
+    #
+    # *Argument*
+    #   * logger_info - value indicating default log level, or JSON string
+    #     of logger names to logger levels, i.e. "{'foo' => 'INFO'}.
+    def parse_and_set_logger_levels(logger_info)
+      log_level_hash = JSON.parse(logger_info) rescue {:default => parse_log_level(logger_info)}
+      set_logger_levels(log_level_hash)
+    end
+
+    # Sets the logger levels the provided hash.  It supports the following formats for
+    # logger levels: 1, "1", "INFO", "Log4r::INFO".
+    #
+    # *Arguments*
+    #   * log_level_hash - hash of logger names to logger levels,
+    #     i.e. { :foo => 'INFO' }
+    def set_logger_levels(log_level_hash)
+      log_level_hash.each do |logger_name, logger_level|
+        logger_name = :default if logger_name == "default"
+        logger_level_value = parse_log_level(logger_level)
+        l = logger(logger_name)
+        l.level = logger_level_value
+      end
+    end
+
+    def configurate
+      if @@log_console || @@log_ignore_configuration
+        Log4r::Logger.root.outputters << Log4r::Outputter.stdout
+      else
+        logging_load_configuration
+        if logging_configuration_looks_bogus
+          Log4r::Logger.root.outputters << Log4r::Outputter.stdout
+        end
+      end
+
+      if @@log_levels
+        set_logger_levels(@@log_levels)
+      end
+
+      if @@log_dump_configuration
+        puts "Log configuration search path:"
+        puts " " + @@log_configuration_search_path.join("\n ")
+        puts "Log configuration files:"
+        puts " " + @@log_configuration_files.join("\n ")
+        puts "Logging Names: #{Log4r::LNAMES.join(', ')}"
+        puts "Yaml section names:"
+        puts " " + @@log_configuration_section_names.join("\n ")
+        loggers = []
+        Log4r::Logger.each do |logger_name, logger|
+          loggers << logger_name
+        end
+        puts "Loggers:"
+        puts "global: #{Log4r::LNAMES[Log4r::Logger.global.level]}"
+        puts "root: #{Log4r::LNAMES[Log4r::Logger['root'].level]} [#{Log4r::Logger['root'].outputters.map{|o| o.name}.join(', ')}]"
+        loggers.sort.reject{|logger_name| ["root", "global"].include? logger_name}.each do |logger_name|
+          puts "#{' ' * logger_name.split('::').length}#{logger_name}: #{Log4r::LNAMES[Log4r::Logger[logger_name].level]} [#{Log4r::Logger[logger_name].outputters.map{|o| o.name}.join(', ')}]"
+        end
+        exit 0
+      end
+    end
+
+  end
+end

data/lib/fiksu-af/logging.rb

@@ -0,0 +1,13 @@
+module Af::Logging
+  def self.included(base)
+    Configurator.singleton
+  end
+
+  def logger(logger_name = :default)
+    return Configurator.singleton.logger(logger_name)
+  end
+
+  def logging_configurator
+    return Configurator.singleton
+  end
+end

data/lib/fiksu-af/option_parser/columnizer.rb

@@ -0,0 +1,36 @@
+module ::Af::OptionParser
+  class Columnizer
+    # Convert an array into a single string, where each item consumes a static
+    # number of characters.  Long fields are truncated and small ones are padded.
+    #
+    # *Arguments*
+    #   * fields - array of objects that respond to "to_s"
+    #   * sized - character count for each field in the new string????
+    def columnized_row(fields, sized)
+      r = []
+      fields.each_with_index do |f, i|
+        r << sprintf("%0-#{sized[i]}s", f.to_s.gsub(/\\n\\r/, '').slice(0, sized[i]))
+      end
+      return r.join('   ')
+    end
+
+    # Converts an array of arrays into a single array of columnized strings.
+    #
+    # *Arguments*
+    #   * rows - arrays to convert
+    #   * options - hash of options, includes:
+    #     :max_width => <integer max width of columns>
+    def columnized(rows, options = {})
+      sized = {}
+      rows.each do |row|
+        row.each_index do |i|
+          value = row[i]
+          sized[i] = [sized[i].to_i, value.to_s.length].max
+          sized[i] = [options[:max_width], sized[i].to_i].min if options[:max_width]
+        end
+      end
+
+      return rows.map { |row| "    " + columnized_row(row, sized).rstrip }
+    end
+  end
+end