ngmoco-request-log-analyzer 1.4.2

data/lib/request_log_analyzer/database.rb

@@ -0,0 +1,102 @@
+require 'rubygems'
+require 'activerecord'
+
+class RequestLogAnalyzer::Database
+
+  def self.const_missing(const) # :nodoc:
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  include RequestLogAnalyzer::Database::Connection
+
+  attr_accessor :file_format
+  attr_reader :line_classes
+
+  def initialize(connection_identifier = nil)
+    @line_classes = []
+    RequestLogAnalyzer::Database::Base.database = self
+    connect(connection_identifier)
+  end
+
+  # Returns the ORM class for the provided line type
+  def get_class(line_type)
+    line_type = line_type.name if line_type.respond_to?(:name)
+    Object.const_get("#{line_type}_line".camelize)
+  end
+
+  def default_classes
+    [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
+  end
+
+  # Loads the ORM classes by inspecting the tables in the current database
+  def load_database_schema!
+    connection.tables.map do |table|
+      case table.to_sym
+      when :warnings then RequestLogAnalyzer::Database::Warning
+      when :sources  then RequestLogAnalyzer::Database::Source
+      when :requests then RequestLogAnalyzer::Database::Request
+      else load_activerecord_class(table)
+      end
+    end
+  end
+
+  # Returns an array of all the ActiveRecord-bases ORM classes for this database
+  def orm_classes
+    default_classes + line_classes
+  end
+
+  # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
+  # a table name or a LineDefinition instance.
+  def load_activerecord_class(linedefinition_or_table)
+
+    case linedefinition_or_table
+    when String, Symbol
+      klass_name = linedefinition_or_table.to_s.singularize.camelize
+      klass      = RequestLogAnalyzer::Database::Base.subclass_from_table(linedefinition_or_table)
+    when RequestLogAnalyzer::LineDefinition
+      klass_name = "#{linedefinition_or_table.name}_line".camelize
+      klass      = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
+    end
+
+    Object.const_set(klass_name, klass)
+    klass = Object.const_get(klass_name)
+    @line_classes << klass
+    return klass
+  end
+
+  def fileformat_classes
+    raise "No file_format provided!" unless file_format
+    line_classes    = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
+    return default_classes + line_classes
+  end
+
+  # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
+  # file format definition. These ORM classes will later be used to create records in the database.
+  def create_database_schema!
+    fileformat_classes.each { |klass| klass.create_table! }
+  end
+
+  # Drops the table of all the ORM classes, and unregisters the classes
+  def drop_database_schema!
+    file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
+    remove_orm_classes!
+  end
+
+  # Registers the default ORM classes in the default namespace
+  def register_default_orm_classes!
+    Object.const_set('Request', RequestLogAnalyzer::Database::Request)
+    Object.const_set('Source',  RequestLogAnalyzer::Database::Source)
+    Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
+  end
+
+  # Unregisters every ORM class constant
+  def remove_orm_classes!
+    orm_classes.each do |klass|
+      if klass.respond_to?(:name) && !klass.name.blank?
+        klass_name = klass.name.split('::').last
+        Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/base.rb

@@ -0,0 +1,115 @@
+class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
+
+  self.abstract_class = true
+
+  def <=>(other)
+    if (source_id.nil? && other.source_id.nil?) || (source_comparison = source_id <=> other.source_id) == 0
+      lineno <=> other.lineno
+    else
+      source_comparison
+    end
+  end
+
+  def line_type
+    self.class.name.underscore.gsub(/_line$/, '').to_sym
+  end
+
+  class_inheritable_accessor :line_definition
+  cattr_accessor :database
+
+  def self.subclass_from_line_definition(definition)
+    klass = Class.new(RequestLogAnalyzer::Database::Base)
+    klass.set_table_name("#{definition.name}_lines")
+
+    klass.line_definition = definition
+
+    # Set relations with requests and sources table
+    klass.belongs_to :request
+    klass.belongs_to :source
+
+    # Serialize complex fields into the database
+    definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
+      klass.send(:serialize, capture[:name], Hash)
+    end
+
+    RequestLogAnalyzer::Database::Request.has_many  "#{definition.name}_lines".to_sym
+    RequestLogAnalyzer::Database::Source.has_many   "#{definition.name}_lines".to_sym
+
+    return klass
+  end
+
+  def self.subclass_from_table(table)
+    raise "Table #{table} not found!" unless database.connection.table_exists?(table)
+
+    klass = Class.new(RequestLogAnalyzer::Database::Base)
+    klass.set_table_name(table)
+
+    if klass.column_names.include?('request_id')
+      klass.belongs_to :request
+      RequestLogAnalyzer::Database::Request.has_many table.to_sym
+    end
+
+    if klass.column_names.include?('source_id')
+      klass.belongs_to :source
+      RequestLogAnalyzer::Database::Source.has_many table.to_sym
+    end
+
+    return klass
+  end
+
+  def self.drop_table!
+    database.connection.remove_index(self.table_name, [:source_id])  rescue nil
+    database.connection.remove_index(self.table_name, [:request_id]) rescue nil
+    database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
+  end
+
+  def self.create_table!
+    raise "No line_definition available to base table schema on!" unless self.line_definition
+
+    unless table_exists?
+      database.connection.create_table(table_name.to_sym) do |t|
+
+        # Default fields
+        t.column :request_id, :integer
+        t.column :source_id,  :integer
+        t.column :lineno,     :integer
+
+        line_definition.captures.each do |capture|
+          # Add a field for every capture
+          t.column(capture[:name], column_type(capture[:type]))
+
+          # If the capture provides other field as well, create columns for them, too
+          capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
+        end
+      end
+    end
+
+    # Add indices to table for more speedy querying
+    database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
+    database.connection.add_index(self.table_name.to_sym, [:source_id])  # rescue
+  end
+
+
+  # Function to determine the column type for a field
+  # TODO: make more robust / include in file-format definition
+  def self.column_type(type_indicator)
+    case type_indicator
+    when :eval;      :text
+    when :hash;      :text
+    when :text;      :text
+    when :string;    :string
+    when :sec;       :double
+    when :msec;      :double
+    when :duration;  :double
+    when :float;     :double
+    when :double;    :double
+    when :integer;   :integer
+    when :int;       :int
+    when :timestamp; :datetime
+    when :datetime;  :datetime
+    when :date;      :date
+    else             :string
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/connection.rb

@@ -0,0 +1,38 @@
+module RequestLogAnalyzer::Database::Connection
+
+  def self.from_string(string)
+    hash = {}
+    if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/
+      string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value }
+    elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/
+      hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = $1, $2, $3, $4, $5
+      hash.delete_if { |k, v| v.nil? }
+    end
+    return hash.empty? ? nil : hash
+  end
+
+  def connect(connection_identifier)
+    if connection_identifier.kind_of?(Hash)
+      RequestLogAnalyzer::Database::Base.establish_connection(connection_identifier)
+    elsif connection_identifier == ':memory:'
+      RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+    elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier)
+      RequestLogAnalyzer::Database::Base.establish_connection(connection_hash)
+    elsif connection_identifier.kind_of?(String) # Normal SQLite 3 database file
+      RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => connection_identifier)
+    elsif connection_identifier.nil?
+      nil
+    else
+      raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
+    end
+  end
+
+  def disconnect
+    RequestLogAnalyzer::Database::Base.remove_connection
+  end
+
+  def connection
+    RequestLogAnalyzer::Database::Base.connection
+  end
+
+end

data/lib/request_log_analyzer/database/request.rb

@@ -0,0 +1,22 @@
+class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
+
+  # Returns an array of all the Line objects of this request in the correct order.
+  def lines
+    @lines ||= begin
+      lines = []
+      self.class.reflections.each { |r, d| lines += self.send(r).all }
+      lines.sort
+    end
+  end
+
+  # Creates the table to store requests in.
+  def self.create_table!
+    unless database.connection.table_exists?(:requests)
+      database.connection.create_table(:requests) do |t|
+        t.column :first_lineno, :integer
+        t.column :last_lineno,  :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/source.rb

@@ -0,0 +1,13 @@
+class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
+
+  def self.create_table!
+    unless database.connection.table_exists?(:sources)
+      database.connection.create_table(:sources) do |t|
+        t.column :filename, :string
+        t.column :mtime,    :datetime
+        t.column :filesize, :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/warning.rb

@@ -0,0 +1,14 @@
+class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
+
+  def self.create_table!
+    unless database.connection.table_exists?(:warnings)
+      database.connection.create_table(:warnings) do |t|
+        t.column  :warning_type, :string, :limit => 30, :null => false
+        t.column  :message, :string
+        t.column  :source_id, :integer
+        t.column  :lineno, :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format.rb

@@ -0,0 +1,160 @@
+module RequestLogAnalyzer::FileFormat
+
+  def self.const_missing(const)
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  # Loads a FileFormat::Base subclass instance.
+  # You can provide:
+  # * A FileFormat instance (which will return itself)
+  # * A FileFormat class (of which an imstance will be returned)
+  # * A filename (from which the FileFormat class is loaded)
+  # * A symbol of a built-in file format (e.g. :rails)
+  def self.load(file_format, *args)
+    klass = nil
+    if file_format.kind_of?(RequestLogAnalyzer::FileFormat::Base)
+      # this already is a file format! return itself
+      return @current_file_format = file_format
+
+    elsif file_format.kind_of?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
+      # a usable class is provided. Use this format class.
+      klass = file_format
+
+    elsif file_format.kind_of?(String) && File.exist?(file_format)
+      # load a format from a ruby file
+      require file_format
+      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+      if RequestLogAnalyzer::FileFormat.const_defined?(const)
+        klass = RequestLogAnalyzer::FileFormat.const_get(const)
+      elsif Object.const_defined?(const)
+        klass = Object.const_get(const)
+      else
+        raise "Cannot load class #{const} from #{file_format}!"
+      end
+
+    else
+      # load a provided file format
+      klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
+    end
+
+    # check the returned klass to see if it can be used
+    raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
+    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
+
+    @current_file_format = klass.create(*args) # return an instance of the class
+  end
+
+  # Base class for all log file format definitions. This class provides functions for subclasses to
+  # define their LineDefinitions and to define a summary report.
+  #
+  # A subclass of this class is instantiated when request-log-analyzer is started and this instance
+  # is shared with all components of the application so they can act on the specifics of the format
+  class Base
+
+    attr_reader :line_definitions, :report_trackers
+
+    ####################################################################################
+    # CLASS METHODS for format definition
+    ####################################################################################
+
+    # Registers the line definer instance for a subclass.
+    def self.inherited(subclass)
+      if subclass.superclass == RequestLogAnalyzer::FileFormat::Base
+
+        # Create aline and report definer for this class
+        subclass.class_eval do
+          instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
+          instance_variable_set(:@report_definer, RequestLogAnalyzer::Aggregator::Summarizer::Definer.new)
+          class << self; attr_accessor :line_definer, :report_definer; end
+        end
+
+        # Create a custom Request class for this file format
+        subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request)) unless subclass.const_defined?('Request')
+      else
+
+        # Copy the line and report definer from the parent class.
+        subclass.class_eval do
+          instance_variable_set(:@line_definer, superclass.line_definer.clone)
+          instance_variable_set(:@report_definer, superclass.report_definer.clone)
+          class << self; attr_accessor :line_definer, :report_definer; end
+        end
+
+        # Create a custom Request class based on the superclass's Request class
+        subclass.const_set('Request', Class.new(subclass.superclass::Request)) unless subclass.const_defined?('Request')
+      end
+    end
+
+    # Specifies a single line defintions.
+    def self.line_definition(name, &block)
+      @line_definer.send(name, &block)
+    end
+
+    # Specifies multiple line definitions at once using a block
+    def self.format_definition(&block)
+      if block_given?
+        yield self.line_definer
+      else
+        return self.line_definer
+      end
+    end
+
+    # Specifies the summary report using a block.
+    def self.report(mode = :append, &block)
+      self.report_definer.reset! if mode == :overwrite
+      yield(self.report_definer)
+    end
+
+    ####################################################################################
+    # Instantiation
+    ####################################################################################
+
+    def self.create(*args)
+      # Ignore arguments
+      return self.new(line_definer.line_definitions, report_definer.trackers)
+    end
+
+    def initialize(line_definitions = {}, report_trackers = [])
+      @line_definitions, @report_trackers = line_definitions, report_trackers
+    end
+
+    ####################################################################################
+    # INSTANCE methods
+    ####################################################################################
+
+    # Returns the Request class of this file format
+    def request_class
+      self.class::Request
+    end
+
+    # Returns a Request instance with the given parsed lines that should be provided as hashes.
+    def request(*hashes)
+      request_class.create(self, *hashes)
+    end
+
+    # Checks whether the line definitions form a valid language.
+    # A file format should have at least a header and a footer line type
+    def valid?
+      line_definitions.any? { |(name, ld)| ld.header } && line_definitions.any? { |(name, ld)| ld.footer }
+    end
+
+    # Returns true if this language captures the given symbol in one of its line definitions
+    def captures?(name)
+      line_definitions.any? { |(name, ld)| ld.captures?(name) }
+    end
+
+    # Function that a file format con implement to monkey patch the environment.
+    # * <tt>controller</tt> The environment is provided as a controller instance
+    def setup_environment(controller)
+    end
+
+    # Parses a line by trying to parse it using every line definition in this file format
+    def parse_line(line, &warning_handler)
+      self.line_definitions.each do |lt, definition|
+        match = definition.matches(line, &warning_handler)
+        return match if match
+      end
+
+      return nil
+    end
+  end
+end