indy 0.3.4 → 0.4.0.pre

data/lib/indy/log_definition.rb

@@ -0,0 +1,115 @@
+class Indy
+
+  class LogDefinition
+
+    attr_accessor :entry_regexp, :entry_fields, :time_format, :multiline
+
+    def initialize(args=:default)
+      case args
+      when :default, {}
+        params_hash = set_defaults
+      when Array, Hash
+        params_hash = parse_enumerable_params(args)
+      end
+      raise ArgumentError, "Values for entry_regexp and/or entry_fields were not supplied" unless (params_hash[:entry_fields] && params_hash[:entry_regexp])
+      if params_hash[:multiline]
+        @entry_regexp = Regexp.new(params_hash[:entry_regexp], Regexp::MULTILINE)
+        @multiline = true
+      else
+        @entry_regexp = Regexp.new(params_hash[:entry_regexp])
+      end
+      @entry_fields = params_hash[:entry_fields]
+      @time_format = params_hash[:time_format]
+      define_struct
+    end
+
+    def set_defaults
+      params_hash = {}
+      params_hash[:entry_regexp] = Indy::LogFormats::DEFAULT_ENTRY_REGEXP
+      params_hash[:entry_fields] = Indy::LogFormats::DEFAULT_ENTRY_FIELDS
+      params_hash
+    end
+
+    def parse_enumerable_params(args)
+      params_hash = {}
+      params_hash.merge!(args)
+      if args.keys.include? :log_format
+        # support 0.3.4 params
+        params_hash[:entry_regexp] = args[:log_format][0]
+        params_hash[:entry_fields] = args[:log_format][1..-1]
+        params_hash.delete :log_format
+      end
+      params_hash
+    end
+
+    #
+    # Return a Struct::Entry object from a hash of values from a log entry
+    #
+    # @param [Hash] entry_hash a hash of :field_name => value pairs for one log entry
+    #
+    def create_struct( entry_hash )
+      values = entry_hash.keys.sort_by{|entry|entry.to_s}.collect {|key| entry_hash[key]}
+      result = Struct::Entry.new( *values )
+      result
+    end
+
+    #
+    # Define Struct::Entry with the fields from @log_definition. Ignore warnings.
+    #
+    def define_struct
+      fields = (@entry_fields + [:raw_entry]).sort_by{|key|key.to_s}
+      verbose = $VERBOSE
+      $VERBOSE = nil
+      Struct.new( "Entry", *fields )
+      $VERBOSE = verbose
+    end
+
+    #
+    # Convert log entry into hash
+    #
+    def entry_hash(values)
+      assert_valid_field_list(values) unless @field_list_is_valid # just do it once
+      raw_entry = values.shift
+      hash = Hash[ *@entry_fields.zip( values ).flatten ]
+      hash[:raw_entry] = raw_entry.strip
+      hash
+    end
+
+
+    #
+    # Return a hash of field=>value pairs for the log entry
+    #
+    # @param [String] raw_entry The raw log entry
+    #
+    def parse_entry(raw_entry)
+      match_data = /#{@entry_regexp}/.match(raw_entry)
+      return nil unless match_data
+      values = match_data.captures
+      entry_hash([raw_entry, values].flatten)
+    end
+
+    #
+    # Return a hash of field=>value pairs for the array of captured values from a log entry
+    #
+    # @param [Array] capture_array The array of values captured by the @log_definition.entry_regexp
+    #
+    def parse_entry_captures( capture_array )
+      entire_entry = capture_array.shift
+      values = capture_array
+      entry_hash([entire_entry, values].flatten)
+    end
+
+    #
+    # Ensure number of fields is expected
+    #
+    def assert_valid_field_list(values)
+      if values.length == @entry_fields.length + 1 # values also includes raw_entry
+        @field_list_is_valid = true
+      else
+        raise ArgumentError, "Field mismatch between log pattern and log data. The data is: '#{values.join(':::')}'"
+      end
+    end
+
+  end
+
+end

data/lib/indy/log_formats.rb

@@ -23,8 +23,8 @@ class Indy
     DEFAULT_APPLICATION = '\w+'
     DEFAULT_MESSAGE = '.+'
 
-    DEFAULT_LOG_REGEXP = /^(#{DEFAULT_DATE_TIME})\s+(#{DEFAULT_SEVERITY_PATTERN})\s+(#{DEFAULT_APPLICATION})\s+-\s+(#{DEFAULT_MESSAGE})$/
-    DEFAULT_LOG_FIELDS = [:time,:severity,:application,:message]
+    DEFAULT_ENTRY_FIELDS = [:time,:severity,:application,:message]
+    DEFAULT_ENTRY_REGEXP = /^(#{DEFAULT_DATE_TIME})\s+(#{DEFAULT_SEVERITY_PATTERN})\s+(#{DEFAULT_APPLICATION})\s+-\s+(#{DEFAULT_MESSAGE})$/
 
     COMMON_FIELDS = [:host, :ident, :authuser, :time, :request, :status, :bytes]
     COMMON_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP}$/
@@ -33,7 +33,10 @@ class Indy
     COMBINED_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP} #{DQUOTE_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP}$/
 
     LOG4R_DEFAULT_FIELDS = [:level, :application, :message]
-    LOG4R_DEFAULT_REGEXP = /(?:\s+)?([A-Z]+) (\S+): (.*)$/
+    LOG4R_DEFAULT_REGEXP = /^(?:\s*)([A-Z]+) (\S+): (.*)$/
+
+    LOG4J_DEFAULT_FIELDS = [:message]
+    LOG4J_DEFAULT_REGEXP = /^(.+?)$/
   end
 
   #
@@ -41,21 +44,26 @@ class Indy
   # e.g.:
   # INFO 2000-09-07 MyApp - Entering APPLICATION.
   #
-  DEFAULT_LOG_FORMAT = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
+  DEFAULT_LOG_FORMAT = {:entry_regexp => LogFormats::DEFAULT_ENTRY_REGEXP, :entry_fields => LogFormats::DEFAULT_ENTRY_FIELDS}
 
   #
   # Uncustomized Log4r log format
   #
-  LOG4R_DEFAULT_FORMAT = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
+  LOG4R_DEFAULT_FORMAT = {:entry_regexp => LogFormats::LOG4R_DEFAULT_REGEXP, :entry_fields => LogFormats::LOG4R_DEFAULT_FIELDS}
+
+  #
+  # Uncustomized Log4j log format (message field only!)
+  #
+  LOG4J_DEFAULT_FORMAT = {:entry_regexp => LogFormats::LOG4J_DEFAULT_REGEXP, :entry_fields => LogFormats::LOG4J_DEFAULT_FIELDS}
 
   #
   # NCSA Common Log Format log format
   #
-  COMMON_LOG_FORMAT = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
+  COMMON_LOG_FORMAT = {:entry_regexp => LogFormats::COMMON_REGEXP, :entry_fields => LogFormats::COMMON_FIELDS}
 
   #
   # NCSA Combined Log Format log format
   #
-  COMBINED_LOG_FORMAT = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
+  COMBINED_LOG_FORMAT = {:entry_regexp => LogFormats::COMBINED_REGEXP, :entry_fields => LogFormats::COMBINED_FIELDS}
 
 end

data/lib/indy/search.rb

@@ -0,0 +1,147 @@
+class Indy
+
+  class Search
+
+    attr_accessor :source
+    attr_accessor :log_definition
+
+    attr_accessor :start_time, :end_time, :inclusive
+
+    def initialize(params_hash)
+      while (param = params_hash.shift) do
+        send("#{param.first}=",param.last)
+      end
+    end
+
+    #
+    # Helper function called by Indy#for, Indy#like and Indy#all
+    #
+    # @param [Symbol] type The symbol :for, :like or :all
+    #
+    # @param [Hash] search_criteria the field to search for as the key and the
+    #        value to compare against the log entries.
+    #
+    def iterate_and_compare(type,search_criteria,&block)
+      results = []
+      results += search do |entry|
+        if type == :all || is_match?(type,entry,search_criteria)
+          result_struct = @log_definition.create_struct(entry)
+          if block_given?
+            block.call(result_struct)
+          else
+            result_struct
+          end
+        end
+      end
+      results.compact
+    end
+
+    #
+    # Search the @source and yield to the block the entry that was found
+    # with @log_definition
+    #
+    # This method is supposed to be used internally.
+    #
+    def search(&block)
+      if @log_definition.multiline
+        multiline_search(&block)
+      else
+        standard_search(&block)
+      end
+    end
+
+    #
+    # Performs #search for line based entries
+    #
+    def standard_search(&block)
+      is_time_search = use_time_criteria?
+      results = []
+      source_lines = (is_time_search ? @source.open([@start_time,@end_time]) : @source.open)
+      source_lines.each do |single_line|
+        hash = @log_definition.parse_entry(single_line)
+        next unless hash
+        next unless Indy::Time.inside_time_window?(hash[:time],@start_time,@end_time,@inclusive) if is_time_search
+        results << (block.call(hash) if block_given?)
+      end
+      results.compact
+    end
+
+    #
+    # Performs #search for multi-line based entries
+    #
+    def multiline_search(&block)
+      is_time_search = use_time_criteria?
+      source_io = StringIO.new( (is_time_search ? @source.open([@start_time,@end_time]) : @source.open ).join("\n") )
+      results = source_io.read.scan(@log_definition.entry_regexp).collect do |entry|
+        hash = @log_definition.parse_entry_captures(entry)
+        next unless hash
+        next unless Indy::Time.inside_time_window?(hash[:time],@start_time,@end_time,@inclusive) if is_time_search
+        block.call(hash) if block_given?
+      end
+      results.compact
+    end
+
+    #
+    #  Return true if start or end time has been set, and a :time field exists
+    #
+    def use_time_criteria?
+      if @start_time || @end_time
+        # ensure both boundaries are set
+        @start_time ||= Indy::Time.forever_ago(@log_definition.time_format)
+        @end_time ||= Indy::Time.forever(@log_definition.time_format)
+      end
+      @start_time && @end_time
+    end
+
+    #
+    # Evaluates if field => value criteria is an exact match on entry
+    #
+    # @param [Hash] result The entry_hash
+    # @param [Hash] search_criteria The field => value criteria to match
+    #
+    def is_match?(type, result, search_criteria)
+      if type == :for
+        search_criteria.reject {|criteria,value| result[criteria] == value }.empty?
+      elsif type == :like
+        search_criteria.reject {|criteria,value| result[criteria] =~ /#{value}/i }.empty?
+      end
+    end
+
+    #
+    # Parse hash to set @start_time, @end_time and @inclusive
+    #
+    def time_scope(params_hash)
+      if params_hash[:time]
+        time_scope_from_direction(params_hash[:direction], params_hash[:span], params_hash[:time])
+      else
+        @start_time = Indy::Time.parse_date(params_hash[:start_time]) if params_hash[:start_time]
+        @end_time = Indy::Time.parse_date(params_hash[:end_time]) if params_hash[:end_time]
+      end
+      @inclusive = params_hash[:inclusive]
+    end
+
+    #
+    # Parse direction, span, and time to set @start_time and @end_time
+    #
+    def time_scope_from_direction(direction, span, time)
+       time = Indy::Time.parse_date(time)
+      span = (span.to_i * 60).seconds if span
+      if direction == :before
+        @end_time = time
+        @start_time = time - span if span
+      elsif direction == :after
+        @start_time = time
+        @end_time = time + span if span
+      end
+    end
+
+    #
+    # Clear time scope settings
+    #
+    def reset_scope
+      @inclusive = @start_time = @end_time = nil
+    end
+
+  end
+
+end

data/lib/indy/source.rb

@@ -14,6 +14,9 @@ class Indy
     # the StringIO object
     attr_reader :io
 
+    # log definition
+    attr_reader :log_definition
+
     # Exception raised when unable to open source
     class Invalid < Exception; end
 
@@ -22,22 +25,30 @@ class Indy
     #
     # @param [String, Hash] param The source content String, filepath String, or :cmd => 'command' Hash
     #
-    def initialize(param)
-      raise Indy::Source::Invalid if param.nil?
-      if param.respond_to?(:keys)
-        set_connection(:cmd, param[:cmd]) if param[:cmd]
-        set_connection(:file, param[:file]) if ( param[:file] and File.size(param[:file].path) > 0 )
-        set_connection(:string, param[:string]) if param[:string]
-      elsif param.respond_to?(:read) and param.respond_to?(:rewind)
-          set_connection(:file, param)
+    def initialize(param,log_definition=nil)
+      raise Indy::Source::Invalid, "No source specified." if param.nil?
+      @log_definition = log_definition || LogDefinition.new()
+      return discover_connection(param) unless param.respond_to?(:keys)
+      if param[:cmd]
+        set_connection(:cmd, param[:cmd])
+      elsif param[:file]
+        set_connection(:file, open_or_return_file(param[:file]))
+      elsif param[:string]
+        set_connection(:string, param[:string])
+      end
+    end
+
+    #
+    # Support source being passed in without key indicating type
+    #
+    def discover_connection(param)
+      if param.respond_to?(:read) and param.respond_to?(:rewind)
+        set_connection(:file, param)
       elsif param.respond_to?(:to_s) and param.respond_to?(:length)
-        # fall back to source being the string passed in
         set_connection(:string, param)
       else
         raise Indy::Source::Invalid
       end
-
-
     end
 
     #
@@ -48,82 +59,164 @@ class Indy
       @connection = value
     end
 
+    def open_or_return_file(param)
+      return param if param.respond_to? :pos
+      file = File.open(param, 'r')
+      raise ArgumentError, "Unable to open file parameter: '#{file}'" unless file.respond_to? :pos
+      file
+    end
+
     #
     # Return a StringIO object to provide access to the underlying log source
     #
-    def open(time_search=nil)
+    def open(time_boundaries=nil)
       begin
+        open_method = ('open_' + @type.to_s).intern
+        self.send(open_method)
+      rescue Exception => e
+        raise Indy::Source::Invalid, "Unable to open log source. (#{e.message})"
+      end
+      load_data
+      scope_by_time(time_boundaries) if time_boundaries
+      @entries
+    end
 
-        case @type
-        when :cmd
-          @io = StringIO.new( exec_command(@connection).read )
-          raise "Failed to execute command (#{@connection})" if @io.nil?
+    def open_cmd
+      @io = StringIO.new(exec_command(@connection).read)
+      raise "Failed to execute command (#{@connection.inspect})" if @io.nil?
+    end
 
-        when :file
-          @connection.rewind
-          @io = StringIO.new(@connection.read)
-          raise "Failed to open file: #{@connection}" if @io.nil?
+    def open_file
+      @connection.rewind
+      @io = StringIO.new(@connection.read)
+      raise "Failed to open file: #{@connection.inspect}" if @io.nil?
+    end
 
-        when :string
-          @io = StringIO.new( @connection )
+    def open_string
+      @io = StringIO.new(@connection)
+      raise "Failed to create StringIO from source (#{@connection.inspect})" if @io.nil?
+    end
 
-        else
-          raise RuntimeError, "Invalid log source type: #{@type.inspect}"
-        end
 
-      rescue Exception => e
-        raise Indy::Source::Invalid, "Unable to open log source. (#{e.message})"
+    #
+    # Return entries that meet time criteria
+    #
+    def scope_by_time(time_boundaries)
+      start_time, end_time = time_boundaries
+      scope_end = num_entries - 1
+      # short circuit the search if possible
+      if (time_at(0) > end_time) or (time_at(-1) < start_time)
+        @entries = []
+        return @entries
+      end
+      scope_begin = find_first(start_time, 0, scope_end)
+      scope_end = find_last(end_time, scope_begin, scope_end)
+      @entries = @entries[scope_begin..scope_end]
+    end
+
+    #
+    # find index of first record to match value
+    #
+    def find_first(value,start,stop)
+      return start if time_at(start) > value
+      find(:first,value,start,stop)
+    end
+
+    #
+    # find index of last record to match value
+    #
+    def find_last(value,start,stop)
+      return stop if time_at(stop) < value
+      find(:last,value,start,stop)
+    end
+
+    #
+    # Find index and time at mid point
+    #
+    def find_middle(start, stop)
+      index = ((stop - start) / 2) + start
+      time = time_at(index)
+      [index, time]
+    end
+
+    #
+    # Step forward or backward by one, looking for the boundary of the value
+    #
+    def find_adjacent(boundary,value,start,stop,mid_index)
+      case boundary
+      when :first
+        (time_at(mid_index,-1) == value) ? find_first(value,start-1,stop) : mid_index
+      when :last
+        (time_at(mid_index,1) == value) ? find_last(value,start,stop+1) : mid_index
       end
+    end
 
-      # scope_by_time(source_io) if time_search
+    #
+    # Return the time of a log entry index, with an optional offset
+    #
+    def time_at(index, delta=0)
+      ::Time.parse(@entries[index + delta])
+    end
 
-      @io
+    #
+    # Binary search for a time condition
+    #
+    def find(boundary,value,start,stop)
+      return start if start == stop
+      mid_index, mid_time = find_middle(start,stop)
+      if mid_time == value
+        find_adjacent(boundary,value,start,stop,mid_index)
+      elsif mid_time > value
+        mid_index -= 1 if mid_index == stop
+        find(boundary, value, start, mid_index)
+      elsif mid_time < value
+        mid_index += 1 if mid_index == start
+        find(boundary, value, mid_index, stop)
+      end
     end
-    
+
     #
     # Execute the source's connection string, returning an IO object
     #
     # @param [String] command_string string of command that will return log contents
     #
     def exec_command(command_string)
-      begin
-        io = IO.popen(command_string)
-        return nil if io.eof?
-      rescue
-        nil
-      end
+      io = IO.popen(command_string)
+      raise Indy::Source::Invalid, "No data returned from command string execution" if io.eof?
       io
     end
 
-    
-
     #
     # the number of lines in the source
     #
-    def num_lines
-      load_data unless @num_lines
-      @num_lines
+    def num_entries
+      load_data unless @num_entries
+      @num_entries
     end
 
     #
     # array of log lines from source
     #
-    def lines
-      load_data unless @lines
-      @lines
+    def entries
+      load_data unless @entries
+      @entries
     end
 
     #
     # read source data and populate instance variables
     #
-    # TODO: hmmm... not called when Source#open is called directly, but #load_data would call open again. :(
-    #
     def load_data
-      self.open
-      @lines = @io.readlines
+      self.open if @io.nil?
+      if @log_definition.multiline
+        entire_log = @io.read
+        @entries = entire_log.scan(@log_definition.entry_regexp).map{|matchdata|matchdata[0]}
+      else
+        @entries = @io.readlines
+      end
       @io.rewind
-      @num_lines = @lines.count
+      @entries.delete_if {|entry| entry.match(/^\s*$/)}
+      @num_entries = @entries.count
     end
 
   end
-end
+end