indy 0.1.5 → 0.2.0

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/History.txt

@@ -1,3 +1,14 @@
+=== 0.2.0 / 2011-03-08
+
+* Support for Multiline log entries. See README.md
+* Time scopes now respect inclusive flag
+* #last() no longer supports number of rows as a parameter. Use :span => minutes.
+* Fixes for Ruby 1.9.2
+
+=== 0.1.6 / 2011-03-07
+
+* Unsupported. Gem version 0.1.6 == 0.2.0
+
 === 0.1.5 / 2011-01-21
 
 * Searching with time scopes (#after, #within, #before) are much faster

data/README.md

@@ -40,16 +40,22 @@ Usage
 
     Indy.search(log_string).for(:message => 'Entering application')
 
-## Specify your Pattern
-
-The default search pattern resembles something you might find:
-
-    YYYY-MM-DD HH:MM:SS SEVERITY APPLICATION_NAME - MESSAGE
+## Log Pattern
 
 ### Default Log Pattern
   
-   Indy.search(source).for(:severity => 'INFO')
-   Indy.search(source).for(:application => 'MyApp', :severity => 'DEBUG')
+The default search pattern follows this form:  
+YYYY-MM-DD HH:MM:SS SEVERITY APPLICATION_NAME - MESSAGE
+
+Which uses this regexp:
+    /^(\d{4}.\d{2}.\d{2}\s+\d{2}.\d{2}.\d{2})\s+(TRACE|DEBUG|INFO|WARN|ERROR|FATAL)\s+(\w+)\s+-\s+(.+)$/
+
+and specifies these fields:  
+    [:time, :severity, :application, :message]
+
+For example:  
+    Indy.search(source).for(:severity => 'INFO')
+    Indy.search(source).for(:application => 'MyApp', :severity => 'DEBUG')
 
 ### Custom Log Pattern
 
@@ -73,6 +79,36 @@ Several log formats have been predefined for ease of configuration. See indy/pat
     #  INFO mylog: This is a message with level INFO
     Indy.new(:source => 'logfile.txt', :pattern => Indy::LOG4R_DEFAULT_PATTERN).for(:application => 'mylog')
 
+### Multiline log entries
+
+By default, Indy assumes that log lines are separated by new lines. Any lines that don't match the active pattern are ignored. To enable multiline log entries you must do two things:
+
+1. Use `Indy.new()` and include the `:multiline => true` parameter
+2. Use a log entry regexp that does not use `$` and/or `\n` to define the end of the entry.
+
+#### Multiline Regexp tips
+
+* Use non-greedy matching when needed: `.*?` instead of `.*`
+* Assuming your log entries do not include a unique line ending, you can use a zero-width positive lookahead assertion to verify that each line is followed by the start of a valid log entry, or the end of the string. e.g.: `(?=^foo|\z)`
+
+Check out [Regexp Extensions](http://www.ruby-doc.org/docs/ProgrammingRuby/html/language.html#UN)
+
+Example:
+
+    # Given this log containing two entries:
+    #
+    # INFO MyApp - Multiline message begins here
+    # and ends here
+    # DEBUG MyOtherApp - Single line message
+    
+    severity_string = 'DEBUG|INFO|WARN|ERROR|FATAL'
+
+    # single line regexp would be:
+    #                  /^(#{severity_string}) (\w+) - (.*)$/
+    multiline_regexp = /^(#{severity_string}) (\w+) - (.*?)(?=^#{severity_string}|\z)/
+
+    Indy.new( :multiline => true, :pattern => [multiline_regexp, :severity, :application, :message], :source => MY_LOG)
+
 ### Explicit Time Format
 
 By default, Indy tries to guess your time format (courtesy of DateTime#parse). If you supply an explicit time format, it will use DateTime#strptime, as well as try to guess.
@@ -94,13 +130,6 @@ This is required when log data uses a non-standard date format, e.g.: U.S. forma
     Indy.search(source).for(:message => 'Entering Application', :application => 'MyApp')
     Indy.search(source).for(:severity => 'INFO', :application => 'MyApp')
 
-### Time Scope
-
-    Indy.search(source).after(:time => '2011-01-13 13:40:00').for(:all)
-    Indy.search(source).before(:time => '2010-12-31 23:59:59').for(:all)
-    Indy.search(source).around(:time => '2011-01-01 00:00:00', :span => 2).for(:all) # 2 minutes around New Year's Eve
-    Indy.search(source).within(:time => ['2011-01-01 00:00:00','2011-02-01 00:00:00']).for(:severity => 'ERROR', :application => 'MyApp')
-
 ### Partial Match
 
     Indy.search(source).like(:message => 'Memory')
@@ -109,17 +138,46 @@ This is required when log data uses a non-standard date format, e.g.: U.S. forma
 
     Indy.search(source).like(:severity => '(?:INFO|DEBUG)', :message => 'Memory')
 
+## Log Scopes
+
+Multiple scope methods can be called on an instance. Use #reset_scope to remove scope constrints on the instance.
+
+### Time Scope
+
+    # After Dec 1
+    Indy.search(source).after(:time => '2010-12-01 23:59:59').for(:all)
+
+    # 20 minutes Around New Year's eve
+    Indy.search(source).around(:time => '2011-01-01 00:00:00', :span => 20).for(:all)
+
+    # After Jan 1 but Before Feb 1
+    @log = Indy.search(source)
+    @log.after(:time => '2011-01-01 00:00:00').before(:time => '2011-02-01 00:00:00')
+    @log.for(:all)
+
+    # Within Jan 1 and Feb 1 (same time scope as above)
+    Indy.search(source).within(:time => ['2011-01-01 00:00:00','2011-02-01 00:00:00']).for(:all)
+
+    # After Jan 1
+    @log = Indy.search(source)
+    @log.after(:time => '2011-01-01 00:00:00')
+    @log.for(:all)
+    # Reset the time scope to include entries before Jan 1
+    @log.reset_scope
+    # Before Feb 1
+    @log.before(:time => '2011-02-01 00:00:00')
+    @log.for(:all)
+
 ## Process the Results
 
-    A ResultSet (Array) is returned by #for, #like, #after, etc.
+A ResultSet is returned by #for and #like, which is an Enumerable containing a hash for each log entry.
 
     entries = Indy.search(source).for(:message => 'Entering Application')
+    entries.first.keys
+    # => [:line, :time, :severity, :application, :message]
 
     Indy.search(source).for(:message => 'Entering Application').each do |entry|
-
-      # each log line entry returned is an OpenStruct object
       puts "[#{entry.time}] #{entry.message}: #{entry.application}"
-
     end
 
 LICENSE

data/Rakefile

@@ -1,9 +1,9 @@
 require 'rake'
 require 'rspec/core'
 require 'rspec/core/rake_task'
+require 'rcov/rcovtask'
 require "cucumber/rake/task"
 require "yard"
-require "city"
 
 desc 'Default: run tests'
 task :default => :test
@@ -55,15 +55,16 @@ task :flog_detail do
   system('find lib -name \*.rb | xargs flog -d')
 end
 
-desc "Generate code coverage"
-RSpec::Core::RakeTask.new(:coverage) do |t|
-  t.pattern = "./spec/**/*_spec.rb" # don't need this, it's default.
-  t.rcov = true
+# Task :rcov -- Run RCOV to Generate code coverage report
+Rcov::RcovTask.new do |t|
+  t.libs << "lib"
+  t.test_files = FileList['spec/*.rb']
   t.rcov_opts = ['--exclude', 'spec', '--exclude', 'gems', '-T']
+  t.verbose = true
 end
 
-
-YARD::Rake::CitydocTask.new do |t|
+# Task :yard -- Generate yard + yard-cucumber docs
+YARD::Rake::YardocTask.new do |t|
   t.files   = ['features/**/*', 'lib/**/*.rb']
   t.options = ['--private']
 end

data/features/README.md

@@ -0,0 +1,4 @@
+Indy
+====
+
+Searching through logs includes before, around, after, and exact matching.

data/features/around_time.feature

@@ -30,5 +30,5 @@ Scenario: Count of entries for a time span before and including a specified time
     Then I expect to have found 2 log entries
 
   Scenario: Count of entries for a time span after and including a specified time
-    When searching the log for all entries 31 minutes after and including the time 2000-09-07 14:17:43
-    Then I expect to have found 3 log entries
+    When searching the log for all entries 30 minutes after and including the time 2000-09-07 14:17:43
+    Then I expect to have found 4 log entries

data/indy.gemspec

@@ -1,6 +1,6 @@
 require File.dirname(__FILE__) + "/lib/indy"
 
-module Indy
+class Indy
 
   def self.show_version_changes(version)
     date = ""
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.authors     = ["Franklin Webber","Brandon Faloona"]
   s.description = %{ Indy is a log archelogy library that treats logs like data structures. Search fixed format or custom logs by field and/or time. }
   s.summary     = "Log Search Library"
-  s.email       = 'franklin.webber@gmail.com'
+  s.email       = 'brandon@faloona.net'
   s.homepage    = "http://github.com/burtlo/Indy"
   s.license     = 'MIT'
 
@@ -40,9 +40,9 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 1.8.5'
   s.add_dependency('activesupport', '>= 2.3.5')
 
-  s.add_development_dependency('cucumber', '>= 0.9.2')
+  s.add_development_dependency('cucumber', '>= 0.10.0')
   s.add_development_dependency('yard', '>= 0.6.4')
-  s.add_development_dependency('cucumber-in-the-yard', '>= 1.7.7')
+  s.add_development_dependency('yard-cucumber', '>= 2.0.0')
   s.add_development_dependency('rspec', '>= 2.4.0')
   s.add_development_dependency('rspec-mocks', '>= 2.4.0')
   s.add_development_dependency('rspec-prof', '>= 0.0.3')
@@ -62,11 +62,12 @@ Gem::Specification.new do |s|
 
   }
 
-  #  exclusions          = [File.join("performance", "large.log")]
-  #  s.files             = `git ls-files`.split("\n") - exclusions
-
+  
   s.rubygems_version  = "1.3.7"
-  s.files             = `git ls-files`.split("\n")
+
+  exclusions          = [File.join("performance", "large.log")]
+  s.files             = `git ls-files`.split("\n") - exclusions
+
   s.extra_rdoc_files  = ["README.md", "History.txt"]
   s.rdoc_options      = ["--charset=UTF-8"]
   s.require_path      = "lib"

data/lib/indy/indy.rb

@@ -4,7 +4,7 @@ class Indy
 
   class InvalidSource < Exception; end
 
-  VERSION = "0.1.5"
+  VERSION = "0.2.0"
 
   #
   # hash with one key (:string, :file, or :cmd) set to the string that defines the log
@@ -22,6 +22,15 @@ class Indy
   #
   attr_accessor :time_format
 
+  #
+  # initialization flag required if multiline log entries are allowed
+  #
+  # @example
+  #
+  #  Indy.new(:source => MY_LOG, :pattern => [MY_REGEXP, FIELD1, FIELD2, FIELD3], :multiline => true)
+  #
+  attr_accessor :multiline
+
   #
   # Initialize Indy.
   #
@@ -34,14 +43,14 @@ class Indy
   #  Indy.new(:time_format => '%m-%d-%Y',:pattern => [LOG_REGEX_PATTERN,:time,:application,:message],:source => LOG_FILENAME)
   #
   def initialize(args)
-    @source = @pattern = @time_format = @log_regexp = @log_fields = nil
+    @source = @pattern = @time_format = @log_regexp = @log_fields = @multiline = nil
     @source = Hash.new
 
     while (arg = args.shift) do
       send("#{arg.first}=",arg.last)
     end
 
-    update_log_pattern(@pattern)
+    update_log_pattern( @pattern )
 
   end
 
@@ -71,7 +80,7 @@ class Indy
     #
     def search(params=nil)
   
-      raise Indy::InvalidSource if params.nil?
+      raise Indy::InvalidSource if params.nil? || params.is_a?(Fixnum)
 
       if params.respond_to?(:keys) && params[:source]
         Indy.new(params)
@@ -111,8 +120,6 @@ class Indy
   def for(search_criteria)
     results = ResultSet.new
 
-    define_struct
-
     case search_criteria
     when Enumerable
       results += _search do |result|
@@ -139,7 +146,6 @@ class Indy
   #
   def like(search_criteria)
     results = ResultSet.new
-    define_struct
 
     results += _search do |result|
       create_struct(result) if search_criteria.reject {|criteria,value| result[criteria] =~ /#{value}/ }.empty?
@@ -152,7 +158,74 @@ class Indy
 
 
   #
-  # After scopes the eventual search to all entries after to this point.
+  # Last() scopes the eventual search to the last N minutes worth of entries.
+  #
+  # @param [Hash] scope_criteria hash describing the amount of time at
+  # the last portion of the source
+  #
+  # @example For last 10 minutes worth of entries
+  #
+  #   Indy.search(LOG_FILE).last(:span => 100).for(:all)
+  #
+  def last(scope_criteria)
+    case scope_criteria
+    when Enumerable
+      raise ArgumentError unless scope_criteria[:span] || scope_criteria[:rows]
+      
+      if scope_criteria[:span]
+        span = (scope_criteria[:span].to_i * 60).seconds
+        starttime = parse_date(last_entry[:_time]) - span
+
+        within(:time => [starttime, forever])
+      end
+    else
+      raise ArgumentError, "Invalid parameter: #{scope_criteria.inspect}"
+    end
+
+    self
+  end
+
+  #
+  # Return a Struct::Line for the last valid entry from the source
+  #
+  def last_entry
+    last_entries(1)
+  end
+
+  #
+  # Return an array of Struct::Line entries for the last N valid entries from the source
+  #
+  # @param [Fixnum] num the number of rows to retrieve
+  #
+  def last_entries(num)
+
+    num_entries = 0
+    result = []
+
+    source_io = open_source
+    source_io.reverse_each do |line|
+
+      hash = parse_line(line)
+
+      set_time(hash) if @time_field
+      
+      if hash
+        num_entries += 1
+        result << hash
+        break if num_entries >= num
+      end
+    end
+
+    warn "No matching lines found in source: #{source_io.class}" if result.empty?
+
+    source_io.close if @source[:file] || @source[:cmd]
+
+    num == 1 ? create_struct(result.first) : result.collect{|e| create_struct(e)}
+  end
+
+
+  #
+  # After() scopes the eventual search to all entries after to this point.
   #
   # @param [Hash] scope_criteria the field to scope for as the key and the
   #        value to compare against the other log messages
@@ -164,7 +237,7 @@ class Indy
   def after(scope_criteria)
     if scope_criteria[:time]
       time = parse_date(scope_criteria[:time])
-      @inclusive = scope_criteria[:inclusive] || false
+      @inclusive = @inclusive || scope_criteria[:inclusive] || nil
 
       if scope_criteria[:span]
         span = (scope_criteria[:span].to_i * 60).seconds
@@ -178,7 +251,15 @@ class Indy
   end
 
   #
-  # Before scopes the eventual search to all entries prior to this point.
+  # reset_time_scope removes any existing start and end times from the instance
+  # Otherwise consecutive calls retain state
+  #
+  def reset_scope
+    @inclusive = @start_time = @end_time = nil
+  end
+    
+  #
+  # Before() scopes the eventual search to all entries prior to this point.
   #
   # @param [Hash] scope_criteria the field to scope for as the key and the
   #        value to compare against the other log messages
@@ -191,7 +272,7 @@ class Indy
   def before(scope_criteria)
     if scope_criteria[:time]
       time = parse_date(scope_criteria[:time])
-      @inclusive = scope_criteria[:inclusive] || false
+      @inclusive = @inclusive || scope_criteria[:inclusive] || nil
 
       if scope_criteria[:span]
         span = (scope_criteria[:span].to_i * 60).seconds
@@ -208,8 +289,8 @@ class Indy
     if scope_criteria[:time]
       time = parse_date(scope_criteria[:time])
 
-      # does @inclusive add any real value to the #around method?
-      @inclusive = scope_criteria[:inclusive] || false
+      @inclusive = nil
+      warn "Ignoring inclusive scope_criteria" if scope_criteria[:inclusive]
 
       half_span = ((scope_criteria[:span].to_i * 60)/2).seconds rescue 300.seconds
       within(:time => [time - half_span, time + half_span])
@@ -220,7 +301,7 @@ class Indy
 
 
   #
-  # Within scopes the eventual search to all entries between two points.
+  # Within() scopes the eventual search to all entries between two points.
   #
   # @param [Hash] scope_criteria the field to scope for as the key and the
   #        value to compare against the other log messages
@@ -233,7 +314,7 @@ class Indy
     if scope_criteria[:time]
       @start_time, @end_time = scope_criteria[:time].collect {|str| parse_date(str) }
 
-      @inclusive = scope_criteria[:inclusive] || false
+      @inclusive = @inclusive || scope_criteria[:inclusive] || nil
     end
 
     self
@@ -288,6 +369,9 @@ class Indy
 
     @time_field = ( @log_fields.include?(:time) ? :time : nil )
 
+    # now that we know the fields
+    define_struct
+
   end
 
   #
@@ -302,27 +386,43 @@ class Indy
     time_search = use_time_criteria?
 
     source_io = open_source
-    results = source_io.collect do |line|
 
-      hash = parse_line(line)
+    if @multiline
+      results = source_io.read.scan(Regexp.new(@log_regexp, Regexp::MULTILINE)).collect do |entry|
 
-      hash ? (line_matched = true) : next
-      
-      if time_search
-        set_time(hash)
-        next unless inside_time_window?(hash)
-      else
-        hash[:_time] = nil if hash
+        hash = parse_line(entry)
+        hash ? (line_matched = true) : next
+
+        if time_search
+          set_time(hash)
+          next unless inside_time_window?(hash)
+        else
+          hash[:_time] = nil if hash
+        end
+
+        block_given? ? block.call(hash) : nil
       end
 
-      block_given? ? block.call(hash) : nil
+    else
+      results = source_io.collect do |line|
+        hash = parse_line(line)
+        hash ? (line_matched = true) : next
+
+        if time_search
+          set_time(hash)
+          next unless inside_time_window?(hash)
+        else
+          hash[:_time] = nil if hash
+        end
+
+        block_given? ? block.call(hash) : nil
+      end
 
     end
 
     warn "No matching lines found in source: #{source_io.class}" unless line_matched
 
-    source_io.close if @source[:file] || @source[:cmd]
-
+    source_io.close if @source[:file] || @source[:cmd]    
     results.compact
   end
 
@@ -365,17 +465,26 @@ class Indy
   #
   def parse_line( line )
 
-    match_data = /#{@log_regexp}/.match(line)
+    if line.kind_of? String
+      match_data = /#{@log_regexp}/.match(line)
+      return nil unless match_data
 
-    if match_data
       values = match_data.captures
-      raise "Field mismatch between log pattern and log data. The data is: '#{values.join(':::')}'" unless values.length == @log_fields.length
+      entire_line = line.strip
 
-      hash = Hash[ *@log_fields.zip( values ).flatten ]
-      hash[:line] = line.strip
+    elsif line.kind_of? Enumerable
 
-      hash
+      entire_line = line.shift
+      values = line 
     end
+    
+    raise "Field mismatch between log pattern and log data. The data is: '#{values.join(':::')}'" unless values.length == @log_fields.length
+
+    hash = Hash[ *@log_fields.zip( values ).flatten ]
+    hash[:line] = entire_line.strip
+
+
+    hash
   end
 
   #
@@ -426,8 +535,8 @@ class Indy
   def parse_date(param)
     return nil unless @time_field
     return param if param.kind_of? Time or param.kind_of? DateTime
-
-    time_string = param[@time_field] ? param[@time_field] : param
+    
+    time_string = param.is_a?(Hash) ? param[@time_field] : param.to_s
 
     if @time_format
       begin