indy 0.4.0.pre → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: daf391715ba62b54e26aceed291bb60ca7e4b44c
-  data.tar.gz: f02335c95b62f424f1325e87b95729da4d3c77ce
+  metadata.gz: 712e234a85c0ab55eeb8b1c47eb7cd0bad77ee55
+  data.tar.gz: 9a366319c64e9f7db69c777c4dc9f66776b26a29
 SHA512:
-  metadata.gz: b5390c28c66aa47a30848e4fd72ea40753b8ae46cbc6c7f66b9f130c8adafbf9909e22d742f76271a311f48a21a3c4f416999de1cecda07485cf9c33a640ce6c
-  data.tar.gz: 5a6328e2c45a0872098192d5a3c73224cd487c54838ab4b7524483182bbe9b4d3dcce521d7065e6ad4782a35e8da45deee988d076fb9b8167980ed8dcb2da4e5
+  metadata.gz: 33b4f8537e0c32a94e3cf654e461a3aace37d51c528b812b1f2308e04e1f857ce1d9d791ee7d6dbd378a02847c04dd45924d146e2789d63416f448969f3a3c23
+  data.tar.gz: c8cb0409964768cd728968feaa82f53852b161aca0ac65fee9ed872946aca7fd2ccc951948f2e0f486495408dc95693c0625ab0ff06f75547e0690fcf53d00a9

data/.travis.yml

@@ -1,8 +1,8 @@
 language: ruby
 rvm:
-  - 2.0.0
+  - 2.2.1
   - 1.9.3
-  - jruby-19mode # JRuby in 1.9 mode
-  - rbx-19mode
   - 1.8.7
-  - ree
+  - ree
+  - rbx-2.2.7
+  - jruby-1.7.20

data/History.txt

@@ -1,8 +1,15 @@
-=== 0.4.0 / unreleased
+=== 0.5.0 / 2015-12-08
 
+* Improved time format handling
+* Multiline :entry_regexp now requires a capture group for the entire entry
 * Faster time scoped searches when using single line log formats
-* result[:entry] had replaced result[:line]
+* result.raw_entry has replaced result.line
 * Indy#all has replaced Indy#for(:all)
+* LogDefinition internal cleanup
+* Restored support for Ruby 1.8.7
+
+=== 0.4.0.pre (unreleased)
+
 
 === 0.3.4 / 2011-08-07
 

data/README.md

@@ -4,6 +4,8 @@ Indy: A Log Archaeology Tool
 Synopsis
 --------
 
+[![build status](https://travis-ci.org/bfaloona/Indy.png)](http://travis-ci.org/bfaloona/Indy)
+
 Log files are often searched for particular strings but are not often treated as data structures.  Indy attempts to deliver log content via more powerful features by allowing the ability to collect segments of a log from particular time; find a particular event; or monitor/reflect on a log to see if a particular event occurred (or not occurred).
 
 Installation
@@ -12,34 +14,36 @@ Installation
 To install Indy use the following command:
 
     $ gem install indy
-    
-(Add `sudo` if you're installing under a POSIX system as root)
-
-Compatibility
--------------
-
-[![build status](https://travis-ci.org/bfaloona/Indy.png)](http://travis-ci.org/bfaloona/Indy)
-
-Indy supports MacOS, *nix, and MS Windows and runs on the following ruby flavors:
-
-  - 1.8.7
-  - 1.9.3
-  - 2.0.0
-  - ree
-  - jruby-19mode
-  - rbx-19mode
 
 Usage
 -----
 
-## Example
+## Basic Example (default log pattern)
+
+    log = Indy.search(:file => './data.log').all.first.message
+    # => "Entering APPLICATION."
 
-    require 'indy'
-    log = Indy.search(File.open('my_log.txt','r')).with(Indy::LOG4R_FORMAT)
-    puts log.all.first.raw_entry
-    # => "2012-09-07 10:01:40 INFO MyApp - Entering APPLICATION."
-    puts log.after(:time => '2012-09-07 20:00:00').for(:application => 'MyApp').first.message
-    # => "Exiting Application"
+## Custom Example
+
+    # Given this log file:
+    #
+    # 2015/09/08 INFO #index Indexing application.
+    # 2015/09/11 ERROR #search Searching application.
+    # ...
+
+    log_contents = File.open('./example.log', 'r').read
+    pattern = /^(\d{4}\/\d{2}\/\d{2})\s+(INFO|DEBUG|WARN|ERROR)\s+(#\S+)\s+(.+)$/
+    fields = [:time, :severity, :method, :message]
+    custom_time_format = '%Y/%m/%d'
+
+    indy = Indy.search(  :source => log_contents,
+                         :entry_regexp => pattern,
+                         :entry_fields => fields,
+                         :time_format => custom_time_format
+               )
+    indy.after(:time => '2015/09/10').like(:severity => 'ERROR') do |entry|
+        puts "#{entry.time} (#{entry.method}) #{entry.message}"
+    end
 
 ## Specify your Source
 
@@ -84,18 +88,21 @@ allowing searches like so:
 
 ### Custom Log Format
 
-If you have a different log format you can brew your own.
-To do so, specify a Regexp pattern that captures each field you want to reference.
-Include it as the first item of your log format array, followed by a list of symbols that name the captured fields.
+Brew your own log format!
+To do so, specify an `:entry_regexp` pattern that captures each field you want to reference.
+Also, specify an `:entry_fields` array of symbols that name the fields captured by your regexp.
+If your date/time format differs from the default (%Y-%m-%d %H:%M:%S), you will need to specify a `:time_format` parameter.
 
     # If your log format is:
-    # HH:MM:SS SEVERITY APPLICATION#METHOD - MESSAGE
-    # Build an appropriate regexp
-    custom_regexp = /^(\d{2}:\d{2}:\d{2})\s*(INFO|DEBUG|WARN|ERROR)\s*([^#]+)#([^\s]+)\s*-\s*(.+)$/
-    # Combine the pattern and the list of fields
-    custom_log_format = [custom_regexp,:time,:severity,:application,:method,:message]
+    # YYYY-MM-DD HH:MM:SS SEVERITY APPLICATION#METHOD - MESSAGE
+
+    # Build a regexp pattern
+    pattern = /^(\d{4}.\d{2}.\d{2}\s+\d{2}.\d{2}.\d{2})\s*(INFO|DEBUG|WARN|ERROR)\s*([^#]+)#([^\s]+)\s*-\s*(.+)$/
+    # List the log fields
+    fields = [:time,:severity,:application,:method,:message]
+
     # Use Indy#with to define your format
-    Indy.search(source).with(custom_log_format).for(:severity => 'INFO', :method => 'allocate')
+    Indy.search(source).with(:entry_regexp => pattern, :entry_fields => fields)
 
 ### Predefined Log Format
 
@@ -107,42 +114,52 @@ Several log formats have been predefined for ease of configuration. See indy/for
     #
     # Example (Log4r)
     #  INFO mylog: This is a message with level INFO
-    Indy.new(:source => log_file, :log_format => Indy::LOG4R_DEFAULT_FORMAT).for(:application => 'mylog')
+    Indy.new(:source => log_file).with(Indy::LOG4R_DEFAULT_FORMAT).for(:application => 'mylog')
 
 ### Multiline log entries
 
-By default, Indy assumes that log lines are separated by new lines. Any lines that don't match the active pattern are ignored. To enable multiline log entries you must do two things:
+By default, Indy assumes that log lines are separated by new lines.
+Any lines that don't match the active pattern are ignored.
+To enable multiline log entries you MUST do these things:
 
 1. Use `Indy.new()` and include the `:multiline => true` parameter
 2. Use a log entry regexp that does not use `$` and/or `\n` to define the end of the entry.
+3. Add a capture group that surrounds one full log entry.
 
 #### Multiline Regexp tips
 
 * Use non-greedy matching when needed: `.*?` instead of `.*`
-* Assuming your log entries do not include a unique line ending, you can use a zero-width positive lookahead assertion to verify that each line is followed by the start of a valid log entry, or the end of the string. e.g.: `(?=^foo|\z)`
+* Assuming your log entries do not include a unique line ending, you can use a zero-width positive lookahead assertion to verify that each line is followed by the start of a valid log entry, or the end of the string. e.g.: if 'foo' starts each entry, use this assertion `(?=^foo|\z)` at the end of your `:entry_regexp`
 
 Check out [Regexp Extensions](http://www.ruby-doc.org/docs/ProgrammingRuby/html/language.html#UN)
 
 Example:
 
-    # Given this log containing two entries:
+    # Given this log containing three entries:
     #
     # INFO MyApp - Multiline message begins here...
     # and ends here
     # DEBUG MyOtherApp - Single line message.
-    # WARN MyOtherApp - Another single line message.
+    # WARN MyOtherApp - A third entry.
     
     severity_string = 'DEBUG|INFO|WARN|ERROR|FATAL'
 
-    # single line regexp would be:
+    multiline_regexp = /^((#{severity_string}) (\w+) - (.*?)(?=^#{severity_string}|\z))/
+
+    # For reference, a single line regexp would be:
     #                  /^(#{severity_string}) (\w+) - (.*)$/
-    multiline_regexp = /^(#{severity_string}) (\w+) - (.*?)(?=^#{severity_string}|\z)/
 
-    Indy.new( :multiline => true, :log_format => [multiline_regexp, :severity, :application, :message], :source => MY_LOG)
+
+    Indy.new( :multiline => true,
+              :entry_regexp => multiline_regexp,
+              :entry_fields => [:severity, :application, :message],
+              :source => MY_LOG
+            )
 
 ### Explicit Time Format
 
-By default, Indy tries to guess your time format (courtesy of DateTime#parse). If you supply an explicit time format, it will use DateTime#strptime, as well as try to guess.
+If not specified, Indy tries to guess your time format (courtesy of DateTime#parse).
+If you supply an explicit time format, it will use DateTime#strptime. If that fails, it will then guess with DateTime#parse.
 
 This is required when log data uses a non-standard date format, e.g.: U.S. format 12-31-2000, and must be used in
 conjunction with :entry_regexp and :entry_fields parameters.
@@ -152,7 +169,9 @@ conjunction with :entry_regexp and :entry_fields parameters.
                 :source => LOG_FILE,
                 :entry_regexp => /\d\d-\d\d-\d\d\d\d .*?/,
                 :entry_fields => [:time, :message]
-    ).all
+            ).all
+
+Format directives are documented in [DateTime#strftime](http://ruby-doc.org/stdlib-2.0.0/libdoc/date/rdoc/DateTime.html#method-i-strftime).
 
 ## Match Criteria
 
@@ -176,7 +195,7 @@ conjunction with :entry_regexp and :entry_fields parameters.
 
 ## Log Scopes
 
-Multiple scope methods can be called on an instance. Use #reset_scope to remove scope constrints on the instance.
+Multiple scope methods can be called on an instance. Use #reset_scope to remove scope constraints on the instance.
 
 ### Time Scope
 
@@ -206,11 +225,12 @@ Multiple scope methods can be called on an instance. Use #reset_scope to remove
 
 ## Process the Results
 
-A ResultSet is returned by #for and #like, which is an Enumerable containing a hash for each log entry.
+An Array is returned by #for and #like, containing a Struct::Entry for each log entry.
+The full entry is available with `entry.raw_entry`.
 
     entries = Indy.search(source).for(:message => 'Entering Application')
-    entries.first.keys
-    # => [:line, :time, :severity, :application, :message]
+    entries.first.members
+    # => ["time", "severity", "application", "message", "raw_entry"]
 
     Indy.search(source).for(:message => 'Entering Application').each do |entry|
       puts "[#{entry.time}] #{entry.message}: #{entry.application}"
@@ -228,15 +248,17 @@ To create a report in /coverage, run
     gem install simplecov
     COVERAGE=true rake test
 
-## Supported Rubies
+Compatibility
+-------------
 
-This library aims to support and is tested against the follow Ruby implementations:
-* Ruby 1.8.7
-* Ruby 1.9.2
-* Rubinius
+Indy supports MacOS, *nix, and MS Windows and runs on the following ruby flavors:
 
-Testing with RVM:
-    $ rvm 1.8.7,1.9.2,rbx exec rake test
+  - 2.2.1
+  - 1.9.3
+  - 1.8.7
+  - ree
+  - jruby-1.7.20
+  - rbx-2.2.7
 
 LICENSE
 -------

data/features/{file.feature → file_object.feature}

@@ -2,7 +2,7 @@
 Feature: Finding log entries in a file
 
   Background:
-    Given the following log file:
+    Given the following log file path:
     """
     spec/data.log
     """

data/features/file_path.feature

@@ -0,0 +1,30 @@
+@application
+Feature: Finding log entries in a file
+
+  Background:
+    Given the following log file object:
+    """
+    spec/data.log
+    """
+
+    
+  Scenario: Count of entries for a specific application
+    When searching the log for the application 'MyApp'
+    Then I expect to have found 2 log entries
+    
+    
+  Scenario: Particular entry for a specific application
+    When searching the log for the application 'MyApp'
+    Then I expect the first entry to be:
+    """
+    2000-09-07 14:07:41 INFO  MyApp - Entering application.
+    """
+    Then I expect the last entry to be:
+    """
+    2000-09-07 14:07:41 INFO  MyApp - Exiting application.
+    """
+
+    
+  Scenario: No entries for a specific application
+    When searching the log for the application 'YourApp'
+    Then I expect to have found no log entries

data/features/log_format_combined.feature

@@ -0,0 +1,15 @@
+@time @after
+Feature: Finding log messages using the combined log format
+  As an Indy user I am able to find data in a file that uses common log format
+
+  Background:
+    Given the following log, using COMBINED_LOG_FORMAT:
+    """
+    127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+    127.0.0.2 - adam [12/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+    127.0.0.3 - larry [13/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+    """
+
+  Scenario: Count of entries after a specified time
+    When searching the log for all entries after the time 2000-10-11 11:00:01
+    Then I expect to have found 2 log entries

data/features/log_format_common.feature

@@ -0,0 +1,19 @@
+@time @after
+Feature: Finding log messages using the common log format
+  As an Indy user I am able to find data in a file that uses common log format
+
+  Background:
+    Given the following log, using COMMON_LOG_FORMAT:
+    """
+    127.0.0.1 - frank [10/Oct/2010:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+    127.0.0.1 - frank [11/Oct/2010:13:56:36 -0700] "GET /apache_aa.gif HTTP/1.0" 200 2327
+    127.0.0.1 - sue [13/Oct/2010:13:56:37 -0700] "GET /apache_bb.gif HTTP/1.0" 200 2328
+    127.0.0.1 - larry [14/Oct/2010:13:56:38 -0700] "GET /apache_cc.gif HTTP/1.0" 200 2329
+    """
+  Scenario: Count of entries after a specified time
+    When searching the log for all entries after the time 2010-10-12 13:56:36
+    Then I expect to have found 2 log entries
+
+  Scenario: Count of entries before a COMMON formatted time
+    When searching the log for all entries before the time 14/Oct/2010 13:56:37 -0700
+    Then I expect to have found 3 log entries

data/features/step_definitions/find_by.steps.rb

@@ -30,6 +30,6 @@ When /^searching the log for the exact match of custom field ([^"]+)\s*"([^"]+)"
 end
 
 Then /^I expect the (first|last|\d+(?:st|nd|rd|th)) entry to be:$/ do |position,expected|
-  @results[position].raw_entry.should == expected
+  expect(@results[position].raw_entry).to eq(expected)
 end
 

data/features/step_definitions/log_file.steps.rb

@@ -1,12 +1,20 @@
 
-Given /^the following log file:$/ do |string|
+Given /^the following log file object:$/ do |string|
   @indy = Indy.search(File.open(string, 'r'))
 end
 
+Given /^the following log file path:$/ do |string|
+  @indy = Indy.search(:file => string)
+end
+
 Given /^the following log:$/ do |string|
   @indy = Indy.search(string)
 end
 
+Given /^the following log, using (COMMON_LOG_FORMAT|COMBINED_LOG_FORMAT):$/ do |log_format, string|
+  @indy = Indy.search(string).with("Indy::#{log_format}")
+end
+
 And /^the custom pattern \(([^\)]+)\):$/ do |fields,pattern|
   @indy = @indy.with({ :entry_regexp => pattern, :entry_fields => fields.split(',').map{|f| f.to_sym} })
-end
+end

data/features/step_definitions/support/transforms.rb

@@ -8,7 +8,7 @@ Transform /^(\d+)$/ do |number|
 end
 
 Then /^I expect to have found (no|\d+) log entr(?:y|ies)$/ do |count|
-  @results.size.should == count
+  expect(@results.size).to eq(count)
 end
 
 Transform /^first$/ do |order|

data/indy.gemspec

@@ -12,28 +12,29 @@ Gem::Specification.new do |s|
   s.license     = 'MIT'
 
   s.platform    = Gem::Platform::RUBY
-  s.required_ruby_version = '>= 1.8.5'
-  s.add_dependency('activesupport', '>= 2.3.5')
-
-  s.add_development_dependency('rake')
-  s.add_development_dependency('i18n')
-  s.add_development_dependency('cucumber', '>= 1.1.0')
-  s.add_development_dependency('yard', '>= 0.7.2')
-  s.add_development_dependency('rspec', '>= 2.9.0')
-  s.add_development_dependency('rspec-mocks', '>= 2.9.0')
-  s.add_development_dependency('rb-fsevent')
-  s.add_development_dependency('ruby_gntp')
-  s.add_development_dependency('growl')                                
+  s.required_ruby_version = '>= 1.8.7'
+  s.add_dependency('activesupport', '~> 3.1.12')
+
+  s.add_development_dependency('rake', '~> 10.4.0')
+  s.add_development_dependency('i18n', '~> 0.6.11')
+  s.add_development_dependency('cucumber', '~> 1.3.20')
+  s.add_development_dependency('yard', '~> 0.8.7.6')
+  s.add_development_dependency('rspec', '~> 3.4.0')
+  s.add_development_dependency('rspec-mocks', '~> 3.4.0')
+  s.add_development_dependency('rb-fsevent', '~> 0.9.0')
+  s.add_development_dependency('ruby_gntp', '~> 0.3.0')
+  s.add_development_dependency('growl', '~> 1.0.0')
 
   unless ENV['TRAVIS'] == 'true'
-    s.add_development_dependency('yard-cucumber', '>= 2.1.1')
-    s.add_development_dependency('flog', '>= 2.5.0')
-    s.add_development_dependency('guard')
+    s.add_development_dependency('yard-cucumber', '~> 2.3.0')
+    s.add_development_dependency('flog', '~> 4.3.0')
+    s.add_development_dependency('guard', '~> 1.8.0')
     unless ENV['RUBY_VERSION'] &&  ENV['RUBY_VERSION'].match(/jruby|rbx/)
-      s.add_development_dependency('guard-rspec')
-      s.add_development_dependency('guard-cucumber')
-      s.add_development_dependency('rspec-prof', '>= 0.0.3')
-      s.add_development_dependency('simplecov', '>= 0.4.0')
+      s.add_development_dependency('guard-rspec', '~> 1.2.0')
+      s.add_development_dependency('guard-cucumber', '~> 1.5.0')
+      s.add_development_dependency('ruby-prof', '~> 0.13.0')
+      s.add_development_dependency('rspec-prof', '~> 0.0.0')
+      s.add_development_dependency('simplecov', '~> 0.11.0')
     end
   end
 
@@ -52,8 +53,6 @@ Gem::Specification.new do |s|
 
   }
 
-  # s.rubygems_version  = "1.6.1"
-
   exclusions          = [File.join("performance", "large.log")]
   s.files             = `git ls-files`.split("\n") - exclusions
   s.test_files       = `git ls-files -- {spec,features,performance}/*`.split("\n")

data/lib/indy/indy.rb

@@ -14,13 +14,12 @@ class Indy
   #  Indy.new(:time_format => '%m-%d-%Y', :entry_regexp => LOG_REGEX_PATTERN, :entry_fields => [:time,:application,:message], :source => LOG_FILE)
   #
   def initialize(args)
-    params_hash = args.dup
-    raise ArgumentError, "Source parameter not specified" unless (params_hash.respond_to?(:keys) && params_hash.keys.include?(:source))
-    source_param = params_hash[:source]
-    params_hash.delete :source
-    log_definition = LogDefinition.new(params_hash)
-    @search = Search.new(:log_definition => log_definition)
-    @search.source = Source.new(source_param,log_definition)
+    params = args.dup
+    raise ArgumentError, "Source parameter not specified" unless (params.respond_to?(:keys) && params.keys.include?(:source))
+    source_param = params[:source]
+    params.delete :source
+    @search = Search.new()
+    @search.source = Source.new( source_param, LogDefinition.new(params) )
   end
 
   class << self
@@ -68,7 +67,10 @@ class Indy
   #  Indy.search(LOG_FILE).with(/^(\d{2}.\d{2}.\d{2})\s*(.+)$/,:time,:message)
   #
   def with(params=:default)
-    @search.log_definition = LogDefinition.new(params)
+    if params.kind_of?(String) && params.match(/^Indy::/)
+      params = params.constantize
+    end
+    @search.source.log_definition = LogDefinition.new(params)
     self
   end
 
@@ -119,8 +121,8 @@ class Indy
     raise ArgumentError, "Unsupported parameter to last(): #{scope_criteria.inspect}" unless scope_criteria.respond_to?(:keys) and scope_criteria[:span]
     span = (scope_criteria[:span].to_i * 60).seconds
     entry = last_entries(1)[0]
-    start_time = Indy::Time.parse_date(entry[:time],@search.log_definition.time_format) - span
-    within(:start_time => start_time, :end_time => Indy::Time.forever(@search.log_definition.time_format))
+    start_time = Indy::Time.parse_date(entry[:time], @search.source.log_definition.time_format) - span
+    within(:start_time => start_time, :end_time => Indy::Time.forever(@search.source.log_definition.time_format))
     self
   end
 
@@ -165,7 +167,7 @@ class Indy
   #
   def around(scope_criteria)
     raise ArgumentError unless scope_criteria.respond_to?(:keys) and scope_criteria[:time]
-    time = Indy::Time.parse_date(scope_criteria[:time])
+    time = Indy::Time.parse_date(scope_criteria[:time], @search.source.log_definition.time_format)
     mid_span = ((scope_criteria[:span].to_i * 60)/2).seconds rescue 300.seconds
     within(:start_time => time - mid_span, :end_time => time + mid_span, :inclusive => nil)
     self
@@ -207,14 +209,14 @@ class Indy
     result = []
     source_io = @search.source.open
     source_io.reverse_each do |entry|
-      hash = @search.log_definition.parse_entry(entry)
+      hash = @search.source.log_definition.parse_entry(entry)
       if hash
         num_entries += 1
         result << hash
         break if num_entries >= num
       end
     end
-    result.collect{|entry| @search.log_definition.create_struct(entry)}
+    result.collect{|entry| @search.source.log_definition.create_struct(entry)}
   end
 
 end