indy 0.1.1 → 0.1.2

data/lib/indy/log_formats.rb

@@ -0,0 +1,39 @@
+class Indy
+
+  #
+  # LogFormats defines the building blocks of Indy's built in log patterns.
+  #
+  # See indy/patterns.rb for the following Constants:
+  # Indy::COMMON_LOG_PATTERN
+  # Indy::COMBINED_LOG_PATTERN
+  # Indy::LOG4R_DEFAULT_PATTERN
+  #
+  module LogFormats
+
+    IPV4_REGEXP = '(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})'
+    SPACE_DELIM_REGEXP = '([^\s]+)'
+    BRACKET_DELIM_REGEXP = '\[([^\]]+)\]'
+    HTTP_REQUEST_REGEXP = '"([A-Z]+ [^\s]+ [^"]+)"'
+    HTTP_STATUS_REGEXP = '(\d{3})'
+    NUMBER_REGEXP = '(\d+)'
+    DQUOTE_DELIM_REGEXP = '"([^"]+)"'
+
+    DEFAULT_DATE_TIME = '\d{4}.\d{2}.\d{2}\s+\d{2}.\d{2}.\d{2}' #"%Y-%m-%d %H:%M:%S"
+    DEFAULT_SEVERITY = [:trace,:debug,:info,:warn,:error,:fatal]
+    DEFAULT_SEVERITY_PATTERN = "(?:#{DEFAULT_SEVERITY.map{|s| s.to_s.upcase}.join("|")})"
+    DEFAULT_APPLICATION = '\w+'
+    DEFAULT_MESSAGE = '.+'
+
+    DEFAULT_LOG_REGEXP = /^(#{DEFAULT_DATE_TIME})\s+(#{DEFAULT_SEVERITY_PATTERN})\s+(#{DEFAULT_APPLICATION})\s+-\s+(#{DEFAULT_MESSAGE})$/
+    DEFAULT_LOG_FIELDS = [:time,:severity,:application,:message]
+
+    COMMON_FIELDS = [:host, :ident, :authuser, :time, :request, :status, :bytes]
+    COMMON_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP}$/
+
+    COMBINED_FIELDS = COMMON_FIELDS + [:referrer, :user_agent]
+    COMBINED_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP} #{DQUOTE_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP}$/
+
+    LOG4R_DEFAULT_FIELDS = [:level, :application, :message]
+    LOG4R_DEFAULT_REGEXP = /(?:\s+)?([A-Z]+) (\S+): (.*)$/
+  end
+end

data/lib/indy/patterns.rb

@@ -0,0 +1,25 @@
+class Indy
+
+  #
+  # Indy default log format @pattern
+  # e.g.:
+  # INFO 2000-09-07 MyApp - Entering APPLICATION.
+  #
+  DEFAULT_LOG_PATTERN = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
+
+  #
+  # Uncustomized Log4r log @pattern
+  #
+  LOG4R_DEFAULT_PATTERN = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
+
+  #
+  # NCSA Common Log Format log @pattern
+  #
+  COMMON_LOG_PATTERN = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
+
+  #
+  # NCSA Combined Log Format log @pattern
+  #
+  COMBINED_LOG_PATTERN = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
+
+end

data/performance/profile_spec.rb

@@ -3,9 +3,9 @@ require "#{File.dirname(__FILE__)}/helper"
 describe "Search Performance" do
 
 
-  context "with a small data set" do
+  context "with a 420 line log file" do
 
-    longer_subject = [
+    log_content = [
       "2000-09-07 14:07:41 INFO  MyApp - Entering application.\n",
       "2000-09-07 14:07:42 DEBUG MyApp - Focusing application.\n",
       "2000-09-07 14:07:43 DEBUG MyApp - Blurring application.\n",
@@ -16,16 +16,16 @@ describe "Search Performance" do
 
     profile :file => STDOUT, :printer => :flat, :min_percent => 1  do
 
-      it "should perform well using #for(:all)" do
-        Indy.search(longer_subject.dup).for(:all)
+      it "should profile code using #for(:all)" do
+        Indy.search(log_content.dup).for(:all)
       end
 
-      it "should perform well using #for(:field => 'value')" do
-        Indy.search(longer_subject.dup).for(:severity => 'INFO')
+      it "should profile code using #for(:field => 'value')" do
+        Indy.search(log_content.dup).for(:severity => 'INFO')
       end
 
-      it "should perform well using #time()" do
-        Indy.search(longer_subject.dup).after(:time => "2000-09-07 14:07:45").for(:all)
+      it "should profile code using #time()" do
+        Indy.search(log_content.dup).after(:time => "2000-09-07 14:07:45").for(:all)
       end
 
     end

data/spec/indy_spec.rb

@@ -89,16 +89,17 @@ describe Indy do
       context "treat it first like a file" do
 
         it "should attempt to open the file" do
-          IO.should_receive(:open).with("possible_file.ext").ordered
+          File.should_receive(:exist?).with("possible_file.ext").ordered
           Indy.search("possible_file.ext")
         end
 
-        it "should not throw an error for an invalid file" do
+        it "should not throw an error for a non-existent file" do
           lambda { Indy.search("possible_file.ext") }.should_not raise_error
         end
 
         it "should return an IO object when there is a file" do
-          IO.should_receive(:open).with("file_exists.ext").and_return(StringIO.new("2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."))
+          File.should_receive(:exist?).with("file_exists.ext").and_return( true )
+          File.should_receive(:open).and_return(StringIO.new("2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."))
           Indy.search("file_exists.ext").for(:application => 'MyApp').length.should == 1
         end
 
@@ -111,13 +112,10 @@ describe Indy do
       context "treat it second like a string" do
 
         it "should attempt to treat it as a string" do
-          expecting_string = mock("String With Expectation")
-          expecting_string.should_receive(:[])
-          expecting_string.should_receive(:to_s).and_return("2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION.")
-
-          IO.should_receive(:open).with(expecting_string).ordered
-
-          Indy.search(expecting_string).for(:application => 'MyApp').length.should == 1
+          string = "2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."
+          string_io = StringIO.new(string)
+          StringIO.should_receive(:new).with(string).ordered.and_return(string_io)
+          Indy.search(string).for(:application => 'MyApp').length.should == 1
         end
 
       end
@@ -161,21 +159,23 @@ describe Indy do
       @indy.should respond_to(:with)
     end
 
-    # http://log4r.rubyforge.org/rdoc/Log4r/rdoc/patternformatter.html
-    it "with() should accept a log4r pattern string without error" do
-      lambda { @indy.with(["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]) }.should_not raise_error
+    it "with() should accept the log4r default pattern const without error" do
+      lambda { @indy.with(Indy::LOG4R_DEFAULT_PATTERN) }.should_not raise_error
     end
 
-    # http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html
-    it "with() should accept a log4j pattern string without error" do
-      lambda { @indy.with(["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message])}.should_not raise_error
+    it "with() should accept :default without error" do
+      lambda { @indy.with(:default) }.should_not raise_error
+    end
+
+    it "with() should accept no params without error" do
+      lambda { @indy.with() }.should_not raise_error
     end
 
     it "should return itself" do
-      @indy.with(["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]).should == @indy
+      @indy.with(:default).should == @indy
     end
 
-    [:for, :search, :like, :matching].each do |method|
+    [:for, :like, :matching].each do |method|
       it "#{method}() should exist" do
         @indy.should respond_to(method)
       end
@@ -190,10 +190,10 @@ describe Indy do
 
     end
 
-    context "_search when given source, param and value" do
+    context "_search" do
 
       before(:each) do
-        @results = @indy.send(:_search, StringIO.new("2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."),[Indy::DEFAULT_LOG_PATTERN, Indy::DEFAULT_LOG_FIELDS].flatten) {|result| result if result[:application] == "MyApp" }
+        @results = @indy.send(:_search) {|result| result if result[:application] == "MyApp" }
       end
 
       it "should not return nil" do

data/spec/log_format_spec.rb

@@ -0,0 +1,58 @@
+require "#{File.dirname(__FILE__)}/helper"
+
+describe Indy do
+
+  context "common logging format" do
+
+    common_log_pattern = {
+      :name => 'common_log_pattern',
+      :source => ["127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb1.gif HTTP/1.0\" 200 2326",
+                  "127.0.0.1 - louie [10/Oct/2000:13:55:37 -0700] \"GET /apache_pb2.gif HTTP/1.0\" 200 2327",
+                  "127.0.0.1 - frank [10/Oct/2000:13:55:38 -0700] \"GET /apache_pb3.gif HTTP/1.0\" 404 300"].join("\n"),
+      :regexp => Indy::LogFormats::COMMON_REGEXP,
+      :fields => Indy::LogFormats::COMMON_FIELDS,
+      :test_field => :authuser
+    }
+
+    combined_log_pattern = {
+      :name => 'combined_log_pattern',
+      :source => ["127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb1.gif HTTP/1.0\" 200 2326 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\"",
+                  "127.0.0.1 - louie [10/Oct/2000:13:55:37 -0700] \"GET /apache_pb2.gif HTTP/1.0\" 200 2327 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\"",
+                  "127.0.0.1 - frank [10/Oct/2000:13:55:38 -0700] \"GET /apache_pb3.gif HTTP/1.0\" 404 300 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\""].join("\n"),
+      :regexp => Indy::LogFormats::COMBINED_REGEXP,
+      :fields => Indy::LogFormats::COMBINED_FIELDS,
+      :test_field => :authuser
+    }
+
+    log4r_default_pattern = {
+      :name => 'log4r_default_pattern',
+      :source => ["DEBUG mylog: This is a message with level DEBUG",
+                  " INFO mylog: This is a message with level INFO",
+                  " WARN louie: This is a message with level WARN",
+                  "ERROR mylog: This is a message with level ERROR",
+                  "FATAL mylog: This is a message with level FATAL"].join("\n"),
+      :regexp => Indy::LogFormats::LOG4R_DEFAULT_REGEXP,
+      :fields => Indy::LogFormats::LOG4R_DEFAULT_FIELDS,
+      :test_field => :application
+    }
+    
+    [ common_log_pattern,
+      combined_log_pattern,
+      log4r_default_pattern ].each do |format|
+      
+      it "#{format[:name]} should work" do
+        indy = Indy.new(:source => format[:source], :pattern => [format[:regexp],format[:fields]].flatten)
+        result = indy.for(format[:test_field] => 'louie')
+        result.count.should == 1
+      end
+
+      it "#{format[:name]} @pattern can be set to the Indy::LogFormat const" do
+        indy = Indy.new(:source => format[:source], :pattern => eval('Indy::' + format[:name].upcase))
+        result = indy.for(format[:test_field] => 'louie')
+        result.count.should == 1
+
+      end
+    end
+  end
+
+end

data/spec/search_spec.rb

@@ -24,6 +24,29 @@ describe Indy do
 
   end
 
+  context "file operations" do
+
+    before(:all) do
+    end
+
+    it "should open and close file once per search" do
+      file_path =  File.join( File.dirname(__FILE__), 'data.log')
+      file_io1 = File.open(file_path)
+      file_io2 = File.open(file_path)
+
+      File.should_receive( :exist? ).with(file_path).ordered.and_return(true)
+      File.should_receive( :open ).ordered.and_return( file_io1 )
+      file_io1.should_receive( :close ).ordered
+      File.should_receive( :open ).ordered.and_return( file_io2 )
+      file_io2.should_receive( :close ).ordered
+
+      @indy = Indy.search(file_path)
+      @indy.for(:all).length.should == 2
+      @indy.for(:all).length.should == 2
+    end
+
+  end
+
   context "search file" do
 
     before(:all) do
@@ -80,15 +103,12 @@ describe Indy do
     it "should execute cmd on each successive search" do
       @indy.for(:application => 'MyApp').length.should == 2
       @indy.for(:severity => 'INFO').length.should == 3
-      @indy.for(:application => 'MyApp').length.should == 2
-    end
 
-    it "should execute cmd on each successive search" do
-      @file.write("\n2000-09-07 14:10:55 INFO  MyApp - really really Exiting APPLICATION.\n")
+      @file.write("\n2000-09-07 14:10:55 DEBUG MyApp - really really Exiting APPLICATION.\n")
       @file.flush
+
       @indy.for(:application => 'MyApp').length.should == 3
-      @indy.for(:severity => 'INFO').length.should == 4
-      @indy.for(:application => 'MyApp').length.should == 3
+      @indy.for(:severity => 'INFO').length.should == 3
     end
 
 

data/spec/time_spec.rb

@@ -13,6 +13,22 @@ describe Indy do
       @indy.send(:parse_date, line_hash).class.should == DateTime
     end
 
+    it "should parse dates when log includes non-conforming data" do
+      logdata = [ "12-03-2000 message1",
+                  "13-03-2000 message2",
+                  "14-03-2000 ",
+                  " message4",
+                  "a14-03-2000 message5",
+                  "14-03-2000 message6\n\n\n\n",
+                  "15-03-2000 message7",
+                  "16-03-2000 message8\r\n",
+                  "17-03-2000 message9"].join("\n")
+      @indy = Indy.new(:source => logdata, :pattern => ['^(\d[^\s]+\d) (.+)$', :time, :message])
+      @indy.after(:time => '13-03-2000')
+      @indy.for(:all).count.should == 4
+    end
+
+
   end
 
   context "non-default time handling" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: indy
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 31
   prerelease: 
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Franklin Webber
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-13 00:00:00 -08:00
+date: 2011-01-17 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -35,7 +35,103 @@ dependencies:
         version: 2.3.5
   type: :runtime
   version_requirements: *id001
-description: " Indy is a log archelogy tool that allows you to search through log files. "
+- !ruby/object:Gem::Dependency 
+  name: cucumber
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 0
+        - 9
+        - 2
+        version: 0.9.2
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 2
+        - 4
+        - 0
+        version: 2.4.0
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rspec-mocks
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 2
+        - 4
+        - 0
+        version: 2.4.0
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rspec-prof
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 0
+        - 0
+        - 3
+        version: 0.0.3
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 41
+        segments: 
+        - 0
+        - 9
+        - 9
+        version: 0.9.9
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: flog
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 2
+        - 5
+        - 0
+        version: 2.5.0
+  type: :development
+  version_requirements: *id007
+description: " Indy is a log archelogy tool that allows you to interact with log data like an object while you search by fields and/or time."
 email: franklin.webber@gmail.com
 executables: []
 
@@ -63,7 +159,6 @@ files:
 - features/exact_mulitple_fields.feature
 - features/exact_time.feature
 - features/file.feature
-- features/log_levels.feature
 - features/message.feature
 - features/multiple_fields.feature
 - features/step_definitions/find_by.steps.rb
@@ -77,12 +172,15 @@ files:
 - indy.gemspec
 - lib/indy.rb
 - lib/indy/indy.rb
+- lib/indy/log_formats.rb
+- lib/indy/patterns.rb
 - lib/indy/result_set.rb
 - performance/helper.rb
 - performance/profile_spec.rb
 - spec/data.log
 - spec/helper.rb
 - spec/indy_spec.rb
+- spec/log_format_spec.rb
 - spec/result_set_spec.rb
 - spec/search_spec.rb
 - spec/time_spec.rb
@@ -90,7 +188,7 @@ has_rdoc: true
 homepage: http://github.com/burtlo/Indy
 licenses: 
 - MIT
-post_install_message: "\n    [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n    Thank you for installing Indy 0.1.1 / 2011-01-13.\n\n    Changes:\n      \n  * Scope search by time (after, before, around, or within)\n  * Explicit format string can be provided to parse non-standard log times\n  * Reopen (or rewind) source for each #search\n  * Source parameter creates an IO object (StringIO, File, or Process)\n  \n\n    [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n  "
+post_install_message: "\n    [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n    Thank you for installing Indy 0.1.2 / 2011-01-18.\n\n    Changes:\n      \n  * Predefined log formats for NCSA Common, NCSA Combined, and Log4r (default)\n  * Source IO is explicitly closed after each #_search\n  * Removed instance method #search; use #for.\n  * Removed instance method #severity\n  \n\n    [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n  "
 rdoc_options: 
 - --charset=UTF-8
 require_paths: