indy 0.2.0 → 0.3.0

data/lib/indy/log_formats.rb

@@ -3,10 +3,9 @@ class Indy
   #
   # LogFormats defines the building blocks of Indy's built in log patterns.
   #
-  # See indy/patterns.rb for the following Constants:
-  # Indy::COMMON_LOG_PATTERN
-  # Indy::COMBINED_LOG_PATTERN
-  # Indy::LOG4R_DEFAULT_PATTERN
+  # Indy::COMMON_LOG_FORMAT
+  # Indy::COMBINED_LOG_FORMAT
+  # Indy::LOG4R_DEFAULT_FORMAT
   #
   module LogFormats
 
@@ -36,4 +35,27 @@ class Indy
     LOG4R_DEFAULT_FIELDS = [:level, :application, :message]
     LOG4R_DEFAULT_REGEXP = /(?:\s+)?([A-Z]+) (\S+): (.*)$/
   end
+
+  #
+  # Indy default log format 
+  # e.g.:
+  # INFO 2000-09-07 MyApp - Entering APPLICATION.
+  #
+  DEFAULT_LOG_FORMAT = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
+
+  #
+  # Uncustomized Log4r log format
+  #
+  LOG4R_DEFAULT_FORMAT = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
+
+  #
+  # NCSA Common Log Format log format
+  #
+  COMMON_LOG_FORMAT = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
+
+  #
+  # NCSA Combined Log Format log format
+  #
+  COMBINED_LOG_FORMAT = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
+
 end

data/lib/indy/notes.txt

@@ -0,0 +1,9 @@
+
+bisect_log (by time)
+
+first_entry._time
+last_entry._time
+log_size
+median_entry
+tail( entries )
+

data/lib/indy/result_set.rb

@@ -1,8 +1,8 @@
-#
-# The current implementation extends an Array.  However, as this
-# has not been tested with a large data set it may need some new
-# Enumerable object that is multi-threaded and performance friendly
-#
-class ResultSet < Array
-
-end
+#
+# The current implementation extends an Array.  However, as this
+# has not been tested with a large data set it may need some new
+# Enumerable object that is multi-threaded and performance friendly
+#
+class ResultSet < Array
+
+end

data/lib/indy/source.rb

@@ -0,0 +1,129 @@
+class Indy
+
+  # 
+  # A StringIO interface to the underlying log source.
+  # 
+  class Source
+
+    # log source type. :cmd, :file, or :string
+    attr_reader :type
+
+    # log source connection string (cmd, filename or log data)
+    attr_reader :connection
+
+    # the StringIO object
+    attr_reader :io
+
+    # Exception raised when unable to open source
+    class Invalid < Exception; end
+
+    ##
+    # Creates a Source object.
+    #
+    # @param [String, Hash] param The source content String, filepath String, or :cmd => 'command' Hash
+    #
+    def initialize(param)
+      raise Indy::Source::Invalid if param.nil?
+      if param.respond_to?(:keys)
+        set_connection(:cmd, param[:cmd]) if param[:cmd]
+        set_connection(:file, param[:file]) if ( param[:file] and param[:file].size > 0 )
+        set_connection(:string, param[:string]) if param[:string]
+      elsif param.respond_to?(:read) and param.respond_to?(:rewind)
+          set_connection(:file, param)
+      elsif param.respond_to?(:to_s) and param.respond_to?(:length)
+        # fall back to source being the string passed in
+        set_connection(:string, param)
+      else
+        raise Indy::Source::Invalid
+      end
+
+
+    end
+
+    #
+    # set the source connection type and connection_string
+    #
+    def set_connection(type, value)
+      @type = type
+      @connection = value
+    end
+
+    #
+    # Return a StringIO object to provide access to the underlying log source
+    #
+    def open(time_search=nil)
+      begin
+
+        case @type
+        when :cmd
+          @io = StringIO.new( exec_command(@connection).read )
+          raise "Failed to execute command (#{@connection})" if @io.nil?
+
+        when :file
+          @connection.rewind
+          @io = StringIO.new(@connection.read)
+          raise "Failed to open file: #{@connection}" if @io.nil?
+
+        when :string
+          @io = StringIO.new( @connection )
+
+        else
+          raise RuntimeError, "Invalid log source type: #{@type.inspect}"
+        end
+
+      rescue Exception => e
+        raise Indy::Source::Invalid, "Unable to open log source. (#{e.message})"
+      end
+
+      # scope_by_time(source_io) if time_search
+
+      @io
+    end
+    
+    #
+    # Execute the source's connection string, returning an IO object
+    #
+    # @param [String] command_string string of command that will return log contents
+    #
+    def exec_command(command_string)
+      begin
+        io = IO.popen(command_string)
+        return nil if io.eof?
+      rescue
+        nil
+      end
+      io
+    end
+
+    
+
+    #
+    # the number of lines in the source
+    #
+    def num_lines
+      load_data unless @num_lines
+      @num_lines
+    end
+
+    #
+    # array of log lines from source
+    #
+    def lines
+      load_data unless @lines
+      @lines
+    end
+
+    #
+    # read source data and populate instance variables
+    #
+    # TODO: hmmm... not called when Source#open is called directly, but #load_data would call open again. :(
+    #
+    def load_data
+      self.open
+      @lines = @io.readlines
+      @io.rewind
+      @num_lines = @lines.count
+    end
+
+  end
+end

data/lib/scanf.rb

@@ -0,0 +1,13 @@
+
+require 'scanf'
+
+f = File.open('spec/multiline.log')
+
+first_capture_regexp = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
+
+full_regexp = "^(#{first_capture_regexp})\\s+([A-Z]+)\\s+(.+?)(?=^#{first_capture_regexp}|\\z)"
+  
+  f.read.scan(/#{full_regexp}/m) do |entry|
+  puts "Entry::::\n#{entry}"
+end
+

data/performance/profile_spec.rb

@@ -4,7 +4,7 @@ describe "Search Performance" do
 
   context "with a 10000 line log file" do
 
-    large_file = "#{File.dirname(__FILE__)}/large.log"
+    large_file = File.open("#{File.dirname(__FILE__)}/large.log", 'r')
     before(:all) do
       @indy = Indy.search(large_file).with([/^\[([^\|]+)\|([^\]]+)\] (.*)$/,:severity, :time, :message])
     end
@@ -33,11 +33,11 @@ describe "Search Performance" do
 
   context "with a 10000 line log file" do
 
-    large_file = "#{File.dirname(__FILE__)}/large.log"
+    large_file = File.open("#{File.dirname(__FILE__)}/large.log", 'r')
     before(:all) do
       @indy = Indy.new(
         :source => large_file,
-        :pattern => [/^\[([^\|]+)\|([^\]]+)\] (.*)$/,:severity, :time, :message],
+        :log_format => [/^\[([^\|]+)\|([^\]]+)\] (.*)$/,:severity, :time, :message],
         :time_format => '%d-%m-%Y %H:%M:%S')
     end
 

data/spec/helper.rb

@@ -1,4 +1,8 @@
+require 'simplecov'
+SimpleCov.start if ENV["COVERAGE"]
+# SimpleCov.root("#{File.dirname(__FILE__)}/../..")
+
 require File.expand_path("#{File.dirname(__FILE__)}/../lib/indy") unless 
   $:.include? File.expand_path("#{File.dirname(__FILE__)}/../lib/indy")
 
-require 'rspec'
+# require 'rspec'

data/spec/indy_spec.rb

@@ -6,30 +6,31 @@ describe 'Indy' do
 
     # http://log4r.rubyforge.org/rdoc/Log4r/rdoc/patternformatter.html
     it "should accept a log4r pattern string without error" do
-      lambda { Indy.new(:pattern => ["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]) }.should_not raise_error
+      Indy.new(:log_format => ["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]).class.should == Indy
     end
 
     # http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html
     it "should accept a log4j pattern string without error" do
-      lambda { Indy.new(:pattern => ["%d [%M] %p %C{1} - %m", :time, :info, :class, :message])}
+      Indy.new(:log_format => ["%d [%M] %p %C{1} - %m", :time, :info, :class, :message]).class.should == Indy
     end
 
     it "should not raise error with non-conforming data" do
-      @indy = Indy.new(:source => " \nfoobar\n\n baz", :pattern => ['([^\s]+) (\w+)', :time, :message])
-      lambda{ @indy.for(:all) }.should_not raise_error
+      @indy = Indy.new(:source => " \nfoobar\n\n baz", :log_format => ['([^\s]+) (\w+)', :time, :message])
+      @indy.for(:all).class.should == Array
     end
 
     it "should accept time_format parameter" do
-      @indy = Indy.new(:time_format => '%d-%m-%Y', :source => "1-13-2000 yes", :pattern => ['^([^\s]+) (\w+)$', :time, :message])
-      lambda{ @indy.for(:all) }.should_not raise_error
+      @indy = Indy.new(:time_format => '%d-%m-%Y', :source => "1-13-2000 yes", :log_format => ['^([^\s]+) (\w+)$', :time, :message])
+      @indy.for(:all).class.should == Array
       @indy.instance_variable_get(:@time_format).should == '%d-%m-%Y'
     end
 
     it "should accept an initialization hash passed to #search" do
       hash = {:time_format => '%d-%m-%Y',
         :source => "1-13-2000 yes",
-        :pattern => ['^([^\s]+) (\w+)$', :time, :message]}
-      lambda{ @indy = Indy.search( hash ) }.should_not raise_error
+        :log_format => ['^([^\s]+) (\w+)$', :time, :message]}
+      @indy = Indy.search(hash)
+      @indy.class.should == Indy
       @indy.for(:all).length.should == 1
     end
 
@@ -39,7 +40,7 @@ describe 'Indy' do
   context 'instance' do
 
     before(:all) do
-      @indy = Indy.new(:source => '1/2/2002 string', :pattern => ['([^\s]+) (\w+)', :time, :message])
+      @indy = Indy.new(:source => '1/2/2002 string', :log_format => ['([^\s]+) (\w+)', :time, :message])
     end
 
     context "method" do
@@ -60,16 +61,18 @@ describe 'Indy' do
 
   context ':search' do
 
+    let(:log_file) { "#{File.dirname(__FILE__)}/data.log" }
+
     it "should be a class method" do
       Indy.should respond_to(:search)
     end
 
     it "should accept a string parameter" do
-      lambda{ Indy.search("String Log") }.should_not raise_error
+      Indy.search("String Log").class.should == Indy
     end
 
     it "should accept a :cmd symbol and a command string parameter" do
-      lambda{ Indy.search(:cmd =>"ls") }.should_not raise_error
+      Indy.search(:cmd =>"ls").class.should == Indy
     end
 
     it "should return an instance of Indy" do
@@ -77,63 +80,46 @@ describe 'Indy' do
       Indy.search(:cmd => "ls").should be_kind_of(Indy)
     end
 
+    it "should return an instance of Indy" do
+      Indy.search(:source => {:cmd => 'ls'}, :log_format => Indy::DEFAULT_LOG_FORMAT).class.should == Indy
+    end
+
+
+    it "should create an instance of Indy::Source" do
+      Indy.search("source string").instance_variable_get(:@source).should be_kind_of(Indy::Source)
+    end
+
     it "the instance should have the source specified" do
       Indy.search("source string").source.should_not be_nil
       Indy.search(:cmd => "ls").source.should_not be_nil
     end
 
-    it "the instance should raise an exception when passed an invalid source: Fixnum" do
-      lambda{ Indy.search(9) }.should raise_error Indy::InvalidSource
+    it "the instance should raise an exception when passed an invalid source" do
+      lambda{ Indy.search(nil) }.should raise_error Indy::Source::Invalid
     end
 
     it "the instance should raise an exception when passed an invalid source: nil" do
-      lambda{ Indy.search(nil) }.should raise_error Indy::InvalidSource
+      lambda{ Indy.search(nil) }.should raise_error Indy::Source::Invalid
     end
 
     it "the instance should raise an exception when the arity is incorrect" do
-      lambda{ Indy.search( ) }.should raise_error Indy::InvalidSource
+      lambda{ Indy.search( ) }.should raise_error Indy::Source::Invalid
     end
 
-    context "for a String" do
-
-      let(:log_file) { "#{File.dirname(__FILE__)}/data.log" }
-
-      context "treat it first like a file" do
-
-        it "should attempt to open the file" do
-          File.should_receive(:exist?).with("possible_file.ext").ordered
-          Indy.search("possible_file.ext")
-        end
-
-        it "should not throw an error for a non-existent file" do
-          lambda { Indy.search("possible_file.ext") }.should_not raise_error
-        end
-
-        it "should return an IO object when there is a file" do
-          File.should_receive(:exist?).with("file_exists.ext").and_return( true )
-          File.should_receive(:open).and_return(StringIO.new("2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."))
-          Indy.search("file_exists.ext").for(:application => 'MyApp').length.should == 1
-        end
-
-        it "should handle a real file" do
-          Indy.search(log_file).for(:application => 'MyApp').length.should == 2
-        end
+    context "treat it second like a string" do
 
+      it "should attempt to treat it as a string" do
+        string = "2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."
+        string_io = StringIO.new(string)
+        StringIO.should_receive(:new).with(string).ordered.and_return(string_io)
+        Indy.search(string).for(:application => 'MyApp').length.should == 1
       end
 
-      context "treat it second like a string" do
-
-        it "should attempt to treat it as a string" do
-          string = "2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."
-          string_io = StringIO.new(string)
-          StringIO.should_receive(:new).with(string).ordered.and_return(string_io)
-          Indy.search(string).for(:application => 'MyApp').length.should == 1
-        end
-
-      end
+    end
 
+    context "with explicit source hash" do
 
-      context "treat it optionally like a command" do
+      context ":cmd" do
 
         it "should attempt open the command" do
           IO.stub!(:popen).with('ssh user@system "bash --login -c \"cat /var/log/standard.log\" "')
@@ -142,7 +128,7 @@ describe 'Indy' do
 
         it "should not throw an error for an invalid command" do
           IO.stub!(:popen).with('an invalid command').and_return('')
-          lambda { Indy.search(:cmd => "an invalid command") }.should_not raise_error
+          Indy.search(:cmd => "an invalid command").class.should == Indy
         end
 
         it "should return an IO object upon a successful command" do
@@ -154,6 +140,32 @@ describe 'Indy' do
           Indy.search(:cmd => "cat #{log_file}").for(:application => 'MyApp').length.should == 2
         end
 
+        it "should return an IO object upon a successful command" do
+          IO.stub!(:popen).with("zzzzzzzzzzzz").and_return('Invalid command')
+          lambda{ Indy.search(:cmd => "zzzzzzzzzzzz").for(:all) }.should raise_error( Indy::Source::Invalid, /Unable to open log source/)
+        end
+
+        it "should raise error for an invalid command" do
+          lambda{ Indy.search(:cmd => "zzzzzzzzzzzz").for(:all) }.should raise_error( Indy::Source::Invalid, /Unable to open log source/)
+        end
+
+      end
+
+      it ":file" do
+        require 'tempfile'
+        file = stub!(:size).and_return(1)
+        lambda{ Indy.search(:file => file) }
+      end
+
+      it ":string" do
+        string = "2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION."
+        string_io = StringIO.new(string)
+        StringIO.should_receive(:new).with(string).ordered.and_return(string_io)
+        Indy.search(:string => string).for(:application => 'MyApp').length.should == 1
+      end
+
+      it "should raise error when invalid" do
+        lambda{ Indy.search(:foo => "a string").for(:all) }.should raise_error( Indy::Source::Invalid )
       end
 
     end
@@ -183,7 +195,7 @@ describe 'Indy' do
               "2000-09-07 14:07:42 DEBUG MyApp - Initializing APPLICATION.",
               "2000-09-07 14:07:43 INFO MyApp - Exiting APPLICATION with data:\nApplications data\nMore data\n\tlast Application data."].join("\n")
       regexp = "^((#{Indy::LogFormats::DEFAULT_DATE_TIME})\\s+(#{Indy::LogFormats::DEFAULT_SEVERITY_PATTERN})\\s+(#{Indy::LogFormats::DEFAULT_APPLICATION})\\s+-\\s+(.*?)(?=#{Indy::LogFormats::DEFAULT_DATE_TIME}|\\z))"
-      @indy = Indy.new(:source => log, :pattern => [regexp, :time,:severity,:application,:message], :multiline => true  )
+      @indy = Indy.new(:source => log, :log_format => [regexp, :time,:severity,:application,:message], :multiline => true  )
     end
 
     it "should find the first row" do
@@ -197,8 +209,51 @@ describe 'Indy' do
       results.length.should == 5
     end
 
+    it "should find using time based search" do
+      results = @indy.before(:time => '2000-09-07 14:07:42', :inclusive => false).for(:all)
+      results.length.should == 2
+    end
+
   end
 
+  context "support for blocks" do
+    
+    def log
+      [ "2000-09-07 14:07:41 INFO MyApp - Entering APPLICATION.",
+              "2000-09-07 14:07:42 DEBUG MyApp - Initializing APPLICATION.",
+              "2000-09-07 14:07:43 INFO MyApp - Exiting APPLICATION."].join("\n")
+    end
+    
+    it "should allow a block with :for and yield on each line of the results using :all" do
+      actual_yield_count = 0
+      Indy.search(log).for(:all) do |result|
+        result.should be_kind_of(Struct::Line)
+        actual_yield_count = actual_yield_count + 1
+      end
+      actual_yield_count.should == 3
+    end
+
+    it "should allow a block with :for and yield on each line of the results" do
+      actual_yield_count = 0
+      Indy.search(log).for(:severity => 'INFO') do |result|
+        result.should be_kind_of(Struct::Line)
+        actual_yield_count = actual_yield_count + 1
+      end
+      actual_yield_count.should == 2
+    end
+
+    it "should allow a block with :like and yield on each line of the results" do
+      actual_yield_count = 0
+      Indy.search(log).like(:message => '\be\S+ing') do |result|
+        result.should be_kind_of(Struct::Line)
+        actual_yield_count = actual_yield_count + 1
+      end
+      actual_yield_count.should == 2
+    end
+
+  end
+  
+  
   context "instance" do
 
     before(:each) do
@@ -213,11 +268,11 @@ describe 'Indy' do
     end
 
     it "with() should accept the log4r default pattern const without error" do
-      lambda { @indy.with(Indy::LOG4R_DEFAULT_PATTERN) }.should_not raise_error
+      @indy.with(Indy::LOG4R_DEFAULT_FORMAT).class.should == Indy
     end
 
     it "with() should accept :default without error" do
-      lambda { @indy.with(:default) }.should_not raise_error
+      @indy.with(:default).class.should == Indy
     end
 
     it "with() should use default log pattern when passed :default" do
@@ -225,7 +280,7 @@ describe 'Indy' do
     end
 
     it "with() should accept no params without error" do
-      lambda { @indy.with() }.should_not raise_error
+      @indy.with().class.should == Indy
     end
 
     it "should return itself" do
@@ -238,7 +293,7 @@ describe 'Indy' do
       end
 
       it "#{method}() should accept a hash of search criteria" do
-        lambda { @indy.send(method,:severity => "INFO") }.should_not raise_error
+        @indy.send(method,:severity => "INFO").class.should == Array
       end
 
       it "#{method}() should return a set of results" do
@@ -300,4 +355,21 @@ describe 'Indy' do
 
   end
 
+  context 'source' do
+    before(:each) do
+      @indy = Indy.search(
+        [ "2000-09-07 14:07:41 INFO  MyApp - Entering APPLICATION.",
+          "2000-09-07 14:08:41 INFO  MyApp - Initializing APPLICATION.",
+          "2000-09-07 14:09:41 INFO  MyApp - Configuring APPLICATION.",
+          "2000-09-07 14:10:50 INFO  MyApp - Running APPLICATION.",
+          "2000-09-07 14:11:42 INFO  MyApp - Exiting APPLICATION.",
+          "2000-09-07 14:12:15 INFO  MyApp - Exiting APPLICATION."
+        ].join("\n") )
+    end
+
+    it "should know how many lines it contains" do
+      @indy.source.send(:num_lines).should == 6
+    end
+  end
+
 end