splog 0.0.3 → 0.0.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    N2IwOThhNGYzZTI5MzVmYTk0YjNhOGFkMzgwOGJhM2JlOTYyMzUzNw==
+    ZTg2MDA1ZDg3MTk4NTgxOTljM2JmZjNkYTNkYzRiNDgzZjIxZDk4Yg==
   data.tar.gz: !binary |-
-    NTkyNGQ3NWM3NWVhNWU3Nzc4NWE1MThiYjYyZDEwYjRmYmY1MTk2OA==
+    ZjU1MmJiOGQxMjU0MzhmYWYyYTBkMjFiZjA5YmJjNDg4MjQxMzAxYw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZmRkMTFlZGY2MjZkNWJmYjczOTJkYTQ1MzYzNDg4ZWMyMzlmMGE0NDA4ZDIw
-    ZTdlYzAzZjFmYzM1MzcxYzFmYmZkYmU0YjlkYTkzYTgyY2ZmMTU0YTQ4Mjk2
-    OWFiYjZlMDIxNTA3NjhhODE5ZTBmZDI3NmIyYjM1NWVjOTI1MjA=
+    MDg1Mjg0ZjRlYzk5NDg5YjMzMDE0ZDFhMWRiN2I4NzU5ZDc5MGIwOGVkODc0
+    YzgyZDg3YzhlZjJhMDhlM2UzMGY5ODIxNWZlZTZjNGI3YzI3ZWYyOWRhOThj
+    Y2I5ZDlmNDY4OTBkMjdkMDQ0N2RkMmE1NGJjNzdiZmM0ZTE0OTI=
   data.tar.gz: !binary |-
-    YjNkODBmOTQ1YzgyMjBiM2UyYjA4NTdlNjQwMzhhZWFiNmYyMTc5ZDE4NWYx
-    NjYzMjZmNDBkMTQyNWRlY2I5NGU1YzlhYjU3MmRiYWI5ZDdjYTZlYjljZDlj
-    ZjY5ZGRmMjA0NTY1Yjg5ZTliYWQ5NDhjNjBhMGQ2Mzk4YjBkZjU=
+    MmY4MzhjNjQwOGM1MjUzNjQ5ODg3YzI2NDQxOTA4ODJjYWQ1ZGU4NDM4NTU3
+    OTJiNjhlNjhlYjkzYjRjNmZkZmQ2MjUyYjdkMzg1OWQ4OWJlYzA5YjRhYzY1
+    MWVhMTA5OTQ0ZWY0YjcwY2M4YTU2MWM0ZjIwYzAzNGQ3ZmRkM2U=

data/README.md

@@ -126,6 +126,10 @@ Persisting the log to mongo.  Set -o with no arg so that no output to stdout.  T
 
     bundle exec rspec test
 
+#### Profiling in dev
+
+    head -n 10000 path_to/server.log | ruby lib/splog-prof.rb -p apache_common -o -v
+
 #### A few performance measurements
 
 Note that everything is done in a streaming manner with Ruby Enumerators.  No file is ever read completely.  At most a file is read two lines at a time.  The memory requirements should be minimal due to this.

data/lib/splog/version.rb

@@ -1,3 +1,3 @@
 module Splog
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/splog-prof.rb

@@ -0,0 +1,12 @@
+require 'splog'
+require 'ruby-prof'
+
+# Profile the code
+RubyProf.start
+parser = Splog::LogParser.new
+parser.cli(ARGV)
+result = RubyProf.stop
+
+# Print a flat profile to text
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(STDOUT)

data/lib/splog.rb

@@ -8,6 +8,7 @@ require 'json'
 require 'enumerator'
 require 'mongo'
 require 'ruby-progressbar'
+require 'open3'
 
 include Mongo
 
@@ -127,13 +128,43 @@ module Splog
     def set_pattern(options)
       @pattern_name = options[:pattern_name]
       begin
+        # MULTILINE to match the \n chars
+        #Regexp::MULTILINE | Regexp::IGNORECASE
         @pattern = @config[options[:pattern_name]]['regex']
+        delim =  @config[@pattern_name].nil? ? "\\s+" : @config[@pattern_name]['delim']
+        c = "#{@pattern.join(delim)}"
+        # Remove the grouped named
+        @pattern_egrep = 'egrep "' + c.gsub(/\?<.*?>/, '') + '"'
+        r = Regexp.new(c, Regexp::MULTILINE)
+        @pattern = r
       rescue => detail
+        #detail.backtrace.each { |e| $stderr.puts e}
+        #$stderr.puts $!
         puts "No pattern matching '#{options[:pattern_name]}' found.  Please choose another name or define this pattern in the your .splog.yaml"
         exit
       end
     end
 
+    def set_pattern_match_forward
+      begin
+        @pattern_match_forward = @config[options[:pattern_name]]['match_forward_regex']
+        # since this is optional only compile if set
+        if @pattern_match_forward
+          delim =  @config[@pattern_name].nil? ? "\\s+" : @config[@pattern_name]['delim']
+          # Remove the grouped named
+          c = "#{@pattern_match_forward.join(delim)}"
+          r = Regexp.new(c, Regexp::MULTILINE)
+          @pattern_match_forward_egrep = 'egrep "' + c.gsub(/\?<.*?>/, '') + '"'
+          @pattern_match_forward = r
+        end
+      rescue => detail
+        #detail.backtrace.each { |e| $stderr.puts e}
+        $stderr.puts $!
+        #puts "No pattern matching '#{options[:pattern_name]}' found.  Please choose another name or define this pattern in the your .splog.yaml"
+        exit
+      end
+    end
+
     def set_mapping(options)
       begin
         tmp = {}
@@ -183,14 +214,17 @@ module Splog
 
     def parse_line(line, opts={})
       res = {}
-      parts = opts[:parts] || @config[@pattern_name]['regex']
+      #parts = opts[:parts] || @config[@pattern_name]['regex']
       begin
-        #pattern = re.compile(r'\s+'.join(parts)+r'\s*\Z')
-        pattern = @config[@pattern_name].has_key?('delim') ? "\\s*#{parts.join(@config[@pattern_name]['delim'])}\\s*" : "\\s*#{parts.join()}\\s*"
-        # MULTILINE to match the \n chars
-        #Regexp::MULTILINE | Regexp::IGNORECASE
-        r = Regexp.new(pattern, Regexp::MULTILINE)
-        m = r.match(line)
+        #pattern = @config[@pattern_name].has_key?('delim') ? "\\s*#{parts.join(@config[@pattern_name]['delim'])}\\s*" : "\\s*#{parts.join()}\\s*"  # was working line
+        #r = Regexp.new(pattern, Regexp::MULTILINE)
+        #m = r.match(line)
+
+        if opts[:regex]
+          m = opts[:regex].match(line)
+        else
+          m = @pattern.match(line)
+        end
         res = {}
         if m
           m.names.each do |group_name|
@@ -239,17 +273,40 @@ module Splog
         parsed_line = nil
         begin
           while enum_ref
+
             line = enum_ref.next
             parsed_line = parse_line(line)
 
             next_line = enum_ref.peek
             # Pass in the 'match_forward_regex' if it exists so the next line can be evaluated in this context
-            parsed_next_line = @config[@pattern_name]['match_forward_regex'].nil? ? parse_line(next_line) : parse_line(next_line, {:parts => @config[@pattern_name]['match_forward_regex']})
+            #parsed_next_line = @pattern_match_forward.nil? ? parse_line(next_line) : parse_line(next_line, {:regex => @pattern_match_forward})
+            #parsed_next_line_test = @pattern_match_forward.nil? ? parse_line(next_line) : parse_line(next_line, {:regex => @pattern_match_forward})
+
+            # Performance optimization here, don't do a full #match only =~ since not all next lines need to be parsed period
+            #parsed_next_line_test = @pattern_match_forward.nil? ? next_line =~ @pattern : next_line =~ @pattern_match_forward
+            #egrep = "echo \"#{next_line}\" | egrep \"#{@pattern_egrep}\""
+            #egrep_fwd = 'echo ' + next_line + ' | egrep ' + @pattern_match_forward_egrep
+            #p egrep
+            #p egrep_fwd
+            #parsed_next_line_test = @pattern_match_forward.nil? ? `#{egrep}` : `#{egrep_fwd}`
+
+            o, e, s = nil
+            begin
+              o, e, s = Open3.capture3(@pattern_match_forward.nil? ? @pattern_egrep : @pattern_match_forward_egrep, :stdin_data=>next_line)
+            rescue Errno::EPIPE
+              #puts "Connection broke!"
+              nil
+            end
 
             ############################################################################################################
             # If the next line matches the match_forward_regex
             ############################################################################################################
-            if parsed_next_line and @config[@pattern_name]['match_forward_regex']
+            #if parsed_next_line and @config[@pattern_name]['match_forward_regex']
+            #if not parsed_next_line_test.nil? and @config[@pattern_name]['match_forward_regex']
+            if s && s.success? and @config[@pattern_name]['match_forward_regex']
+
+              # Do the actual match now that we know it matches
+              parsed_next_line = @pattern_match_forward.nil? ? parse_line(next_line) : parse_line(next_line, {:regex => @pattern_match_forward})
 
               # If the current_working_line does not yet exist, set it to the latest parsed line
               if current_working_line.nil? and parsed_line
@@ -266,7 +323,8 @@ module Splog
               while true
                 # Only peek here to not advance the enum unnecessarily
                 sub_line = enum_ref.peek
-                parsed_sub_line = @config[@pattern_name]['match_forward_regex'].nil? ? nil : parse_line(sub_line, {:parts => @config[@pattern_name]['match_forward_regex']})
+                #parsed_sub_line = @config[@pattern_name]['match_forward_regex'].nil? ? nil : parse_line(sub_line, {:regex => @pattern_match_forward})
+                parsed_sub_line = @pattern_match_forward.nil? ? nil : parse_line(sub_line, {:regex => @pattern_match_forward})
                 if parsed_sub_line
                   # if matched advance the enum and add the data to the current working line
                   enum_ref.next
@@ -293,8 +351,19 @@ module Splog
               while true
                 # Only peek here to not advance the enum unnecessarily
                 sub_line = enum_ref.peek
-                parsed_sub_line = parse_line(sub_line)
-                if parsed_sub_line.nil? and @config[@pattern_name]['unmatched_append_key_name']
+
+                # TODO this can be optimized too since I'm attmpting to not match it!  I don't even read the parsed_sub_line
+                #parsed_sub_line = parse_line(sub_line)
+                o, e, s = nil
+                begin
+                  o, e, s = Open3.capture3(@pattern_egrep, :stdin_data=>sub_line)
+                rescue Errno::EPIPE
+                  #puts "Connection broke!"
+                  nil
+                end
+
+                #if parsed_sub_line.nil? and @config[@pattern_name]['unmatched_append_key_name']
+                if (s.nil? or not s.success?) && @config[@pattern_name]['unmatched_append_key_name']
                   # if unmatched advance the enum and add the data to the current working line
                   enum_ref.next
                   current_working_line[@config[@pattern_name]['unmatched_append_key_name']] << sub_line
@@ -446,6 +515,7 @@ module Splog
         load_dot_file
 
         set_pattern(options)
+        set_pattern_match_forward
         set_mapping(options)
 
         # Total line count, if file input we can easily do wc -l on the file.  If $stdin we can allow allow a user defined

data/splog.gemspec

@@ -27,4 +27,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '~> 2.6'
+  spec.add_development_dependency 'ruby-prof'
 end

data/test/examples/jboss/multiline_match_unmatch_server.log

@@ -7,3 +7,4 @@ unmatched line 2 added to previous
 03 Oct 2013 20:16:55,308 ERROR [stderr] (MSC service thread 1-3) Should be parsed log line # 3
 03 Oct 2013 20:16:55,308 ERROR [stderr] (MSC service thread 1-3) Should be parsed log line # 4
 
+03 Oct 2013 20:16:55,308 ERROR [stderr] (MSC service thread 1-3) Should be parsed log line # 5

data/test/splog_spec.rb

@@ -201,7 +201,7 @@ describe Splog::LogParser do
     log_entry_two['Category'].should eql('org.jboss.as.connector.subsystems.datasources')
     log_entry_two['Date'].to_s.should eql('2013-10-03 18:33:00 UTC')
     log_entry_two['Date'].should be_a(Time)
-    log_entry_two['Message'].should eql("JBAS010403: Deploying JDBC-compliant driver class org.h2.Driver (version 1.3)\n\n")
+    log_entry_two['Message'].should eql("JBAS010403: Deploying JDBC-compliant driver class org.h2.Driver (version 1.3)\n")
     log_entry_two['Priority'].should eql('INFO')
     log_entry_two['Thread'].should eql('ServerService Thread Pool -- 57')
   end
@@ -219,10 +219,9 @@ describe Splog::LogParser do
     # Get an enumerable from the parser
     pe = parser.parse(e)
     parsed_lines = pe.to_a
-    parsed_lines.length.should eql(4)
+    parsed_lines.length.should eql(5)
   end
 
-
   it 'should properly hash the 50 lines in the sample access log' do
     # Match subsequent lines and add them to a previous line
     test_dir = Dir.pwd.match(/.*?splog$/) ? 'test/' : ''

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: splog
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Samuel Mendenhall
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-11 00:00:00.000000000 Z
+date: 2013-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -94,6 +94,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Parse any log file with yml defined regex rules
 email:
 - Samuel.Mendenhall@gmail.com
@@ -110,6 +124,7 @@ files:
 - bin/splog
 - examples/.splog.yml
 - examples/access_log
+- lib/splog-prof.rb
 - lib/splog.rb
 - lib/splog/version.rb
 - splog.gemspec