sherlog-holmes 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f7d4009b72347473254b169427804b5dfb0ed36
-  data.tar.gz: 9cad2d96b5b02ac7fedef91ba8b39c32a9c8d3eb
+  metadata.gz: 03f964fa6606b37a8676bd7dad10ad3c92d52416
+  data.tar.gz: d749eb7ce9c8768f55143c77f69b0b5fe286fa92
 SHA512:
-  metadata.gz: 940283aab12cf1fd5ab55d81addca4812e3715bb411367294d2e7d64de9c4d94368ba0cf9a8b83db56962704cbfccf1ec7e37f887b5cc1e22b74ac0cefd72634
-  data.tar.gz: d105d4e8a515179053a92ae15149c54a3b3b80c001a3b5f201135402c176e3a98f2cd6a460572cfafcd91c9222819c672763cf580fa4596ff3971149c4ee860e
+  metadata.gz: 6569c425092b97809a0073b0e6f2019b72df7eb6674c6c775537333a8c45078e956d1ac621326a51e4ebb4042bb2ab1369981f16213a15b27293abd34fd4e012
+  data.tar.gz: 623ed201b563e60d0d5e9930c0f5b2ff0f3cc953833ad7f94e51297eeeb30f231d300274487e27182591e5ed62e4da1068398659d9017149db3f219a4afaf779

data/.idea/misc.xml

@@ -1,12 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="MavenImportPreferences">
-    <option name="generalSettings">
-      <MavenGeneralSettings>
-        <option name="mavenHome" value="Bundled (Maven 3)" />
-      </MavenGeneralSettings>
-    </option>
-  </component>
   <component name="ProjectLevelVcsManager" settingsEditedManually="false">
     <OptionsSetting value="true" id="Add" />
     <OptionsSetting value="true" id="Remove" />

data/README.md

@@ -29,16 +29,16 @@ Sherlog works by grabbing every line of a log and parsing it into a simple struc
 - Category
 - Origin
 - Message
-- Exception (if present)
-- Stacktrace (if present)
+- Exception
+- Stacktrace
 
-You need to supply a regular expression that maps that fields to match your log entry. Here is an example:
+You need to supply a regular expression that maps those fields in order to match your log entry. Here is an example:
 
 ```regexp
 (<?level>\w+)\s(<?category>\s+)\s(<?message>.+)
 ```
 
-Notice that you don't need to define every field, just the ones you need to filter.
+Notice that you don't need to define every field, just the ones shown in your log.
 
 Patterns for exception and stacktrace should be defined separately. The exception pattern is used only in the message field. Here is a complete example of a pattern configuration:
 
@@ -49,16 +49,28 @@ jboss:
   stacktrace: ^(\s+at)|(Caused by\:)|(\s+\.{3}\s\d+\smore)
 ```
 
-The configuration should contain a unique id and at least a pattern for the log **entry**. Place you configuration file in a `*.yml` file inside you `$HOME/.sherlog/patterns` directory and you're ready to go!
+The configuration should contain a unique id and at least a pattern for the log **entry**. Place you configuration file in a `*.yml` file inside your `$HOME/.sherlog/patterns` directory and you're ready to go!
 
 ## Usage
 
 Shelog Holmes provides the command line tool `sherlog`. You can use this to pass a log, the filters you need to apply and the process that needs to be executed (like showing the filtered entries or counting the exceptions):
 
+### Config Options
+
 `-p, --patterns FILE`
 
 Additionally to having definitions in your `$HOME/.sherlog` directory, you can pass a definition file from anywhere in your machine and Sherlog will scan and register the definitions.
 
+`--encode ENCODE`
+
+This sets the encode to use while reading the log file.
+
+`-t, --type TYPE`
+
+This will manually set the patterns definitions. If you don't specify this option, Sherlog will try the mapped ones with the first input line.
+
+### Filter Options
+
 `-c, --category EXPRESSION`
 
 This will filter entries using the category field. You can use the wildcard `*` here.
@@ -85,6 +97,8 @@ This will filter entries using the exception field. You can use the wildcard `*`
 
 This will filter entries with exceptions, regardless the kind.
 
+### Logical Options
+
 `--and`
 
 This will use the **AND** operation to connect the next filter. This is the default operation.
@@ -93,13 +107,53 @@ This will use the **AND** operation to connect the next filter. This is the defa
 
 This will use the **OR** operation to connect the next filter.
 
-    sherlog --level WARN --or --level ERROR --and --any-exception
+`--not`
+
+This will negate the next filter.
+
+```
+sherlog --level WARN --or --not --level INFO --and --any-exception
+```
 
 This is equivalent to:
 
-    (WARN || ERROR) && EXCEPTION
+    (WARN || ! INFO) && EXCEPTION
+
+*NOTICE: try not to do fuzzy logics with this operators*
+
+### Operation Options
+
+`--print`
+
+This will instruct Sherlog to print every filtered entry. This is useful to reduce that crazy log file into a sane one.
+
+```
+$ sherlog --level ERROR --print crazy-log-file.log > sane-log-file.log
+```
+
+`--no-stacktrace`
+
+This will instruct Sherlog to not print stacktraces for entries. This only has effect if used with `--print`.
+
+`--count GROUPS...`
+
+Set this and Sherlog will count the number of entries per level, category, origin or exception. The possible parameters are (separated by a `,`):
+
+- `levels`: counts the number of entries per level
+- `categories`: counts the number of entries per category
+- `origins`: counts the number of entries per origin
+- `exception`: counts the number of entries per exception
+- `all`: counts all groups
+
+```
+$ sherlog --count levels,categories log-file.log
+```
+
+## Built-in Patterns
+
+Currently, Sherlog has the following patterns:
 
-*NOTICE: you cannot do some fuzzy logics with these operations because there are no options to define the precedence*
+- `jboss`: matches Wildfly | EAP logs
 
 ## License
 

data/bin/sherlog

@@ -32,22 +32,50 @@ include Sherlog
 @stacktrace = true
 @filter = nil
 @operation = :and
-@listeners = []
+@listeners = {}
+@groups = []
 
 def add_filter(filter)
+  filter = filter.negate if @negate
   if @filter
     @filter = @filter.send @operation, filter
   else
     @filter = filter
   end
+  @negate = false
+end
+
+def print_table(name, data)
+  table = Yummi::Table::new
+  table.title = name.to_s.capitalize
+  table.header = %w{Name Count}
+  table.colorize :name, with: 'bold.white'
+  table.colorize :count, with: 'magenta'
+  data.each do |name, count|
+    table << [name, count]
+  end
+  table.print
+  puts
 end
 
 @opts.banner = 'Usage: sherlog [options] <logfile>'
 
+@opts.separator "\n  Config Options\n".bold.white
+
 @opts.on '-p FILE', '--patterns FILE', 'Configures the patterns' do |file|
   Sherlog.load_patterns file
 end
 
+@opts.on '--encode ENCODE', 'Sets the encode to read the log' do |encode|
+  ENV['SHERLOG_FILE_ENCODE'] = encode
+end
+
+@opts.on '-t TYPE', '--type TYPE', 'Sets the type of the log (for loading patterns)' do |type|
+  @type = type.to_sym
+end
+
+@opts.separator "\n  Filter Options\n".bold.white
+
 @opts.on '-c EXPRESSION', '--category EXPRESSION', 'Set the category filter expression' do |expression|
   add_filter Filter::category(expression)
 end
@@ -72,6 +100,8 @@ end
   add_filter Filter::exceptions
 end
 
+@opts.separator "\n  Logical Options\n".bold.white
+
 @opts.on '--and', 'Sets the next filter to use the AND operator' do
   @operation = :and
 end
@@ -80,14 +110,32 @@ end
   @operation = :or
 end
 
-@opts.on '-t TYPE', '--type TYPE', 'Sets the type of the log (for loading patterns)' do |type|
-  @type = type.to_sym
+@opts.on '--not', 'Negates the next filter' do
+  @negate = true
 end
 
+@opts.separator "\n  Operation Options\n".bold.white
+
 @opts.on '--print', 'Prints the filtered entries' do
-  @listeners << lambda do |entry|
-    puts entry.raw_content
-  end
+  @listeners[:print] = PrintListener::new
+end
+
+@opts.on '--no-stacktrace', 'Do not print stacktraces (use with --print)' do
+  @no_stacktrace = true
+end
+
+@opts.on '--count GROUPS...',
+         'Counts entries and print the result by group (levels, categories, origins, exceptions or all)',
+         Array do |groups|
+  all_groups = [:levels, :categories, :origins, :exceptions]
+  @groups = if groups == ['all']
+              all_groups
+            else
+              all_groups.find_all do |name|
+                groups.index name.to_s
+              end
+            end
+  @listeners[:count] = CountListener::new
 end
 
 @opts.on '-h', '--help', 'Shows the usage help' do
@@ -97,11 +145,13 @@ end
 
 @opts.parse! ARGV
 
+@listeners[:print].hide_stacktrace if @no_stacktrace and @listeners[:print]
+
 file = ARGV.first
 parser = Sherlog.parser @type if @type
 parser ||= Parser::new
 parser.filter @filter
-@listeners.each do |listener|
+@listeners.values.each do |listener|
   parser.on_new_entry listener
 end
 begin
@@ -112,6 +162,9 @@ begin
       parser.parse line.chomp
     end
   end
+  @groups.each do |group|
+    print_table group, @listeners[:count].send(group).sort_by { |name, count| -count }
+  end
 rescue Interrupt
   exit
 end

data/changelog.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## v0.3.0
+
+- Operation to count entries
+- Entries can now have multiple exceptions (but the first one will be returned by `#exception`)
+- Option to hide stacktrace while printing entries
+- Option to set the encode
+- Option to negate a filter
+
+## v0.2.0
+
+- Patterns configurable through files
+- Code Refactor
+- Specs
+
+## v0.1.0
+
+- Initial version

data/conf/patterns/jboss.yml

@@ -1,4 +1,4 @@
 jboss:
-  entry: (?<time>[0-9,.:]+)\s+(?<level>\w+)\s+\[(?<category>\S+)\]\s\((?<origin>[^)]+)\)?\s?(?<message>.+)
+  entry: (?<time>(\d{4}-\d{2}-\d{2}\s)?(\d{2}:\d{2}:\d{2},\d{3}))\s+(?<level>\w+)\s+\[(?<category>\S+)\]\s\((?<origin>[^)]+)\)?\s?(?<message>.+)
   exception: (?<exception>\w+(\.\w+)+(Exception|Error))
   stacktrace: ^(\s+at)|(Caused by\:)|(\s+\.{3}\s\d+\smore)

data/lib/sherlog_holmes.rb

@@ -28,6 +28,8 @@ require_relative 'sherlog_holmes/result'
 require_relative 'sherlog_holmes/entry'
 require_relative 'sherlog_holmes/filter'
 require_relative 'sherlog_holmes/parser'
+require_relative 'sherlog_holmes/listeners/print_listener'
+require_relative 'sherlog_holmes/listeners/count_listener'
 
 module Sherlog
 
@@ -61,3 +63,5 @@ module Sherlog
   end
 
 end
+
+ENV['SHERLOG_FILE_ENCODE'] ||= Encoding.default_external.name

data/lib/sherlog_holmes/entry.rb

@@ -23,7 +23,7 @@
 module Sherlog
   class Entry
 
-    attr_accessor :time, :level, :category, :origin, :message, :exception, :stacktrace, :raw_content
+    attr_accessor :time, :level, :category, :origin, :message, :exceptions, :stacktrace, :raw_content
 
     def initialize(params = {})
       @time = params[:time]
@@ -31,12 +31,18 @@ module Sherlog
       @category = params[:category]
       @origin = params[:origin]
       @message = params[:message]
-      @exception = params[:exception]
+      @exceptions = [params[:exception]] if params[:exception]
+      @exceptions ||= params[:exceptions]
+      @exceptions ||= []
       @stacktrace = []
     end
 
     def exception?
-      !@exception.nil?
+      !@exceptions.empty?
+    end
+
+    def exception
+      @exceptions.first
     end
 
     def <<(line)

data/lib/sherlog_holmes/filter.rb

@@ -50,6 +50,12 @@ module Sherlog
       end
     end
 
+    def negate
+      Filter::new do |entry|
+        !self.accept?(entry)
+      end
+    end
+
     def self.expression(expression)
       Filter::new do |object|
         wildcard_at_start = expression.start_with? '*'
@@ -92,7 +98,9 @@ module Sherlog
 
     def self.exception(expression)
       Filter::new do |entry|
-        expression(expression).accept? entry.exception
+        entry.exceptions.find do |exception|
+          expression(expression).accept? exception
+        end
       end
     end
 

data/lib/sherlog_holmes/listeners/count_listener.rb

@@ -0,0 +1,62 @@
+#                         The MIT License
+#
+# Copyright (c) 2015 Marcelo "Ataxexe" Guimarães <ataxexe@devnull.tools>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Sherlog
+
+  class CountListener
+    attr_reader :levels, :categories, :origins, :exceptions
+
+    def initialize
+      @levels = {}
+      @categories = {}
+      @origins = {}
+      @exceptions = {}
+    end
+
+    def call(entry)
+      initialize_counters entry
+      count entry
+    end
+
+    private
+
+    def initialize_counters(entry)
+      @levels[entry.level] ||= 0 if entry.level
+      @categories[entry.category] ||= 0 if entry.category
+      @origins[entry.origin] ||= 0 if entry.origin
+      entry.exceptions.each do |exception|
+        @exceptions[exception] ||= 0
+      end
+    end
+
+    def count(entry)
+      @levels[entry.level] += 1 if entry.level
+      @categories[entry.category] += 1 if entry.category
+      @origins[entry.origin] += 1 if entry.origin
+      entry.exceptions.each do |exception|
+        @exceptions[exception] += 1
+      end
+    end
+
+  end
+
+end

data/lib/sherlog_holmes/listeners/print_listener.rb

@@ -0,0 +1,47 @@
+#                         The MIT License
+#
+# Copyright (c) 2015 Marcelo "Ataxexe" Guimarães <ataxexe@devnull.tools>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Sherlog
+
+  class PrintListener
+
+    def initialize(target = $stdout)
+      @target = target
+    end
+
+    def hide_stacktrace
+      @hide_stacktrace = true
+    end
+
+    def call(entry)
+      if @hide_stacktrace and not entry.stacktrace.empty?
+        content = entry.raw_content
+        message_end = content.index(entry.stacktrace.first)
+        @target << content[0...message_end].chomp << $/
+      else
+        @target << entry.raw_content.chomp << $/
+      end
+    end
+
+  end
+
+end

data/lib/sherlog_holmes/parser.rb

@@ -64,7 +64,7 @@ module Sherlog
           notify entry
           entry = Entry::new entry_data
           entry.raw_content = line.chomp
-          entry.exception = Regexp.last_match[:exception] if @patterns[:exception] =~ entry.message
+          entry.exceptions << Regexp.last_match[:exception] if @patterns[:exception] =~ entry.message
           entry = nil unless @filter.accept? entry
         else
           if entry
@@ -73,6 +73,7 @@ module Sherlog
             else
               entry << line.chomp
             end
+            entry.exceptions << Regexp.last_match[:exception] if @patterns[:exception] =~ line
             entry.raw_content << $/ << line.chomp
           end
         end
@@ -84,7 +85,7 @@ module Sherlog
 
     def foreach(input, &block)
       if File.exist? input
-        IO.foreach input, &block
+        IO.foreach input, encoding: ENV['SHERLOG_FILE_ENCODE'], &block
       else
         input.each_line &block
       end

data/lib/sherlog_holmes/version.rb

@@ -21,5 +21,5 @@
 # THE SOFTWARE.
 
 module Sherlog
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/sherlog-holmes.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Ataxexe']
   spec.email         = ['ataxexe@devnull.tools']
 
-  spec.summary       = %q{A gem to analyse log files}
+  spec.summary       = %q{The best companion for a log detective!}
   spec.homepage      = 'https://github.com/devnull-tools/sherlog-holmes'
   spec.license       = 'MIT'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sherlog-holmes
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ataxexe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-08 00:00:00.000000000 Z
+date: 2015-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yummi
@@ -92,10 +92,13 @@ files:
 - README.md
 - Rakefile
 - bin/sherlog
+- changelog.md
 - conf/patterns/jboss.yml
 - lib/sherlog_holmes.rb
 - lib/sherlog_holmes/entry.rb
 - lib/sherlog_holmes/filter.rb
+- lib/sherlog_holmes/listeners/count_listener.rb
+- lib/sherlog_holmes/listeners/print_listener.rb
 - lib/sherlog_holmes/parser.rb
 - lib/sherlog_holmes/result.rb
 - lib/sherlog_holmes/version.rb
@@ -123,5 +126,5 @@ rubyforge_project:
 rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
-summary: A gem to analyse log files
+summary: The best companion for a log detective!
 test_files: []