logstash-cli 0.0.4 → 0.0.5

data/README.md

@@ -46,13 +46,13 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
       [--meta=META]                  # Meta Logstash fields to show
                                      # Default: type,message
       [--to=TO]                      # End date
-                                     # Default: 2012-05-11
-      [--delim=DELIM]                # csv delimiter
+                                     # Default: Today in YYYY-MM-DD form
+      [--delim=DELIM]                # plain or csv delimiter
                                      # Default: |
       [--format=FORMAT]              # Format to use for exporting
                                      # Default: csv
       [--from=FROM]                  # Begin date
-                                     # Default: 2012-05-11
+                                     # Default: Today in YYYY-MM-DD form
       [--size=SIZE]                  # Number of results to return
                                      # Default: 500
       [--esurl=ESURL]                # URL to connect to elasticsearch
@@ -90,7 +90,7 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
                                            # Default: rawlogs
           [--password=PASSWORD]            # Password to connect to AMQP
                                            # Default: foo
-          [--delim=DELIM]                  # csv delimiter
+          [--delim=DELIM]                  # plain or csv delimiter
                                            # Default: |
           [--exchange-type=EXCHANGE_TYPE]  # Exchange Type
                                            # Default: direct
@@ -98,6 +98,34 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
 
     Stream a live feed via AMQP
 
+### Count
+
+    Usage:
+      logstash-cli count PATTERN --countfield=COUNTFIELD
+
+    Options:
+      [--meta=META]                  # Meta Logstash fields to show
+      [--last=LAST]                  # Specify period since now f.i. 1d
+      [--from=FROM]                  # Begin date
+                                     # Default: Today in YYYY-MM-DD form
+      [--delim=DELIM]                # plain or csv delimiter
+                                     # Default: |
+      --countfield=COUNTFIELD        # Logstash field to count
+      [--countsize=COUNTSIZE]        # Number of most frequent values to return
+                                     # Default: 50
+      [--format=FORMAT]              # Format to use for exporting (plain,csv,json)
+                                     # Default: csv
+      [--to=TO]                      # End date
+                                     # Default: Today in YYYY-MM-DD form
+      [--fields=FIELDS]              # Logstash fields to show
+      [--size=SIZE]                  # Number of results per index to show
+                                     # Default: 10
+      [--esurl=ESURL]                # URL to connect to elasticsearch
+                                     # Default: http://localhost:9200
+      [--index-prefix=INDEX_PREFIX]  # Logstash index prefix
+                                     # Default: logstash-
+
+    Return most frequent values of a field within a pattern and optionally show associated fields
 
 ## Examples
 
@@ -105,6 +133,8 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
 
     $ logstash-cli tail --amqpurl="amqp://logger-1.jedi.be:5672" --key="program.sshd"
 
+    $ logstash-cli count --esurl="http://logger-1.jedi.be:9200" '@message:jedi4ever' --countfield=program
+
 ## TODO
 
 - find a way to query existing instances

data/lib/logstash-cli/cli.rb

@@ -1,5 +1,4 @@
 require 'rubygems'
-require 'bundler/setup'
 require 'rack'
 require 'amqp'
 require 'tire'
@@ -21,7 +20,7 @@ module LogstashCli
     method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
     method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
     method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
-    method_option :format , :default => 'csv', :desc => "Format to use for exporting (plain,csv,json)"
+    method_option :format , :default => 'plain', :desc => "Format to use for exporting (plain,csv,json)"
     method_option :size , :default => 500, :desc => "Number of results to return"
     method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
     method_option :meta , :default => "type,message", :desc => "Meta Logstash fields to show"
@@ -47,13 +46,31 @@ module LogstashCli
     method_option :auto_delete, :default => false, :desc => "Autodelete Exchange or not" , :type => :boolean
     method_option :persistent, :default => false, :desc => "Persistent Exchange or not", :type => :boolean
     method_option :key, :default => '#', :desc => "Routing key"
-    method_option :format , :default => 'csv', :desc => "Format to use for exporting (plain,csv,json)"
+    method_option :format , :default => 'plain', :desc => "Format to use for exporting (plain,csv,json)"
     method_option :meta, :default => "timestamp,type,message", :desc => "Meta Logstash fields to show"
-    method_option :delim, :default => "|", :desc => "csv delimiter"
+    method_option :delim, :default => "|", :desc => "plain or csv delimiter"
 
     def tail()
       _tail(options)
     end
 
+    desc "count PATTERN", "Return most frequent values of a field within a pattern and optionally show associated fields"
+    method_option :esurl , :default => 'http://localhost:9200', :desc => "URL to connect to elasticsearch"
+    method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
+    method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
+    method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
+    method_option :format , :default => 'csv', :desc => "Format to use for exporting (plain,csv,json)"
+    method_option :size , :default => 10, :desc => "Number of results per index to show"
+    method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
+    method_option :meta , :default => "", :desc => "Meta Logstash fields to show"
+    method_option :fields , :default => "", :desc => "Logstash fields to show"
+    method_option :countfield , :required => true, :desc => "Logstash field to count"
+    method_option :countsize , :default => 50, :desc => "Number of most frequent values to return"
+    method_option :delim , :default => "|", :desc => "plain or csv delimiter"
+
+    def count(pattern)
+      _count(pattern,options)
+    end
+
   end
 end

data/lib/logstash-cli/command/count.rb

@@ -0,0 +1,108 @@
+require 'date'
+
+require 'yajl/json_gem'
+
+module Count
+
+  def _count(pattern,options)
+    es_url = options[:esurl]
+    index_prefix = options[:index_prefix]
+
+    from = options[:from]
+    to = options[:to]
+    metafields = options[:meta].split(',')
+    fields = options[:fields].split(',')
+
+    countfield = options[:countfield]
+    countsize = options[:countsize]
+
+    begin
+      unless options[:last].nil?
+        days = options[:last].match(/(\d*)d/)[1].to_i
+        to_date = Date.today
+        from_date = to_date - days
+        from = from_date.to_s
+        to = to_date.to_s
+      end
+
+      from_date = Date.parse(from)
+      to_date = Date.parse(to)
+    rescue Exception => ex
+      $stderr.puts "Something went wrong while parsing the dates: currently only dates are supported with last. Be sure to add the suffix 'd' "+ex
+      exit -1
+    end
+
+    $stderr.puts "Searching #{es_url}[#{index_prefix}#{from_date}..#{index_prefix}#{to_date}] - #{pattern}"
+
+    # We reverse the order of working ourselves through the index
+    (from_date..to_date).sort.reverse.to_a.each do |date|
+
+      es_index = index_prefix+date.to_s.gsub('-','.')
+
+      begin
+        Tire.configure {url es_url}
+        search = Tire.search(es_index) do
+          query do
+            string "#{pattern}"
+          end
+          facet "#{countfield}" do
+             terms countfield, :size => countsize
+          end
+        end
+      rescue Exception => e
+        $stderr.puts e
+        $stderr.puts "\nSomething went wrong with the search. This is usually due to lucene query parsing"
+        exit
+      end
+
+      # Results per index to show
+      result_size = options[:size]
+
+      begin
+        results = search.results.facets[countfield]
+
+        header = [ countfield, results['total'] ]
+        puts _format(header, options)
+
+        results['terms'].each do |terms|
+          puts _format(terms.values, options)
+
+          unless fields.empty? and metafields.empty?
+            term = terms['term']
+            begin
+              Tire.configure {url es_url}
+              search = Tire.search(es_index) do
+                query do
+                  string "#{pattern}"
+                end
+                filter :terms, countfield => [term]
+                size result_size
+              end
+            rescue Exception => e
+              $stderr.puts e
+              $stderr.puts "\nSomething went wrong with the search. This is usually due to lucene query parsing"
+              exit
+            end
+
+            search.results.each do |log|
+              result = Array.new
+
+              metafields.each do |metafield|
+                result << log["@#{metafield}".to_sym]
+              end
+
+              fields.each do |field|
+                result << log[:@fields][field.to_sym]
+              end
+
+              puts _format(result, options)
+              result = []
+            end
+          end
+        end
+      rescue ::Tire::Search::SearchRequestFailed => e
+        $stderr.puts e.message
+      end
+    end
+  end
+end

data/lib/logstash-cli/command/grep.rb

@@ -77,14 +77,7 @@ module Grep
             result << res[:@fields][field.to_sym]
           end
 
-          output = case options[:format]
-                   when 'csv' then result.to_csv({:col_sep => options[:delim]})
-                   when 'json' then result.to_json
-                   when 'plain' then result.join(options[:delim])
-                   end
-          #tstamp = Time.iso8601(res[:@timestamp]).localtime.iso8601
-
-          puts output
+          puts _format(result, options)
           result = []
         end
       rescue ::Tire::Search::SearchRequestFailed => e

data/lib/logstash-cli/command/tail.rb

@@ -44,13 +44,7 @@ module Tail
               result << parsed_message["@#{metafield}"]
             end
 
-            output = case options[:format]
-                     when 'csv' then result.to_csv({:col_sep => options[:delim]})
-                     when 'json' then result.to_json
-                     when 'plain' then result.join(options[:delim])
-                     end
-
-            puts output
+            puts _format(result, options)
             result = []
           end
         end

data/lib/logstash-cli/command.rb

@@ -1,9 +1,20 @@
 require 'logstash-cli/command/grep'
 require 'logstash-cli/command/tail'
+require 'logstash-cli/command/count'
 
 module LogstashCli
   module Command
     include Grep
     include Tail
+    include Count
+
+    def _format(result,options)
+      output = case options[:format]
+                 when 'csv' then result.to_csv({:col_sep => options[:delim]})
+                 when 'json' then result.to_json
+                 when 'plain' then result.join(options[:delim])
+               end
+      return output
+    end
   end
 end

data/lib/logstash-cli/version.rb

@@ -1,3 +1,3 @@
 module LogstashCli
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-cli
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: 
   segments: 
   - 0
   - 0
-  - 4
-  version: 0.0.4
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Patrick Debois
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-25 00:00:00 Z
+date: 2012-08-13 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement 
@@ -150,6 +150,7 @@ files:
 - lib/logstash-cli.rb
 - lib/logstash-cli/cli.rb
 - lib/logstash-cli/command.rb
+- lib/logstash-cli/command/count.rb
 - lib/logstash-cli/command/grep.rb
 - lib/logstash-cli/command/tail.rb
 - lib/logstash-cli/version.rb