logstash-cli 0.0.2 → 0.0.3

data/README.md

@@ -33,6 +33,8 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
 
 ## Commandline Options
 
+### Grep
+
     Usage:
       logstash-cli grep PATTERN
 
@@ -59,15 +61,54 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
 
     Search logstash for a pattern
 
+### Tail
+
+Usage:
+  logstash-cli tail
+
+Options:
+      [--host=HOST]                    # Host to connect to AMQP
+                                       # Default: localhost
+  --amqpurl, [--url=URL]               # Alternate way to specify settings via an AMQP Url f.i. amqp://logstash:foopass@localhost:5672. 
+ This takes precendence over other settings. Note that username and password need to be percentage encoded(URL encoded) in case of special characters
+      [--auto-delete]                  # Autodelete Exchange or not
+      [--vhost=VHOST]                  # VHost to connect to AMQP
+                                       # Default: /
+      [--persistent]                   # Persistent Exchange or not
+      [--ssl]                          # Enable SSL to connect to AMQP
+      [--user=USER]                    # User to connect to AMQP
+                                       # Default: logstash
+      [--meta=META]                    # Meta Logstash fields to show
+                                       # Default: timestamp,type,message
+      [--format=FORMAT]                # Format to use for exporting (plain,csv,json)
+                                       # Default: csv
+      [--key=KEY]                      # Routing key
+                                       # Default: #
+      [--port=PORT]                    # Port to connect to AMQP
+                                       # Default: 5672
+      [--exchange=EXCHANGE]            # Exchange name
+                                       # Default: rawlogs
+      [--password=PASSWORD]            # Password to connect to AMQP
+                                       # Default: foo
+      [--delim=DELIM]                  # csv delimiter
+                                       # Default: |
+      [--exchange-type=EXCHANGE_TYPE]  # Exchange Type
+                                       # Default: direct
+      [--durable]                      # Durable Exchange or not
+
+Stream a live feed via AMQP
+
+
 ## Examples
 
     $ logstash-cli grep --esurl="http://logger-1.jedi.be:9200" '@message:jedi4ever AND program:sshd' --last 5d --format csv --delim ':'
 
+    $ logstash-cli tail --amqpurl="amqp://logger-1.jedi.be:5672" --key="program.sshd"
+
 ## TODO
 
 - find a way to query existing instances
 - specify last 15m 
 - find a way to get the results by streaming instead of loading all in memory (maybe pagination will help here)
-- live tailing the output
 - produce ascii histograms
 - or sparklines

data/lib/logstash-cli/cli.rb

@@ -1,10 +1,11 @@
 require 'rubygems'
 require 'bundler/setup'
 require 'rack'
+require 'amqp'
 require 'tire'
 require 'time'
 require 'fastercsv'
-require 'logstash-cli/command/grep'
+require 'logstash-cli/command'
 
 module LogstashCli
   class CLI < Thor
@@ -16,15 +17,39 @@ module LogstashCli
     method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
     method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
     method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
-    method_option :format , :default => 'csv', :desc => "Format to use for exporting"
+    method_option :format , :default => 'csv', :desc => "Format to use for exporting (plain,csv,json)"
     method_option :size , :default => 500, :desc => "Number of results to return"
     method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
     method_option :meta , :default => "type,message", :desc => "Meta Logstash fields to show"
-    method_option :fields , :default => "message,program", :desc => "Logstash Fields to show"
-    method_option :delim , :default => "|", :desc => "csv delimiter"
+    method_option :fields , :default => "", :desc => "Logstash Fields to show"
+    method_option :delim , :default => "|", :desc => "plain or csv delimiter"
+
     def grep(pattern)
       _grep(pattern,options)
     end
 
+    desc "tail", "Stream a live feed via AMQP"
+    method_option :url,  :desc => "Alternate way to specify settings via an AMQP Url f.i. amqp://logstash:foopass@localhost:5672. \n This takes precendence over other settings. Note that username and password need to be percentage encoded(URL encoded) in case of special characters",:aliases => "\--amqpurl"
+    method_option :user, :default => 'logstash', :desc => "User to connect to AMQP"
+    method_option :password, :default => 'foo', :desc => "Password to connect to AMQP"
+    method_option :vhost, :default => '/', :desc => "VHost to connect to AMQP"
+    method_option :port, :default => 5672, :desc => "Port to connect to AMQP"
+    method_option :host, :default => 'localhost' , :desc => "Host to connect to AMQP"
+    method_option :ssl, :default => false , :desc => "Enable SSL to connect to AMQP", :type => :boolean
+
+    method_option :exchange, :default => 'rawlogs', :desc => "Exchange name"
+    method_option :exchange_type, :default => 'direct', :desc => "Exchange Type"
+    method_option :durable, :default => false, :desc => "Durable Exchange or not", :type => :boolean
+    method_option :auto_delete, :default => false, :desc => "Autodelete Exchange or not" , :type => :boolean
+    method_option :persistent, :default => false, :desc => "Persistent Exchange or not", :type => :boolean
+    method_option :key, :default => '#', :desc => "Routing key"
+    method_option :format , :default => 'csv', :desc => "Format to use for exporting (plain,csv,json)"
+    method_option :meta, :default => "timestamp,type,message", :desc => "Meta Logstash fields to show"
+    method_option :delim, :default => "|", :desc => "csv delimiter"
+
+    def tail()
+      _tail(options)
+    end
+
   end
 end

data/lib/logstash-cli/command/grep.rb

@@ -2,7 +2,7 @@ require 'date'
 
 require 'yajl/json_gem'
 
-module LogstashCli::Command
+module Grep
 
   def _grep(pattern,options)
     es_url = options[:esurl]
@@ -80,6 +80,7 @@ module LogstashCli::Command
           output = case options[:format]
                    when 'csv' then result.to_csv({:col_sep => options[:delim]})
                    when 'json' then result.to_json
+                   when 'plain' then result.join(options[:delim])
                    end
           #tstamp = Time.iso8601(res[:@timestamp]).localtime.iso8601
 

data/lib/logstash-cli/command/tail.rb

@@ -0,0 +1,67 @@
+require 'yajl/json_gem'
+
+module Tail
+
+  def _tail(options)
+    amqp_url = options[:url]
+
+    amqp_user = options[:user]
+    amqp_password = options[:password]
+    amqp_vhost = options[:vhost]
+    amqp_port = options[:port]
+    amqp_host = options[:host]
+    amqp_ssl = options[:ssl]
+
+    exchange_name = options[:exchange]
+    exchange_type = options[:exchange_type]
+    persistent = options[:persistent]
+    durable = options[:durable]
+    auto_delete = options[:autodelete]
+    routing_key = options[:key]
+    metafields = options[:meta].split(',')
+
+    begin
+      #connection = AMQP.connect(AMQP_OPTS.merge(:username => "amqp_gem", :password => "amqp_gem_password", :vhost => "amqp_gem_testbed"))
+      settings= { :host =>  amqp_host, :vhost => amqp_vhost, :port => amqp_port,
+                  :user => amqp_user, :password => amqp_password ,
+                  :ssl => amqp_ssl }
+
+      # Aqmp url can override settings
+      unless amqp_url.nil?
+        settings = aqmp_url
+      end
+
+      AMQP.start(settings) do |connection, open_ok|
+        channel = AMQP::Channel.new(connection, :auto_recovery => true)
+
+        channel.queue("", :auto_delete => auto_delete, :peristent => persistent , :durable => durable)   do |queue, declare_ok|
+          queue.bind(exchange_name, :routing_key => routing_key)
+          queue.subscribe do |payload|
+            parsed_message = JSON.parse(payload)
+            result = Array.new
+
+            metafields.each do |metafield|
+              result << parsed_message["@#{metafield}"]
+            end
+
+            output = case options[:format]
+                     when 'csv' then result.to_csv({:col_sep => options[:delim]})
+                     when 'json' then result.to_json
+                     when 'plain' then result.join(options[:delim])
+                     end
+
+            puts output
+            result = []
+          end
+        end
+      end
+    rescue AMQP::PossibleAuthenticationFailureError => ex
+      puts "Possible Authentication error:\nthe AMQP connection URL used is #{amqp_url}\n\nDetail Info:\n#{ex}"
+      exit -1
+    rescue Exception => ex
+      puts "Error occured: #{ex}"
+      exit -1
+    end
+    trap("INT") { puts "Shutting down..."; connection.close { EM.stop };exit }
+  end
+end

data/lib/logstash-cli/command.rb

@@ -0,0 +1,7 @@
+require 'logstash-cli/command/grep'
+require 'logstash-cli/command/tail'
+
+module LogstashCli::Command
+  include Grep
+  include Tail
+end

data/lib/logstash-cli/version.rb

@@ -1,3 +1,3 @@
 module LogstashCli
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/logstashcli.gemspec

@@ -9,14 +9,14 @@ Gem::Specification.new do |s|
   s.email       = ["patrick.debois@jedi.be"]
   s.homepage    = "http://github.com/jedi4ever/logstash-cli/"
   s.summary     = %q{CLI interface to logstash}
-  s.description = %q{CLI inteface to logstash}
+  s.description = %q{CLI interface to logstash}
 
   s.required_rubygems_version = ">= 1.3.6"
   s.rubyforge_project         = "logstash-cli"
 
   s.add_dependency "tire"
   s.add_dependency "thor"
-#  s.add_dependency "amqp"
+  s.add_dependency "amqp"
   s.add_dependency "rack"
   s.add_dependency "yajl-ruby"
   s.add_dependency "fastercsv"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-cli
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: 
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Patrick Debois
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-05-24 00:00:00 Z
+date: 2012-06-24 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement 
@@ -56,7 +56,7 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  name: rack
+  name: amqp
   version_requirements: *id003
   prerelease: false
 - !ruby/object:Gem::Dependency 
@@ -70,7 +70,7 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  name: yajl-ruby
+  name: rack
   version_requirements: *id004
   prerelease: false
 - !ruby/object:Gem::Dependency 
@@ -84,7 +84,7 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  name: fastercsv
+  name: yajl-ruby
   version_requirements: *id005
   prerelease: false
 - !ruby/object:Gem::Dependency 
@@ -98,11 +98,25 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  name: json
+  name: fastercsv
   version_requirements: *id006
   prerelease: false
 - !ruby/object:Gem::Dependency 
   requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: json
+  version_requirements: *id007
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -115,9 +129,9 @@ dependencies:
         version: 1.0.0
   type: :development
   name: bundler
-  version_requirements: *id007
+  version_requirements: *id008
   prerelease: false
-description: CLI inteface to logstash
+description: CLI interface to logstash
 email: 
 - patrick.debois@jedi.be
 executables: 
@@ -135,8 +149,9 @@ files:
 - bin/logstash-cli
 - lib/logstash-cli.rb
 - lib/logstash-cli/cli.rb
+- lib/logstash-cli/command.rb
 - lib/logstash-cli/command/grep.rb
-- lib/logstash-cli/command/live.rb
+- lib/logstash-cli/command/tail.rb
 - lib/logstash-cli/version.rb
 - logstashcli.gemspec
 homepage: http://github.com/jedi4ever/logstash-cli/

data/lib/logstash-cli/command/live.rb

@@ -1,139 +0,0 @@
-#!/usr/bin/env ruby
-# vim:filetype=ruby
-require 'rubygems'
-require 'bundler/setup'
-begin
-  require 'tire'
-  require 'slop'
-  require 'time'
-rescue LoadError
-  puts "You seem to be missing some key libraries"
-  puts "please run `gem install bundler --no-ri --no-rdoc; bundle install`"
-end
-
-begin
-  require 'yajl/json_gem'
-rescue LoadError
-  puts "`gem install yajl-ruby for better performance"
-end
-
-opts = Slop.parse do
-  banner "Usage: search.rb -i <index> -f <facility>"
-  on :i, :index=, "index to search (default: logstash-#{Time.now.strftime('%Y.%m.%d')})", :default => "logstash-#{Time.now.strftime('%Y.%m.%d')}"
-  on :f, :facility=, "REQUIRED: Facility(application name) to use", nil
-  on :s, :size=, "number of results to return (default: 500)", true, :default => 500
-  on :c, :class_name=, "optional class name to narrow results by", nil
-  on :g, :grep=, "optional search on message content. Warning!!! This is slow", true
-  on :exceptions, "toggle exception search. Warning!!! This is slow", :default => false
-  on :live, "runs in live logging mode. This is A LOT of output. Please use non-wildcard facilities", :default => false
-  on :fields=, Array, "optional comma-separated list of fields to display (tstamp, msg, file, class_name, service) in order (default: tstamp,service,msg)"
-  on :h, :help, 'Print this help message', :tail => true do
-    #puts help
-    puts <<-EOF
-
---------
-Examples
---------
-- last 10 results log entries from curation including timestamp, classname and message:
-
-search.rb -f curation* -s 10 --fields tstamp,class_name,msg
-
-- last 5 entries for class name com.va (note the quote around * in the -f option):
-
-search.rb -f "*" -s 5 -c com.va.*
-
-- last 20 entries everywhere with timestamp, service name and message
-
-search.rb -f "*" -s 20 --fields tstamp,service,msg
-
-- last 5 exceptions everywhere
-
-search.rb -f "*" -s 5 --exceptions
-
-- live tail tracker_web
-
-search.rb -f tracker_web --live
-
-- live tail foo with custom display
-
-search.rb -f foo --live --fields tstamp,service,class_name,msg
-
-    EOF
-    exit
-  end
-end
-
-if opts[:live]
-  require 'amqp'
-  require 'yajl/json_gem'
-  #https://github.com/ruby-amqp/amqp/pull/74
-  #URL naming scheme
-  AMQP.start("amqp://logstash:bar@localhost:7777") do |connection, open_ok|
-    channel = AMQP::Channel.new(connection, :auto_recovery => true)
-    exchange_name = "rawlogs"
-
-    channel.queue("", :auto_delete => true, :durable => false) do |queue, declare_ok|
-      queue.bind(exchange_name, :routing_key => opts[:facility])
-      queue.subscribe do |payload|
-        parsed_message = JSON.parse(payload)
-        require 'pp'
-        pp parsed_message
-        service = parsed_message["@fields"]["facility"]
-        class_name = parsed_message["@fields"]["_logger"]
-        file = parsed_message["@fields"]["file"]
-        msg = parsed_message["@message"]
-        #msg = parsed_message["@fields"]["full_message"]
-        tstamp = Time.iso8601(parsed_message["@timestamp"]).localtime.iso8601
-        fields = opts[:fields] || ["tstamp", "service", "msg"]
-        vals = fields.map {|x| x == fields[0] ? "\e[1m[#{eval(x)}]\e[0m" : eval(x)}
-        display = vals.join(" - ")
-        puts display
-      end
-    end
-
-    trap("INT") { puts "Shutting down..."; connection.close { EM.stop };exit }
-  end
-end
-
-if opts[:facility].nil?
-  puts "Facility (matches name of service) CANNOT be empty!"
-  require 'pp'
-  pp opts
-  exit
-end
-
-begin
-  Tire.configure {url "http://localhost:9200"}
-  puts opts[:index]
-  search = Tire.search(opts[:index]) do
-    query do
-      boolean do
-        must { string 'HOSTNAME:shipper'}
-        #must { string "facility:#{opts[:facility]}" } unless opts[:facility].nil?
-        #must { string "_logger:#{opts[:class_name]}" } unless opts[:class_name].nil?
-        #must { string "full_message:Exception*" } if opts[:exceptions]
-        must { string "message:#{opts[:grep]}*" } if opts[:grep]
-      end
-    end
-    sort do
-      by :@timestamp, 'desc'
-    end
-    size opts[:size]
-  end
-rescue Exception => e
-  puts "\nSomething went wrong with the search. This is usually do to lucene query parsing of the 'grep' option"
-  exit
-end
-
-search.results.sort {|a,b| a[:@timestamp] <=> b[:@timestamp] }.each do |res|
-  service = res[:@fields][:facility]
-  class_name = res[:@fields][:_logger]
-  file = res[:@fields][:file]
-  #msg = res[:@fields][:full_message]
-  msg = res[:@fields][:message]
-  tstamp = Time.iso8601(res[:@timestamp]).localtime.iso8601
-  fields = opts[:fields] || ["tstamp", "service", "msg"]
-  vals = fields.map {|x| x == fields[0] ? "\e[1m[#{eval(x)}]\e[0m" : eval(x)}
-  display = vals.join(" - ")
-  puts display
-end