logstash-cli 0.0.1

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.rvmrc

@@ -0,0 +1,6 @@
+#this uses rvm
+rvm use 1.8.7
+rvm_gemset_create_on_use_flag=1
+rvm gemset use logstash-cli
+alias rake="bundle exec rake"
+alias logstash-cli="bundle exec bin/logstash-cli"

data/Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+
+gem "logstash-cli", :path => "."
+
+group :test do
+  gem "rake"
+end
+
+gemspec

data/README.md

@@ -0,0 +1,55 @@
+** Work in progress **
+
+## Description
+
+A cli tool to query an elasticsearch host for logstash information.
+Because let's face it, we're CLI junkies :)
+
+Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/1388077>
+
+## Installation
+Installation using the usual steps
+
+Install rvm , (no gem yet)
+
+    $ git clone thisrepo
+    $ cd thisrepo
+    $ gem install bundler
+    $ bundle install
+
+## Usage
+
+    Usage:
+      logstash-cli grep PATTERN
+
+    Options:
+      [--index-prefix=INDEX_PREFIX]  # Logstash index prefix
+                                     # Default: logstash-
+      [--fields=FIELDS]              # Logstash Fields to show
+                                     # Default: message,program
+      [--meta=META]                  # Meta Logstash fields to show
+                                     # Default: type,message
+      [--to=TO]                      # End date
+                                     # Default: 2012-05-11
+      [--delim=DELIM]                # csv delimiter
+                                     # Default: |
+      [--format=FORMAT]              # Format to use for exporting
+                                     # Default: csv
+      [--from=FROM]                  # Begin date
+                                     # Default: 2012-05-11
+      [--size=SIZE]                  # Number of results to return
+                                     # Default: 500
+      [--esurl=ESURL]                # URL to connect to elasticsearch
+                                     # Default: http://localhost:9200
+      [--last=LAST]                  # Specify period since now f.i. 1d
+
+    Search logstash for a pattern
+
+## Examples
+
+    $ logstash-cli grep --esurl="http://logger-1.jedi.be:9200" '@message:jedi4ever AND program:sshd' --last 5d --format csv --delim ':'
+## TODO
+
+- find a way to query existing instances
+- specify last 15m 
+- find a way to get the results by streaming instead of loading all in memory (maybe pagination will help here)

data/bin/logstash-cli

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+require 'logstash-cli'
+
+# Disable color if the proper argument was passed
+shell = ARGV.include?("--no-color") ? Thor::Shell::Basic.new : Thor::Base.shell.new
+
+# Start the CLI
+::LogstashCli::CLI.start(ARGV)

data/lib/logstash-cli/cli.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'rack'
+require 'tire'
+require 'time'
+require 'fastercsv'
+require 'logstash-cli/command/grep'
+
+module LogstashCli
+  class CLI < Thor
+
+    include LogstashCli::Command
+
+    desc "grep PATTERN", "Search logstash for a pattern"
+    method_option :esurl , :default => 'http://localhost:9200', :desc => "URL to connect to elasticsearch"
+    method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
+    method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
+    method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
+    method_option :format , :default => 'csv', :desc => "Format to use for exporting"
+    method_option :size , :default => 500, :desc => "Number of results to return"
+    method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
+    method_option :meta , :default => "type,message", :desc => "Meta Logstash fields to show"
+    method_option :fields , :default => "message,program", :desc => "Logstash Fields to show"
+    method_option :delim , :default => "|", :desc => "csv delimiter"
+    def grep(pattern)
+      _grep(pattern,options)
+    end
+
+  end
+end

data/lib/logstash-cli/command/grep.rb

@@ -0,0 +1,82 @@
+require 'date'
+
+require 'yajl/json_gem'
+
+module LogstashCli::Command
+
+  def _grep(pattern,options)
+    es_url = options[:esurl]
+    index_prefix =  options[:index_prefix]
+
+    from = options[:from]
+    to  = options[:to]
+    metafields = options[:meta].split(',')
+    fields = options[:fields].split(',')
+
+    begin
+      unless options[:last].nil?
+        days = options[:last].match(/(\d*)d/)[1].to_i
+        to_date = Date.today
+        from_date = to_date - days
+        from = from_date.to_s
+        to = to_date.to_s
+      end
+
+      from_date = Date.parse(from)
+      to_date = Date.parse(to)
+    rescue Exception => ex
+      $stderr.puts "Something went wrong while parsing the dates: currently only dates are supported with last. Be sure to add the suffix 'd' "+ex
+      exit -1
+    end
+
+    $stderr.puts "Searching #{es_url}[#{index_prefix}#{from_date}..#{index_prefix}#{to_date}] - #{pattern}"
+
+    (from_date..to_date).to_a.each do |date|
+      es_index = index_prefix+date.to_s.gsub('-','.')
+
+      result_size = options[:size]
+
+      begin
+        Tire.configure {url es_url}
+        search = Tire.search(es_index) do
+          query do
+            string "#{pattern}"
+          end
+          sort do
+            by :@timestamp, 'desc'
+          end
+          size result_size
+        end
+      rescue Exception => e
+        $stderr.puts e
+        $stderr.puts "\nSomething went wrong with the search. This is usually due to lucene query parsing of the 'grep' option"
+        exit
+      end
+
+      begin
+        result = Array.new
+        search.results.sort {|a,b| a[:@timestamp] <=> b[:@timestamp] }.each do |res|
+
+          metafields.each do |metafield|
+            result << res["@#{metafield}".to_sym]
+          end
+
+          fields.each do |field|
+            result << res[:@fields][field.to_sym]
+          end
+
+          output = case options[:format]
+            when 'csv' then result.to_csv({:col_sep => options[:delim]})
+            when 'json' then result.to_json
+          end
+          #tstamp = Time.iso8601(res[:@timestamp]).localtime.iso8601
+
+          puts output
+          result = []
+        end
+      rescue ::Tire::Search::SearchRequestFailed => e
+        $stderr.puts e.message
+      end
+    end
+  end
+end

data/lib/logstash-cli/command/live.rb

@@ -0,0 +1,139 @@
+#!/usr/bin/env ruby
+# vim:filetype=ruby
+require 'rubygems'
+require 'bundler/setup'
+begin
+  require 'tire'
+  require 'slop'
+  require 'time'
+rescue LoadError
+  puts "You seem to be missing some key libraries"
+  puts "please run `gem install bundler --no-ri --no-rdoc; bundle install`"
+end
+
+begin
+  require 'yajl/json_gem'
+rescue LoadError
+  puts "`gem install yajl-ruby for better performance"
+end
+
+opts = Slop.parse do
+  banner "Usage: search.rb -i <index> -f <facility>"
+  on :i, :index=, "index to search (default: logstash-#{Time.now.strftime('%Y.%m.%d')})", :default => "logstash-#{Time.now.strftime('%Y.%m.%d')}"
+  on :f, :facility=, "REQUIRED: Facility(application name) to use", nil
+  on :s, :size=, "number of results to return (default: 500)", true, :default => 500
+  on :c, :class_name=, "optional class name to narrow results by", nil
+  on :g, :grep=, "optional search on message content. Warning!!! This is slow", true
+  on :exceptions, "toggle exception search. Warning!!! This is slow", :default => false
+  on :live, "runs in live logging mode. This is A LOT of output. Please use non-wildcard facilities", :default => false
+  on :fields=, Array, "optional comma-separated list of fields to display (tstamp, msg, file, class_name, service) in order (default: tstamp,service,msg)"
+  on :h, :help, 'Print this help message', :tail => true do
+    #puts help
+    puts <<-EOF
+
+--------
+Examples
+--------
+- last 10 results log entries from curation including timestamp, classname and message:
+
+search.rb -f curation* -s 10 --fields tstamp,class_name,msg
+
+- last 5 entries for class name com.va (note the quote around * in the -f option):
+
+search.rb -f "*" -s 5 -c com.va.*
+
+- last 20 entries everywhere with timestamp, service name and message
+
+search.rb -f "*" -s 20 --fields tstamp,service,msg
+
+- last 5 exceptions everywhere
+
+search.rb -f "*" -s 5 --exceptions
+
+- live tail tracker_web
+
+search.rb -f tracker_web --live
+
+- live tail foo with custom display
+
+search.rb -f foo --live --fields tstamp,service,class_name,msg
+
+    EOF
+    exit
+  end
+end
+
+if opts[:live]
+  require 'amqp'
+  require 'yajl/json_gem'
+  #https://github.com/ruby-amqp/amqp/pull/74
+  #URL naming scheme
+  AMQP.start("amqp://logstash:bar@localhost:7777") do |connection, open_ok|
+    channel = AMQP::Channel.new(connection, :auto_recovery => true)
+    exchange_name = "rawlogs"
+
+    channel.queue("", :auto_delete => true, :durable => false) do |queue, declare_ok|
+      queue.bind(exchange_name, :routing_key => opts[:facility])
+      queue.subscribe do |payload|
+        parsed_message = JSON.parse(payload)
+        require 'pp'
+        pp parsed_message
+        service = parsed_message["@fields"]["facility"]
+        class_name = parsed_message["@fields"]["_logger"]
+        file = parsed_message["@fields"]["file"]
+        msg = parsed_message["@message"]
+        #msg = parsed_message["@fields"]["full_message"]
+        tstamp = Time.iso8601(parsed_message["@timestamp"]).localtime.iso8601
+        fields = opts[:fields] || ["tstamp", "service", "msg"]
+        vals = fields.map {|x| x == fields[0] ? "\e[1m[#{eval(x)}]\e[0m" : eval(x)}
+        display = vals.join(" - ")
+        puts display
+      end
+    end
+
+    trap("INT") { puts "Shutting down..."; connection.close { EM.stop };exit }
+  end
+end
+
+if opts[:facility].nil?
+  puts "Facility (matches name of service) CANNOT be empty!"
+  require 'pp'
+  pp opts
+  exit
+end
+
+begin
+  Tire.configure {url "http://localhost:9200"}
+  puts opts[:index]
+  search = Tire.search(opts[:index]) do
+    query do
+      boolean do
+        must { string 'HOSTNAME:shipper'}
+        #must { string "facility:#{opts[:facility]}" } unless opts[:facility].nil?
+        #must { string "_logger:#{opts[:class_name]}" } unless opts[:class_name].nil?
+        #must { string "full_message:Exception*" } if opts[:exceptions]
+        must { string "message:#{opts[:grep]}*" } if opts[:grep]
+      end
+    end
+    sort do
+      by :@timestamp, 'desc'
+    end
+    size opts[:size]
+  end
+rescue Exception => e
+  puts "\nSomething went wrong with the search. This is usually do to lucene query parsing of the 'grep' option"
+  exit
+end
+
+search.results.sort {|a,b| a[:@timestamp] <=> b[:@timestamp] }.each do |res|
+  service = res[:@fields][:facility]
+  class_name = res[:@fields][:_logger]
+  file = res[:@fields][:file]
+  #msg = res[:@fields][:full_message]
+  msg = res[:@fields][:message]
+  tstamp = Time.iso8601(res[:@timestamp]).localtime.iso8601
+  fields = opts[:fields] || ["tstamp", "service", "msg"]
+  vals = fields.map {|x| x == fields[0] ? "\e[1m[#{eval(x)}]\e[0m" : eval(x)}
+  display = vals.join(" - ")
+  puts display
+end

data/lib/logstash-cli/version.rb

@@ -0,0 +1,3 @@
+module LogstashCli
+  VERSION = "0.0.1"
+end

data/lib/logstash-cli.rb

@@ -0,0 +1,3 @@
+require 'thor'
+
+require 'logstash-cli/cli'

data/logstashcli.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path("../lib/logstash-cli/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "logstash-cli"
+  s.version     = LogstashCli::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Patrick Debois"]
+  s.email       = ["patrick.debois@jedi.be"]
+  s.homepage    = "http://github.com/jedi4ever/logstash-cli/"
+  s.summary     = %q{CLI interface to logstash}
+  s.description = %q{CLI inteface to logstash}
+
+  s.required_rubygems_version = ">= 1.3.6"
+  s.rubyforge_project         = "logstash-cli"
+
+  s.add_dependency "tire"
+  s.add_dependency "thor"
+#  s.add_dependency "amqp"
+  s.add_dependency "rack"
+  s.add_dependency "yajl-ruby"
+  s.add_dependency "fastercsv"
+  s.add_dependency "json"
+  #s.add_dependency "amqp-utils"
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+
+  s.files        = `git ls-files`.split("\n")
+  s.executables  = `git ls-files`.split("\n").map{ |f| f =~ /^bin\/(.*)/ ? $1 : nil }.compact
+  s.require_path = 'lib'
+end
+

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification 
+name: logstash-cli
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Patrick Debois
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-05-11 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: tire
+  version_requirements: *id001
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: thor
+  version_requirements: *id002
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: rack
+  version_requirements: *id003
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: yajl-ruby
+  version_requirements: *id004
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: fastercsv
+  version_requirements: *id005
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: json
+  version_requirements: *id006
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :development
+  name: bundler
+  version_requirements: *id007
+  prerelease: false
+description: CLI inteface to logstash
+email: 
+- patrick.debois@jedi.be
+executables: 
+- logstash-cli
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rvmrc
+- Gemfile
+- README.md
+- bin/logstash-cli
+- lib/logstash-cli.rb
+- lib/logstash-cli/cli.rb
+- lib/logstash-cli/command/grep.rb
+- lib/logstash-cli/command/live.rb
+- lib/logstash-cli/version.rb
+- logstashcli.gemspec
+homepage: http://github.com/jedi4ever/logstash-cli/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: logstash-cli
+rubygems_version: 1.8.12
+signing_key: 
+specification_version: 3
+summary: CLI interface to logstash
+test_files: []
+