logstash-cli 0.0.6 → 0.0.7

data/.rvmrc

@@ -1,5 +1,5 @@
 #this uses rvm
-rvm use 1.8.7
+rvm use 1.9.2
 rvm_gemset_create_on_use_flag=1
 rvm gemset use logstash-cli
 alias rake="bundle exec rake"

data/README.md

@@ -25,6 +25,10 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
 
 ### Using the Gem
 
+    # If you no rvm
+    $ bundle exec bin/logstash-cli
+
+    # If you have rvm , there is an alias in .rvmrc
     $ logstash-cli
 
 ### Using the Github version - through bundler
@@ -46,18 +50,18 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
       [--meta=META]                  # Meta Logstash fields to show
                                      # Default: type,message
       [--to=TO]                      # End date
-                                     # Default: Today in YYYY-MM-DD form
+                                     # Default: Today in YYYY-MM-DD HH:MM:SS form (the time is optional)
       [--delim=DELIM]                # plain or csv delimiter
                                      # Default: |
       [--format=FORMAT]              # Format to use for exporting
                                      # Default: csv
       [--from=FROM]                  # Begin date
-                                     # Default: Today in YYYY-MM-DD form
+                                     # Default: Today in YYYY-MM-DD HH:MM:SS form (the time is optional)
       [--size=SIZE]                  # Number of results to return
                                      # Default: 500
       [--esurl=ESURL]                # URL to connect to elasticsearch
                                      # Default: http://localhost:9200
-      [--last=LAST]                  # Specify period since now f.i. 1d
+      [--last=LAST]                  # Specify period since now (Examples: 10min, 3hrs, 4days, 1wk, 1yr)
 
     Search logstash for a pattern
 
@@ -138,7 +142,6 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
 ## TODO
 
 - find a way to query existing instances
-- specify last 15m 
 - find a way to get the results by streaming instead of loading all in memory (maybe pagination will help here)
 - produce ascii histograms
 - or sparklines

data/lib/logstash-cli/cli.rb

@@ -18,11 +18,11 @@ module LogstashCli
     desc "grep PATTERN", "Search logstash for a pattern"
     method_option :esurl , :default => 'http://localhost:9200', :desc => "URL to connect to elasticsearch"
     method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
-    method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
-    method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
+    method_option :from , :default => nil, :desc => "Begin date and (optionally) time."
+    method_option :to, :default => nil, :desc => "End date and (optionally) time"
     method_option :format , :default => 'plain', :desc => "Format to use for exporting (plain,csv,json)"
     method_option :size , :default => 500, :desc => "Number of results to return"
-    method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
+    method_option :last , :default => '1d', :desc => "Specify period since now f.i. 1d"
     method_option :meta , :default => "type,message", :desc => "Meta Logstash fields to show"
     method_option :fields , :default => "", :desc => "Logstash Fields to show"
     method_option :delim , :default => "|", :desc => "plain or csv delimiter"

data/lib/logstash-cli/command/count.rb

@@ -65,7 +65,8 @@ module Count
         puts _format(header, options)
 
         results['terms'].each do |terms|
-          puts _format(terms.values, options)
+          result = [ terms['term'], terms['count'] ]
+          puts _format(result, options)
 
           unless fields.empty? and metafields.empty?
             term = terms['term']

data/lib/logstash-cli/command/grep.rb

@@ -4,32 +4,60 @@ require 'yajl/json_gem'
 
 module Grep
 
+  def self.indexes_from_interval(from, to)
+    (from.to_date..to.to_date).sort.map do |date|
+      date.to_s.gsub('-', '.')
+    end
+  end
+
+  # Very naive time range description parsing.
+  def self.parse_time_range(desc)
+    /(?<value>\d+)\s*(?<units>\w*)/ =~ desc
+    value = value.to_i
+    start = case units.to_s.downcase
+            when 'm', 'min', 'mins', 'minute', 'minutes'
+              DateTime.now - (value/(60*24.0))
+            when 'h', 'hr', 'hrs', 'hour', 'hours'
+              DateTime.now - (value/24.0)
+            when 'd', 'day', 'days'
+              DateTime.now - value
+            when 'w', 'wk', 'wks', 'week', 'weeks'
+              DateTime.now - (7.0*value)
+            when 'y', 'yr', 'yrs', 'year', 'years'
+              DateTime.now - (365.0*value)
+            else
+              raise ArgumentError
+            end
+    [start, DateTime.now]
+  end
+
   def _grep(pattern,options)
     es_url = options[:esurl]
     index_prefix =  options[:index_prefix]
-
-    from = options[:from]
-    to  = options[:to]
     metafields = options[:meta].split(',')
     fields = options[:fields].split(',')
 
     begin
-      unless options[:last].nil?
-        days = options[:last].match(/(\d*)d/)[1].to_i
-        to_date = Date.today
-        from_date = to_date - days
-        from = from_date.to_s
-        to = to_date.to_s
+      if options[:last].nil?
+        from_time = DateTime.parse(options[:from])
+        to_time = DateTime.parse(options[:to])
+      else
+        from_time, to_time = Grep.parse_time_range(options[:last])
       end
-
-      from_date = Date.parse(from)
-      to_date = Date.parse(to)
-    rescue Exception => ex
-      $stderr.puts "Something went wrong while parsing the dates: currently only dates are supported with last. Be sure to add the suffix 'd' "+ex
+    rescue ArgumentError
+      $stderr.puts "Something went wrong while parsing the date range."
       exit -1
     end
 
-    $stderr.puts "Searching #{es_url}[#{index_prefix}#{from_date}..#{index_prefix}#{to_date}] - #{pattern}"
+    index_range = Grep.indexes_from_interval(from_time, to_time).map do |i|
+      "#{index_prefix}#{i}"
+    end
+
+    $stderr.puts "Searching #{es_url}[#{index_range.first}..#{index_range.last}] - #{pattern}"
+
+    # Reformat time interval to match logstash's internal timestamp'
+    from = from_time.to_time.utc.strftime('%FT%T')
+    to = to_time.to_time.utc.strftime('%FT%T')
 
     # Total of results to show
     total_result_size = options[:size]
@@ -40,19 +68,17 @@ module Grep
     running_result_size = total_result_size.to_i
 
     # We reverse the order of working ourselves through the index
-    (from_date..to_date).sort.reverse.to_a.each do |date|
-
-      es_index = index_prefix+date.to_s.gsub('-','.')
-
+    index_range.reverse.each do |idx|
       begin
         Tire.configure {url es_url}
-        search = Tire.search(es_index) do
+        search = Tire.search(idx) do
           query do
             string "#{pattern}"
           end
           sort do
             by :@timestamp, 'desc'
           end
+          filter "range", "@timestamp" => { "from" => from, "to" => to}
           size running_result_size
         end
       rescue Exception => e
@@ -81,7 +107,8 @@ module Grep
           result = []
         end
       rescue ::Tire::Search::SearchRequestFailed => e
-        $stderr.puts e.message
+        # If we got a 404 it likely means we simply don't have logs for that day, not failing over necessarily.
+        $stderr.puts e.message unless search.response.code == 404
       end
     end
   end

data/lib/logstash-cli/command/tail.rb

@@ -36,7 +36,7 @@ module Tail
 
         channel = AMQP::Channel.new(connection, :auto_recovery => true)
 
-        channel.queue("", :auto_delete => auto_delete, :peristent => persistent , :durable => durable)   do |queue, declare_ok|
+        channel.queue("", :auto_delete => auto_delete, :persistent => persistent , :durable => durable)   do |queue, declare_ok|
           queue.bind(exchange_name, :routing_key => routing_key)
           queue.subscribe do |payload|
             parsed_message = JSON.parse(payload)

data/lib/logstash-cli/version.rb

@@ -1,3 +1,3 @@
 module LogstashCli
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/logstash-cli/command/grep_spec.rb

@@ -0,0 +1,58 @@
+require_relative '../../spec_helper.rb'
+require_relative '../../../lib/logstash-cli/command/grep.rb'
+
+describe Grep do
+  it "determines index range needed for a given date range" do
+    Grep.indexes_from_interval(DateTime.now, DateTime.now).should == [DateTime.now.to_date.to_s.gsub('-', '.')]
+  end
+
+  it "calculates time range required to search last X minutes" do
+    # The to_i removes fractions seconds.  Unsure on the best way to test this.
+    diff = 10*(1/60.0*24.0)
+    Grep.parse_time_range("10m").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10mins").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 min").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 minute").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 minutes").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+  end
+
+  it "calculates time range required to search last X hours" do
+    diff = 10*(1/24.0)
+    Grep.parse_time_range("10h").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10hr").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10hrs").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 hour").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 hours").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+  end
+
+  it "calculates time range required to search last X days" do
+    diff = 10
+    Grep.parse_time_range("10d").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10days").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 day").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+  end
+
+  it "calculates time range required to search last X weeks" do
+    diff = 10*7
+    Grep.parse_time_range("10w").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10wk").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 wks").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 week").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 weeks").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+  end
+
+  it "calculates time range required to search last X years" do
+    diff = 10*365
+    Grep.parse_time_range("10y").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 yr").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 yrs").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 year").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+    Grep.parse_time_range("10 years").map{|t| t.to_time.to_i}.should == [DateTime.now - diff, DateTime.now].map{|t| t.to_time.to_i}
+  end
+
+  it "raises an ArgumentError if it can't cacluate the time range" do
+    lambda {Grep.parse_time_range("two-weeks")}.should raise_error ArgumentError
+    lambda {Grep.parse_time_range("10parsecs")}.should raise_error ArgumentError
+  end
+
+end

metadata

@@ -1,146 +1,152 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: logstash-cli
-version: !ruby/object:Gem::Version 
-  hash: 19
+version: !ruby/object:Gem::Version
+  version: 0.0.7
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 6
-  version: 0.0.6
 platform: ruby
-authors: 
+authors:
 - Patrick Debois
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-08-21 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
+date: 2012-12-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: tire
-  version_requirements: *id001
-  prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  name: thor
-  version_requirements: *id002
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  name: amqp
-  version_requirements: *id003
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  name: rack
-  version_requirements: *id004
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  name: yajl-ruby
-  version_requirements: *id005
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  name: fastercsv
-  version_requirements: *id006
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fastercsv
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: json
-  version_requirements: *id007
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
-  name: bundler
-  version_requirements: *id008
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: CLI interface to logstash
-email: 
+email:
 - patrick.debois@jedi.be
-executables: 
+executables:
 - logstash-cli
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - .rvmrc
 - Gemfile
@@ -155,40 +161,33 @@ files:
 - lib/logstash-cli/command/tail.rb
 - lib/logstash-cli/version.rb
 - logstashcli.gemspec
+- spec/logstash-cli/command/grep_spec.rb
+- spec/spec_helper.rb
 homepage: http://github.com/jedi4ever/logstash-cli/
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: 3090991630995914937
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 23
-      segments: 
-      - 1
-      - 3
-      - 6
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-
 rubyforge_project: logstash-cli
-rubygems_version: 1.8.12
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: CLI interface to logstash
 test_files: []
-