logstash-cli 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +34 -4
- data/lib/logstash-cli/cli.rb +21 -4
- data/lib/logstash-cli/command/count.rb +108 -0
- data/lib/logstash-cli/command/grep.rb +1 -8
- data/lib/logstash-cli/command/tail.rb +1 -7
- data/lib/logstash-cli/command.rb +11 -0
- data/lib/logstash-cli/version.rb +1 -1
- metadata +5 -4
data/README.md
CHANGED
@@ -46,13 +46,13 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
|
|
46
46
|
[--meta=META] # Meta Logstash fields to show
|
47
47
|
# Default: type,message
|
48
48
|
[--to=TO] # End date
|
49
|
-
# Default:
|
50
|
-
[--delim=DELIM] # csv delimiter
|
49
|
+
# Default: Today in YYYY-MM-DD form
|
50
|
+
[--delim=DELIM] # plain or csv delimiter
|
51
51
|
# Default: |
|
52
52
|
[--format=FORMAT] # Format to use for exporting
|
53
53
|
# Default: csv
|
54
54
|
[--from=FROM] # Begin date
|
55
|
-
# Default:
|
55
|
+
# Default: Today in YYYY-MM-DD form
|
56
56
|
[--size=SIZE] # Number of results to return
|
57
57
|
# Default: 500
|
58
58
|
[--esurl=ESURL] # URL to connect to elasticsearch
|
@@ -90,7 +90,7 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
|
|
90
90
|
# Default: rawlogs
|
91
91
|
[--password=PASSWORD] # Password to connect to AMQP
|
92
92
|
# Default: foo
|
93
|
-
[--delim=DELIM] # csv delimiter
|
93
|
+
[--delim=DELIM] # plain or csv delimiter
|
94
94
|
# Default: |
|
95
95
|
[--exchange-type=EXCHANGE_TYPE] # Exchange Type
|
96
96
|
# Default: direct
|
@@ -98,6 +98,34 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
|
|
98
98
|
|
99
99
|
Stream a live feed via AMQP
|
100
100
|
|
101
|
+
### Count
|
102
|
+
|
103
|
+
Usage:
|
104
|
+
logstash-cli count PATTERN --countfield=COUNTFIELD
|
105
|
+
|
106
|
+
Options:
|
107
|
+
[--meta=META] # Meta Logstash fields to show
|
108
|
+
[--last=LAST] # Specify period since now f.i. 1d
|
109
|
+
[--from=FROM] # Begin date
|
110
|
+
# Default: Today in YYYY-MM-DD form
|
111
|
+
[--delim=DELIM] # plain or csv delimiter
|
112
|
+
# Default: |
|
113
|
+
--countfield=COUNTFIELD # Logstash field to count
|
114
|
+
[--countsize=COUNTSIZE] # Number of most frequent values to return
|
115
|
+
# Default: 50
|
116
|
+
[--format=FORMAT] # Format to use for exporting (plain,csv,json)
|
117
|
+
# Default: csv
|
118
|
+
[--to=TO] # End date
|
119
|
+
# Default: Today in YYYY-MM-DD form
|
120
|
+
[--fields=FIELDS] # Logstash fields to show
|
121
|
+
[--size=SIZE] # Number of results per index to show
|
122
|
+
# Default: 10
|
123
|
+
[--esurl=ESURL] # URL to connect to elasticsearch
|
124
|
+
# Default: http://localhost:9200
|
125
|
+
[--index-prefix=INDEX_PREFIX] # Logstash index prefix
|
126
|
+
# Default: logstash-
|
127
|
+
|
128
|
+
Return most frequent values of a field within a pattern and optionally show associated fields
|
101
129
|
|
102
130
|
## Examples
|
103
131
|
|
@@ -105,6 +133,8 @@ Mucho inspired by a gist of the eminent @lusis - <https://gist.github.com/138807
|
|
105
133
|
|
106
134
|
$ logstash-cli tail --amqpurl="amqp://logger-1.jedi.be:5672" --key="program.sshd"
|
107
135
|
|
136
|
+
$ logstash-cli count --esurl="http://logger-1.jedi.be:9200" '@message:jedi4ever' --countfield=program
|
137
|
+
|
108
138
|
## TODO
|
109
139
|
|
110
140
|
- find a way to query existing instances
|
data/lib/logstash-cli/cli.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'rubygems'
|
2
|
-
require 'bundler/setup'
|
3
2
|
require 'rack'
|
4
3
|
require 'amqp'
|
5
4
|
require 'tire'
|
@@ -21,7 +20,7 @@ module LogstashCli
|
|
21
20
|
method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
|
22
21
|
method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
|
23
22
|
method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
|
24
|
-
method_option :format , :default => '
|
23
|
+
method_option :format , :default => 'plain', :desc => "Format to use for exporting (plain,csv,json)"
|
25
24
|
method_option :size , :default => 500, :desc => "Number of results to return"
|
26
25
|
method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
|
27
26
|
method_option :meta , :default => "type,message", :desc => "Meta Logstash fields to show"
|
@@ -47,13 +46,31 @@ module LogstashCli
|
|
47
46
|
method_option :auto_delete, :default => false, :desc => "Autodelete Exchange or not" , :type => :boolean
|
48
47
|
method_option :persistent, :default => false, :desc => "Persistent Exchange or not", :type => :boolean
|
49
48
|
method_option :key, :default => '#', :desc => "Routing key"
|
50
|
-
method_option :format , :default => '
|
49
|
+
method_option :format , :default => 'plain', :desc => "Format to use for exporting (plain,csv,json)"
|
51
50
|
method_option :meta, :default => "timestamp,type,message", :desc => "Meta Logstash fields to show"
|
52
|
-
method_option :delim, :default => "|", :desc => "csv delimiter"
|
51
|
+
method_option :delim, :default => "|", :desc => "plain or csv delimiter"
|
53
52
|
|
54
53
|
def tail()
|
55
54
|
_tail(options)
|
56
55
|
end
|
57
56
|
|
57
|
+
desc "count PATTERN", "Return most frequent values of a field within a pattern and optionally show associated fields"
|
58
|
+
method_option :esurl , :default => 'http://localhost:9200', :desc => "URL to connect to elasticsearch"
|
59
|
+
method_option :index_prefix , :default => "logstash-", :desc => "Logstash index prefix"
|
60
|
+
method_option :from , :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "Begin date"
|
61
|
+
method_option :to, :default => "#{Time.now.strftime('%Y-%m-%d')}", :desc => "End date"
|
62
|
+
method_option :format , :default => 'csv', :desc => "Format to use for exporting (plain,csv,json)"
|
63
|
+
method_option :size , :default => 10, :desc => "Number of results per index to show"
|
64
|
+
method_option :last , :default => nil, :desc => "Specify period since now f.i. 1d"
|
65
|
+
method_option :meta , :default => "", :desc => "Meta Logstash fields to show"
|
66
|
+
method_option :fields , :default => "", :desc => "Logstash fields to show"
|
67
|
+
method_option :countfield , :required => true, :desc => "Logstash field to count"
|
68
|
+
method_option :countsize , :default => 50, :desc => "Number of most frequent values to return"
|
69
|
+
method_option :delim , :default => "|", :desc => "plain or csv delimiter"
|
70
|
+
|
71
|
+
def count(pattern)
|
72
|
+
_count(pattern,options)
|
73
|
+
end
|
74
|
+
|
58
75
|
end
|
59
76
|
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require 'date'
|
2
|
+
|
3
|
+
require 'yajl/json_gem'
|
4
|
+
|
5
|
+
module Count
|
6
|
+
|
7
|
+
def _count(pattern,options)
|
8
|
+
es_url = options[:esurl]
|
9
|
+
index_prefix = options[:index_prefix]
|
10
|
+
|
11
|
+
from = options[:from]
|
12
|
+
to = options[:to]
|
13
|
+
metafields = options[:meta].split(',')
|
14
|
+
fields = options[:fields].split(',')
|
15
|
+
|
16
|
+
countfield = options[:countfield]
|
17
|
+
countsize = options[:countsize]
|
18
|
+
|
19
|
+
begin
|
20
|
+
unless options[:last].nil?
|
21
|
+
days = options[:last].match(/(\d*)d/)[1].to_i
|
22
|
+
to_date = Date.today
|
23
|
+
from_date = to_date - days
|
24
|
+
from = from_date.to_s
|
25
|
+
to = to_date.to_s
|
26
|
+
end
|
27
|
+
|
28
|
+
from_date = Date.parse(from)
|
29
|
+
to_date = Date.parse(to)
|
30
|
+
rescue Exception => ex
|
31
|
+
$stderr.puts "Something went wrong while parsing the dates: currently only dates are supported with last. Be sure to add the suffix 'd' "+ex
|
32
|
+
exit -1
|
33
|
+
end
|
34
|
+
|
35
|
+
$stderr.puts "Searching #{es_url}[#{index_prefix}#{from_date}..#{index_prefix}#{to_date}] - #{pattern}"
|
36
|
+
|
37
|
+
# We reverse the order of working ourselves through the index
|
38
|
+
(from_date..to_date).sort.reverse.to_a.each do |date|
|
39
|
+
|
40
|
+
es_index = index_prefix+date.to_s.gsub('-','.')
|
41
|
+
|
42
|
+
begin
|
43
|
+
Tire.configure {url es_url}
|
44
|
+
search = Tire.search(es_index) do
|
45
|
+
query do
|
46
|
+
string "#{pattern}"
|
47
|
+
end
|
48
|
+
facet "#{countfield}" do
|
49
|
+
terms countfield, :size => countsize
|
50
|
+
end
|
51
|
+
end
|
52
|
+
rescue Exception => e
|
53
|
+
$stderr.puts e
|
54
|
+
$stderr.puts "\nSomething went wrong with the search. This is usually due to lucene query parsing"
|
55
|
+
exit
|
56
|
+
end
|
57
|
+
|
58
|
+
# Results per index to show
|
59
|
+
result_size = options[:size]
|
60
|
+
|
61
|
+
begin
|
62
|
+
results = search.results.facets[countfield]
|
63
|
+
|
64
|
+
header = [ countfield, results['total'] ]
|
65
|
+
puts _format(header, options)
|
66
|
+
|
67
|
+
results['terms'].each do |terms|
|
68
|
+
puts _format(terms.values, options)
|
69
|
+
|
70
|
+
unless fields.empty? and metafields.empty?
|
71
|
+
term = terms['term']
|
72
|
+
begin
|
73
|
+
Tire.configure {url es_url}
|
74
|
+
search = Tire.search(es_index) do
|
75
|
+
query do
|
76
|
+
string "#{pattern}"
|
77
|
+
end
|
78
|
+
filter :terms, countfield => [term]
|
79
|
+
size result_size
|
80
|
+
end
|
81
|
+
rescue Exception => e
|
82
|
+
$stderr.puts e
|
83
|
+
$stderr.puts "\nSomething went wrong with the search. This is usually due to lucene query parsing"
|
84
|
+
exit
|
85
|
+
end
|
86
|
+
|
87
|
+
search.results.each do |log|
|
88
|
+
result = Array.new
|
89
|
+
|
90
|
+
metafields.each do |metafield|
|
91
|
+
result << log["@#{metafield}".to_sym]
|
92
|
+
end
|
93
|
+
|
94
|
+
fields.each do |field|
|
95
|
+
result << log[:@fields][field.to_sym]
|
96
|
+
end
|
97
|
+
|
98
|
+
puts _format(result, options)
|
99
|
+
result = []
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
rescue ::Tire::Search::SearchRequestFailed => e
|
104
|
+
$stderr.puts e.message
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -77,14 +77,7 @@ module Grep
|
|
77
77
|
result << res[:@fields][field.to_sym]
|
78
78
|
end
|
79
79
|
|
80
|
-
|
81
|
-
when 'csv' then result.to_csv({:col_sep => options[:delim]})
|
82
|
-
when 'json' then result.to_json
|
83
|
-
when 'plain' then result.join(options[:delim])
|
84
|
-
end
|
85
|
-
#tstamp = Time.iso8601(res[:@timestamp]).localtime.iso8601
|
86
|
-
|
87
|
-
puts output
|
80
|
+
puts _format(result, options)
|
88
81
|
result = []
|
89
82
|
end
|
90
83
|
rescue ::Tire::Search::SearchRequestFailed => e
|
@@ -44,13 +44,7 @@ module Tail
|
|
44
44
|
result << parsed_message["@#{metafield}"]
|
45
45
|
end
|
46
46
|
|
47
|
-
|
48
|
-
when 'csv' then result.to_csv({:col_sep => options[:delim]})
|
49
|
-
when 'json' then result.to_json
|
50
|
-
when 'plain' then result.join(options[:delim])
|
51
|
-
end
|
52
|
-
|
53
|
-
puts output
|
47
|
+
puts _format(result, options)
|
54
48
|
result = []
|
55
49
|
end
|
56
50
|
end
|
data/lib/logstash-cli/command.rb
CHANGED
@@ -1,9 +1,20 @@
|
|
1
1
|
require 'logstash-cli/command/grep'
|
2
2
|
require 'logstash-cli/command/tail'
|
3
|
+
require 'logstash-cli/command/count'
|
3
4
|
|
4
5
|
module LogstashCli
|
5
6
|
module Command
|
6
7
|
include Grep
|
7
8
|
include Tail
|
9
|
+
include Count
|
10
|
+
|
11
|
+
def _format(result,options)
|
12
|
+
output = case options[:format]
|
13
|
+
when 'csv' then result.to_csv({:col_sep => options[:delim]})
|
14
|
+
when 'json' then result.to_json
|
15
|
+
when 'plain' then result.join(options[:delim])
|
16
|
+
end
|
17
|
+
return output
|
18
|
+
end
|
8
19
|
end
|
9
20
|
end
|
data/lib/logstash-cli/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 21
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 5
|
10
|
+
version: 0.0.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Patrick Debois
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-
|
18
|
+
date: 2012-08-13 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
requirement: &id001 !ruby/object:Gem::Requirement
|
@@ -150,6 +150,7 @@ files:
|
|
150
150
|
- lib/logstash-cli.rb
|
151
151
|
- lib/logstash-cli/cli.rb
|
152
152
|
- lib/logstash-cli/command.rb
|
153
|
+
- lib/logstash-cli/command/count.rb
|
153
154
|
- lib/logstash-cli/command/grep.rb
|
154
155
|
- lib/logstash-cli/command/tail.rb
|
155
156
|
- lib/logstash-cli/version.rb
|