logstash-input-proc 0.1.1 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/ElasticSearch/templates/pidstats.json +185 -0
  3. data/README.md +8 -118
  4. data/documentation.md +6 -0
  5. data/lib/logstash/inputs/proc.rb +39 -25
  6. data/logstash-input-proc.gemspec +1 -1
  7. data/quickstart.md +45 -0
  8. metadata +5 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9b39878e4ae4d55b561d66b98e5df734d9de7482
4
- data.tar.gz: 7158cef5bf169956201c15bbd540aaeff5f1fb89
3
+ metadata.gz: 1a7521ce81c7e07305cdbef8c821907fa0ea7843
4
+ data.tar.gz: 64a7d10f0fae20db1bc98f6d7454451ea1083999
5
5
  SHA512:
6
- metadata.gz: 53acb7d9a31f969d1aae2cc30bcf9766bbae43751a044f02f41f88e03af3b712482bac18011341bba4d794ffa79407aa8802c7f986448af42c1adad78d8c8855
7
- data.tar.gz: e832d5ab85d648286d9ee9b58e66de06abf50843e3e57c8a59c901ab497b900956b1415bba3c70cbb05eebbdf8a44c7a86b9c00457002750d8bd59f583f3f685
6
+ metadata.gz: 80939c69de0a1520c46fdbe9606c6468714472879b0d30f70bdf7db816e7dae0a50343ac8088b2e54152e9454f1f52a573705d423c01cc0c387857da2d711540
7
+ data.tar.gz: 2d0651bd5ed03e5ce16b129e47cd5fc607547409e1747e5eb98d1fec6bdb6e4a90266fbcac02881d4ed4a6996a667c1c40312751d05571e527f41987c4901cf9
data/ElasticSearch/templates/pidstats.json ADDED
@@ -0,0 +1,185 @@
1
+ {
2
+ "template": "pidstats*",
3
+ "order": 10,
4
+ "aliases": {
5
+ "pidstats": [
6
+ {
7
+ "add": {
8
+ "index": "pidstats"
9
+ }
10
+ }
11
+ ]
12
+ },
13
+ "mappings": {
14
+ "pidstats": {
15
+ "properties": {
16
+ "uid": {
17
+ "type": "long"
18
+ },
19
+ "tty_nr": {
20
+ "type": "long"
21
+ },
22
+ "nlwp": {
23
+ "type": "long"
24
+ },
25
+ "utime": {
26
+ "type": "long"
27
+ },
28
+ "startcode": {
29
+ "type": "long"
30
+ },
31
+ "rt_priority": {
32
+ "type": "long"
33
+ },
34
+ "egid": {
35
+ "type": "long"
36
+ },
37
+ "stime": {
38
+ "type": "long"
39
+ },
40
+ "state": {
41
+ "type": "string"
42
+ },
43
+ "sigcatch": {
44
+ "type": "long"
45
+ },
46
+ "kstkeip": {
47
+ "type": "long"
48
+ },
49
+ "nice": {
50
+ "type": "long"
51
+ },
52
+ "signal": {
53
+ "type": "long"
54
+ },
55
+ "nswap": {
56
+ "type": "long"
57
+ },
58
+ "sigignore": {
59
+ "type": "long"
60
+ },
61
+ "vsize": {
62
+ "type": "long"
63
+ },
64
+ "pgrp": {
65
+ "type": "long"
66
+ },
67
+ "endcode": {
68
+ "type": "long"
69
+ },
70
+ "kstkesp": {
71
+ "type": "long"
72
+ },
73
+ "priority": {
74
+ "type": "long"
75
+ },
76
+ "name": {
77
+ "type": "string"
78
+ },
79
+ "rss": {
80
+ "type": "long"
81
+ },
82
+ "cmajflt": {
83
+ "type": "long"
84
+ },
85
+ "blocked": {
86
+ "type": "long"
87
+ },
88
+ "processor": {
89
+ "type": "long"
90
+ },
91
+ "rlim": {
92
+ "type": "string"
93
+ },
94
+ "tpgid": {
95
+ "type": "long"
96
+ },
97
+ "euid": {
98
+ "type": "long"
99
+ },
100
+ "comm": {
101
+ "type": "string"
102
+ },
103
+ "starttime": {
104
+ "type": "long"
105
+ },
106
+ "flags": {
107
+ "type": "long"
108
+ },
109
+ "ppid": {
110
+ "type": "long"
111
+ },
112
+ "cmdline": {
113
+ "type": "string"
114
+ },
115
+ "environ": {
116
+ "type": "object"
117
+ },
118
+ "pid": {
119
+ "type": "long"
120
+ },
121
+ "fd": {
122
+ "type": "object"
123
+ },
124
+ "startstack": {
125
+ "type": "long"
126
+ },
127
+ "minflt": {
128
+ "type": "long"
129
+ },
130
+ "io": {
131
+ "type": "object"
132
+ },
133
+ "exit_signal": {
134
+ "type": "long"
135
+ },
136
+ "cstime": {
137
+ "type": "long"
138
+ },
139
+ "itrealvalue": {
140
+ "type": "long"
141
+ },
142
+ "wchan": {
143
+ "type": "string"
144
+ },
145
+ "session": {
146
+ "type": "long"
147
+ },
148
+ "majflt": {
149
+ "type": "long"
150
+ },
151
+ "gid": {
152
+ "type": "long"
153
+ },
154
+ "policy": {
155
+ "type": "long"
156
+ },
157
+ "cutime": {
158
+ "type": "long"
159
+ },
160
+ "cnswap": {
161
+ "type": "long"
162
+ },
163
+ "cminflt": {
164
+ "type": "long"
165
+ }
166
+ }
167
+ },
168
+ "@timestamp": {
169
+ "format": "dateOptionalTime",
170
+ "type": "date"
171
+ },
172
+ "host": {
173
+ "type": "string"
174
+ },
175
+ "file": {
176
+ "type": "string"
177
+ },
178
+ "type": {
179
+ "type": "string"
180
+ },
181
+ "@version": {
182
+ "type": "string"
183
+ }
184
+ }
185
+ }
data/README.md CHANGED
@@ -1,130 +1,20 @@
1
1
  ![](https://github.com/eperry/logstash-input-proc/wiki/MemInfoDashboard.png)
2
2
 
3
- # Logstash Plugin
3
+ # Logstash Input Proc Parser
4
+ ##[Home page](http://eperry.github.io/logstash-input-proc/)
4
5
 
5
- This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
6
+ This is a plugin for with [Logstash](https://github.com/elasticsearch/logstash).
6
7
 
7
8
  This plugin is to read the /proc virtual file system , decode the files in it.
8
9
  I am using the following pages for reference
9
10
 
10
- - http://man7.org/linux/man-pages/man5/proc.5.html
11
-
12
-
13
-
14
-
15
11
  ## Documentation
12
+ [Documentation](https://github.com/eperry/logstash-input-proc/wiki/documentation)
16
13
 
14
+ [Quickstart](https://github.com/eperry/logstash-input-proc/wiki/quickstart)
17
15
 
18
- ### 1. Plugin Developement and Testing
19
-
20
- #### Code
21
- - To get started, you'll need JRuby with the Bundler gem installed.
22
- ```sh
23
- bundle install
24
- ```
25
-
26
- - Then clone this repo
27
- - You will need to either clone the logstash repo or download the binary
28
-
29
-
30
-
31
- ### 2. Running the unpublished Plugin in Logstash
32
-
33
- #### 2.1 Run in a local Logstash clone
34
-
35
- - Edit Logstash `Gemfile` and add the local plugin path, for example:
36
- ```ruby
37
- gem "logstash-input-proc", :path => "/your/local/logstash-input-proc"
38
- ```
39
- - Install plugin
40
- ```sh
41
- bin/plugin install --no-verify
42
- ```
43
- - install Ruby Debug
44
- ```sh
45
- bin/plugin install logstash-codec-rubydebug
46
- ```
47
- - Run Logstash with your plugin
48
- ```sh
49
- bin/logstash -e 'input {proc {interval=>60}} output { stdout{ codec=>"rubydebug"}}'
50
- ```
51
- At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
52
-
53
- #### 2.2 Run in an installed Logstash
54
-
55
- You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
56
-
57
- - Build your plugin gem
58
- ```sh
59
- gem build logstash-output-proc.gemspec
60
- ```
61
- - Install the plugin from the Logstash home
62
- ```sh
63
- bin/plugin install /your/local/plugin/logstash-input-proc.gem
64
- ```
65
- - Start Logstash and proceed to test the plugin
66
- -
67
- # Example Config all features enabled
68
- ```ruby
69
- input {
70
- proc {
71
- interval=>60
72
- vmstats =>{ }
73
- loadavg =>{ }
74
- meminfo =>{ }
75
- pidstats =>{
76
- user => "root"
77
- }
78
-
79
- }
80
- }
81
-
82
- output {
83
- stdout{
84
- codec=>"rubydebug"
85
- }
86
- }
87
- ```
88
- #Example Minimal
89
-
90
- ```ruby
91
- input {
92
- proc {
93
- interval=>60
94
- meminfo =>{ }
95
- }
96
- }
97
-
98
- output {
99
- stdout{
100
- codec=>"rubydebug"
101
- }
102
- }
103
- ```
104
-
105
- ## 3.0 Kibana Dashboards
106
-
107
- Still a work in progress but I have saved a copy of the Kibana 4.1 dashboards I have created
108
- in the ~/Kibana Directory, you should be able to import them from the Kibana->settings->Objects pages
109
-
110
- These dashboards are right now a way of me validating the data loaded in elasticsearch is usable and provide an example for others to work off of. They work with the setup
111
- of elasticsearch as defined below.
112
-
113
-
114
- ## 4.0 Elasticsearch Templates
115
-
116
- In the ~/ElasitcSearch Directory are all the Elasticsearch templates I am developing to work with this plugin.
117
- While they may not be exactly what you need they are a good start.
118
-
119
- I load them via the ${ES_HOME/config/templates directory but feel free to load them in your preffered way
16
+ [Development](https://github.com/eperry/logstash-input-proc/wiki/development)
120
17
 
121
- These templates are based on the fact that your indexes for the data are created like so:
122
- ```
123
- output {
124
- elasticsearch {
125
- host => localhost
126
- index => "%{type}-%{+YYYY.MM.dd}"
127
- }
128
- }
129
- ```
18
+ [kibana](https://github.com/eperry/logstash-input-proc/wiki/kibana)
130
19
 
20
+ [ElasticSearch](https://github.com/eperry/logstash-input-proc/wiki/es)
data/documentation.md ADDED
@@ -0,0 +1,6 @@
1
+ This plugin is to read the /proc virtual file system , decode the files in it.
2
+ I am using the following pages for reference
3
+
4
+ - http://man7.org/linux/man-pages/man5/proc.5.html
5
+
6
+
data/lib/logstash/inputs/proc.rb CHANGED
@@ -13,7 +13,7 @@ class LogStash::Inputs::Proc < LogStash::Inputs::Base
13
13
  config_name "proc"
14
14
 
15
15
  # If undefined, Logstash will complain, even if codec is unused.
16
- default :codec, "plain"
16
+ default :codec, "json"
17
17
 
18
18
  # The message string to use in the event.
19
19
  #config :message, :validate => :string, :default => "Hello World!"
@@ -106,7 +106,6 @@ def readPidStats(queue)
106
106
  fuid = Etc.getpwnam(@pidstats["user"]).uid
107
107
  @logger.info? && @logger.info("Filtering userid =" + @pidstats["user"] )
108
108
  end
109
- process = Hash.new
110
109
  #Loosely based on the GEM ProcTable which was based on the Perl Module ProcTable
111
110
  Dir.foreach("/proc"){ |file|
112
111
  next if file =~ /\D/ # Skip non-numeric directories
@@ -114,24 +113,24 @@ def readPidStats(queue)
114
113
  fileUid = File.stat("/proc/"+file).uid
115
114
  next if fileUid != fuid
116
115
  end
117
-
116
+ process = Hash.new
118
117
  # Get /proc/<pid>/cmdline information. Strip out embedded nulls.
119
118
  begin
120
119
  data = IO.read("/proc/#{file}/cmdline").tr("\000", ' ').strip
121
120
  process["cmdline"] = data
122
121
  rescue
123
- next # Process terminated, on to the next process
122
+ # Ignore and move on.
124
123
  end
125
-
126
124
  # Get /proc/<pid>/cwd information
127
- process["cwd"] = File.readlink("/proc/#{file}/cwd") rescue nil
125
+ process["cwd"] = File.readlink("/proc/#{file}/cwd") rescue
128
126
 
129
127
  # Get /proc/<pid>/environ information. Environment information
130
128
  # is represented as a Hash, with the environment variable as the
131
129
  # key and its value as the hash value.
132
- process["environ"] = Hash.new
130
+
133
131
 
134
132
  begin
133
+ process["environ"] = Hash.new
135
134
  IO.read("/proc/#{file}/environ").split("\0").each{ |str|
136
135
  key, value = str.split('=')
137
136
  process["environ"][key] = value
@@ -141,7 +140,7 @@ def readPidStats(queue)
141
140
  end
142
141
 
143
142
  # Get /proc/<pid>/exe information
144
- process["exe"] = File.readlink("/proc/#{file}/exe") rescue nil
143
+ process["exe"] = File.readlink("/proc/#{file}/exe") rescue
145
144
 
146
145
  # Get /proc/<pid>/fd information. File descriptor information
147
146
  # is represented as a Hash, with the fd as the key, and its
@@ -150,23 +149,41 @@ def readPidStats(queue)
150
149
 
151
150
  begin
152
151
  Dir.foreach("/proc/#{file}/fd/") { |fd|
153
- process["fd"][fd] = File.readlink("/proc/#{file}/fd/"+fd) rescue nil
152
+ process["fd"][fd] = File.readlink("/proc/#{file}/fd/"+fd) rescue process["fd"] = []
154
153
  }
155
- rescue
156
- process["fd"] = ""
154
+ rescue
157
155
  # # Ignore and move on
158
156
  end
159
157
 
160
158
  # Get /proc/<pid>/root information
161
- process["root"] = File.readlink("/proc/#{file}/root") rescue nil
162
-
163
- # Get /proc/<pid>/stat information
164
- stat = IO.read("/proc/#{file}/stat") rescue next
159
+ process["root"] = File.readlink("/proc/#{file}/root") rescue
165
160
 
166
161
  # Get number of LWP, one directory for each in /proc/<pid>/task/
167
162
  # Every process has at least one thread, so if we fail to read the task directory, set nlwp to 1.
168
- process["nlwp"] = Dir.glob("/proc/#{file}/task/*").length rescue process["nlwp"] = 1
163
+ process["nlwp"] = Dir.glob("/proc/#{file}/task/*").length rescue process["nlwp"] = 1
164
+
165
+ # cat /proc/3828/io
166
+ # rchar: 323934931
167
+ # wchar: 323929600
168
+ # syscr: 632687
169
+ # syscw: 632675
170
+ # read_bytes: 0
171
+ # write_bytes: 323932160
172
+ # cancelled_write_bytes: 0
173
+ begin
174
+ process["io"] = Hash.new
169
175
 
176
+ IO.foreach("/proc/#{file}/io") do |line|
177
+ key, value = line.split(/[:\s]+/)
178
+ process["io"][key] = value.to_i
179
+ end
180
+ rescue
181
+ # Ignore and move on.
182
+ end
183
+
184
+ # Get /proc/<pid>/stat information
185
+ begin
186
+ stat = IO.read("/proc/#{file}/stat")
170
187
  # Deal with spaces in comm name. Courtesy of Ara Howard.
171
188
  re = %r/\([^\)]+\)/
172
189
  comm = stat[re]
@@ -174,7 +191,6 @@ def readPidStats(queue)
174
191
  stat[re] = comm
175
192
 
176
193
  stat = stat.split
177
-
178
194
  process["pid"] = stat[0].to_i
179
195
  process["comm"] = stat[1].tr('()','') # Remove parens
180
196
  process["state"] = stat[2]
@@ -199,7 +215,7 @@ def readPidStats(queue)
199
215
  process["starttime"] = stat[21].to_i
200
216
  process["vsize"] = stat[22].to_i
201
217
  process["rss"] = stat[23].to_i
202
- process["rlim"] = stat[24].to_i
218
+ process["rlim"] = stat[24].to_s
203
219
  process["startcode"] = stat[25].to_i
204
220
  process["endcode"] = stat[26].to_i
205
221
  process["startstack"] = stat[27].to_i
@@ -209,13 +225,16 @@ def readPidStats(queue)
209
225
  process["blocked"] = stat[31].to_i
210
226
  process["sigignore"] = stat[32].to_i
211
227
  process["sigcatch"] = stat[33].to_i
212
- process["wchan"] = stat[34].to_i
228
+ process["wchan"] = stat[34].to_s
213
229
  process["nswap"] = stat[35].to_i
214
230
  process["cnswap"] = stat[36].to_i
215
231
  process["exit_signal"] = stat[37].to_i
216
232
  process["processor"] = stat[38].to_i
217
233
  process["rt_priority"] = stat[39].to_i
218
234
  process["policy"] = stat[40].to_i
235
+ rescue
236
+
237
+ end
219
238
  # Get /proc/<pid>/status information (name, uid, euid, gid, egid)
220
239
  begin
221
240
  IO.foreach("/proc/#{file}/status") do |line|
@@ -231,17 +250,13 @@ def readPidStats(queue)
231
250
  end
232
251
  end
233
252
  rescue Errno::ESRCH, Errno::ENOENT
234
- next
235
253
  end
236
254
 
237
255
  # If cmdline is empty use comm instead
238
256
  process["cmdline"] = process["comm"] if process["cmdline.empty?"]
239
-
240
-
241
257
  event = LogStash::Event.new( "file" => "/proc" ,"host" => @host, "type" => "pidstats" , "process" => process);
242
258
  decorate(event)
243
259
  queue << event
244
-
245
260
  }
246
261
 
247
262
 
@@ -462,7 +477,6 @@ def readWireless(queue)
462
477
  lines.each { |line|
463
478
  #@logger.info? && @logger.info("LINE: "+line)
464
479
  m = line.strip.split(/[:\s]+/)
465
- #puts(m)
466
480
  if (m && m.length >= 11 )
467
481
  event = LogStash::Event.new(
468
482
  "raw" => line,
@@ -553,7 +567,7 @@ end
553
567
  sleep(sleeptime)
554
568
  end
555
569
  rescue => exception
556
- #puts exception.message
570
+ puts exception.message
557
571
  puts exception.backtrace
558
572
  raise
559
573
  end # rescue
data/logstash-input-proc.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-proc'
3
- s.version = '0.1.1'
3
+ s.version = '0.1.2'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "This plugin is for reading the /proc of a linux filesystem"
6
6
  s.description = "Plugin is used with logstash"
data/quickstart.md ADDED
@@ -0,0 +1,45 @@
1
+ # Logstash Plugin
2
+
3
+ ## To install for use
4
+ ```
5
+ ${LS_HOME}/bin/plugin install logstash-input-proc
6
+ ```
7
+
8
+ ## Example Config all features enabled
9
+ ```ruby
10
+ input {
11
+ proc {
12
+ interval=>60
13
+ vmstats =>{ }
14
+ loadavg =>{ }
15
+ meminfo =>{ }
16
+ pidstats =>{
17
+ user => "root"
18
+ }
19
+
20
+ }
21
+ }
22
+
23
+ output {
24
+ stdout{
25
+ codec=>"rubydebug"
26
+ }
27
+ }
28
+ ```
29
+ ##Example Minimal
30
+
31
+ ```ruby
32
+ input {
33
+ proc {
34
+ interval=>60
35
+ meminfo =>{ }
36
+ }
37
+ }
38
+
39
+ output {
40
+ stdout{
41
+ codec=>"rubydebug"
42
+ }
43
+ }
44
+ ```
45
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-proc
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Edward Perry
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-20 00:00:00.000000000 Z
11
+ date: 2015-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
@@ -82,15 +82,18 @@ files:
82
82
  - ElasticSearch/README.md
83
83
  - ElasticSearch/templates/loadavg.json
84
84
  - ElasticSearch/templates/meminfo.json
85
+ - ElasticSearch/templates/pidstats.json
85
86
  - ElasticSearch/templates/vmstats.json
86
87
  - Gemfile
87
88
  - Kibana/READEME.md
88
89
  - Kibana/export.json
89
90
  - README.md
90
91
  - Rakefile
92
+ - documentation.md
91
93
  - lib/logstash/inputs/proc.rb
92
94
  - logstash-input-proc.gemspec
93
95
  - logstash.conf
96
+ - quickstart.md
94
97
  - spec/inputs/proc_spec.rb
95
98
  homepage: http://eperry.github.io/logstash-input-proc/
96
99
  licenses: