logstash-filter-dissect 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/BUILD_INSTRUCTIONS.md +23 -0
- data/CHANGELOG.md +4 -1
- data/Gemfile +0 -3
- data/README.md +3 -0
- data/VERSION +1 -0
- data/lib/jruby-dissect-library_jars.rb +4 -0
- data/lib/logstash/filters/dissect.rb +118 -79
- data/logstash-filter-dissect.gemspec +10 -5
- data/spec/filters/dissect_spec.rb +39 -69
- data/spec/spec_helper.rb +1 -0
- data/vendor/jars/org/jruby/jruby-core/1.7.25/jruby-core-1.7.25.jar +0 -0
- data/vendor/jars/org/logstash/dissect/jruby-dissect-library/1.0.5/jruby-dissect-library-1.0.5.jar +0 -0
- data/vendor/jars/org/logstash/dissect/jruby-dissect-library/1.0.7/jruby-dissect-library-1.0.7.jar +0 -0
- metadata +44 -11
- data/lib/jars/jruby-dissect-library.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4066828d764fb5671602ea92941a48c3d1c87adc
|
4
|
+
data.tar.gz: e9dd175b2a351effd59fc4c654d5c9be675a1aa7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bf533ed8fd5dc76a2ca2f1dcbb4df512935bc5c0d8fa8976c1d5422762c7932d574801bd54c8ace3f7fb142a10bd2ecd1d9b29eadef31e66232a6463b2ce843e
|
7
|
+
data.tar.gz: 4175f2160f80bbfb5fa18b20f50e9a89283d611a7f3a3188737244dfe89d01fb1abb6aacbac554c8fc894df214089e0ed2e9bc52a92ab9a5b5dcdcb3f9e9924e
|
@@ -0,0 +1,23 @@
|
|
1
|
+
These instructions are for the JAVA build NOT Ruby Gem build and publish.
|
2
|
+
|
3
|
+
#### History
|
4
|
+
This is a Logstash plugin with a large amount of Java code.
|
5
|
+
|
6
|
+
The Java source is dependent on the logstash-core jar and the logstash-core-event jar.
|
7
|
+
These jars can be found in the gems that are resolved via the `s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"` line in the gemspec.
|
8
|
+
So different versions of these gems can be installed as time goes by. The gradle build dependencies cannot refer to a static location and version.
|
9
|
+
|
10
|
+
These Rake tasks allow for Gradle to be dependent on Rubygems.
|
11
|
+
- `rake vendor` - this task calls "./gradlew vendor" but only after have done the "bundle_install" rake task
|
12
|
+
- `rake bundle_install` - this task calls `bundle install` then puts the output of `bundle show logstash-core` and `bundle show logstash-core-event` into the "gradle.properties" file.
|
13
|
+
|
14
|
+
#### Procedure
|
15
|
+
Each time a new gem version of `logstash-core-plugin-api` is released we need to run `rake vendor` to verify that any changes to the Java in logstash-core and logstash-core-event are still compatible with their usage in this plugin.
|
16
|
+
|
17
|
+
Run `rake vendor`
|
18
|
+
|
19
|
+
#### Travis
|
20
|
+
Travis is set to use `rake write_gradle_properties` before its build task
|
21
|
+
|
22
|
+
#### More Information
|
23
|
+
See the Gradle "vendor" task to understand how the the jar is generated.
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.0.7
|
@@ -3,109 +3,134 @@ require "logstash/filters/base"
|
|
3
3
|
require "logstash/namespace"
|
4
4
|
|
5
5
|
require "java"
|
6
|
-
require "
|
6
|
+
require "jruby-dissect-library_jars"
|
7
7
|
require "jruby_dissector"
|
8
8
|
|
9
|
-
#
|
10
|
-
#
|
11
|
-
# The
|
12
|
-
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
23
|
-
|
24
|
-
#
|
25
|
-
|
26
|
-
#
|
27
|
-
# [
|
9
|
+
# ==== *Dissect or how to de-structure text*
|
10
|
+
#
|
11
|
+
# The Dissect filter is a kind of split operation. Unlike a regular split operation where one delimiter is applied to the whole string, this operation applies a set of delimiters # to a string value. +
|
12
|
+
# Dissect does not use regular expressions and is very fast. +
|
13
|
+
# However, if the structure of your text varies from line to line then Grok is more suitable. +
|
14
|
+
# There is a hybrid case where Dissect can be used to de-structure the section of the line that is reliably repeated and then Grok can be used on the remaining field values with # more regex predictability and less overall work to do. +
|
15
|
+
#
|
16
|
+
# A set of fields and delimiters is called a *dissection*.
|
17
|
+
#
|
18
|
+
# The dissection is described using a set of `%{}` sections:
|
19
|
+
# ....
|
20
|
+
# %{a} - %{b} - %{c}
|
21
|
+
# ....
|
22
|
+
#
|
23
|
+
# A *field* is the text from `%` to `}` inclusive.
|
24
|
+
#
|
25
|
+
# A *delimiter* is the text between `}` and `%` characters.
|
26
|
+
#
|
27
|
+
# [NOTE]
|
28
|
+
# delimiters can't contain these `}{%` characters.
|
29
|
+
#
|
30
|
+
# The config might look like this:
|
31
|
+
# ....
|
28
32
|
# filter {
|
29
33
|
# dissect {
|
30
34
|
# mapping => {
|
31
|
-
# "message" => "%{
|
35
|
+
# "message" => "%{ts} %{+ts} %{+ts} %{src} %{} %{prog}[%{pid}]: %{msg}"
|
32
36
|
# }
|
33
37
|
# }
|
34
38
|
# }
|
35
|
-
|
36
|
-
# When dissecting a string
|
37
|
-
#
|
38
|
-
|
39
|
-
# The
|
40
|
-
#
|
41
|
-
# `%{
|
42
|
-
# `%{
|
43
|
-
#
|
44
|
-
|
45
|
-
#
|
46
|
-
#
|
47
|
-
#
|
48
|
-
|
49
|
-
#
|
50
|
-
# The found value is recorded internally but not added to the Event.
|
39
|
+
# ....
|
40
|
+
# When dissecting a string from left to right, text is captured upto the first delimiter - this captured text is stored in the first field. This is repeated for each field/# delimiter pair thereafter until the last delimiter is reached, then *the remaining text is stored in the last field*. +
|
41
|
+
#
|
42
|
+
# *The Key:* +
|
43
|
+
# The key is the text between the `%{` and `}`, exclusive of the ?, +, & prefixes and the ordinal suffix. +
|
44
|
+
# `%{?aaa}` - key is `aaa` +
|
45
|
+
# `%{+bbb/3}` - key is `bbb` +
|
46
|
+
# `%{&ccc}` - key is `ccc` +
|
47
|
+
#
|
48
|
+
# *Normal field notation:* +
|
49
|
+
# The found value is added to the Event using the key. +
|
50
|
+
# `%{some_field}` - a normal field has no prefix or suffix
|
51
|
+
#
|
52
|
+
# *Skip field notation:* +
|
53
|
+
# The found value is stored internally but not added to the Event. +
|
51
54
|
# The key, if supplied, is prefixed with a `?`.
|
52
|
-
#
|
53
|
-
# `%{
|
54
|
-
|
55
|
-
#
|
56
|
-
#
|
57
|
-
#
|
58
|
-
# The
|
59
|
-
# The
|
60
|
-
#
|
61
|
-
#
|
62
|
-
#
|
63
|
-
#
|
64
|
-
#
|
65
|
-
#
|
66
|
-
|
67
|
-
#
|
68
|
-
#
|
69
|
-
#
|
70
|
-
#
|
71
|
-
#
|
72
|
-
#
|
55
|
+
#
|
56
|
+
# `%{}` is an empty skip field.
|
57
|
+
#
|
58
|
+
# `%{?foo}` is a named skip field.
|
59
|
+
#
|
60
|
+
# *Append field notation:* +
|
61
|
+
# The value is appended to another value or stored if its the first field seen. +
|
62
|
+
# The key is prefixed with a `+`. +
|
63
|
+
# The final value is stored in the Event using the key. +
|
64
|
+
#
|
65
|
+
# [NOTE]
|
66
|
+
# ====
|
67
|
+
# The delimiter found before the field is appended with the value. +
|
68
|
+
# If no delimiter is found before the field, a single space character is used.
|
69
|
+
# ====
|
70
|
+
#
|
71
|
+
# `%{+some_field}` is an append field. +
|
72
|
+
# `%{+some_field/2}` is an append field with an order modifier.
|
73
|
+
#
|
74
|
+
# An order modifier, `/digits`, allows one to reorder the append sequence. +
|
75
|
+
# e.g. for a text of `1 2 3 go`, this `%{+a/2} %{+a/1} %{+a/4} %{+a/3}` will build a key/value of `a => 2 1 go 3` +
|
76
|
+
# Append fields without an order modifier will append in declared order. +
|
77
|
+
# e.g. for a text of `1 2 3 go`, this `%{a} %{b} %{+a}` will build two key/values of `a => 1 3 go, b => 2` +
|
78
|
+
#
|
79
|
+
# *Indirect field notation:* +
|
80
|
+
# The found value is added to the Event using the found value of another field as the key. +
|
81
|
+
# The key is prefixed with a `&`. +
|
82
|
+
# `%{&some_field}` - an indirect field where the key is indirectly sourced from the value of `some_field`. +
|
83
|
+
# e.g. for a text of `error: some_error, some_description`, this `error: %{?err}, %{&err}` will build a key/value of `some_error => description`.
|
84
|
+
#
|
85
|
+
# [NOTE]
|
86
|
+
# for append and indirect field the key can refer to a field that already exists in the event before dissection.
|
87
|
+
#
|
88
|
+
# [NOTE]
|
89
|
+
# use a Skip field if you do not want the indirection key/value stored.
|
90
|
+
#
|
73
91
|
# e.g. for a text of `google: 77.98`, this `%{?a}: %{&a}` will build a key/value of `google => 77.98`.
|
74
|
-
|
75
|
-
#
|
76
|
-
#
|
77
|
-
#
|
78
|
-
# `%{
|
79
|
-
|
80
|
-
#
|
81
|
-
#
|
92
|
+
#
|
93
|
+
# [NOTE]
|
94
|
+
# ===============================
|
95
|
+
# append and indirect cannot be combined and will fail validation. +
|
96
|
+
# `%{+&something}` - will add a value to the `&something` key, probably not the intended outcome. +
|
97
|
+
# `%{&+something}` will add a value to the `+something` key, again probably unintended. +
|
98
|
+
# ===============================
|
99
|
+
#
|
100
|
+
# *Delimiter repetition:* +
|
101
|
+
# In the source text if a field has variable width padded with delimiters, the padding will be ignored. +
|
82
102
|
# e.g. for texts of:
|
83
|
-
#
|
103
|
+
# ....
|
84
104
|
# 00000043 ViewReceiver I
|
85
105
|
# 000000b3 Peer I
|
86
|
-
#
|
87
|
-
#
|
106
|
+
# ....
|
107
|
+
# with a dissection of `%{a} %{b} %{c}`; the padding is ignored, `event.get([c]) -> "I"`
|
88
108
|
#
|
89
|
-
#
|
90
|
-
#
|
91
|
-
#
|
109
|
+
# [NOTE]
|
110
|
+
# ====
|
111
|
+
# You probably want to use this filter inside an `if` block. +
|
112
|
+
# This ensures that the event contains a field value with a suitable structure for the dissection.
|
113
|
+
# ====
|
114
|
+
#
|
115
|
+
# For example...
|
116
|
+
# ....
|
92
117
|
# filter {
|
93
118
|
# if [type] == "syslog" or "syslog" in [tags] {
|
94
119
|
# dissect {
|
95
120
|
# mapping => {
|
96
|
-
# "message" => "%{
|
121
|
+
# "message" => "%{ts} %{+ts} %{+ts} %{src} %{} %{prog}[%{pid}]: %{msg}"
|
97
122
|
# }
|
98
123
|
# }
|
99
124
|
# }
|
100
125
|
# }
|
126
|
+
# ....
|
101
127
|
|
102
128
|
module LogStash module Filters class Dissect < LogStash::Filters::Base
|
103
129
|
|
104
130
|
config_name "dissect"
|
105
131
|
|
106
|
-
# A hash of dissections of field => value
|
107
|
-
# A later dissection can be done on
|
108
|
-
# or they can be independent.
|
132
|
+
# A hash of dissections of `field => value` +
|
133
|
+
# A later dissection can be done on values from a previous dissection or they can be independent.
|
109
134
|
#
|
110
135
|
# For example
|
111
136
|
# [source, ruby]
|
@@ -118,11 +143,25 @@ module LogStash module Filters class Dissect < LogStash::Filters::Base
|
|
118
143
|
# }
|
119
144
|
# }
|
120
145
|
#
|
121
|
-
# This is useful if you want to keep the field `description` also
|
146
|
+
# This is useful if you want to keep the field `description` but also
|
122
147
|
# dissect it some more.
|
148
|
+
|
123
149
|
config :mapping, :validate => :hash, :default => {}
|
124
150
|
|
125
|
-
#
|
151
|
+
# With this setting `int` and `float` datatype conversions can be specified. +
|
152
|
+
# These will be done after all `mapping` dissections have taken place. +
|
153
|
+
# Feel free to use this setting on its own without a `mapping` section. +
|
154
|
+
#
|
155
|
+
# For example
|
156
|
+
# [source, ruby]
|
157
|
+
# filter {
|
158
|
+
# dissect {
|
159
|
+
# convert_datatype => {
|
160
|
+
# cpu => "float"
|
161
|
+
# code => "int"
|
162
|
+
# }
|
163
|
+
# }
|
164
|
+
# }
|
126
165
|
config :convert_datatype, :validate => :hash, :default => {}
|
127
166
|
|
128
167
|
# Append values to the `tags` field when dissection fails
|
@@ -131,7 +170,7 @@ module LogStash module Filters class Dissect < LogStash::Filters::Base
|
|
131
170
|
public
|
132
171
|
|
133
172
|
def register
|
134
|
-
|
173
|
+
@dissector = LogStash::Dissector.new(@mapping)
|
135
174
|
end
|
136
175
|
|
137
176
|
def filter(event)
|
@@ -1,8 +1,10 @@
|
|
1
|
+
DISSECT_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(DISSECT_VERSION)
|
2
|
+
|
1
3
|
Gem::Specification.new do |s|
|
2
4
|
s.name = 'logstash-filter-dissect'
|
3
|
-
s.version
|
5
|
+
s.version = DISSECT_VERSION
|
4
6
|
s.licenses = ['Apache License (2.0)']
|
5
|
-
s.summary = "This dissect filter will
|
7
|
+
s.summary = "This dissect filter will de-structure text into multiple fields."
|
6
8
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
9
|
s.authors = ["Elastic"]
|
8
10
|
s.email = 'info@elastic.co'
|
@@ -10,7 +12,7 @@ Gem::Specification.new do |s|
|
|
10
12
|
s.require_paths = ["lib"]
|
11
13
|
|
12
14
|
# Files
|
13
|
-
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
15
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','VERSION','LICENSE','NOTICE.TXT']
|
14
16
|
# Tests
|
15
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
18
|
|
@@ -18,6 +20,9 @@ Gem::Specification.new do |s|
|
|
18
20
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
19
21
|
|
20
22
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency
|
22
|
-
s.
|
23
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 2.1.1', '<= 2.99'
|
24
|
+
s.add_runtime_dependency 'jar-dependencies'
|
25
|
+
|
26
|
+
s.add_development_dependency 'rspec'
|
27
|
+
s.add_development_dependency 'logstash-devutils'
|
23
28
|
end
|
@@ -26,6 +26,16 @@ describe LogStash::Filters::Dissect do
|
|
26
26
|
@msgs.push(msg[0])
|
27
27
|
@hashes.push(msg[1])
|
28
28
|
end
|
29
|
+
|
30
|
+
def fatal(*msg)
|
31
|
+
@msgs.push(msg[0])
|
32
|
+
@hashes.push(msg[1])
|
33
|
+
end
|
34
|
+
|
35
|
+
def trace(*msg)
|
36
|
+
@msgs.push(msg[0])
|
37
|
+
@hashes.push(msg[1])
|
38
|
+
end
|
29
39
|
end
|
30
40
|
|
31
41
|
describe "Basic dissection" do
|
@@ -74,6 +84,31 @@ describe LogStash::Filters::Dissect do
|
|
74
84
|
end
|
75
85
|
end
|
76
86
|
|
87
|
+
describe "Basic dissection with multibyte Unicode characters" do
|
88
|
+
let(:config) do <<-CONFIG
|
89
|
+
filter {
|
90
|
+
dissect {
|
91
|
+
mapping => {
|
92
|
+
message => "[%{occurred_at}]྿྿྿%{code}྿%{service}྿྿྿྿%{?ic}=%{&ic}%྿྿%{svc_message}"
|
93
|
+
}
|
94
|
+
convert_datatype => {
|
95
|
+
cpu => "float"
|
96
|
+
code => "int"
|
97
|
+
}
|
98
|
+
}
|
99
|
+
}
|
100
|
+
CONFIG
|
101
|
+
end
|
102
|
+
|
103
|
+
sample("message" => "[25/05/16 09:10:38:425 BST]྿྿྿00000001྿SystemOut྿྿྿྿cpu=95.43%྿྿java.lang:type=MemoryPool,name=class storage") do
|
104
|
+
expect(subject.get("occurred_at")).to eq("25/05/16 09:10:38:425 BST")
|
105
|
+
expect(subject.get("code")).to eq(1)
|
106
|
+
expect(subject.get("service")).to eq("SystemOut")
|
107
|
+
expect(subject.get("cpu")).to eq(95.43)
|
108
|
+
expect(subject.get("svc_message")).to eq("java.lang:type=MemoryPool,name=class storage")
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
77
112
|
describe "Basic dissection with failing datatype conversion" do
|
78
113
|
subject(:filter) { LogStash::Filters::Dissect.new(config) }
|
79
114
|
|
@@ -92,7 +127,7 @@ describe LogStash::Filters::Dissect do
|
|
92
127
|
let(:loggr) { LoggerMock.new }
|
93
128
|
|
94
129
|
before(:each) do
|
95
|
-
filter.logger
|
130
|
+
filter.class.instance_variable_set("@logger", loggr)
|
96
131
|
end
|
97
132
|
|
98
133
|
it "tags and log messages are created" do
|
@@ -145,7 +180,7 @@ describe LogStash::Filters::Dissect do
|
|
145
180
|
let(:loggr) { LoggerMock.new }
|
146
181
|
|
147
182
|
before(:each) do
|
148
|
-
filter.logger
|
183
|
+
filter.class.instance_variable_set("@logger", loggr)
|
149
184
|
end
|
150
185
|
|
151
186
|
it "does not raise any exceptions" do
|
@@ -162,11 +197,6 @@ describe LogStash::Filters::Dissect do
|
|
162
197
|
describe "valid field format handling" do
|
163
198
|
subject(:filter) { LogStash::Filters::Dissect.new(config) }
|
164
199
|
let(:config) { {"mapping" => {"message" => "%{+timestamp/2} %{+timestamp/1} %{?no_name} %{&no_name} %{} %{program}[%{pid}]: %{msg}"}}}
|
165
|
-
let(:loggr) { LoggerMock.new }
|
166
|
-
|
167
|
-
before(:each) do
|
168
|
-
filter.logger = loggr
|
169
|
-
end
|
170
200
|
|
171
201
|
it "does not raise an error in register" do
|
172
202
|
expect{filter.register}.not_to raise_exception
|
@@ -175,16 +205,11 @@ describe LogStash::Filters::Dissect do
|
|
175
205
|
|
176
206
|
describe "invalid field format handling" do
|
177
207
|
subject(:filter) { LogStash::Filters::Dissect.new(config) }
|
178
|
-
let(:loggr) { LoggerMock.new }
|
179
|
-
|
180
|
-
before(:each) do
|
181
|
-
filter.logger = loggr
|
182
|
-
end
|
183
208
|
|
184
209
|
context "when field is defined as Append and Indirect (+&)" do
|
185
210
|
let(:config) { {"mapping" => {"message" => "%{+×tamp}"}}}
|
186
211
|
it "raises an error in register" do
|
187
|
-
msg = "org.logstash.dissect.InvalidFieldException: Field cannot prefix with both Append and Indirect Prefix (+&): +×tamp"
|
212
|
+
msg = "org.logstash.dissect.fields.InvalidFieldException: Field cannot prefix with both Append and Indirect Prefix (+&): +×tamp"
|
188
213
|
expect{filter.register}.to raise_exception(LogStash::FieldFormatError, msg)
|
189
214
|
end
|
190
215
|
end
|
@@ -192,64 +217,9 @@ describe LogStash::Filters::Dissect do
|
|
192
217
|
context "when field is defined as Indirect and Append (&+)" do
|
193
218
|
let(:config) { {"mapping" => {"message" => "%{&+timestamp}"}}}
|
194
219
|
it "raises an error in register" do
|
195
|
-
msg = "org.logstash.dissect.InvalidFieldException: Field cannot prefix with both Append and Indirect Prefix (&+): &+timestamp"
|
220
|
+
msg = "org.logstash.dissect.fields.InvalidFieldException: Field cannot prefix with both Append and Indirect Prefix (&+): &+timestamp"
|
196
221
|
expect{filter.register}.to raise_exception(LogStash::FieldFormatError, msg)
|
197
222
|
end
|
198
223
|
end
|
199
224
|
end
|
200
|
-
|
201
|
-
describe "baseline performance test", :performance => true do
|
202
|
-
event_count = 1000000
|
203
|
-
min_rate = 30000
|
204
|
-
|
205
|
-
max_duration = event_count / min_rate
|
206
|
-
cfg_base = <<-CONFIG
|
207
|
-
input {
|
208
|
-
generator {
|
209
|
-
count => #{event_count}
|
210
|
-
message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
|
211
|
-
}
|
212
|
-
}
|
213
|
-
output { null { } }
|
214
|
-
CONFIG
|
215
|
-
|
216
|
-
config(cfg_base)
|
217
|
-
start = Time.now.to_f
|
218
|
-
agent do
|
219
|
-
duration = (Time.now.to_f - start)
|
220
|
-
puts "\n\ninputs/generator baseline rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s\n\n"
|
221
|
-
insist { duration } < max_duration
|
222
|
-
end
|
223
|
-
end
|
224
|
-
|
225
|
-
describe "dissect performance test", :performance => true do
|
226
|
-
event_count = 1000000
|
227
|
-
min_rate = 30000
|
228
|
-
max_duration = event_count / min_rate
|
229
|
-
|
230
|
-
cfg_filter = <<-CONFIG
|
231
|
-
input {
|
232
|
-
generator {
|
233
|
-
count => #{event_count}
|
234
|
-
message => "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
|
235
|
-
}
|
236
|
-
}
|
237
|
-
filter {
|
238
|
-
dissect {
|
239
|
-
mapping => {
|
240
|
-
"message" => "%{timestamp} %{+timestamp} %{+timestamp} %{logsource} %{program}[%{pid}]: %{msg}"
|
241
|
-
}
|
242
|
-
}
|
243
|
-
}
|
244
|
-
output { null { } }
|
245
|
-
CONFIG
|
246
|
-
|
247
|
-
config(cfg_filter)
|
248
|
-
start = Time.now.to_f
|
249
|
-
agent do
|
250
|
-
duration = (Time.now.to_f - start)
|
251
|
-
puts "\n\nfilters/dissect rate: #{"%02.0f/sec" % (event_count / duration)}, elapsed: #{duration}s\n\n"
|
252
|
-
insist { duration } < event_count / min_rate
|
253
|
-
end
|
254
|
-
end
|
255
225
|
end
|
data/spec/spec_helper.rb
CHANGED
Binary file
|
data/vendor/jars/org/logstash/dissect/jruby-dissect-library/1.0.5/jruby-dissect-library-1.0.5.jar
ADDED
Binary file
|
data/vendor/jars/org/logstash/dissect/jruby-dissect-library/1.0.7/jruby-dissect-library-1.0.7.jar
ADDED
Binary file
|
metadata
CHANGED
@@ -1,21 +1,21 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-dissect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-10-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
16
|
- - ">="
|
17
17
|
- !ruby/object:Gem::Version
|
18
|
-
version:
|
18
|
+
version: 2.1.1
|
19
19
|
- - "<="
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '2.99'
|
@@ -26,30 +26,59 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 2.1.1
|
30
30
|
- - "<="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
35
35
|
requirements:
|
36
|
-
- - "
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
name: jar-dependencies
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
37
51
|
- !ruby/object:Gem::Version
|
38
|
-
version:
|
52
|
+
version: '0'
|
53
|
+
name: rspec
|
54
|
+
prerelease: false
|
55
|
+
type: :development
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
39
67
|
name: logstash-devutils
|
40
68
|
prerelease: false
|
41
69
|
type: :development
|
42
70
|
version_requirements: !ruby/object:Gem::Requirement
|
43
71
|
requirements:
|
44
|
-
- - "
|
72
|
+
- - ">="
|
45
73
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
74
|
+
version: '0'
|
47
75
|
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
48
76
|
email: info@elastic.co
|
49
77
|
executables: []
|
50
78
|
extensions: []
|
51
79
|
extra_rdoc_files: []
|
52
80
|
files:
|
81
|
+
- BUILD_INSTRUCTIONS.md
|
53
82
|
- CHANGELOG.md
|
54
83
|
- CONTRIBUTORS
|
55
84
|
- DEVELOPER.md
|
@@ -57,11 +86,15 @@ files:
|
|
57
86
|
- LICENSE
|
58
87
|
- NOTICE.TXT
|
59
88
|
- README.md
|
60
|
-
-
|
89
|
+
- VERSION
|
90
|
+
- lib/jruby-dissect-library_jars.rb
|
61
91
|
- lib/logstash/filters/dissect.rb
|
62
92
|
- logstash-filter-dissect.gemspec
|
63
93
|
- spec/filters/dissect_spec.rb
|
64
94
|
- spec/spec_helper.rb
|
95
|
+
- vendor/jars/org/jruby/jruby-core/1.7.25/jruby-core-1.7.25.jar
|
96
|
+
- vendor/jars/org/logstash/dissect/jruby-dissect-library/1.0.5/jruby-dissect-library-1.0.5.jar
|
97
|
+
- vendor/jars/org/logstash/dissect/jruby-dissect-library/1.0.7/jruby-dissect-library-1.0.7.jar
|
65
98
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
66
99
|
licenses:
|
67
100
|
- Apache License (2.0)
|
@@ -84,10 +117,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
84
117
|
version: '0'
|
85
118
|
requirements: []
|
86
119
|
rubyforge_project:
|
87
|
-
rubygems_version: 2.
|
120
|
+
rubygems_version: 2.4.8
|
88
121
|
signing_key:
|
89
122
|
specification_version: 4
|
90
|
-
summary: This dissect filter will
|
123
|
+
summary: This dissect filter will de-structure text into multiple fields.
|
91
124
|
test_files:
|
92
125
|
- spec/filters/dissect_spec.rb
|
93
126
|
- spec/spec_helper.rb
|
Binary file
|