logstash-codec-multiline 3.0.3 → 3.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 26c71113b37e1292bbf9459af6716c724ee0f432
4
- data.tar.gz: 6d75faf254ff4be96e8f37a1818a573e357e60be
3
+ metadata.gz: 0b83041012aa7d4c15fe3b86d61f5b626d1b71ff
4
+ data.tar.gz: 33fd5598cc32b5f1ea942593def59627757abc0a
5
5
  SHA512:
6
- metadata.gz: e75491ca1f12432d886c53ab7d6aa4a4d65bb4f77ce71381b063fb779a0fc3564ad860287c91ac07484241d8ddabf6217c743eef868e9776fe51b76a53725377
7
- data.tar.gz: 450aed6a82000a797973c1bdb19dc031cdd7ac0acde846af8e088adc0f6df8371cc9ed73c010068b5079cb1d7ca61f178ab8b588685f0a2999727da4fd95f92c
6
+ metadata.gz: fe261395e4de7c05f9694647e5bf5fdfa3da3b863b3034101beafa730560ce8f6c93b3cf10a4307381ef7b29acbb0021a88fcfe41cec2e541164d41526c3c850
7
+ data.tar.gz: db6015c51acade7a6d185393377124c7a58948e1381372bb36e2804e99f135321f7a7881810a2aeb8f7c4587d1b502a68785278dc2e98cf725516ae1dfaa92a6
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 3.0.3
2
+ - Docs: Add note indicating that the multiline codec should not be used with input plugins that support multiple hosts
3
+
1
4
  ## 3.0.2
2
5
  - Fix log levels
3
6
 
data/Gemfile CHANGED
@@ -1,4 +1,11 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
- # Specify your gem's dependencies in logstash-mass_effect.gemspec
4
3
  gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
6
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
7
+
8
+ if Dir.exist?(logstash_path) && use_logstash_source
9
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
11
+ end
@@ -0,0 +1,222 @@
1
+ :plugin: multiline
2
+ :type: codec
3
+
4
+ ///////////////////////////////////////////
5
+ START - GENERATED VARIABLES, DO NOT EDIT!
6
+ ///////////////////////////////////////////
7
+ :version: %VERSION%
8
+ :release_date: %RELEASE_DATE%
9
+ :changelog_url: %CHANGELOG_URL%
10
+ :include_path: ../../../../logstash/docs/include
11
+ ///////////////////////////////////////////
12
+ END - GENERATED VARIABLES, DO NOT EDIT!
13
+ ///////////////////////////////////////////
14
+
15
+ [id="plugins-{type}-{plugin}"]
16
+
17
+ === Multiline codec plugin
18
+
19
+ include::{include_path}/plugin_header.asciidoc[]
20
+
21
+ ==== Description
22
+
23
+ The multiline codec will collapse multiline messages and merge them into a
24
+ single event.
25
+
26
+ IMPORTANT: If you are using a Logstash input plugin that supports multiple
27
+ hosts, such as the <<plugins-inputs-beats>> input plugin, you should not use
28
+ the multiline codec to handle multiline events. Doing so may result in the
29
+ mixing of streams and corrupted event data. In this situation, you need to
30
+ handle multiline events before sending the event data to Logstash.
31
+
32
+ The original goal of this codec was to allow joining of multiline messages
33
+ from files into a single event. For example, joining Java exception and
34
+ stacktrace messages into a single event.
35
+
36
+ The config looks like this:
37
+ [source,ruby]
38
+ input {
39
+ stdin {
40
+ codec => multiline {
41
+ pattern => "pattern, a regexp"
42
+ negate => "true" or "false"
43
+ what => "previous" or "next"
44
+ }
45
+ }
46
+ }
47
+
48
+ The `pattern` should match what you believe to be an indicator that the field
49
+ is part of a multi-line event.
50
+
51
+ The `what` must be `previous` or `next` and indicates the relation
52
+ to the multi-line event.
53
+
54
+ The `negate` can be `true` or `false` (defaults to `false`). If `true`, a
55
+ message not matching the pattern will constitute a match of the multiline
56
+ filter and the `what` will be applied. (vice-versa is also true)
57
+
58
+ For example, Java stack traces are multiline and usually have the message
59
+ starting at the far-left, with each subsequent line indented. Do this:
60
+ [source,ruby]
61
+ input {
62
+ stdin {
63
+ codec => multiline {
64
+ pattern => "^\s"
65
+ what => "previous"
66
+ }
67
+ }
68
+ }
69
+
70
+ This says that any line starting with whitespace belongs to the previous line.
71
+
72
+ Another example is to merge lines not starting with a date up to the previous
73
+ line..
74
+ [source,ruby]
75
+ input {
76
+ file {
77
+ path => "/var/log/someapp.log"
78
+ codec => multiline {
79
+ # Grok pattern names are valid! :)
80
+ pattern => "^%{TIMESTAMP_ISO8601} "
81
+ negate => true
82
+ what => "previous"
83
+ }
84
+ }
85
+ }
86
+
87
+ This says that any line not starting with a timestamp should be merged with the previous line.
88
+
89
+ One more common example is C line continuations (backslash). Here's how to do that:
90
+ [source,ruby]
91
+ input {
92
+ stdin {
93
+ codec => multiline {
94
+ pattern => "\\$"
95
+ what => "next"
96
+ }
97
+ }
98
+ }
99
+
100
+ This says that any line ending with a backslash should be combined with the
101
+ following line.
102
+
103
+
104
+ [id="plugins-{type}s-{plugin}-options"]
105
+ ==== Multiline Codec Configuration Options
106
+
107
+ [cols="<,<,<",options="header",]
108
+ |=======================================================================
109
+ |Setting |Input type|Required
110
+ | <<plugins-{type}s-{plugin}-auto_flush_interval>> |<<number,number>>|No
111
+ | <<plugins-{type}s-{plugin}-charset>> |<<string,string>>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No
112
+ | <<plugins-{type}s-{plugin}-max_bytes>> |<<bytes,bytes>>|No
113
+ | <<plugins-{type}s-{plugin}-max_lines>> |<<number,number>>|No
114
+ | <<plugins-{type}s-{plugin}-multiline_tag>> |<<string,string>>|No
115
+ | <<plugins-{type}s-{plugin}-negate>> |<<boolean,boolean>>|No
116
+ | <<plugins-{type}s-{plugin}-pattern>> |<<string,string>>|Yes
117
+ | <<plugins-{type}s-{plugin}-patterns_dir>> |<<array,array>>|No
118
+ | <<plugins-{type}s-{plugin}-what>> |<<string,string>>, one of `["previous", "next"]`|Yes
119
+ |=======================================================================
120
+
121
+ &nbsp;
122
+
123
+ [id="plugins-{type}s-{plugin}-auto_flush_interval"]
124
+ ===== `auto_flush_interval`
125
+
126
+ * Value type is <<number,number>>
127
+ * There is no default value for this setting.
128
+
129
+ The accumulation of multiple lines will be converted to an event when either a
130
+ matching new line is seen or there has been no new data appended for this many
131
+ seconds. No default. If unset, no auto_flush. Units: seconds
132
+
133
+ [id="plugins-{type}s-{plugin}-charset"]
134
+ ===== `charset`
135
+
136
+ * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale`
137
+ * Default value is `"UTF-8"`
138
+
139
+ The character encoding used in this input. Examples include `UTF-8`
140
+ and `cp1252`
141
+
142
+ This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
143
+ or in another character set other than `UTF-8`.
144
+
145
+ This only affects "plain" format logs since JSON is `UTF-8` already.
146
+
147
+ [id="plugins-{type}s-{plugin}-max_bytes"]
148
+ ===== `max_bytes`
149
+
150
+ * Value type is <<bytes,bytes>>
151
+ * Default value is `"10 MiB"`
152
+
153
+ The accumulation of events can make logstash exit with an out of memory error
154
+ if event boundaries are not correctly defined. This settings make sure to flush
155
+ multiline events after reaching a number of bytes, it is used in combination
156
+ max_lines.
157
+
158
+ [id="plugins-{type}s-{plugin}-max_lines"]
159
+ ===== `max_lines`
160
+
161
+ * Value type is <<number,number>>
162
+ * Default value is `500`
163
+
164
+ The accumulation of events can make logstash exit with an out of memory error
165
+ if event boundaries are not correctly defined. This settings make sure to flush
166
+ multiline events after reaching a number of lines, it is used in combination
167
+ max_bytes.
168
+
169
+ [id="plugins-{type}s-{plugin}-multiline_tag"]
170
+ ===== `multiline_tag`
171
+
172
+ * Value type is <<string,string>>
173
+ * Default value is `"multiline"`
174
+
175
+ Tag multiline events with a given tag. This tag will only be added
176
+ to events that actually have multiple lines in them.
177
+
178
+ [id="plugins-{type}s-{plugin}-negate"]
179
+ ===== `negate`
180
+
181
+ * Value type is <<boolean,boolean>>
182
+ * Default value is `false`
183
+
184
+ Negate the regexp pattern ('if not matched').
185
+
186
+ [id="plugins-{type}s-{plugin}-pattern"]
187
+ ===== `pattern`
188
+
189
+ * This is a required setting.
190
+ * Value type is <<string,string>>
191
+ * There is no default value for this setting.
192
+
193
+ The regular expression to match.
194
+
195
+ [id="plugins-{type}s-{plugin}-patterns_dir"]
196
+ ===== `patterns_dir`
197
+
198
+ * Value type is <<array,array>>
199
+ * Default value is `[]`
200
+
201
+ Logstash ships by default with a bunch of patterns, so you don't
202
+ necessarily need to define this yourself unless you are adding additional
203
+ patterns.
204
+
205
+ Pattern files are plain text with format:
206
+ [source,ruby]
207
+ NAME PATTERN
208
+
209
+ For example:
210
+ [source,ruby]
211
+ NUMBER \d+
212
+
213
+ [id="plugins-{type}s-{plugin}-what"]
214
+ ===== `what`
215
+
216
+ * This is a required setting.
217
+ * Value can be any of: `previous`, `next`
218
+ * There is no default value for this setting.
219
+
220
+ If the pattern matched, does event belong to the next or previous event?
221
+
222
+
@@ -7,6 +7,12 @@ require "logstash/codecs/auto_flush"
7
7
  # The multiline codec will collapse multiline messages and merge them into a
8
8
  # single event.
9
9
  #
10
+ # IMPORTANT: If you are using a Logstash input plugin that supports multiple
11
+ # hosts, such as the <<plugins-inputs-beats>> input plugin, you should not use
12
+ # the multiline codec to handle multiline events. Doing so may result in the
13
+ # mixing of streams and corrupted event data. In this situation, you need to
14
+ # handle multiline events before sending the event data to Logstash.
15
+ #
10
16
  # The original goal of this codec was to allow joining of multiline messages
11
17
  # from files into a single event. For example, joining Java exception and
12
18
  # stacktrace messages into a single event.
@@ -57,7 +63,7 @@ require "logstash/codecs/auto_flush"
57
63
  # # Grok pattern names are valid! :)
58
64
  # pattern => "^%{TIMESTAMP_ISO8601} "
59
65
  # negate => true
60
- # what => previous
66
+ # what => "previous"
61
67
  # }
62
68
  # }
63
69
  # }
@@ -66,11 +72,12 @@ require "logstash/codecs/auto_flush"
66
72
  #
67
73
  # One more common example is C line continuations (backslash). Here's how to do that:
68
74
  # [source,ruby]
69
- # filter {
70
- # multiline {
71
- # type => "somefiletype"
72
- # pattern => "\\$"
73
- # what => "next"
75
+ # input {
76
+ # stdin {
77
+ # codec => multiline {
78
+ # pattern => "\\$"
79
+ # what => "next"
80
+ # }
74
81
  # }
75
82
  # }
76
83
  #
@@ -128,8 +135,8 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
128
135
  config :max_bytes, :validate => :bytes, :default => "10 MiB"
129
136
 
130
137
  # The accumulation of multiple lines will be converted to an event when either a
131
- # matching new line is seen or there has been no new data appended for this time
132
- # auto_flush_interval. No default. If unset, no auto_flush. Units: seconds
138
+ # matching new line is seen or there has been no new data appended for this many
139
+ # seconds. No default. If unset, no auto_flush. Units: seconds
133
140
  config :auto_flush_interval, :validate => :number
134
141
 
135
142
  public
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-multiline'
4
- s.version = '3.0.3'
4
+ s.version = '3.0.5'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
11
11
  s.require_paths = ["lib"]
12
12
 
13
13
  # Files
14
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
15
15
 
16
16
  # Tests
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -1,4 +1,5 @@
1
1
  # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
2
3
  require "logstash/codecs/auto_flush"
3
4
  require "logstash/codecs/multiline"
4
5
  require_relative "../supports/helpers.rb"
@@ -1,4 +1,5 @@
1
1
  # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
2
3
  require "logstash/codecs/multiline"
3
4
  require "logstash/event"
4
5
  require "insist"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-multiline
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-09-15 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,7 @@ files:
84
84
  - LICENSE
85
85
  - NOTICE.TXT
86
86
  - README.md
87
+ - docs/index.asciidoc
87
88
  - lib/logstash/codecs/auto_flush.rb
88
89
  - lib/logstash/codecs/identity_map_codec.rb
89
90
  - lib/logstash/codecs/multiline.rb