logstash-codec-multiline 3.0.3 → 3.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 26c71113b37e1292bbf9459af6716c724ee0f432
4
- data.tar.gz: 6d75faf254ff4be96e8f37a1818a573e357e60be
3
+ metadata.gz: 0b83041012aa7d4c15fe3b86d61f5b626d1b71ff
4
+ data.tar.gz: 33fd5598cc32b5f1ea942593def59627757abc0a
5
5
  SHA512:
6
- metadata.gz: e75491ca1f12432d886c53ab7d6aa4a4d65bb4f77ce71381b063fb779a0fc3564ad860287c91ac07484241d8ddabf6217c743eef868e9776fe51b76a53725377
7
- data.tar.gz: 450aed6a82000a797973c1bdb19dc031cdd7ac0acde846af8e088adc0f6df8371cc9ed73c010068b5079cb1d7ca61f178ab8b588685f0a2999727da4fd95f92c
6
+ metadata.gz: fe261395e4de7c05f9694647e5bf5fdfa3da3b863b3034101beafa730560ce8f6c93b3cf10a4307381ef7b29acbb0021a88fcfe41cec2e541164d41526c3c850
7
+ data.tar.gz: db6015c51acade7a6d185393377124c7a58948e1381372bb36e2804e99f135321f7a7881810a2aeb8f7c4587d1b502a68785278dc2e98cf725516ae1dfaa92a6
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 3.0.3
2
+ - Docs: Add note indicating that the multiline codec should not be used with input plugins that support multiple hosts
3
+
1
4
  ## 3.0.2
2
5
  - Fix log levels
3
6
 
data/Gemfile CHANGED
@@ -1,4 +1,11 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
- # Specify your gem's dependencies in logstash-mass_effect.gemspec
4
3
  gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
6
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
7
+
8
+ if Dir.exist?(logstash_path) && use_logstash_source
9
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
11
+ end
@@ -0,0 +1,222 @@
1
+ :plugin: multiline
2
+ :type: codec
3
+
4
+ ///////////////////////////////////////////
5
+ START - GENERATED VARIABLES, DO NOT EDIT!
6
+ ///////////////////////////////////////////
7
+ :version: %VERSION%
8
+ :release_date: %RELEASE_DATE%
9
+ :changelog_url: %CHANGELOG_URL%
10
+ :include_path: ../../../../logstash/docs/include
11
+ ///////////////////////////////////////////
12
+ END - GENERATED VARIABLES, DO NOT EDIT!
13
+ ///////////////////////////////////////////
14
+
15
+ [id="plugins-{type}-{plugin}"]
16
+
17
+ === Multiline codec plugin
18
+
19
+ include::{include_path}/plugin_header.asciidoc[]
20
+
21
+ ==== Description
22
+
23
+ The multiline codec will collapse multiline messages and merge them into a
24
+ single event.
25
+
26
+ IMPORTANT: If you are using a Logstash input plugin that supports multiple
27
+ hosts, such as the <<plugins-inputs-beats>> input plugin, you should not use
28
+ the multiline codec to handle multiline events. Doing so may result in the
29
+ mixing of streams and corrupted event data. In this situation, you need to
30
+ handle multiline events before sending the event data to Logstash.
31
+
32
+ The original goal of this codec was to allow joining of multiline messages
33
+ from files into a single event. For example, joining Java exception and
34
+ stacktrace messages into a single event.
35
+
36
+ The config looks like this:
37
+ [source,ruby]
38
+ input {
39
+ stdin {
40
+ codec => multiline {
41
+ pattern => "pattern, a regexp"
42
+ negate => "true" or "false"
43
+ what => "previous" or "next"
44
+ }
45
+ }
46
+ }
47
+
48
+ The `pattern` should match what you believe to be an indicator that the field
49
+ is part of a multi-line event.
50
+
51
+ The `what` must be `previous` or `next` and indicates the relation
52
+ to the multi-line event.
53
+
54
+ The `negate` can be `true` or `false` (defaults to `false`). If `true`, a
55
+ message not matching the pattern will constitute a match of the multiline
56
+ filter and the `what` will be applied. (vice-versa is also true)
57
+
58
+ For example, Java stack traces are multiline and usually have the message
59
+ starting at the far-left, with each subsequent line indented. Do this:
60
+ [source,ruby]
61
+ input {
62
+ stdin {
63
+ codec => multiline {
64
+ pattern => "^\s"
65
+ what => "previous"
66
+ }
67
+ }
68
+ }
69
+
70
+ This says that any line starting with whitespace belongs to the previous line.
71
+
72
+ Another example is to merge lines not starting with a date up to the previous
73
+ line..
74
+ [source,ruby]
75
+ input {
76
+ file {
77
+ path => "/var/log/someapp.log"
78
+ codec => multiline {
79
+ # Grok pattern names are valid! :)
80
+ pattern => "^%{TIMESTAMP_ISO8601} "
81
+ negate => true
82
+ what => "previous"
83
+ }
84
+ }
85
+ }
86
+
87
+ This says that any line not starting with a timestamp should be merged with the previous line.
88
+
89
+ One more common example is C line continuations (backslash). Here's how to do that:
90
+ [source,ruby]
91
+ input {
92
+ stdin {
93
+ codec => multiline {
94
+ pattern => "\\$"
95
+ what => "next"
96
+ }
97
+ }
98
+ }
99
+
100
+ This says that any line ending with a backslash should be combined with the
101
+ following line.
102
+
103
+
104
+ [id="plugins-{type}s-{plugin}-options"]
105
+ ==== Multiline Codec Configuration Options
106
+
107
+ [cols="<,<,<",options="header",]
108
+ |=======================================================================
109
+ |Setting |Input type|Required
110
+ | <<plugins-{type}s-{plugin}-auto_flush_interval>> |<<number,number>>|No
111
+ | <<plugins-{type}s-{plugin}-charset>> |<<string,string>>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No
112
+ | <<plugins-{type}s-{plugin}-max_bytes>> |<<bytes,bytes>>|No
113
+ | <<plugins-{type}s-{plugin}-max_lines>> |<<number,number>>|No
114
+ | <<plugins-{type}s-{plugin}-multiline_tag>> |<<string,string>>|No
115
+ | <<plugins-{type}s-{plugin}-negate>> |<<boolean,boolean>>|No
116
+ | <<plugins-{type}s-{plugin}-pattern>> |<<string,string>>|Yes
117
+ | <<plugins-{type}s-{plugin}-patterns_dir>> |<<array,array>>|No
118
+ | <<plugins-{type}s-{plugin}-what>> |<<string,string>>, one of `["previous", "next"]`|Yes
119
+ |=======================================================================
120
+
121
+ &nbsp;
122
+
123
+ [id="plugins-{type}s-{plugin}-auto_flush_interval"]
124
+ ===== `auto_flush_interval`
125
+
126
+ * Value type is <<number,number>>
127
+ * There is no default value for this setting.
128
+
129
+ The accumulation of multiple lines will be converted to an event when either a
130
+ matching new line is seen or there has been no new data appended for this many
131
+ seconds. No default. If unset, no auto_flush. Units: seconds
132
+
133
+ [id="plugins-{type}s-{plugin}-charset"]
134
+ ===== `charset`
135
+
136
+ * Value can be any of: `ASCII-8BIT`, `UTF-8`, `US-ASCII`, `Big5`, `Big5-HKSCS`, `Big5-UAO`, `CP949`, `Emacs-Mule`, `EUC-JP`, `EUC-KR`, `EUC-TW`, `GB2312`, `GB18030`, `GBK`, `ISO-8859-1`, `ISO-8859-2`, `ISO-8859-3`, `ISO-8859-4`, `ISO-8859-5`, `ISO-8859-6`, `ISO-8859-7`, `ISO-8859-8`, `ISO-8859-9`, `ISO-8859-10`, `ISO-8859-11`, `ISO-8859-13`, `ISO-8859-14`, `ISO-8859-15`, `ISO-8859-16`, `KOI8-R`, `KOI8-U`, `Shift_JIS`, `UTF-16BE`, `UTF-16LE`, `UTF-32BE`, `UTF-32LE`, `Windows-31J`, `Windows-1250`, `Windows-1251`, `Windows-1252`, `IBM437`, `IBM737`, `IBM775`, `CP850`, `IBM852`, `CP852`, `IBM855`, `CP855`, `IBM857`, `IBM860`, `IBM861`, `IBM862`, `IBM863`, `IBM864`, `IBM865`, `IBM866`, `IBM869`, `Windows-1258`, `GB1988`, `macCentEuro`, `macCroatian`, `macCyrillic`, `macGreek`, `macIceland`, `macRoman`, `macRomania`, `macThai`, `macTurkish`, `macUkraine`, `CP950`, `CP951`, `IBM037`, `stateless-ISO-2022-JP`, `eucJP-ms`, `CP51932`, `EUC-JIS-2004`, `GB12345`, `ISO-2022-JP`, `ISO-2022-JP-2`, `CP50220`, `CP50221`, `Windows-1256`, `Windows-1253`, `Windows-1255`, `Windows-1254`, `TIS-620`, `Windows-874`, `Windows-1257`, `MacJapanese`, `UTF-7`, `UTF8-MAC`, `UTF-16`, `UTF-32`, `UTF8-DoCoMo`, `SJIS-DoCoMo`, `UTF8-KDDI`, `SJIS-KDDI`, `ISO-2022-JP-KDDI`, `stateless-ISO-2022-JP-KDDI`, `UTF8-SoftBank`, `SJIS-SoftBank`, `BINARY`, `CP437`, `CP737`, `CP775`, `IBM850`, `CP857`, `CP860`, `CP861`, `CP862`, `CP863`, `CP864`, `CP865`, `CP866`, `CP869`, `CP1258`, `Big5-HKSCS:2008`, `ebcdic-cp-us`, `eucJP`, `euc-jp-ms`, `EUC-JISX0213`, `eucKR`, `eucTW`, `EUC-CN`, `eucCN`, `CP936`, `ISO2022-JP`, `ISO2022-JP2`, `ISO8859-1`, `ISO8859-2`, `ISO8859-3`, `ISO8859-4`, `ISO8859-5`, `ISO8859-6`, `CP1256`, `ISO8859-7`, `CP1253`, `ISO8859-8`, `CP1255`, `ISO8859-9`, `CP1254`, `ISO8859-10`, `ISO8859-11`, `CP874`, `ISO8859-13`, `CP1257`, `ISO8859-14`, `ISO8859-15`, `ISO8859-16`, `CP878`, `MacJapan`, `ASCII`, `ANSI_X3.4-1968`, `646`, `CP65000`, `CP65001`, `UTF-8-MAC`, `UTF-8-HFS`, `UCS-2BE`, `UCS-4BE`, `UCS-4LE`, `CP932`, `csWindows31J`, `SJIS`, `PCK`, `CP1250`, `CP1251`, `CP1252`, `external`, `locale`
137
+ * Default value is `"UTF-8"`
138
+
139
+ The character encoding used in this input. Examples include `UTF-8`
140
+ and `cp1252`
141
+
142
+ This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
143
+ or in another character set other than `UTF-8`.
144
+
145
+ This only affects "plain" format logs since JSON is `UTF-8` already.
146
+
147
+ [id="plugins-{type}s-{plugin}-max_bytes"]
148
+ ===== `max_bytes`
149
+
150
+ * Value type is <<bytes,bytes>>
151
+ * Default value is `"10 MiB"`
152
+
153
+ The accumulation of events can make logstash exit with an out of memory error
154
+ if event boundaries are not correctly defined. This settings make sure to flush
155
+ multiline events after reaching a number of bytes, it is used in combination
156
+ max_lines.
157
+
158
+ [id="plugins-{type}s-{plugin}-max_lines"]
159
+ ===== `max_lines`
160
+
161
+ * Value type is <<number,number>>
162
+ * Default value is `500`
163
+
164
+ The accumulation of events can make logstash exit with an out of memory error
165
+ if event boundaries are not correctly defined. This settings make sure to flush
166
+ multiline events after reaching a number of lines, it is used in combination
167
+ max_bytes.
168
+
169
+ [id="plugins-{type}s-{plugin}-multiline_tag"]
170
+ ===== `multiline_tag`
171
+
172
+ * Value type is <<string,string>>
173
+ * Default value is `"multiline"`
174
+
175
+ Tag multiline events with a given tag. This tag will only be added
176
+ to events that actually have multiple lines in them.
177
+
178
+ [id="plugins-{type}s-{plugin}-negate"]
179
+ ===== `negate`
180
+
181
+ * Value type is <<boolean,boolean>>
182
+ * Default value is `false`
183
+
184
+ Negate the regexp pattern ('if not matched').
185
+
186
+ [id="plugins-{type}s-{plugin}-pattern"]
187
+ ===== `pattern`
188
+
189
+ * This is a required setting.
190
+ * Value type is <<string,string>>
191
+ * There is no default value for this setting.
192
+
193
+ The regular expression to match.
194
+
195
+ [id="plugins-{type}s-{plugin}-patterns_dir"]
196
+ ===== `patterns_dir`
197
+
198
+ * Value type is <<array,array>>
199
+ * Default value is `[]`
200
+
201
+ Logstash ships by default with a bunch of patterns, so you don't
202
+ necessarily need to define this yourself unless you are adding additional
203
+ patterns.
204
+
205
+ Pattern files are plain text with format:
206
+ [source,ruby]
207
+ NAME PATTERN
208
+
209
+ For example:
210
+ [source,ruby]
211
+ NUMBER \d+
212
+
213
+ [id="plugins-{type}s-{plugin}-what"]
214
+ ===== `what`
215
+
216
+ * This is a required setting.
217
+ * Value can be any of: `previous`, `next`
218
+ * There is no default value for this setting.
219
+
220
+ If the pattern matched, does event belong to the next or previous event?
221
+
222
+
@@ -7,6 +7,12 @@ require "logstash/codecs/auto_flush"
7
7
  # The multiline codec will collapse multiline messages and merge them into a
8
8
  # single event.
9
9
  #
10
+ # IMPORTANT: If you are using a Logstash input plugin that supports multiple
11
+ # hosts, such as the <<plugins-inputs-beats>> input plugin, you should not use
12
+ # the multiline codec to handle multiline events. Doing so may result in the
13
+ # mixing of streams and corrupted event data. In this situation, you need to
14
+ # handle multiline events before sending the event data to Logstash.
15
+ #
10
16
  # The original goal of this codec was to allow joining of multiline messages
11
17
  # from files into a single event. For example, joining Java exception and
12
18
  # stacktrace messages into a single event.
@@ -57,7 +63,7 @@ require "logstash/codecs/auto_flush"
57
63
  # # Grok pattern names are valid! :)
58
64
  # pattern => "^%{TIMESTAMP_ISO8601} "
59
65
  # negate => true
60
- # what => previous
66
+ # what => "previous"
61
67
  # }
62
68
  # }
63
69
  # }
@@ -66,11 +72,12 @@ require "logstash/codecs/auto_flush"
66
72
  #
67
73
  # One more common example is C line continuations (backslash). Here's how to do that:
68
74
  # [source,ruby]
69
- # filter {
70
- # multiline {
71
- # type => "somefiletype"
72
- # pattern => "\\$"
73
- # what => "next"
75
+ # input {
76
+ # stdin {
77
+ # codec => multiline {
78
+ # pattern => "\\$"
79
+ # what => "next"
80
+ # }
74
81
  # }
75
82
  # }
76
83
  #
@@ -128,8 +135,8 @@ module LogStash module Codecs class Multiline < LogStash::Codecs::Base
128
135
  config :max_bytes, :validate => :bytes, :default => "10 MiB"
129
136
 
130
137
  # The accumulation of multiple lines will be converted to an event when either a
131
- # matching new line is seen or there has been no new data appended for this time
132
- # auto_flush_interval. No default. If unset, no auto_flush. Units: seconds
138
+ # matching new line is seen or there has been no new data appended for this many
139
+ # seconds. No default. If unset, no auto_flush. Units: seconds
133
140
  config :auto_flush_interval, :validate => :number
134
141
 
135
142
  public
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-multiline'
4
- s.version = '3.0.3'
4
+ s.version = '3.0.5'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
11
11
  s.require_paths = ["lib"]
12
12
 
13
13
  # Files
14
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
15
15
 
16
16
  # Tests
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -1,4 +1,5 @@
1
1
  # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
2
3
  require "logstash/codecs/auto_flush"
3
4
  require "logstash/codecs/multiline"
4
5
  require_relative "../supports/helpers.rb"
@@ -1,4 +1,5 @@
1
1
  # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
2
3
  require "logstash/codecs/multiline"
3
4
  require "logstash/event"
4
5
  require "insist"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-multiline
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-09-15 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,7 @@ files:
84
84
  - LICENSE
85
85
  - NOTICE.TXT
86
86
  - README.md
87
+ - docs/index.asciidoc
87
88
  - lib/logstash/codecs/auto_flush.rb
88
89
  - lib/logstash/codecs/identity_map_codec.rb
89
90
  - lib/logstash/codecs/multiline.rb