logstash-codec-json_lines 3.1.0 → 3.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/docs/index.asciidoc +9 -0
  4. data/lib/logstash/codecs/json_lines.rb +11 -1
  5. data/logstash-codec-json_lines.gemspec +1 -1
  6. data/spec/codecs/json_lines_spec.rb +17 -0
  7. metadata +10 -11
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5169e72b2dd6bd4e67d98b39d45e9774df075b494b459f17612812413f8964d4
4
- data.tar.gz: 0c8e8cdcf9507c5f16389c175e6b29460149cb9c0fa80428605099330b8d7e83
3
+ metadata.gz: 4cb965f19d6c842e0b12c227328b4d962db8ea8e56eb4a123686cab8aa875bda
4
+ data.tar.gz: ff600109f192b060f14109a869a8aeb3d4f02bc70bc1742f831bbb244449beca
5
5
  SHA512:
6
- metadata.gz: a8421053d2d9387e7c9b33ad384f69f28e3fb5074a23ea9f5528d8493f81ea84e6a077e608c10550dc9496bd41fdbb75d0f0eb4a69ade501c124b34ce860b9a5
7
- data.tar.gz: 3b9cc5e8991eccb8c9d522602e8fdeaea2452bee93aad9449ebbba211cb1bcc2e69f861d26cefd810e326f9bbfe2b7e642f311ce593f1caab8a34bce3f692d57
6
+ metadata.gz: b94b09d11ce52fcf38d7a2d026563e4fa1c29ae99d3ce0ab1eb5c07b4ed10712ee99d5b759d026f583768e015b4758784738354453fd2eb2fb63a5385af99aae
7
+ data.tar.gz: 2f0ded04c8eb2a5b499c24b079286c16d7b4e1d722c9563e0f15b0ec1c14ebf817830a394d1a306d952aa1c30b10471bc59ffe62e8da307abb804b931151d9f6
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 3.2.1
2
+ - Raise the default value of `decode_size_limit_bytes` up to 512 MB. [#46](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/46)
3
+
4
+ ## 3.2.0
5
+ - Add decode_size_limit_bytes option to limit the maximum size of each JSON line that can be parsed.[#43](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/43)
6
+
1
7
  ## 3.1.0
2
8
  - Feat: event `target => namespace` support (ECS) [#41](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/41)
3
9
  - Refactor: dropped support for old Logstash versions (< 6.0)
data/docs/index.asciidoc CHANGED
@@ -35,6 +35,7 @@ Therefore this codec cannot work with line oriented inputs.
35
35
  |=======================================================================
36
36
  |Setting |Input type|Required
37
37
  | <<plugins-{type}s-{plugin}-charset>> |<<string,string>>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No
38
+ | <<plugins-{type}s-{plugin}-decode_size_limit_bytes>> |<<string,string>>|No
38
39
  | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
39
40
  | <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
40
41
  | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
@@ -58,6 +59,14 @@ actual encoding of the text and logstash will convert it for you.
58
59
 
59
60
  For nxlog users, you'll want to set this to `CP1252`
60
61
 
62
+ [id="plugins-{type}s-{plugin}-decode_size_limit_bytes"]
63
+ ===== `decode_size_limit_bytes`
64
+
65
+ * Value type is <<string,string>>
66
+ * Default value is 512 MB
67
+
68
+ Maximum number of bytes for a single line before stop processing.
69
+
61
70
  [id="plugins-{type}s-{plugin}-delimiter"]
62
71
  ===== `delimiter`
63
72
 
data/lib/logstash/codecs/json_lines.rb CHANGED
@@ -28,6 +28,8 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
28
28
 
29
29
  config_name "json_lines"
30
30
 
31
+ DEFAULT_DECODE_SIZE_LIMIT_BYTES = 512 * (1024 * 1024)
32
+
31
33
  # The character encoding used in this codec. Examples include `UTF-8` and
32
34
  # `CP1252`
33
35
  #
@@ -42,6 +44,11 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
42
44
  # Change the delimiter that separates lines
43
45
  config :delimiter, :validate => :string, :default => "\n"
44
46
 
47
+ # Maximum number of bytes for a single line before a fatal exception is raised
48
+ # which will stop Logstash.
49
+ # The default is 512MB which is quite large for a JSON document
50
+ config :decode_size_limit_bytes, :validate => :number, :default => DEFAULT_DECODE_SIZE_LIMIT_BYTES # 512MB
51
+
45
52
  # Defines a target field for placing decoded fields.
46
53
  # If this setting is omitted, data gets stored at the root (top level) of the event.
47
54
  # The target is only relevant while decoding data into a new event.
@@ -50,7 +57,10 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
50
57
  public
51
58
 
52
59
  def register
53
- @buffer = FileWatch::BufferedTokenizer.new(@delimiter)
60
+ if decode_size_limit_bytes == DEFAULT_DECODE_SIZE_LIMIT_BYTES
61
+ deprecation_logger.deprecated "The default value for `decode_size_limit_bytes`, currently at 512Mb, will be lowered in a future version to prevent Out of Memory errors from abnormally large messages or missing delimiters. Please set a value that reflects the largest expected message size (e.g. 20971520 for 20Mb)"
62
+ end
63
+ @buffer = FileWatch::BufferedTokenizer.new(@delimiter, @decode_size_limit_bytes)
54
64
  @converter = LogStash::Util::Charset.new(@charset)
55
65
  @converter.logger = @logger
56
66
  end
data/logstash-codec-json_lines.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-json_lines'
4
- s.version = '3.1.0'
4
+ s.version = '3.2.1'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Reads and writes newline-delimited JSON"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
data/spec/codecs/json_lines_spec.rb CHANGED
@@ -118,6 +118,23 @@ describe LogStash::Codecs::JSONLines, :ecs_compatibility_support do
118
118
  end
119
119
  end
120
120
 
121
+ describe "decode_size_limits_bytes" do
122
+ let(:codec_options) { { "decode_size_limit_bytes" => 20 * 1024 * 1024 } } # lower the default to avoid OOM errors in tests
123
+ let(:maximum_payload) { "a" * subject.decode_size_limit_bytes }
124
+
125
+ it "should not raise an error if the number of bytes is not exceeded" do
126
+ expect {
127
+ subject.decode(maximum_payload)
128
+ }.not_to raise_error
129
+ end
130
+
131
+ it "should raise an error if the max bytes are exceeded" do
132
+ expect {
133
+ subject.decode(maximum_payload << "z")
134
+ }.to raise_error(java.lang.IllegalStateException, "input buffer full")
135
+ end
136
+ end
137
+
121
138
  end
122
139
 
123
140
  context "#encode" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-json_lines
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.0
4
+ version: 3.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-01 00:00:00.000000000 Z
11
+ date: 2024-09-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
20
20
  - !ruby/object:Gem::Version
21
21
  version: '2.99'
22
22
  name: logstash-core-plugin-api
23
- prerelease: false
24
23
  type: :runtime
24
+ prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
27
  - - ">="
@@ -37,8 +37,8 @@ dependencies:
37
37
  - !ruby/object:Gem::Version
38
38
  version: '1.3'
39
39
  name: logstash-mixin-ecs_compatibility_support
40
- prerelease: false
41
40
  type: :runtime
41
+ prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
@@ -54,8 +54,8 @@ dependencies:
54
54
  - !ruby/object:Gem::Version
55
55
  version: 1.0.1
56
56
  name: logstash-mixin-event_support
57
- prerelease: false
58
57
  type: :runtime
58
+ prerelease: false
59
59
  version_requirements: !ruby/object:Gem::Requirement
60
60
  requirements:
61
61
  - - "~>"
@@ -71,8 +71,8 @@ dependencies:
71
71
  - !ruby/object:Gem::Version
72
72
  version: '1.0'
73
73
  name: logstash-mixin-validator_support
74
- prerelease: false
75
74
  type: :runtime
75
+ prerelease: false
76
76
  version_requirements: !ruby/object:Gem::Requirement
77
77
  requirements:
78
78
  - - "~>"
@@ -85,8 +85,8 @@ dependencies:
85
85
  - !ruby/object:Gem::Version
86
86
  version: 2.1.0
87
87
  name: logstash-codec-line
88
- prerelease: false
89
88
  type: :development
89
+ prerelease: false
90
90
  version_requirements: !ruby/object:Gem::Requirement
91
91
  requirements:
92
92
  - - ">="
@@ -99,8 +99,8 @@ dependencies:
99
99
  - !ruby/object:Gem::Version
100
100
  version: '0'
101
101
  name: logstash-devutils
102
- prerelease: false
103
102
  type: :development
103
+ prerelease: false
104
104
  version_requirements: !ruby/object:Gem::Requirement
105
105
  requirements:
106
106
  - - ">="
@@ -113,8 +113,8 @@ dependencies:
113
113
  - !ruby/object:Gem::Version
114
114
  version: '0'
115
115
  name: insist
116
- prerelease: false
117
116
  type: :development
117
+ prerelease: false
118
118
  version_requirements: !ruby/object:Gem::Requirement
119
119
  requirements:
120
120
  - - ">="
@@ -159,8 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
159
159
  - !ruby/object:Gem::Version
160
160
  version: '0'
161
161
  requirements: []
162
- rubyforge_project:
163
- rubygems_version: 2.6.13
162
+ rubygems_version: 3.3.26
164
163
  signing_key:
165
164
  specification_version: 4
166
165
  summary: Reads and writes newline-delimited JSON