logstash-codec-json_lines 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/CONTRIBUTORS +1 -0
- data/lib/logstash/codecs/json_lines.rb +19 -13
- data/logstash-codec-json_lines.gemspec +1 -1
- data/spec/codecs/json_lines_spec.rb +36 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4be023e15a5cf6663bc8bb2538583c6c8a8c306f
|
4
|
+
data.tar.gz: f9c0752674c58ff41cb8ea3f628782399f1dfbb4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7f37ba24a814fb4603c79c33ebc97fb5e05b29b32144cdd4da8f1f759d91b991844bbc22df5c2aa78e8deed600a7a9d77e776cca756d7cb194c532bdc4b4a09
|
7
|
+
data.tar.gz: c180c5b6db9f074c79336c79862ae56fd0d8d06cbe953e5ac1d240361e920d4f85c2be65ef708c98ec5934fa2df2580977436a545728e74653d4f00c12259340
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
@@ -11,6 +11,7 @@ Contributors:
|
|
11
11
|
* Richard Pijnenburg (electrical)
|
12
12
|
* Suyog Rao (suyograo)
|
13
13
|
* Tal Levy (talevy)
|
14
|
+
* Guy Boertje (guyboertje)
|
14
15
|
|
15
16
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
16
17
|
Logstash, and you aren't on the list above and want to be, please let us know
|
@@ -4,9 +4,12 @@ require "logstash/codecs/line"
|
|
4
4
|
require "logstash/json"
|
5
5
|
|
6
6
|
# This codec will decode streamed JSON that is newline delimited.
|
7
|
-
# For decoding line-oriented JSON payload in the redis or file inputs,
|
8
|
-
# for example, use the json codec instead.
|
9
7
|
# Encoding will emit a single JSON string ending in a `\n`
|
8
|
+
# NOTE: Do not use this codec if your source input is line-oriented JSON, for
|
9
|
+
# example, redis or file inputs. Rather, use the json codec.
|
10
|
+
# More info: This codec is expecting to receive a stream (string) of newline
|
11
|
+
# terminated lines. The file input will produce a line string without a newline.
|
12
|
+
# Therefore this codec cannot work with line oriented inputs.
|
10
13
|
class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
|
11
14
|
config_name "json_lines"
|
12
15
|
|
@@ -23,30 +26,33 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
|
|
23
26
|
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
|
24
27
|
|
25
28
|
public
|
29
|
+
|
26
30
|
def initialize(params={})
|
27
31
|
super(params)
|
28
32
|
@lines = LogStash::Codecs::Line.new
|
29
33
|
@lines.charset = @charset
|
30
34
|
end
|
31
35
|
|
32
|
-
public
|
33
36
|
def decode(data)
|
34
|
-
|
35
37
|
@lines.decode(data) do |event|
|
36
|
-
|
37
|
-
yield LogStash::Event.new(LogStash::Json.load(event["message"]))
|
38
|
-
rescue LogStash::Json::ParserError => e
|
39
|
-
@logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
|
40
|
-
yield LogStash::Event.new("message" => event["message"], "tags" => ["_jsonparsefailure"])
|
41
|
-
end
|
38
|
+
yield guard(event, data)
|
42
39
|
end
|
43
40
|
end # def decode
|
44
41
|
|
45
|
-
public
|
46
42
|
def encode(event)
|
47
43
|
# Tack on a \n for now because previously most of logstash's JSON
|
48
44
|
# outputs emitted one per line, and whitespace is OK in json.
|
49
|
-
@on_event.call(event, event.to_json
|
45
|
+
@on_event.call(event, "#{event.to_json}#{NL}")
|
50
46
|
end # def encode
|
51
47
|
|
52
|
-
|
48
|
+
private
|
49
|
+
|
50
|
+
def guard(event, data)
|
51
|
+
begin
|
52
|
+
LogStash::Event.new(LogStash::Json.load(event["message"]))
|
53
|
+
rescue LogStash::Json::ParserError => e
|
54
|
+
LogStash::Event.new("message" => event["message"], "tags" => ["_jsonparsefailure"])
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end # class LogStash::Codecs::JSONLines
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-json_lines'
|
4
|
-
s.version = '1.0.
|
4
|
+
s.version = '1.0.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "This codec will decode streamed JSON that is newline delimited."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -101,4 +101,40 @@ describe LogStash::Codecs::JSONLines do
|
|
101
101
|
insist { got_event }
|
102
102
|
end
|
103
103
|
end
|
104
|
+
|
105
|
+
context 'reading from a simulated multiline json file without last newline' do
|
106
|
+
let(:input) do
|
107
|
+
%{{"field": "value1"}
|
108
|
+
{"field": "value2"}}
|
109
|
+
end
|
110
|
+
|
111
|
+
let(:collector) { Array.new }
|
112
|
+
|
113
|
+
it 'should generate one event' do
|
114
|
+
subject.decode(input) do |event|
|
115
|
+
collector.push(event)
|
116
|
+
end
|
117
|
+
expect(collector.size).to eq(1)
|
118
|
+
expect(collector.first['field']).to eq('value1')
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
context 'reading from a simulated multiline json file with last newline' do
|
123
|
+
let(:input) do
|
124
|
+
%{{"field": "value1"}
|
125
|
+
{"field": "value2"}
|
126
|
+
}
|
127
|
+
end
|
128
|
+
|
129
|
+
let(:collector) { Array.new }
|
130
|
+
|
131
|
+
it 'should generate two events' do
|
132
|
+
subject.decode(input) do |event|
|
133
|
+
collector.push(event)
|
134
|
+
end
|
135
|
+
expect(collector.size).to eq(2)
|
136
|
+
expect(collector.first['field']).to eq('value1')
|
137
|
+
expect(collector.last['field']).to eq('value2')
|
138
|
+
end
|
139
|
+
end
|
104
140
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-json_lines
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
@@ -97,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
97
97
|
version: '0'
|
98
98
|
requirements: []
|
99
99
|
rubyforge_project:
|
100
|
-
rubygems_version: 2.
|
100
|
+
rubygems_version: 2.1.9
|
101
101
|
signing_key:
|
102
102
|
specification_version: 4
|
103
103
|
summary: This codec will decode streamed JSON that is newline delimited.
|