logstash-codec-rfc6587 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/codecs/rfc6587.rb +15 -11
- data/logstash-codec-rfc6587.gemspec +1 -1
- data/spec/codecs/rfc6587_spec.rb +109 -104
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d4b77071d659535df03e355c6608120d7d198b08eaccd52da29e5bc320933fb3
|
4
|
+
data.tar.gz: 7d00398f882083289ea13f6c75bdbd728d2b3a11ad6499e3c6b230c26082ca7a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 73a1b782ef74480634c426b4b900a418672f7afe10a4b252588d9a0d3d79496f7b5396cf32f5411f653b3416ab13db9b8138e89304967f0f3b1664136c054166
|
7
|
+
data.tar.gz: c4a2988537b9aebc49f961dbb186389310acc507da1d69057bbe8b27bac3c9bd15fd3c14bc2c50117345466d0be82c723c3493b94224476c13470fddc0f3591a
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
|
2
3
|
require "logstash/codecs/base"
|
3
4
|
require "logstash/util/charset"
|
4
5
|
|
@@ -11,14 +12,13 @@ require 'logstash/plugin_mixins/event_support/event_factory_adapter'
|
|
11
12
|
#
|
12
13
|
# Encoding behavior: TBD
|
13
14
|
class LogStash::Codecs::Rfc6587 < LogStash::Codecs::Base
|
14
|
-
|
15
15
|
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
16
16
|
include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
|
17
17
|
|
18
18
|
config_name "rfc6587"
|
19
19
|
|
20
20
|
## Set the desired text format for encoding.
|
21
|
-
#config :format, :validate => :string
|
21
|
+
# config :format, :validate => :string
|
22
22
|
|
23
23
|
# The character encoding used in this input. Examples include `UTF-8`
|
24
24
|
# and `cp1252`
|
@@ -35,6 +35,8 @@ class LogStash::Codecs::Rfc6587 < LogStash::Codecs::Base
|
|
35
35
|
def initialize(*params)
|
36
36
|
super
|
37
37
|
|
38
|
+
raise "Delimitter must be 1 character long, but got '#{@delimiter}'" if @delimiter.length != 1
|
39
|
+
|
38
40
|
@original_field = ecs_select[disabled: nil, v1: '[event][original]']
|
39
41
|
end
|
40
42
|
|
@@ -48,13 +50,15 @@ class LogStash::Codecs::Rfc6587 < LogStash::Codecs::Base
|
|
48
50
|
|
49
51
|
def read(data)
|
50
52
|
header = ""
|
51
|
-
while c = data.
|
52
|
-
next if ["\0", "\x00"].include?(c)
|
53
|
+
while (c = data.getc)
|
54
|
+
next if ["\0", "\x00"].include?(c) # ignore null characters
|
53
55
|
raise "Unknown header character '#{c}'" if not [@delimiter, nil, c.to_i.to_s].include?(c)
|
56
|
+
|
54
57
|
header += c if c
|
55
58
|
break if c == @delimiter or data.eof or c == nil
|
56
59
|
end
|
57
60
|
raise "Unknown header '#{header}'" if header != "" and header.to_i == 0
|
61
|
+
|
58
62
|
if data.eof
|
59
63
|
@leftover = header
|
60
64
|
return
|
@@ -64,6 +68,7 @@ class LogStash::Codecs::Rfc6587 < LogStash::Codecs::Base
|
|
64
68
|
line = ""
|
65
69
|
to_read.times do
|
66
70
|
break if (c = data.getc) == nil
|
71
|
+
|
67
72
|
line += c
|
68
73
|
end
|
69
74
|
|
@@ -82,22 +87,22 @@ class LogStash::Codecs::Rfc6587 < LogStash::Codecs::Base
|
|
82
87
|
|
83
88
|
def decode(data)
|
84
89
|
data = StringIO.new @leftover + data
|
85
|
-
while line = read(data)
|
90
|
+
while (line = read(data))
|
86
91
|
yield new_event_from_line(line)
|
87
92
|
end
|
88
93
|
end
|
89
94
|
|
90
95
|
def flush(&block)
|
91
|
-
#remainder = @buffer.flush
|
92
|
-
#if !remainder.empty?
|
96
|
+
# remainder = @buffer.flush
|
97
|
+
# if !remainder.empty?
|
93
98
|
# block.call new_event_from_line(remainder)
|
94
|
-
#end
|
99
|
+
# end
|
95
100
|
end
|
96
101
|
|
97
102
|
def encode(event)
|
98
103
|
raise "Not implemented"
|
99
|
-
#encoded = @format ? event.sprintf(@format) : event.to_s
|
100
|
-
|
104
|
+
# encoded = @format ? event.sprintf(@format) : event.to_s
|
105
|
+
# @on_event.call(event, encoded + @delimiter)
|
101
106
|
end
|
102
107
|
|
103
108
|
private
|
@@ -108,5 +113,4 @@ class LogStash::Codecs::Rfc6587 < LogStash::Codecs::Base
|
|
108
113
|
event.set @original_field, message.dup.freeze if @original_field
|
109
114
|
event
|
110
115
|
end
|
111
|
-
|
112
116
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-rfc6587'
|
4
|
-
s.version = '0.1.
|
4
|
+
s.version = '0.1.5'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads lines as described in rfc6587"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/codecs/rfc6587_spec.rb
CHANGED
@@ -7,12 +7,11 @@ require "logstash/event"
|
|
7
7
|
require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
|
8
8
|
|
9
9
|
describe LogStash::Codecs::Rfc6587, :ecs_compatibility_support do
|
10
|
-
|
11
10
|
subject do
|
12
11
|
next LogStash::Codecs::Rfc6587.new
|
13
12
|
end
|
14
13
|
|
15
|
-
#context "#encode" do
|
14
|
+
# context "#encode" do
|
16
15
|
# let (:event) {LogStash::Event.new({"message" => "hello world", "host" => "test"})}
|
17
16
|
|
18
17
|
# it "should return a default date formatted line" do
|
@@ -46,111 +45,65 @@ describe LogStash::Codecs::Rfc6587, :ecs_compatibility_support do
|
|
46
45
|
# subject.encode(event)
|
47
46
|
# end
|
48
47
|
# end
|
49
|
-
#end
|
48
|
+
# end # context #encode
|
50
49
|
|
51
50
|
context "#decode" do
|
52
|
-
|
53
|
-
it "should return an event from an ascii string" do
|
54
|
-
decoded = false
|
55
|
-
subject.decode("11 hello world") do |e|
|
56
|
-
decoded = true
|
57
|
-
insist { e.is_a?(LogStash::Event) }
|
58
|
-
insist { e.get("message") } == "hello world"
|
59
|
-
end
|
60
|
-
insist { decoded } == true
|
61
|
-
end
|
62
|
-
|
63
|
-
it "should return nothing if input data is empty" do
|
64
|
-
decoded = false
|
65
|
-
subject.decode("") do |e|
|
66
|
-
decoded = true
|
67
|
-
end
|
68
|
-
insist { decoded } == false
|
69
|
-
end
|
70
|
-
|
71
|
-
it "should return nothing if input data is only '\\0' or '\\x00'" do
|
72
|
-
decoded = false
|
73
|
-
subject.decode("\0\x00") do |e|
|
74
|
-
decoded = true
|
75
|
-
end
|
76
|
-
insist { decoded } == false
|
77
|
-
end
|
78
|
-
|
79
|
-
it "should return an event from an ascii string prefixed with \\0" do
|
80
|
-
decoded = false
|
81
|
-
subject.decode("\x004 test") do |e|
|
82
|
-
decoded = true
|
83
|
-
insist { e.is_a?(LogStash::Event) }
|
84
|
-
insist { e.get("message") } == "test"
|
85
|
-
end
|
86
|
-
insist { decoded } == true
|
87
|
-
end
|
88
|
-
|
89
|
-
it "should contain correct results when input contains newline" do
|
90
|
-
result = []
|
91
|
-
subject.decode("5 line17 line2\n!5 line3") { |e| result << e }
|
92
|
-
subject.flush { |e| result << e }
|
93
|
-
expect(result.size).to eq(3)
|
94
|
-
expect(result[0].get("message")).to eq("line1")
|
95
|
-
expect(result[1].get("message")).to eq("line2\n!")
|
96
|
-
expect(result[2].get("message")).to eq("line3")
|
97
|
-
end
|
98
|
-
|
99
|
-
it "should parse partial batches" do
|
100
|
-
result = []
|
101
|
-
subject.decode("5 line15 line") { |e| result << e }
|
102
|
-
subject.decode("25 line3") { |e| result << e }
|
103
|
-
subject.flush { |e| result << e }
|
104
|
-
expect(result.size).to eq(3)
|
105
|
-
expect(result[0].get("message")).to eq("line1")
|
106
|
-
expect(result[1].get("message")).to eq("line2")
|
107
|
-
expect(result[2].get("message")).to eq("line3")
|
108
|
-
end
|
109
|
-
|
110
51
|
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
111
|
-
|
112
52
|
before(:each) do
|
113
53
|
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
114
54
|
end
|
115
55
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
expect( e.get("message") ).to eql message
|
123
|
-
expect( e.get("message").encoding ).to eql Encoding.find('UTF-8')
|
124
|
-
expect( e.get("message").valid_encoding? ).to be true
|
56
|
+
it "should return an event from an ascii string" do
|
57
|
+
decoded = false
|
58
|
+
subject.decode("11 hello world") do |e|
|
59
|
+
decoded = true
|
60
|
+
insist { e.is_a?(LogStash::Event) }
|
61
|
+
insist { e.get("message") } == "hello world"
|
125
62
|
end
|
63
|
+
insist { decoded } == true
|
126
64
|
end
|
127
65
|
|
128
|
-
it "
|
129
|
-
|
130
|
-
|
66
|
+
it "should return nothing if input data is empty" do
|
67
|
+
decoded = false
|
68
|
+
subject.decode("") do
|
69
|
+
decoded = true
|
131
70
|
end
|
132
|
-
|
71
|
+
insist { decoded } == false
|
72
|
+
end
|
133
73
|
|
134
|
-
|
74
|
+
it "should return nothing if input data is only '\\0' or '\\x00'" do
|
75
|
+
decoded = false
|
76
|
+
subject.decode("\0\x00") do
|
77
|
+
decoded = true
|
78
|
+
end
|
79
|
+
insist { decoded } == false
|
80
|
+
end
|
135
81
|
|
136
|
-
|
137
|
-
|
138
|
-
|
82
|
+
it "should return an event from an ascii string prefixed with \\0" do
|
83
|
+
decoded = false
|
84
|
+
subject.decode("\x004 test") do |e|
|
85
|
+
decoded = true
|
86
|
+
insist { e.is_a?(LogStash::Event) }
|
87
|
+
insist { e.get("message") } == "test"
|
88
|
+
end
|
89
|
+
insist { decoded } == true
|
139
90
|
end
|
140
91
|
|
141
|
-
it "should
|
142
|
-
line = "4 item4 item"
|
143
|
-
raw = "12|#{line}"
|
92
|
+
it "should contain correct results when input contains newline" do
|
144
93
|
result = []
|
145
|
-
subject.decode(
|
94
|
+
subject.decode("5 line17 line2\n!5 line3") { |e| result << e }
|
146
95
|
subject.flush { |e| result << e }
|
147
|
-
expect(result.size).to eq(
|
148
|
-
expect(result[0].get("message")).to eq(
|
96
|
+
expect(result.size).to eq(3)
|
97
|
+
expect(result[0].get("message")).to eq("line1")
|
98
|
+
expect(result[1].get("message")).to eq("line2\n!")
|
99
|
+
expect(result[2].get("message")).to eq("line3")
|
149
100
|
end
|
150
101
|
|
151
|
-
it "should
|
102
|
+
it "should parse partial batches" do
|
152
103
|
result = []
|
153
|
-
subject.decode("5
|
104
|
+
subject.decode("5 line15") { |e| result << e }
|
105
|
+
subject.decode(" line25 ") { |e| result << e }
|
106
|
+
subject.decode("line3") { |e| result << e }
|
154
107
|
subject.flush { |e| result << e }
|
155
108
|
expect(result.size).to eq(3)
|
156
109
|
expect(result[0].get("message")).to eq("line1")
|
@@ -158,21 +111,73 @@ describe LogStash::Codecs::Rfc6587, :ecs_compatibility_support do
|
|
158
111
|
expect(result[2].get("message")).to eq("line3")
|
159
112
|
end
|
160
113
|
|
161
|
-
|
162
|
-
|
114
|
+
it "should return an event from a valid utf-8 string" do
|
115
|
+
subject.decode("7 München") do |e|
|
116
|
+
expect(e).to be_a LogStash::Event
|
117
|
+
expect(e.get("message")).to eql "München"
|
118
|
+
expect(e.get("message").encoding).to eql Encoding.find('UTF-8')
|
119
|
+
expect(e.get("message").valid_encoding?).to be true
|
120
|
+
end
|
121
|
+
end
|
163
122
|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
123
|
+
it "sets event.original in ECS mode" do
|
124
|
+
subject.decode("7 München") do |event|
|
125
|
+
expect(event.get("[event][original]")).to eql "München"
|
126
|
+
end
|
127
|
+
end if ecs_select.active_mode != :disabled
|
128
|
+
|
129
|
+
context "when using custom :delimiter" do
|
130
|
+
subject do
|
131
|
+
next LogStash::Codecs::Rfc6587.new("delimiter" => "|")
|
132
|
+
end
|
133
|
+
|
134
|
+
it "should fail if delimitter has an invalid length" do
|
135
|
+
expect {
|
136
|
+
LogStash::Codecs::Rfc6587.new("delimiter" => "")
|
137
|
+
}.to raise_error(RuntimeError,
|
138
|
+
"Delimitter must be 1 character long, but got ''")
|
139
|
+
expect {
|
140
|
+
LogStash::Codecs::Rfc6587.new("delimiter" => "xx")
|
141
|
+
}.to raise_error(RuntimeError,
|
142
|
+
"Delimitter must be 1 character long, but got 'xx'")
|
143
|
+
end
|
144
|
+
|
145
|
+
it "should not break lines by '<number><space><line>'" do
|
146
|
+
line = "4 item4 item"
|
147
|
+
raw = "12|#{line}"
|
148
|
+
result = []
|
149
|
+
subject.decode(raw) { |e| result << e }
|
150
|
+
subject.flush { |e| result << e }
|
151
|
+
expect(result.size).to eq(1)
|
152
|
+
expect(result[0].get("message")).to eq(line)
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should break lines by that delimiter" do
|
156
|
+
result = []
|
157
|
+
subject.decode("5|line15|line25|line3") { |e| result << e }
|
158
|
+
subject.flush { |e| result << e }
|
159
|
+
expect(result.size).to eq(3)
|
160
|
+
expect(result[0].get("message")).to eq("line1")
|
161
|
+
expect(result[1].get("message")).to eq("line2")
|
162
|
+
expect(result[2].get("message")).to eq("line3")
|
163
|
+
end
|
164
|
+
|
165
|
+
context "and it being unicode" do
|
166
|
+
subject do
|
167
|
+
next LogStash::Codecs::Rfc6587.new("delimiter" => "÷")
|
168
|
+
end
|
169
|
+
|
170
|
+
it "should produce correct events" do
|
171
|
+
line = "4÷item4÷item"
|
172
|
+
result = []
|
173
|
+
subject.decode(line) { |e| result << e }
|
174
|
+
subject.flush { |e| result << e }
|
175
|
+
expect(result.size).to eq(2)
|
176
|
+
expect(result[0].get("message")).to eq('item')
|
177
|
+
expect(result[1].get("message")).to eq('item')
|
178
|
+
end
|
179
|
+
end # context unicode delimiter
|
180
|
+
end # context custom delimiter
|
181
|
+
end # ecs compat
|
182
|
+
end # context #decode
|
183
|
+
end # describe
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-rfc6587
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2024-10-
|
12
|
+
date: 2024-10-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -21,8 +21,8 @@ dependencies:
|
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2.99'
|
23
23
|
name: logstash-core-plugin-api
|
24
|
-
prerelease: false
|
25
24
|
type: :runtime
|
25
|
+
prerelease: false
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
27
27
|
requirements:
|
28
28
|
- - ">="
|
@@ -38,8 +38,8 @@ dependencies:
|
|
38
38
|
- !ruby/object:Gem::Version
|
39
39
|
version: '1.0'
|
40
40
|
name: logstash-mixin-event_support
|
41
|
-
prerelease: false
|
42
41
|
type: :runtime
|
42
|
+
prerelease: false
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
@@ -52,8 +52,8 @@ dependencies:
|
|
52
52
|
- !ruby/object:Gem::Version
|
53
53
|
version: '1.3'
|
54
54
|
name: logstash-mixin-ecs_compatibility_support
|
55
|
-
prerelease: false
|
56
55
|
type: :runtime
|
56
|
+
prerelease: false
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
@@ -66,8 +66,8 @@ dependencies:
|
|
66
66
|
- !ruby/object:Gem::Version
|
67
67
|
version: '0'
|
68
68
|
name: logstash-devutils
|
69
|
-
prerelease: false
|
70
69
|
type: :development
|
70
|
+
prerelease: false
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
@@ -80,8 +80,8 @@ dependencies:
|
|
80
80
|
- !ruby/object:Gem::Version
|
81
81
|
version: '0'
|
82
82
|
name: insist
|
83
|
-
prerelease: false
|
84
83
|
type: :development
|
84
|
+
prerelease: false
|
85
85
|
version_requirements: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - ">="
|