fluent-plugin-grep 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +6 -0
- data/fluent-plugin-grep.gemspec +1 -1
- data/lib/fluent/plugin/out_grep.rb +27 -2
- data/spec/out_grep_spec.rb +18 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01c08ca3372864d443be685dbe0ef91252644612
|
4
|
+
data.tar.gz: 6f7ec217ec20b890536a09de99a7fc28261c69e5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7b9b72ba0eb1ae64d472f18f8c57afb53ec0f7383f20d5a2c68bd7face9eac46761376542d79b40206fe2acdfb36e786dd8008020153c7021a911c0300d1f24b
|
7
|
+
data.tar.gz: 444709c11deead3423c8b5601744ecf6b420f1f95277b4fb2ff10627875a5172b52023425d5da8fd11a7619139b0fb69961f0c7a344e2bf25dbfb3b0e79185f0
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -26,6 +26,12 @@ Then, output bocomes as belows:
|
|
26
26
|
grep.syslog.host1: {"foo":"bar","message":"2013/01/13T07:02:13.232645 WARN POST /auth"}
|
27
27
|
grep.syslog.host1: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
|
28
28
|
|
29
|
+
## Parameters
|
30
|
+
|
31
|
+
- replace_invalid_sequence
|
32
|
+
|
33
|
+
Replace invalid byte sequence in UTF-8 with '?' character if `true`
|
34
|
+
|
29
35
|
## ChangeLog
|
30
36
|
|
31
37
|
See [CHANGELOG.md](CHANGELOG.md) for details.
|
data/fluent-plugin-grep.gemspec
CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-grep"
|
6
|
-
s.version = "0.0.
|
6
|
+
s.version = "0.0.2"
|
7
7
|
s.authors = ["Naotoshi SEO"]
|
8
8
|
s.email = ["sonots@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/sonots/fluent-plugin-grep"
|
@@ -6,6 +6,7 @@ class Fluent::GrepOutput < Fluent::Output
|
|
6
6
|
config_param :exclude, :string, :default => nil
|
7
7
|
config_param :tag, :string, :default => nil
|
8
8
|
config_param :add_tag_prefix, :string, :default => 'grep'
|
9
|
+
config_param :replace_invalid_sequence, :bool, :default => false
|
9
10
|
|
10
11
|
def configure(conf)
|
11
12
|
super
|
@@ -20,11 +21,35 @@ class Fluent::GrepOutput < Fluent::Output
|
|
20
21
|
|
21
22
|
es.each do |time,record|
|
22
23
|
value = record[@input_key]
|
23
|
-
next
|
24
|
-
next if @exclude and @exclude.match(value)
|
24
|
+
next unless match(value)
|
25
25
|
Fluent::Engine.emit(emit_tag, time, record)
|
26
26
|
end
|
27
27
|
|
28
28
|
chain.next
|
29
29
|
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def match(string)
|
34
|
+
begin
|
35
|
+
return false if @regexp and !@regexp.match(string)
|
36
|
+
return false if @exclude and @exclude.match(string)
|
37
|
+
rescue ArgumentError => e
|
38
|
+
unless e.message.index("invalid byte sequence in") == 0
|
39
|
+
raise
|
40
|
+
end
|
41
|
+
string = replace_invalid_byte(string)
|
42
|
+
return false if @regexp and !@regexp.match(string)
|
43
|
+
return false if @exclude and @exclude.match(string)
|
44
|
+
end
|
45
|
+
return true
|
46
|
+
end
|
47
|
+
|
48
|
+
def replace_invalid_byte(string)
|
49
|
+
replace_options = { invalid: :replace, undef: :replace, replace: '?' }
|
50
|
+
original_encoding = string.encoding
|
51
|
+
temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
|
52
|
+
string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
|
53
|
+
end
|
54
|
+
|
30
55
|
end
|
data/spec/out_grep_spec.rb
CHANGED
@@ -114,6 +114,24 @@ describe Fluent::GrepOutput do
|
|
114
114
|
end
|
115
115
|
it { emit }
|
116
116
|
end
|
117
|
+
|
118
|
+
context 'replace_invalid_sequence' do
|
119
|
+
let(:config) do
|
120
|
+
CONFIG + %[
|
121
|
+
regexp WARN
|
122
|
+
replace_invalid_sequence true
|
123
|
+
]
|
124
|
+
end
|
125
|
+
let(:messages) do
|
126
|
+
[
|
127
|
+
"\xff".force_encoding('UTF-8'),
|
128
|
+
]
|
129
|
+
end
|
130
|
+
before do
|
131
|
+
Fluent::Engine.stub(:now).and_return(time)
|
132
|
+
end
|
133
|
+
it { expect { emit }.not_to raise_error(ArgumentError) }
|
134
|
+
end
|
117
135
|
end
|
118
136
|
|
119
137
|
end
|