fluent-plugin-grep 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8bc4b1bc5d03ba51926b8d8dbc5039c0a45696c3
4
- data.tar.gz: 4e6e77f9ef105730ee530720773ee98cc54f7e1c
3
+ metadata.gz: 01c08ca3372864d443be685dbe0ef91252644612
4
+ data.tar.gz: 6f7ec217ec20b890536a09de99a7fc28261c69e5
5
5
  SHA512:
6
- metadata.gz: a9e7fdf029afe36c65abd1016c85ffbaa31701f0b1ba4e678c63e7e986fa79766b21c391c30120936d9f8120689e0ea81f52e749a459da11c48ab50402adb11e
7
- data.tar.gz: 17984c87fc1b8ab48b4b7b334fa328c9ad6e702035188843b44d2bb2687ba3dd0765fdb28a9b5a437b150d3b911ca099e6d6fb19a51bcc495eb5dc08eadda449
6
+ metadata.gz: 7b9b72ba0eb1ae64d472f18f8c57afb53ec0f7383f20d5a2c68bd7face9eac46761376542d79b40206fe2acdfb36e786dd8008020153c7021a911c0300d1f24b
7
+ data.tar.gz: 444709c11deead3423c8b5601744ecf6b420f1f95277b4fb2ff10627875a5172b52023425d5da8fd11a7619139b0fb69961f0c7a344e2bf25dbfb3b0e79185f0
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.0.2
2
+
3
+ Features:
4
+
5
+ - Add `replace_invalid_sequence` option
6
+
1
7
  ## 0.0.1
2
8
 
3
9
  First version
data/README.md CHANGED
@@ -26,6 +26,12 @@ Then, output bocomes as belows:
26
26
  grep.syslog.host1: {"foo":"bar","message":"2013/01/13T07:02:13.232645 WARN POST /auth"}
27
27
  grep.syslog.host1: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
28
28
 
29
+ ## Parameters
30
+
31
+ - replace_invalid_sequence
32
+
33
+ Replace invalid byte sequence in UTF-8 with '?' character if `true`
34
+
29
35
  ## ChangeLog
30
36
 
31
37
  See [CHANGELOG.md](CHANGELOG.md) for details.
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-grep"
6
- s.version = "0.0.1"
6
+ s.version = "0.0.2"
7
7
  s.authors = ["Naotoshi SEO"]
8
8
  s.email = ["sonots@gmail.com"]
9
9
  s.homepage = "https://github.com/sonots/fluent-plugin-grep"
@@ -6,6 +6,7 @@ class Fluent::GrepOutput < Fluent::Output
6
6
  config_param :exclude, :string, :default => nil
7
7
  config_param :tag, :string, :default => nil
8
8
  config_param :add_tag_prefix, :string, :default => 'grep'
9
+ config_param :replace_invalid_sequence, :bool, :default => false
9
10
 
10
11
  def configure(conf)
11
12
  super
@@ -20,11 +21,35 @@ class Fluent::GrepOutput < Fluent::Output
20
21
 
21
22
  es.each do |time,record|
22
23
  value = record[@input_key]
23
- next if @regexp and !@regexp.match(value)
24
- next if @exclude and @exclude.match(value)
24
+ next unless match(value)
25
25
  Fluent::Engine.emit(emit_tag, time, record)
26
26
  end
27
27
 
28
28
  chain.next
29
29
  end
30
+
31
+ private
32
+
33
+ def match(string)
34
+ begin
35
+ return false if @regexp and !@regexp.match(string)
36
+ return false if @exclude and @exclude.match(string)
37
+ rescue ArgumentError => e
38
+ unless e.message.index("invalid byte sequence in") == 0
39
+ raise
40
+ end
41
+ string = replace_invalid_byte(string)
42
+ return false if @regexp and !@regexp.match(string)
43
+ return false if @exclude and @exclude.match(string)
44
+ end
45
+ return true
46
+ end
47
+
48
+ def replace_invalid_byte(string)
49
+ replace_options = { invalid: :replace, undef: :replace, replace: '?' }
50
+ original_encoding = string.encoding
51
+ temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
52
+ string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
53
+ end
54
+
30
55
  end
@@ -114,6 +114,24 @@ describe Fluent::GrepOutput do
114
114
  end
115
115
  it { emit }
116
116
  end
117
+
118
+ context 'replace_invalid_sequence' do
119
+ let(:config) do
120
+ CONFIG + %[
121
+ regexp WARN
122
+ replace_invalid_sequence true
123
+ ]
124
+ end
125
+ let(:messages) do
126
+ [
127
+ "\xff".force_encoding('UTF-8'),
128
+ ]
129
+ end
130
+ before do
131
+ Fluent::Engine.stub(:now).and_return(time)
132
+ end
133
+ it { expect { emit }.not_to raise_error(ArgumentError) }
134
+ end
117
135
  end
118
136
 
119
137
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-grep
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi SEO