fluent-plugin-concat 2.2.3 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0a119cbc6b5774f81bde44b530ece7292c1f83635d6577155baad7d3c6d9544c
4
- data.tar.gz: 92c703271e63894f9b56c8a4d3d22c68c79c2ee6e219adb9b934fbb05a7464cf
3
+ metadata.gz: ee5e1936b898d90227550ba4baadba4f3d64d31577a1b0923aa634a0b3a49e6c
4
+ data.tar.gz: 59be67a9067f3589e9728702a4b4a8bb9f25e8915eedd4aa308dc297e710ba46
5
5
  SHA512:
6
- metadata.gz: cc59b890e0a82eca4a27663f2705f1ae4bf98e2d89c626696f90f523b73310df779cb26b92fdf544aa0e149f3c3e4b97569ea7291475b828ed2378082ec3a0bf
7
- data.tar.gz: 6b7eca55c66e0a038faf738ff718952a57bbf382881ffc7c03663984ede6f8e34d3219b737f4922ec83e1e82d82d823d64ad4a943e96dc9d8581fac1c78a81ad
6
+ metadata.gz: 2827392627d1d6d1eba547bb1d167fd86d9ebb88cb0c5aa98db6cb124a254da0af841de55a9ef4f437548a6c4175202f8bd751647ee711d40b2fd3607400f829
7
+ data.tar.gz: c827c2d9b14b509abd801428ac9aae29f34d99b12e6359e2152eb4b4716a7204dddf3638a9e9124da0eaee0d17fb8edd5e2ff18e6c3ff7fba5be4b000fad097f
data/NEWS.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # ChangeLog
2
2
 
3
+ ## v2.3.0
4
+
5
+ ### Improvements
6
+
7
+ * Support `partial_key`, `partial_value` to concatenate split log lines. See #46, #52
8
+
3
9
  ## v2.2.3
4
10
 
5
11
  ### Fixes
data/README.md CHANGED
@@ -29,16 +29,16 @@ Or install it yourself as:
29
29
 
30
30
  ## Configuration
31
31
 
32
- **key** (required)
32
+ **key** (string) (required)
33
33
 
34
34
  The key for part of multiline log.
35
35
 
36
- **separator**
36
+ **separator** (string)
37
37
 
38
38
  The separator of lines.
39
39
  Default value is `"\n"`.
40
40
 
41
- **n\_lines**
41
+ **n\_lines** (integer)
42
42
 
43
43
  The number of lines.
44
44
  This is exclusive with `multiline_start_regex`.
@@ -58,18 +58,33 @@ This is exclusive with `n_lines.`
58
58
  The regexp to match continuous lines.
59
59
  This is exclusive with `n_lines.`
60
60
 
61
- **stream\_identity\_key**
61
+ **stream\_identity\_key** (string)
62
62
 
63
63
  The key to determine which stream an event belongs to.
64
64
 
65
- **flush\_interval**
65
+ **flush\_interval** (integer)
66
66
 
67
67
  The number of seconds after which the last received event log will be flushed.
68
68
  If specified 0, wait for next line forever.
69
69
 
70
- **use\_first\_timestamp**
70
+ **use\_first\_timestamp** (bool)
71
71
 
72
72
  Use timestamp of first record when buffer is flushed.
73
+ Default value is `false`.
74
+
75
+ **partial\_key** (string)
76
+
77
+ The field name that is the reference to concatenate records
78
+
79
+ **partial\_value** (string)
80
+
81
+ The value stored in the field specified by partial_key that represent partial log
82
+
83
+ **keep\_partial\_key** (bool)
84
+
85
+ If true, keep partial_key in concatenated records
86
+ Default value is `false`.
87
+
73
88
 
74
89
  ## Usage
75
90
 
@@ -137,6 +152,51 @@ Handle single line JSON from Docker containers.
137
152
  </filter>
138
153
  ```
139
154
 
155
+ Handle Docker's `partial_message`.
156
+
157
+ ```aconf
158
+ <filter>
159
+ @type concat
160
+ key message
161
+ partial_key partial_message
162
+ partial_value true
163
+ </filter>
164
+ ```
165
+
166
+ Handle containerd/cri in Kubernetes.
167
+
168
+ ```aconf
169
+ <source>
170
+ @type tail
171
+ path /var/log/containers/*.log
172
+ <parse>
173
+ @type regexp
174
+ expression /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z) (?<output>\w+) (?<partial_flag>[FP]) (?<message>.+)$/
175
+ </parse>
176
+ tag k8s
177
+ @label @CONCAT
178
+ </source>
179
+
180
+ <label @CONCAT>
181
+ <filter k8s>
182
+ @type concat
183
+ key message
184
+ partial_key partial_flag
185
+ partial_value P
186
+ </filter>
187
+ <match k8s>
188
+ @type relabel
189
+ @label @OUTPUT
190
+ </match>
191
+ </label>
192
+
193
+ <label @OUTPUT>
194
+ <match>
195
+ @type stdout
196
+ </match>
197
+ </label>
198
+ ```
199
+
140
200
  ## Contributing
141
201
 
142
202
  1. Fork it
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-concat"
7
- spec.version = "2.2.3"
7
+ spec.version = "2.3.0"
8
8
  spec.authors = ["Kenji Okimoto"]
9
9
  spec.email = ["okimoto@clear-code.com"]
10
10
 
@@ -26,6 +26,12 @@ module Fluent::Plugin
26
26
  config_param :timeout_label, :string, default: nil
27
27
  desc "Use timestamp of first record when buffer is flushed"
28
28
  config_param :use_first_timestamp, :bool, default: false
29
+ desc "The field name that is the reference to concatenate records"
30
+ config_param :partial_key, :string, default: nil
31
+ desc "The value stored in the field specified by partial_key that represent partial log"
32
+ config_param :partial_value, :string, default: nil
33
+ desc "If true, keep partial_key in concatenated records"
34
+ config_param :keep_partial_key, :bool, default: false
29
35
 
30
36
  class TimeoutError < StandardError
31
37
  end
@@ -43,17 +49,28 @@ module Fluent::Plugin
43
49
  def configure(conf)
44
50
  super
45
51
 
46
- if @n_lines && (@multiline_start_regexp || @multiline_end_regexp || @continuous_line_regexp)
47
- raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp/continuous_line_regexp are exclusive"
52
+ if @n_lines && (@multiline_start_regexp || @multiline_end_regexp)
53
+ raise Fluent::ConfigError, "n_lines and multiline_start_regexp/multiline_end_regexp are exclusive"
48
54
  end
49
- if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil?
55
+ if @n_lines.nil? && @multiline_start_regexp.nil? && @multiline_end_regexp.nil? && @partial_key.nil?
50
56
  raise Fluent::ConfigError, "Either n_lines or multiline_start_regexp or multiline_end_regexp is required"
51
57
  end
58
+ if @partial_key && @n_lines
59
+ raise Fluent::ConfigError, "partial_key and n_lines are exclusive"
60
+ end
61
+ if @partial_key && (@multiline_start_regexp || @multiline_end_regexp)
62
+ raise Fluent::ConfigError, "partial_key and multiline_start_regexp/multiline_end_regexp are exclusive"
63
+ end
64
+ if @partial_key && @partial_value.nil?
65
+ raise Fluent::ConfigError, "partial_value is required when partial_key is specified"
66
+ end
52
67
 
53
68
  @mode = nil
54
69
  case
55
70
  when @n_lines
56
71
  @mode = :line
72
+ when @partial_key
73
+ @mode = :partial
57
74
  when @multiline_start_regexp || @multiline_end_regexp
58
75
  @mode = :regexp
59
76
  if @multiline_start_regexp
@@ -96,7 +113,9 @@ module Fluent::Plugin
96
113
  unless flushed_es.empty?
97
114
  flushed_es.each do |_time, new_record|
98
115
  time = _time if @use_first_timestamp
99
- new_es.add(time, record.merge(new_record))
116
+ merged_record = record.merge(new_record)
117
+ merged_record.delete(@partial_key) unless @keep_partial_key
118
+ new_es.add(time, merged_record)
100
119
  end
101
120
  end
102
121
  rescue => e
@@ -128,6 +147,8 @@ module Fluent::Plugin
128
147
  case @mode
129
148
  when :line
130
149
  process_line(stream_identity, tag, time, record)
150
+ when :partial
151
+ process_partial(stream_identity, tag, time, record)
131
152
  when :regexp
132
153
  process_regexp(stream_identity, tag, time, record)
133
154
  end
@@ -144,6 +165,18 @@ module Fluent::Plugin
144
165
  new_es
145
166
  end
146
167
 
168
+ def process_partial(stream_identity, tag, time, record)
169
+ new_es = Fluent::MultiEventStream.new
170
+ @buffer[stream_identity] << [tag, time, record]
171
+ unless @partial_value == record[@partial_key]
172
+ new_time, new_record = flush_buffer(stream_identity)
173
+ time = new_time if @use_first_timestamp
174
+ new_record.delete(@partial_key)
175
+ new_es.add(time, new_record)
176
+ end
177
+ new_es
178
+ end
179
+
147
180
  def process_regexp(stream_identity, tag, time, record)
148
181
  new_es = Fluent::MultiEventStream.new
149
182
  case
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-concat
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.3
4
+ version: 2.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kenji Okimoto
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-15 00:00:00.000000000 Z
11
+ date: 2018-07-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd