fluent-plugin-concat 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6e0e8f69b86fd07cc91f4045b25ac65cb7355130
4
- data.tar.gz: 3927ef6dd92afbe5807ce1dbf34c77a190e723b4
3
+ metadata.gz: aba37ca7a793288cddef2c986f31caec1c492b4f
4
+ data.tar.gz: beaa009f9c3bc35bc38e39390918bdd29e53361b
5
5
  SHA512:
6
- metadata.gz: 3f2e2eeaa5321ab37aaa86cc2974e1239121e84dd9c8fd4a80aa809ac0f9483307cacb50ff003b6ab749c21fc56ff6d00c0913367fd4294b2ff82f493609c4ee
7
- data.tar.gz: c27abd17cef7c40df3cd09d63e8ca61a5d2d87a6d465cd5f4df50e5246b9c933a593a987fba967ecf51132098f5857fc8128bc7e026a03b96496d5fbb7463acb
6
+ metadata.gz: f62bffeed9583c683da11a8b6b7f58f4ab45fd1d39905e46a8cfa11747e96a185f5464933c9cc989aa466dd199215855b4ca2cb5a4510d9b54b13b51f6f1eb59
7
+ data.tar.gz: 569e6b2393a83a0303cab98319ddcd412f6c11245ac26dde9524ff5bf12cbd7c82fae9765e901a84cfa834b8514978d0219f726ea5b8b34418b2b26bfeea4ea5
data/README.md CHANGED
@@ -52,6 +52,11 @@ The key to determine which stream an event belongs to.
52
52
  **flush\_interval**
53
53
 
54
54
  The number of seconds after which the last received event log will be flushed.
55
+ If specified 0, wait for next line forever.
56
+
57
+ **use\_first\_timestamp**
58
+
59
+ Use timestamp of first record when buffer is flushed.
55
60
 
56
61
  ## Usage
57
62
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-concat"
7
- spec.version = "0.5.0"
7
+ spec.version = "0.6.0"
8
8
  spec.authors = ["Kenji Okimoto"]
9
9
  spec.email = ["okimoto@clear-code.com"]
10
10
 
@@ -14,10 +14,12 @@ module Fluent
14
14
  config_param :multiline_end_regexp, :string, default: nil
15
15
  desc "The key to determine which stream an event belongs to"
16
16
  config_param :stream_identity_key, :string, default: nil
17
- desc "The interval between data flushes"
17
+ desc "The interval between data flushes, 0 means disable timeout"
18
18
  config_param :flush_interval, :time, default: 60
19
19
  desc "The label name to handle timeout"
20
20
  config_param :timeout_label, :string, default: nil
21
+ desc "Use timestamp of first record when buffer is flushed"
22
+ config_param :use_first_timestamp, :bool, default: false
21
23
 
22
24
  class TimeoutError < StandardError
23
25
  end
@@ -74,8 +76,11 @@ module Fluent
74
76
  new_es = MultiEventStream.new
75
77
  es.each do |time, record|
76
78
  begin
77
- new_record = process(tag, time, record)
78
- new_es.add(time, record.merge(new_record)) if new_record
79
+ new_time, new_record = process(tag, time, record)
80
+ if new_record
81
+ time = new_time if @use_first_timestamp
82
+ new_es.add(time, record.merge(new_record))
83
+ end
79
84
  rescue => e
80
85
  router.emit_error_event(tag, time, record, e)
81
86
  end
@@ -86,6 +91,7 @@ module Fluent
86
91
  private
87
92
 
88
93
  def on_timer
94
+ return if @flush_interval <= 0
89
95
  return if @finished
90
96
  flush_timeout_buffer
91
97
  end
@@ -104,19 +110,20 @@ module Fluent
104
110
  return flush_buffer(stream_identity)
105
111
  end
106
112
  when :regexp
107
- case
108
- when firstline?(record[@key])
113
+ if firstline?(record[@key])
109
114
  if @buffer[stream_identity].empty?
110
115
  @buffer[stream_identity] << [tag, time, record]
111
116
  else
112
117
  return flush_buffer(stream_identity, [tag, time, record])
113
118
  end
114
- when lastline?(record[@key])
115
- @buffer[stream_identity] << [tag, time, record]
119
+ end
120
+ if lastline?(record[@key])
121
+ @buffer[stream_identity] << [tag, time, record] unless firstline?(record[@key])
116
122
  return flush_buffer(stream_identity)
117
- else
123
+ end
124
+ if !firstline?(record[@key]) && !lastline?(record[@key])
118
125
  if @buffer[stream_identity].empty?
119
- return record
126
+ return [time, record]
120
127
  else
121
128
  # Continuation of the previous line
122
129
  @buffer[stream_identity] << [tag, time, record]
@@ -127,7 +134,7 @@ module Fluent
127
134
  end
128
135
 
129
136
  def firstline?(text)
130
- !!@multiline_start_regexp.match(text)
137
+ @multiline_start_regexp && !!@multiline_start_regexp.match(text)
131
138
  end
132
139
 
133
140
  def lastline?(text)
@@ -136,13 +143,13 @@ module Fluent
136
143
 
137
144
  def flush_buffer(stream_identity, new_element = nil)
138
145
  lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
139
- _tag, _time, last_record = @buffer[stream_identity].last
146
+ _tag, time, first_record = @buffer[stream_identity].first
140
147
  new_record = {
141
148
  @key => lines.join(@separator)
142
149
  }
143
150
  @buffer[stream_identity] = []
144
151
  @buffer[stream_identity] << new_element if new_element
145
- last_record.merge(new_record)
152
+ [time, first_record.merge(new_record)]
146
153
  end
147
154
 
148
155
  def flush_timeout_buffer
@@ -150,11 +157,11 @@ module Fluent
150
157
  timeout_stream_identities = []
151
158
  @timeout_map.each do |stream_identity, previous_timestamp|
152
159
  next if @flush_interval > (now - previous_timestamp)
153
- flushed_record = flush_buffer(stream_identity)
160
+ time, flushed_record = flush_buffer(stream_identity)
154
161
  timeout_stream_identities << stream_identity
155
162
  tag = stream_identity.split(":").first
156
163
  message = "Timeout flush: #{stream_identity}"
157
- handle_timeout_error(tag, now, flushed_record, message)
164
+ handle_timeout_error(tag, @use_first_timestamp ? time : now, flushed_record, message)
158
165
  log.info(message)
159
166
  end
160
167
  @timeout_map.reject! do |stream_identity, _|
@@ -170,7 +177,7 @@ module Fluent
170
177
  new_record = {
171
178
  @key => lines.join(@separator)
172
179
  }
173
- tag, time, record = elements.last
180
+ tag, time, record = elements.first
174
181
  message = "Flush remaining buffer: #{stream_identity}"
175
182
  handle_timeout_error(tag, time, record.merge(new_record), message)
176
183
  log.info(message)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-concat
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kenji Okimoto
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-23 00:00:00.000000000 Z
11
+ date: 2016-08-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -139,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
139
139
  version: '0'
140
140
  requirements: []
141
141
  rubyforge_project:
142
- rubygems_version: 2.5.1
142
+ rubygems_version: 2.6.4
143
143
  signing_key:
144
144
  specification_version: 4
145
145
  summary: Fluentd Filter plugin to concat multiple event messages