fluent-plugin-buffer-event_limited 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6dac5d961204427f1720db8a996c381430953103
|
4
|
+
data.tar.gz: f17424fa7ae74a0bb472f2f5b4bc0141d4a82670
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 78739c15f9b891f7131fa018f07bbecda52010522810434941b5c90d557afa98ad2bcf710dc5e060c823199b8fd69289ca78e9598002e0b9c22af12f9a2e14b3
|
7
|
+
data.tar.gz: 7437256f6949ee8c5ebcc040cac28ac6a54316794c7f8dbeec0a6af1af3b23231718cfa04e0a4d3fe9261313240d56be3a57d2f2f2f7d8f39adbeabd870c749d
|
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |spec|
|
4
4
|
spec.name = "fluent-plugin-buffer-event_limited"
|
5
|
-
spec.version = "0.1.
|
6
|
-
spec.authors = [
|
7
|
-
spec.email = [
|
8
|
-
spec.description = %
|
9
|
-
spec.summary = %
|
5
|
+
spec.version = "0.1.6"
|
6
|
+
spec.authors = ['Gergo Sulymosi']
|
7
|
+
spec.email = ['gergo.sulymosi@gmail.com']
|
8
|
+
spec.description = %{Fluentd memory buffer plugin with many types of chunk limits}
|
9
|
+
spec.summary = %{Alternative file buffer plugin for Fluentd to limit events in a buffer not it's size}
|
10
10
|
spec.homepage = "https://github.com/trekdemo/fluent-plugin-buffer-event_limited"
|
11
11
|
spec.license = "APLv2"
|
12
12
|
|
@@ -9,16 +9,30 @@ module Fluent
|
|
9
9
|
@data = data.to_str.freeze
|
10
10
|
end
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
# Partition the data into required sizes
|
13
|
+
def each_slice(target_sizes)
|
14
|
+
target_size = target_sizes.next
|
15
|
+
slice_size = 0
|
16
|
+
slice_data = ''
|
17
|
+
|
18
|
+
reader.each do |event|
|
19
|
+
if slice_size == target_size
|
20
|
+
yield(slice_data, slice_size)
|
21
|
+
|
22
|
+
target_size = target_sizes.next
|
23
|
+
slice_size = 0
|
24
|
+
slice_data = ''
|
25
|
+
end
|
26
|
+
|
27
|
+
slice_data << pack(event)
|
28
|
+
slice_size += 1
|
29
|
+
end
|
15
30
|
|
16
|
-
|
17
|
-
records.dup
|
31
|
+
yield(slice_data, slice_size)
|
18
32
|
end
|
19
33
|
|
20
34
|
def size
|
21
|
-
@size ||=
|
35
|
+
@size ||= reader.each.reduce(0) { |c, _| c + 1 }
|
22
36
|
end
|
23
37
|
|
24
38
|
alias_method :to_str, :data
|
@@ -26,8 +40,12 @@ module Fluent
|
|
26
40
|
|
27
41
|
private
|
28
42
|
|
29
|
-
def
|
30
|
-
MessagePack
|
43
|
+
def pack(event)
|
44
|
+
MessagePack.pack(event)
|
45
|
+
end
|
46
|
+
|
47
|
+
def reader
|
48
|
+
@reader ||= MessagePack::Unpacker.new(StringIO.new(data))
|
31
49
|
end
|
32
50
|
end
|
33
51
|
|
@@ -60,44 +78,6 @@ module Fluent
|
|
60
78
|
|
61
79
|
config_param :buffer_chunk_records_limit, :integer, :default => Float::INFINITY
|
62
80
|
|
63
|
-
def emit(key, data, chain)
|
64
|
-
data = MessagePackFormattedBufferData.new(data)
|
65
|
-
key = key.to_s
|
66
|
-
flush_trigger = false
|
67
|
-
|
68
|
-
synchronize do
|
69
|
-
# Get the active chunk if it exists
|
70
|
-
chunk = (@map[key] ||= new_chunk(key))
|
71
|
-
|
72
|
-
# Partition the data into chunks that can be written into new chunks
|
73
|
-
events = data.as_events
|
74
|
-
[
|
75
|
-
events.shift(chunk.remaining_capacity),
|
76
|
-
*events.each_slice(@buffer_chunk_records_limit)
|
77
|
-
].each do |event_group|
|
78
|
-
chunk, queue_size = rotate_chunk!(chunk, key)
|
79
|
-
# Trigger flush only when we put the first chunk into it
|
80
|
-
flush_trigger ||= (queue_size == 0)
|
81
|
-
|
82
|
-
chain.next
|
83
|
-
chunk.write(
|
84
|
-
event_group.map { |d| MessagePack.pack(d) }.join(''),
|
85
|
-
event_group.size
|
86
|
-
)
|
87
|
-
end
|
88
|
-
|
89
|
-
return flush_trigger
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
def new_chunk(key)
|
94
|
-
encoded_key = encode_key(key)
|
95
|
-
path, tsuffix = make_path(encoded_key, 'b')
|
96
|
-
unique_id = tsuffix_to_unique_id(tsuffix)
|
97
|
-
|
98
|
-
chunk_factory(key, path, unique_id, 'a+')
|
99
|
-
end
|
100
|
-
|
101
81
|
# Copied here from
|
102
82
|
# https://github.com/fluent/fluentd/blob/d3ae305b6e7521fafac6ad30c6b0a8763c363b65/lib/fluent/plugin/buf_file.rb#L128-L165
|
103
83
|
def resume
|
@@ -134,8 +114,40 @@ module Fluent
|
|
134
114
|
return queue, map
|
135
115
|
end
|
136
116
|
|
117
|
+
def emit(key, data, chain)
|
118
|
+
data = MessagePackFormattedBufferData.new(data)
|
119
|
+
key = key.to_s
|
120
|
+
flush_trigger = false
|
121
|
+
|
122
|
+
synchronize do
|
123
|
+
# Get the current open chunk
|
124
|
+
chunk = (@map[key] ||= new_chunk(key))
|
125
|
+
|
126
|
+
data.each_slice(chunk_sizes(chunk.remaining_capacity)) do |data, size|
|
127
|
+
chain.next
|
128
|
+
chunk.write(data, size)
|
129
|
+
chunk, queue_size = rotate_chunk!(chunk, key)
|
130
|
+
flush_trigger ||= (queue_size == 0)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
return flush_trigger
|
135
|
+
end
|
136
|
+
|
137
|
+
def new_chunk(key)
|
138
|
+
encoded_key = encode_key(key)
|
139
|
+
path, tsuffix = make_path(encoded_key, 'b')
|
140
|
+
unique_id = tsuffix_to_unique_id(tsuffix)
|
141
|
+
|
142
|
+
chunk_factory(key, path, unique_id, 'a+')
|
143
|
+
end
|
144
|
+
|
137
145
|
private
|
138
146
|
|
147
|
+
def chunk_factory(key, path, uniq_id, mode)
|
148
|
+
EventLimitedBufferChunk.new(key, path, uniq_id, @buffer_chunk_records_limit, mode)
|
149
|
+
end
|
150
|
+
|
139
151
|
def rotate_chunk!(chunk, key)
|
140
152
|
queue_size = nil
|
141
153
|
return chunk unless chunk.full?
|
@@ -150,12 +162,15 @@ module Fluent
|
|
150
162
|
return chunk, queue_size
|
151
163
|
end
|
152
164
|
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
def
|
158
|
-
|
165
|
+
# Generates infinite sequence with and initial value followed by the chunk
|
166
|
+
# limit
|
167
|
+
#
|
168
|
+
# Eg.: [2, 5, 5, 5, 5, 5, ...]
|
169
|
+
def chunk_sizes(initial_size)
|
170
|
+
Enumerator.new do |y|
|
171
|
+
y << initial_size
|
172
|
+
y << @buffer_chunk_records_limit while true
|
173
|
+
end
|
159
174
|
end
|
160
175
|
end
|
161
176
|
end
|
@@ -73,24 +73,24 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
|
|
73
73
|
d.emit({"a" => 1})
|
74
74
|
assert_equal 1, count_buffer_events.call
|
75
75
|
|
76
|
-
(2..
|
77
|
-
assert_equal
|
76
|
+
(2..8).each { |i| d.emit({"a" => i}) }
|
77
|
+
assert_equal 8, count_buffer_events.call
|
78
78
|
|
79
79
|
chain = DummyChain.new
|
80
80
|
tag = d.instance.instance_variable_get(:@tag)
|
81
81
|
time = Time.now.to_i
|
82
82
|
|
83
83
|
# flush_trigger false
|
84
|
-
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" =>
|
85
|
-
assert_equal
|
84
|
+
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 9}), chain), "Shouldn't trigger flush"
|
85
|
+
assert_equal 9, count_buffer_events.call
|
86
86
|
|
87
87
|
# flush_trigger true
|
88
|
-
assert buffer.emit(tag, d.instance.format(tag, time, {"a" =>
|
89
|
-
assert_equal
|
88
|
+
assert buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain), "Should trigger flush"
|
89
|
+
assert_equal 0, count_buffer_events.call # new chunk
|
90
90
|
|
91
91
|
# flush_trigger false
|
92
|
-
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" =>
|
93
|
-
assert_equal
|
92
|
+
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 11}), chain), "Shouldn't trigger flush"
|
93
|
+
assert_equal 1, count_buffer_events.call
|
94
94
|
end
|
95
95
|
|
96
96
|
def test_emit_with_oversized_streams
|
metadata
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-buffer-event_limited
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
- TAGOMORI Satoshi
|
8
7
|
- Gergo Sulymosi
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
@@ -83,7 +82,6 @@ dependencies:
|
|
83
82
|
version: 0.10.42
|
84
83
|
description: Fluentd memory buffer plugin with many types of chunk limits
|
85
84
|
email:
|
86
|
-
- tagomoris@gmail.com
|
87
85
|
- gergo.sulymosi@gmail.com
|
88
86
|
executables: []
|
89
87
|
extensions: []
|
@@ -124,7 +122,8 @@ rubyforge_project:
|
|
124
122
|
rubygems_version: 2.4.5.1
|
125
123
|
signing_key:
|
126
124
|
specification_version: 4
|
127
|
-
summary: Alternative file buffer plugin for Fluentd to
|
125
|
+
summary: Alternative file buffer plugin for Fluentd to limit events in a buffer not
|
126
|
+
it's size
|
128
127
|
test_files:
|
129
128
|
- test/plugin/dummy_chain.rb
|
130
129
|
- test/plugin/test_buf_event_limited.rb
|