fluent-plugin-buffer-event_limited 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.travis.yml +6 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +61 -0
- data/Rakefile +11 -0
- data/fluent-plugin-buffer-event_limited.gemspec +22 -0
- data/lib/fluent/plugin/buf_event_limited.rb +36 -0
- data/test/plugin/dummy_chain.rb +4 -0
- data/test/plugin/test_buf_event_limited.rb +78 -0
- data/test/plugin/test_event_recorder_buffered_output.rb +24 -0
- data/test/test_helper.rb +14 -0
- metadata +118 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: bde16a6ec9ee256c8eb1d8eff67a03f15857a67a
|
4
|
+
data.tar.gz: 93341c7a6f0b72d0003e907e31195a0e4703eddc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d83a6398da1b8e118cffbfb221802396d6da93b530176f526c59b64ce565d69c1975de6b52ba389d7d82707bdc060745c83a14a5f59a762d7907b7c486ad4dce
|
7
|
+
data.tar.gz: 71ee96f41c83236bac796a0184d4dae7a6a9d532ffaf42599a4f2378091ffef9cbf25751033523b0e220c1a9acdb211622670222c555c359c5d0b501cbaea3ad
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012- TAGOMORI Satoshi
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
# fluent-plugin-buffer-event_limited
|
2
|
+
|
3
|
+
[Fluentd](http://fluentd.org) buffer plugin on memory to flush with many types of chunk limit methods:
|
4
|
+
* events count limit in chunk
|
5
|
+
|
6
|
+
These options are to decrease latency from emit to write, and to control chunk sizes and flush sizes.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Do `gem install fluent-plugin-buffer-event_limited` or `fluent-gem ...`.
|
11
|
+
|
12
|
+
## Configuration
|
13
|
+
|
14
|
+
EventLimited buffer plugin can be enabled with all of buffered output plugins.
|
15
|
+
|
16
|
+
To flush chunks per 100 records, configure like this:
|
17
|
+
|
18
|
+
```
|
19
|
+
<match data.**>
|
20
|
+
type any_buffered_output_plugin
|
21
|
+
buffer_type event_limited
|
22
|
+
buffer_chunk_records_limit 100
|
23
|
+
# other options...
|
24
|
+
</match>
|
25
|
+
```
|
26
|
+
|
27
|
+
Options of `buffer_type file` are also available:
|
28
|
+
```
|
29
|
+
<match data.**>
|
30
|
+
type any_buffered_output_plugin
|
31
|
+
buffer_type event_limited
|
32
|
+
buffer_chunk_limit 10M
|
33
|
+
buffer_chunk_records_limit 100
|
34
|
+
# other options...
|
35
|
+
</match>
|
36
|
+
```
|
37
|
+
|
38
|
+
### For less delay
|
39
|
+
|
40
|
+
For more frequently flushing, use `flush_interval` and `try_flush_interval` with floating point values on Fluentd v0.10.42 or later:
|
41
|
+
```
|
42
|
+
<match data.**>
|
43
|
+
type any_buffered_output_plugin
|
44
|
+
buffer_type event_limited
|
45
|
+
buffer_chunk_records_limit 100
|
46
|
+
# other options...
|
47
|
+
flush_interval 0.5
|
48
|
+
try_flush_interval 0.1 # 0.6sec delay for worst case
|
49
|
+
</match>
|
50
|
+
```
|
51
|
+
|
52
|
+
## TODO
|
53
|
+
|
54
|
+
* more limit patterns
|
55
|
+
* patches welcome!
|
56
|
+
|
57
|
+
## Copyright
|
58
|
+
|
59
|
+
* Copyright (c) 2013- TAGOMORI Satoshi (tagomoris)
|
60
|
+
* License
|
61
|
+
* Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
Gem::Specification.new do |spec|
|
4
|
+
spec.name = "fluent-plugin-buffer-event_limited"
|
5
|
+
spec.version = "0.1.0"
|
6
|
+
spec.authors = ["TAGOMORI Satoshi", 'Gergo Sulymosi']
|
7
|
+
spec.email = ["tagomoris@gmail.com", 'gergo.sulymosi@gmail.com']
|
8
|
+
spec.description = %q{Fluentd memory buffer plugin with many types of chunk limits}
|
9
|
+
spec.summary = %q{Alternative file buffer plugin for Fluentd to realize less delay}
|
10
|
+
spec.homepage = "https://github.com/trekdemo/fluent-plugin-buffer-event_limited"
|
11
|
+
spec.license = "APLv2"
|
12
|
+
|
13
|
+
spec.files = `git ls-files`.split($/)
|
14
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
15
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
16
|
+
spec.require_paths = ["lib"]
|
17
|
+
|
18
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
19
|
+
spec.add_development_dependency "rake"
|
20
|
+
spec.add_development_dependency "test-unit"
|
21
|
+
spec.add_runtime_dependency "fluentd", ">= 0.10.42"
|
22
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'fluent/plugin/buf_file'
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class EventLimitedBufferChunk < FileBufferChunk
|
5
|
+
attr_reader :record_counter
|
6
|
+
|
7
|
+
def initialize(key, path, unique_id, mode = "a+", symlink_path = nil)
|
8
|
+
super
|
9
|
+
@record_counter = 0
|
10
|
+
end
|
11
|
+
|
12
|
+
def <<(data)
|
13
|
+
super
|
14
|
+
@record_counter += 1
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class EventLimitedFileBuffer < FileBuffer
|
19
|
+
Fluent::Plugin.register_buffer('event_limited', self)
|
20
|
+
|
21
|
+
config_param :buffer_chunk_records_limit, :integer, :default => Float::INFINITY
|
22
|
+
|
23
|
+
def new_chunk(key)
|
24
|
+
encoded_key = encode_key(key)
|
25
|
+
path, tsuffix = make_path(encoded_key, 'b')
|
26
|
+
unique_id = tsuffix_to_unique_id(tsuffix)
|
27
|
+
|
28
|
+
EventLimitedBufferChunk.new(key, path, unique_id, 'a+', @symlink_path)
|
29
|
+
end
|
30
|
+
|
31
|
+
def storable?(chunk, data)
|
32
|
+
chunk.record_counter < @buffer_chunk_records_limit &&
|
33
|
+
(chunk.size + data.bytesize) <= @buffer_chunk_limit
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require_relative '../test_helper'
|
2
|
+
require_relative 'test_event_recorder_buffered_output'
|
3
|
+
require_relative 'dummy_chain'
|
4
|
+
|
5
|
+
class EventLimitedFileBufferTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@buffer = Tempfile.new('event-limited-file-buffer')
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
@buffer.close
|
12
|
+
@buffer.unlink
|
13
|
+
end
|
14
|
+
|
15
|
+
def default_config
|
16
|
+
%[
|
17
|
+
buffer_type event_limited
|
18
|
+
flush_interval 0.1
|
19
|
+
try_flush_interval 0.03
|
20
|
+
buffer_chunk_records_limit 10
|
21
|
+
buffer_path #{@buffer.path}
|
22
|
+
]
|
23
|
+
end
|
24
|
+
|
25
|
+
def create_driver(conf = default_config, tag = 'test')
|
26
|
+
Fluent::Test::OutputTestDriver
|
27
|
+
.new(Fluent::TestEventRecorderBufferedOutput, tag)
|
28
|
+
.configure(conf)
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_plugin_configuration
|
32
|
+
output = create_driver.instance
|
33
|
+
buffer = output.instance_variable_get(:@buffer)
|
34
|
+
|
35
|
+
assert output # successfully configured
|
36
|
+
assert_equal 0.1, output.flush_interval
|
37
|
+
assert_equal 0.03, output.try_flush_interval
|
38
|
+
assert_equal 10, buffer.buffer_chunk_records_limit
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_emit
|
42
|
+
d = create_driver
|
43
|
+
|
44
|
+
buffer = d.instance.instance_variable_get(:@buffer)
|
45
|
+
assert buffer
|
46
|
+
buffer.start
|
47
|
+
|
48
|
+
assert_nil buffer.instance_variable_get(:@map)['']
|
49
|
+
|
50
|
+
d.emit({"a" => 1})
|
51
|
+
assert_equal 1, buffer.instance_variable_get(:@map)[''].record_counter
|
52
|
+
|
53
|
+
d.emit({"a" => 2}); d.emit({"a" => 3}); d.emit({"a" => 4})
|
54
|
+
d.emit({"a" => 5}); d.emit({"a" => 6}); d.emit({"a" => 7});
|
55
|
+
d.emit({"a" => 8});
|
56
|
+
assert_equal 8, buffer.instance_variable_get(:@map)[''].record_counter
|
57
|
+
|
58
|
+
chain = DummyChain.new
|
59
|
+
tag = d.instance.instance_variable_get(:@tag)
|
60
|
+
time = Time.now.to_i
|
61
|
+
|
62
|
+
# flush_trigger false
|
63
|
+
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 9}), chain)
|
64
|
+
assert_equal 9, buffer.instance_variable_get(:@map)[''].record_counter
|
65
|
+
|
66
|
+
# flush_trigger false
|
67
|
+
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain)
|
68
|
+
assert_equal 10, buffer.instance_variable_get(:@map)[''].record_counter
|
69
|
+
|
70
|
+
# flush_trigger true
|
71
|
+
assert buffer.emit(tag, d.instance.format(tag, time, {"a" => 11}), chain)
|
72
|
+
assert_equal 1, buffer.instance_variable_get(:@map)[''].record_counter # new chunk
|
73
|
+
|
74
|
+
# flush_trigger false
|
75
|
+
assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 12}), chain)
|
76
|
+
assert_equal 2, buffer.instance_variable_get(:@map)[''].record_counter
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Fluent
|
2
|
+
class TestEventRecorderBufferedOutput < BufferedOutput
|
3
|
+
Fluent::Plugin.register_output('test_event_recorder', self)
|
4
|
+
|
5
|
+
attr_reader :written
|
6
|
+
|
7
|
+
def start
|
8
|
+
super
|
9
|
+
@written = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def format(tag, time, record)
|
13
|
+
[tag, time, record.merge({'format_time' => Time.now.to_f})].to_msgpack
|
14
|
+
end
|
15
|
+
|
16
|
+
def write(chunk)
|
17
|
+
chunk.msgpack_each do |(tag, time, record)|
|
18
|
+
@written.push([tag, time, record.merge({'write_time' => Time.now.to_f})])
|
19
|
+
end
|
20
|
+
|
21
|
+
true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
unless ENV.has_key?('VERBOSE')
|
6
|
+
class NullLogger
|
7
|
+
def method_missing(method, *args); end
|
8
|
+
end
|
9
|
+
$log = NullLogger.new
|
10
|
+
end
|
11
|
+
|
12
|
+
require 'fluent/test'
|
13
|
+
require 'fluent/plugin/buf_event_limited'
|
14
|
+
Fluent::Test.setup
|
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-buffer-event_limited
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- TAGOMORI Satoshi
|
8
|
+
- Gergo Sulymosi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2015-10-23 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1.3'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.3'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: test-unit
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: fluentd
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 0.10.42
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.10.42
|
70
|
+
description: Fluentd memory buffer plugin with many types of chunk limits
|
71
|
+
email:
|
72
|
+
- tagomoris@gmail.com
|
73
|
+
- gergo.sulymosi@gmail.com
|
74
|
+
executables: []
|
75
|
+
extensions: []
|
76
|
+
extra_rdoc_files: []
|
77
|
+
files:
|
78
|
+
- ".gitignore"
|
79
|
+
- ".travis.yml"
|
80
|
+
- Gemfile
|
81
|
+
- LICENSE.txt
|
82
|
+
- README.md
|
83
|
+
- Rakefile
|
84
|
+
- fluent-plugin-buffer-event_limited.gemspec
|
85
|
+
- lib/fluent/plugin/buf_event_limited.rb
|
86
|
+
- test/plugin/dummy_chain.rb
|
87
|
+
- test/plugin/test_buf_event_limited.rb
|
88
|
+
- test/plugin/test_event_recorder_buffered_output.rb
|
89
|
+
- test/test_helper.rb
|
90
|
+
homepage: https://github.com/trekdemo/fluent-plugin-buffer-event_limited
|
91
|
+
licenses:
|
92
|
+
- APLv2
|
93
|
+
metadata: {}
|
94
|
+
post_install_message:
|
95
|
+
rdoc_options: []
|
96
|
+
require_paths:
|
97
|
+
- lib
|
98
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: '0'
|
108
|
+
requirements: []
|
109
|
+
rubyforge_project:
|
110
|
+
rubygems_version: 2.4.5.1
|
111
|
+
signing_key:
|
112
|
+
specification_version: 4
|
113
|
+
summary: Alternative file buffer plugin for Fluentd to realize less delay
|
114
|
+
test_files:
|
115
|
+
- test/plugin/dummy_chain.rb
|
116
|
+
- test/plugin/test_buf_event_limited.rb
|
117
|
+
- test/plugin/test_event_recorder_buffered_output.rb
|
118
|
+
- test/test_helper.rb
|