fluent-plugin-buffer-pullpool 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 5a042d29887fdbb84c7c15b72c91a02b507118c8
4
+ data.tar.gz: 591e5419b7a7d8330c95fe8549b17324cdb945ac
5
+ SHA512:
6
+ metadata.gz: 7dcfad83e69f51b2e4cd5fd5c9f3df6f8d80a5de8de63e9464f248fa7bb20ef50aac7ad4c8f948ec351e1e5603a51278268eb3b836ad9cfc229da4dd36b066c2
7
+ data.tar.gz: d1f869cd4a019dae6ff0e3663003efed963d2fe0db392992927c424d298ab33fe9b0b7082578802d30eb75de14f69e6cd52f75c9dade11be7e14cb2d4c3ac739
@@ -0,0 +1,22 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-buffer-pullpool.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 TAGOMORI Satoshi
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,36 @@
1
+ # fluent-plugin-buffer-pullpool
2
+
3
+ [Fluentd](http://fluentd.org) file buffer plugin to store events until output plugin's pull actions.
4
+
5
+ **NOTICE:** PullPool buffer plugin works just as same as normal file buffer plugin for normal buffered/time-sliced output plugins. Use this plugin with output plugins written to use this plugin.
6
+
7
+ ## Installation
8
+
9
+ Do `gem install fluent-plugin-buffer-pullpool` or `fluent-gem ...`.
10
+
11
+ ## Configuration
12
+
13
+ PullPool buffer plugin has options just same with File buffer plugin. `buffer_path` option must be specified.
14
+
15
+ ```
16
+ <match data.**>
17
+ type buffered_plugin_to_use_pullpool
18
+
19
+ buffer_type pullpool
20
+ buffer_path /home/myuser/tmp/pullbuffer
21
+ buffer_chunk_limit 256M
22
+ buffer_queue_limit 256
23
+ </match>
24
+ ```
25
+
26
+ Specifing short time for `flush_interval` makes a lot of chunk files.
27
+
28
+ ## TODO
29
+
30
+ * patches welcome!
31
+
32
+ ## Copyright
33
+
34
+ * Copyright (c) 2014- TAGOMORI Satoshi (tagomoris)
35
+ * License
36
+ * Apache License, Version 2.0
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,21 @@
1
+ # coding: utf-8
2
+
3
+ Gem::Specification.new do |spec|
4
+ spec.name = "fluent-plugin-buffer-pullpool"
5
+ spec.version = "0.0.1"
6
+ spec.authors = ["TAGOMORI Satoshi"]
7
+ spec.email = ["tagomoris@gmail.com"]
8
+ spec.summary = %q{Fluentd file buffer plugin to buffer data for pullers}
9
+ spec.description = %q{Alternative file buffer plugin to store data to wait to be pulled by plugin}
10
+ spec.homepage = "https://github.com/tagomoris/fluent-plugin-buffer-pullpool"
11
+ spec.license = "APLv2"
12
+
13
+ spec.files = `git ls-files -z`.split("\x0")
14
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
15
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
16
+ spec.require_paths = ["lib"]
17
+
18
+ spec.add_development_dependency "bundler", "~> 1.6"
19
+ spec.add_development_dependency "rake"
20
+ spec.add_runtime_dependency "fluentd"
21
+ end
@@ -0,0 +1,108 @@
1
+ require 'fluent/plugin/buf_file'
2
+
3
+ module Fluent
4
+ class PullBufferChunk < FileBufferChunk
5
+ attr_accessor :flushed
6
+
7
+ def initialize(key, path, unique_id, mode="a+", symlink_path = nil)
8
+ super
9
+ @flushed = false
10
+ end
11
+
12
+ def purge
13
+ # do not anything if not flushed yet: call actual_purge explicitly.
14
+ if @flushed
15
+ actual_purge
16
+ end
17
+ end
18
+
19
+ def actual_purge
20
+ @file.close
21
+ File.unlink(@path) rescue nil # TODO: check @path exists or not, and remove-or-rename if exists
22
+ end
23
+ end
24
+
25
+ class PullPoolBuffer < FileBuffer
26
+ Plugin.register_buffer('pullpool', self)
27
+
28
+ def initialize
29
+ super
30
+ @mutex = Mutex.new
31
+ @pool = []
32
+ end
33
+
34
+ def write_chunk(chunk, out)
35
+ if out.respond_to?(:write) # for normal BufferedOutput plugins: works just same as FileBuffer
36
+ out.write(chunk)
37
+ chunk.flushed = true
38
+ else
39
+ @pool.push(chunk)
40
+ end
41
+ end
42
+
43
+ def pull_chunks
44
+ raise "BUG: block not given" unless block_given?
45
+ chunks = nil
46
+ @mutex.synchronize do
47
+ chunks = @pool
48
+ @pool = []
49
+ end
50
+ chunks.each do |chunk|
51
+ begin
52
+ yield chunk
53
+ chunk.actual_purge
54
+ rescue => e
55
+ # TODO: Chunks should be returned to @pool ?
56
+ # It may make infinite error loops. Hmmm....
57
+ end
58
+ end
59
+ end
60
+
61
+ def new_chunk(key) # copy&paste from FileBuffer, but s/FileBufferChunk/PullBufferChunk/
62
+ encoded_key = encode_key(key)
63
+ path, tsuffix = make_path(encoded_key, "b")
64
+ unique_id = tsuffix_to_unique_id(tsuffix)
65
+ PullBufferChunk.new(key, path, unique_id, "a+", @symlink_path)
66
+ end
67
+
68
+ def resume # copy&paste from FileBuffer, but s/FileBufferChunk/PullBufferChunk/
69
+ maps = []
70
+ queues = []
71
+
72
+ Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
73
+ match = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
74
+ if m = PATH_MATCH.match(match)
75
+ key = decode_key(m[1])
76
+ bq = m[2]
77
+ tsuffix = m[3]
78
+ timestamp = m[3].to_i(16)
79
+ unique_id = tsuffix_to_unique_id(tsuffix)
80
+
81
+ if bq == 'b'
82
+ chunk = PullBufferChunk.new(key, path, unique_id, "a+")
83
+ maps << [timestamp, chunk]
84
+ elsif bq == 'q'
85
+ chunk = PullBufferChunk.new(key, path, unique_id, "r")
86
+ queues << [timestamp, chunk]
87
+ end
88
+ end
89
+ }
90
+
91
+ map = {}
92
+ maps.sort_by {|(timestamp,chunk)|
93
+ timestamp
94
+ }.each {|(timestamp,chunk)|
95
+ map[chunk.key] = chunk
96
+ }
97
+
98
+ queue = queues.sort_by {|(timestamp,chunk)|
99
+ timestamp
100
+ }.map {|(timestamp,chunk)|
101
+ chunk
102
+ }
103
+
104
+ return queue, map
105
+ end
106
+
107
+ end
108
+ end
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/buf_pullpool'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,53 @@
1
+ require 'json'
2
+
3
+ module Fluent
4
+ class DummyBuffered1Output < BufferedOutput
5
+ Fluent::Plugin.register_output('pullpool_test1', self)
6
+
7
+ attr_reader :written
8
+
9
+ def start
10
+ super
11
+ @written = []
12
+ end
13
+
14
+ def format(tag, time, record)
15
+ [tag, time, record.merge({"format_time" => Time.now.to_f})].to_json + "\n"
16
+ end
17
+
18
+ def write(chunk)
19
+ chunk_lines = chunk.read.split("\n").select{|line| not line.empty?}
20
+ chunk_lines.each do |line|
21
+ tag, time, record = JSON.parse(line)
22
+ record.update({"write_time" => Time.now.to_f})
23
+ @written.push record
24
+ end
25
+ end
26
+ end
27
+
28
+ class DummyBuffered2Output < BufferedOutput
29
+ Fluent::Plugin.register_output('pullpool_test2', self)
30
+
31
+ attr_reader :written
32
+
33
+ def start
34
+ super
35
+ @written = []
36
+ end
37
+
38
+ def format(tag, time, record)
39
+ [tag, time, record.merge({"format_time" => Time.now.to_f})].to_json + "\n"
40
+ end
41
+
42
+ def execute_pull
43
+ @buffer.pull_chunks do |chunk|
44
+ chunk_lines = chunk.read.split("\n").select{|line| not line.empty?}
45
+ chunk_lines.each do |line|
46
+ tag, time, record = JSON.parse(line)
47
+ record.update({"write_time" => Time.now.to_f})
48
+ @written.push record
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,106 @@
1
+ require 'helper'
2
+ require_relative 'dummy_output'
3
+
4
+ class DummyChain
5
+ def next
6
+ end
7
+ end
8
+
9
+ class PullPoolBufferTest < Test::Unit::TestCase
10
+ CONFIG = %[
11
+ buffer_type pullpool
12
+ buffer_path /tmp/pullpooltest
13
+ flush_interval 1s
14
+ ]
15
+
16
+ def create_driver1(conf=CONFIG, tag='test')
17
+ Fluent::Test::OutputTestDriver.new(Fluent::DummyBuffered1Output, tag).configure(conf)
18
+ end
19
+
20
+ def create_driver2(conf=CONFIG, tag='test')
21
+ Fluent::Test::OutputTestDriver.new(Fluent::DummyBuffered2Output, tag).configure(conf)
22
+ end
23
+
24
+ def test_configure
25
+ d = create_driver1
26
+ assert d.instance # successfully configured
27
+
28
+ d = create_driver2
29
+ assert d.instance # successfully configured
30
+ end
31
+
32
+ def test_emit1
33
+ d = create_driver1
34
+ buffer = d.instance.instance_eval{ @buffer }
35
+ assert buffer
36
+ d.instance.start
37
+
38
+ chain = DummyChain.new
39
+ tag = d.instance.instance_eval{ @tag }
40
+ time = Time.now.to_i
41
+
42
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 1}), chain)
43
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 2}), chain)
44
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 3}), chain)
45
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 4}), chain)
46
+
47
+ d.instance.instance_eval{ @next_flush_time = Time.now.to_i - 30 }
48
+ d.instance.try_flush
49
+
50
+ assert_equal 4, d.instance.written.size
51
+ end
52
+
53
+
54
+ def test_emit2
55
+ d = create_driver2
56
+ buffer = d.instance.instance_eval{ @buffer }
57
+ assert buffer
58
+ d.instance.start
59
+
60
+ chain = DummyChain.new
61
+ tag = d.instance.instance_eval{ @tag }
62
+ time = Time.now.to_i
63
+
64
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 1}), chain)
65
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 2}), chain)
66
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 3}), chain)
67
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 4}), chain)
68
+
69
+ d.instance.instance_eval{ @next_flush_time = Time.now.to_i - 30 }
70
+ d.instance.try_flush
71
+
72
+ assert_equal 0, d.instance.written.size
73
+
74
+ d.instance.execute_pull
75
+ assert_equal 4, d.instance.written.size
76
+
77
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 5}), chain)
78
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 6}), chain)
79
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 7}), chain)
80
+
81
+ d.instance.instance_eval{ @next_flush_time = Time.now.to_i - 30 }
82
+ d.instance.try_flush
83
+
84
+ assert_equal 4, d.instance.written.size
85
+
86
+ d.instance.execute_pull
87
+ assert_equal 7, d.instance.written.size
88
+
89
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 8}), chain)
90
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 9}), chain)
91
+
92
+ d.instance.instance_eval{ @next_flush_time = Time.now.to_i - 30 }
93
+ d.instance.try_flush
94
+
95
+ buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain)
96
+
97
+ d.instance.instance_eval{ @next_flush_time = Time.now.to_i - 30 }
98
+ d.instance.try_flush
99
+
100
+ assert_equal 7, d.instance.written.size
101
+
102
+ d.instance.execute_pull
103
+ assert_equal 10, d.instance.written.size
104
+
105
+ end
106
+ end
metadata ADDED
@@ -0,0 +1,101 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-buffer-pullpool
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - TAGOMORI Satoshi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-07-26 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.6'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fluentd
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: Alternative file buffer plugin to store data to wait to be pulled by
56
+ plugin
57
+ email:
58
+ - tagomoris@gmail.com
59
+ executables: []
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - Gemfile
65
+ - LICENSE.txt
66
+ - README.md
67
+ - Rakefile
68
+ - fluent-plugin-buffer-pullpool.gemspec
69
+ - lib/fluent/plugin/buf_pullpool.rb
70
+ - test/helper.rb
71
+ - test/plugin/dummy_output.rb
72
+ - test/plugin/test_buf_pullpool.rb
73
+ homepage: https://github.com/tagomoris/fluent-plugin-buffer-pullpool
74
+ licenses:
75
+ - APLv2
76
+ metadata: {}
77
+ post_install_message:
78
+ rdoc_options: []
79
+ require_paths:
80
+ - lib
81
+ required_ruby_version: !ruby/object:Gem::Requirement
82
+ requirements:
83
+ - - ">="
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ required_rubygems_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ requirements: []
92
+ rubyforge_project:
93
+ rubygems_version: 2.2.2
94
+ signing_key:
95
+ specification_version: 4
96
+ summary: Fluentd file buffer plugin to buffer data for pullers
97
+ test_files:
98
+ - test/helper.rb
99
+ - test/plugin/dummy_output.rb
100
+ - test/plugin/test_buf_pullpool.rb
101
+ has_rdoc: