fluent-plugin-flowcounter-simple 0.0.4 → 0.1.0

This diff has not been reviewed by any users.
Log in in order to be able to vote.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: ea1531a6eca03db802d05acb32f7ac39cfd29576
4
- data.tar.gz: 6db938ffaa1c09dc074816b6491fc8dfa752b9f1
2
+ SHA256:
3
+ metadata.gz: 4f4d2e85abdbdf5c7b9e3bca8de7740066e6337262216bb09d7cd93e668c5977
4
+ data.tar.gz: beef12e960f130d3c1710d63e346c7524d410224e1548fc4c3bcc38643ed1ed8
5
5
  SHA512:
6
- metadata.gz: 3564af10781be5858eed85223f0657581a7140e729d994fba324bbb0222532d524408a1821182352316657f73f01182254daa62cad3e5de94257ace2b96e7939
7
- data.tar.gz: 7b327631a4d0cb1a082d26e0bebb3674549cf4b37184de7057993d942181d1e26d44c6481934d4272235758869383e192d71237cc582e860595535e5e8a0a8ce
6
+ metadata.gz: cb0cddb0135a5e1b8013d93490edcc872af9f2fe4d4f39206155704f00807a16a0b904e8bb3a5ad4049cdb8c5e81ecfc72bc8ffb581c6709abfcf4a1b64677fc
7
+ data.tar.gz: a1f1b6b1450ebae1c6269232924acb9769367dd13f31a4fa176ced494168f4aa6c42769f448247b15499195fbf3c33e5c6e509b88b67448e1bfa30c351c420f0
@@ -1,8 +1,11 @@
1
1
  rvm:
2
- - 1.9.3
3
- - 2.0.0
4
- - 2.1
5
- - 2.2
2
+ - 2.4.*
3
+ - 2.5.*
4
+ - 2.6.*
5
+ - 2.7.*
6
+
6
7
  gemfile:
7
8
  - Gemfile
8
- - Gemfile.v0.10
9
+
10
+ before_install:
11
+ - gem update bundler
@@ -1,3 +1,7 @@
1
+ ## 0.1.0 (2020/06/30)
2
+
3
+ * Use fluentd v1 API (repeatedly)
4
+
1
5
  ## 0.0.4 (2015/04/24)
2
6
 
3
7
  Enhancements:
data/README.md CHANGED
@@ -2,21 +2,15 @@
2
2
 
3
3
  Simple Fluentd Plugin to count number of messages and outputs to log
4
4
 
5
- ## Output Plugin Configuration
5
+ ## Requirements
6
6
 
7
- <match foo.bar.**>
8
- type flowcounter_simple
9
- unit second
10
- </match>
11
-
12
- This plugin does not emit, just writes counts into the log file as
13
-
14
- plugin:out_flowcounter_simple count:30 indicator:num unit:second
7
+ | fluent-plugin-flowcounter-simple | fluentd | ruby |
8
+ |-------------------|---------|------|
9
+ | >= 0.1.0 | >= v1.0 | >= 2.4 |
10
+ | < 0.0.4 | >= v0.12.0 | >= 2.1 |
15
11
 
16
12
  ## Filter Plugin Configuration
17
13
 
18
- Fluentd >= v0.12
19
-
20
14
  ```apache
21
15
  <filter foo.bar.**>
22
16
  type flowcounter_simple
@@ -28,6 +22,17 @@ This filter plugin pass through records, and writes counts into the log file as
28
22
 
29
23
  plugin:out_flowcounter_simple count:30 indicator:num unit:second
30
24
 
25
+ ## Output Plugin Configuration
26
+
27
+ <match foo.bar.**>
28
+ type flowcounter_simple
29
+ unit second
30
+ </match>
31
+
32
+ This plugin does not emit, just writes counts into the log file as
33
+
34
+ plugin:out_flowcounter_simple count:30 indicator:num unit:second
35
+
31
36
  ## Parameters
32
37
 
33
38
  - unit
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-flowcounter-simple"
4
- gem.version = "0.0.4"
4
+ gem.version = "0.1.0"
5
5
  gem.authors = ["Naotoshi Seo"]
6
6
  gem.email = ["sonots@gmail.com"]
7
7
  gem.summary = %q{Simple Fluentd Plugin to count number of messages and outputs to log}
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
14
14
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
15
15
  gem.require_paths = ["lib"]
16
16
 
17
- gem.add_runtime_dependency "fluentd"
17
+ gem.add_runtime_dependency "fluentd", [">= 1.0"]
18
18
  gem.add_development_dependency "rake"
19
19
  gem.add_development_dependency "pry"
20
20
  gem.add_development_dependency "pry-nav"
@@ -1,21 +1,15 @@
1
- # I'm lazy to to implement filter plugin. Use output plugin instance
2
- require_relative 'out_flowcounter_simple'
3
- require 'forwardable'
1
+ require 'fluent/plugin/filter'
2
+ require_relative 'flowcounter_simple'
4
3
 
5
- class Fluent::FlowCounterSimpleFilter < Fluent::Filter
6
- Fluent::Plugin.register_filter('flowcounter_simple', self)
4
+ module Fluent::Plugin
5
+ class FlowCounterSimpleFilter < Filter
6
+ Fluent::Plugin.register_filter('flowcounter_simple', self)
7
7
 
8
- extend Forwardable
9
- attr_reader :output
10
- def_delegators :@output, :configure, :start, :shutdown, :flush_emit
8
+ include ::Fluent::FlowcounterSimple
11
9
 
12
- def initialize
13
- super
14
- @output = Fluent::FlowCounterSimpleOutput.new
10
+ def filter_stream(tag, es)
11
+ process_count(tag, es)
12
+ es
13
+ end
15
14
  end
16
-
17
- def filter_stream(tag, es)
18
- @output.emit(tag, es, Fluent::NullOutputChain.instance)
19
- es
20
- end
21
- end if defined?(Fluent::Filter)
15
+ end
@@ -0,0 +1,100 @@
1
+ require 'fluent/plugin_helper/thread'
2
+
3
+ module Fluent
4
+ module FlowcounterSimple
5
+ attr_accessor :last_checked
6
+
7
+ def self.included(klass)
8
+ klass.helpers :thread
9
+ klass.config_param :indicator, :string, :default => 'num'
10
+ klass.config_param :unit, :string, :default => 'second'
11
+ klass.config_param :comment, :string, :default => nil
12
+ end
13
+
14
+ def configure(conf)
15
+ super
16
+
17
+ @indicator_proc =
18
+ case @indicator
19
+ when 'num' then Proc.new { |es| es.size }
20
+ when 'byte' then Proc.new { |es|
21
+ count = 0
22
+ es.each { |time, record|
23
+ count += record.to_msgpack.size
24
+ }
25
+ count
26
+ }
27
+ else
28
+ raise Fluent::ConfigError, "flowcounter-simple count allows num/byte"
29
+ end
30
+ @unit =
31
+ case @unit
32
+ when 'second' then :second
33
+ when 'minute' then :minute
34
+ when 'hour' then :hour
35
+ when 'day' then :day
36
+ else
37
+ raise Fluent::ConfigError, "flowcounter-simple unit allows second/minute/hour/day"
38
+ end
39
+ @tick =
40
+ case @unit
41
+ when :second then 1
42
+ when :minute then 60
43
+ when :hour then 3600
44
+ when :day then 86400
45
+ else
46
+ raise Fluent::ConfigError, "@unit must be one of second/minute/hour/day"
47
+ end
48
+
49
+ @type_str = self.is_a?(Fluent::Plugin::Filter) ? 'filter' : 'out'
50
+ @output_proc =
51
+ if @comment
52
+ Proc.new { |count| "plugin:#{@type_str}_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}\tcomment:#{@comment}" }
53
+ else
54
+ Proc.new { |count| "plugin:#{@type_str}_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}" }
55
+ end
56
+
57
+ @count = 0
58
+ @mutex = Mutex.new
59
+ end
60
+
61
+ def start
62
+ super
63
+ thread_create(:flowcounter_simple_watch, &method(:watch))
64
+ end
65
+
66
+ def shutdown
67
+ super
68
+ end
69
+
70
+ def countup(count)
71
+ @mutex.synchronize {
72
+ @count = (@count || 0) + count
73
+ }
74
+ end
75
+
76
+ def flush_emit(step)
77
+ count, @count = @count, 0
78
+ if count > 0
79
+ log.info @output_proc.call(count)
80
+ end
81
+ end
82
+
83
+ def watch
84
+ # instance variable, and public accessable, for test
85
+ @last_checked = Fluent::EventTime.now
86
+ while thread_current_running?
87
+ sleep 0.1
88
+ if Fluent::EventTime.now - @last_checked >= @tick
89
+ now = Fluent::EventTime.now
90
+ flush_emit(now - @last_checked)
91
+ @last_checked = now
92
+ end
93
+ end
94
+ end
95
+
96
+ def process_count(tag, es)
97
+ countup(@indicator_proc.call(es))
98
+ end
99
+ end
100
+ end
@@ -1,106 +1,22 @@
1
- class Fluent::FlowCounterSimpleOutput < Fluent::Output
2
- Fluent::Plugin.register_output('flowcounter_simple', self)
1
+ require 'fluent/plugin/output'
2
+ require_relative 'flowcounter_simple'
3
3
 
4
- # To support log_level option implemented by Fluentd v0.10.43
5
- unless method_defined?(:log)
6
- define_method("log") { $log }
7
- end
8
-
9
- config_param :indicator, :string, :default => 'num'
10
- config_param :unit, :string, :default => 'second'
11
- config_param :comment, :string, :default => nil
12
-
13
- attr_accessor :last_checked
14
-
15
- def configure(conf)
16
- super
17
-
18
- @indicator_proc =
19
- case @indicator
20
- when 'num' then Proc.new {|record| 1 }
21
- when 'byte' then Proc.new {|record| record.to_msgpack.size }
22
- else
23
- raise Fluent::ConfigError, "flowcounter-simple count allows num/byte"
24
- end
25
- @unit =
26
- case @unit
27
- when 'second' then :second
28
- when 'minute' then :minute
29
- when 'hour' then :hour
30
- when 'day' then :day
31
- else
32
- raise Fluent::ConfigError, "flowcounter-simple unit allows second/minute/hour/day"
33
- end
34
- @tick =
35
- case @unit
36
- when :second then 1
37
- when :minute then 60
38
- when :hour then 3600
39
- when :day then 86400
40
- else
41
- raise RuntimeError, "@unit must be one of second/minute/hour/day"
42
- end
4
+ module Fluent::Plugin
5
+ class FlowCounterSimpleOutput < Output
6
+ Fluent::Plugin.register_output('flowcounter_simple', self)
43
7
 
44
- @output_proc =
45
- if @comment
46
- Proc.new {|count| "plugin:out_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}\tcomment:#{@comment}" }
47
- else
48
- Proc.new {|count| "plugin:out_flowcounter_simple\tcount:#{count}\tindicator:#{@indicator}\tunit:#{@unit}" }
49
- end
8
+ include ::Fluent::FlowcounterSimple
50
9
 
51
- @count = 0
52
- @mutex = Mutex.new
53
- end
54
-
55
- def start
56
- super
57
- start_watch
58
- end
59
-
60
- def shutdown
61
- super
62
- @watcher.terminate
63
- @watcher.join
64
- end
65
-
66
- def countup(count)
67
- @mutex.synchronize {
68
- @count = (@count || 0) + count
69
- }
70
- end
71
-
72
- def flush_emit(step)
73
- count, @count = @count, 0
74
- if count > 0
75
- log.info @output_proc.call(count)
10
+ def prefer_buffered_processing
11
+ false
76
12
  end
77
- end
78
13
 
79
- def start_watch
80
- # for internal, or tests only
81
- @watcher = Thread.new(&method(:watch))
82
- end
83
-
84
- def watch
85
- # instance variable, and public accessable, for test
86
- @last_checked = Fluent::Engine.now
87
- while true
88
- sleep 0.1
89
- if Fluent::Engine.now - @last_checked >= @tick
90
- now = Fluent::Engine.now
91
- flush_emit(now - @last_checked)
92
- @last_checked = now
93
- end
14
+ def multi_workers_ready?
15
+ true
94
16
  end
95
- end
96
17
 
97
- def emit(tag, es, chain)
98
- count = 0
99
- es.each {|time,record|
100
- count += @indicator_proc.call(record)
101
- }
102
- countup(count)
103
-
104
- chain.next
18
+ def process(tag, es)
19
+ process_count(tag, es)
20
+ end
105
21
  end
106
22
  end
@@ -1,27 +1,28 @@
1
1
  require 'rubygems'
2
2
  require 'bundler'
3
- begin
4
- Bundler.setup(:default, :development)
5
- rescue Bundler::BundlerError => e
6
- $stderr.puts e.message
7
- $stderr.puts "Run `bundle install` to install missing gems"
8
- exit e.status_code
9
- end
3
+
10
4
  require 'test/unit'
5
+ require 'test/unit/rr'
11
6
 
12
7
  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
8
  $LOAD_PATH.unshift(File.dirname(__FILE__))
9
+
14
10
  require 'fluent/test'
15
- require 'fluent/plugin/out_flowcounter_simple'
16
- require 'fluent/plugin/filter_flowcounter_simple'
17
11
 
18
12
  class Test::Unit::TestCase
19
13
  def capture_log(log)
20
- tmp = log.out
21
- log.out = StringIO.new
22
- yield
23
- return log.out.string
24
- ensure
25
- log.out = tmp
14
+ if defined?(Fluent::Test::TestLogger) and log.is_a?(Fluent::Test::TestLogger) # v0.14
15
+ yield
16
+ log.out.logs.join("\n")
17
+ else
18
+ begin
19
+ tmp = log.out
20
+ log.out = StringIO.new
21
+ yield
22
+ return log.out.string
23
+ ensure
24
+ log.out = tmp
25
+ end
26
+ end
26
27
  end
27
28
  end
@@ -1,5 +1,6 @@
1
1
  require_relative '../helper'
2
- require "test/unit/rr"
2
+ require 'fluent/test/driver/filter'
3
+ require 'fluent/plugin/filter_flowcounter_simple'
3
4
 
4
5
  class FlowCounterSimpleFilterTest < Test::Unit::TestCase
5
6
  include Fluent
@@ -14,7 +15,7 @@ class FlowCounterSimpleFilterTest < Test::Unit::TestCase
14
15
  ]
15
16
 
16
17
  def create_driver(conf = CONFIG)
17
- Fluent::Test::FilterTestDriver.new(Fluent::FlowCounterSimpleFilter).configure(conf, true)
18
+ Fluent::Test::Driver::Filter.new(Fluent::Plugin::FlowCounterSimpleFilter).configure(conf)
18
19
  end
19
20
 
20
21
  def test_filter
@@ -26,24 +27,20 @@ class FlowCounterSimpleFilterTest < Test::Unit::TestCase
26
27
  d = create_driver
27
28
  filtered, out = filter(d, msgs)
28
29
  assert_equal msgs, filtered
29
- assert( out.include?("count:20"), out )
30
+ assert { out.include?("count:20") }
30
31
  end
31
32
 
32
33
  private
33
34
 
34
35
  def filter(d, msgs)
35
- stub(d.instance.output).start
36
- stub(d.instance.output).shutdown
37
- d.run {
36
+ d.run(default_tag: 'test') {
38
37
  msgs.each {|msg|
39
- d.filter(msg, @time)
38
+ d.feed(msg)
40
39
  }
41
40
  }
42
- out = capture_log(d.instance.output.log) do
41
+ out = capture_log(d.instance.log) do
43
42
  d.instance.flush_emit(0)
44
43
  end
45
- filtered = d.filtered_as_array
46
- filtered_msgs = filtered.map {|m| m[2] }
47
- [filtered_msgs, out]
44
+ [d.filtered_records, out]
48
45
  end
49
- end if defined?(Fluent::Filter)
46
+ end
@@ -1,4 +1,6 @@
1
1
  require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_flowcounter_simple'
2
4
 
3
5
  class FlowCounterSimpleOutputTest < Test::Unit::TestCase
4
6
  def setup
@@ -10,60 +12,60 @@ class FlowCounterSimpleOutputTest < Test::Unit::TestCase
10
12
  ]
11
13
 
12
14
  def create_driver(conf=CONFIG,tag='test')
13
- Fluent::Test::OutputTestDriver.new(Fluent::FlowCounterSimpleOutput, tag).configure(conf)
15
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::FlowCounterSimpleOutput).configure(conf)
14
16
  end
15
17
 
16
18
  def test_configure
17
19
  assert_nothing_raised {
18
- d = create_driver('')
20
+ create_driver('')
19
21
  }
20
22
  assert_nothing_raised {
21
- d = create_driver(CONFIG)
23
+ create_driver(CONFIG)
22
24
  }
23
25
  assert_nothing_raised {
24
- d = create_driver(CONFIG + %[indicator num])
26
+ create_driver(CONFIG + %[indicator num])
25
27
  }
26
28
  assert_nothing_raised {
27
- d = create_driver(CONFIG + %[indicator byte])
29
+ create_driver(CONFIG + %[indicator byte])
28
30
  }
29
31
  end
30
32
 
31
33
  def test_num
32
- d1 = create_driver(CONFIG, 'test.tag1')
33
- d1.run do
34
+ d1 = create_driver(CONFIG)
35
+ d1.run(default_tag: 'test.tag1') do
34
36
  10.times do
35
- d1.emit({'message'=> 'a' * 100})
36
- d1.emit({'message'=> 'b' * 100})
37
- d1.emit({'message'=> 'c' * 100})
37
+ d1.feed({'message'=> 'a' * 100})
38
+ d1.feed({'message'=> 'b' * 100})
39
+ d1.feed({'message'=> 'c' * 100})
38
40
  end
39
41
  end
40
42
  out = capture_log(d1.instance.log) { d1.instance.flush_emit(60) }
41
- assert( out.include?("count:30"), out )
43
+ assert { out.include?("count:30") }
42
44
  end
43
45
 
44
46
  def test_byte
45
- d1 = create_driver(CONFIG + %[indicator byte], 'test.tag1')
46
- d1.run do
47
+ d1 = create_driver(CONFIG + %[indicator byte])
48
+ d1.run(default_tag: 'test.tag1') do
47
49
  10.times do
48
- d1.emit({'message'=> 'a' * 100})
49
- d1.emit({'message'=> 'b' * 100})
50
- d1.emit({'message'=> 'c' * 100})
50
+ d1.feed({'message'=> 'a' * 100})
51
+ d1.feed({'message'=> 'b' * 100})
52
+ d1.feed({'message'=> 'c' * 100})
51
53
  end
52
54
  end
53
55
  out = capture_log(d1.instance.log) { d1.instance.flush_emit(60) }
54
- assert( out.include?("count:3360"), out )
56
+ assert { out =~ /count:\d+\tindicator:byte\tunit:second/ }
55
57
  end
56
58
 
57
59
  def test_comment
58
- d1 = create_driver(CONFIG + %[comment foobar], 'test.tag1')
59
- d1.run do
60
+ d1 = create_driver(CONFIG + %[comment foobar])
61
+ d1.run(default_tag: 'test.tag1') do
60
62
  1.times do
61
- d1.emit({'message'=> 'a' * 100})
62
- d1.emit({'message'=> 'b' * 100})
63
- d1.emit({'message'=> 'c' * 100})
63
+ d1.feed({'message'=> 'a' * 100})
64
+ d1.feed({'message'=> 'b' * 100})
65
+ d1.feed({'message'=> 'c' * 100})
64
66
  end
65
67
  end
66
68
  out = capture_log(d1.instance.log) { d1.instance.flush_emit(60) }
67
- assert( out.include?("comment:foobar"), out )
69
+ assert { out.include?("comment:foobar") }
68
70
  end
69
71
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-flowcounter-simple
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-23 00:00:00.000000000 Z
11
+ date: 2020-06-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: '1.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -105,7 +105,6 @@ files:
105
105
  - ".travis.yml"
106
106
  - CHANGELOG.md
107
107
  - Gemfile
108
- - Gemfile.v0.10
109
108
  - LICENSE.txt
110
109
  - README.md
111
110
  - Rakefile
@@ -113,6 +112,7 @@ files:
113
112
  - examples/output.conf
114
113
  - fluent-plugin-flowcounter-simple.gemspec
115
114
  - lib/fluent/plugin/filter_flowcounter_simple.rb
115
+ - lib/fluent/plugin/flowcounter_simple.rb
116
116
  - lib/fluent/plugin/out_flowcounter_simple.rb
117
117
  - test/helper.rb
118
118
  - test/plugin/test_filter_flowcounter_simple.rb
@@ -136,8 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
136
136
  - !ruby/object:Gem::Version
137
137
  version: '0'
138
138
  requirements: []
139
- rubyforge_project:
140
- rubygems_version: 2.2.2
139
+ rubygems_version: 3.0.3
141
140
  signing_key:
142
141
  specification_version: 4
143
142
  summary: Simple Fluentd Plugin to count number of messages and outputs to log
@@ -1,4 +0,0 @@
1
- source "http://rubygems.org"
2
-
3
- gemspec
4
- gem 'fluentd', '~> 0.10.0'